Support Pyodide 0.28 and bump to Python >=3.13

.github/workflows/main.yml

@@ -9,6 +9,9 @@ on:
 permissions:
   contents: read
 
+env:
+  FORCE_COLOR: 3
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -20,10 +23,10 @@ jobs:
           fetch-tags: true
           persist-credentials: false
 
-      - name: Set up Python 3.10
+      - name: Set up Python 3.13
         uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
-          python-version: "3.10"
+          python-version: "3.13"
 
       - name: Install dependencies
         run: |
@@ -35,15 +38,14 @@ jobs:
 
       - uses: actions/setup-node@1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a # v4.2.0
         with:
-          node-version: "20.6.1"
+          node-version: "22"
 
       - name: Install the pyodide npm package
-        run: |
-          npm install [email protected]
+        run: npm install [email protected]
 
       - name: Test package + examples
         run: |
-          pytest --cov=pyodide_pack -n 2 --cov-report=xml
+          pytest --cov=pyodide_pack -n auto --cov-report=xml
 
       - name: Upload coverage
         uses: codecov/codecov-action@0565863a31f2c772f9f0395002a31e3f06189574 # v5.4.0

.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
-          python-version: "3.12"
+          python-version: "3.13"
 
       - name: Build the distribution packages
         run: |

.readthedocs.yml

@@ -12,6 +12,6 @@ python:
       path: .
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.13"

CHANGELOG.md

@@ -12,8 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
    `pyproject.toml` files
    [#35](https://github.com/pyodide/pyodide-pack/pull/35)
 
-
- - Add `pyidide minify` command to minify the Python packages with AST rewrites by,
+ - Add `pyodide minify` command to minify the Python packages with AST rewrites by,
    removing comments and docstrings
    [#23](https://github.com/pyodide/pyodide-pack/pull/23)
 
@@ -31,8 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
- - Added support for Pyodide 0.24.0. This is now the minimal supported version of Pyodide.
-   [#26](https://github.com/pyodide/pyodide-pack/pull/26)
+ - Added support for Pyodide 0.27.3. This is now the minimal supported version of Pyodide.
+   [#57](https://github.com/pyodide/pyodide-pack/pull/57)
 
 ## Fixed
 

README.md

@@ -15,16 +15,17 @@ Each of these approaches have different tradeoffs, and can be used separately or
 
 ## Install
 
-Pyodide-pack requires Python 3.10+ and can be installed via pip:
+`pyodide-pack` requires Python >=3.13 and can be installed via `pip`:
 
 ```
 pip install pyodide-pack
 ```
 
-(optionally) For elimation of unused modules via runtime detection, run NodeJS  needs to be installed together with Pyodide 0.24.0+:
+(optionally) For elimation of unused modules via runtime detection, run Node.js, needs to be installed together with Pyodide 0.28.0a3 and later:
 
+<!-- TODO update this to stable when ready -->
 ```bash
-npm install pyodide@">=0.24.0"
+npm install pyodide@">=0.28.0a3"
 ```
 
 ## Usage

docs/conf.py

@@ -26,7 +26,7 @@
     "sphinx_autodoc_typehints",
 ]
 intersphinx_mapping = {
-    "python": ("https://docs.python.org/3.10", None),
+    "python": ("https://docs.python.org/3.13", None),
     "pyodide": ("https://pyodide.org/en/stable/", None),
 }
 

docs/module-elimination-at-runtime.md

@@ -17,61 +17,64 @@
 
 2. Create the package bundle,
 
-   ```bash
-   pyodide pack examples/pandas/app.py
-   ```
-   which would produce the following output
+    ```bash
+    pyodide pack examples/pandas/app.py
+    ```
+
+    which would produce the following output:
+
+    ```
+    Running pyodide-pack on examples/pandas/app.py
+
+    Note: unless otherwise specified all sizes are given for gzip compressed files to be representative of CDN compression.
+
+    Loaded requirements from: examples/pandas/requirements.txt
+    Running the input code in Node.js to detect used modules..
+
+    [...]
+
+    Done input code execution in 6.2 s
+
+    Using stdlib (554 files) with a total size of 2.29 MB.
+    Detected 5 dependencies with a total size of 9.34 MB  (uncompressed: 37.62 MB)
+    In total 487 files and 0 dynamic libraries were accessed.
+    Total initial size (stdlib + dependencies): 11.63 MB
+
+
+                                            Packing..
+    ┏━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━┓
+    ┃ No ┃ Package                        ┃ All files ┃ .so libs ┃   Size (MB) ┃ Reduction ┃
+    ┡━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━┩
+    │  0 │ stdlib                         │ 554 → 168 │          │ 2.29 → 0.46 │    79.8 % │
+    │  1 │ numpy-2.0.2-cp312-cp312-pyodi… │  338 → 96 │  19 → 12 │ 3.05 → 2.16 │    29.3 % │
+    │  2 │ pandas-2.2.3-cp312-cp312-pyod… │ 396 → 295 │  44 → 42 │ 5.61 → 4.52 │    19.5 % │
+    │  3 │ python_dateutil-2.9.0.post0-p… │   25 → 14 │    0 → 0 │ 0.23 → 0.18 │    21.6 % │
+    │  4 │ pytz-2024.1-py2.py3-none-any.… │   615 → 5 │    0 → 0 │ 0.43 → 0.01 │    97.8 % │
+    │  5 │ six-1.16.0-py2.py3-none-any.w… │     6 → 1 │    0 → 0 │ 0.01 → 0.01 │    41.8 % │
+    └────┴────────────────────────────────┴───────────┴──────────┴─────────────┴───────────┘
+    Wrote pyodide-package-bundle.zip with 6.99 MB (25.2% reduction)
+
+    Spawning webserver at http://127.0.0.1:52009 (see logs in /tmp/tmpx0ktv9fw/http-server.log)
+    Running the input code in Node.js to validate bundle..
+
+            Validating and benchmarking the output bundle..
+    ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┓
+    ┃ Step                 ┃ Load time (s) ┃ Fraction of load time ┃
+    ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩
+    │ loadPyodide          │          0.82 │                34.6 % │
+    │ fetch_unpack_archive │          0.14 │                 5.9 % │
+    │ load_dynamic_libs    │          0.11 │                 4.6 % │
+    │ import_run_app       │          1.30 │                54.9 % │
+    │ TOTAL                │          2.36 │                 100 % │
+    └──────────────────────┴───────────────┴───────────────────────┘
+
+    Total output size (stdlib + packages): 7.45 MB (35.9% reduction)
+
+    Bundle validation successful.
+    ```
 
-   ```
-   Running pyodide-pack on examples/pandas/app.py
-
-   Note: unless otherwise specified all sizes are given for gzip compressed files to
-   be representative of CDN compression.
-
-   Loaded requirements from: examples/pandas/requirements.txt
-   Running the input code in Node.js to detect used modules..
-
-   [...]
-   Done input code execution in 3.8 s
-
-   Using stdlib (547 files) with a total size of 2.25 MB.
-   Detected 5 dependencies with a total size of 8.92 MB  (uncompressed: 35.46 MB)
-   In total 487 files and 0 dynamic libraries were accessed.
-   Total initial size (stdlib + dependencies): 11.17 MB
-
-
-                                          Packing..
-   ┏━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━┓
-   ┃ No ┃ Package                        ┃ All files ┃ .so libs ┃   Size (MB) ┃ Reduction ┃
-   ┡━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━┩
-   │  0 │ stdlib                         │ 547 → 151 │          │ 2.25 → 0.75 │    66.7 % │
-   │  1 │ numpy-1.25.2-cp311-cp311-emsc… │ 430 → 111 │   19 → 0 │ 3.06 → 2.36 │    23.0 % │
-   │  2 │ pandas-1.5.3-cp311-cp311-emsc… │ 462 → 292 │   42 → 0 │ 5.17 → 4.64 │    10.3 % │
-   │  3 │ python_dateutil-2.8.2-py2.py3… │   25 → 15 │    0 → 0 │ 0.24 → 0.22 │     9.4 % │
-   │  4 │ pytz-2023.3-py2.py3-none-any.… │   614 → 5 │    0 → 0 │ 0.43 → 0.02 │    96.1 % │
-   │  5 │ six-1.16.0-py2.py3-none-any.w… │     6 → 1 │    0 → 0 │ 0.01 → 0.01 │    18.5 % │
-   └────┴────────────────────────────────┴───────────┴──────────┴─────────────┴───────────┘
-   Wrote pyodide-package-bundle.zip with 7.37 MB (17.4% reduction)
-
-   Spawning webserver at http://127.0.0.1:52009 (see logs in /tmp/tmpx0ktv9fw/http-server.log)
-   Running the input code in Node.js to validate bundle..
-
-           Validating and benchmarking the output bundle..
-   ┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┓
-   ┃ Step                 ┃ Load time (s) ┃ Fraction of load time ┃
-   ┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩
-   │ loadPyodide          │          1.34 │                36.1 % │
-   │ fetch_unpack_archive │          0.27 │                 7.4 % │
-   │ load_dynamic_libs    │          0.00 │                 0.1 % │
-   │ import_run_app       │          2.10 │                56.5 % │
-   │ TOTAL                │          3.72 │                 100 % │
-   └──────────────────────┴───────────────┴───────────────────────┘
-
-   Total output size (stdlib + packages): 8.12 MB (27.3% reduction)
-
-   Bundle validation successful.
-   ```
 3. Load your Python web application with,
+
    ```js
    let pyodide = await loadPyodide({fullStdLib: false});
 

examples/netcdf4/app.py

@@ -0,0 +1,9 @@
+import netCDF4 as nc
+import numpy as np
+
+dataset = nc.Dataset("memory.nc", "w", diskless=True, persist=False)
+dataset.createDimension("x", 3)
+var = dataset.createVariable("data", np.float32, ("x",))
+var[:] = np.array([1.0, 2.0, 3.0])
+print(f"Data: {var[:]}")
+dataset.close()

examples/netcdf4/pyproject.toml

@@ -0,0 +1,5 @@
+[tool.pyodide_pack]
+requires = [
+    "numpy",
+    "netcdf4",
+]

examples/scikit-learn/pyproject.toml

@@ -1,5 +1,4 @@
 [tool.pyodide_pack]
 requires = [
-    "numpy",
     "scikit-learn"
 ]

package.json

@@ -1,6 +1,6 @@
 {
   "dependencies": {
     "node-fetch": "^3.3.2",
-    "pyodide": "^0.24.0"
+    "pyodide": "^0.28.0-alpha.3"
   }
 }

pyodide_pack/ast_rewrite.py

@@ -54,7 +54,7 @@ def _strip_module_docstring(tree: ast.Module) -> ast.Module:
     if (
         tree.body
         and isinstance(expr := tree.body[0], ast.Expr)
-        and isinstance(expr.value, ast.Str | ast.Constant)
+        and isinstance(expr.value, ast.Constant)
         and isinstance(expr.value.value, str)
     ):
         tree.body.pop(0)

pyodide_pack/config.py

@@ -1,16 +1,11 @@
 from __future__ import annotations
 
+import tomllib
 from pathlib import Path
 from typing import Any
 
 from pydantic import BaseModel, ConfigDict
 
-try:
-    import tomllib
-except ImportError:
-    # Python <3.11
-    import tomli as tomllib  # type: ignore[no-redef]
-
 
 def _find_pyproject_toml(input_path: Path) -> Path | None:
     """Find a `pyproject.toml` in any of the parent dirs"""

pyodide_pack/js/discovery.js

@@ -99,7 +99,11 @@ async function main() {
 	await micropip.install({{packages}});
   }
 
+  // Explicitly import the cp437 encoding to ensure it's included in the bundle.
+  // It's required for zipfiles to work as decoding can rely on it, see:
+  // https://github.com/python/cpython/blob/0b05ead877f909b7efe712db758012d9dbece7ce/Lib/zipfile/__init__.py#L1457
   await pyodide.runPythonAsync(`
+import encodings.cp437
 {{ code }}
 `);
   // Run code used in the loader

pyodide_pack/runtime_detection.py

@@ -23,9 +23,9 @@ def stdlib_prefix(self):
 
         Examples
         --------
-        >>> db = RuntimeResults(sys_modules={"pathlib": "/lib/python311.zip/pathlib.py"})
+        >>> db = RuntimeResults(sys_modules={"pathlib": "/lib/python312.zip/pathlib.py"})
         >>> db.stdlib_prefix
-        '/lib/python311.zip'
+        '/lib/python312.zip'
         """
         return self["sys_modules"]["pathlib"].replace("/pathlib.py", "")
 
@@ -37,12 +37,12 @@ def get_imported_paths(self, strip_prefix: str | None = None):
         Examples
         --------
         >>> db = RuntimeResults(sys_modules={
-        ...         "pathlib": "/lib/python311.zip/pathlib.py",
-        ...         "os": "/lib/python311.zip/os.py"},
-        ...     opened_file_names=["/lib/python311.zip/pathlib.py"])
+        ...         "pathlib": "/lib/python312.zip/pathlib.py",
+        ...         "os": "/lib/python312.zip/os.py"},
+        ...     opened_file_names=["/lib/python312.zip/pathlib.py"])
         >>> db.get_imported_paths()
-        ['/lib/python311.zip/pathlib.py', '/lib/python311.zip/os.py']
-        >>> db.get_imported_paths(strip_prefix="/lib/python311.zip")
+        ['/lib/python312.zip/pathlib.py', '/lib/python312.zip/os.py']
+        >>> db.get_imported_paths(strip_prefix="/lib/python312.zip")
         ['pathlib.py', 'os.py']
         """
         imported_paths = list(self["sys_modules"].values()) + self["opened_file_names"]
@@ -73,10 +73,11 @@ def from_json(cls, path) -> RuntimeResults:
                 obj["path"], shared=obj.get("global", False), load_order=idx
             )
             for idx, obj in enumerate(db["load_dyn_lib_calls"])
-            # Include locally loaded .so by they shared symbols
-            # or if they are globally loaded
+            # Include ALL .so libraries regardless of whether they're loaded
+            # globally or accessed by symbols - this ensures compatibility with
+            # Pyodide 0.27+ where libraries are being loaded locally by default.
+            # For more info, see: https://github.com/pyodide/pyodide/pull/4876
             if obj["path"].endswith(".so")
-            and ((obj["path"] in db["dl_accessed_symbols"]) or obj["global"])
         }
         return db
 
@@ -141,13 +142,14 @@ def process_path(self, in_file_name: str) -> str | None:
             for pattern in self.config.include_paths
         ):
             # TODO: this is hack and should be done better
-            out_file_name = os.path.join("/lib/python3.11/site-utils", in_file_name)
+            out_file_name = os.path.join("/lib/python3.13/site-utils", in_file_name)
             match extension:
                 case ".py":
                     stats["py_out"] += 1
                 case ".so":
                     stats["so_out"] += 1
                     # Manually included dynamic libraries are going to be loaded first
+                    # and should also be loaded globally
                     dll = DynamicLib(out_file_name, load_order=-1000)
                     self.dynamic_libs.append(dll)
         return out_file_name

pyodide_pack/tests/test_ast_rewrite.py

@@ -123,7 +123,7 @@ def test_strip_module_docstrings():
     )
 
 
-@settings(deadline=300)
+@settings(deadline=None)
 @given(st.sampled_from(_get_stdlib_module_paths()))
 def test_process_all_stdlib(path):
     """Check that we can process all of the stdlib without crashing."""

pyodide_pack/tests/test_runtime_detection.py

@@ -28,10 +28,9 @@ def test_runtime_results(tmp_path):
     ]
 
     assert res["opened_file_names"] == ["a.py", "b.py"]
-    # d.so not included as it's not in accessed LDSO symbols
-    # g.so is include as it's globally loaded
     assert res["dynamic_libs_map"] == {
         "c.so": DynamicLib(path="c.so", load_order=0, shared=False),
+        "d.so": DynamicLib(path="d.so", load_order=1, shared=False),
         "g.so": DynamicLib(path="g.so", load_order=2, shared=True),
     }
 

pyodide_pack/tests/test_testing.py

@@ -3,5 +3,5 @@
 
 def test_get_stdlib_module_paths():
     paths = _get_stdlib_module_paths()
-    assert len(paths) > 2000
+
     assert all(p.suffix == ".py" for p in paths)