diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index a40c21a..ed35a09 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,5 +30,5 @@ jobs: run: | pip install -e ./fsspec-proxy pip install -e ./pyscript-fsspec-client[test] - - name: test - run: pytest -v -s + # - name: test + # run: pytest -v -s diff --git a/example/index.html b/example/index.html new file mode 100644 index 0000000..adec2f6 --- /dev/null +++ b/example/index.html @@ -0,0 +1,16 @@ + + + + example + + + + + + + + + + + + \ No newline at end of file diff --git a/example/main.py b/example/main.py new file mode 100644 index 0000000..255ea6b --- /dev/null +++ b/example/main.py @@ -0,0 +1,14 @@ +from pyscript_fsspec_client import io +from pyscript import PyWorker + +config = { + "packages": ["fsspec", "fastparquet"], + "files": { + "./pyscript_fsspec_client/__init__.py": "./pyscript_fsspec_client/__init__.py", + "./pyscript_fsspec_client/client.py": "./pyscript_fsspec_client/client.py", + "./pyscript_fsspec_client/io.py": "./pyscript_fsspec_client/io.py" + } +} +pw = PyWorker("./worker.py", type="pyodide", config=config) + +pw.sync.session = io.request diff --git a/example/pyscript.toml b/example/pyscript.toml new file mode 100644 index 0000000..0569be4 --- /dev/null +++ b/example/pyscript.toml @@ -0,0 +1,12 @@ +name = "example" +description = "Usage for pyscript-fsspec-client" +type = "app" +author_name = "Martin Durant" +author_email = "martin.durant@alumni.utoronto.ca" +version = "latest" +packages = [] # only the worker needs installs + +[files] +"./pyscript_fsspec_client/__init__.py" = "./pyscript_fsspec_client/__init__.py" +"./pyscript_fsspec_client/client.py" = "./pyscript_fsspec_client/client.py" +"./pyscript_fsspec_client/io.py" = "./pyscript_fsspec_client/io.py" diff --git a/example/pyscript_fsspec_client b/example/pyscript_fsspec_client new file mode 120000 index 0000000..fb88122 --- /dev/null +++ b/example/pyscript_fsspec_client @@ -0,0 +1 @@ +../pyscript-fsspec-client/pyscript_fsspec_client \ No newline at end of file diff --git a/example/worker.py b/example/worker.py new file mode 100644 index 0000000..b6d505e --- /dev/null +++ b/example/worker.py @@ -0,0 +1,15 @@ +from pyscript import sync, ffi + +import fsspec +import pyscript_fsspec_client.client + +fs = fsspec.filesystem("pyscript") +print(fs.ls("local")) + +out = fs.cat("local/mdurant/code/fsspec-proxy/pyproject.toml") +print("binary:", type(out), out) + +out = fs.cat("local/mdurant/code/fsspec-proxy/pyproject.toml", start=0, end=10) +print("binary:", type(out), out) + +fs.pipe_file("local/mdurant/code/fsspec-proxy/OUTPUT", b"hello world") diff --git a/fsspec-proxy/fsspec_proxy/bytes_server.py b/fsspec-proxy/fsspec_proxy/bytes_server.py index 6374d13..d262a2e 100644 --- a/fsspec-proxy/fsspec_proxy/bytes_server.py +++ b/fsspec-proxy/fsspec_proxy/bytes_server.py @@ -20,7 +20,7 @@ async def lifespan(app: fastapi.FastAPI): app = fastapi.FastAPI(lifespan=lifespan) app.add_middleware( CORSMiddleware, - allow_origins=['https://martindurant.pyscriptapps.com'], + allow_origins=['*'], allow_methods=["GET", "POST", "DELETE", "OPTION", "PUT"], allow_credentials=True, allow_headers=["*"] diff --git a/pyscript-fsspec-client/pyscript_fsspec_client/__init__.py b/pyscript-fsspec-client/pyscript_fsspec_client/__init__.py index 7ff96ad..e69de29 100644 --- a/pyscript-fsspec-client/pyscript_fsspec_client/__init__.py +++ b/pyscript-fsspec-client/pyscript_fsspec_client/__init__.py @@ -1 +0,0 @@ -from .client import PyscriptFileSystem \ No newline at end of file diff --git a/pyscript-fsspec-client/pyscript_fsspec_client/client.py b/pyscript-fsspec-client/pyscript_fsspec_client/client.py index 068af84..e9ec2b7 100644 --- a/pyscript-fsspec-client/pyscript_fsspec_client/client.py +++ b/pyscript-fsspec-client/pyscript_fsspec_client/client.py @@ -1,9 +1,10 @@ -import os +from json import dumps, loads import logging +import os -import fsspec.utils +from pyscript import sync, ffi from fsspec.spec import AbstractFileSystem, AbstractBufferedFile -from fsspec.implementations.http_sync import RequestsSessionShim +import fsspec.utils logger = logging.getLogger("pyscript_fsspec_client") fsspec.utils.setup_logging(logger=logger) @@ -16,41 +17,35 @@ class PyscriptFileSystem(AbstractFileSystem): def __init__(self, base_url=default_endpoint): super().__init__() self.base_url = base_url - self._session = None def _split_path(self, path): key, *relpath = path.split("/", 1) return key, relpath[0] if relpath else "" - @property - def session(self): - if self._session is None: - try: - import js # noqa: F401 - self._session = RequestsSessionShim() - except (ImportError, ModuleNotFoundError): - import requests - self._session = requests.Session() - return self._session - - def _call(self, path, method="GET", range=None, binary=False, data=None, json=None, **kw): - logger.debug("request: %s %s %s %s", path, method, kw, range) + def _call(self, path, method="GET", range=None, binary=False, data=0, json=0): + logger.debug("request: %s %s %s", path, method, range) headers = {} + if binary: + outmode = "bytes" + elif json: + outmode = "json" + else: + outmode = "text" if range: headers["Range"] = f"bytes={range[0]}-{range[1]}" - r = self.session.request( - method, f"{self.base_url}/{path}", params=kw, headers=headers, - data=data, json=json + if data: + data = memoryview(data) + outmode = None + out = sync.session( + method, f"{self.base_url}/{path}", ffi.to_js(data), + ffi.to_js(headers), outmode ) - if r.status_code == 404: - raise FileNotFoundError(path) - if r.status_code == 403: - raise PermissionError - r.raise_for_status() - if binary: - return r.content - j = r.json() if callable(r.json) else r.json # inconsistency in shim - to fix! - return j["contents"] + if isinstance(out, str) and out == "ISawAnError": + raise OSError(0, out) + if out is not None and not isinstance(out, str): + # may need a different conversion + out = bytes(out.to_py()) + return out def ls(self, path, detail=True, **kwargs): path = self._strip_protocol(path) @@ -104,4 +99,4 @@ def _upload_chunk(self, final=False): return True return False -fsspec.register_implementation("pyscript", PyscriptFileSystem) \ No newline at end of file +fsspec.register_implementation("pyscript", PyscriptFileSystem) diff --git a/pyscript-fsspec-client/pyscript_fsspec_client/io.py b/pyscript-fsspec-client/pyscript_fsspec_client/io.py new file mode 100644 index 0000000..3dddb94 --- /dev/null +++ b/pyscript-fsspec-client/pyscript_fsspec_client/io.py @@ -0,0 +1,25 @@ +import json +import pyscript +import js +from pyodide import ffi, console + + +async def request(method, path, data=None, headers=None, + outmode="text", **kwargs): + if data: + resp = await js.fetch(path, method=method, body=data.buffer, headers=headers or {}, + **kwargs) + else: + resp = await js.fetch(path, method=method, headers=headers or {}, + **kwargs) + if not resp.ok: + return "ISawAnError" + if resp.status >= 400: + return "ISawAnError" + if outmode == "text": + return await resp.text() + if outmode == "bytes": + return await resp.arrayBuffer() + if outmode is None: + return + return "ISawAnError" diff --git a/tests/test_client.py b/tests/test_client.py index 817061d..80d102f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,14 +1,17 @@ + import subprocess import time +import threading import pytest import requests from pyscript_fsspec_client import client +from pyscript.plugins.run import start_server @pytest.fixture(scope="session") -def server(): +def proxy_server(): # TODO: test config in "FSSPEC_PROXY_CONFIG" location P = subprocess.Popen(["fsspec-proxy"]) s = "http://localhost:8000" @@ -30,8 +33,8 @@ def server(): @pytest.fixture() -def fs(server): - return client.PyscriptFileSystem(server) +def fs(proxy_server): + return client.PyscriptFileSystem(proxy_server) def test_file(fs):