diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1c679a9..3dc945a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,9 @@ name: docs on: workflow_dispatch: push: - pull_request: + branches: + - main + # pull_request: defaults: @@ -38,7 +40,7 @@ jobs: environment-name: pyjs-wasm condarc: | channels: - - https://repo.mamba.pm/emscripten-forge + - https://repo.mamba.pm/emscripten-forge-4x - conda-forge - name: build the docs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d3575b5..8fe8ef3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,11 +19,7 @@ jobs: strategy: fail-fast: false - matrix: - include: - - emsdk_ver: "3.1.73" - python_version: "3.13" - pybind11_version: "<3" + steps: - uses: actions/checkout@v2 @@ -34,7 +30,7 @@ jobs: environment-name: pyjs-wasm condarc: | channels: - - https://repo.prefix.dev/emscripten-forge-dev + - https://repo.prefix.dev/emscripten-forge-4x - conda-forge @@ -48,14 +44,14 @@ jobs: micromamba create -n pyjs-build-wasm \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-dev\ + -c https://repo.prefix.dev/emscripten-forge-4x \ -c https://repo.mamba.pm/conda-forge \ --yes \ - python=${{matrix.python_version}} \ - "pybind11${{matrix.pybind11_version}}" \ + "python=3.13" \ + "pybind11<3" \ nlohmann_json pybind11_json numpy \ pytest bzip2 sqlite zlib zstd libffi \ - exceptiongroup emscripten-abi==${{matrix.emsdk_ver}} \ + exceptiongroup emscripten-abi>=4 \ openssl liblzma @@ -87,10 +83,10 @@ jobs: micromamba activate pyjs-wasm micromamba create -n pyjs-build-wasm-with-numpy \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-dev\ + -c https://repo.prefix.dev/emscripten-forge-4x \ -c https://repo.mamba.pm/conda-forge \ --yes \ - "python=${{matrix.python_version}}" pytest numpy exceptiongroup + "python=3.13" pytest numpy exceptiongroup - name: Test in browser-main @@ -100,7 +96,7 @@ jobs: browser-main \ --conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm-with-numpy \ --mount $(pwd)/tests:/tests \ - --mount $(pwd)/module/pyjs:/lib/python${{matrix.python_version}}/site-packages/pyjs \ + --mount $(pwd)/module/pyjs:/lib/python3.13/site-packages/pyjs \ --script main.py \ --work-dir /tests \ --pyjs-dir $(pwd)/build \ @@ -116,7 +112,7 @@ jobs: browser-worker \ --conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm-with-numpy \ --mount $(pwd)/tests:/tests \ - --mount $(pwd)/module/pyjs:/lib/python${{matrix.python_version}}/site-packages/pyjs \ + --mount $(pwd)/module/pyjs:/lib/python3.13/site-packages/pyjs \ --script main.py \ --work-dir /tests \ --pyjs-dir $(pwd)/build \ @@ -128,10 +124,10 @@ jobs: micromamba activate pyjs-wasm micromamba create -n pyjs-build-wasm-no-numpy \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-dev\ + -c https://repo.prefix.dev/emscripten-forge-4x \ -c https://repo.mamba.pm/conda-forge \ --yes \ - "python=${{matrix.python_version}}" pytest exceptiongroup + "python=3.13" pytest exceptiongroup - name: Test in browser-main-no-numpy run: | @@ -142,7 +138,7 @@ jobs: browser-main \ --conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm-no-numpy \ --mount $(pwd)/tests:/tests \ - --mount $(pwd)/module/pyjs:/lib/python${{matrix.python_version}}/site-packages/pyjs \ + --mount $(pwd)/module/pyjs:/lib/python3.13/site-packages/pyjs \ --script main.py \ --work-dir /tests \ --pyjs-dir $(pwd)/build \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ea2e36..f0ac6b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,14 +115,16 @@ target_link_libraries(pyjs PRIVATE ${PYTHON_UTIL_LIBS} pybind11::embed) target_compile_options(pyjs PUBLIC --std=c++17 + PUBLIC -msimd128 PUBLIC -Wno-deprecated - PUBLIC "SHELL: -fexceptions" + PUBLIC "SHELL: -fwasm-exceptions" ) target_link_options(pyjs - PRIVATE -lembind PUBLIC -Wno-unused-command-line-argument - PUBLIC "SHELL: -fexceptions" + PUBLIC "SHELL: -fwasm-exceptions" + PUBLIC "SHELL: -lembind" + # PUBLIC "SHELL: -fsanitize=address" #PUBLIC "SHELL:-s EXPORT_EXCEPTION_HANDLING_HELPERS" #PUBLIC "SHELL:-s EXCEPTION_CATCHING_ALLOWED=['we only want to allow exception handling in side modules']" ) @@ -198,44 +200,54 @@ target_link_libraries(pyjs_runtime_browser PRIVATE pyjs pybind11::embed ${PYTHO target_compile_options(pyjs_runtime_browser PUBLIC --std=c++17 PUBLIC -Wno-deprecated + PUBLIC -msimd128 PUBLIC "SHELL: -s ENVIRONMENT=${ENVIRONMENT}" - PUBLIC "SHELL: -fexceptions" - #PUBLIC "SHELL:-s EXPORT_EXCEPTION_HANDLING_HELPERS" + PUBLIC "SHELL: -fwasm-exceptions" + PUBLIC "SHELL: -sSUPPORT_LONGJMP" + PUBLIC "SHELL: -s EXPORT_EXCEPTION_HANDLING_HELPERS" PUBLIC "SHELL: -s FORCE_FILESYSTEM" - PUBLIC "SHELL: -s LZ4=1" - PUBLIC "SHELL: -flto" - # PUBLIC "SHELL: -s WASM_BIGINT" + PUBLIC "SHELL: -s EXPORT_ALL=1" + # PUBLIC "SHELL: -flto" + # PUBLIC "SHELL: -s MAXIMUM_MEMORY=2GB" + # PUBLIC "SHELL: -fsanitize=address" ) target_link_options(pyjs_runtime_browser - PRIVATE -lembind PUBLIC -Wno-unused-command-line-argument PUBLIC "SHELL: -s MODULARIZE=1" PUBLIC "SHELL: -s EXPORT_NAME=\"createModule\"" - PUBLIC "SHELL: -s EXPORT_ES6=0" - PUBLIC "SHELL: -s USE_ES6_IMPORT_META=0" - PUBLIC "SHELL: -s DEMANGLE_SUPPORT=0" - PUBLIC "SHELL: -s ASSERTIONS=0" + PUBLIC "SHELL: -s EXPORT_ALL=1" + PUBLIC "SHELL: -s EXPORT_EXCEPTION_HANDLING_HELPERS" + # PUBLIC "SHELL: -s EXPORT_ES6=0" + # PUBLIC "SHELL: -s DEMANGLE_SUPPORT=0" + # PUBLIC "SHELL: -s ASSERTIONS=0" PUBLIC "SHELL: -s ALLOW_MEMORY_GROWTH=1" - PUBLIC "SHELL: -s EXIT_RUNTIME=1" - PUBLIC "SHELL: -s WASM=1" + # PUBLIC "SHELL: -s EXIT_RUNTIME=1" + PUBLIC "SHELL: -s INITIAL_MEMORY=20971520" + PUBLIC "SHELL: -s MAXIMUM_MEMORY=4GB" + # PUBLIC "SHELL: -s WASM=1" PUBLIC "SHELL: -s USE_PTHREADS=0" PUBLIC "SHELL: -s ENVIRONMENT=${ENVIRONMENT}" - PUBLIC "SHELL: -fexceptions" + PUBLIC "SHELL: -fwasm-exceptions" + PUBLIC "SHELL: -sSUPPORT_LONGJMP" PUBLIC "SHELL: -s MAIN_MODULE=1" - PUBLIC "SHELL: -s ENVIRONMENT=${ENVIRONMENT}" - PUBLIC "SHELL: -s TOTAL_STACK=16mb" - PUBLIC "SHELL: -s INITIAL_MEMORY=64mb" + # PUBLIC "SHELL: -s MAXIMUM_MEMORY=2GB" PUBLIC "SHELL: -s FORCE_FILESYSTEM" - PUBLIC "SHELL: -s LZ4=1" PUBLIC "SHELL: --post-js pyjs_post.js" PUBLIC "SHELL: --pre-js pyjs_pre.js" - PUBLIC "SHELL: -flto" - PUBLIC "SHELL: -lidbfs.js" - PUBLIC "SHELL: -s WASM_BIGINT" + # PUBLIC "SHELL: -flto" + # PUBLIC "SHELL: -lidbfs.js" + PUBLIC "SHELL: -lembind" + # PUBLIC "SHELL: -fsanitize=address" ) +# target_link_options(pyjs_runtime_browser PRIVATE +# "-sEXPORTED_FUNCTIONS=['_emscripten_dlopen_promise','_dlerror','FS']" +# ) + + + install(TARGETS pyjs_runtime_browser DESTINATION ${CMAKE_INSTALL_PREFIX}/lib_js/pyjs) install(FILES diff --git a/build_mkdocs.sh b/build_mkdocs.sh index f97b0b6..19700a1 100755 --- a/build_mkdocs.sh +++ b/build_mkdocs.sh @@ -18,7 +18,7 @@ if [ ! -d "$WASM_ENV_PREFIX" ]; then echo "Creating wasm env $WASM_ENV_NAME" micromamba create -n $WASM_ENV_NAME \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-dev\ + -c https://repo.prefix.dev/emscripten-forge-4x\ -c https://repo.prefix.dev/conda-forge \ --yes \ python=$PYTHON_VERSION "pybind11<3" nlohmann_json pybind11_json numpy \ diff --git a/environment-dev.yml b/environment-dev.yml index 676239d..ef42e07 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -1,7 +1,8 @@ name: pyjs-wasm channels: - conda-forge - - https://repo.prefix.dev/emscripten-forge-dev + - https://repo.prefix.dev/emscripten-forge-4x + dependencies: - cmake - pip @@ -19,7 +20,7 @@ dependencies: - mkdocstrings - mkdocstrings-python - mkdocs-material - - empack >=3.2.0 + - empack >=6.0.0 - jupyter_server # to enable contents - jupyterlite - jupyterlite-xeus >= 3.1.8 @@ -27,7 +28,7 @@ dependencies: - notebook >=7,<8 # to include the extension to switch between JupyterLab and Notebook # pyjs_code_runner dev deps - hatchling - - emscripten_emscripten-wasm32 + - emscripten_emscripten-wasm32 == 4.0.9 - pip: - JLDracula - pyjs_code_runner >= 3.0 diff --git a/include/pyjs/post_js/fixes.js b/include/pyjs/post_js/fixes.js index 3260295..32060e6 100644 --- a/include/pyjs/post_js/fixes.js +++ b/include/pyjs/post_js/fixes.js @@ -7,3 +7,11 @@ Module['PATH'] = PATH Module['LDSO'] = LDSO Module['getDylinkMetadata'] = getDylinkMetadata Module['loadDynamicLibrary'] = loadDynamicLibrary +Module['getPromise'] = getPromise +Module['promiseMap'] = promiseMap + +Module['stackSave'] = stackSave +Module['stackRestore'] = stackRestore +Module['runtimeKeepalivePush'] = runtimeKeepalivePush +Module['UTF8ToString'] = UTF8ToString +Module['stringToUTF8OnStack'] = stringToUTF8OnStack \ No newline at end of file diff --git a/include/pyjs/pre_js/dynload/dynload.js b/include/pyjs/pre_js/dynload/dynload.js index 8ee2482..bd9b341 100644 --- a/include/pyjs/pre_js/dynload/dynload.js +++ b/include/pyjs/pre_js/dynload/dynload.js @@ -1,17 +1,3 @@ -const memoize = (fn) => { - let cache = {}; - return (...args) => { - let n = args[0]; - if (n in cache) { - return cache[n]; - } else { - let result = fn(n); - cache[n] = result; - return result; - } - }; -}; - function createLock() { let _lock = Promise.resolve(); @@ -26,215 +12,110 @@ function createLock() { return acquireLock; } -function isInSharedLibraryPath(prefix, libPath){ - if (libPath.startsWith("/")){ - const dirname = libPath.substring(0, libPath.lastIndexOf("/")); - if(prefix == "/"){ - return (dirname == `/lib`); - } - else{ - return (dirname == `${prefix}/lib`); - } - } - else{ - return false; + _lock = createLock(); + + + + // * local + // * global + // * undefined // when we dont + function libraryType (path) { + if (path.includes("cpython-3") && path.includes("-wasm32-emscripten.so")) { + return "local"; } -} + return undefined; + } + + function getFilenameFromPath(path) { + const parts = path.split("/"); + return parts[parts.length - 1]; + } async function loadDynlibsFromPackage( prefix, python_version, pkg_file_name, - pkg_is_shared_library, dynlibPaths, - ) { - - // for(const path of dynlibPaths){ - // console.log(path); - // } - - // assume that shared libraries of a package are located in .libs directory, - // following the convention of auditwheel. - if(prefix == "/"){ - var sitepackages = `/lib/python${python_version[0]}.${python_version[1]}/site-packages` - } - else{ - var sitepackages = `${prefix}/lib/python${python_version[0]}.${python_version[1]}/site-packages` - } - const auditWheelLibDir = `${sitepackages}/${ - pkg_file_name.split("-")[0] - }.libs`; - - // This prevents from reading large libraries multiple times. - const readFileMemoized = memoize(Module.FS.readFile); - - const forceGlobal = !!pkg_is_shared_library; - - - - let dynlibs = []; - - if (forceGlobal) { - dynlibs = dynlibPaths.map((path) => { - return { - path: path, - global: true, - }; - }); - } else { - const globalLibs = calculateGlobalLibs( - dynlibPaths, - readFileMemoized, - ); - - dynlibs = dynlibPaths.map((path) => { - const global = globalLibs.has(Module.PATH.basename(path)); - return { - path: path, - global: global || !! pkg_is_shared_library || isInSharedLibraryPath(prefix, path) || path.startsWith(auditWheelLibDir), - }; - }); - } - - dynlibs.sort((lib1, lib2) => Number(lib2.global) - Number(lib1.global)); - - for (const { path, global } of dynlibs) { - await loadDynlib(prefix, path, global, [auditWheelLibDir], readFileMemoized); - } - } - -function createDynlibFS( - prefix, - lib, - searchDirs, - readFileFunc ) { - const dirname = lib.substring(0, lib.lastIndexOf("/")); - let _searchDirs = searchDirs || []; - - if(prefix == "/"){ - _searchDirs = _searchDirs.concat([dirname], [`/lib`]); - } - else{ - _searchDirs = _searchDirs.concat([dirname], [`${prefix}/lib`]); - } + if(prefix != "/") + { + throw `only root prefix / is supported for loading shared libraries from packages, got: ${prefix}`; + } + // console.log("load shared objects from package:", pkg_file_name, dynlibPaths); + for (let i = 0; i < dynlibPaths.length; i++) { + let path = dynlibPaths[i]; - const resolvePath = (path) => { - //console.log("resolvePath", path); - - if (Module.PATH.basename(path) !== Module.PATH.basename(lib)) { - //console.debug(`Searching a library from ${path}, required by ${lib}`); + const isSymlink = FS.isLink(FS.lstat(path)); + if (isSymlink) { + continue; } - for (const dir of _searchDirs) { - const fullPath = Module.PATH.join2(dir, path); - //console.log("SERARCHING", fullPath); - if (Module.FS.findObject(fullPath) !== null) { - //console.log("FOUND", fullPath); - return fullPath; - } - } - return path; - }; + const releaseDynlibLock = await _lock(); + try { - let readFile = (path) => - Module.FS.readFile(resolvePath(path)); + // RTLD_LAZY 1 + // RTLD_NOW 2 + // RTLD_NOLOAD 4 + // RTLD_NODELETE 4096 + // RTLD_GLOBAL 256 + // RTLD_LOCAL 0 - if (readFileFunc !== undefined) { - readFile = (path) => readFileFunc(resolvePath(path)); - } - - const fs = { - findObject: (path, dontResolveLastLink) => { - let obj = Module.FS.findObject(resolvePath(path), dontResolveLastLink); - - if (obj === null) { - console.debug(`Failed to find a library: ${resolvePath(path)}`); + const libt = libraryType(path); + let flag = 2; // RTLD_NOW + if(libt === "local"){ + flag = 2 /* RTLD_NOW */ | 0 /* RTLD_LOCAL */; + } + else if(libt === "global"){ + flag = 2 /* RTLD_NOW */ | 256 /* RTLD_GLOBAL */; } - return obj; - }, - readFile: readFile, - }; + const stack = Module.stackSave(); + const pathUTF8 = Module.stringToUTF8OnStack(path); + try { + const pid = Module._emscripten_dlopen_promise(pathUTF8, flag); + Module.stackRestore(stack); + const promise = Module.getPromise(pid); + Module.promiseMap.free(pid); + // time it + const start = performance.now(); - return fs; -} + + await promise; -function calculateGlobalLibs( - libs, - readFileFunc -) { - let readFile = Module.FS.readFile; - if (readFileFunc !== undefined) { - readFile = readFileFunc; - } - const globalLibs = new Set(); - libs.forEach((lib) => { - const binary = readFile(lib); - const needed = Module.getDylinkMetadata(binary).neededDynlibs; - needed.forEach((lib) => { - globalLibs.add(lib); - }); - }); + const end = performance.now(); - return globalLibs; -} -// Emscripten has a lock in the corresponding code in library_browser.js. I -// don't know why we need it, but quite possibly bad stuff will happen without -// it. -const acquireDynlibLock = createLock(); + } catch (e) { + const dll_error_ptr = Module._dlerror(); + if (dll_error_ptr === 0) { + throw Error("unknown error loading shared library"); + } + const error = Module.UTF8ToString( + dll_error_ptr, + 512, // Use enough space for the error message + ); + const error_msg = error.trim(); -async function loadDynlib(prefix, lib, global, searchDirs, readFileFunc) { - if (searchDirs === undefined) { - searchDirs = []; - } - const releaseDynlibLock = await acquireDynlibLock(); - - try { - const fs = createDynlibFS(prefix, lib, searchDirs, readFileFunc); - - const libName = Module.PATH.basename(lib); - - // contains cpython-3 and with wasm32-emscripten - const is_cython_lib = libName.includes("cpython-3") && libName.includes("wasm32-emscripten"); - - // load cython library from full path - const load_name = is_cython_lib ? lib : libName; - - await Module.loadDynamicLibrary(load_name, { - loadAsync: true, - nodelete: true, - allowUndefined: true, - global: global && !is_cython_lib, - fs: fs - }) - - const dsoOnlyLibName = Module.LDSO.loadedLibsByName[libName]; - const dsoFullLib = Module.LDSO.loadedLibsByName[lib]; - - if(!dsoOnlyLibName && !dsoFullLib){ - console.execption(`Failed to load ${libName} from ${lib} LDSO not found`); - } - if(!is_cython_lib){ - if (!dsoOnlyLibName) { - Module.LDSO.loadedLibsByName[libName] = dsoFullLib - } - - if(!dsoFullLib){ - Module.LDSO.loadedLibsByName[lib] = dsoOnlyLibName; + throw new Error(`error loading shared library ${path} from package ${pkg_file_name}: ${error_msg}`); + } + + + } catch (e) { + throw e; + } finally { + console.log("release dynlib lock"); + releaseDynlibLock(); } - } finally { - releaseDynlibLock(); + } } diff --git a/include/pyjs/pre_js/init.js b/include/pyjs/pre_js/init.js index 9b11133..1651d1b 100644 --- a/include/pyjs/pre_js/init.js +++ b/include/pyjs/pre_js/init.js @@ -19,32 +19,57 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { var p = await Module['_wait_run_dependencies'](); if(prefix == "/"){ - Module.setenv("PYTHONHOME", `/`); - Module.setenv("PYTHONPATH", `/lib/python${version_str}/site-packages:/usr/lib/python${version_str}`); - + Module.setenv("LANG", "en_US.UTF-8"); + + // LC_COLLATE="C" + // LC_CTYPE="UTF-8" + // LC_MESSAGES="C" + // LC_MONETARY="C" + // LC_NUMERIC="C" + // LC_TIME="C" + // Module.setenv("LC_COLLATE", "C"); + // Module.setenv("LC_CTYPE", "UTF-8"); + // Module.setenv("LC_MESSAGES", "C"); + // Module.setenv("LC_MONETARY", "C"); + // Module.setenv("LC_NUMERIC", "C"); + // Module.setenv("LC_TIME", "C"); + + const pypath = `/lib/python${version_str}/site-packages:/usr/lib/python${version_str}/site-packages`; var side_path = `/lib/python${version_str}/site-packages`; + + console.log("PYTHONPATH",pypath); + console.log("SIDE_PATH",side_path); + Module.setenv("PYTHONHOME", `/`); + Module.setenv("PYTHONPATH", pypath); + Module.create_directories(side_path); } else{ - Module.setenv("PYTHONHOME", prefix); - Module.setenv("PYTHONPATH", `${prefix}/lib/python${version_str}/site-packages:/usr/lib/python${version_str}`); - var side_path = `${prefix}/lib/python${version_str}/site-packages`; + throw new Error("prefix must be / in wasm build"); } - Module.create_directories(side_path); - Module["_interpreter"] = new Module["_Interpreter"]() - var default_scope = Module["main_scope"]() + // Module["_interpreter"] = new Module["_Interpreter"]() + console.log("initialize interpreter"); + Module["_initialize_interpreter"](); + console.log("interpreter initialized"); + + + + var default_scope = Module["main_scope"]() Module["default_scope"] = default_scope; - Module['_py_objects'].push(Module["default_scope"]); Module['_py_objects'].push(Module["_interpreter"]); Module['exec'] = function(code, globals=default_scope, locals=default_scope) { + if(globals === undefined){ + globals = Module["globals"]() + } let ret = Module._exec(code, globals, locals) + // console.error("exec done"); if (ret.has_err) { throw ret } @@ -53,6 +78,9 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { Module['eval'] = function(code, globals=default_scope, locals=default_scope) { + if(globals === undefined){ + globals = Module["globals"]() + } let ret = Module._eval(code, globals, locals) if (ret.has_err) { throw ret @@ -62,6 +90,9 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { }; Module['eval_file'] = function(file, globals=default_scope, locals=default_scope) { + if(globals === undefined){ + globals = Module["globals"]() + } let ret = Module._eval_file(file, globals, locals) if (ret.has_err) { throw ret @@ -125,7 +156,6 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { Module['pyobject'].prototype.get = function(...keys) { - let types = keys.map(Module['_get_type_string']) let ret = this._raw_getitem(keys, types, keys.length) if (ret.has_err) { @@ -134,7 +164,7 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { return ret['ret'] } }; - if(verbose){console.log("in init phase 2 done");} + if(verbose){console.log("in init phase 1 done!!");} } Module['init_phase_2'] = function(prefix, python_version, verbose) { @@ -174,7 +204,7 @@ except Exception as e: return await Module._py_async_exec_eval.py_call(script, globals, locals) } if(verbose){console.log("assign pyobjects IV");} - Module._add_resolve_done_callback = Module.exec_eval(` + Module.exec(` import asyncio def _add_resolve_done_callback(future, resolve, reject): ensured_future = asyncio.ensure_future(future) @@ -185,8 +215,9 @@ def _add_resolve_done_callback(future, resolve, reject): reject(repr(err)) ensured_future.add_done_callback(done) -_add_resolve_done_callback `) + + Module._add_resolve_done_callback = Module.eval(`_add_resolve_done_callback`) Module._py_objects.push(Module._add_resolve_done_callback); diff --git a/include/pyjs/pre_js/load_pkg.js b/include/pyjs/pre_js/load_pkg.js index 9b89651..1687af3 100644 --- a/include/pyjs/pre_js/load_pkg.js +++ b/include/pyjs/pre_js/load_pkg.js @@ -34,7 +34,6 @@ def _py_untar(tarball_path, target_dir): WASM_BINARY_MAGIC = b"\\0asm" with file_path.open(mode="rb") as file: return file.read(4) == WASM_BINARY_MAGIC - target_dir = target_dir if target_dir == "": @@ -64,7 +63,7 @@ def _py_untar(tarball_path, target_dir): return s `) let shared_libs = Module.eval(`_py_untar("${tarball_path}", "${target_dir}")`) - + return JSON.parse(shared_libs) } @@ -247,8 +246,11 @@ Module["bootstrap_from_empack_packed_environment"] = async function let version = python_package.version.split(".").map(x => parseInt(x)); - if(verbose){console.log("start init_phase_1");} + await Module.init_phase_1(prefix, version, verbose); + + console.log("bootstrapping python done") + } @@ -276,20 +278,46 @@ Module["bootstrap_from_empack_packed_environment"] = async function let python_version = python_package.version.split(".").map(x => parseInt(x)); // fetch init python itself - console.log("--bootstrap_python"); if(verbose){ console.log("bootstrap_python"); } - let python_is_ready_promise = bootstrap_python(prefix, package_tarballs_root_url, python_package, verbose); + + let python_is_ready_promise = undefined; + try{ + python_is_ready_promise = bootstrap_python(prefix, package_tarballs_root_url, python_package, verbose); + console.log("waiting for python to be ready"); + await python_is_ready_promise; + console.log("python is ready"); + } + catch(e){ + console.log("error while bootstrapping python", e) + throw e + } // create array with size if(verbose){ console.log("fetchAndUntarAll"); } - let shared_libs = await Promise.all([ - ...packages.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)), - ...all_mount_points.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)) - ]); + // let shared_libs = await Promise.all([ + // ...packages.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)), + // ...all_mount_points.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)) + // ]); + + // console.log("shared_libs:", shared_libs); + + + // same as above, but sequentially to debug better + let shared_libs = [] + for(let pkg of packages){ + console.log("fetching and untarring package", pkg.name) + let sl = await fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose) + shared_libs.push(sl) + } + for(let pkg of all_mount_points){ + console.log("fetching and untarring mount point", pkg.name) + let sl = await fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose) + shared_libs.push(sl) + } if(verbose){ console.log("init_phase_2"); @@ -299,6 +327,14 @@ Module["bootstrap_from_empack_packed_environment"] = async function if(verbose){ console.log("init shared"); } + + + // in a worker we dont need to pre-load shared libraries + const is_worker = (typeof WorkerGlobalScope !== 'undefined') && (self instanceof WorkerGlobalScope); + if(is_worker){ + // skip_loading_shared_libs = true; + } + if(!skip_loading_shared_libs){ // instantiate all packages for (let i = 0; i < packages.length; i++) { @@ -309,16 +345,19 @@ Module["bootstrap_from_empack_packed_environment"] = async function for (let j = 0; j < shared_libs[i].length; j++) { let sl = shared_libs[i][j]; } + console.log(`loading #${shared_libs[i].length} shared libraries for package ${packages[i].name}`); await Module._loadDynlibsFromPackage( prefix, python_version, packages[i].name, - false, shared_libs[i] - ) + ); + console.log(`asd loading shared libraries for package ${packages[i].name} done`); } } } + console.log("loading shared libraries done PUSH"); + Module.runtimeKeepalivePush(); if(verbose){ console.log("done bootstrapping");} } diff --git a/src/convert.cpp b/src/convert.cpp index 86c2fcd..92e3cf6 100644 --- a/src/convert.cpp +++ b/src/convert.cpp @@ -15,14 +15,15 @@ namespace pyjs std::pair implicit_py_to_js(py::object& py_ret) { // py::module_ pyjs = py::module_::import("pyjs_utils"); - // const std::string info = pyjs.attr("implicit_convert_info")(py_ret).cast(); + // const std::string info = pyjs.attr("implicit_convert_info")(py_ret).cast(); const std::string info = py_ret.get_type().attr("__name__").str(); if (info == "int") { - return std::make_pair(em::val(py_ret.cast()),false); + auto emval = em::val(py_ret.cast()); + return std::make_pair(std::move(emval),false); } else if (info == "str") { diff --git a/src/export_js_module.cpp b/src/export_js_module.cpp index 5b15a39..95cefea 100644 --- a/src/export_js_module.cpp +++ b/src/export_js_module.cpp @@ -12,6 +12,9 @@ #include #include +#include +#include + namespace pyjs { @@ -19,12 +22,13 @@ namespace pyjs namespace py = pybind11; - em::val eval(const std::string & code, py::object & globals, py::object & locals) - { em::val ret = em::val::object(); + em::val eval(const std::string & code, py::object & globals , py::object & locals) + { + em::val ret = em::val::object();; + ret.set("has_err", false); try { py::object py_ret = py::eval(code, globals, locals); - ret.set("has_err",em::val(false)); auto [jsval, is_proxy] = implicit_py_to_js(py_ret); ret.set("ret",jsval); ret.set("is_proxy",is_proxy); @@ -44,10 +48,10 @@ namespace pyjs em::val exec(const std::string & code, py::object & globals, py::object & locals) { em::val ret = em::val::object(); + ret.set("has_err", false); try { py::exec(code, globals, locals); - ret.set("has_err",em::val(false)); return ret; } catch (py::error_already_set& e) @@ -55,12 +59,13 @@ namespace pyjs ret.set("has_err",em::val(true)); ret.set("message",em::val(std::string(e.what()))); ret.set("error",em::val(std::string(e.what()))); + return ret; } } - em::val eval_file(const std::string & filename, py::object & globals, py::object & locals) + em::val eval_file(const std::string & filename, py::object & globals, py::object & locals) { em::val ret = em::val::object(); try @@ -111,6 +116,12 @@ namespace pyjs .constructor<>() ; + em::function("_initialize_interpreter", +[]() + { + py::initialize_interpreter(false, 0, nullptr, false); + } + ); + em::function("_eval", &eval); em::function("_exec", &exec); em::function("_eval_file", &eval_file); @@ -126,15 +137,22 @@ namespace pyjs // py-object (proxy) export_py_object(); - // main scope + // // main scope em::function("main_scope",em::select_overload( []()->py::object{ + std::cout<<"get scope"<py::object{ + return py::globals(); + }); + em::function("cout", em::select_overload([](const std::string& val) { std::cout << val; })); @@ -142,6 +160,16 @@ namespace pyjs em::function("extract_exception_message", &extract_exception_message); + // em::function("_emscripten_dlopen_promise", +[](const std::string& filename, int flags) + // { + // return reinterpret_cast(emscripten_dlopen_promise(filename.c_str(), flags)); + // }); + + // em::function("_dlerror", +[]() + // { + // return std::string(dlerror()); + // }); + } } diff --git a/src/export_py_object.cpp b/src/export_py_object.cpp index 91cea56..b6e3a1a 100644 --- a/src/export_py_object.cpp +++ b/src/export_py_object.cpp @@ -88,7 +88,6 @@ namespace pyjs // implicit to py - py::list py_args; for (std::size_t i = 0; i < n_keys; ++i) { @@ -141,6 +140,7 @@ namespace pyjs em::select_overload( [](py::object& pyobject) -> em::val { + std::cout<<"call 0-ary"<( [](py::object& pyobject, em::val arg1) -> em::val { + std::cout<<"call 1-ary"< (1 << 30)) { // >1GB is suspicious + std::cout<<"Invalid filesize in tar header: "< 0) { bytes_read = fread(buff, 1, 512, a); if (bytes_read < 512) { @@ -277,6 +283,7 @@ namespace pyjs f = NULL; } } + std::cout<<"HERE WE ARE"<