diff --git a/.github/workflows/stdlib.yml b/.github/workflows/stdlib.yml new file mode 100644 index 00000000..8ff674cb --- /dev/null +++ b/.github/workflows/stdlib.yml @@ -0,0 +1,36 @@ +name: Stdlib +on: + push: + branches: [main, master] + pull_request: + +jobs: + test: + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + with: + submodules: true + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + - run: cargo test -p quickjs-jit-stdlib + - run: cargo test -p quickjs-jit-stdlib --all-features + - uses: actions/setup-python@v6 + with: + python-version: '3.11' + - run: python scripts/check-stdlib-jit.py + + msrv: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + submodules: true + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: '1.89.0' + - run: cargo check -p quickjs-jit-stdlib diff --git a/CHANGELOG.md b/CHANGELOG.md index 52281ad5..464ef8b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Add `quickjs-jit-stdlib`, a thin facade over external LLRT standard modules, with host-owned JIT compatibility and no bundled LLRT sources. + - Add pre-generated bindings for `riscv64gc-unknown-linux-gnu` and `riscv64a23-unknown-linux-gnu` - JIT M2: Tier 1 and Tier 2 now support the remaining comparison, bitwise, shift, `%`, unary numeric, constant, stack-shuffle, tail-call, `null`, and empty-string opcodes; the JIT ABI diff --git a/Cargo.toml b/Cargo.toml index 75d4be6f..02041c21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "jit", "benchmarks", "macro", + "stdlib", "examples/native-module", "examples/module-loader", "examples/import-attributes", diff --git a/README.md b/README.md index 0f169577..43171b66 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,11 @@ and use Cargo source patches from that revision for both `quickjs-jit-core` and `quickjs-jit-sys`. Do not combine the published 0.12.2 runtime with 0.12.3 core or sys crates. +The optional [`quickjs-jit-stdlib`](stdlib/README.md) package provides a thin +facade over external LLRT Buffer, Crypto, Path, URL and Zlib dependencies. +It contains no LLRT implementation sources. JIT hosts retain their application-owned +`rquickjs` compatibility patch. Stdlib requires Rust 1.89+ and is Git-only for now. + [![github](https://img.shields.io/badge/github-longbridge/rquickjs-8da0cb.svg?style=for-the-badge&logo=github)](https://github.com/longbridge/rquickjs) [![crates](https://img.shields.io/crates/v/quickjs-jit.svg?style=for-the-badge&color=fc8d62&logo=rust)](https://crates.io/crates/quickjs-jit) [![docs](https://img.shields.io/badge/docs.rs-quickjs--jit-66c2a5?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyByb2xlPSJpbWciIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgdmlld0JveD0iMCAwIDUxMiA1MTIiPjxwYXRoIGZpbGw9IiNmNWY1ZjUiIGQ9Ik00ODguNiAyNTAuMkwzOTIgMjE0VjEwNS41YzAtMTUtOS4zLTI4LjQtMjMuNC0zMy43bC0xMDAtMzcuNWMtOC4xLTMuMS0xNy4xLTMuMS0yNS4zIDBsLTEwMCAzNy41Yy0xNC4xIDUuMy0yMy40IDE4LjctMjMuNCAzMy43VjIxNGwtOTYuNiAzNi4yQzkuMyAyNTUuNSAwIDI2OC45IDAgMjgzLjlWMzk0YzAgMTMuNiA3LjcgMjYuMSAxOS45IDMyLjJsMTAwIDUwYzEwLjEgNS4xIDIyLjEgNS4xIDMyLjIgMGwxMDMuOS01MiAxMDMuOSA1MmMxMC4xIDUuMSAyMi4xIDUuMSAzMi4yIDBsMTAwLTUwYzEyLjItNi4xIDE5LjktMTguNiAxOS45LTMyLjJWMjgzLjljMC0xNS05LjMtMjguNC0yMy40LTMzLjd6TTM1OCAyMTQuOGwtODUgMzEuOXYtNjguMmw4NS0zN3Y3My4zek0xNTQgMTA0LjFsMTAyLTM4LjIgMTAyIDM4LjJ2LjZsLTEwMiA0MS40LTEwMi00MS40di0uNnptODQgMjkxLjFsLTg1IDQyLjV2LTc5LjFsODUtMzguOHY3NS40em0wLTExMmwtMTAyIDQxLjQtMTAyLTQxLjR2LS42bDEwMi0zOC4yIDEwMiAzOC4ydi42em0yNDAgMTEybC04NSA0Mi41di03OS4xbDg1LTM4Ljh2NzUuNHptMC0xMTJsLTEwMiA0MS40LTEwMi00MS40di0uNmwxMDItMzguMiAxMDIgMzguMnYuNnoiPjwvcGF0aD48L3N2Zz4K)](https://docs.rs/quickjs-jit) diff --git a/scripts/check-stdlib-jit.py b/scripts/check-stdlib-jit.py new file mode 100644 index 00000000..4919712d --- /dev/null +++ b/scripts/check-stdlib-jit.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Test the facade from an external JIT host with an application-owned patch.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import tomllib + +root = Path(__file__).resolve().parents[1] +bindings = tomllib.loads((root / "Cargo.toml").read_text()) +version = bindings["package"]["version"] + +with tempfile.TemporaryDirectory(prefix="stdlib-jit-host-") as directory: + host = Path(directory) + compat = host / "compat" + (compat / "src").mkdir(parents=True) + # The fixture belongs to this temporary application, never to the library. + features = {name: [f"upstream/{name}"] for name in bindings["features"]} + (compat / "Cargo.toml").write_text( + '[package]\nname = "rquickjs"\n' + f'version = "{version}"\nedition = "2021"\npublish = false\n' + '[dependencies]\nupstream = { package = "quickjs-jit", ' + f'path = {json.dumps(str(root))}, default-features = false }}\n' + '[features]\n' + + ''.join(f'{name} = {json.dumps(values)}\n' for name, values in features.items()) + ) + (compat / "src/lib.rs").write_text('#![no_std]\npub use upstream::*;\n') + (host / "tests").mkdir() + shutil.copyfile(root / "stdlib/tests/modules.rs", host / "tests/modules.rs") + (host / "Cargo.toml").write_text( + '[package]\nname = "stdlib-jit-host"\nversion = "0.0.0"\n' + 'edition = "2021"\npublish = false\n[workspace]\n' + '[dependencies]\nrquickjs = { package = "quickjs-jit", ' + f'path = {json.dumps(str(root))}, features = ["futures", "loader", "macro", "half"] }}\n' + f'quickjs-jit-stdlib = {{ path = {json.dumps(str(root / "stdlib"))} }}\n' + 'tokio = { version = "1", features = ["macros", "rt", "time"] }\n' + '[features]\nparallel = ["quickjs-jit-stdlib/parallel", "rquickjs/parallel"]\n' + '[patch.crates-io]\nrquickjs = { path = "compat" }\n' + ) + env = dict(os.environ) + env.setdefault("CARGO_TARGET_DIR", str(root / "target/stdlib-jit-host")) + metadata = json.loads(subprocess.check_output( + ["cargo", "metadata", "--format-version", "1"], cwd=host, env=env + )) + packages = metadata["packages"] + assert not any(p["name"] in {"rquickjs-core", "rquickjs-sys", "rquickjs-macro"} + for p in packages), "Host resolved original bindings alongside JIT" + assert sum(p["name"] == "quickjs-jit-sys" for p in packages) == 1 + llrt = [p for p in packages if p["name"].startswith("llrt_")] + assert llrt and all((p["source"] or "").startswith("git+https://github.com/awslabs/llrt") + for p in llrt), "LLRT must remain an external Git dependency" + for flags in ([], ["--all-features"]): + subprocess.run(["cargo", "test", *flags], cwd=host, env=env, check=True) diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml new file mode 100644 index 00000000..7f326c1d --- /dev/null +++ b/stdlib/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "quickjs-jit-stdlib" +version = "0.12.7" +edition = "2021" +rust-version = "1.89" +license = "MIT" +description = "Thin facade for LLRT standard modules used with quickjs-jit" +repository = "https://github.com/longbridge/quickjs-jit" +readme = "README.md" +publish = false + +[features] +parallel = ["rquickjs/parallel"] + +[dependencies] +# Use LLRT's binding identity. JIT hosts must patch rquickjs at their workspace root. +rquickjs = { version = "0.12.2", default-features = false, features = ["futures", "half", "loader", "macro", "std"] } +llrt_buffer = { git = "https://github.com/awslabs/llrt", rev = "7b95c82a9b15e7ddfb2778eca4b5a63111e74f51" } +llrt_crypto = { git = "https://github.com/awslabs/llrt", rev = "7b95c82a9b15e7ddfb2778eca4b5a63111e74f51", default-features = false, features = ["crypto-rust"] } +llrt_path = { git = "https://github.com/awslabs/llrt", rev = "7b95c82a9b15e7ddfb2778eca4b5a63111e74f51" } +llrt_url = { git = "https://github.com/awslabs/llrt", rev = "7b95c82a9b15e7ddfb2778eca4b5a63111e74f51" } +llrt_zlib = { git = "https://github.com/awslabs/llrt", rev = "7b95c82a9b15e7ddfb2778eca4b5a63111e74f51", default-features = false, features = ["compression-rust"] } + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt", "time"] } diff --git a/stdlib/README.md b/stdlib/README.md new file mode 100644 index 00000000..e3962196 --- /dev/null +++ b/stdlib/README.md @@ -0,0 +1,63 @@ +# quickjs-jit-stdlib + +A thin facade over the external LLRT `buffer`, `crypto`, `path`, `url` and `zlib` +crates. This package contains dependency declarations, re-exports and context +initialization only; LLRT sources remain in the upstream repository and are +fetched by Cargo. Dependencies are pinned to LLRT revision +`7b95c82a9b15e7ddfb2778eca4b5a63111e74f51`, using RustCrypto and Rust +brotli/flate2 with zstd's native backend. + +```rust +use quickjs_jit_stdlib as stdlib; +use rquickjs::{Context, Module, Runtime}; + +let runtime = Runtime::new()?; +runtime.set_loader(stdlib::resolver(), stdlib::loader()); +let context = Context::full(&runtime)?; +context.with(|ctx| { + stdlib::init(&ctx)?; + Module::evaluate(ctx, "app", "import { Buffer } from 'buffer';")? + .finish::<()>() +})?; +# Ok::<(), rquickjs::Error>(()) +``` + +Hosts can compose their own loader with `buffer::BufferModule`, +`crypto::CryptoModule`, `path::PathModule`, `url::UrlModule` and +`zlib::ZlibModule`. Call `init` once per fresh context to install globals. +The host owns Tokio execution and QuickJS job polling for asynchronous APIs. +This package does not install filesystem, process, network or fetch modules. +Rust 1.89 or newer is required. The `parallel` feature forwards to `rquickjs`. + +## JIT integration and GPUI Shell migration + +The facade uses the same `rquickjs` package identity as LLRT. Without a host +patch it uses upstream rquickjs, not quickjs-jit. For a JIT application, retain +an application-owned compatibility crate named `rquickjs` that re-exports +`quickjs-jit` and forwards the required Cargo features. Its quickjs-jit dependency +must use exactly the same source and revision as the host's binding dependency. + +GPUI Shell should replace its five direct `llrt_*` dependencies with this package, +use the corresponding `quickjs_jit_stdlib::{buffer,crypto,path,url,zlib}` paths, +and retain its existing local `rquickjs-compat` crate and root patch: + +```toml +# Application workspace root; patches in library manifests do not propagate. +[patch.crates-io] +rquickjs = { path = "crates/shell/rquickjs-compat" } +``` + +The three global initializers can be replaced by `quickjs_jit_stdlib::init(ctx)`. +Permission checks and host-specific loaders stay in the application. This package +does not ship a compatibility crate or apply a workspace patch. + +## Distribution and verification + +This package is currently Git-only (`publish = false`). A single public facade +does not remove LLRT's transitive dependencies: crates.io publication still needs +compatible registry releases and a solution for the binding compatibility layer. + +Run `cargo test -p quickjs-jit-stdlib --all-features` to test the facade with +upstream rquickjs. With Python 3.11+, run `python3 scripts/check-stdlib-jit.py` to test an isolated +external host against the local quickjs-jit bindings, using a temporary host-owned +compatibility crate. Neither command copies LLRT sources into this repository. diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs new file mode 100644 index 00000000..b8c88892 --- /dev/null +++ b/stdlib/src/lib.rs @@ -0,0 +1,45 @@ +#![doc = include_str!("../README.md")] + +pub use llrt_buffer as buffer; +pub use llrt_crypto as crypto; +pub use llrt_path as path; +pub use llrt_url as url; +pub use llrt_zlib as zlib; + +use rquickjs::{ + loader::{BuiltinResolver, ModuleLoader}, + Ctx, Result, +}; + +/// Module names provided by this facade. +pub const MODULE_NAMES: &[&str] = &["buffer", "crypto", "path", "url", "zlib"]; + +/// A resolver for the five standard modules. Compose it with host resolvers. +pub fn resolver() -> BuiltinResolver { + MODULE_NAMES + .iter() + .fold(BuiltinResolver::default(), |resolver, name| { + resolver.with_module(*name) + }) +} + +/// A loader for the five standard modules. Compose it with host loaders. +pub fn loader() -> ModuleLoader { + ModuleLoader::default() + .with_module("buffer", buffer::BufferModule) + .with_module("crypto", crypto::CryptoModule) + .with_module("path", path::PathModule) + .with_module("url", url::UrlModule) + .with_module("zlib", zlib::ZlibModule) +} + +/// Install the Buffer, URL and Crypto globals into a fresh context, in order. +/// +/// Call once per context before evaluating a module using this library. +/// This does not install host I/O modules, start Tokio, or drive pending jobs. +pub fn init(ctx: &Ctx<'_>) -> Result<()> { + buffer::init(ctx)?; + url::init(ctx)?; + crypto::init(ctx)?; + Ok(()) +} diff --git a/stdlib/tests/modules.rs b/stdlib/tests/modules.rs new file mode 100644 index 00000000..ef8dccde --- /dev/null +++ b/stdlib/tests/modules.rs @@ -0,0 +1,77 @@ +use quickjs_jit_stdlib as stdlib; +use rquickjs::{CatchResultExt, Context, Module, Runtime}; + +fn evaluate(source: &str) { + let runtime = Runtime::new().unwrap(); + runtime.set_loader(stdlib::resolver(), stdlib::loader()); + let context = Context::full(&runtime).unwrap(); + context.with(|ctx| { + stdlib::init(&ctx).unwrap(); + Module::evaluate(ctx.clone(), "consumer", source) + .unwrap() + .finish::<()>() + .catch(&ctx) + .unwrap(); + }); +} + +#[test] +fn modules_share_buffer_url_and_quickjs_types() { + evaluate( + r#" + import { Buffer as ImportedBuffer } from 'buffer'; + import path from 'path'; + import { URL as ImportedURL } from 'url'; + if (ImportedBuffer !== Buffer || ImportedURL !== URL) throw Error('different globals'); + if (Buffer.from('hello').toString('base64') !== 'aGVsbG8=') throw Error('buffer'); + if (path.normalize('./file') !== 'file') throw Error('path'); + if (new URL('/child?q=1', 'https://example.com/base').hostname !== 'example.com') throw Error('url'); + "#, + ); +} + +#[test] +fn crypto_hash_and_compression_roundtrip() { + evaluate( + r#" + import { createHash } from 'crypto'; + import { gzipSync, gunzipSync } from 'zlib'; + const digest = createHash('sha256').update('abc').digest('hex'); + if (digest !== 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad') throw Error(digest); + const input = Buffer.from('external standard library'); + if (gunzipSync(gzipSync(input)).toString() !== input.toString()) throw Error('compression'); + "#, + ); +} + +#[test] +fn module_definitions_can_be_composed_by_a_host() { + let _: rquickjs::loader::ModuleLoader = rquickjs::loader::ModuleLoader::default() + .with_module("buffer", stdlib::buffer::BufferModule) + .with_module("crypto", stdlib::crypto::CryptoModule) + .with_module("path", stdlib::path::PathModule) + .with_module("url", stdlib::url::UrlModule) + .with_module("zlib", stdlib::zlib::ZlibModule); +} + +#[tokio::test] +async fn async_blob_crypto_and_compression_use_the_host_executor() { + let runtime = rquickjs::AsyncRuntime::new().unwrap(); + runtime + .set_loader(stdlib::resolver(), stdlib::loader()) + .await; + let context = rquickjs::AsyncContext::full(&runtime).await.unwrap(); + context.async_with(async |ctx| { + stdlib::init(&ctx).unwrap(); + Module::evaluate(ctx.clone(), "async-consumer", r#" + import { gzip, gunzip } from 'zlib'; + const text = 'async standard modules'; + if (await new Blob([text]).text() !== text) throw Error('Blob.text'); + const digest = await crypto.subtle.digest('SHA-256', Buffer.from('abc')); + if (Buffer.from(digest).toString('hex') !== 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad') throw Error('subtle.digest'); + const compressed = await new Promise((resolve, reject) => gzip(Buffer.from(text), (error, value) => error ? reject(error) : resolve(value))); + const plain = await new Promise((resolve, reject) => gunzip(compressed, (error, value) => error ? reject(error) : resolve(value))); + if (plain.toString() !== text) throw Error('async gzip'); + "#).unwrap().into_future::<()>().await.catch(&ctx).unwrap(); + }).await; +}