From c040215fd686c491d0d9664521e00aec91af97de Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Sun, 13 Sep 2026 20:17:59 -0400 Subject: [PATCH] harness: adapt pinned Wasmtime component tests and async guests --- .github/justfile | 2 + .github/workflows/ci.yml | 5 +- .gitignore | 1 + Cargo.lock | 1 + crates/testgen/Cargo.toml | 1 + crates/testgen/src/main.rs | 238 ++++++- crates/testgen/tests/cli.rs | 131 ++++ docs/architecture.md | 9 +- docs/references.md | 10 +- harness/README.md | 62 ++ harness/deno.json | 4 +- harness/src/runner.ts | 38 +- harness/src/runtime-executor.ts | 13 +- harness/src/wasmtime-classifier.ts | 54 ++ harness/src/wasmtime-expectations.ts | 692 ++++++++++++++++++++ harness/src/wasmtime-spectest.ts | 80 +++ harness/tests/runner_unit_test.ts | 37 +- harness/tests/wasmtime_expectations_test.ts | 145 ++++ harness/tests/wasmtime_subprocess_test.ts | 61 ++ justfile | 25 +- runtime/tests/wasmtime/public_guests.ts | 317 +++++++++ tools/wasmtime-guests/.gitignore | 1 + tools/wasmtime-guests/build.ts | 154 +++++ tools/wasmtime/generate.ts | 27 + tools/wasmtime/metadata.json | 591 +++++++++++++++++ tools/wasmtime/run.ts | 204 ++++++ tools/wasmtime/source.ts | 58 ++ tools/wasmtime/subprocess.ts | 76 +++ tools/wasmtime/worker.ts | 23 + 29 files changed, 3005 insertions(+), 55 deletions(-) create mode 100644 crates/testgen/tests/cli.rs create mode 100644 harness/src/wasmtime-classifier.ts create mode 100644 harness/src/wasmtime-expectations.ts create mode 100644 harness/src/wasmtime-spectest.ts create mode 100644 harness/tests/wasmtime_expectations_test.ts create mode 100644 harness/tests/wasmtime_subprocess_test.ts create mode 100644 runtime/tests/wasmtime/public_guests.ts create mode 100644 tools/wasmtime-guests/.gitignore create mode 100644 tools/wasmtime-guests/build.ts create mode 100644 tools/wasmtime/generate.ts create mode 100644 tools/wasmtime/metadata.json create mode 100644 tools/wasmtime/run.ts create mode 100644 tools/wasmtime/source.ts create mode 100644 tools/wasmtime/subprocess.ts create mode 100644 tools/wasmtime/worker.ts diff --git a/.github/justfile b/.github/justfile index 6fa72b92..0ca09d24 100644 --- a/.github/justfile +++ b/.github/justfile @@ -63,6 +63,8 @@ core: @just gha::_step examples @just gha::_step test-translate @just gha::_step conformance + @just gha::_step test-wasmtime + @just gha::_step test-wasmtime-guests @just gha::_step sched-seeds # Page and worker realms share per-engine expectations; WebKit is best-effort. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81face4c..bd8093e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,10 @@ jobs: submodules: true # third_party/component-model: testgen + conformance corpus - uses: dtolnay/rust-toolchain@stable with: - targets: wasm32-unknown-unknown,wasm32-wasip2 + # wasm32-wasip1 builds the upstream Wasmtime async guests + # (tools/wasmtime-guests/build.ts); the corpus itself is not + # cargo-built, only converted (docs/architecture.md §11). + targets: wasm32-unknown-unknown,wasm32-wasip1,wasm32-wasip2 - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@v2 with: diff --git a/.gitignore b/.gitignore index 45e46010..db1fe6d0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # @polyengine/translator packaged asset (copied by `just shim`) translator/translator_shim.wasm harness/generated/ +harness/generated-wasmtime/ examples/guests/build/ node_modules/ dist/ diff --git a/Cargo.lock b/Cargo.lock index 3f48d98f..3e5ad776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -570,6 +570,7 @@ version = "0.0.0" dependencies = [ "anyhow", "json-from-wast", + "serde", "serde_json", "wast", ] diff --git a/crates/testgen/Cargo.toml b/crates/testgen/Cargo.toml index 61030495..687f58d8 100644 --- a/crates/testgen/Cargo.toml +++ b/crates/testgen/Cargo.toml @@ -12,4 +12,5 @@ path = "src/main.rs" anyhow = "1" json-from-wast = "0.258.0" serde_json = { version = "1", features = ["preserve_order"] } +serde = { version = "1", features = ["derive"] } wast = "258.0.0" diff --git a/crates/testgen/src/main.rs b/crates/testgen/src/main.rs index 3e28db0d..67489682 100644 --- a/crates/testgen/src/main.rs +++ b/crates/testgen/src/main.rs @@ -4,7 +4,7 @@ //! as libraries. See harness/README.md for the pipeline and schema documentation. //! //! Usage: -//! testgen [--test-dir DIR] [--out-dir DIR] [SUBDIR...] +//! testgen [--test-dir DIR] [--out-dir DIR] [--source-prefix PREFIX] [SUBDIR...] //! //! Defaults (resolved relative to the repository root, so this works from //! any working directory): @@ -18,6 +18,13 @@ use anyhow::{bail, Context, Result}; use std::path::{Path, PathBuf}; use std::process::ExitCode; +#[derive(serde::Serialize)] +struct SupplementaryFile { + path: String, + source: String, + directives: std::collections::BTreeMap, +} + fn main() -> ExitCode { match run() { Ok(failures) if failures == 0 => ExitCode::SUCCESS, @@ -46,14 +53,24 @@ fn run() -> Result { let mut test_dir = root.join("third_party/component-model/test"); let mut out_dir = root.join("harness/generated"); let mut subdirs: Vec = Vec::new(); + let mut source_prefix = "third_party/component-model/test".to_string(); + let mut source_revision: Option = None; let mut args = std::env::args().skip(1); while let Some(arg) = args.next() { match arg.as_str() { - "--test-dir" => test_dir = PathBuf::from(args.next().context("--test-dir needs a value")?), + "--test-dir" => { + test_dir = PathBuf::from(args.next().context("--test-dir needs a value")?) + } "--out-dir" => out_dir = PathBuf::from(args.next().context("--out-dir needs a value")?), + "--source-prefix" => { + source_prefix = args.next().context("--source-prefix needs a value")? + } + "--source-revision" => { + source_revision = Some(args.next().context("--source-revision needs a value")?) + } "--help" | "-h" => { - println!("usage: testgen [--test-dir DIR] [--out-dir DIR] [SUBDIR...]"); + println!("usage: testgen [--test-dir DIR] [--out-dir DIR] [--source-prefix PREFIX] [SUBDIR...]"); return Ok(0); } s if s.starts_with('-') => bail!("unknown flag: {s}"), @@ -64,6 +81,23 @@ fn run() -> Result { if !test_dir.is_dir() { bail!("test dir not found: {}", test_dir.display()); } + let canonical_input = test_dir.canonicalize()?; + let canonical_output = if out_dir.exists() { + // Resolve the complete existing path: its final component may itself + // be a symlink back into the input tree. + out_dir.canonicalize()? + } else { + out_dir.parent().unwrap_or(&out_dir).canonicalize()?.join( + out_dir + .file_name() + .context("output directory has no final component")?, + ) + }; + if canonical_input.starts_with(&canonical_output) + || canonical_output.starts_with(&canonical_input) + { + bail!("input and output directories must not overlap"); + } // Deterministic subdir set: sorted, filtered to requested names. let mut found: Vec = std::fs::read_dir(&test_dir) @@ -72,11 +106,25 @@ fn run() -> Result { .filter(|e| e.path().is_dir()) .map(|e| e.file_name().to_string_lossy().into_owned()) .collect(); + if std::fs::read_dir(&test_dir)? + .filter_map(|e| e.ok()) + .any(|e| e.path().extension().is_some_and(|ext| ext == "wast")) + { + found.push(String::new()); + } found.sort(); if !subdirs.is_empty() { for want in &subdirs { if !found.contains(want) { - bail!("no such test subdirectory: {want} (available: {})", found.join(", ")); + bail!( + "no such test subdirectory: {want} (available: {})", + found + .iter() + .filter(|s| !s.is_empty()) + .cloned() + .collect::>() + .join(", ") + ); } } found.retain(|d| subdirs.contains(d)); @@ -85,25 +133,36 @@ fn run() -> Result { let mut converted = 0usize; let mut total_commands = 0usize; let mut failures: Vec<(PathBuf, anyhow::Error)> = Vec::new(); + let mut generated_files = Vec::new(); + let mut supplementary_files = Vec::new(); + + if subdirs.is_empty() && out_dir.exists() { + std::fs::remove_dir_all(&out_dir) + .with_context(|| format!("cleaning {}", out_dir.display()))?; + } + std::fs::create_dir_all(&out_dir)?; for sub in &found { let in_sub = test_dir.join(sub); let out_sub = out_dir.join(sub); // Regenerate from scratch so deleted/renamed wast files leave no // stale outputs behind. - if out_sub.exists() { + if !sub.is_empty() && out_sub.exists() { std::fs::remove_dir_all(&out_sub) .with_context(|| format!("cleaning {}", out_sub.display()))?; } std::fs::create_dir_all(&out_sub) .with_context(|| format!("creating {}", out_sub.display()))?; - let mut wast_files: Vec = std::fs::read_dir(&in_sub) - .with_context(|| format!("reading {}", in_sub.display()))? - .filter_map(|e| e.ok()) - .map(|e| e.path()) - .filter(|p| p.extension().is_some_and(|e| e == "wast")) - .collect(); + let mut wast_files = if sub.is_empty() { + std::fs::read_dir(&test_dir)? + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|e| e == "wast")) + .collect() + } else { + recursive_wast_files(&in_sub)? + }; wast_files.sort(); for wast_path in wast_files { @@ -113,10 +172,18 @@ fn run() -> Result { .to_string_lossy() .into_owned(); // Stable, machine-independent source reference. + let relative = wast_path + .strip_prefix(&test_dir) + .expect("discovered below test dir"); let source_rel = format!( - "third_party/component-model/test/{sub}/{}", - wast_path.file_name().unwrap().to_string_lossy() + "{}/{}", + source_prefix.trim_end_matches('/'), + relative.to_string_lossy().replace('\\', "/") ); + let relative_parent = relative.parent().unwrap_or(Path::new("")); + let artifact_dir = out_dir.join(relative_parent); + std::fs::create_dir_all(&artifact_dir) + .with_context(|| format!("creating {}", artifact_dir.display()))?; let text = std::fs::read_to_string(&wast_path) .with_context(|| format!("reading {}", wast_path.display()))?; @@ -140,13 +207,25 @@ fn run() -> Result { Ok(wast) => { let n_artifacts = wast.wasms.len(); for (filename, bytes) in &wast.wasms { - std::fs::write(out_sub.join(filename), bytes) - .with_context(|| format!("writing {sub}/{filename}"))?; + std::fs::write(artifact_dir.join(filename), bytes).with_context(|| { + format!("writing {}/{filename}", relative_parent.display()) + })?; } let mut json = serde_json::to_string_pretty(&wast)?; json.push('\n'); - std::fs::write(out_sub.join(format!("{stem}.json")), json) - .with_context(|| format!("writing {sub}/{stem}.json"))?; + std::fs::write(artifact_dir.join(format!("{stem}.json")), json).with_context( + || format!("writing {}/{stem}.json", relative_parent.display()), + )?; + let generated = relative + .with_extension("json") + .to_string_lossy() + .replace('\\', "/"); + generated_files.push(generated.clone()); + supplementary_files.push(SupplementaryFile { + path: generated, + source: source_rel.clone(), + directives: parse_directives(&text)?, + }); println!( "converted {source_rel}: {} commands, {} artifacts", wast.commands.len(), @@ -162,22 +241,19 @@ fn run() -> Result { // Manifest: lets consumers (e.g. a browser runner without directory // listings) discover the generated JSON files. Sorted, deterministic. - let mut json_files: Vec = Vec::new(); - for sub in &found { - let out_sub = out_dir.join(sub); - let mut files: Vec = std::fs::read_dir(&out_sub)? - .filter_map(|e| e.ok()) - .map(|e| e.file_name().to_string_lossy().into_owned()) - .filter(|f| f.ends_with(".json")) - .map(|f| format!("{sub}/{f}")) - .collect(); - files.sort(); - json_files.extend(files); - } - let manifest = serde_json::json!({ "files": json_files }); + generated_files.sort(); + supplementary_files.sort_by(|a, b| a.path.cmp(&b.path)); + let manifest = serde_json::json!({ "files": generated_files }); let mut manifest_str = serde_json::to_string_pretty(&manifest)?; manifest_str.push('\n'); std::fs::write(out_dir.join("manifest.json"), manifest_str)?; + let metadata = serde_json::json!({ + "source_revision": source_revision, + "files": supplementary_files, + }); + let mut metadata_str = serde_json::to_string_pretty(&metadata)?; + metadata_str.push('\n'); + std::fs::write(out_dir.join("supplementary-metadata.json"), metadata_str)?; println!( "testgen: converted {converted} wast file(s), {total_commands} commands, {} failure(s)", @@ -189,6 +265,108 @@ fn run() -> Result { Ok(failures.len()) } +fn parse_directives(text: &str) -> Result> { + let mut directives = std::collections::BTreeMap::new(); + for line in text + .lines() + .take_while(|line| line.trim().is_empty() || line.starts_with(";;!")) + { + let Some(setting) = line.strip_prefix(";;!") else { + continue; + }; + let (key, value) = setting.split_once('=').context("invalid ;;! directive")?; + const KNOWN: &[&str] = &[ + "bulk_memory", + "component_model_async", + "component_model_async_stackful", + "component_model_error_context", + "component_model_fixed_length_lists", + "component_model_gc", + "component_model_implements", + "component_model_map", + "component_model_memory64", + "component_model_more_async_builtins", + "component_model_threading", + "exceptions", + "function_references", + "gc", + "gc_types", + "hogs_memory", + "memory64", + "multi_memory", + "reference_types", + ]; + if !KNOWN.contains(&key.trim()) { + bail!("unknown ;;! directive {}", key.trim()); + } + let value = match value.trim() { + "true" => true, + "false" => false, + other => bail!("unknown ;;! value {other:?} for {}", key.trim()), + }; + if directives.insert(key.trim().to_string(), value).is_some() { + bail!("duplicate ;;! directive {}", key.trim()); + } + } + Ok(directives) +} + +fn recursive_wast_files(root: &Path) -> Result> { + recursive_files_with_extension(root, "wast") +} + +fn recursive_files_with_extension(root: &Path, extension: &str) -> Result> { + let mut files = Vec::new(); + let mut dirs = vec![root.to_path_buf()]; + while let Some(dir) = dirs.pop() { + for entry in + std::fs::read_dir(&dir).with_context(|| format!("reading {}", dir.display()))? + { + let path = entry?.path(); + if path.is_dir() { + dirs.push(path); + } else if path.extension().is_some_and(|e| e == extension) { + files.push(path); + } + } + } + files.sort(); + Ok(files) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn recursive_discovery_preserves_distinct_parent_paths() { + let root = std::env::temp_dir().join(format!("polyengine-testgen-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("a/nested")).unwrap(); + std::fs::create_dir_all(root.join("b")).unwrap(); + std::fs::write(root.join("a/nested/same.wast"), "").unwrap(); + std::fs::write(root.join("b/same.wast"), "").unwrap(); + let got = recursive_wast_files(&root).unwrap(); + assert_eq!( + got, + vec![root.join("a/nested/same.wast"), root.join("b/same.wast")] + ); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn directives_are_strict() { + let got = parse_directives( + ";;! component_model_async = true\n;;! component_model_implements = false\n(component)", + ) + .unwrap(); + assert_eq!(got["component_model_async"], true); + assert_eq!(got["component_model_implements"], false); + assert!(parse_directives(";;! gc = maybe").is_err()); + assert!(parse_directives(";;! imaginary = true").is_err()); + } +} + fn pretty(mut e: wast::Error, path: &str, text: &str) -> anyhow::Error { e.set_path(std::path::Path::new(path)); e.set_text(text); diff --git a/crates/testgen/tests/cli.rs b/crates/testgen/tests/cli.rs new file mode 100644 index 00000000..20a04262 --- /dev/null +++ b/crates/testgen/tests/cli.rs @@ -0,0 +1,131 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn scratch(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("polyengine-testgen-{name}-{}", std::process::id())) +} + +fn write(path: &Path, text: &str) { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); +} + +fn run(input: &Path, output: &Path, selection: &[&str]) { + let status = Command::new(env!("CARGO_BIN_EXE_testgen")) + .args([ + "--test-dir", + input.to_str().unwrap(), + "--out-dir", + output.to_str().unwrap(), + ]) + .args(selection) + .status() + .unwrap(); + assert!(status.success()); +} + +fn manifest(output: &Path) -> Vec { + serde_json::from_str::( + &fs::read_to_string(output.join("manifest.json")).unwrap(), + ) + .unwrap()["files"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap().to_string()) + .collect() +} + +#[test] +fn root_nested_collisions_cleanup_and_subset_selection() { + let base = scratch("cli"); + let input = base.join("input"); + let output = base.join("output"); + let _ = fs::remove_dir_all(&base); + write(&input.join("same.wast"), "(component)"); + write(&input.join("a/same.wast"), "(component)"); + write(&input.join("b/same.wast"), "(component)"); + run(&input, &output, &[]); + assert_eq!( + manifest(&output), + ["a/same.json", "b/same.json", "same.json"] + ); + + fs::remove_file(input.join("a/same.wast")).unwrap(); + run(&input, &output, &[]); + assert_eq!(manifest(&output), ["b/same.json", "same.json"]); + assert!(!output.join("a/same.json").exists()); + + write(&input.join("a/other.wast"), "(component)"); + run(&input, &output, &["a"]); + assert_eq!(manifest(&output), ["a/other.json"]); + fs::remove_dir_all(base).unwrap(); +} + +#[test] +fn refuses_overlapping_input_and_output() { + let base = scratch("overlap"); + let input = base.join("input"); + write(&input.join("a/x.wast"), "(component)"); + let status = Command::new(env!("CARGO_BIN_EXE_testgen")) + .args([ + "--test-dir", + input.to_str().unwrap(), + "--out-dir", + input.join("out").to_str().unwrap(), + ]) + .status() + .unwrap(); + assert!(!status.success()); + fs::remove_dir_all(base).unwrap(); +} + +#[cfg(unix)] +#[test] +fn refuses_output_symlink_to_input_without_removing_source() { + use std::os::unix::fs::symlink; + + let base = scratch("output-symlink"); + let input = base.join("input"); + let output = base.join("output"); + let source = input.join("a/x.wast"); + let _ = fs::remove_dir_all(&base); + write(&source, "(component)"); + symlink(&input, &output).unwrap(); + + let status = Command::new(env!("CARGO_BIN_EXE_testgen")) + .args([ + "--test-dir", + input.to_str().unwrap(), + "--out-dir", + output.to_str().unwrap(), + "a", + ]) + .status() + .unwrap(); + assert!(!status.success()); + assert_eq!(fs::read_to_string(&source).unwrap(), "(component)"); + fs::remove_dir_all(base).unwrap(); +} + +#[cfg(unix)] +#[test] +fn subset_cleanup_does_not_follow_output_subdirectory_symlink() { + use std::os::unix::fs::symlink; + + let base = scratch("subdir-symlink"); + let input = base.join("input"); + let output = base.join("output"); + let source = input.join("a/x.wast"); + let _ = fs::remove_dir_all(&base); + write(&source, "(component)"); + fs::create_dir_all(&output).unwrap(); + symlink(input.join("a"), output.join("a")).unwrap(); + + run(&input, &output, &["a"]); + assert_eq!(fs::read_to_string(&source).unwrap(), "(component)"); + assert!(!output.join("a").is_symlink()); + assert!(output.join("a/x.json").is_file()); + fs::remove_dir_all(base).unwrap(); +} diff --git a/docs/architecture.md b/docs/architecture.md index 3db54d5d..eba9bbd3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -545,8 +545,13 @@ for deployment guidance. | `runtime/tests/conventions/` | Committed transcripts of the public host ABI, gated with protocol versioning | Wasmtime's component-model tests are supplementary reference material -([references.md](references.md)), not an additional corpus executed by the -current generation or gate paths. +([references.md](references.md)): a pinned Deno lane (`just test-wasmtime`, +`just test-wasmtime-guests`; see [harness/README.md](../harness/README.md#supplementary-wasmtime-coverage)) +converts and classifies Wasmtime's own WAST and drives two of its upstream +async guest binaries through the public embedder API, separately from — and +never merged into — the official corpus's generation, xfails, or engine-shell +and browser lanes above. It remains supplementary corroboration, not a claim +of full conformance against Wasmtime. `crates/testgen` uses `wast` and `json-from-wast` to convert WAST into JSON commands and wasm binaries. The TS harness distinguishes core modules diff --git a/docs/references.md b/docs/references.md index 7f7e9e3b..07102950 100644 --- a/docs/references.md +++ b/docs/references.md @@ -57,8 +57,14 @@ The revision is declared in [Cargo.toml](../Cargo.toml) and resolved in - [Adapter translation](https://github.com/bytecodealliance/wasmtime/blob/4675ee16b703b33948073a5ff6b961367371e7a1/crates/environ/src/component/translate/adapt.rs): how component linkage is translated into FACT adapters. - [Component-model tests at the same revision](https://github.com/bytecodealliance/wasmtime/tree/4675ee16b703b33948073a5ff6b961367371e7a1/tests/misc_testsuite/component-model): - supplementary reference material, not a corpus executed by the current - project gates or an independent check of the reused frontend. + converted and classified separately by `just test-wasmtime` (see + [harness/README.md](../harness/README.md#supplementary-wasmtime-coverage)); + supplementary reference material, not merged into the official corpus or + an independent check of the reused frontend. +- [Async test-programs guests at the same revision](https://github.com/bytecodealliance/wasmtime/tree/4675ee16b703b33948073a5ff6b961367371e7a1/crates/test-programs/src/bin): + `async_round_trip_stackless` and `async_short_reads`, built by + `just wasmtime-guests` and driven through the public embedder API by + `just test-wasmtime-guests`. ## Toolchain crates (pinned versions in lockfiles) diff --git a/harness/README.md b/harness/README.md index 98efd10c..0142a49b 100644 --- a/harness/README.md +++ b/harness/README.md @@ -68,6 +68,68 @@ All generated suite directories, including `async/` and `values/`, are run. Component-level value imports/exports remain outside the runtime's stated parity scope; do not confuse that feature with ordinary canonical-ABI values. +## Supplementary Wasmtime coverage + +`tools/wasmtime/` converts Wasmtime's own `tests/misc_testsuite/component-model` +WAST at the same `wasmtime-environ` revision pinned in `Cargo.lock`, and runs it +through this harness under a separate classifier +(`src/wasmtime-classifier.ts`, `src/wasmtime-expectations.ts`). It is +supplementary reference material (docs/architecture.md §11), not the official +Component Model corpus above: its expectations, exclusions, and results never +mix with `src/xfail.ts` or `generated/`. + +```sh +just test-wasmtime # from the repo root +just test-wasmtime-guests # public API scenarios, FIFO and seeded +``` + +From `harness/`, `deno task wasmtime` runs generation, shim-check, focused +harness tests, and the classifying runner. Guest builds require the Rust +`wasm32-wasip1` and `wasm32-unknown-unknown` targets plus `wasm-tools`. + +Results, including full per-class and per-file counts, are written to +`harness/generated-wasmtime/results.json` — read that file rather than a fixed +number from this document; counts shift as the pinned revision or translator +changes. As of the pinned revision above, the run classifies every command +executed: no unexpected failures or unexplained skips. Known-failure classes +and their tracking issues are declared in +`harness/src/wasmtime-expectations.ts` (`WASMTIME_FAILURE_CLASSES`); most map +to [#372](https://github.com/polymorph-components/polyengine/issues/372) +(runtime-semantics, diagnostic-mismatch, imported-module, cascade, +provider-control, exception-handling — plan v0 / diagnostic gaps against +Wasmtime's own assertions, not the spec corpus above). `deferred-threads` +skips map to [#12](https://github.com/polymorph-components/polyengine/issues/12). +A handful of files are excluded outright (unbounded memory stress, GC, or +Wasmtime-specific validation configuration this translator doesn't share) — +see `WASMTIME_EXCLUSIONS` for the current list and reasons; the run fails if an +exclusion goes stale (the file gone from the manifest). + +`just test-wasmtime-guests` builds the two upstream async guest binaries this +WAST corpus doesn't cover as executables — `async_round_trip_stackless` and +`async_short_reads` from `crates/test-programs/src/bin/` at the same locked +revision — and drives them through the public embedder API +(`runtime/tests/wasmtime/public_guests.ts`), once under FIFO scheduling and +once under `POLYENGINE_SCHED_SEED=1`. `just wasmtime-guests` +(`tools/wasmtime-guests/build.ts`) does the build alone: it clones the locked +revision into an ignored scratch checkout, builds with a separate +`CARGO_TARGET_DIR`, and refuses dirty source inputs, replacing a clean scratch +checkout when the pin changes. The cached +upstream checkout under cargo's git cache is read, never written. One of the +two guest scenarios asserts resource `own` ownership transfer across a +short-read stream (source wrappers invalidated after transfer, returned +wrappers exclusively own the value, drop is idempotent); this exercises the +public wrapper API's ownership bookkeeping, not Wasmtime's internal +resource-table representation, which this project makes no claim about. + +The WAST host provider uses raw resource reps, which do not expose the Rust +`Resource::owned()` flag. Its corresponding host-side ownership assertions +are not reproduced; the guest scenarios exercise public ownership transfer. + +Both gates report source provenance (`Cargo.lock`'s `wasmtime-environ` git +revision) and fail on drift — a stale locked revision, a modified guest +checkout, or a guest artifact that doesn't hash-match its build's own +provenance — rather than silently running an unreviewed rebuild. + ## Execution [`CommandExecutor`](src/executor.ts) separates command bookkeeping from engine diff --git a/harness/deno.json b/harness/deno.json index 7a94a090..8040ae0b 100644 --- a/harness/deno.json +++ b/harness/deno.json @@ -1,8 +1,10 @@ { "tasks": { "gen": "cargo run -q -p testgen --manifest-path ../Cargo.toml", + "gen-wasmtime": "deno run --allow-read=.. --allow-run ../tools/wasmtime/generate.ts", "shim-check": "test -f ../target/wasm32-unknown-unknown/release/translator_shim.wasm || cargo build -p translator-shim --manifest-path ../Cargo.toml --target wasm32-unknown-unknown --release", "test": "deno test --allow-read=.. --allow-env=CONFORMANCE_EXECUTOR,POLYENGINE_SCHED_SEED tests/", - "conformance": "deno task gen && deno task shim-check && deno task test" + "conformance": "deno task gen && deno task shim-check && deno task test", + "wasmtime": "deno task gen-wasmtime && deno task shim-check && deno test --allow-run --allow-read=.. tests/wasmtime_expectations_test.ts tests/wasmtime_subprocess_test.ts && deno run --allow-read=.. --allow-write=generated-wasmtime --allow-run --allow-env=POLYENGINE_SCHED_SEED ../tools/wasmtime/run.ts" } } diff --git a/harness/src/runner.ts b/harness/src/runner.ts index 69bb3209..e23af404 100644 --- a/harness/src/runner.ts +++ b/harness/src/runner.ts @@ -1,13 +1,7 @@ // JSON command runner: executes one testgen-generated command file against a // CommandExecutor, classifying every command as passed / failed / skipped. -import type { - Action, - ArtifactRef, - Command, - Kind, - WastJson, -} from "./schema.ts"; +import type { Action, ArtifactRef, Command, Kind, WastJson } from "./schema.ts"; import { type Artifact, type CommandExecutor, @@ -35,7 +29,8 @@ export function artifactKind(bytes: Uint8Array): Kind { const isModule = bytes.length >= 8 && bytes[0] === 0x00 && bytes[1] === 0x61 && bytes[2] === 0x73 && bytes[3] === 0x6d && - bytes[4] === 0x01 && bytes[5] === 0x00 && bytes[6] === 0x00 && bytes[7] === 0x00; + bytes[4] === 0x01 && bytes[5] === 0x00 && bytes[6] === 0x00 && + bytes[7] === 0x00; return isModule ? "module" : "component"; } @@ -189,7 +184,9 @@ class FileRunner { command.expected, outcome.values.length === 0 ? undefined - : (outcome.values.length === 1 ? outcome.values[0] : outcome.values), + : (outcome.values.length === 1 + ? outcome.values[0] + : outcome.values), ); if (mismatch !== undefined) throw new Error(mismatch); return undefined; @@ -230,7 +227,14 @@ class FileRunner { try { await this.executor.instantiate(artifact, "trap"); } catch (e) { - if (e instanceof TrapError) return undefined; + if (e instanceof TrapError) { + if (!trapMatches(command.text, e.message)) { + throw new Error( + `expected instantiation trap "${command.text}", got "${e.message}"`, + ); + } + return undefined; + } throw e; } throw new Error( @@ -302,7 +306,9 @@ class UnsupportedDirective extends Error {} * (V8/SpiderMonkey/JSC differ, e.g. for `unreachable`) is normalized here * against the suite's expected (typically wasmtime-worded) text instead. */ -const TRAP_MESSAGE_EQUIVALENTS: Array<[expectedPrefix: string, actualSubstrings: string[]]> = [ +const TRAP_MESSAGE_EQUIVALENTS: Array< + [expectedPrefix: string, actualSubstrings: string[]] +> = [ // resources/handle-table.wast: runtime/src/cabi/handles.ts Table.get/free. ["unknown handle index", ["table index out of range", "table entry empty"]], // resources/handle-table.wast: runtime/src/cabi/handles.ts lift/lowerOwn @@ -319,7 +325,11 @@ const TRAP_MESSAGE_EQUIVALENTS: Array<[expectedPrefix: string, actualSubstrings: // against the lowercase expected text). [ "wasm trap: wasm `unreachable` instruction executed", - ["unreachable", "unreachable executed", "Unreachable code should not be executed"], + [ + "unreachable", + "unreachable executed", + "Unreachable code should not be executed", + ], ], // async/big-interleaving-test.wast:836 asserts the SHORT form, plain // "unreachable". V8's and SpiderMonkey's spellings contain it as a @@ -340,7 +350,9 @@ const TRAP_MESSAGE_EQUIVALENTS: Array<[expectedPrefix: string, actualSubstrings: export function trapMatches(expected: string, actual: string): boolean { if (actual.includes(expected)) return true; for (const [prefix, actuals] of TRAP_MESSAGE_EQUIVALENTS) { - if (expected.startsWith(prefix) && actuals.some((a) => actual.includes(a))) { + if ( + expected.startsWith(prefix) && actuals.some((a) => actual.includes(a)) + ) { return true; } } diff --git a/harness/src/runtime-executor.ts b/harness/src/runtime-executor.ts index da80a130..968a9b71 100644 --- a/harness/src/runtime-executor.ts +++ b/harness/src/runtime-executor.ts @@ -12,6 +12,7 @@ import type { WireExport } from "@polyengine/runtime/plan"; import { Translator } from "@polyengine/runtime/shim"; import { type ComponentHandle, + type HostImports, instantiateComponent, } from "../../runtime/src/exec/mod.ts"; import { @@ -107,17 +108,22 @@ type Definition = export class RuntimeExecutor implements CommandExecutor { readonly #translator: Translator; + readonly #imports: HostImports; readonly #core = new CoreOnlyExecutor(); #definitions: Definition[] = []; #namedDefinitions = new Map(); #registered = new Map(); - private constructor(translator: Translator) { + private constructor(translator: Translator, imports: HostImports) { this.#translator = translator; + this.#imports = imports; } - static async create(shimWasm: Uint8Array): Promise { - return new RuntimeExecutor(await Translator.create(shimWasm)); + static async create( + shimWasm: Uint8Array, + imports: HostImports = {}, + ): Promise { + return new RuntimeExecutor(await Translator.create(shimWasm), imports); } validate( @@ -166,6 +172,7 @@ export class RuntimeExecutor implements CommandExecutor { plan, componentBytes: bytes, adapters, + imports: this.#imports, // The wast `invoke` directive is a BLOCKING call: an async-typed // export that goes idle with its task unresolved is a deadlock for // this runner, not a Promise to leave pending (#292). wasmtime's diff --git a/harness/src/wasmtime-classifier.ts b/harness/src/wasmtime-classifier.ts new file mode 100644 index 00000000..a53d5604 --- /dev/null +++ b/harness/src/wasmtime-classifier.ts @@ -0,0 +1,54 @@ +import type { CommandResult } from "./runner.ts"; +import { + expectedWasmtimeFailure, + expectedWasmtimeSkip, + WASMTIME_EXPECTATIONS, + WASMTIME_SKIP_EXPECTATIONS, +} from "./wasmtime-expectations.ts"; + +export type Classified = + | { status: "passed" } + | { status: "known-failure"; class: string } + | { status: "skip"; class: string } + | { status: "unexpected"; detail: string }; + +export function classify(file: string, result: CommandResult): Classified { + if (result.status === "passed") { + if ( + WASMTIME_EXPECTATIONS.some((e) => + e.file === file && e.line === result.line + ) || WASMTIME_SKIP_EXPECTATIONS.some((e) => + e.file === file && e.line === result.line + ) + ) { + return { status: "unexpected", detail: "stale expectation passed" }; + } + return { status: "passed" }; + } + if (result.status === "failed") { + const expected = expectedWasmtimeFailure( + file, + result.line, + result.detail ?? "", + ); + return expected === undefined + ? { + status: "unexpected", + detail: result.detail ?? "failure without detail", + } + : { status: "known-failure", class: expected.class }; + } + const expected = expectedWasmtimeSkip( + file, + result.line, + result.reason, + result.detail ?? "", + ); + if (expected !== undefined) { + return { status: "skip", class: expected.class }; + } + return { + status: "unexpected", + detail: `unclassified skip: ${result.detail}`, + }; +} diff --git a/harness/src/wasmtime-expectations.ts b/harness/src/wasmtime-expectations.ts new file mode 100644 index 00000000..e8419e4b --- /dev/null +++ b/harness/src/wasmtime-expectations.ts @@ -0,0 +1,692 @@ +export interface WasmtimeExpectation { + file: string; + line: number; + cause: string; + class: string; + status: "failed" | "skipped"; +} + +export interface WasmtimeExpectationGroup { + class: string; + files: Array<{ + file: string; + rows: Array<{ + lines: number[]; + cause: string; + status: "failed" | "skipped"; + }>; + }>; +} + +export interface WasmtimeFailureClass { + reason: string; + issue: string; +} + +export const WASMTIME_FAILURE_CLASSES: Readonly< + Record +> = { + "runtime-semantics": { + reason: "runtime result or state differs from the WAST assertion", + issue: "https://github.com/polymorph-components/polyengine/issues/372", + }, + "diagnostic-mismatch": { + reason: "trap diagnostics have not been proved equivalent", + issue: "https://github.com/polymorph-components/polyengine/issues/372", + }, + "imported-module": { + reason: "plan v0 cannot instantiate an imported core module", + issue: "https://github.com/polymorph-components/polyengine/issues/372", + }, + "provider-control": { + reason: "Wasmtime-native test control is unavailable and is not emulated", + issue: "https://github.com/polymorph-components/polyengine/issues/372", + }, + "exception-handling": { + reason: + "WebAssembly exception behavior is not represented by the harness verdict", + issue: "https://github.com/polymorph-components/polyengine/issues/372", + }, + "cascade": { + reason: "command has no current instance after its owning setup failure", + issue: "https://github.com/polymorph-components/polyengine/issues/372", + }, + "deferred-threads": { + reason: "deferred thread.new-indirect support is not implemented", + issue: "https://github.com/polymorph-components/polyengine/issues/12", + }, +}; + +export const WASMTIME_EXPECTATION_GROUPS: readonly WasmtimeExpectationGroup[] = + [ + { + class: "cascade", + files: [ + { + file: "alias-region-reexported-entities-to-imported-module.json", + rows: [{ + lines: [101, 102, 103], + cause: "Error: no current instance", + status: "failed", + }], + }, + { + file: "async/cancel-starting-subtask-does-not-leak.json", + rows: [{ + lines: [103], + cause: "Error: no current instance", + status: "failed", + }], + }, + { + file: "async/context-in-compositions.json", + rows: [{ + lines: [435, 436, 437], + cause: + "Error: expected return, got trap: cannot enter component instance 2 — instance poisoned by: Trap: guest trapped: unreachable", + status: "failed", + }], + }, + { + file: "async/context-in-resource-drop.json", + rows: [{ + lines: [325, 326, 327, 328], + cause: "Error: no current instance", + status: "failed", + }], + }, + { + file: "async/join-during-sync-read.json", + rows: [{ + lines: [66], + cause: "Error: no current instance", + status: "failed", + }], + }, + { + file: "async/stream-big-read-and-writes.json", + rows: [{ + lines: [43], + cause: + 'Error: expected trap "stream read/write count too large", got "cannot enter component instance 0 — instance poisoned by: RangeError: Invalid array length"', + status: "failed", + }], + }, + { + file: "async/task-deletion.json", + rows: [{ + lines: [323, 324, 325, 326, 327, 328, 329, 330, 331], + cause: "Error: no current instance", + status: "failed", + }], + }, + { + file: "async/task-return-traps.json", + rows: [{ + lines: [56, 91], + cause: "Error: no current instance", + status: "failed", + }], + }, + ], + }, + { + class: "diagnostic-mismatch", + files: [ + { + file: "async/backpressure-overflow.json", + rows: [{ + lines: [36], + cause: + 'Error: expected trap "backpressure counter overflow", got "backpressure counter underflow"', + status: "failed", + }], + }, + { + file: "async/cancel-host.json", + rows: [{ + lines: [256], + cause: + 'Error: expected trap "`subtask.cancel` called after terminal status delivered", got "subtask.cancel on a subtask whose resolution was already delivered"', + status: "failed", + }], + }, + { + file: "async/cancel-sync-and-waitable.json", + rows: [{ + lines: [68], + cause: + 'Error: expected trap "waitable cannot be used synchronously while added to a waitable set", got "future.cancel-write: synchronous cancel on an end that is in a waitable set"', + status: "failed", + }, { + lines: [121], + cause: + 'Error: expected trap "waitable cannot be used synchronously while added to a waitable set", got "future.cancel-read: synchronous cancel on an end that is in a waitable set"', + status: "failed", + }, { + lines: [174], + cause: + 'Error: expected trap "waitable cannot be used synchronously while added to a waitable set", got "stream.cancel-write: synchronous cancel on an end that is in a waitable set"', + status: "failed", + }, { + lines: [227], + cause: + 'Error: expected trap "waitable cannot be used synchronously while added to a waitable set", got "stream.cancel-read: synchronous cancel on an end that is in a waitable set"', + status: "failed", + }], + }, + { + file: "async/future-read.json", + rows: [{ + lines: [65], + cause: + 'Error: expected trap "wasm trap: cannot block a synchronous task before returning", got "wasm trap: deadlock detected: event loop cannot make further progress (export \'run\': every suspended activation is waiting on a suspension only this scheduler could resume, and none is ready)"', + status: "failed", + }], + }, + { + file: "async/intra-futures.json", + rows: [{ + lines: [55], + cause: + 'Error: expected trap "cannot read from and write to intra-component future/stream with non-numeric payload", got "cannot read from and write to intra-component future"', + status: "failed", + }], + }, + { + file: "async/intra-streams.json", + rows: [{ + lines: [56], + cause: + 'Error: expected trap "cannot read from and write to intra-component future/stream with non-numeric payload", got "cannot read from and write to intra-component stream"', + status: "failed", + }], + }, + { + file: "async/stream-cancel-finished-op.json", + rows: [{ + lines: [232, 234, 236], + cause: + 'Error: expected trap "cannot read after being notified that the writable end dropped", got "cannot read from stream after being notified that the writable end dropped"', + status: "failed", + }, { + lines: [239, 241, 243], + cause: + 'Error: expected trap "cannot write after being notified that the readable end dropped", got "cannot write to stream after being notified that the readable end dropped"', + status: "failed", + }, { + lines: [247, 249, 251], + cause: + 'Error: expected trap "cannot write after being notified that the readable end dropped", got "cannot write to future after previous write succeeded or readable end dropped"', + status: "failed", + }], + }, + { + file: "async/subtask-wait.json", + rows: [{ + lines: [82], + cause: + 'Error: expected trap "wasm `unreachable` instruction executed", got "guest trapped: unreachable"', + status: "failed", + }], + }, + { + file: "async/sync-and-async-waitable.json", + rows: [{ + lines: [123], + cause: + 'Error: expected trap "waitable cannot be used synchronously while added to a waitable set", got "synchronous future copy on an end that is in a waitable set"', + status: "failed", + }], + }, + { + file: "async/sync-call-context-trap.json", + rows: [{ + lines: [47], + cause: + 'Error: expected trap "wasm `unreachable` instruction executed", got "guest trapped: unreachable"', + status: "failed", + }], + }, + { + file: "async/task-return-traps.json", + rows: [{ + lines: [19, 104], + cause: + 'Error: expected trap "async-lifted export failed to produce a result", got "task finished all threads without resolving"', + status: "failed", + }, { + lines: [118], + cause: + 'Error: expected trap "invalid `task.return` signature and/or options for current task", got "task.return with a result type that is not the task\'s result type"', + status: "failed", + }, { + lines: [135, 150], + cause: + 'Error: expected trap "invalid `task.return` signature and/or options for current task", got "task.return with canonical options differing from the task\'s"', + status: "failed", + }], + }, + { + file: "async/trap-if-done.json", + rows: [{ + lines: [599, 601, 603, 605, 608, 610, 612, 614], + cause: + 'Error: expected trap "cannot write after being notified that the readable end dropped", got "cannot write to future after previous write succeeded or readable end dropped"', + status: "failed", + }, { + lines: [643, 645, 647, 649], + cause: + 'Error: expected trap "cannot write after being notified that the readable end dropped", got "cannot write to stream after being notified that the readable end dropped"', + status: "failed", + }, { + lines: [652, 654, 658, 660], + cause: + 'Error: expected trap "cannot read after being notified that the writable end dropped", got "cannot read from stream after being notified that the writable end dropped"', + status: "failed", + }], + }, + { + file: "exceptions.json", + rows: [{ + lines: [50, 127, 161, 237], + cause: + 'Error: expected trap "uncaught exception propagated out of component", got "guest trapped: unreachable"', + status: "failed", + }], + }, + { + file: "resources.json", + rows: [{ + lines: [927], + cause: + 'Error: expected trap "cannot remove owned resource while borrowed", got "handle still lent out"', + status: "failed", + }], + }, + { + file: "strings.json", + rows: [{ + lines: [21, 23], + cause: + 'Error: expected trap "string pointer not aligned to 2", got "misaligned string pointer"', + status: "failed", + }], + }, + { + file: "trap.json", + rows: [{ + lines: [30], + cause: + 'Error: expected trap "wasm `unreachable` instruction executed", got "guest trapped: unreachable"', + status: "failed", + }], + }, + { + file: "types.json", + rows: [{ + lines: [378], + cause: + 'Error: expected trap "discriminant 2 out of range [0..2)", got "invalid variant discriminant"', + status: "failed", + }], + }, + ], + }, + { + class: "exception-handling", + files: [ + { + file: "async/exceptions.json", + rows: [{ + lines: [68, 70, 144, 146, 216], + cause: "[object WebAssembly.Exception]", + status: "failed", + }, { + lines: [218], + cause: + 'Error: expected trap "thrown Wasm exception", got "guest trapped: unreachable"', + status: "failed", + }], + }, + ], + }, + { + class: "imported-module", + files: [ + { + file: "alias-region-reexported-entities-to-imported-module.json", + rows: [{ + lines: [14], + cause: + "TranslateError: translator error [unsupported]: imported-module instantiation (InstantiateModule::Import) is not supported in plan v0 (contracts/plan-format.md open items)", + status: "failed", + }], + }, + { + file: "instance.json", + rows: [{ + lines: [216, 224], + cause: + "TranslateError: translator error [unsupported]: imported-module instantiation (InstantiateModule::Import) is not supported in plan v0 (contracts/plan-format.md open items)", + status: "failed", + }], + }, + { + file: "modules.json", + rows: [{ + lines: [316, 417], + cause: + "TranslateError: translator error [unsupported]: imported-module instantiation (InstantiateModule::Import) is not supported in plan v0 (contracts/plan-format.md open items)", + status: "failed", + }], + }, + { + file: "nested.json", + rows: [{ + lines: [149, 219], + cause: + "TranslateError: translator error [unsupported]: imported-module instantiation (InstantiateModule::Import) is not supported in plan v0 (contracts/plan-format.md open items)", + status: "failed", + }], + }, + ], + }, + { + class: "provider-control", + files: [ + { + file: "async/cancel-starting-subtask-does-not-leak.json", + rows: [{ + lines: [9], + cause: + "PlanError: host import 'wasmtime/set-max-table-capacity' not provided (no key 'wasmtime' in imports)", + status: "failed", + }], + }, + { + file: "async/context-in-resource-drop.json", + rows: [{ + lines: [248], + cause: + "PlanError: host import 'wasmtime/gc' not provided (no key 'wasmtime' in imports)", + status: "failed", + }], + }, + { + file: "instance.json", + rows: [{ + lines: [287, 294, 301], + cause: + "PlanError: host import 'I1/r' not provided (no key 'I1' in imports)", + status: "failed", + }, { + lines: [308, 315], + cause: + "PlanError: host import 'I2/r' not provided (no key 'I2' in imports)", + status: "failed", + }, { + lines: [322], + cause: + "PlanError: host import 'I3/r' not provided (no key 'I3' in imports)", + status: "failed", + }], + }, + ], + }, + { + class: "runtime-semantics", + files: [ + { + file: "async/context-in-compositions.json", + rows: [{ + lines: [155, 434], + cause: + "Error: expected return, got trap: guest trapped: unreachable", + status: "failed", + }], + }, + { + file: "async/error-context.json", + rows: [{ + lines: [86], + cause: "AssertionError: store out of bounds", + status: "failed", + }], + }, + { + file: "async/futures.json", + rows: [{ + lines: [54, 64], + cause: + "AssertionError: suspension mode jspi wrapped imports without wrapping any entry (entries=false, imports=true) — a Suspending import reached from a non-promising activation traps unconditionally (jspi pin (c))", + status: "failed", + }], + }, + { + file: "async/stackful.json", + rows: [{ + lines: [110, 132], + cause: + "AssertionError: suspension mode jspi wrapped imports without wrapping any entry (entries=false, imports=true) — a Suspending import reached from a non-promising activation traps unconditionally (jspi pin (c))", + status: "failed", + }], + }, + { + file: "async/stream-big-read-and-writes.json", + rows: [{ + lines: [42], + cause: "RangeError: Invalid array length", + status: "failed", + }], + }, + { + file: "async/streams.json", + rows: [{ + lines: [73, 83], + cause: + "AssertionError: suspension mode jspi wrapped imports without wrapping any entry (entries=false, imports=true) — a Suspending import reached from a non-promising activation traps unconditionally (jspi pin (c))", + status: "failed", + }], + }, + { + file: "async/sync-call-context-slots.json", + rows: [{ + lines: [76, 152], + cause: + "Error: expected return, got trap: guest trapped: unreachable", + status: "failed", + }], + }, + { + file: "async/sync-call-context.json", + rows: [{ + lines: [61, 126, 207, 298, 389, 447, 544], + cause: + "Error: expected return, got trap: guest trapped: unreachable", + status: "failed", + }], + }, + { + file: "async/task-builtins.json", + rows: [{ + lines: [34, 56, 74], + cause: + "AssertionError: suspension mode jspi wrapped imports without wrapping any entry (entries=false, imports=true) — a Suspending import reached from a non-promising activation traps unconditionally (jspi pin (c))", + status: "failed", + }, { + lines: [201, 466, 723], + cause: + "Error: expected return, got trap: guest trapped: unreachable", + status: "failed", + }], + }, + { + file: "import.json", + rows: [{ + lines: [8], + cause: + "Error: expected instantiation link-error, but component instantiated successfully", + status: "failed", + }], + }, + { + file: "instance.json", + rows: [{ + lines: [79], + cause: "RuntimeError: unreachable", + status: "failed", + }], + }, + { + file: "linking.json", + rows: [{ + lines: [2, 11, 14, 17], + cause: + "Error: expected instantiation link-error, but component instantiated successfully", + status: "failed", + }], + }, + { + file: "modules.json", + rows: [{ + lines: [ + 26, + 43, + 90, + 100, + 120, + 127, + 134, + 141, + 150, + 157, + 164, + 171, + 178, + 185, + 211, + 218, + 225, + 232, + 241, + 248, + 255, + 262, + 269, + 276, + ], + cause: + "Error: expected instantiation link-error, but component instantiated successfully", + status: "failed", + }, { + lines: [299], + cause: + "TranslateError: translator error [unsupported]: re-exporting an imported module is not supported (export 'm2'); module imports have no instantiation story yet (the Export::ModuleImport rejection, contracts/plan-format.md schema notes)", + status: "failed", + }], + }, + { + file: "resources.json", + rows: [{ + lines: [167], + cause: + "PlanError: host import 'host/missing' must be a HostResourceType (the component imports a resource type); got undefined", + status: "failed", + }, { + lines: [174], + cause: + "PlanError: host import 'host/return-three' must be a HostResourceType (the component imports a resource type); got a function", + status: "failed", + }, { + lines: [201], + cause: + "Error: expected instantiation link-error, but component instantiated successfully", + status: "failed", + }], + }, + { + file: "types.json", + rows: [{ + lines: [339], + cause: + "TranslateError: translator error [unsupported]: unsupported type export: component", + status: "failed", + }, { + lines: [348], + cause: + "TranslateError: translator error [unsupported]: unsupported type export: instance", + status: "failed", + }], + }, + ], + }, + ] as const; + +export const WASMTIME_EXPECTATIONS: readonly WasmtimeExpectation[] = + WASMTIME_EXPECTATION_GROUPS.flatMap((group) => + group.files.flatMap((file) => + file.rows.flatMap((row) => + row.lines.map((line) => ({ + line, + cause: row.cause, + status: row.status, + file: file.file, + class: group.class, + })) + ) + ) + ); + +export const WASMTIME_SKIP_EXPECTATIONS: readonly WasmtimeExpectation[] = [ + "async/join-during-sync-read.json:8", + "async/task-deletion.json:11", + "async/task-return-traps.json:21", + "async/task-return-traps.json:58", +].map((key) => { + const split = key.lastIndexOf(":"); + return { + file: key.slice(0, split), + line: Number(key.slice(split + 1)), + status: "skipped" as const, + class: "deferred-threads", + cause: + "pending component runtime: pending-capability: instantiate: component requires host trampoline 'thread-new-indirect' — needs the \"task-core\" capability, not yet implemented in the current executor (contracts/intrinsics.md §B)", + }; +}); + +export const WASMTIME_EXCLUSIONS: Readonly> = { + "big-strings.json": + "upstream ;;! hogs_memory=true; bounded gate excludes memory stress", + "memory64.json": + "upstream ;;! hogs_memory=true; bounded gate excludes memory stress", + "async/streams-massive-send.json": + "upstream memory stress cannot start reliably under a bounded V8 heap; not executed", + "gc/empty.json": + "upstream ;;! component_model_gc/gc=true; GC is feature-disabled", + "implements-disabled.json": + "upstream ;;! component_model_implements=false requests disabled validation, but translator configuration enables it", +}; + +export function expectedWasmtimeFailure( + file: string, + line: number, + detail: string, +): WasmtimeExpectation | undefined { + return WASMTIME_EXPECTATIONS.find((e) => + e.status === "failed" && e.file === file && e.line === line && + detail === e.cause + ); +} + +export function expectedWasmtimeSkip( + file: string, + line: number, + reason: string | undefined, + detail: string, +): WasmtimeExpectation | undefined { + return WASMTIME_SKIP_EXPECTATIONS.find((e) => + e.file === file && e.line === line && reason === "pending-runtime" && + detail === e.cause + ); +} diff --git a/harness/src/wasmtime-spectest.ts b/harness/src/wasmtime-spectest.ts new file mode 100644 index 00000000..582409da --- /dev/null +++ b/harness/src/wasmtime-spectest.ts @@ -0,0 +1,80 @@ +import { suspending } from "@polyengine/protocol"; +import { + type HostImports, + hostResourceType, +} from "../../runtime/src/exec/mod.ts"; + +export interface SpectestProbe { + readonly imports: HostImports; + readonly counters: { readonly drops: number; readonly lastDrop: number }; +} + +/** Port of locked wasmtime crates/wast/src/spectest.rs:90-224. + * Raw HostImports expose reps but not Wasmtime's `Resource::owned()` bit, so + * those upstream ownership assertions are not duplicated here. */ +export function wasmtimeSpectest(sourceFile = ""): SpectestProbe { + const state = { drops: 0, lastDrop: 0 }; + const resource1 = hostResourceType({ + name: "host.resource1", + dtor: (rep) => { + state.drops++; + state.lastDrop = rep; + }, + }); + return { + counters: state, + imports: { + "host-echo-u32": async (v: unknown) => v, + "host-return-two": () => 2, + host: { + "return-three": () => 3, + nested: { "return-four": () => 4 }, + resource1, + resource2: hostResourceType({ name: "host.resource2" }), + "resource1-again": resource1, + "[constructor]resource1": (rep: unknown) => rep, + "[static]resource1.assert": (rep: unknown, expected: unknown) => { + if (rep !== expected) { + throw new Error(`resource rep ${rep} != ${expected}`); + } + }, + "[static]resource1.last-drop": () => state.lastDrop, + "[static]resource1.drops": () => state.drops, + "[method]resource1.simple": (rep: unknown, expected: unknown) => { + if (rep !== expected) { + throw new Error(`resource rep ${rep} != ${expected}`); + } + }, + "[method]resource1.take-borrow": () => undefined, + "[method]resource1.take-own": () => undefined, + "never-return": () => new Promise(() => {}), + "return-two-slowly": maybeSuspending( + sourceFile === "async/cancel-host.json", + async () => { + await Promise.resolve(); + return 2; + }, + ), + "echo-slowly": maybeSuspending( + sourceFile === "async/cancel-host.json", + async (v: unknown) => { + await Promise.resolve(); + return v; + }, + ), + "[method]resource1.never-return": maybeSuspending( + sourceFile === "async/cancel-host.json", + () => new Promise(() => {}), + ), + "return-hi": () => "hi", + }, + }, + }; +} + +function maybeSuspending( + enabled: boolean, + fn: F, +): F { + return enabled ? suspending(fn) : fn; +} diff --git a/harness/tests/runner_unit_test.ts b/harness/tests/runner_unit_test.ts index 5e88462b..2a8db793 100644 --- a/harness/tests/runner_unit_test.ts +++ b/harness/tests/runner_unit_test.ts @@ -7,6 +7,8 @@ import type { WastJson } from "../src/schema.ts"; import { CoreOnlyExecutor } from "../src/executor.ts"; +import type { CommandExecutor } from "../src/executor.ts"; +import { TrapError } from "../src/executor.ts"; import { runWastJson, trapMatches } from "../src/runner.ts"; // (module) - the empty core module, hand-encoded. @@ -14,7 +16,17 @@ const EMPTY_CORE_MODULE = new Uint8Array([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0]); // A core module with a valid preamble (sniffs as kind "module") but an // invalid trailing section byte (0xff is not a valid section id), so // WebAssembly.validate rejects it on content, not preamble. -const INVALID_SECTION_MODULE = new Uint8Array([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0, 0xff]); +const INVALID_SECTION_MODULE = new Uint8Array([ + 0, + 0x61, + 0x73, + 0x6d, + 1, + 0, + 0, + 0, + 0xff, +]); // (component) - the empty component: core preamble with version 0x0d, // layer 0x0001. const EMPTY_COMPONENT = new Uint8Array([0, 0x61, 0x73, 0x6d, 0x0d, 0, 1, 0]); @@ -43,7 +55,11 @@ function assertEq(actual: unknown, expected: unknown, what: string) { Deno.test("environment: V8 validates core modules but no component binaries", () => { assertEq(WebAssembly.validate(EMPTY_CORE_MODULE), true, "core valid"); - assertEq(WebAssembly.validate(INVALID_SECTION_MODULE), false, "invalid section"); + assertEq( + WebAssembly.validate(INVALID_SECTION_MODULE), + false, + "invalid section", + ); // The load-bearing fact behind skip("pending-runtime"): the JS API rejects // the component layer preamble outright, so `validate === false` carries // no information about a component's actual validity. @@ -171,6 +187,23 @@ Deno.test("a genuinely invalid core module fails assert-free module command", as assertEq(result.results[0].status, "failed", "status"); }); +Deno.test("assert_uninstantiable rejects an unrelated trap cause", async () => { + const executor = new CoreOnlyExecutor() as CommandExecutor; + executor.instantiate = () => Promise.reject(new TrapError("different cause")); + const result = await runWastJson( + doc([{ + type: "assert_uninstantiable", + line: 1, + filename: "comp.0.wasm", + module_type: "binary", + text: "expected cause", + }]), + load, + executor, + ); + assertEq(result.results[0].status, "failed", "status"); +}); + // TRAP_MESSAGE_EQUIVALENTS: the core `unreachable` trap row. The runtime // (runtime/src/exec/boundary.ts mapCoreException) passes each JS engine's raw // trap text through untouched; this table is where the suite's diff --git a/harness/tests/wasmtime_expectations_test.ts b/harness/tests/wasmtime_expectations_test.ts new file mode 100644 index 00000000..115e8ac1 --- /dev/null +++ b/harness/tests/wasmtime_expectations_test.ts @@ -0,0 +1,145 @@ +import { + expectedWasmtimeFailure, + WASMTIME_EXPECTATIONS, + WASMTIME_FAILURE_CLASSES, + WASMTIME_SKIP_EXPECTATIONS, +} from "../src/wasmtime-expectations.ts"; +import { classify } from "../src/wasmtime-classifier.ts"; + +function assert(condition: boolean, message: string): void { + if (!condition) throw new Error(message); +} + +Deno.test("Wasmtime expectation requires exact file, line, and failure cause", () => { + const entry = WASMTIME_EXPECTATIONS[0]; + assert(entry !== undefined, "fixture expectation missing"); + assert( + expectedWasmtimeFailure( + entry.file, + entry.line, + entry.cause, + ) === entry, + "matching cause was rejected", + ); + assert( + expectedWasmtimeFailure( + entry.file, + entry.line, + `prefix ${entry.cause} suffix`, + ) === undefined, + "non-exact cause was accepted", + ); + assert( + expectedWasmtimeFailure(entry.file, entry.line + 1_000_000, entry.cause) === + undefined, + "wrong line was accepted", + ); + assert( + expectedWasmtimeFailure(entry.file, entry.line, "different failure") === + undefined, + "wrong cause was accepted", + ); +}); + +Deno.test("every expectation names a declared failure class", () => { + assert( + WASMTIME_EXPECTATIONS.every((e) => + WASMTIME_FAILURE_CLASSES[e.class] !== undefined + ), + "expectation has an undeclared failure class", + ); +}); + +Deno.test("classifier rejects stale passes and unexpected skips", () => { + const entry = WASMTIME_EXPECTATIONS[0]; + assert( + classify(entry.file, { + line: entry.line, + type: "assert_return", + status: "passed", + }).status === + "unexpected", + "stale pass accepted", + ); + assert( + classify("x.json", { + line: 1, + type: "future-directive", + status: "skipped", + detail: "new reason", + }).status === "unexpected", + "unknown skip accepted", + ); +}); + +Deno.test("skip expectations require exact line and full cause", () => { + const entry = WASMTIME_SKIP_EXPECTATIONS[0]; + const base = { + type: "module", + status: "skipped" as const, + reason: "pending-runtime" as const, + }; + assert( + classify(entry.file, { ...base, line: entry.line, detail: entry.cause }) + .status === "skip", + "exact skip rejected", + ); + assert( + classify(entry.file, { + ...base, + line: entry.line + 1_000_000, + detail: entry.cause, + }).status === "unexpected", + "wrong skip line accepted", + ); + assert( + classify(entry.file, { + ...base, + line: entry.line, + detail: entry.cause + " changed", + }).status === "unexpected", + "changed skip cause accepted", + ); + assert( + classify(entry.file, { + ...base, + line: entry.line, + reason: "pending-capability", + detail: entry.cause, + }).status === "unexpected", + "changed skip reason accepted", + ); + assert( + classify(entry.file, { line: entry.line, type: "module", status: "passed" }) + .status === "unexpected", + "skip-to-pass accepted", + ); +}); + +Deno.test("expectation inventory has no duplicates and every class is declared", () => { + const all = [...WASMTIME_EXPECTATIONS, ...WASMTIME_SKIP_EXPECTATIONS]; + const keys = all.map((e) => `${e.status}:${e.file}:${e.line}`); + assert(new Set(keys).size === keys.length, "duplicate expectation row"); + assert( + all.every((e) => WASMTIME_FAILURE_CLASSES[e.class] !== undefined), + "missing expectation class", + ); +}); + +Deno.test("spectest resource probe preserves rep and destructor counters", async () => { + const { wasmtimeSpectest } = await import("../src/wasmtime-spectest.ts"); + const probe = wasmtimeSpectest(); + const host = probe.imports.host as Record< + string, + (...args: unknown[]) => unknown + >; + host["[static]resource1.assert"](7, 7); + const resource = host.resource1 as unknown as { + options: { dtor: (rep: number) => void }; + }; + resource.options.dtor(7); + assert( + probe.counters.drops === 1 && probe.counters.lastDrop === 7, + "resource counters lost", + ); +}); diff --git a/harness/tests/wasmtime_subprocess_test.ts b/harness/tests/wasmtime_subprocess_test.ts new file mode 100644 index 00000000..4ea7e1c4 --- /dev/null +++ b/harness/tests/wasmtime_subprocess_test.ts @@ -0,0 +1,61 @@ +import { + runChild, + validateWorkerResult, +} from "../../tools/wasmtime/subprocess.ts"; + +const canRun = + (await Deno.permissions.query({ name: "run" })).state === "granted"; + +Deno.test({ + name: "subprocess timeout kills and reaps the child", + ignore: !canRun, + fn: async () => { + const started = performance.now(); + const outcome = await runChild( + new Deno.Command(Deno.execPath(), { + args: ["eval", "setTimeout(() => {}, 10000)"], + stdout: "piped", + stderr: "piped", + }), + 25, + ); + if (outcome !== "timeout") throw new Error("child did not time out"); + if (performance.now() - started > 2_000) { + throw new Error("child was not reaped"); + } + }, +}); + +Deno.test("malformed worker result is rejected", () => { + const doc = { + source_filename: "x.wast", + commands: [{ line: 1, type: "module" }], + }; + if ( + validateWorkerResult(doc, { source: "x.wast", results: [] }) === undefined + ) throw new Error("missing command accepted"); + if ( + validateWorkerResult(doc, { + source: "other", + results: [{ line: 1, type: "module" }], + }) === undefined + ) throw new Error("wrong provenance accepted"); + if ( + validateWorkerResult(doc, { + source: "x.wast", + results: [{ line: 1, type: "module", status: "mystery" }], + }) === undefined + ) throw new Error("unknown status accepted"); + if ( + validateWorkerResult(doc, { + source: "x.wast", + results: [{ line: 1, type: "module", status: "skipped", reason: 7 }], + }) === undefined + ) throw new Error("malformed skip reason accepted"); + if ( + validateWorkerResult(doc, { + source: "x.wast", + results: [{ line: 1, type: "module", status: "failed", detail: {} }], + }) === undefined + ) throw new Error("malformed detail accepted"); +}); diff --git a/justfile b/justfile index 3e7c73bc..80cf5b29 100644 --- a/justfile +++ b/justfile @@ -9,7 +9,7 @@ default: ci: (gha::core) (gha::browser) # Full pre-commit gates, including local consumer smokes (docs/consumers.md). -gates: version-guard-local fmt-check lint build test-rust test-protocol test-runtime test-wasi test-sockets-node test-ct-runner test-bundle test-version-guard publish-check test-npm examples test-translate conformance sched-seeds shells browsers smoke-tls smoke-c0 +gates: version-guard-local fmt-check lint build test-rust test-protocol test-runtime test-wasi test-sockets-node test-ct-runner test-bundle test-version-guard publish-check test-npm examples test-translate conformance test-wasmtime test-wasmtime-guests sched-seeds shells browsers smoke-tls smoke-c0 # Fast sanity: builds + native tests + type-checks, no suites. check: fmt-check lint build test-rust @@ -150,6 +150,28 @@ test-npm: npm-build fixtures conformance: cd harness && deno task conformance +# Supplementary Wasmtime WAST gate (docs/architecture.md §11): the pinned +# wasmtime-environ revision's own component-model WAST, converted and +# classified alongside — never merged into — the official corpus's xfails. +# `deno task wasmtime` chains generation, the focused expectation-inventory +# tests and the classifying CLI; results land in +# harness/generated-wasmtime/results.json. +test-wasmtime: shim + cd harness && deno task wasmtime + +# Build the two upstream Wasmtime async guests (round-trip, short reads) +# from the same locked wasmtime-environ revision, into the ignored +# tools/wasmtime-guests/build/ directory. +wasmtime-guests: + deno run -A tools/wasmtime-guests/build.ts + +# Public-embedder round-trip/short-read scenarios adapted from those guests +# (runtime/tests/wasmtime/public_guests.ts). FIFO first, then the seeded +# scheduler order, sharing the same prepared guest artifacts. +test-wasmtime-guests: shim wasmtime-guests + deno test --config runtime/deno.json --allow-read=.,/tmp --allow-write=/tmp --allow-env=POLYENGINE_SCHED_SEED runtime/tests/wasmtime/public_guests.ts + POLYENGINE_SCHED_SEED=1 deno test --config runtime/deno.json --allow-read=.,/tmp --allow-write=/tmp --allow-env=POLYENGINE_SCHED_SEED runtime/tests/wasmtime/public_guests.ts + # Scheduler-order sensitivity (docs/architecture.md §6) — spec-allowed # nondeterminism; FIFO when POLYENGINE_SCHED_SEED is unset. # The affected suites re-run under seeded-shuffle scheduling. @@ -157,6 +179,7 @@ sched-seeds: shim fixtures corpus cd runtime && POLYENGINE_SCHED_SEED=1 deno task test cd runtime && POLYENGINE_SCHED_SEED=4242 deno task test cd harness && POLYENGINE_SCHED_SEED=1 deno task conformance + cd harness && POLYENGINE_SCHED_SEED=1 deno task wasmtime # ----- engine lanes ----------------------------------------------------------- diff --git a/runtime/tests/wasmtime/public_guests.ts b/runtime/tests/wasmtime/public_guests.ts new file mode 100644 index 00000000..90034487 --- /dev/null +++ b/runtime/tests/wasmtime/public_guests.ts @@ -0,0 +1,317 @@ +// This deliberately lacks a `_test.ts` suffix: the focused gate names it +// explicitly, while ordinary runtime test discovery remains build-independent. +// Public-embedder adaptations of Wasmtime's async round-trip and short-read +// scenarios from crates/test-programs/src/bin and +// crates/misc/component-async-tests/wit in the Cargo.lock-selected checkout. +// The host assertions intentionally cover portable API behavior, not Wasmtime's +// native Accessor styles or internal counters. + +import { assertEq } from "../support/asserts.ts"; +import { instantiate } from "../../src/embedder/mod.ts"; +import { Translator } from "../../src/shim/mod.ts"; +import type { Stream } from "@polyengine/protocol"; +import { wasi } from "../../../wasi/src/mod.ts"; + +const root = new URL("../../../", import.meta.url); +const build = "tools/wasmtime-guests/build/"; + +async function required(rel: string, command: string): Promise { + try { + return await Deno.readFile(new URL(rel, root)); + } catch (cause) { + throw new Error(`missing required artifact ${rel}; run: ${command}`, { + cause, + }); + } +} + +async function requiredText(rel: string, command: string): Promise { + try { + return await Deno.readTextFile(new URL(rel, root)); + } catch (cause) { + throw new Error(`missing required artifact ${rel}; run: ${command}`, { + cause, + }); + } +} + +async function withTimeout( + label: string, + promise: Promise, + milliseconds = 10_000, +): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timed out after ${milliseconds}ms`)), + milliseconds, + ); + }); + try { + return await Promise.race([promise, timeout]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +const buildGuests = "deno run -A tools/wasmtime-guests/build.ts"; +const shim = await required( + "target/wasm32-unknown-unknown/release/translator_shim.wasm", + "cargo build -p translator-shim --release --target wasm32-unknown-unknown", +); +const translator = await Translator.create(shim); + +const sourcePaths = [ + "crates/test-programs/src/bin/async_round_trip_stackless.rs", + "crates/test-programs/src/bin/async_short_reads.rs", + "crates/misc/component-async-tests/wit/test.wit", + "crates/wasi-preview1-component-adapter", +]; +const provenance = JSON.parse( + await requiredText(`${build}provenance.json`, buildGuests), +) as { + revision?: string; + sources?: string[]; + artifacts?: Record; +}; +const cargoLock = await Deno.readTextFile(new URL("Cargo.lock", root)); +const environBlock = cargoLock.split("[[package]]").find((block) => + block.includes('name = "wasmtime-environ"') +); +const lockedRevision = environBlock?.match( + /source = "git\+[^"#]+(?:\?[^"#]+)?#([0-9a-f]{40})"/, +)?.[1]; +if ( + provenance.revision === undefined || + provenance.revision !== lockedRevision || + JSON.stringify(provenance.sources) !== JSON.stringify(sourcePaths) +) { + throw new Error( + `stale Wasmtime guest artifacts for revision ${provenance.revision}; run: ${buildGuests}`, + ); +} + +async function guest(name: string, imports: Record = {}) { + const componentBytes = await required( + `${build}${name}.component.wasm`, + buildGuests, + ); + const digest = Array.from( + new Uint8Array( + await crypto.subtle.digest("SHA-256", new Uint8Array(componentBytes)), + ), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + if (provenance.artifacts?.[name] !== digest) { + throw new Error( + `stale or modified guest artifact ${name}; run: ${buildGuests}`, + ); + } + return await instantiate({ componentBytes, translator }, { + ...wasi(), + ...imports, + }); +} + +type Deferred = { + promise: Promise; + resolve(value: T): void; +}; + +function deferred(): Deferred { + let resolve!: (value: T) => void; + const promise = new Promise((done) => resolve = done); + return { promise, resolve }; +} + +function observed(promise: Promise) { + const settled = deferred(); + let state: "pending" | "fulfilled" | "rejected" = "pending"; + const value = promise.then( + (result) => { + state = "fulfilled"; + settled.resolve(); + return result; + }, + (error) => { + state = "rejected"; + settled.resolve(); + throw error; + }, + ); + return { value, settled: settled.promise, state: () => state }; +} + +function timedTest(name: string, fn: () => Promise): void { + Deno.test(name, () => withTimeout(name, fn(), 20_000)); +} + +async function assertRemainsPending( + label: string, + settled: Promise, +): Promise { + const won = await Promise.race([ + settled.then(() => "settled" as const), + new Promise<"turn">((resolve) => setTimeout(() => resolve("turn"), 0)), + ]); + assertEq(won, "turn", `${label} settled before its release`); +} + +timedTest( + "wasmtime guest: three stackless calls remain independently outstanding", + async () => { + // Upstream scenario: crates/misc/component-async-tests/tests/scenario/ + // round_trip.rs:513-525 starts three calls before joining their results. + const pending = new Map>(); + const allArrived = deferred(); + const c = await guest("async_round_trip_stackless", { + "local:local/baz": { + foo(input: string): Promise { + const result = deferred(); + pending.set(input, result); + if (pending.size === 3) allArrived.resolve(); + return result.promise; + }, + }, + }); + const foo = c.exports["local:local/baz"].foo as ( + value: string, + ) => Promise; + + const calls = ["alpha", "beta", "gamma"].map((input) => + observed(foo(input)) + ); + await withTimeout("three host imports to arrive", allArrived.promise); + assertEq([...pending.keys()], [ + "alpha - entered guest", + "beta - entered guest", + "gamma - entered guest", + ]); + + const settle = async (call: number, input: string) => { + const hostInput = `${input} - entered guest`; + pending.get(hostInput)!.resolve( + `${hostInput} - entered host - exited host`, + ); + return await withTimeout(`${input} call to publish`, calls[call].value); + }; + assertEq( + await settle(2, "gamma"), + "gamma - entered guest - entered host - exited host - exited guest", + ); + assertEq(calls[0].state(), "pending"); + assertEq(calls[1].state(), "pending"); + assertEq( + await settle(0, "alpha"), + "alpha - entered guest - entered host - exited host - exited guest", + ); + assertEq(calls[1].state(), "pending"); + assertEq( + await settle(1, "beta"), + "beta - entered guest - entered host - exited host - exited guest", + ); + assertEq(await Promise.all(calls.map((call) => call.value)), [ + "alpha - entered guest - entered host - exited host - exited guest", + "beta - entered guest - entered host - exited host - exited guest", + "gamma - entered guest - entered host - exited host - exited guest", + ]); + }, +); + +for (const delayed of [false, true]) { + timedTest( + `wasmtime guest: resource short reads preserve ownership (${ + delayed ? "delayed" : "immediate" + })`, + async () => { + const c = await guest("async_short_reads"); + const api = c.exports["local:local/short-reads"]; + // Upstream scenario: crates/misc/component-async-tests/tests/scenario/ + // streams.rs:492-528 transfers five owns in, consumes one at a time, + // then calls each returned resource. We add public-wrapper drop checks. + const labels = ["a", "b", "c", "d", "e"]; + const things = labels.map((label) => new api.Thing(label)); + let returned = false; + const production = deferred(); + const producerEntered = deferred(); + async function* source() { + producerEntered.resolve(); + if (delayed) await production.promise; + try { + yield things; + } finally { + returned = true; + } + } + + const outputPromise = api.shortReads(source()) as Promise< + Stream< + InstanceType< + typeof api.Thing + > + > + >; + const output = await withTimeout("short-reads export", outputPromise); + const received: InstanceType[] = []; + if (delayed) { + await withTimeout("stream producer to park", producerEntered.promise); + const firstRead = observed(output.read(1)); + await assertRemainsPending("first read", firstRead.settled); + production.resolve(); + const first = await withTimeout( + "first delayed short read", + firstRead.value, + ); + assertEq(first.length, 1); + received.push(first[0]); + } + for (let i = delayed ? 1 : 0; i < labels.length; i++) { + const consumerReady = deferred(); + const consume = observed((async () => { + await consumerReady.promise; + return await output.read(1); + })()); + if (delayed) { + await assertRemainsPending(`short read ${i + 1}`, consume.settled); + } + consumerReady.resolve(); + const chunk = await withTimeout(`short read ${i + 1}`, consume.value); + assertEq(chunk.length, 1, "consumer forces one-element short reads"); + received.push(chunk[0]); + } + assertEq(await withTimeout("short-read EOF", output.read(1)), []); + assertEq(returned, true, "producer was consumed and finalized"); + assertEq( + await withTimeout( + "returned resource reads", + Promise.all(received.map((thing) => thing.get())), + ), + labels, + ); + + // own moved through host -> guest -> host. Source wrappers were + // invalidated by transfer; returned wrappers exclusively own each value. + for (const sourceThing of things) { + let failed = false; + try { + await sourceThing.get(); + } catch { + failed = true; + } + assertEq(failed, true, "source own wrapper was consumed"); + } + for (const thing of received) { + thing.drop(); + thing.drop(); + let failed = false; + try { + await thing.get(); + } catch { + failed = true; + } + assertEq(failed, true, "returned own is invalid after idempotent drop"); + } + output.drop(); + }, + ); +} diff --git a/tools/wasmtime-guests/.gitignore b/tools/wasmtime-guests/.gitignore new file mode 100644 index 00000000..567609b1 --- /dev/null +++ b/tools/wasmtime-guests/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/tools/wasmtime-guests/build.ts b/tools/wasmtime-guests/build.ts new file mode 100644 index 00000000..4984662f --- /dev/null +++ b/tools/wasmtime-guests/build.ts @@ -0,0 +1,154 @@ +// Build the two upstream Wasmtime guests used by runtime/tests/wasmtime. +// The dependency checkout is discovered from this repository's locked cargo +// graph; it is never written. The guest sources are read directly from +// crates/test-programs/src/bin/{async_round_trip_stackless,async_short_reads}.rs +// and crates/misc/component-async-tests/wit at that revision. A detached local +// clone and all cargo output live under the ignored build directory. + +import { resolveWasmtimeSource } from "../wasmtime/source.ts"; + +const root = new URL("../../", import.meta.url); +const here = new URL("./", import.meta.url); +const buildDir = new URL("build/", here); +const checkout = new URL("build/source/", here); +const target = new URL("build/target/", here); +const sources = [ + "crates/test-programs/src/bin/async_round_trip_stackless.rs", + "crates/test-programs/src/bin/async_short_reads.rs", + "crates/misc/component-async-tests/wit/test.wit", + "crates/wasi-preview1-component-adapter", +]; + +async function run( + command: string, + args: string[], + options: Deno.CommandOptions = {}, +): Promise { + const output = await new Deno.Command(command, { + ...options, + args, + stdout: "piped", + stderr: "inherit", + }).output(); + if (!output.success) { + throw new Error(`${command} ${args.join(" ")} exited ${output.code}`); + } + return new TextDecoder().decode(output.stdout).trim(); +} + +const wasmToolsVersion = await run("wasm-tools", ["--version"]); + +const { path: source, rev: revision } = await resolveWasmtimeSource(); +if ((await run("git", ["rev-parse", "HEAD"], { cwd: source })) !== revision) { + throw new Error(`cargo cache checkout is not locked revision ${revision}`); +} +// Cargo git checkouts may contain unrelated uninitialized submodule entries. +// Only committed objects are cloned, but reject changes to every direct source +// selected for this build rather than silently stamping them as the clean rev. +const sourceDirty = await run( + "git", + ["status", "--porcelain", "--", ...sources], + { cwd: source }, +); +if (sourceDirty !== "") { + throw new Error(`refusing dirty cargo cache checkout at ${source}`); +} + +await Deno.mkdir(buildDir, { recursive: true }); +try { + const actual = await run("git", ["rev-parse", "HEAD"], { cwd: checkout }); + const dirty = await run("git", ["status", "--porcelain"], { + cwd: checkout, + }); + if (dirty !== "") { + throw new Error( + `refusing dirty guest checkout at ${checkout.pathname}`, + ); + } + if (actual !== revision) { + await Deno.remove(checkout, { recursive: true }); + } +} catch (error) { + if (!(error instanceof Deno.errors.NotFound)) throw error; +} +try { + await Deno.stat(new URL(".git", checkout)); +} catch (error) { + if (!(error instanceof Deno.errors.NotFound)) throw error; + await run("git", ["clone", "--no-checkout", source, checkout.pathname]); + await run("git", ["checkout", "--detach", revision], { cwd: checkout }); +} +if ((await run("git", ["rev-parse", "HEAD"], { cwd: checkout })) !== revision) { + throw new Error(`guest checkout is not locked revision ${revision}`); +} +if ((await run("git", ["status", "--porcelain"], { cwd: checkout })) !== "") { + throw new Error(`refusing dirty guest checkout at ${checkout.pathname}`); +} + +const env = { + CARGO_TARGET_DIR: target.pathname, +}; +await run( + "cargo", + [ + "build", + "--locked", + "--release", + "--target=wasm32-wasip1", + "--package=test-programs", + "--bin=async_round_trip_stackless", + "--bin=async_short_reads", + ], + { cwd: checkout, env }, +); +await run( + "cargo", + [ + "build", + "--locked", + "--release", + "--target=wasm32-unknown-unknown", + "--package=wasi-preview1-component-adapter", + "--no-default-features", + "--features=command", + ], + { cwd: checkout, env }, +); + +const adapter = new URL( + "wasm32-unknown-unknown/release/wasi_snapshot_preview1.wasm", + target, +).pathname; +const artifacts: Record = {}; +for (const name of ["async_round_trip_stackless", "async_short_reads"]) { + const core = new URL(`wasm32-wasip1/release/${name}.wasm`, target).pathname; + const component = new URL(`${name}.component.wasm`, buildDir).pathname; + await run("wasm-tools", [ + "component", + "new", + core, + "--adapt", + `wasi_snapshot_preview1=${adapter}`, + "-o", + component, + ]); + await run("wasm-tools", [ + "validate", + "--features=component-model,cm-async", + component, + ]); + const digest = await crypto.subtle.digest( + "SHA-256", + await Deno.readFile(component), + ); + artifacts[name] = Array.from( + new Uint8Array(digest), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + console.log(`built ${component} from wasmtime ${revision}`); +} +await Deno.writeTextFile( + new URL("provenance.json", buildDir), + JSON.stringify({ revision, wasmToolsVersion, sources, artifacts }, null, 2) + + "\n", +); diff --git a/tools/wasmtime/generate.ts b/tools/wasmtime/generate.ts new file mode 100644 index 00000000..e1ddec94 --- /dev/null +++ b/tools/wasmtime/generate.ts @@ -0,0 +1,27 @@ +import { resolveWasmtimeSource } from "./source.ts"; +import { dirname, fromFileUrl, join } from "jsr:@std/path@1"; + +const root = join(dirname(fromFileUrl(import.meta.url)), "..", ".."); +const source = await resolveWasmtimeSource(); +const testDir = `${source.path}/tests/misc_testsuite/component-model`; +const outDir = join(root, "harness", "generated-wasmtime"); +const command = new Deno.Command("cargo", { + cwd: root, + args: [ + "run", + "-q", + "-p", + "testgen", + "--", + "--test-dir", + testDir, + "--out-dir", + outDir, + "--source-prefix", + `wasmtime@${source.rev}/tests/misc_testsuite/component-model`, + "--source-revision", + source.rev, + ], +}); +const status = await command.spawn().status; +if (!status.success) Deno.exit(status.code); diff --git a/tools/wasmtime/metadata.json b/tools/wasmtime/metadata.json new file mode 100644 index 00000000..4c37da5e --- /dev/null +++ b/tools/wasmtime/metadata.json @@ -0,0 +1,591 @@ +{ + "source_revision": "4675ee16b703b33948073a5ff6b961367371e7a1", + "files": [ + { + "path": "adapter.json", + "directives": { + "multi_memory": true + } + }, + { + "path": "alias-region-known-imported-adapter-memory.json", + "directives": { + "multi_memory": true + } + }, + { + "path": "alias-region-known-imported-canonical-abi-memory.json", + "directives": {} + }, + { + "path": "alias-region-known-imported-entities.json", + "directives": { + "gc": true + } + }, + { + "path": "alias-region-multiple-instantiations.json", + "directives": { + "gc": true + } + }, + { + "path": "alias-region-reexported-ambiguous-entities.json", + "directives": { + "gc": true + } + }, + { + "path": "alias-region-reexported-entities-to-imported-module.json", + "directives": { + "gc": true + } + }, + { + "path": "alias-region-reexported-known-entities.json", + "directives": { + "gc": true + } + }, + { + "path": "aliasing.json", + "directives": {} + }, + { + "path": "async/async-builtins.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true + } + }, + { + "path": "async/backpressure-deadlock.json", + "directives": { + "component_model_async": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/backpressure-overflow.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/callback-yield-then-exit.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "async/cancel-host.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/cancel-sibling-subtask.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "async/cancel-starting-subtask-does-not-leak.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "async/cancel-sync-and-waitable.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "reference_types": true + } + }, + { + "path": "async/context-in-compositions.json", + "directives": { + "component_model_async": true, + "multi_memory": true + } + }, + { + "path": "async/context-in-resource-drop.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/drop-deadlock.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/drop-host.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/drop-waitable-set-stackful.json", + "directives": { + "component_model_async": true, + "component_model_async_stackful": true, + "reference_types": true + } + }, + { + "path": "async/error-context.json", + "directives": { + "component_model_async": true, + "component_model_error_context": true + } + }, + { + "path": "async/exceptions.json", + "directives": { + "component_model_async": true, + "exceptions": true, + "reference_types": true + } + }, + { + "path": "async/fused.json", + "directives": { + "component_model_async": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/future-cancel-read-dropped.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/future-cancel-write-completed.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/future-cancel-write-dropped.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/future-drop-writable-after-notified-drop.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "async/future-read.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/futures-must-write.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/futures-must-write2.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/futures.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true + } + }, + { + "path": "async/intra-futures.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true + } + }, + { + "path": "async/intra-streams.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "multi_memory": true + } + }, + { + "path": "async/join-during-sync-read.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "component_model_threading": true + } + }, + { + "path": "async/lift.json", + "directives": { + "component_model_async": true, + "component_model_async_stackful": true + } + }, + { + "path": "async/lower.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/many-params-with-retptr.json", + "directives": { + "bulk_memory": true, + "component_model_async": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/partial-stream-copies.json", + "directives": { + "component_model_async": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/reenter-during-yield.json", + "directives": { + "component_model_async": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/reentrance.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "async/stackful.json", + "directives": { + "component_model_async": true, + "component_model_async_stackful": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/stream-big-read-and-writes.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "async/stream-cancel-finished-op.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/stream-zero-ops.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "async/streams-massive-send.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "reference_types": true + } + }, + { + "path": "async/streams.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/subtask-wait.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "async/sync-and-async-waitable.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "reference_types": true + } + }, + { + "path": "async/sync-call-context-slots.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "component_model_threading": true + } + }, + { + "path": "async/sync-call-context-trap.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true + } + }, + { + "path": "async/sync-call-context.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true + } + }, + { + "path": "async/sync-streams.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "reference_types": true + } + }, + { + "path": "async/task-builtins.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/task-deletion.json", + "directives": { + "component_model_async": true, + "component_model_async_stackful": true, + "component_model_more_async_builtins": true, + "component_model_threading": true, + "reference_types": true + } + }, + { + "path": "async/task-return-traps.json", + "directives": { + "component_model_async": true, + "component_model_async_stackful": true, + "component_model_threading": true + } + }, + { + "path": "async/trap-if-done.json", + "directives": { + "component_model_async": true, + "component_model_more_async_builtins": true, + "reference_types": true + } + }, + { + "path": "async/trap-if-transfer-in-waitable-set.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/wait-forever.json", + "directives": { + "component_model_async": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/wait-forever2.json", + "directives": { + "component_model_async": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "async/waitable-set-stale-entry.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "async/yield-when-cancelled.json", + "directives": { + "component_model_async": true, + "reference_types": true + } + }, + { + "path": "big-strings.json", + "directives": { + "hogs_memory": true, + "multi_memory": true + } + }, + { + "path": "enum_discriminant.json", + "directives": {} + }, + { + "path": "enums.json", + "directives": {} + }, + { + "path": "error-context-trap-in-post-return.json", + "directives": { + "component_model_error_context": true + } + }, + { + "path": "exceptions.json", + "directives": { + "bulk_memory": true, + "exceptions": true, + "function_references": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "fixed_length_lists.json", + "directives": { + "component_model_fixed_length_lists": true, + "multi_memory": true + } + }, + { + "path": "gc/empty.json", + "directives": { + "component_model_gc": true, + "gc": true, + "gc_types": true, + "multi_memory": true, + "reference_types": true + } + }, + { + "path": "implements-disabled.json", + "directives": { + "component_model_implements": false + } + }, + { + "path": "implements.json", + "directives": { + "component_model_implements": true + } + }, + { + "path": "import.json", + "directives": {} + }, + { + "path": "instance.json", + "directives": {} + }, + { + "path": "linking.json", + "directives": {} + }, + { + "path": "map-types.json", + "directives": { + "component_model_map": true + } + }, + { + "path": "memory64.json", + "directives": { + "bulk_memory": true, + "component_model_memory64": true, + "hogs_memory": true, + "memory64": true, + "multi_memory": true + } + }, + { + "path": "modules.json", + "directives": { + "exceptions": true, + "reference_types": true + } + }, + { + "path": "nested-many-instantiations.json", + "directives": {} + }, + { + "path": "nested.json", + "directives": {} + }, + { + "path": "resources.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "restrictions.json", + "directives": {} + }, + { + "path": "simple.json", + "directives": {} + }, + { + "path": "string-transcode-invalid.json", + "directives": { + "multi_memory": true + } + }, + { + "path": "strings.json", + "directives": { + "multi_memory": true + } + }, + { + "path": "tags.json", + "directives": { + "exceptions": true + } + }, + { + "path": "trap.json", + "directives": { + "component_model_async": true + } + }, + { + "path": "types.json", + "directives": {} + } + ] +} diff --git a/tools/wasmtime/run.ts b/tools/wasmtime/run.ts new file mode 100644 index 00000000..5ec288cd --- /dev/null +++ b/tools/wasmtime/run.ts @@ -0,0 +1,204 @@ +import { classify } from "../../harness/src/wasmtime-classifier.ts"; +import { + WASMTIME_EXCLUSIONS, + WASMTIME_EXPECTATIONS, + WASMTIME_SKIP_EXPECTATIONS, +} from "../../harness/src/wasmtime-expectations.ts"; +import type { FileResult } from "../../harness/src/runner.ts"; +import { dirname, fromFileUrl, join } from "jsr:@std/path@1"; +import { runChild, validateWorkerResult } from "./subprocess.ts"; + +const repo = join(dirname(fromFileUrl(import.meta.url)), "..", ".."); +const generated = join(repo, "harness", "generated-wasmtime"); +const manifest = JSON.parse( + await Deno.readTextFile(join(generated, "manifest.json")), +) as { files: string[] }; +const generatedMetadata = JSON.parse( + await Deno.readTextFile(join(generated, "supplementary-metadata.json")), +) as { source_revision: string; files: unknown[] }; +const reviewedMetadata = JSON.parse( + await Deno.readTextFile(join(repo, "tools/wasmtime/metadata.json")), +) as { source_revision: string; files: unknown[] }; +const reviewedFiles = (reviewedMetadata.files as Array<{ path: string }>).map(( + f, +) => f.path); +if (JSON.stringify(manifest.files) !== JSON.stringify(reviewedFiles)) { + throw new Error("Wasmtime inventory drift"); +} +if (reviewedMetadata.source_revision !== generatedMetadata.source_revision) { + throw new Error("Wasmtime reviewed-corpus revision drift"); +} +const generatedReviewedShape = { + source_revision: generatedMetadata.source_revision, + files: + (generatedMetadata.files as Array<{ path: string; directives: unknown }>) + .map((f) => ({ + path: f.path, + directives: f.directives, + })), +}; +if ( + JSON.stringify(generatedReviewedShape) !== JSON.stringify(reviewedMetadata) +) { + throw new Error("Wasmtime directive/source metadata drift"); +} +for (const file of Object.keys(WASMTIME_EXCLUSIONS)) { + if (!manifest.files.includes(file)) { + throw new Error(`stale exclusion: ${file}`); + } +} + +const counts = { + inventory: manifest.files.length, + passed: 0, + setupOnly: 0, + assertions: 0, + knownFailures: {} as Record, + skips: {} as Record, + excludedFiles: 0, + excludedCommands: 0, + infrastructureFailures: 0, + infrastructureCommands: 0, + totalCommands: 0, + unexpected: 0, +}; +const failures: string[] = []; +const seenExpectations = new Set(); +const seenSkips = new Set(); +const fileResults: Array> = []; +for (const file of manifest.files) { + const doc = JSON.parse(await Deno.readTextFile(join(generated, file))) as { + source_filename: string; + commands: Array<{ line: number; type: string }>; + }; + const exclusion = WASMTIME_EXCLUSIONS[file]; + counts.totalCommands += doc.commands.length; + if (exclusion !== undefined) { + counts.excludedFiles++; + counts.excludedCommands += doc.commands.length; + console.log(`EXCLUDED ${file}: ${exclusion}`); + fileResults.push({ + file, + status: "excluded", + reason: exclusion, + commands: doc.commands.length, + }); + continue; + } + const command = new Deno.Command(Deno.execPath(), { + cwd: repo, + args: [ + "run", + "--no-lock", + "--allow-read", + "--allow-env=POLYENGINE_SCHED_SEED", + join(repo, "tools/wasmtime/worker.ts"), + file, + ], + stdout: "piped", + stderr: "piped", + }); + const outcome = await runChild(command, 30_000); + if (outcome === "timeout") { + counts.infrastructureFailures++; + counts.infrastructureCommands += doc.commands.length; + failures.push(`${file}: infrastructure timeout`); + fileResults.push({ + file, + status: "infrastructure-failure", + reason: "timeout", + }); + continue; + } + if (!outcome.success) { + counts.infrastructureFailures++; + counts.infrastructureCommands += doc.commands.length; + failures.push( + `${file}: worker failed: ${new TextDecoder().decode(outcome.stderr)}`, + ); + fileResults.push({ + file, + status: "infrastructure-failure", + reason: "worker exit", + }); + continue; + } + let result: FileResult; + try { + result = JSON.parse(new TextDecoder().decode(outcome.stdout)) as FileResult; + } catch (error) { + counts.infrastructureFailures++; + counts.infrastructureCommands += doc.commands.length; + failures.push(`${file}: malformed worker JSON: ${error}`); + fileResults.push({ + file, + status: "infrastructure-failure", + reason: "malformed JSON", + }); + continue; + } + const malformed = validateWorkerResult(doc, result); + if (malformed !== undefined) { + counts.infrastructureFailures++; + counts.infrastructureCommands += doc.commands.length; + failures.push(`${file}: ${malformed}`); + fileResults.push({ + file, + status: "infrastructure-failure", + reason: malformed, + }); + continue; + } + for (const command of result.results) { + const verdict = classify(file, command); + if (verdict.status === "passed") { + counts.passed++; + if ( + command.type === "module" || command.type === "module_definition" || + command.type === "module_instance" + ) counts.setupOnly++; + else counts.assertions++; + } else if (verdict.status === "known-failure") { + seenExpectations.add(`${file}:${command.line}`); + counts.knownFailures[verdict.class] = + (counts.knownFailures[verdict.class] ?? 0) + 1; + } else if (verdict.status === "skip") { + seenSkips.add(`${file}:${command.line}`); + counts.skips[verdict.class] = (counts.skips[verdict.class] ?? 0) + 1; + } else { + counts.unexpected++; + failures.push(`${file}:${command.line}: ${verdict.detail}`); + } + } + fileResults.push({ file, status: "executed", results: result.results }); +} +for (const expected of WASMTIME_EXPECTATIONS) { + const key = `${expected.file}:${expected.line}`; + if (!seenExpectations.has(key)) { + failures.push(`${key}: stale or unreachable expectation`); + } +} +for (const expected of WASMTIME_SKIP_EXPECTATIONS) { + const key = `${expected.file}:${expected.line}`; + if (!seenSkips.has(key)) { + failures.push(`${key}: stale or unreachable skip expectation`); + } +} +const accounted = counts.passed + + Object.values(counts.knownFailures).reduce((a, b) => a + b, 0) + + Object.values(counts.skips).reduce((a, b) => a + b, 0) + + counts.excludedCommands + counts.unexpected + counts.infrastructureCommands; +if (accounted !== counts.totalCommands) { + failures.push( + `command accounting mismatch: ${accounted} != ${counts.totalCommands}`, + ); +} +await Deno.writeTextFile( + join(generated, "results.json"), + JSON.stringify({ counts, files: fileResults, failures }, null, 2) + "\n", +); +console.log(JSON.stringify(counts, null, 2)); +if (failures.length > 0) { + console.error(failures.join("\n")); + Deno.exit(1); +} diff --git a/tools/wasmtime/source.ts b/tools/wasmtime/source.ts new file mode 100644 index 00000000..7e52b235 --- /dev/null +++ b/tools/wasmtime/source.ts @@ -0,0 +1,58 @@ +import { dirname, fromFileUrl, join } from "jsr:@std/path@1"; + +const REPO_ROOT = join(dirname(fromFileUrl(import.meta.url)), "..", ".."); + +/** Resolve the exact Wasmtime checkout selected by Cargo.lock. */ +export interface WasmtimeSource { + path: string; + rev: string; +} + +interface MetadataPackage { + name: string; + source: string | null; + manifest_path: string; +} + +export async function resolveWasmtimeSource(): Promise { + const command = new Deno.Command("cargo", { + cwd: REPO_ROOT, + args: ["metadata", "--locked", "--format-version", "1"], + stdout: "piped", + stderr: "piped", + }); + const output = await command.output(); + if (!output.success) { + throw new Error( + `cargo metadata --locked failed: ${ + new TextDecoder().decode(output.stderr) + }`, + ); + } + const metadata = JSON.parse(new TextDecoder().decode(output.stdout)) as { + packages: MetadataPackage[]; + }; + const pkg = metadata.packages.find((p) => p.name === "wasmtime-environ"); + if (pkg === undefined || pkg.source === null) { + throw new Error( + "locked wasmtime-environ package is absent from cargo metadata", + ); + } + const precise = new URLSearchParams( + pkg.source.split("?")[1]?.split("#")[0] ?? "", + ).get("rev"); + const locked = pkg.source.split("#")[1]; + const rev = locked ?? precise; + if (rev === null || !/^[0-9a-f]{40}$/.test(rev)) { + throw new Error( + `wasmtime-environ source has no precise revision: ${pkg.source}`, + ); + } + // wasmtime-environ lives at /crates/environ/Cargo.toml. + const path = dirname(dirname(dirname(pkg.manifest_path))); + return { path, rev }; +} + +if (import.meta.main) { + console.log(JSON.stringify(await resolveWasmtimeSource())); +} diff --git a/tools/wasmtime/subprocess.ts b/tools/wasmtime/subprocess.ts new file mode 100644 index 00000000..c7b0988f --- /dev/null +++ b/tools/wasmtime/subprocess.ts @@ -0,0 +1,76 @@ +export async function runChild( + command: Deno.Command, + timeoutMs: number, +): Promise { + const child = command.spawn(); + const outputPromise = child.output(); + let timer: ReturnType | undefined; + const timeout = new Promise<"timeout">((resolve) => { + timer = setTimeout(() => resolve("timeout"), timeoutMs); + }); + const outcome = await Promise.race([outputPromise, timeout]).finally(() => + clearTimeout(timer) + ); + if (outcome !== "timeout") return outcome; + try { + child.kill("SIGKILL"); + } catch (error) { + if (!(error instanceof Deno.errors.NotFound)) throw error; + } + await outputPromise; + return "timeout"; +} + +export function validateWorkerResult( + doc: { + source_filename: string; + commands: Array<{ line: number; type: string }>; + }, + value: unknown, +): string | undefined { + if (typeof value !== "object" || value === null) { + return "worker result is not an object"; + } + const result = value as { source?: unknown; results?: unknown }; + if (result.source !== doc.source_filename) { + return "worker provenance mismatch"; + } + if (!Array.isArray(result.results)) return "worker results is not an array"; + if (result.results.length !== doc.commands.length) { + return "worker command count mismatch"; + } + for (let i = 0; i < doc.commands.length; i++) { + const row = result.results[i] as { + line?: unknown; + type?: unknown; + status?: unknown; + detail?: unknown; + reason?: unknown; + }; + if ( + typeof row !== "object" || row === null || + row.line !== doc.commands[i].line || row.type !== doc.commands[i].type + ) return `worker command mismatch at index ${i}`; + if ( + row.status !== "passed" && row.status !== "failed" && + row.status !== "skipped" + ) { + return `worker command status malformed at index ${i}`; + } + if (row.detail !== undefined && typeof row.detail !== "string") { + return `worker command detail malformed at index ${i}`; + } + if ( + row.reason !== undefined && row.reason !== "pending-runtime" && + row.reason !== "pending-capability" && + row.reason !== "unsupported-directive" + ) return `worker command reason malformed at index ${i}`; + if (row.status === "skipped" && row.reason === undefined) { + return `worker skipped command has no reason at index ${i}`; + } + if (row.status !== "skipped" && row.reason !== undefined) { + return `worker non-skipped command has a reason at index ${i}`; + } + } + return undefined; +} diff --git a/tools/wasmtime/worker.ts b/tools/wasmtime/worker.ts new file mode 100644 index 00000000..debda862 --- /dev/null +++ b/tools/wasmtime/worker.ts @@ -0,0 +1,23 @@ +import type { WastJson } from "../../harness/src/schema.ts"; +import { runWastJson } from "../../harness/src/runner.ts"; +import { RuntimeExecutor } from "../../harness/src/runtime-executor.ts"; +import { wasmtimeSpectest } from "../../harness/src/wasmtime-spectest.ts"; +import { dirname, fromFileUrl, join } from "jsr:@std/path@1"; + +const file = Deno.args[0]; +if (file === undefined) throw new Error("worker needs a generated file path"); +const repo = join(dirname(fromFileUrl(import.meta.url)), "..", ".."); +const generated = join(repo, "harness", "generated-wasmtime"); +const doc = JSON.parse( + await Deno.readTextFile(join(generated, file)), +) as WastJson; +const shim = await Deno.readFile( + join(repo, "target/wasm32-unknown-unknown/release/translator_shim.wasm"), +); +const dir = dirname(file) === "." ? "" : dirname(file); +const result = await runWastJson( + doc, + (name) => Deno.readFile(join(generated, dir, name)), + await RuntimeExecutor.create(shim, wasmtimeSpectest(file).imports), +); +console.log(JSON.stringify(result));