From 1919daf286b308b31fcd65ea794619a2d47af443 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 16:04:54 +0800 Subject: [PATCH 01/64] feat(mega-evme): rename --rpc.rate-limit to --rpc.cu-per-sec Canonical flag is now --rpc.cu-per-sec; --rpc.rate-limit remains a visible alias. Clarify that the value is a CU/s budget, not RPS, and warn once at provider build when retries are on and the budget is <100. --- bin/mega-evme/src/common/provider/mod.rs | 64 ++++++++++++++++++- bin/mega-evme/tests/provider.rs | 14 ++++ docs/mega-evme/commands/run.md | 9 ++- docs/mega-evme/commands/tx.md | 4 +- .../configuration/state-management.md | 10 +-- 5 files changed, 90 insertions(+), 11 deletions(-) diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 94369a6f..5cc2c677 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -135,8 +135,11 @@ pub struct RpcArgs { #[arg(long = "rpc.backoff-ms", default_value_t = 1_000)] pub backoff_ms: u64, - /// Compute units per second budget passed to the retry layer's rate-limit accounting. - #[arg(long = "rpc.rate-limit", default_value_t = 660)] + /// Compute-unit budget (CU/s) for the retry layer's rate-limit accounting. + /// This is NOT requests per second: each RPC method costs multiple compute units. + /// A single-digit value will heavily self-throttle. Default (660) matches typical + /// public-endpoint budgets. + #[arg(long = "rpc.cu-per-sec", visible_alias = "rpc.rate-limit", default_value_t = 660)] pub compute_units_per_sec: u64, } @@ -155,6 +158,9 @@ impl RpcArgs { EvmeError::RpcError(format!("Invalid RPC URL '{}': {}", rpc_url_str, e)) })?; + // Once per provider build (not per client: resolve_chain_id also builds a client). + self.maybe_warn_low_cu_per_sec(); + // 1. Resolve chain id (always needed by downstream consumers). let chain_id = self.resolve_chain_id(url.clone()).await?; @@ -304,6 +310,9 @@ impl RpcArgs { EvmeError::RpcError(format!("Invalid RPC URL '{}': {}", rpc_url_str, e)) })?; + // Once per provider build (capture builds a single client; keep the same entry point). + self.maybe_warn_low_cu_per_sec(); + // Load existing envelope if the file exists. let existing_envelope = if path.exists() { let env = CacheFileEnvelope::load(path)?; @@ -402,6 +411,17 @@ impl RpcArgs { } } + /// Emit the low CU/s warning at most once per networked provider build. + /// + /// Not placed in [`Self::build_client`]: the standard path builds a throwaway + /// client for chain-id resolution and then the real client, so a warning there + /// would fire twice. + fn maybe_warn_low_cu_per_sec(&self) { + if let Some(msg) = cu_per_sec_warning(self.max_retries, self.compute_units_per_sec) { + warn!("{msg}"); + } + } + /// Resolve the chain ID by issuing `eth_chainId` against a throwaway /// cache-less provider using the configured retry policy. async fn resolve_chain_id(&self, url: reqwest::Url) -> Result { @@ -413,6 +433,23 @@ impl RpcArgs { } } +/// Threshold below which a configured CU/s budget is considered dangerously low. +/// Values under this with retries enabled produce a one-shot warning at provider build. +const CU_PER_SEC_WARN_THRESHOLD: u64 = 100; + +/// Return a warning message when the retry layer is enabled and the CU/s budget is +/// below [`CU_PER_SEC_WARN_THRESHOLD`]. Used so the trigger rule is unit-testable +/// without capturing log output. +fn cu_per_sec_warning(max_retries: u32, compute_units_per_sec: u64) -> Option { + (max_retries > 0 && compute_units_per_sec < CU_PER_SEC_WARN_THRESHOLD).then(|| { + format!( + "--rpc.cu-per-sec is set to {compute_units_per_sec}, which is a compute-unit \ + budget (CU/s) for the retry layer's rate-limit accounting, NOT requests per \ + second; such a low budget will heavily self-throttle RPC traffic" + ) + }) +} + /// `clap` value parser that rejects empty and whitespace-only path arguments /// at parse time. /// @@ -495,4 +532,27 @@ mod tests { let expected = expected_root.join("mega-evme").join("rpc").join("rpc-cache-11155420.json"); assert_eq!(path, expected); } + + /// Warn when retries are on and CU/s is below the threshold. + #[test] + fn test_cu_per_sec_warning_fires_below_threshold_with_retries() { + let msg = cu_per_sec_warning(5, 99).expect("should warn at 99 with retries on"); + assert!(msg.contains("99"), "message should include the configured value: {msg}"); + assert!(msg.contains("NOT requests per second") || msg.contains("NOT requests"), "{msg}"); + assert!(msg.contains("self-throttle"), "{msg}"); + } + + /// Silent at the threshold boundary (100) and at the production default (660). + #[test] + fn test_cu_per_sec_warning_silent_at_or_above_threshold() { + assert!(cu_per_sec_warning(5, 100).is_none()); + assert!(cu_per_sec_warning(5, 660).is_none()); + } + + /// Silent when the retry layer is disabled, even with a low CU/s budget. + #[test] + fn test_cu_per_sec_warning_silent_when_retries_disabled() { + assert!(cu_per_sec_warning(0, 1).is_none()); + assert!(cu_per_sec_warning(0, 99).is_none()); + } } diff --git a/bin/mega-evme/tests/provider.rs b/bin/mega-evme/tests/provider.rs index f575964f..6a4ab185 100644 --- a/bin/mega-evme/tests/provider.rs +++ b/bin/mega-evme/tests/provider.rs @@ -24,6 +24,7 @@ use common::{test_rpc_args, test_rpc_args_cached, MockRpcServer}; #[test] fn test_rpc_args_parses_all_new_flags() { + // Keep `--rpc.rate-limit` here so the visible alias stays pin-tested. let args = RpcArgs::parse_from([ "mega-evme", "--rpc", @@ -51,6 +52,19 @@ fn test_rpc_args_parses_all_new_flags() { assert_eq!(args.compute_units_per_sec, 1234); } +/// Canonical flag name `--rpc.cu-per-sec` parses into the same field. +#[test] +fn test_rpc_args_parses_cu_per_sec_flag() { + let args = RpcArgs::parse_from([ + "mega-evme", + "--rpc", + "https://example.test/rpc", + "--rpc.cu-per-sec", + "1234", + ]); + assert_eq!(args.compute_units_per_sec, 1234); +} + /// `--rpc.cache-dir ""` (and whitespace-only) must be rejected at parse /// time. The alternative — silently landing in `PathBuf::from("")` and /// writing the cache to CWD — is a footgun since the same command run diff --git a/docs/mega-evme/commands/run.md b/docs/mega-evme/commands/run.md index 3204eb69..cfefe35b 100644 --- a/docs/mega-evme/commands/run.md +++ b/docs/mega-evme/commands/run.md @@ -54,7 +54,7 @@ Each group is documented on its own page. | Chain and spec | `--spec`, `--chain-id` | [Chain and Spec](../configuration/chain-and-spec.md) | | Block environment | `--block.number`, `--block.coinbase`, `--block.timestamp`, `--block.gaslimit`, `--block.basefee`, `--block.difficulty`, `--block.prevrandao`, `--block.blobexcessgas` | [Block Environment](../configuration/block-environment.md) | | SALT buckets | `--bucket-capacity` | [SALT Buckets](../configuration/salt-buckets.md) | -| RPC cache / retry | `--rpc.cache-size`, `--rpc.cache-dir`, `--rpc.no-cache-file`, `--rpc.clear-cache`, `--rpc.max-retries`, `--rpc.backoff-ms`, `--rpc.rate-limit` | [RPC Cache and Retry](../configuration/state-management.md#rpc-cache-and-retry) | +| RPC cache / retry | `--rpc.cache-size`, `--rpc.cache-dir`, `--rpc.no-cache-file`, `--rpc.clear-cache`, `--rpc.max-retries`, `--rpc.backoff-ms`, `--rpc.cu-per-sec` | [RPC Cache and Retry](../configuration/state-management.md#rpc-cache-and-retry) | | Tracing | `--trace`, `--tracer`, `--trace.output`, and tracer-specific flags | [Tracing Overview](../tracing/overview.md) | | Output | `--json` | See [JSON output](#json-output) below | @@ -322,8 +322,11 @@ RPC Options: --rpc.backoff-ms Fixed sleep (ms) between retries; no exponential backoff [default: 1000] - --rpc.rate-limit - Compute-units-per-second budget for the retry layer [default: 660] + --rpc.cu-per-sec + Compute-unit budget (CU/s) for the retry layer's rate-limit accounting. This is NOT requests per second: each RPC method costs multiple compute units. A single-digit value will heavily self-throttle. Default (660) matches typical public-endpoint budgets + + [default: 660] + [alias: --rpc.rate-limit] Chain Options: --spec diff --git a/docs/mega-evme/commands/tx.md b/docs/mega-evme/commands/tx.md index ce45e33b..08bca30f 100644 --- a/docs/mega-evme/commands/tx.md +++ b/docs/mega-evme/commands/tx.md @@ -210,7 +210,9 @@ RPC Options: --rpc.clear-cache Delete the current chain's cache file before loading --rpc.max-retries Max transport retries; 0 disables [default: 5] --rpc.backoff-ms Fixed retry sleep in ms [default: 1000] - --rpc.rate-limit Retry-layer compute-units-per-second budget [default: 660] + --rpc.cu-per-sec + Compute-unit budget (CU/s) for the retry layer's rate-limit accounting (NOT requests/s) [default: 660] + [alias: --rpc.rate-limit] Chain Options: --spec Spec [default: Rex7] diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index 86584df8..c22e598f 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -228,11 +228,11 @@ The default cache directory is the platform cache directory: ### Retry Flags -| Flag | Type | Default | Description | -| ------------------------- | ----- | ------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| `--rpc.max-retries ` | `u32` | `5` | Maximum retry attempts for failing RPC requests. Retries on HTTP 429/503, rate-limit errors, and transport failures. `0` to disable. | -| `--rpc.backoff-ms ` | `u64` | `1000` | Fixed sleep duration in milliseconds between retry attempts (no exponential backoff). | -| `--rpc.rate-limit ` | `u64` | `660` | Compute units per second budget for the retry layer's rate-limit accounting. | +| Flag | Type | Default | Description | +| ------------------------- | ----- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--rpc.max-retries ` | `u32` | `5` | Maximum retry attempts for failing RPC requests. Retries on HTTP 429/503, rate-limit errors, and transport failures. `0` to disable. | +| `--rpc.backoff-ms ` | `u64` | `1000` | Fixed sleep duration in milliseconds between retry attempts (no exponential backoff). | +| `--rpc.cu-per-sec ` | `u64` | `660` | Compute-unit budget (CU/s) for the retry layer's rate-limit accounting — not requests per second. Alias: `--rpc.rate-limit`. Values below 100 with retries enabled emit a warning. | ### Examples From c41ff35df4fc0f6451906f531e37870adc453984 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 16:18:07 +0800 Subject: [PATCH 02/64] feat(mega-evme): replace --rpc.cache-size with --rpc.cache-max-entries Delete --rpc.cache-size and introduce --rpc.cache-max-entries (default 0 = never evict) so verification workloads keep large RPC caches. The cache layer is always installed; 0 maps to u32::MAX capacity. Capture/replay conflict lists and docs updated. --- bin/mega-evme/src/common/provider/mod.rs | 88 +++++++++++-------- bin/mega-evme/tests/common/mod.rs | 9 +- .../fixtures/test_run_rpc_args_parse.json | 2 +- bin/mega-evme/tests/provider.rs | 59 +++++++++---- bin/mega-evme/tests/state.rs | 6 +- docs/mega-evme/commands/replay.md | 4 +- docs/mega-evme/commands/run.md | 6 +- docs/mega-evme/commands/tx.md | 2 +- .../configuration/state-management.md | 12 +-- 9 files changed, 111 insertions(+), 77 deletions(-) diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 5cc2c677..5bd105c5 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -45,11 +45,11 @@ pub type OpProvider = DynProvider; /// Return value of the `RpcArgs::build_*_provider` methods. #[derive(Debug)] pub struct BuildProviderOutput { - /// Configured OP-stack provider. Already wrapped with the retry layer and (unless - /// the cache is disabled) the in-memory cache layer. + /// Configured OP-stack provider. Already wrapped with the retry layer and the + /// in-memory cache layer. pub provider: OpProvider, /// Clean-exit cache persistence handle. Call [`RpcCacheStore::persist`] on the - /// success path; no-op when the cache is disabled. + /// success path; no-op when on-disk persistence is disabled (`--rpc.no-cache-file`). pub cache_store: RpcCacheStore, /// Chain id resolved during provider construction. Always populated — /// comes from `eth_chainId` (standard/capture) or the envelope (replay). @@ -75,12 +75,12 @@ pub struct RpcArgs { /// If the file already exists, its entries are loaded and merged; /// missing entries are fetched via the RPC endpoint and persisted on clean exit. /// Cannot be used with --rpc.replay-file, --rpc.cache-dir, --rpc.clear-cache, - /// --rpc.no-cache-file, or --rpc.cache-size. + /// --rpc.no-cache-file, or --rpc.cache-max-entries. #[arg( long = "rpc.capture-file", value_parser = parse_non_empty_path, requires = "rpc_url", - conflicts_with_all = ["replay_file", "cache_dir", "clear_cache", "no_cache_file", "cache_size"], + conflicts_with_all = ["replay_file", "cache_dir", "clear_cache", "no_cache_file", "cache_max_entries"], )] pub capture_file: Option, @@ -88,18 +88,18 @@ pub struct RpcArgs { /// Cannot be used with `--rpc`. /// Any RPC miss is a hard error; the file is never written. /// Cannot be used with --rpc.capture-file, --rpc.cache-dir, --rpc.clear-cache, - /// --rpc.no-cache-file, or --rpc.cache-size. + /// --rpc.no-cache-file, or --rpc.cache-max-entries. #[arg( long = "rpc.replay-file", value_parser = parse_non_empty_path, - conflicts_with_all = ["rpc_url", "capture_file", "cache_dir", "clear_cache", "no_cache_file", "cache_size"], + conflicts_with_all = ["rpc_url", "capture_file", "cache_dir", "clear_cache", "no_cache_file", "cache_max_entries"], )] pub replay_file: Option, - /// Maximum number of items to keep in the in-memory RPC LRU cache. - /// Set to 0 to disable the cache layer entirely. - #[arg(id = "cache_size", long = "rpc.cache-size", default_value_t = 10_000)] - pub cache_size: u32, + /// Maximum number of items in the in-memory RPC LRU cache (and therefore what + /// gets persisted to the cache file). `0` = unlimited (never evict). Default is `0`. + #[arg(id = "cache_max_entries", long = "rpc.cache-max-entries", default_value_t = 0)] + pub cache_max_entries: u32, /// Directory for per-chain RPC cache files. /// @@ -112,8 +112,7 @@ pub struct RpcArgs { #[arg(long = "rpc.cache-dir", value_parser = parse_non_empty_path)] pub cache_dir: Option, - /// Disable on-disk cache persistence. The in-memory LRU cache still applies — use - /// `--rpc.cache-size 0` to disable that too. + /// Disable on-disk cache persistence. The in-memory LRU cache still applies. #[arg(long = "rpc.no-cache-file")] pub no_cache_file: bool, @@ -164,32 +163,17 @@ impl RpcArgs { // 1. Resolve chain id (always needed by downstream consumers). let chain_id = self.resolve_chain_id(url.clone()).await?; - // 2. Fast path: cache fully disabled. - if self.cache_size == 0 { - let provider = build_bare_op_provider(self.build_retry_client(url)); - info!( - rpc_url = %rpc_url_str, - max_retries = self.max_retries, - backoff_ms = self.backoff_ms, - "Built RPC provider (cache disabled)", - ); - return Ok(BuildProviderOutput { - provider, - cache_store: RpcCacheStore::noop(), - chain_id, - external_env: None, - }); - } - - // 3. Resolve on-disk cache path (None when disk persistence is disabled). + // 2. Resolve on-disk cache path (None when disk persistence is disabled). let cache_path = if self.no_cache_file { None } else { Some(resolve_cache_path(self.cache_dir.as_deref(), chain_id)?) }; - // 4. Build the cache layer and (optionally) the disk store. - let cache_layer = CacheLayer::new(self.cache_size); + // 3. Build the cache layer and (optionally) the disk store. + // Cache layer is always installed; 0 max entries means unlimited (never evict). + let max_items = cache_max_entries_capacity(self.cache_max_entries); + let cache_layer = CacheLayer::new(max_items); let cache = cache_layer.cache(); let cache_store = match cache_path { Some(path) => { @@ -215,6 +199,10 @@ impl RpcArgs { } } if path.exists() { + // Oversized-load warning is intentionally omitted: alloy's + // `SharedCache` has no public entry-count API (`len` / loaded-count), + // so a file-with-N-entries-over-cap warning cannot be emitted + // without re-implementing load or file-size heuristics (rejected). if let Err(err) = cache.load_cache(path.clone()) { warn!( path = %path.display(), @@ -228,7 +216,7 @@ impl RpcArgs { None => RpcCacheStore::noop(), }; - // 5. Build the cached provider. + // 4. Build the cached provider. let client = self.build_retry_client(url); let provider = ProviderBuilder::new() .disable_recommended_fillers() @@ -238,7 +226,8 @@ impl RpcArgs { info!( rpc_url = %rpc_url_str, - cache_size = self.cache_size, + cache_max_entries = self.cache_max_entries, + cache_capacity = max_items, max_retries = self.max_retries, backoff_ms = self.backoff_ms, "Built RPC provider", @@ -466,12 +455,23 @@ fn parse_non_empty_path(s: &str) -> std::result::Result { } } +/// Map `--rpc.cache-max-entries` to the capacity passed to [`CacheLayer::new`]. +/// +/// `0` means unlimited (never evict) and is represented as [`u32::MAX`]. +/// Nonzero values pass through unchanged. +fn cache_max_entries_capacity(cache_max_entries: u32) -> u32 { + if cache_max_entries == 0 { + u32::MAX + } else { + cache_max_entries + } +} + /// Build a cache-less [`OpProvider`] from an already-configured `RpcClient`. /// -/// Used by the cache-disabled fast path and by the throwaway chain-id fetch. -/// The cache-enabled path builds its provider inline because the cache layer -/// has to be inserted into the `ProviderBuilder` chain before the client is -/// attached. +/// Used by the throwaway chain-id fetch. The standard path builds its provider +/// inline because the cache layer has to be inserted into the `ProviderBuilder` +/// chain before the client is attached. fn build_bare_op_provider(client: RpcClient) -> OpProvider { DynProvider::new( ProviderBuilder::new() @@ -555,4 +555,14 @@ mod tests { assert!(cu_per_sec_warning(0, 1).is_none()); assert!(cu_per_sec_warning(0, 99).is_none()); } + + /// `0` maps to unlimited capacity (`u32::MAX`); nonzero values pass through. + #[test] + fn test_cache_max_entries_capacity_mapping() { + assert_eq!(cache_max_entries_capacity(0), u32::MAX); + assert_eq!(cache_max_entries_capacity(1), 1); + assert_eq!(cache_max_entries_capacity(256), 256); + assert_eq!(cache_max_entries_capacity(10_000), 10_000); + assert_eq!(cache_max_entries_capacity(u32::MAX), u32::MAX); + } } diff --git a/bin/mega-evme/tests/common/mod.rs b/bin/mega-evme/tests/common/mod.rs index 54dfc716..c0cd0626 100644 --- a/bin/mega-evme/tests/common/mod.rs +++ b/bin/mega-evme/tests/common/mod.rs @@ -109,7 +109,7 @@ impl MockRpcServer { /// Build [`RpcArgs`] for a test pointed at `url` with the on-disk cache disabled. /// -/// Defaults: `--rpc.cache-size 0` (no cache layer, no disk persistence), +/// Defaults: `--rpc.no-cache-file` (in-memory LRU still applies; no disk persistence), /// 1ms backoff, production rate limit. `build_provider` still calls /// `eth_chainId`, so the caller must mount a mock for it. Pass `Some(n)` to /// override `--rpc.max-retries`; `None` keeps the production default. @@ -118,8 +118,7 @@ pub(crate) fn test_rpc_args(url: &str, max_retries: Option) -> RpcArgs { "mega-evme".into(), "--rpc".into(), url.into(), - "--rpc.cache-size".into(), - "0".into(), + "--rpc.no-cache-file".into(), "--rpc.backoff-ms".into(), "1".into(), "--rpc.rate-limit".into(), @@ -134,7 +133,7 @@ pub(crate) fn test_rpc_args(url: &str, max_retries: Option) -> RpcArgs { /// Build [`RpcArgs`] for a test that exercises the on-disk cache path. /// -/// Sets `--rpc.cache-size 256` and an explicit `--rpc.cache-dir`. The caller +/// Sets `--rpc.cache-max-entries 256` and an explicit `--rpc.cache-dir`. The caller /// must mount a mock `eth_chainId` response on the server so that /// `build_provider`'s `resolve_chain_id` call succeeds — use /// [`MockRpcServer::respond_eth_chain_id`] for this. @@ -147,7 +146,7 @@ pub(crate) fn test_rpc_args_cached( "mega-evme".into(), "--rpc".into(), url.into(), - "--rpc.cache-size".into(), + "--rpc.cache-max-entries".into(), "256".into(), "--rpc.cache-dir".into(), cache_dir.to_str().expect("cache_dir utf-8").to_string(), diff --git a/bin/mega-evme/tests/fixtures/test_run_rpc_args_parse.json b/bin/mega-evme/tests/fixtures/test_run_rpc_args_parse.json index 0e5fb157..e8d43539 100644 --- a/bin/mega-evme/tests/fixtures/test_run_rpc_args_parse.json +++ b/bin/mega-evme/tests/fixtures/test_run_rpc_args_parse.json @@ -3,7 +3,7 @@ "args": [ "run", "0x604260005260206000f3", - "--rpc.cache-size", "100", + "--rpc.cache-max-entries", "100", "--rpc.max-retries", "3", "--rpc.backoff-ms", "500", "--rpc.rate-limit", "660" diff --git a/bin/mega-evme/tests/provider.rs b/bin/mega-evme/tests/provider.rs index 6a4ab185..2a475c91 100644 --- a/bin/mega-evme/tests/provider.rs +++ b/bin/mega-evme/tests/provider.rs @@ -29,7 +29,7 @@ fn test_rpc_args_parses_all_new_flags() { "mega-evme", "--rpc", "https://example.test/rpc", - "--rpc.cache-size", + "--rpc.cache-max-entries", "256", "--rpc.cache-dir", "/tmp/example-cache", @@ -43,7 +43,7 @@ fn test_rpc_args_parses_all_new_flags() { "1234", ]); assert_eq!(args.rpc_url, Some("https://example.test/rpc".to_string())); - assert_eq!(args.cache_size, 256); + assert_eq!(args.cache_max_entries, 256); assert_eq!(args.cache_dir, Some(PathBuf::from("/tmp/example-cache"))); assert!(args.no_cache_file); assert!(args.clear_cache); @@ -52,6 +52,26 @@ fn test_rpc_args_parses_all_new_flags() { assert_eq!(args.compute_units_per_sec, 1234); } +/// The removed `--rpc.cache-size` flag must fail to parse (pin the deletion). +#[test] +fn test_rpc_args_rejects_removed_cache_size_flag() { + let err = RpcArgs::try_parse_from([ + "mega-evme", + "--rpc", + "https://example.test/rpc", + "--rpc.cache-size", + "100", + ]) + .expect_err("removed --rpc.cache-size must not parse"); + let msg = err.to_string(); + assert!( + msg.contains("unexpected argument") || + msg.contains("unknown") || + msg.contains("cache-size"), + "error must reject the removed flag, got: {msg}", + ); +} + /// Canonical flag name `--rpc.cu-per-sec` parses into the same field. #[test] fn test_rpc_args_parses_cu_per_sec_flag() { @@ -94,7 +114,7 @@ fn test_rpc_args_rejects_empty_cache_dir() { fn test_rpc_args_default_values() { let args = RpcArgs::parse_from(["mega-evme"]); assert_eq!(args.rpc_url, None); - assert_eq!(args.cache_size, 10_000); + assert_eq!(args.cache_max_entries, 0, "default is unlimited (never evict)"); assert_eq!(args.cache_dir, None); assert!(!args.no_cache_file); assert!(!args.clear_cache); @@ -105,16 +125,18 @@ fn test_rpc_args_default_values() { // ─── build_provider shape variants ─────────────────────────────────────────── -/// `--rpc.cache-size 0`: noop store, but `chain_id` is still resolved. +/// Default `--rpc.cache-max-entries 0` (unlimited) still resolves `chain_id` +/// and installs the in-memory cache layer; with `--rpc.no-cache-file` the +/// disk store is a no-op. #[tokio::test(flavor = "multi_thread")] -async fn test_build_provider_without_cache() { +async fn test_build_provider_default_unlimited_with_no_cache_file() { let server = MockRpcServer::start().await; server.respond_eth_chain_id(4326, 1).await; - let args = RpcArgs::parse_from(["mega-evme", "--rpc", &server.uri(), "--rpc.cache-size", "0"]); + let args = RpcArgs::parse_from(["mega-evme", "--rpc", &server.uri(), "--rpc.no-cache-file"]); let BuildProviderOutput { cache_store, chain_id, .. } = args.build_provider().await.expect("build_provider"); - assert!(cache_store.is_noop(), "cache_size == 0 must produce a no-op store"); - assert_eq!(chain_id, 4326, "chain_id must be resolved even when cache is disabled"); + assert!(cache_store.is_noop(), "--rpc.no-cache-file must produce a no-op store"); + assert_eq!(chain_id, 4326, "chain_id must be resolved with unlimited cache default"); cache_store.persist().expect("persist"); } @@ -128,7 +150,7 @@ async fn test_build_provider_no_cache_file_skips_persistence() { "mega-evme", "--rpc", &server.uri(), - "--rpc.cache-size", + "--rpc.cache-max-entries", "100", "--rpc.no-cache-file", ]); @@ -150,13 +172,16 @@ async fn test_build_provider_with_cache_names_file_from_fetched_chain_id() { let BuildProviderOutput { cache_store, .. } = args.build_provider().await.expect("build_provider"); - assert!(!cache_store.is_noop(), "cache_size > 0 + cache_dir must produce a real store"); + assert!( + !cache_store.is_noop(), + "cache_dir without --rpc.no-cache-file must produce a real store" + ); assert_eq!(cache_store.cache_path(), Some(dir.path().join("rpc-cache-4326.json").as_path())); } #[tokio::test(flavor = "multi_thread")] async fn test_build_provider_invalid_url() { - let args = RpcArgs::parse_from(["mega-evme", "--rpc", "not a url", "--rpc.cache-size", "0"]); + let args = RpcArgs::parse_from(["mega-evme", "--rpc", "not a url", "--rpc.no-cache-file"]); let err = args.build_provider().await.expect_err("build_provider should fail"); match err { EvmeError::RpcError(msg) => { @@ -183,7 +208,7 @@ async fn test_build_provider_fetches_chain_id_from_rpc() { "mega-evme", "--rpc", &server.uri(), - "--rpc.cache-size", + "--rpc.cache-max-entries", "256", "--rpc.cache-dir", dir.path().to_str().unwrap(), @@ -215,7 +240,7 @@ async fn test_build_provider_chain_id_rpc_failure_is_hard_error() { "mega-evme", "--rpc", &server.uri(), - "--rpc.cache-size", + "--rpc.cache-max-entries", "256", "--rpc.cache-dir", dir.path().to_str().unwrap(), @@ -320,7 +345,7 @@ async fn test_build_provider_clear_cache_deletes_file_before_load() { "mega-evme", "--rpc", &server.uri(), - "--rpc.cache-size", + "--rpc.cache-max-entries", "256", "--rpc.cache-dir", dir.path().to_str().unwrap(), @@ -372,7 +397,7 @@ async fn test_build_provider_clear_cache_hard_errors_on_unlink_failure() { "mega-evme", "--rpc", &server.uri(), - "--rpc.cache-size", + "--rpc.cache-max-entries", "256", "--rpc.cache-dir", dir.path().to_str().unwrap(), @@ -845,7 +870,7 @@ fn test_capture_file_mutex_with_other_cache_flags() { (&["--rpc.cache-dir", "/tmp/cache"], "--rpc.cache-dir"), (&["--rpc.clear-cache"], "--rpc.clear-cache"), (&["--rpc.no-cache-file"], "--rpc.no-cache-file"), - (&["--rpc.cache-size", "256"], "--rpc.cache-size"), + (&["--rpc.cache-max-entries", "256"], "--rpc.cache-max-entries"), ]; for (extra_flags, label) in cases { let mut argv = @@ -870,7 +895,7 @@ fn test_replay_file_mutex_with_rpc_and_cache_flags() { (&["--rpc.cache-dir", "/tmp/cache"], "--rpc.cache-dir"), (&["--rpc.clear-cache"], "--rpc.clear-cache"), (&["--rpc.no-cache-file"], "--rpc.no-cache-file"), - (&["--rpc.cache-size", "256"], "--rpc.cache-size"), + (&["--rpc.cache-max-entries", "256"], "--rpc.cache-max-entries"), ]; for (extra_flags, label) in cases { let mut argv = vec!["mega-evme", "--rpc.replay-file", "/tmp/replay.json"]; diff --git a/bin/mega-evme/tests/state.rs b/bin/mega-evme/tests/state.rs index bb811a91..3a88a614 100644 --- a/bin/mega-evme/tests/state.rs +++ b/bin/mega-evme/tests/state.rs @@ -200,7 +200,7 @@ async fn test_create_initial_state_fork_real_rpc_smoke() { "mega-evme", "--rpc", &rpc_url, - "--rpc.cache-size", + "--rpc.cache-max-entries", "256", "--rpc.cache-dir", dir.path().to_str().unwrap(), @@ -272,7 +272,7 @@ async fn test_create_initial_state_fork_real_rpc_storage_cache_hit() { "mega-evme", "--rpc", &rpc_url, - "--rpc.cache-size", + "--rpc.cache-max-entries", "256", "--rpc.cache-dir", dir.path().to_str().unwrap(), @@ -304,7 +304,7 @@ async fn test_create_initial_state_fork_real_rpc_storage_cache_hit() { "mega-evme", "--rpc", &rpc_url, - "--rpc.cache-size", + "--rpc.cache-max-entries", "256", "--rpc.cache-dir", dir.path().to_str().unwrap(), diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index dd56ea11..0bc407eb 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -55,7 +55,7 @@ The updated set of entries is persisted back to the same file on clean exit. The file also embeds an external-environment snapshot — currently the set of `--bucket-capacity` values in effect — so the captured fixture is self-contained. If `--bucket-capacity` is not passed on a subsequent run, the previous envelope's values are reused; passing `--bucket-capacity` overrides them. -`--rpc.capture-file` is mutually exclusive with `--rpc.replay-file`, `--rpc.cache-dir`, `--rpc.clear-cache`, `--rpc.no-cache-file`, and `--rpc.cache-size`. +`--rpc.capture-file` is mutually exclusive with `--rpc.replay-file`, `--rpc.cache-dir`, `--rpc.clear-cache`, `--rpc.no-cache-file`, and `--rpc.cache-max-entries`. ### `--rpc.replay-file ` @@ -67,7 +67,7 @@ Any request that is not present in the fixture aborts the run with a hard error Bucket-capacity data is read from the fixture envelope, so `--bucket-capacity` is neither required nor accepted with `--rpc.replay-file`. Passing `--bucket-capacity` together with `--rpc.replay-file` is rejected; to regenerate a fixture with new capacities, re-run in capture mode. -`--rpc.replay-file` is mutually exclusive with `--rpc`, `--rpc.capture-file`, `--rpc.cache-dir`, `--rpc.clear-cache`, `--rpc.no-cache-file`, and `--rpc.cache-size`. +`--rpc.replay-file` is mutually exclusive with `--rpc`, `--rpc.capture-file`, `--rpc.cache-dir`, `--rpc.clear-cache`, `--rpc.no-cache-file`, and `--rpc.cache-max-entries`. ### Examples diff --git a/docs/mega-evme/commands/run.md b/docs/mega-evme/commands/run.md index cfefe35b..e34437e5 100644 --- a/docs/mega-evme/commands/run.md +++ b/docs/mega-evme/commands/run.md @@ -54,7 +54,7 @@ Each group is documented on its own page. | Chain and spec | `--spec`, `--chain-id` | [Chain and Spec](../configuration/chain-and-spec.md) | | Block environment | `--block.number`, `--block.coinbase`, `--block.timestamp`, `--block.gaslimit`, `--block.basefee`, `--block.difficulty`, `--block.prevrandao`, `--block.blobexcessgas` | [Block Environment](../configuration/block-environment.md) | | SALT buckets | `--bucket-capacity` | [SALT Buckets](../configuration/salt-buckets.md) | -| RPC cache / retry | `--rpc.cache-size`, `--rpc.cache-dir`, `--rpc.no-cache-file`, `--rpc.clear-cache`, `--rpc.max-retries`, `--rpc.backoff-ms`, `--rpc.cu-per-sec` | [RPC Cache and Retry](../configuration/state-management.md#rpc-cache-and-retry) | +| RPC cache / retry | `--rpc.cache-max-entries`, `--rpc.cache-dir`, `--rpc.no-cache-file`, `--rpc.clear-cache`, `--rpc.max-retries`, `--rpc.backoff-ms`, `--rpc.cu-per-sec` | [RPC Cache and Retry](../configuration/state-management.md#rpc-cache-and-retry) | | Tracing | `--trace`, `--tracer`, `--trace.output`, and tracer-specific flags | [Tracing Overview](../tracing/overview.md) | | Output | `--json` | See [JSON output](#json-output) below | @@ -304,8 +304,8 @@ RPC Options: --rpc.replay-file (replay command only) Serve JSON-RPC from a captured fixture; not usable as a run/tx offline-fork path - --rpc.cache-size - Max items in the in-memory RPC LRU cache; 0 disables it [default: 10000] + --rpc.cache-max-entries + Max items in the in-memory RPC LRU cache (and therefore the cache file); 0 = unlimited (never evict) [default: 0] --rpc.cache-dir Directory for per-chain RPC cache files (default: platform cache dir) diff --git a/docs/mega-evme/commands/tx.md b/docs/mega-evme/commands/tx.md index 08bca30f..9504869b 100644 --- a/docs/mega-evme/commands/tx.md +++ b/docs/mega-evme/commands/tx.md @@ -204,7 +204,7 @@ RPC Options: [aliases: --rpc-url] [compat alias: --fork.rpc] --rpc.capture-file (replay command only) capture to fixture; not usable as a run/tx offline path --rpc.replay-file (replay command only) serve from fixture; not usable as a run/tx offline path - --rpc.cache-size In-memory RPC LRU cache size; 0 disables [default: 10000] + --rpc.cache-max-entries In-memory RPC LRU max entries; 0 = unlimited (never evict) [default: 0] --rpc.cache-dir Per-chain RPC cache directory (default: platform cache dir) --rpc.no-cache-file Disable on-disk cache persistence --rpc.clear-cache Delete the current chain's cache file before loading diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index c22e598f..777d8b46 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -219,12 +219,12 @@ The default cache directory is the platform cache directory: ### Cache Flags -| Flag | Type | Default | Description | -| ------------------------ | ----- | ------------------ | ------------------------------------------------------------------------------------------------------------------------ | -| `--rpc.cache-size ` | `u32` | `10000` | Maximum number of items in the in-memory RPC LRU cache. Set to `0` to disable the cache layer entirely. | -| `--rpc.cache-dir ` | path | Platform cache dir | Directory for per-chain cache files. Each chain's cache is stored as `{cache_dir}/rpc-cache-{chain_id}.json`. | -| `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies — use `--rpc.cache-size 0` to disable that too. | -| `--rpc.clear-cache` | flag | `false` | Delete the current chain's cache file before loading it. Recovery path for a polluted or corrupt cache. | +| Flag | Type | Default | Description | +| ----------------------------- | ----- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--rpc.cache-max-entries ` | `u32` | `0` | Maximum number of items in the in-memory RPC LRU cache (and therefore what is persisted to the cache file). `0` = unlimited (never evict). Default. | +| `--rpc.cache-dir ` | path | Platform cache dir | Directory for per-chain cache files. Each chain's cache is stored as `{cache_dir}/rpc-cache-{chain_id}.json`. | +| `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies. | +| `--rpc.clear-cache` | flag | `false` | Delete the current chain's cache file before loading it. Recovery path for a polluted or corrupt cache. | ### Retry Flags From 65d03aaca1fa45a2bea971a197ee1e4bb70e7322 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 16:32:12 +0800 Subject: [PATCH 03/64] fix(mega-evme): map cache-max-entries 0 to 1_048_576, not u32::MAX Alloy SharedCache preallocates its LRU hash table to full capacity, so mapping "unlimited" to u32::MAX caused multi-GB RSS on every default online run. Cap at 2^20 entries (cheap preallocation, covers far more than any observed corpus) and add a construction-reality unit test. --- bin/mega-evme/src/common/provider/mod.rs | 45 ++++++++++++++++--- docs/mega-evme/commands/run.md | 2 +- docs/mega-evme/commands/tx.md | 2 +- .../configuration/state-management.md | 12 ++--- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 5bd105c5..0ebac3ff 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -97,7 +97,9 @@ pub struct RpcArgs { pub replay_file: Option, /// Maximum number of items in the in-memory RPC LRU cache (and therefore what - /// gets persisted to the cache file). `0` = unlimited (never evict). Default is `0`. + /// gets persisted to the cache file). `0` = effectively unlimited (caps at + /// 1,048,576 entries; the cache index is preallocated proportional to the + /// cap). Default is `0`. #[arg(id = "cache_max_entries", long = "rpc.cache-max-entries", default_value_t = 0)] pub cache_max_entries: u32, @@ -171,7 +173,8 @@ impl RpcArgs { }; // 3. Build the cache layer and (optionally) the disk store. - // Cache layer is always installed; 0 max entries means unlimited (never evict). + // Cache layer is always installed; 0 max entries maps to + // EFFECTIVELY_UNLIMITED_CACHE_ENTRIES. let max_items = cache_max_entries_capacity(self.cache_max_entries); let cache_layer = CacheLayer::new(max_items); let cache = cache_layer.cache(); @@ -455,13 +458,25 @@ fn parse_non_empty_path(s: &str) -> std::result::Result { } } +/// Cap used when `--rpc.cache-max-entries 0` ("effectively unlimited") is requested. +/// +/// Alloy's `SharedCache` preallocates its LRU hash table to full capacity +/// (`lru::LruCache::with_hasher` → `HashMap::with_capacity_and_hasher`), so a true +/// `u32::MAX` mapping would preallocate ~2^33 hash buckets and balloon RSS by multi-GB +/// on every default-config online run. 2^20 entries preallocates ~2^21 pointer-sized +/// buckets (tens of MB) while covering ~2,600 blocks' worth of RPC entries per process +/// (a full mainnet block is ~200 entries; the largest real merged corpus to date was +/// 15,294). +const EFFECTIVELY_UNLIMITED_CACHE_ENTRIES: u32 = 1_048_576; + /// Map `--rpc.cache-max-entries` to the capacity passed to [`CacheLayer::new`]. /// -/// `0` means unlimited (never evict) and is represented as [`u32::MAX`]. +/// `0` means effectively unlimited and is approximated by +/// [`EFFECTIVELY_UNLIMITED_CACHE_ENTRIES`] (see that constant's rationale). /// Nonzero values pass through unchanged. fn cache_max_entries_capacity(cache_max_entries: u32) -> u32 { if cache_max_entries == 0 { - u32::MAX + EFFECTIVELY_UNLIMITED_CACHE_ENTRIES } else { cache_max_entries } @@ -556,13 +571,31 @@ mod tests { assert!(cu_per_sec_warning(0, 99).is_none()); } - /// `0` maps to unlimited capacity (`u32::MAX`); nonzero values pass through. + /// `0` maps to the effectively-unlimited cap; nonzero values pass through. #[test] fn test_cache_max_entries_capacity_mapping() { - assert_eq!(cache_max_entries_capacity(0), u32::MAX); + assert_eq!(cache_max_entries_capacity(0), EFFECTIVELY_UNLIMITED_CACHE_ENTRIES); + assert_eq!(cache_max_entries_capacity(0), 1_048_576); assert_eq!(cache_max_entries_capacity(1), 1); assert_eq!(cache_max_entries_capacity(256), 256); assert_eq!(cache_max_entries_capacity(10_000), 10_000); assert_eq!(cache_max_entries_capacity(u32::MAX), u32::MAX); } + + /// Construction-reality check: actually allocate the cache at the mapped "unlimited" + /// capacity, insert, and read back. Preallocating `u32::MAX` would hang/OOM here + /// (hashbrown ctrl-byte memset over ~2^33 buckets); this test must complete quickly. + #[test] + fn test_cache_layer_constructs_at_effectively_unlimited_capacity() { + let layer = CacheLayer::new(cache_max_entries_capacity(0)); + assert_eq!(layer.max_items(), EFFECTIVELY_UNLIMITED_CACHE_ENTRIES); + + let cache = layer.cache(); + assert_eq!(cache.max_items(), EFFECTIVELY_UNLIMITED_CACHE_ENTRIES); + + let key = alloy_primitives::B256::repeat_byte(0xab); + let value = r#"{"result":"0x1"}"#.to_string(); + cache.put(key, value.clone()).expect("put into SharedCache"); + assert_eq!(cache.get(&key).as_deref(), Some(value.as_str())); + } } diff --git a/docs/mega-evme/commands/run.md b/docs/mega-evme/commands/run.md index e34437e5..91172155 100644 --- a/docs/mega-evme/commands/run.md +++ b/docs/mega-evme/commands/run.md @@ -305,7 +305,7 @@ RPC Options: (replay command only) Serve JSON-RPC from a captured fixture; not usable as a run/tx offline-fork path --rpc.cache-max-entries - Max items in the in-memory RPC LRU cache (and therefore the cache file); 0 = unlimited (never evict) [default: 0] + Max items in the in-memory RPC LRU cache (and therefore the cache file); 0 = effectively unlimited (caps at 1,048,576 entries; the cache index is preallocated proportional to the cap) [default: 0] --rpc.cache-dir Directory for per-chain RPC cache files (default: platform cache dir) diff --git a/docs/mega-evme/commands/tx.md b/docs/mega-evme/commands/tx.md index 9504869b..08685d9f 100644 --- a/docs/mega-evme/commands/tx.md +++ b/docs/mega-evme/commands/tx.md @@ -204,7 +204,7 @@ RPC Options: [aliases: --rpc-url] [compat alias: --fork.rpc] --rpc.capture-file (replay command only) capture to fixture; not usable as a run/tx offline path --rpc.replay-file (replay command only) serve from fixture; not usable as a run/tx offline path - --rpc.cache-max-entries In-memory RPC LRU max entries; 0 = unlimited (never evict) [default: 0] + --rpc.cache-max-entries In-memory RPC LRU max entries; 0 = effectively unlimited (caps at 1,048,576 entries; the cache index is preallocated proportional to the cap) [default: 0] --rpc.cache-dir Per-chain RPC cache directory (default: platform cache dir) --rpc.no-cache-file Disable on-disk cache persistence --rpc.clear-cache Delete the current chain's cache file before loading diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index 777d8b46..d1db51e8 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -219,12 +219,12 @@ The default cache directory is the platform cache directory: ### Cache Flags -| Flag | Type | Default | Description | -| ----------------------------- | ----- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--rpc.cache-max-entries ` | `u32` | `0` | Maximum number of items in the in-memory RPC LRU cache (and therefore what is persisted to the cache file). `0` = unlimited (never evict). Default. | -| `--rpc.cache-dir ` | path | Platform cache dir | Directory for per-chain cache files. Each chain's cache is stored as `{cache_dir}/rpc-cache-{chain_id}.json`. | -| `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies. | -| `--rpc.clear-cache` | flag | `false` | Delete the current chain's cache file before loading it. Recovery path for a polluted or corrupt cache. | +| Flag | Type | Default | Description | +| ----------------------------- | ----- | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--rpc.cache-max-entries ` | `u32` | `0` | Maximum number of items in the in-memory RPC LRU cache (and therefore what is persisted to the cache file). `0` = effectively unlimited (caps at 1,048,576 entries; the cache index is preallocated proportional to the cap). Default. | +| `--rpc.cache-dir ` | path | Platform cache dir | Directory for per-chain cache files. Each chain's cache is stored as `{cache_dir}/rpc-cache-{chain_id}.json`. | +| `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies. | +| `--rpc.clear-cache` | flag | `false` | Delete the current chain's cache file before loading it. Recovery path for a polluted or corrupt cache. | ### Retry Flags From 836024ec49e6f4277dd49ed0070fc51bce31955f Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 16:26:35 +0800 Subject: [PATCH 04/64] feat(evme): add batch replay via --tx-file and --block Replay many transactions in a single process: build one provider and one RPC cache, group targets by their containing block, and execute each block once while recording every target's result. Batch mode emits NDJSON with --json (one line per target, result or error entry), exits non-zero when a target hit an infrastructure failure, and rejects the flags that only have single-transaction semantics (fixture dump, overrides, forced spec, trace, state dump). The single-transaction path is unchanged; its output stays byte-identical. Command dispatch now maps errors instead of propagating with ?, so the error handler runs and diagnostics stay off stdout. --- bin/mega-evme/src/cmd.rs | 19 +- bin/mega-evme/src/replay/batch.rs | 680 ++++++++++++++++++++++++++++ bin/mega-evme/src/replay/cmd.rs | 369 +++++++++++++-- bin/mega-evme/src/replay/mod.rs | 1 + bin/mega-evme/tests/replay_batch.rs | 210 +++++++++ docs/mega-evme/commands/replay.md | 132 +++++- 6 files changed, 1348 insertions(+), 63 deletions(-) create mode 100644 bin/mega-evme/src/replay/batch.rs create mode 100644 bin/mega-evme/tests/replay_batch.rs diff --git a/bin/mega-evme/src/cmd.rs b/bin/mega-evme/src/cmd.rs index 736f257a..68a1030d 100644 --- a/bin/mega-evme/src/cmd.rs +++ b/bin/mega-evme/src/cmd.rs @@ -45,19 +45,14 @@ impl MainCmd { // Initialize logging first self.log.init(); + // Map instead of `?`: `?` inside an arm returns from `run` directly and + // skips the handler below, leaving the error to be printed on stdout by + // the binary's fallback — which corrupts machine-readable output such as + // the batch replay NDJSON stream. match self.command { - Commands::Run(cmd) => { - cmd.run().await?; - Ok(()) - } - Commands::Tx(cmd) => { - cmd.run().await?; - Ok(()) - } - Commands::Replay(cmd) => { - cmd.run().await?; - Ok(()) - } + Commands::Run(cmd) => cmd.run().await.map_err(Error::from), + Commands::Tx(cmd) => cmd.run().await.map_err(Error::from), + Commands::Replay(cmd) => cmd.run().await.map_err(Error::from), } .inspect_err(|e| { error!(err = ?e, "Error executing command"); diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs new file mode 100644 index 00000000..19e98779 --- /dev/null +++ b/bin/mega-evme/src/replay/batch.rs @@ -0,0 +1,680 @@ +//! Batch replay driver: replay many transactions inside a single process. +//! +//! The single-transaction path ([`super::cmd`]) builds a provider, forks state at +//! the parent block, and executes one block per process. Verifying a large corpus +//! that way pays the provider/cache setup once per transaction, which dominates +//! the actual EVM work. This module reuses one provider and one RPC cache for the +//! whole run, groups the requested transactions by their containing block, and +//! executes each block exactly once while recording the result of every target it +//! passes through. +//! +//! Every RPC call issued here has the same shape as the single-transaction path +//! (`eth_getTransactionByHash`, `eth_getBlockByNumber` with hash-only bodies, and +//! the state reads behind [`EvmeState::new_forked`]), so an offline envelope +//! captured by single-transaction replays serves batch runs without a miss. + +use std::{ + collections::{BTreeMap, HashSet}, + str::FromStr, + time::{Duration, Instant}, +}; + +use alloy_consensus::{BlockHeader, Transaction as _}; +use alloy_primitives::{Address, B256}; +use alloy_provider::Provider; +use alloy_rpc_types_eth::Block; +use mega_evm::{ + alloy_evm::{block::BlockExecutor, Evm, EvmEnv}, + alloy_op_evm::block::OpAlloyReceiptBuilder, + revm::{ + context::{result::ExecutionResult, ContextTr}, + database::{states::bundle_state::BundleRetention, StateBuilder}, + DatabaseRef, + }, + BlockLimits, MegaBlockExecutionCtx, MegaBlockExecutorFactory, MegaEvmFactory, MegaHaltReason, + MegaHardforks, +}; +use op_alloy_rpc_types::Transaction; +use serde::Serialize; +use tracing::{debug, info, warn}; + +use crate::{ + common::{ + op_receipt_to_tx_receipt, print_execution_summary, print_receipt, EvmeExternalEnvs, + ExecutionSummary, OpTxReceipt, + }, + replay::get_hardfork_config, + ChainArgs, EvmeState, +}; + +use super::{cmd::retrieve_block_env, ReplayError, Result}; + +/// What a batch run was asked to replay. +#[derive(Debug)] +pub(super) enum BatchMode { + /// Transaction hashes read from `--tx-file`, in file order. + TxList(Vec), + /// Every transaction of the block given by `--block`. + Block(u64), +} + +/// Why a target transaction produced no execution result. +/// +/// Execution outcomes (success, revert, halt) are normal results and never map +/// to one of these kinds. +#[derive(Debug, Clone, Copy)] +enum BatchErrorKind { + /// The transaction hash is unknown to the endpoint. + NotFound, + /// The transaction exists but is not mined yet (no block number). + Pending, + /// An RPC call failed or returned nothing. + Rpc, + /// The block executor rejected the transaction or the block setup failed. + Execution, +} + +impl BatchErrorKind { + /// Wire name used in the NDJSON error line and the human-readable output. + const fn as_str(self) -> &'static str { + match self { + Self::NotFound => "not_found", + Self::Pending => "pending", + Self::Rpc => "rpc", + Self::Execution => "execution", + } + } +} + +/// Outcome of a single target transaction. +enum BatchEntry { + /// The transaction executed and produced a result (success, revert, or halt). + Executed(Box), + /// The transaction could not be executed. + Failed(FailedTx), +} + +impl BatchEntry { + /// Hash of the target this entry reports on. + const fn tx_hash(&self) -> B256 { + match self { + Self::Executed(tx) => tx.tx_hash, + Self::Failed(tx) => tx.tx_hash, + } + } +} + +/// A target transaction that ran to completion. +struct ExecutedTx { + tx_hash: B256, + block_number: u64, + tx_index: u64, + exec_result: ExecutionResult, + contract_address: Option
, + exec_time: Duration, + receipt: OpTxReceipt, +} + +/// A target transaction that hit an infrastructure failure. +struct FailedTx { + tx_hash: B256, + kind: BatchErrorKind, + message: String, +} + +/// One block's worth of work. +struct BlockJob { + /// Number of the block holding the targets. + number: u64, + /// Block body, present when planning already fetched it (`--block`). + block: Option>, + /// Hashes of the transactions whose results are reported. + targets: Vec, +} + +/// A target that executed, awaiting the receipt harvested by `finish()`. +struct PendingTarget { + tx_hash: B256, + tx_index: u64, + /// Position of this transaction among the block's committed transactions. + commit_index: usize, + exec_result: ExecutionResult, + exec_time: Duration, + gas_used: u64, + pre_execution_nonce: u64, + from: Address, + to: Option
, + effective_gas_price: u128, +} + +/// NDJSON line for a target that produced an execution result. +#[derive(Serialize)] +struct BatchResultLine<'a> { + tx_hash: B256, + block_number: u64, + tx_index: u64, + #[serde(flatten)] + summary: &'a ExecutionSummary, +} + +/// NDJSON line for a target that produced an infrastructure error. +#[derive(Serialize)] +struct BatchErrorLine<'a> { + tx_hash: B256, + error: BatchErrorBody<'a>, +} + +/// Error payload of a [`BatchErrorLine`]. +#[derive(Serialize)] +struct BatchErrorBody<'a> { + kind: &'static str, + message: &'a str, +} + +/// Replay every requested transaction, reporting one entry per target. +/// +/// Returns an error when at least one target produced an infrastructure error +/// entry, so the process exits non-zero; execution outcomes never fail the run. +pub(super) async fn run

( + provider: &P, + chain_id: u64, + mode: &BatchMode, + external_envs: EvmeExternalEnvs, + json: bool, +) -> Result<()> +where + P: Provider + Clone + std::fmt::Debug, +{ + let start = Instant::now(); + let mut replayed = 0usize; + let mut failed = 0usize; + + let jobs = match mode { + BatchMode::Block(number) => { + let block = fetch_block(provider, *number).await?; + let targets: Vec = block.transactions.hashes().collect(); + info!(block = number, tx_count = targets.len(), "Batch replay of a whole block"); + vec![BlockJob { number: *number, block: Some(block), targets }] + } + BatchMode::TxList(hashes) => { + let (jobs, failures) = resolve_targets(provider, hashes).await; + info!( + requested = hashes.len(), + blocks = jobs.len(), + unresolved = failures.len(), + "Batch replay of a transaction list", + ); + for failure in failures { + failed += 1; + emit(&BatchEntry::Failed(failure), json); + } + jobs + } + }; + + for job in jobs { + for entry in replay_block(provider, chain_id, job, external_envs.clone()).await { + match entry { + BatchEntry::Executed(_) => replayed += 1, + BatchEntry::Failed(_) => failed += 1, + } + emit(&entry, json); + } + } + + info!(replayed, failed, elapsed = ?start.elapsed(), "Batch replay finished"); + + if failed > 0 { + return Err(ReplayError::Other(format!( + "{failed} of {} target transaction(s) failed to replay", + replayed + failed + ))); + } + Ok(()) +} + +/// Resolve each requested hash to its containing block. +/// +/// Returns the per-block jobs in ascending block order, plus the failures for +/// hashes that could not be resolved (in the order they were requested). +async fn resolve_targets

(provider: &P, hashes: &[B256]) -> (Vec, Vec) +where + P: Provider, +{ + let mut grouped: BTreeMap> = BTreeMap::new(); + let mut failures = Vec::new(); + + for hash in hashes { + match provider.get_transaction_by_hash(*hash).await { + Err(e) => failures.push(FailedTx { + tx_hash: *hash, + kind: BatchErrorKind::Rpc, + message: format!("Failed to fetch transaction: {e}"), + }), + Ok(None) => failures.push(FailedTx { + tx_hash: *hash, + kind: BatchErrorKind::NotFound, + message: "Transaction not found".to_string(), + }), + Ok(Some(tx)) => match tx.block_number { + Some(number) => grouped.entry(number).or_default().push(*hash), + None => failures.push(FailedTx { + tx_hash: *hash, + kind: BatchErrorKind::Pending, + message: "Transaction is pending (no block number)".to_string(), + }), + }, + } + } + + let jobs = grouped + .into_iter() + .map(|(number, targets)| BlockJob { number, block: None, targets }) + .collect(); + (jobs, failures) +} + +/// Replay one block, reporting an entry for every target it was asked about. +/// +/// The block is executed exactly once: every transaction runs in order, and each +/// target's result is recorded before the transaction is committed. Receipts are +/// harvested from the finished block, which is why the block's entries are only +/// produced once the block is done. +async fn replay_block

( + provider: &P, + chain_id: u64, + job: BlockJob, + external_envs: EvmeExternalEnvs, +) -> Vec +where + P: Provider + Clone + std::fmt::Debug, +{ + let BlockJob { number, block, targets } = job; + + if number == 0 { + return fail_all(&targets, BatchErrorKind::Rpc, "Block 0 has no parent block to fork from"); + } + + let block = match block { + Some(block) => block, + None => match fetch_block(provider, number).await { + Ok(block) => block, + Err(e) => return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()), + }, + }; + let parent_block = match fetch_block(provider, number - 1).await { + Ok(block) => block, + Err(e) => return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()), + }; + + let hardforks = get_hardfork_config(chain_id); + let timestamp = block.header.timestamp(); + let spec = hardforks.spec_id(timestamp); + let chain_args = ChainArgs { chain_id, spec: spec.to_string() }; + debug!(block = number, chain_id, spec = %spec, "Block configuration"); + + let cfg_env = match chain_args.create_cfg_env() { + Ok(cfg) => cfg, + Err(e) => return fail_all(&targets, BatchErrorKind::Execution, &e.to_string()), + }; + let block_env = match retrieve_block_env(&block) { + Ok(env) => env, + Err(e) => return fail_all(&targets, BatchErrorKind::Execution, &e.to_string()), + }; + let evm_env = EvmEnv::new(cfg_env, block_env); + + let Some(hardfork) = hardforks.hardfork(timestamp) else { + let message = format!("No `MegaHardfork` active at block timestamp: {timestamp}"); + return fail_all(&targets, BatchErrorKind::Execution, &message); + }; + let block_limits = + BlockLimits::from_hardfork_and_block_gas_limit(hardfork, block.header.gas_limit()); + let block_ctx = MegaBlockExecutionCtx::new( + parent_block.hash(), + block.header.parent_beacon_block_root(), + block.header.extra_data().clone(), + block_limits, + ); + + info!(block = number, fork_block = parent_block.header.number(), "Forking state for block"); + let mut database = match EvmeState::new_forked( + provider.clone(), + Some(parent_block.header.number()), + Default::default(), + Default::default(), + ) + .await + { + Ok(database) => database, + Err(e) => return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()), + }; + + let evm_factory = MegaEvmFactory::new().with_external_env_factory(external_envs); + let block_executor_factory = + MegaBlockExecutorFactory::new(&hardforks, evm_factory, OpAlloyReceiptBuilder::default()); + let mut state = StateBuilder::new().with_database(&mut database).with_bundle_update().build(); + let mut block_executor = block_executor_factory.create_executor(&mut state, block_ctx, evm_env); + + if let Err(e) = block_executor.apply_pre_execution_changes() { + let message = format!("Block execution error: {e}"); + return fail_all(&targets, BatchErrorKind::Execution, &message); + } + + let target_set: HashSet = targets.iter().copied().collect(); + let tx_hashes: Vec = block.transactions.hashes().collect(); + let mut pending: Vec = Vec::new(); + let mut committed = 0usize; + + // Run the block's transactions in order. Any failure aborts the block: the + // executor state no longer matches the chain, so the remaining targets + // cannot be replayed faithfully. + let loop_result: Result<()> = async { + for (tx_index, tx_hash) in tx_hashes.iter().enumerate() { + let tx = provider + .get_transaction_by_hash(*tx_hash) + .await + .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? + .ok_or(ReplayError::TransactionNotFound(*tx_hash))?; + + let is_target = target_set.contains(tx_hash); + let start = Instant::now(); + let pre_execution_nonce = if is_target { + block_executor + .evm() + .db_ref() + .basic_ref(tx.inner.inner.signer())? + .map(|acc| acc.nonce) + .unwrap_or(0) + } else { + 0 + }; + + let outcome = block_executor + .run_transaction(tx.as_recovered()) + .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + // Record the target's result before committing, mirroring the + // single-transaction path. + let exec_result = is_target.then(|| outcome.inner.result.clone()); + let gas_used = block_executor + .commit_transaction_outcome(outcome) + .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + let commit_index = committed; + committed += 1; + + if let Some(exec_result) = exec_result { + pending.push(PendingTarget { + tx_hash: *tx_hash, + tx_index: tx_index as u64, + commit_index, + exec_result, + exec_time: start.elapsed(), + gas_used, + pre_execution_nonce, + from: tx.inner.inner.signer(), + to: tx.inner.inner.to(), + effective_gas_price: tx.inner.effective_gas_price.unwrap_or(0), + }); + } + } + Ok(()) + } + .await; + + // Finish the block even when it aborted midway: targets that already ran + // still have a receipt worth reporting. + let mut entries = Vec::with_capacity(targets.len()); + match block_executor.finish() { + Ok((evm, block_result)) => { + let (db, _) = evm.finish(); + db.merge_transitions(BundleRetention::Reverts); + let receipts = block_result.receipts; + // Receipts are pushed one per committed transaction; index from the + // end so any receipt produced before the first transaction (now or + // later) cannot shift the mapping. + let offset = receipts.len().saturating_sub(committed); + let block_hash = block.hash(); + for target in pending { + let Some(envelope) = receipts.get(offset + target.commit_index) else { + entries.push(failure( + target.tx_hash, + BatchErrorKind::Execution, + format!("No receipt produced for transaction index {}", target.tx_index), + )); + continue; + }; + let contract_address = (target.to.is_none() && envelope.is_success()) + .then(|| target.from.create(target.pre_execution_nonce)); + let receipt = op_receipt_to_tx_receipt( + envelope, + number, + timestamp, + target.from, + target.to, + contract_address, + target.effective_gas_price, + target.gas_used, + Some(target.tx_hash), + Some(block_hash), + target.tx_index, + ); + entries.push(BatchEntry::Executed(Box::new(ExecutedTx { + tx_hash: target.tx_hash, + block_number: number, + tx_index: target.tx_index, + exec_result: target.exec_result, + contract_address, + exec_time: target.exec_time, + receipt, + }))); + } + } + Err(e) => { + let message = format!("Block execution error: {e}"); + for target in pending { + entries.push(failure(target.tx_hash, BatchErrorKind::Execution, message.clone())); + } + } + } + + // Any target that produced no entry either sat behind the abort or is not + // part of this block at all. + let (kind, message) = match loop_result { + Ok(()) => (BatchErrorKind::NotFound, format!("Transaction is not part of block {number}")), + Err(e) => { + warn!(block = number, error = %e, "Aborted block replay; skipping its remaining targets"); + (classify(&e), e.to_string()) + } + }; + let reported: HashSet = entries.iter().map(BatchEntry::tx_hash).collect(); + for tx_hash in &targets { + if !reported.contains(tx_hash) { + entries.push(failure(*tx_hash, kind, message.clone())); + } + } + + entries +} + +/// Fetch a block by number, using the same call shape as the single-transaction path. +async fn fetch_block

(provider: &P, number: u64) -> Result> +where + P: Provider, +{ + provider + .get_block_by_number(number.into()) + .await + .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? + .ok_or(ReplayError::BlockNotFound(number)) +} + +/// Map an error raised while replaying a block onto a reported error kind. +const fn classify(err: &ReplayError) -> BatchErrorKind { + match err { + ReplayError::TransactionNotFound(_) => BatchErrorKind::NotFound, + ReplayError::RpcError(_) | ReplayError::RpcTransportError(_) => BatchErrorKind::Rpc, + _ => BatchErrorKind::Execution, + } +} + +/// Build a failure entry. +fn failure(tx_hash: B256, kind: BatchErrorKind, message: String) -> BatchEntry { + BatchEntry::Failed(FailedTx { tx_hash, kind, message }) +} + +/// Report the same failure for every target of a block that never started. +fn fail_all(targets: &[B256], kind: BatchErrorKind, message: &str) -> Vec { + targets.iter().map(|hash| failure(*hash, kind, message.to_string())).collect() +} + +/// Write one entry to stdout: a compact NDJSON line, or the human-readable +/// summary used by the single-transaction path. +fn emit(entry: &BatchEntry, json: bool) { + if json { + let line = match entry { + BatchEntry::Executed(tx) => { + let mut summary = + ExecutionSummary::from_result(&tx.exec_result, tx.contract_address); + summary.receipt = + Some(serde_json::to_value(&tx.receipt).expect("failed to serialize receipt")); + serde_json::to_string(&BatchResultLine { + tx_hash: tx.tx_hash, + block_number: tx.block_number, + tx_index: tx.tx_index, + summary: &summary, + }) + } + BatchEntry::Failed(tx) => serde_json::to_string(&BatchErrorLine { + tx_hash: tx.tx_hash, + error: BatchErrorBody { kind: tx.kind.as_str(), message: &tx.message }, + }), + }; + println!("{}", line.expect("failed to serialize output")); + return; + } + + match entry { + BatchEntry::Executed(tx) => { + println!(); + println!( + "=== Transaction {} (block {}, index {}) ===", + tx.tx_hash, tx.block_number, tx.tx_index + ); + print_execution_summary(&tx.exec_result, tx.contract_address, tx.exec_time); + print_receipt(&tx.receipt); + } + BatchEntry::Failed(tx) => { + println!(); + println!("=== Transaction {} ===", tx.tx_hash); + println!("Error ({}): {}", tx.kind.as_str(), tx.message); + } + } +} + +/// Parse the newline-separated transaction hash list behind `--tx-file`. +/// +/// Blank lines and `#`-prefixed comment lines are ignored. Duplicates are +/// dropped, keeping the first occurrence. +pub(super) fn parse_tx_hash_list(contents: &str) -> Result> { + let mut hashes = Vec::new(); + let mut seen = HashSet::new(); + + for (index, raw_line) in contents.lines().enumerate() { + let line = raw_line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let line_number = index + 1; + let hash = B256::from_str(line).map_err(|e| { + ReplayError::InvalidInput(format!( + "invalid transaction hash on line {line_number}: '{line}' ({e})" + )) + })?; + if seen.insert(hash) { + hashes.push(hash); + } else { + warn!( + tx_hash = %hash, + line = line_number, + "Duplicate transaction hash in --tx-file; replaying it once", + ); + } + } + + Ok(hashes) +} + +/// `clap` value parser for `--block`, accepting decimal or `0x`-prefixed hex. +pub(super) fn parse_block_number(value: &str) -> std::result::Result { + let trimmed = value.trim(); + match trimmed.strip_prefix("0x").or_else(|| trimmed.strip_prefix("0X")) { + Some(hex) => u64::from_str_radix(hex, 16) + .map_err(|e| format!("invalid hex block number '{value}': {e}")), + None => trimmed.parse::().map_err(|e| format!("invalid block number '{value}': {e}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const HASH_A: &str = "0xde3d56dc739484166b8af1bea757bf7e3e9a4b9a0fb62d722703345570dfc1d6"; + const HASH_B: &str = "0x323ddc8e67dfc134284d78c65f3c1dc7ff45ba1db02eeaf62e211ae3253478ef"; + + #[test] + fn test_parse_tx_hash_list_skips_blanks_and_comments() { + let contents = + format!("# leading comment\n\n{HASH_A}\n \n # indented comment\n\t{HASH_B} \n\n"); + + let hashes = parse_tx_hash_list(&contents).expect("should parse"); + + assert_eq!(hashes, vec![B256::from_str(HASH_A).unwrap(), B256::from_str(HASH_B).unwrap()]); + } + + #[test] + fn test_parse_tx_hash_list_deduplicates_preserving_order() { + let contents = format!("{HASH_B}\n{HASH_A}\n{HASH_B}\n"); + + let hashes = parse_tx_hash_list(&contents).expect("should parse"); + + assert_eq!(hashes, vec![B256::from_str(HASH_B).unwrap(), B256::from_str(HASH_A).unwrap()]); + } + + #[test] + fn test_parse_tx_hash_list_reports_offending_line_number() { + let contents = format!("# comment\n\n{HASH_A}\nnot-a-hash\n"); + + let err = parse_tx_hash_list(&contents).expect_err("should reject the invalid hash"); + + let message = err.to_string(); + assert!(message.contains("line 4"), "error should name the line, got: {message}"); + assert!(message.contains("not-a-hash"), "error should quote the line, got: {message}"); + } + + #[test] + fn test_parse_tx_hash_list_accepts_empty_input() { + let hashes = parse_tx_hash_list("# only a comment\n\n").expect("should parse"); + assert!(hashes.is_empty()); + } + + #[test] + fn test_parse_block_number_decimal() { + assert_eq!(parse_block_number("22945844"), Ok(22_945_844)); + assert_eq!(parse_block_number(" 22945844 "), Ok(22_945_844)); + assert_eq!(parse_block_number("0"), Ok(0)); + } + + #[test] + fn test_parse_block_number_hex() { + assert_eq!(parse_block_number("0x15e2034"), Ok(22_945_844)); + assert_eq!(parse_block_number("0X15E2034"), Ok(22_945_844)); + } + + #[test] + fn test_parse_block_number_rejects_garbage() { + assert!(parse_block_number("").is_err()); + assert!(parse_block_number("0x").is_err()); + assert!(parse_block_number("0xzz").is_err()); + assert!(parse_block_number("-1").is_err()); + assert!(parse_block_number("12.5").is_err()); + } +} diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 571d7714..04d3f676 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -1,10 +1,10 @@ -use std::{str::FromStr, time::Instant}; +use std::{path::PathBuf, str::FromStr, time::Instant}; use alloy_consensus::{BlockHeader, Transaction as _}; use alloy_primitives::{B256, U256}; use alloy_provider::Provider; use alloy_rpc_types_eth::Block; -use clap::Parser; +use clap::{ArgGroup, Parser}; use mega_evm::{ alloy_evm::{block::BlockExecutor, Evm, EvmEnv}, alloy_op_evm::block::OpAlloyReceiptBuilder, @@ -26,20 +26,40 @@ use crate::{ common::{ op_receipt_to_tx_receipt, parse_bucket_capacity, print_execution_summary, print_execution_trace, print_receipt, BuildProviderOutput, EvmeExternalEnvs, EvmeOutcome, - ExecutionSummary, ExternalEnvSnapshot, OpTxReceipt, RpcCacheStore, TxOverrideArgs, + ExecutionSummary, ExternalEnvSnapshot, OpTxReceipt, RpcCacheStore, TracerType, + TxOverrideArgs, }, replay::get_hardfork_config, run, ChainArgs, EvmeState, }; -use super::{ReplayError, Result}; +use super::{batch, ReplayError, Result}; /// Replay a transaction from RPC #[derive(Parser, Debug)] +#[command(group( + ArgGroup::new("replay_target").required(true).args(["tx_hash", "tx_file", "block"]) +))] pub struct Cmd { /// Transaction hash to replay #[arg(value_name = "TX_HASH")] - pub tx_hash: B256, + pub tx_hash: Option, + + /// Replay every transaction hash listed in the given file, one per line. + /// + /// Blank lines and `#`-prefixed comment lines are ignored, and duplicates are + /// replayed once. All hashes are replayed in a single process: transactions + /// are grouped by their containing block and each block is executed once. + /// Batch mode does not support `--dump-fixture`, transaction overrides, + /// `--override.spec`, tracing, or state dumps. + #[arg(long = "tx-file", value_name = "PATH")] + pub tx_file: Option, + + /// Replay every transaction of the given block (decimal or `0x`-prefixed hex). + /// + /// Same batch semantics and restrictions as `--tx-file`. + #[arg(long = "block", value_name = "N", value_parser = batch::parse_block_number)] + pub block: Option, /// RPC configuration #[command(flatten)] @@ -104,6 +124,7 @@ pub(super) struct ReplayOutcome { /// Intermediate context fetched from RPC before execution. struct ReplayContext { + tx_hash: B256, target_tx: Transaction, parent_block: Block, block: Block, @@ -111,46 +132,31 @@ struct ReplayContext { preceding_tx_hashes: Vec, } +/// What the command was asked to replay, resolved from the target argument group. +enum ReplayMode { + /// The single transaction named by the positional `TX_HASH`. + Single(B256), + /// Many transactions replayed in one process (`--tx-file` / `--block`). + Batch(batch::BatchMode), +} + impl Cmd { - /// Replay a historical transaction. + /// Replay one or more historical transactions. pub async fn run(&self) -> Result<()> { - // Pure input validation — reject before any network/state work. A dumped - // fixture must represent the on-chain transaction, so it can neither apply - // transaction overrides nor force a spec: both would make the recorded - // execution a what-if, not the on-chain one. - if self.dump_fixture.is_some() { - if self.tx_override_args.has_overrides() { - return Err(ReplayError::Other( - "--dump-fixture cannot be combined with transaction overrides (the \ - isolated execution would not represent the on-chain transaction)" - .to_string(), - )); - } - if self.spec_override.is_some() { - return Err(ReplayError::Other( - "--dump-fixture cannot be combined with --override.spec (the fixture \ - must record the spec auto-detected for the on-chain block, not a \ - manually forced one)" - .to_string(), - )); - } - } + self.validate()?; + let mode = self.resolve_mode()?; let mut pctx = self.resolve_provider().await?; - let rctx = self.fetch_replay_context(&pctx.provider, pctx.chain_id).await?; - let (external_envs, env_snapshot) = self.resolve_external_envs(&pctx)?; // Execute, report, and (for --dump-fixture) finalize/write — but defer // error propagation until the cache store has persisted: in capture mode // an execution or dump-gate failure is exactly the case you'd want to // debug offline, so the captured RPC responses must not be discarded. - let run_result = self.execute_and_report(&pctx.provider, &rctx, external_envs).await; + let run_result = match &mode { + ReplayMode::Single(tx_hash) => self.run_single(&mut pctx, *tx_hash).await, + ReplayMode::Batch(batch_mode) => self.run_batch(&mut pctx, batch_mode).await, + }; - // Hand the effective external-env snapshot to the store before the final - // persist; no-op unless this is a fixture-capture store. - if let Some(snapshot) = env_snapshot { - pctx.cache_store.set_external_env(snapshot); - } let persist_result = pctx.cache_store.persist(); match run_result { Ok(()) => Ok(persist_result?), @@ -168,6 +174,141 @@ impl Cmd { } } + /// Pure input validation — reject before any network/state work. + fn validate(&self) -> Result<()> { + if self.is_batch() { + self.validate_batch_args()?; + } + + // A dumped fixture must represent the on-chain transaction, so it can + // neither apply transaction overrides nor force a spec: both would make + // the recorded execution a what-if, not the on-chain one. + if self.dump_fixture.is_some() { + if self.tx_override_args.has_overrides() { + return Err(ReplayError::Other( + "--dump-fixture cannot be combined with transaction overrides (the \ + isolated execution would not represent the on-chain transaction)" + .to_string(), + )); + } + if self.spec_override.is_some() { + return Err(ReplayError::Other( + "--dump-fixture cannot be combined with --override.spec (the fixture \ + must record the spec auto-detected for the on-chain block, not a \ + manually forced one)" + .to_string(), + )); + } + } + + Ok(()) + } + + /// Whether this invocation selects a batch of transactions. + fn is_batch(&self) -> bool { + self.tx_file.is_some() || self.block.is_some() + } + + /// Reject the single-transaction-only flags in batch mode. + /// + /// Batch mode reports one summary per transaction; per-transaction artifacts + /// (fixture, trace, state dump) and what-if knobs (overrides, forced spec) + /// have no meaningful batch semantics, so they are rejected up front rather + /// than silently ignored. + fn validate_batch_args(&self) -> Result<()> { + const MODE: &str = "batch replay (--tx-file / --block)"; + + if self.dump_fixture.is_some() { + return Err(ReplayError::Other(format!( + "--dump-fixture is not supported by {MODE}; dump a fixture by replaying a \ + single transaction" + ))); + } + if self.tx_override_args.has_overrides() { + return Err(ReplayError::Other(format!( + "transaction overrides (--override.gas-limit / --override.value / \ + --override.input / --override.input-file) are not supported by {MODE}" + ))); + } + if self.spec_override.is_some() { + return Err(ReplayError::Other(format!( + "--override.spec is not supported by {MODE}; each block's spec is \ + auto-detected from its timestamp" + ))); + } + if has_trace_args(&self.trace_args) { + return Err(ReplayError::Other(format!( + "trace options (--trace / --trace.output / --tracer / --trace.*) are not \ + supported by {MODE}" + ))); + } + if has_dump_args(&self.dump_args) { + return Err(ReplayError::Other(format!( + "state dump options (--dump / --dump.output) are not supported by {MODE}" + ))); + } + + Ok(()) + } + + /// Resolve the target argument group into an execution mode. + /// + /// Reads `--tx-file` from disk here so a malformed list fails before any + /// provider is built. + fn resolve_mode(&self) -> Result { + if let Some(path) = &self.tx_file { + let contents = std::fs::read_to_string(path).map_err(|e| { + ReplayError::InvalidInput(format!( + "Failed to read --tx-file '{}': {e}", + path.display() + )) + })?; + let hashes = batch::parse_tx_hash_list(&contents)?; + if hashes.is_empty() { + return Err(ReplayError::InvalidInput(format!( + "--tx-file '{}' contains no transaction hashes", + path.display() + ))); + } + return Ok(ReplayMode::Batch(batch::BatchMode::TxList(hashes))); + } + if let Some(number) = self.block { + return Ok(ReplayMode::Batch(batch::BatchMode::Block(number))); + } + match self.tx_hash { + Some(tx_hash) => Ok(ReplayMode::Single(tx_hash)), + // Unreachable through clap (the target group is required), but the + // library API can construct `Cmd` directly. + None => Err(ReplayError::InvalidInput( + "'mega-evme replay' requires a TX_HASH, '--tx-file ', or '--block '" + .to_string(), + )), + } + } + + /// Replay the single transaction named by the positional argument. + async fn run_single(&self, pctx: &mut ProviderContext, tx_hash: B256) -> Result<()> { + let rctx = self.fetch_replay_context(&pctx.provider, tx_hash, pctx.chain_id).await?; + let external_envs = self.apply_external_envs(pctx)?; + self.execute_and_report(&pctx.provider, &rctx, external_envs).await + } + + /// Replay a batch of transactions through the shared per-block driver. + async fn run_batch(&self, pctx: &mut ProviderContext, mode: &batch::BatchMode) -> Result<()> { + let external_envs = self.apply_external_envs(pctx)?; + batch::run(&pctx.provider, pctx.chain_id, mode, external_envs, self.output_args.json).await + } + + /// Resolve the external environment and hand the capture snapshot to the + /// cache store, which persists it on exit. + fn apply_external_envs(&self, pctx: &mut ProviderContext) -> Result { + let (external_envs, env_snapshot) = self.resolve_external_envs(pctx)?; + if let Some(snapshot) = env_snapshot { + pctx.cache_store.set_external_env(snapshot); + } + Ok(external_envs) + } + /// Execute the replay, print the results, and (for `--dump-fixture`) /// finalize and write the fixture. /// @@ -225,16 +366,21 @@ impl Cmd { } /// Fetch the transaction, its block, and preceding transaction hashes from the provider. - async fn fetch_replay_context

(&self, provider: &P, chain_id: u64) -> Result + async fn fetch_replay_context

( + &self, + provider: &P, + tx_hash: B256, + chain_id: u64, + ) -> Result where P: Provider, { - info!(tx_hash = %self.tx_hash, "Fetching transaction"); + info!(tx_hash = %tx_hash, "Fetching transaction"); let target_tx = provider - .get_transaction_by_hash(self.tx_hash) + .get_transaction_by_hash(tx_hash) .await .map_err(|e| ReplayError::RpcError(format!("Failed to fetch transaction: {e}")))? - .ok_or_else(|| ReplayError::TransactionNotFound(self.tx_hash))?; + .ok_or_else(|| ReplayError::TransactionNotFound(tx_hash))?; debug!(block_number = ?target_tx.block_number, "Transaction found"); let (state_base_block, block_number, is_pending) = if let Some(n) = target_tx.block_number { @@ -267,7 +413,7 @@ impl Cmd { let mut preceding_tx_hashes = vec![]; if !is_pending { for hash in block.transactions.hashes() { - if hash == self.tx_hash { + if hash == tx_hash { break; } preceding_tx_hashes.push(hash); @@ -276,7 +422,7 @@ impl Cmd { debug!(chain_id, preceding_count = preceding_tx_hashes.len(), "Replay context ready"); - Ok(ReplayContext { target_tx, parent_block, block, chain_id, preceding_tx_hashes }) + Ok(ReplayContext { tx_hash, target_tx, parent_block, block, chain_id, preceding_tx_hashes }) } /// Build the external environment and (for capture mode) the envelope snapshot. @@ -400,10 +546,10 @@ impl Cmd { oracle_storage.sort_unstable(); let mega_env = state_test::types::MegaEnv { bucket_capacities, oracle_storage }; let receipt = provider - .get_transaction_receipt(self.tx_hash) + .get_transaction_receipt(ctx.tx_hash) .await .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? - .ok_or(ReplayError::TransactionNotFound(self.tx_hash))?; + .ok_or(ReplayError::TransactionNotFound(ctx.tx_hash))?; // Anchor the receipt to the replayed block: across a reorg or a // load-balanced endpoint serving divergent views, the receipt can // describe a different inclusion than the block fetched earlier, @@ -664,12 +810,36 @@ impl Cmd { } } +/// Whether any trace option was set on the command line. +/// +/// `--tracer` carries a default, so it counts as set only when it names a +/// non-default tracer. +fn has_trace_args(args: &run::TraceArgs) -> bool { + args.trace || + args.trace_output_file.is_some() || + !matches!(args.tracer, TracerType::Opcode) || + args.trace_opcode_disable_memory || + args.trace_opcode_disable_stack || + args.trace_opcode_disable_storage || + args.trace_opcode_enable_return_data || + args.trace_call_only_top_call || + args.trace_call_with_log || + args.trace_prestate_diff_mode || + args.trace_prestate_disable_code || + args.trace_prestate_disable_storage +} + +/// Whether any state dump option was set on the command line. +fn has_dump_args(args: &run::StateDumpArgs) -> bool { + args.dump || args.dump_output_file.is_some() +} + /// Build a [`BlockEnv`] from the RPC block header. /// /// Reads `excess_blob_gas` directly from the header rather than using a /// hardcoded default, so blob-fee-sensitive opcodes (e.g. `BLOBBASEFEE`) /// match on-chain semantics during replay. -fn retrieve_block_env(block: &Block) -> Result { +pub(super) fn retrieve_block_env(block: &Block) -> Result { let mut block_env = BlockEnv { number: U256::from(block.number()), beneficiary: block.header.beneficiary(), @@ -704,6 +874,117 @@ mod tests { use alloy_rpc_types_eth::Header as RpcHeader; use mega_evm::revm::context_interface::block::BlobExcessGasAndPrice; + const TX: &str = "0x323ddc8e67dfc134284d78c65f3c1dc7ff45ba1db02eeaf62e211ae3253478ef"; + const RPC: [&str; 2] = ["--rpc.replay-file", "/tmp/envelope.json"]; + + /// Parse a `replay` invocation, prefixing the shared offline RPC flags. + fn parse(extra: &[&str]) -> std::result::Result { + let mut argv = vec!["replay"]; + argv.extend_from_slice(&RPC); + argv.extend_from_slice(extra); + Cmd::try_parse_from(argv) + } + + /// Parse a batch invocation carrying one extra flag, and return the + /// validation error message it must be rejected with. + fn batch_rejection(extra: &[&str]) -> String { + let mut argv = vec!["--block", "22945844"]; + argv.extend_from_slice(extra); + let cmd = parse(&argv).expect("flags should parse"); + cmd.validate().expect_err("batch mode must reject the flag").to_string() + } + + #[test] + fn test_replay_target_group_accepts_each_form() { + assert!(matches!( + parse(&[TX]).expect("positional").resolve_mode().expect("mode"), + ReplayMode::Single(_) + )); + assert!(matches!( + parse(&["--block", "0x15e2034"]).expect("block").resolve_mode().expect("mode"), + ReplayMode::Batch(batch::BatchMode::Block(22_945_844)), + )); + // `--tx-file` reads the file, so only the parse is checked here. + assert_eq!( + parse(&["--tx-file", "/tmp/list.txt"]).expect("tx-file").tx_file, + Some(PathBuf::from("/tmp/list.txt")), + ); + } + + #[test] + fn test_replay_target_group_is_required() { + let err = parse(&[]).expect_err("a replay target is required"); + assert_eq!(err.kind(), clap::error::ErrorKind::MissingRequiredArgument); + } + + #[test] + fn test_replay_target_group_is_mutually_exclusive() { + for extra in [ + vec![TX, "--block", "1"], + vec![TX, "--tx-file", "/tmp/list.txt"], + vec!["--block", "1", "--tx-file", "/tmp/list.txt"], + ] { + let err = parse(&extra).expect_err("targets must be mutually exclusive"); + assert_eq!( + err.kind(), + clap::error::ErrorKind::ArgumentConflict, + "unexpected error for {extra:?}: {err}" + ); + } + } + + #[test] + fn test_batch_rejects_single_transaction_only_flags() { + for (extra, expected) in [ + (vec!["--dump-fixture", "/tmp/f.json"], "--dump-fixture"), + (vec!["--override.gas-limit", "50000"], "transaction overrides"), + (vec!["--override.value", "1ether"], "transaction overrides"), + (vec!["--override.input", "0xdeadbeef"], "transaction overrides"), + (vec!["--override.input-file", "/tmp/in.hex"], "transaction overrides"), + (vec!["--override.spec", "Rex4"], "--override.spec"), + (vec!["--trace"], "trace options"), + (vec!["--trace.output", "/tmp/t.json"], "trace options"), + (vec!["--tracer", "call"], "trace options"), + (vec!["--trace.call.with-log"], "trace options"), + (vec!["--trace.prestate.diff-mode"], "trace options"), + (vec!["--dump"], "state dump options"), + (vec!["--dump.output", "/tmp/s.json"], "state dump options"), + ] { + let message = batch_rejection(&extra); + assert!( + message.contains(expected) && message.contains("batch replay"), + "unexpected rejection for {extra:?}: {message}" + ); + } + } + + #[test] + fn test_batch_accepts_the_flags_it_supports() { + parse(&["--block", "1", "--json"]).expect("parse").validate().expect("--json is allowed"); + parse(&["--tx-file", "/tmp/list.txt"]) + .expect("parse") + .validate() + .expect("plain batch replay is allowed"); + } + + /// The single-transaction path keeps accepting every flag batch mode rejects. + #[test] + fn test_single_transaction_path_keeps_all_flags() { + for extra in [ + vec!["--trace", "--tracer", "call"], + vec!["--dump", "--dump.output", "/tmp/s.json"], + vec!["--override.spec", "Rex4"], + vec!["--override.gas-limit", "50000"], + ] { + let mut argv = vec![TX]; + argv.extend_from_slice(&extra); + parse(&argv) + .expect("parse") + .validate() + .unwrap_or_else(|e| panic!("single-transaction replay must accept {extra:?}: {e}")); + } + } + fn make_block(excess_blob_gas: Option) -> Block { let inner = ConsensusHeader { excess_blob_gas, ..Default::default() }; Block::empty(RpcHeader::new(inner)) diff --git a/bin/mega-evme/src/replay/mod.rs b/bin/mega-evme/src/replay/mod.rs index 8424e9fa..ab8d0522 100644 --- a/bin/mega-evme/src/replay/mod.rs +++ b/bin/mega-evme/src/replay/mod.rs @@ -3,6 +3,7 @@ //! This module provides functionality to replay historical transactions //! by fetching them from an RPC endpoint and re-executing them. +mod batch; mod cmd; mod fixture; mod hardforks; diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs new file mode 100644 index 00000000..35425b2b --- /dev/null +++ b/bin/mega-evme/tests/replay_batch.rs @@ -0,0 +1,210 @@ +//! Offline integration tests for `mega-evme replay --block` / `--tx-file`. +//! +//! These run against an RPC capture envelope large enough to replay whole +//! blocks, which is too big to commit; point `MEGA_EVME_TEST_ENVELOPE` at one +//! and run them explicitly: +//! +//! ```bash +//! MEGA_EVME_TEST_ENVELOPE= cargo test -p mega-evme -- --ignored +//! ``` +//! +//! They are `#[ignore]`d so CI, which has no envelope, skips them. + +use std::process::Command; + +/// Block fully covered by the envelope, and its transaction count. +const BLOCK: u64 = 22_945_844; +const BLOCK_TX_COUNT: usize = 23; + +/// Sample transactions of `BLOCK`: the index-0 deposit, a mid-block call, and +/// the last transaction. +const BLOCK_TXS: [(&str, u64); 3] = [ + ("0xde3d56dc739484166b8af1bea757bf7e3e9a4b9a0fb62d722703345570dfc1d6", 0), + ("0x323ddc8e67dfc134284d78c65f3c1dc7ff45ba1db02eeaf62e211ae3253478ef", 3), + ("0xb6a0b7a302c741f64b8e46861a3dcb2d5c1047f6f2cb89a35b5c2183c96296b7", 22), +]; + +/// Last transaction of the envelope's second block. +const OTHER_BLOCK: u64 = 22_945_853; +const OTHER_BLOCK_TX: &str = "0x18302160f2395069a44e1654d173fa9eed95ead8f922f12bfe07b6bdcc0a14f2"; +const OTHER_BLOCK_TX_INDEX: u64 = 23; + +/// Path of the offline envelope, or a skip message when it is not configured. +fn envelope() -> String { + std::env::var("MEGA_EVME_TEST_ENVELOPE").expect( + "set MEGA_EVME_TEST_ENVELOPE to an RPC capture covering the replayed blocks; \ + these tests are #[ignore]d precisely because that envelope is not committed", + ) +} + +fn mega_evme() -> Command { + Command::new(env!("CARGO_BIN_EXE_mega-evme")) +} + +/// Run `replay` offline and return its stdout, asserting the exit status. +fn replay(args: &[&str], expect_success: bool) -> String { + let envelope = envelope(); + let mut cmd = mega_evme(); + cmd.args(["replay", "--rpc.replay-file", &envelope]); + cmd.args(args); + let output = cmd.output().expect("failed to run mega-evme"); + assert_eq!( + output.status.success(), + expect_success, + "unexpected exit status for {args:?}\nstderr: {}", + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8(output.stdout).expect("stdout is utf-8") +} + +/// Parse NDJSON stdout into one JSON value per line. +fn ndjson(stdout: &str) -> Vec { + stdout + .lines() + .map(|line| { + assert!(!line.trim().is_empty(), "NDJSON output must not contain blank lines"); + serde_json::from_str(line) + .unwrap_or_else(|e| panic!("stdout line is not compact JSON ({e}): {line}")) + }) + .collect() +} + +/// `--block N --json` emits exactly one NDJSON line per transaction of the +/// block, in transaction order, and exits 0. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_block_emits_one_ndjson_line_per_transaction() { + let stdout = replay(&["--block", &BLOCK.to_string(), "--json"], true); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), BLOCK_TX_COUNT, "expected one line per transaction of the block"); + for (index, line) in lines.iter().enumerate() { + assert_eq!( + line["block_number"].as_u64(), + Some(BLOCK), + "every line must report the replayed block: {line}" + ); + assert_eq!( + line["tx_index"].as_u64(), + Some(index as u64), + "lines must be ordered by transaction index: {line}" + ); + assert!(line["tx_hash"].is_string(), "line must carry the transaction hash: {line}"); + assert!(line["receipt"].is_object(), "line must carry the receipt: {line}"); + assert!(line.get("error").is_none(), "line must not be an error entry: {line}"); + // Batch mode rejects the trace/dump flags, so those fields never appear. + assert!(line.get("trace").is_none(), "batch output must carry no trace: {line}"); + assert!(line.get("state").is_none(), "batch output must carry no state dump: {line}"); + } +} + +/// A batch line and a single-transaction replay of the same transaction must +/// agree on the execution outcome. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_batch_matches_single_transaction_replay() { + let batch = ndjson(&replay(&["--block", &BLOCK.to_string(), "--json"], true)); + + for (tx_hash, tx_index) in BLOCK_TXS { + let single: serde_json::Value = serde_json::from_str(&replay(&["--json", tx_hash], true)) + .expect("single-transaction output is JSON"); + let line = batch + .iter() + .find(|line| line["tx_hash"] == tx_hash) + .unwrap_or_else(|| panic!("batch output is missing {tx_hash}")); + + assert_eq!(line["tx_index"].as_u64(), Some(tx_index), "wrong index for {tx_hash}"); + for field in ["success", "gas_used", "logs_count"] { + assert_eq!( + line[field], single[field], + "batch and single-transaction replay disagree on {field} for {tx_hash}", + ); + } + } +} + +/// `--tx-file` replays transactions from several blocks in one process and +/// reports them ordered by (block, transaction index). +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_tx_file_spans_blocks_in_order() { + // Deliberately unordered, with a comment, a blank line, and a duplicate. + let list = format!( + "# sample corpus\n{OTHER_BLOCK_TX}\n\n{}\n {}\n{}\n{}\n", + BLOCK_TXS[2].0, BLOCK_TXS[0].0, BLOCK_TXS[1].0, BLOCK_TXS[0].0, + ); + let path = std::env::temp_dir().join(format!("mega_evme_tx_list_{}.txt", std::process::id())); + std::fs::write(&path, list).expect("write tx list"); + + let stdout = replay(&["--tx-file", path.to_str().unwrap(), "--json"], true); + let _ = std::fs::remove_file(&path); + let lines = ndjson(&stdout); + + let observed: Vec<(u64, u64, &str)> = lines + .iter() + .map(|line| { + ( + line["block_number"].as_u64().expect("block number"), + line["tx_index"].as_u64().expect("transaction index"), + line["tx_hash"].as_str().expect("transaction hash"), + ) + }) + .collect(); + let expected: Vec<(u64, u64, &str)> = vec![ + (BLOCK, BLOCK_TXS[0].1, BLOCK_TXS[0].0), + (BLOCK, BLOCK_TXS[1].1, BLOCK_TXS[1].0), + (BLOCK, BLOCK_TXS[2].1, BLOCK_TXS[2].0), + (OTHER_BLOCK, OTHER_BLOCK_TX_INDEX, OTHER_BLOCK_TX), + ]; + + assert_eq!(observed, expected, "results must be ordered by (block, transaction index)"); +} + +/// A hash that cannot be resolved is reported as an error entry, the remaining +/// targets still replay, and the process exits non-zero. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_tx_file_reports_unresolved_targets_and_exits_nonzero() { + let unknown = "0x0000000000000000000000000000000000000000000000000000000000000001"; + let path = + std::env::temp_dir().join(format!("mega_evme_tx_list_bad_{}.txt", std::process::id())); + std::fs::write(&path, format!("{unknown}\n{}\n", BLOCK_TXS[1].0)).expect("write tx list"); + + let stdout = replay(&["--tx-file", path.to_str().unwrap(), "--json"], false); + let _ = std::fs::remove_file(&path); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), 2, "every target gets exactly one line, including failures"); + assert_eq!(lines[0]["tx_hash"].as_str(), Some(unknown)); + assert!(lines[0]["error"]["kind"].is_string(), "failure line carries an error kind"); + assert!(lines[0]["error"]["message"].is_string(), "failure line carries a message"); + assert_eq!(lines[1]["tx_hash"].as_str(), Some(BLOCK_TXS[1].0)); + assert_eq!(lines[1]["success"].as_bool(), Some(true), "the resolvable target still replays"); +} + +/// Batch mode rejects the single-transaction-only flags before doing any work. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_batch_rejects_single_transaction_flags() { + let envelope = envelope(); + for (extra, expected) in [ + (vec!["--dump-fixture", "/tmp/should-not-exist.json"], "--dump-fixture"), + (vec!["--override.gas-limit", "50000"], "transaction overrides"), + (vec!["--override.spec", "Rex4"], "--override.spec"), + (vec!["--trace"], "trace options"), + (vec!["--dump"], "state dump options"), + ] { + let mut cmd = mega_evme(); + cmd.args(["replay", "--rpc.replay-file", &envelope, "--block", &BLOCK.to_string()]); + cmd.args(&extra); + let output = cmd.output().expect("failed to run mega-evme"); + + assert!(!output.status.success(), "batch mode must reject {extra:?}"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains(expected) && stderr.contains("batch replay"), + "unexpected error for {extra:?}: {stderr}" + ); + assert!(output.stdout.is_empty(), "a rejected batch run must print nothing on stdout"); + } +} diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 0bc407eb..56575c17 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -1,24 +1,31 @@ --- -description: Fetch and re-execute an on-chain transaction with optional overrides and tracing. +description: Fetch and re-execute one or many on-chain transactions with optional overrides and tracing. --- # replay -Re-execute a historical transaction locally using an RPC endpoint or a previously captured fixture file. +Re-execute historical transactions locally using an RPC endpoint or a previously captured fixture file. In online mode, `mega-evme` fetches the transaction, block environment, and pre-state from the RPC and re-executes locally. In offline mode (`--rpc.replay-file`), all data is served from a local fixture captured by an earlier run — no network access is required. +`replay` has two modes. +The single-transaction mode replays the transaction named by the positional `TX_HASH` and supports the full option set (overrides, tracing, state dumps, fixture dumps). +[Batch mode](#batch-replay) (`--tx-file` / `--block`) replays many transactions in one process and reports one summary per transaction. + ## Usage ``` -mega-evme replay [OPTIONS] +mega-evme replay [OPTIONS] |--block > ``` +Exactly one replay target is required: the positional `TX_HASH`, `--tx-file`, or `--block`. +The three are mutually exclusive. + ## Arguments ### `TX_HASH` -The transaction hash to replay (32-byte hex, required). +The transaction hash to replay (32-byte hex). `mega-evme` re-executes the transaction locally using state and block context sourced from either an RPC endpoint or a local fixture file. This gives you a fully reproducible execution without needing a local archive node. @@ -35,6 +42,108 @@ Required for online replay and capture mode; omit when using `--rpc.replay-file` mega-evme replay --rpc https://mainnet.megaeth.com/rpc ``` +## Batch Replay + +Replaying a corpus of transactions one process at a time pays for provider construction, chain-id resolution, and RPC cache parsing once per transaction — work that dominates the actual EVM execution. +Batch mode does all of it once. + +A batch run builds a single provider and a single RPC cache, groups the requested transactions by their containing block, and processes the blocks in ascending order. +Each block is executed exactly once: state is forked at the parent block, pre-execution changes are applied, and every transaction of the block runs in order, with each requested transaction's result recorded before it is committed. +The RPC cache is persisted once, on exit, even if some transactions failed — the captured responses are the artifact you need to debug the failure offline. + +Batch mode issues the same RPC calls as single-transaction replay, so an offline envelope captured by single-transaction runs serves a batch run without a cache miss. + +### `--tx-file ` + +Replay every transaction hash listed in ``, one per line. + +Blank lines and lines whose first non-whitespace character is `#` are ignored. +A hash listed more than once is replayed once. +A line that is not a valid 32-byte hex hash aborts the run before any network access, naming the offending line number. + +### `--block ` + +Replay every transaction of block `N`, given in decimal or `0x`-prefixed hex. + +### Restrictions + +Batch mode reports one summary per transaction and has no meaningful semantics for per-transaction artifacts or what-if knobs, so the following are rejected up front with an explanatory error rather than silently ignored: + +- `--dump-fixture` +- Transaction overrides (`--override.gas-limit`, `--override.value`, `--override.input`, `--override.input-file`) +- `--override.spec` — each block's spec is auto-detected from its timestamp +- All trace options (`--trace`, `--trace.output`, `--tracer`, `--trace.*`) +- All state dump options (`--dump`, `--dump.output`) + +Single-transaction replay keeps accepting all of them. + +### Output + +With `--json`, batch mode writes NDJSON: exactly one compact, single-line JSON object per requested transaction, in processing order (ascending block, then transaction index). + +A transaction that executed is reported as its `tx_hash`, `block_number`, and `tx_index`, followed by the same fields the single-transaction JSON output carries (`success`, `gas_used`, `logs_count`, and the optional `output` / `contract_address` / `revert_reason` / `halt_reason`) and its `receipt`. +Both shapes below are expanded for readability; on the wire each object occupies exactly one line. + +```json +{ + "tx_hash": "0x…", + "block_number": 22945844, + "tx_index": 3, + "success": true, + "gas_used": 81740, + "logs_count": 0, + "receipt": { "…": "…" } +} +``` + +A transaction that could not be executed is reported as an error entry instead: + +```json +{ + "tx_hash": "0x…", + "error": { "kind": "not_found", "message": "Transaction not found" } +} +``` + +`kind` is one of `not_found` (unknown hash), `pending` (mined into no block yet), `rpc` (an RPC call failed), or `execution` (block setup or the block executor rejected the transaction). +Execution outcomes are not errors: a reverted or halted transaction is a normal result line with `success: false`. + +Without `--json`, each transaction is printed with a header naming its hash, block, and index, followed by the same summary and receipt the single-transaction mode prints. +A final one-line summary (transactions replayed, transactions failed, elapsed time) is logged at `INFO` level, so pass `-vvv` to see it. + +### Exit Status + +A batch run exits `0` when every requested transaction produced an execution result, and `1` when any of them produced an error entry. +The NDJSON stream is written to stdout in both cases; diagnostics go to stderr. + +### Examples + +Replay a whole block offline and stream the results as NDJSON: + +```bash +mega-evme replay --rpc.replay-file ./fixtures/blocks.json --block 22945844 --json +``` + +Replay a corpus of transactions against a live RPC, one process for the lot: + +```bash +mega-evme replay --rpc https://mainnet.megaeth.com/rpc --tx-file ./corpus.txt --json > results.ndjson +``` + +Where `corpus.txt` looks like: + +``` +# regression corpus, refreshed 2026-08-03 +0xde3d56dc739484166b8af1bea757bf7e3e9a4b9a0fb62d722703345570dfc1d6 +0x323ddc8e67dfc134284d78c65f3c1dc7ff45ba1db02eeaf62e211ae3253478ef +``` + +Count the transactions that did not succeed: + +```bash +jq -c 'select(.error != null or .success == false)' results.ndjson | wc -l +``` + ## RPC Cache File `mega-evme replay` supports a transport-level JSON-RPC fixture mechanism that records every request/response pair to a single file and serves them back on later runs without touching the network. @@ -181,18 +290,21 @@ All of that context comes from the RPC. `replay` supports the following shared option groups. See the linked pages for full details. +Options marked _(single transaction only)_ are rejected in [batch mode](#batch-replay). +- **Batch replay** — Replay many transactions in one process via `--tx-file` / `--block`. + See [Batch Replay](#batch-replay) above. - **SALT buckets** — Configure SALT bucket capacity for dynamic storage gas pricing. See [SALT Buckets](../configuration/salt-buckets.md). -- **State dump** — Dump or load pre/post-state snapshots. +- **State dump** _(single transaction only)_ — Dump or load pre/post-state snapshots. See [State Management](../configuration/state-management.md). - **RPC cache file** — Single-file JSON-RPC capture and offline replay via `--rpc.capture-file` / `--rpc.replay-file`. See [RPC Cache File](#rpc-cache-file) above. - **RPC cache / retry** — Per-chain response cache, retry, and rate-limit settings. See [RPC Cache and Retry](../configuration/state-management.md#rpc-cache-and-retry). -- **Tracing** — Emit execution traces (call traces, opcode traces, gas profiles, etc.). +- **Tracing** _(single transaction only)_ — Emit execution traces (call traces, opcode traces, gas profiles, etc.). See [Tracing Overview](../tracing/overview.md). -- **Fixture dump** — Write a self-validating EEST state-test fixture via `--dump-fixture`. +- **Fixture dump** _(single transaction only)_ — Write a self-validating EEST state-test fixture via `--dump-fixture`. See [Self-Validating Fixture Dump](#self-validating-fixture-dump) above. - **Throughput benchmark** — Dump a fixture (`--dump-fixture`) and time it with `state-test --bench`. See [Throughput Benchmark](#throughput-benchmark) above. @@ -232,6 +344,12 @@ mega-evme replay --rpc https://mainnet.megaeth.com/rpc --override.input 0xdeadbe mega-evme replay --rpc https://mainnet.megaeth.com/rpc --override.spec Rex2 0xabc123... ``` +**Replay a whole block as NDJSON** + +```bash +mega-evme replay --rpc https://mainnet.megaeth.com/rpc --block 22945844 --json +``` + ## See Also - [`run`](./run.md) — Execute raw EVM bytecode locally without fetching from RPC From 88ca0a6ec35cbad051e8265d6dbe709a0d47429c Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 16:51:28 +0800 Subject: [PATCH 05/64] feat(mega-evme): concurrent-safe cache persist and cache merge Share --rpc.cache-dir across processes via exclusive sidecar lock plus re-read-merge on persist (provider cache and capture envelopes). Add mega-evme cache merge for offline consolidation of both shapes. --- Cargo.lock | 33 ++ bin/mega-evme/Cargo.toml | 1 + bin/mega-evme/src/cache/merge.rs | 444 ++++++++++++++++++ bin/mega-evme/src/cache/mod.rs | 262 +++++++++++ bin/mega-evme/src/cmd.rs | 3 + .../src/common/provider/cache_store.rs | 344 ++++++++++++-- bin/mega-evme/src/lib.rs | 2 + docs/mega-evme/SUMMARY.md | 1 + docs/mega-evme/commands/cache.md | 76 +++ .../configuration/state-management.md | 19 + docs/mega-evme/overview.md | 1 + 11 files changed, 1141 insertions(+), 45 deletions(-) create mode 100644 bin/mega-evme/src/cache/merge.rs create mode 100644 bin/mega-evme/src/cache/mod.rs create mode 100644 docs/mega-evme/commands/cache.md diff --git a/Cargo.lock b/Cargo.lock index 67952e73..b0aba7a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,6 +2192,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -3110,6 +3120,7 @@ dependencies = [ "alloy-transport-http", "clap", "dirs", + "fs2", "mega-evm", "mega-evme", "mega-state-test", @@ -5726,6 +5737,22 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -5735,6 +5762,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" diff --git a/bin/mega-evme/Cargo.toml b/bin/mega-evme/Cargo.toml index 1f893ce0..d90c939a 100644 --- a/bin/mega-evme/Cargo.toml +++ b/bin/mega-evme/Cargo.toml @@ -48,6 +48,7 @@ op-alloy-rpc-types.workspace = true # misc clap = { workspace = true, features = ["default", "env"] } dirs.workspace = true +fs2 = { version = "0.4", default-features = false } reqwest = "0.13" serde.workspace = true serde_json.workspace = true diff --git a/bin/mega-evme/src/cache/merge.rs b/bin/mega-evme/src/cache/merge.rs new file mode 100644 index 00000000..1d7137ad --- /dev/null +++ b/bin/mega-evme/src/cache/merge.rs @@ -0,0 +1,444 @@ +//! Pure merge helpers for provider-cache and capture-envelope JSON shapes. +//! +//! Used by the `cache merge` subcommand and by lock-protected merge-on-persist +//! in [`crate::common::provider`]'s cache store. + +use std::{ + collections::BTreeMap, + fs, + io::Write as _, + path::{Path, PathBuf}, +}; + +use alloy_primitives::B256; +use serde::{Deserialize, Serialize}; + +use crate::common::{EvmeError, Result}; + +/// Current on-disk envelope schema version (must match capture/replay). +pub(crate) const ENVELOPE_VERSION: u32 = 1; + +/// One `{key, value}` entry shared by provider-cache files and envelope `cache` arrays. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct CacheKv { + /// Request fingerprint (typically `keccak256` of method + params). + pub key: B256, + /// Serialized JSON-RPC response body. + pub value: String, +} + +/// Detected on-disk shape of a cache file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CacheShape { + /// JSON array of `{key, value}` (provider `--rpc.cache-dir` files). + Provider, + /// `{version, chain_id, cache, external_env?}` capture envelope. + Envelope, +} + +/// Minimal envelope view used for merge (independent of the store type). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub(crate) struct EnvelopeDoc { + /// Schema version (must match [`ENVELOPE_VERSION`] for this build). + pub version: u32, + /// Chain ID recorded at capture time. + pub chain_id: u64, + /// Transport-level cache entries. + pub cache: Vec, + /// Optional external-env snapshot (SALT buckets, …). + #[serde(default)] + pub external_env: Option, +} + +/// External-env snapshot fields needed for envelope merge. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct ExternalEnvDoc { + /// SALT bucket capacity pairs `(bucket_id, capacity)`. + #[serde(default)] + pub bucket_capacities: Vec<(u32, u64)>, +} + +/// Path of the advisory lock sidecar for `target` (`.lock`). +pub(crate) fn lock_sidecar_path(target: &Path) -> PathBuf { + let mut os = target.as_os_str().to_owned(); + os.push(".lock"); + PathBuf::from(os) +} + +/// Union `base` with `overlay` by key; overlay wins on collision. +/// +/// Output is sorted by key for deterministic files. +pub(crate) fn merge_kv_entries(base: Vec, overlay: Vec) -> Vec { + let mut map: BTreeMap = BTreeMap::new(); + for e in base { + map.insert(e.key, e.value); + } + for e in overlay { + map.insert(e.key, e.value); + } + map.into_iter().map(|(key, value)| CacheKv { key, value }).collect() +} + +/// Detect whether `value` is a provider-cache array or a capture envelope. +pub(crate) fn detect_shape(value: &serde_json::Value, path: &Path) -> Result { + if value.is_array() { + // Validate array elements look like {key, value} when non-empty. + if let Some(arr) = value.as_array() { + for (i, el) in arr.iter().enumerate() { + if !el.is_object() || el.get("key").is_none() || el.get("value").is_none() { + return Err(EvmeError::InvalidInput(format!( + "Provider-cache entry {i} in '{}' is not a {{key, value}} object", + path.display() + ))); + } + } + } + return Ok(CacheShape::Provider); + } + if let Some(obj) = value.as_object() { + if obj.contains_key("version") && obj.contains_key("chain_id") && obj.contains_key("cache") + { + return Ok(CacheShape::Envelope); + } + } + Err(EvmeError::InvalidInput(format!( + "Unrecognized cache file shape in '{}': expected a JSON array of {{key, value}} \ + or a capture envelope {{version, chain_id, cache, ...}}", + path.display() + ))) +} + +/// Read and parse a provider-cache file (JSON array). Missing file → empty vec. +/// +/// Corrupt / unreadable content returns `Err` so callers can degrade or hard-fail. +pub(crate) fn read_provider_cache(path: &Path) -> Result> { + if !path.exists() { + return Ok(Vec::new()); + } + let content = fs::read_to_string(path).map_err(|e| { + EvmeError::InvalidInput(format!("Failed to read cache file {}: {e}", path.display())) + })?; + let value: serde_json::Value = serde_json::from_str(&content).map_err(|e| { + EvmeError::InvalidInput(format!("Failed to parse cache file {}: {e}", path.display())) + })?; + match detect_shape(&value, path)? { + CacheShape::Provider => serde_json::from_value(value).map_err(|e| { + EvmeError::InvalidInput(format!( + "Failed to decode provider-cache entries in {}: {e}", + path.display() + )) + }), + CacheShape::Envelope => Err(EvmeError::InvalidInput(format!( + "Expected provider-cache array in '{}', found capture envelope", + path.display() + ))), + } +} + +/// Read and parse a capture envelope. Missing file is an error for callers that +/// require a document; use [`try_read_envelope`] for best-effort re-read. +pub(crate) fn read_envelope(path: &Path) -> Result { + let content = fs::read_to_string(path).map_err(|e| { + EvmeError::FixtureError(format!("Failed to read envelope {}: {e}", path.display())) + })?; + let value: serde_json::Value = serde_json::from_str(&content).map_err(|e| { + EvmeError::FixtureError(format!("Failed to parse envelope {}: {e}", path.display())) + })?; + match detect_shape(&value, path).map_err(|e| EvmeError::FixtureError(e.to_string()))? { + CacheShape::Envelope => { + let doc: EnvelopeDoc = serde_json::from_value(value).map_err(|e| { + EvmeError::FixtureError(format!( + "Failed to decode envelope {}: {e}", + path.display() + )) + })?; + if doc.version != ENVELOPE_VERSION { + return Err(EvmeError::FixtureError(format!( + "Unsupported cache file version {} in '{}'; expected {ENVELOPE_VERSION}", + doc.version, + path.display(), + ))); + } + Ok(doc) + } + CacheShape::Provider => Err(EvmeError::FixtureError(format!( + "Expected capture envelope in '{}', found provider-cache array", + path.display() + ))), + } +} + +/// Merge two provider-cache entry lists (overlay wins). +pub(crate) fn merge_provider_lists(base: Vec, overlay: Vec) -> Vec { + merge_kv_entries(base, overlay) +} + +/// Merge `ours` over `on_disk` for envelope persist (ours wins on key collision; +/// `external_env`: keep ours if set, else on-disk). +/// +/// Returns an error if `chain_id` or `version` disagree. +pub(crate) fn merge_envelope_for_persist( + on_disk: &EnvelopeDoc, + ours: &EnvelopeDoc, + path: &Path, +) -> Result { + if on_disk.version != ours.version { + return Err(EvmeError::FixtureError(format!( + "Envelope version mismatch when merging '{}': on-disk {}, ours {}", + path.display(), + on_disk.version, + ours.version, + ))); + } + if on_disk.chain_id != ours.chain_id { + return Err(EvmeError::FixtureError(format!( + "Envelope chain_id mismatch when merging '{}': on-disk {}, ours {}", + path.display(), + on_disk.chain_id, + ours.chain_id, + ))); + } + Ok(EnvelopeDoc { + version: ours.version, + chain_id: ours.chain_id, + cache: merge_kv_entries(on_disk.cache.clone(), ours.cache.clone()), + external_env: ours.external_env.clone().or_else(|| on_disk.external_env.clone()), + }) +} + +/// Merge multiple envelope inputs for the `cache merge` subcommand. +/// +/// All inputs must share `version` and `chain_id`. Later inputs win on cache +/// key collision. Non-null `external_env` values must be identical when more +/// than one is present. +pub(crate) fn merge_envelopes_cli(docs: &[(PathBuf, EnvelopeDoc)]) -> Result { + let Some((_, first)) = docs.first() else { + return Err(EvmeError::InvalidInput("cache merge requires at least one input file".into())); + }; + let version = first.version; + let chain_id = first.chain_id; + if version != ENVELOPE_VERSION { + return Err(EvmeError::InvalidInput(format!( + "Unsupported envelope version {version} in '{}'; expected {ENVELOPE_VERSION}", + docs[0].0.display(), + ))); + } + + let mut cache = Vec::new(); + let mut external_env: Option = None; + + for (path, doc) in docs { + if doc.version != version { + return Err(EvmeError::InvalidInput(format!( + "Envelope version mismatch: '{}' has version {}, expected {version}", + path.display(), + doc.version, + ))); + } + if doc.chain_id != chain_id { + return Err(EvmeError::InvalidInput(format!( + "Envelope chain_id mismatch: '{}' has chain_id {}, expected {chain_id}", + path.display(), + doc.chain_id, + ))); + } + cache = merge_kv_entries(cache, doc.cache.clone()); + if let Some(ref ext) = doc.external_env { + match &external_env { + None => external_env = Some(ext.clone()), + Some(prev) if prev != ext => { + return Err(EvmeError::InvalidInput(format!( + "Conflicting external_env snapshots while merging '{}'", + path.display(), + ))); + } + Some(_) => {} + } + } + } + + Ok(EnvelopeDoc { version, chain_id, cache, external_env }) +} + +/// Atomically write `entries` as a provider-cache JSON array to `path`. +pub(crate) fn write_provider_cache_atomic(path: &Path, entries: &[CacheKv]) -> Result<()> { + let dir = path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(dir).map_err(|e| { + EvmeError::InvalidInput(format!("Failed to create directory {}: {e}", dir.display())) + })?; + let serialized = serde_json::to_vec(entries) + .map_err(|e| EvmeError::InvalidInput(format!("Failed to serialize provider cache: {e}")))?; + write_bytes_atomic(path, &serialized) + .map_err(|e| EvmeError::InvalidInput(format!("Failed to write {}: {e}", path.display()))) +} + +/// Atomically write an envelope document (pretty-printed, matching capture). +pub(crate) fn write_envelope_atomic(path: &Path, doc: &EnvelopeDoc) -> Result<()> { + let dir = path.parent().unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(dir).map_err(|e| { + EvmeError::FixtureError(format!( + "Failed to create cache file directory {}: {e}", + dir.display() + )) + })?; + let serialized = serde_json::to_string_pretty(doc).map_err(|e| { + EvmeError::FixtureError(format!("Failed to serialize envelope for {}: {e}", path.display())) + })?; + write_bytes_atomic(path, serialized.as_bytes()).map_err(|e| { + EvmeError::FixtureError(format!("Failed to persist envelope to {}: {e}", path.display())) + }) +} + +/// Temp-file + rename write. +pub(crate) fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + let dir = path.parent().unwrap_or_else(|| Path::new(".")); + let mut tmp = tempfile::NamedTempFile::new_in(dir).map_err(|e| { + std::io::Error::other(format!("failed to create temp file in {}: {e}", dir.display())) + })?; + tmp.write_all(bytes)?; + tmp.flush()?; + tmp.persist(path).map_err(|e| { + std::io::Error::other(format!( + "failed to rename temp file into {}: {}", + path.display(), + e.error, + )) + })?; + Ok(()) +} + +/// Load any supported cache file and return its shape + entry count. +pub(crate) fn load_cache_file(path: &Path) -> Result<(CacheShape, LoadedCache)> { + let content = fs::read_to_string(path) + .map_err(|e| EvmeError::InvalidInput(format!("Failed to read {}: {e}", path.display())))?; + let value: serde_json::Value = serde_json::from_str(&content) + .map_err(|e| EvmeError::InvalidInput(format!("Failed to parse {}: {e}", path.display())))?; + let shape = detect_shape(&value, path)?; + match shape { + CacheShape::Provider => { + let entries: Vec = serde_json::from_value(value).map_err(|e| { + EvmeError::InvalidInput(format!( + "Failed to decode provider cache {}: {e}", + path.display() + )) + })?; + Ok((shape, LoadedCache::Provider(entries))) + } + CacheShape::Envelope => { + let doc: EnvelopeDoc = serde_json::from_value(value).map_err(|e| { + EvmeError::InvalidInput(format!( + "Failed to decode envelope {}: {e}", + path.display() + )) + })?; + if doc.version != ENVELOPE_VERSION { + return Err(EvmeError::InvalidInput(format!( + "Unsupported cache file version {} in '{}'; expected {ENVELOPE_VERSION}", + doc.version, + path.display(), + ))); + } + Ok((shape, LoadedCache::Envelope(doc))) + } + } +} + +/// Parsed cache file payload. +#[derive(Debug)] +pub(crate) enum LoadedCache { + Provider(Vec), + Envelope(EnvelopeDoc), +} + +impl LoadedCache { + pub(crate) fn entry_count(&self) -> usize { + match self { + Self::Provider(e) => e.len(), + Self::Envelope(d) => d.cache.len(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn kv(byte: u8, val: &str) -> CacheKv { + CacheKv { key: B256::repeat_byte(byte), value: val.to_string() } + } + + #[test] + fn test_merge_kv_union_and_ours_wins() { + let base = vec![kv(1, "a"), kv(2, "b")]; + let overlay = vec![kv(2, "B"), kv(3, "c")]; + let merged = merge_kv_entries(base, overlay); + assert_eq!(merged.len(), 3); + assert_eq!(merged[0], kv(1, "a")); + assert_eq!(merged[1], kv(2, "B")); // overlay wins + assert_eq!(merged[2], kv(3, "c")); + } + + #[test] + fn test_detect_shape_provider_and_envelope() { + let arr = serde_json::json!([{"key": B256::ZERO, "value": "x"}]); + assert_eq!(detect_shape(&arr, Path::new("p.json")).unwrap(), CacheShape::Provider); + let env = serde_json::json!({ + "version": 1, + "chain_id": 1, + "cache": [] + }); + assert_eq!(detect_shape(&env, Path::new("e.json")).unwrap(), CacheShape::Envelope); + } + + #[test] + fn test_merge_envelope_for_persist_union_and_ext() { + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "disk")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10)] }), + }; + let ours = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "ours"), kv(2, "new")], + external_env: None, + }; + let merged = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap(); + assert_eq!(merged.cache, vec![kv(1, "ours"), kv(2, "new")]); + assert_eq!(merged.external_env, Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10)] })); + } + + #[test] + fn test_merge_envelope_for_persist_chain_id_mismatch() { + let on_disk = EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: None }; + let ours = EnvelopeDoc { version: 1, chain_id: 2, cache: vec![], external_env: None }; + let err = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap_err(); + assert!(err.to_string().contains("chain_id")); + } + + #[test] + fn test_merge_envelopes_cli_conflict_external_env() { + let a = EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(1, "a")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 1)] }), + }; + let b = EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(2, "b")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 2)] }), + }; + let docs = vec![(PathBuf::from("a.json"), a), (PathBuf::from("b.json"), b)]; + let err = merge_envelopes_cli(&docs).unwrap_err(); + assert!(err.to_string().contains("external_env")); + } + + #[test] + fn test_lock_sidecar_path_suffix() { + let p = Path::new("/tmp/rpc-cache-1.json"); + assert_eq!(lock_sidecar_path(p), PathBuf::from("/tmp/rpc-cache-1.json.lock")); + } +} diff --git a/bin/mega-evme/src/cache/mod.rs b/bin/mega-evme/src/cache/mod.rs new file mode 100644 index 00000000..a00ff861 --- /dev/null +++ b/bin/mega-evme/src/cache/mod.rs @@ -0,0 +1,262 @@ +//! Top-level `cache` subcommand group (`mega-evme cache …`). +//! +//! Currently ships `cache merge` for consolidating per-worker provider-cache +//! files or capture envelopes after historical sharded campaigns. + +mod merge; + +use std::path::PathBuf; + +use clap::{Parser, Subcommand}; + +use crate::common::{EvmeError, Result}; + +pub(crate) use merge::{ + lock_sidecar_path, merge_envelope_for_persist, merge_kv_entries, merge_provider_lists, + read_envelope, read_provider_cache, write_bytes_atomic, write_envelope_atomic, + write_provider_cache_atomic, CacheKv, EnvelopeDoc, ExternalEnvDoc, ENVELOPE_VERSION, +}; + +use merge::{load_cache_file, merge_envelopes_cli, CacheShape, LoadedCache}; + +/// `mega-evme cache` — offline cache-file utilities. +#[derive(Parser, Debug)] +pub struct Cmd { + /// Cache utility subcommand (`merge`, …). + #[command(subcommand)] + pub command: CacheCommands, +} + +/// Cache utility subcommands. +#[derive(Subcommand, Debug)] +pub enum CacheCommands { + /// Merge provider-cache files or capture envelopes into one output file. + Merge(MergeArgs), +} + +/// Arguments for `mega-evme cache merge`. +#[derive(Parser, Debug)] +pub struct MergeArgs { + /// Input cache files (provider-cache arrays or capture envelopes; not mixed). + #[arg(required = true, num_args = 1.., value_name = "INPUT")] + pub inputs: Vec, + + /// Destination path for the merged file (written atomically via temp + rename). + #[arg(long, short = 'o', value_name = "FILE")] + pub output: PathBuf, +} + +impl Cmd { + /// Dispatch the cache subcommand. + pub fn run(self) -> Result<()> { + match self.command { + CacheCommands::Merge(args) => args.run(), + } + } +} + +impl MergeArgs { + /// Merge inputs into `--output` and print a one-line summary. + pub fn run(self) -> Result<()> { + if self.inputs.is_empty() { + return Err(EvmeError::InvalidInput( + "cache merge requires at least one input file".into(), + )); + } + + let mut loaded: Vec<(PathBuf, CacheShape, LoadedCache)> = + Vec::with_capacity(self.inputs.len()); + for path in &self.inputs { + let (shape, data) = load_cache_file(path)?; + loaded.push((path.clone(), shape, data)); + } + + let first_shape = loaded[0].1; + for (path, shape, _) in &loaded { + if *shape != first_shape { + return Err(EvmeError::InvalidInput(format!( + "Mixed cache file shapes: '{}' is {:?}, but the first input is {:?}. \ + Merge provider-cache files and capture envelopes in separate invocations.", + path.display(), + shape, + first_shape, + ))); + } + } + + let total_in: usize = loaded.iter().map(|(_, _, d)| d.entry_count()).sum(); + let input_count = loaded.len(); + + let unique_out = match first_shape { + CacheShape::Provider => { + let mut acc = Vec::new(); + for (_, _, data) in loaded { + let LoadedCache::Provider(entries) = data else { unreachable!() }; + // Later inputs win on collision. + acc = merge_provider_lists(acc, entries); + } + let unique = acc.len(); + write_provider_cache_atomic(&self.output, &acc)?; + unique + } + CacheShape::Envelope => { + let docs: Vec<(PathBuf, EnvelopeDoc)> = loaded + .into_iter() + .map(|(path, _, data)| { + let LoadedCache::Envelope(doc) = data else { unreachable!() }; + (path, doc) + }) + .collect(); + let merged = merge_envelopes_cli(&docs)?; + let unique = merged.cache.len(); + write_envelope_atomic(&self.output, &merged)?; + unique + } + }; + + println!( + "Merged {input_count} inputs ({total_in} entries in) → {unique_out} unique entries out" + ); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use alloy_primitives::B256; + use tempfile::tempdir; + + use super::*; + use crate::cache::merge::{merge_envelopes_cli, CacheKv, EnvelopeDoc, ExternalEnvDoc}; + + fn write(path: &std::path::Path, content: &str) { + fs::write(path, content).expect("write"); + } + + fn kv(byte: u8, val: &str) -> CacheKv { + CacheKv { key: B256::repeat_byte(byte), value: val.to_string() } + } + + #[test] + fn test_cache_merge_provider_union_later_wins() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let b = dir.path().join("b.json"); + let out = dir.path().join("out.json"); + + let entries_a = vec![kv(1, "from-a"), kv(2, "a")]; + let entries_b = vec![kv(2, "from-b"), kv(3, "b")]; + write(&a, &serde_json::to_string(&entries_a).unwrap()); + write(&b, &serde_json::to_string(&entries_b).unwrap()); + + MergeArgs { inputs: vec![a, b], output: out.clone() }.run().expect("merge"); + + let merged: Vec = + serde_json::from_str(&fs::read_to_string(&out).unwrap()).unwrap(); + assert_eq!(merged, vec![kv(1, "from-a"), kv(2, "from-b"), kv(3, "b")]); + } + + #[test] + fn test_cache_merge_envelope_union() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let b = dir.path().join("b.json"); + let out = dir.path().join("out.json"); + + let env_a = EnvelopeDoc { + version: 1, + chain_id: 4326, + cache: vec![kv(1, "a")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 100)] }), + }; + let env_b = EnvelopeDoc { + version: 1, + chain_id: 4326, + cache: vec![kv(1, "b"), kv(2, "b")], + external_env: None, + }; + write(&a, &serde_json::to_string_pretty(&env_a).unwrap()); + write(&b, &serde_json::to_string_pretty(&env_b).unwrap()); + + MergeArgs { inputs: vec![a, b], output: out.clone() }.run().expect("merge"); + + let merged: EnvelopeDoc = serde_json::from_str(&fs::read_to_string(&out).unwrap()).unwrap(); + assert_eq!(merged.chain_id, 4326); + assert_eq!(merged.cache, vec![kv(1, "b"), kv(2, "b")]); + assert_eq!(merged.external_env, Some(ExternalEnvDoc { bucket_capacities: vec![(1, 100)] })); + } + + #[test] + fn test_cache_merge_rejects_mixed_shapes() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let b = dir.path().join("b.json"); + let out = dir.path().join("out.json"); + + write(&a, &serde_json::to_string(&vec![kv(1, "a")]).unwrap()); + write( + &b, + &serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![], + external_env: None, + }) + .unwrap(), + ); + + let err = MergeArgs { inputs: vec![a, b], output: out }.run().unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("Mixed") || msg.contains("shape"), "msg={msg}"); + } + + #[test] + fn test_cache_merge_rejects_chain_id_mismatch() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let b = dir.path().join("b.json"); + let out = dir.path().join("out.json"); + + write( + &a, + &serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![], + external_env: None, + }) + .unwrap(), + ); + write( + &b, + &serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 2, + cache: vec![], + external_env: None, + }) + .unwrap(), + ); + + let err = MergeArgs { inputs: vec![a, b], output: out }.run().unwrap_err(); + assert!(err.to_string().contains("chain_id")); + } + + #[test] + fn test_cache_merge_rejects_version_mismatch() { + let docs = vec![ + ( + PathBuf::from("a.json"), + EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: None }, + ), + ( + PathBuf::from("b.json"), + EnvelopeDoc { version: 2, chain_id: 1, cache: vec![], external_env: None }, + ), + ]; + let err = merge_envelopes_cli(&docs).unwrap_err(); + assert!(err.to_string().contains("version")); + } +} diff --git a/bin/mega-evme/src/cmd.rs b/bin/mega-evme/src/cmd.rs index 68a1030d..1fcd1eb2 100644 --- a/bin/mega-evme/src/cmd.rs +++ b/bin/mega-evme/src/cmd.rs @@ -26,6 +26,8 @@ pub enum Commands { Tx(crate::tx::Cmd), /// Replay a transaction from RPC Replay(crate::replay::Cmd), + /// Offline RPC cache utilities (`cache merge`, …) + Cache(crate::cache::Cmd), } /// Error types for the main command system @@ -53,6 +55,7 @@ impl MainCmd { Commands::Run(cmd) => cmd.run().await.map_err(Error::from), Commands::Tx(cmd) => cmd.run().await.map_err(Error::from), Commands::Replay(cmd) => cmd.run().await.map_err(Error::from), + Commands::Cache(cmd) => cmd.run().map_err(Error::from), } .inspect_err(|e| { error!(err = ?e, "Error executing command"); diff --git a/bin/mega-evme/src/common/provider/cache_store.rs b/bin/mega-evme/src/common/provider/cache_store.rs index ef7c717e..692a263c 100644 --- a/bin/mega-evme/src/common/provider/cache_store.rs +++ b/bin/mega-evme/src/common/provider/cache_store.rs @@ -8,19 +8,36 @@ //! //! The envelope is v1. Forward-incompatible changes bump `ENVELOPE_VERSION`; //! additive fields use `#[serde(default)]` instead. +//! +//! # Concurrent cache-dir sharing +//! +//! Persist takes an exclusive advisory lock on a sidecar `.lock`, re-reads +//! the target file, merges in-memory entries over on-disk ones (ours win on key +//! collision), then writes via temp-file + atomic rename. Multiple processes may +//! therefore share one `--rpc.cache-dir` without losing each other's entries. +//! The lock sidecar is left in place after the process exits (the flock is released +//! when the lock file handle is closed). use std::{ fmt, fs, - io::Write as _, + fs::OpenOptions, path::{Path, PathBuf}, }; use alloy_provider::layers::SharedCache; +use fs2::FileExt as _; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; use super::transport::TransportCache; -use crate::common::{EvmeError, Result}; +use crate::{ + cache::{ + lock_sidecar_path, merge_envelope_for_persist, merge_kv_entries, read_envelope, + read_provider_cache, write_bytes_atomic, write_envelope_atomic, CacheKv, EnvelopeDoc, + ExternalEnvDoc, ENVELOPE_VERSION, + }, + common::{EvmeError, Result}, +}; /// Clean-exit cache persistence handle. /// @@ -145,6 +162,11 @@ impl RpcCacheStore { /// For fixture-capture stores, any `external_env` snapshot previously /// attached via [`Self::set_external_env`] is written into the envelope. /// + /// Persist takes an exclusive advisory lock on `.lock`, re-reads the + /// on-disk file (a sibling process may have written since load), and merges + /// our in-memory entries over the on-disk ones (ours win on key collision) + /// before the atomic write. + /// /// - **`ProviderCache`**: best-effort — failures are warn-logged and swallowed. /// - **`FixtureCapture`**: hard error — the fixture is the primary output of capture mode. /// - **No-op**: returns `Ok(())`. @@ -189,40 +211,106 @@ impl fmt::Debug for RpcCacheStore { } } -/// Atomically persist `cache` to `target` via a temp file + rename. +/// RAII exclusive lock on the sidecar file for `target`. +/// +/// The lock is released when this guard is dropped (file handle closed). +/// The sidecar file itself is left on disk. +struct ExclusiveFileLock { + _file: fs::File, +} + +/// Acquire an exclusive advisory lock on `.lock`, blocking until held. +/// +/// The sidecar is created if missing and left in place after unlock. +fn acquire_exclusive_lock(target: &Path) -> std::io::Result { + let lock_path = lock_sidecar_path(target); + if let Some(parent) = lock_path.parent() { + fs::create_dir_all(parent)?; + } + // truncate(false): the sidecar is only a flock target; keep any existing bytes. + let file = + OpenOptions::new().create(true).read(true).write(true).truncate(false).open(&lock_path)?; + file.lock_exclusive()?; + Ok(ExclusiveFileLock { _file: file }) +} + +/// Atomically persist `cache` to `target` via lock + re-read-merge + temp rename. /// /// All error paths include `target` in the returned [`std::io::Error`] so the /// warn-log in [`RpcCacheStore::persist`] identifies which file failed. +/// +/// Lock acquisition failure degrades to an unlocked write with a `warn!`. +/// A missing or corrupt on-disk file during re-read degrades to persisting +/// our entries only (with a `warn!` for corrupt). fn save_cache_atomic(cache: &SharedCache, target: &Path) -> std::io::Result<()> { + let _guard = match acquire_exclusive_lock(target) { + Ok(g) => Some(g), + Err(err) => { + warn!( + path = %target.display(), + error = %err, + "Failed to acquire RPC cache lock; persisting without lock", + ); + None + } + }; + let dir = target.parent().unwrap_or_else(|| Path::new(".")); - let tmp = tempfile::NamedTempFile::new_in(dir).map_err(|e| { - std::io::Error::other(format!("failed to create temp file in {}: {e}", dir.display())) + fs::create_dir_all(dir).map_err(|e| { + std::io::Error::other(format!("failed to create directory {}: {e}", dir.display())) })?; - let tmp_path = tmp.path().to_path_buf(); - // alloy's save_cache takes a PathBuf, not a Write. - cache.save_cache(tmp_path).map_err(|e| { + // SharedCache has no iteration API — dump our entries to a temp file and re-read. + let our_tmp = tempfile::NamedTempFile::new_in(dir).map_err(|e| { + std::io::Error::other(format!("failed to create temp file in {}: {e}", dir.display())) + })?; + let our_tmp_path = our_tmp.path().to_path_buf(); + cache.save_cache(our_tmp_path.clone()).map_err(|e| { std::io::Error::other(format!("failed to save cache for {}: {e}", target.display())) })?; - // Atomic rename. persist() consumes the NamedTempFile without deleting it. - tmp.persist(target).map_err(|e| { + let our_entries: Vec = match fs::read_to_string(&our_tmp_path) + .map_err(|e| e.to_string()) + .and_then(|s| serde_json::from_str(&s).map_err(|e| e.to_string())) + { + Ok(entries) => entries, + Err(err) => { + return Err(std::io::Error::other(format!( + "failed to re-read our cache dump for {}: {err}", + target.display() + ))); + } + }; + // Drop the NamedTempFile so it is unlinked; we only needed the dump bytes. + drop(our_tmp); + + let disk_entries = match read_provider_cache(target) { + Ok(entries) => entries, + Err(err) => { + warn!( + path = %target.display(), + error = %err, + "Failed to re-read on-disk RPC cache during merge; persisting our entries only", + ); + Vec::new() + } + }; + + let merged = merge_kv_entries(disk_entries, our_entries); + let serialized = serde_json::to_vec(&merged).map_err(|e| { std::io::Error::other(format!( - "failed to rename temp file into {}: {}", - target.display(), - e.error, + "failed to serialize merged cache for {}: {e}", + target.display() )) })?; + write_bytes_atomic(target, &serialized)?; Ok(()) } -/// Envelope version accepted by this build. -const ENVELOPE_VERSION: u32 = 1; - /// On-disk envelope format shared by `--rpc.capture-file` (write) and /// `--rpc.replay-file` (read). Contains a transport-level cache dump, /// chain ID, and optional external environment snapshot. -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub(super) struct CacheFileEnvelope { /// Schema version (currently always 1, reserved for future format changes). version: u32, @@ -274,41 +362,75 @@ impl CacheFileEnvelope { Ok(envelope) } - /// Atomically write this envelope to `path`. + /// Atomically write this envelope to `path` under a lock, merging with any + /// on-disk envelope already present (ours win on cache key collision; + /// `external_env` keeps ours if set, else the on-disk one). + /// + /// Lock contention blocks until the lock is free. Failure to create/acquire + /// the lock degrades to an unlocked write with a `warn!`. Write failures + /// remain hard errors. pub(super) fn save(&self, path: &Path) -> Result<()> { - let dir = path.parent().unwrap_or_else(|| Path::new(".")); - fs::create_dir_all(dir).map_err(|e| { - EvmeError::FixtureError(format!( - "Failed to create cache file directory {}: {e}", - dir.display() - )) - })?; + let _guard = match acquire_exclusive_lock(path) { + Ok(g) => Some(g), + Err(err) => { + warn!( + path = %path.display(), + error = %err, + "Failed to acquire envelope lock; persisting without lock", + ); + None + } + }; + + let ours = self.to_merge_doc()?; + let to_write = if path.exists() { + match read_envelope(path) { + Ok(on_disk) => merge_envelope_for_persist(&on_disk, &ours, path)?, + // Version / chain_id / shape mismatches are hard errors (primary capture output). + // Corrupt or unreadable JSON degrades to ours-only with a warning. + Err(err) => { + let msg = err.to_string(); + let hard = msg.contains("chain_id") || + msg.contains("version") || + msg.contains("Unsupported") || + msg.contains("Expected capture envelope") || + msg.contains("provider-cache"); + if hard { + return Err(err); + } + warn!( + path = %path.display(), + error = %err, + "Failed to re-read on-disk envelope during merge; persisting our entries only", + ); + ours + } + } + } else { + ours + }; - let serialized = serde_json::to_string_pretty(self).map_err(|e| { - EvmeError::FixtureError(format!( - "Failed to serialize envelope for {}: {e}", - path.display() - )) - })?; + write_envelope_atomic(path, &to_write) + } - let mut tmp = tempfile::NamedTempFile::new_in(dir).map_err(|e| { - EvmeError::FixtureError(format!("Failed to create temp file in {}: {e}", dir.display())) - })?; - tmp.write_all(serialized.as_bytes()) - .map_err(|e| EvmeError::FixtureError(format!("Failed to write envelope: {e}")))?; - tmp.persist(path).map_err(|e| { - EvmeError::FixtureError(format!( - "Failed to persist envelope to {}: {e}", - path.display() - )) + fn to_merge_doc(&self) -> Result { + let cache: Vec = serde_json::from_value(self.cache.clone()).map_err(|e| { + EvmeError::FixtureError(format!("Failed to decode envelope cache entries: {e}")) })?; - - Ok(()) + Ok(EnvelopeDoc { + version: self.version, + chain_id: self.chain_id, + cache, + external_env: self + .external_env + .as_ref() + .map(|e| ExternalEnvDoc { bucket_capacities: e.bucket_capacities.clone() }), + }) } } /// Snapshot of mega-evm external environment inputs not derivable from RPC. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct ExternalEnvSnapshot { /// SALT bucket capacity pairs `(bucket_id, capacity)`. #[serde(default)] @@ -317,7 +439,8 @@ pub struct ExternalEnvSnapshot { #[cfg(test)] mod tests { - use alloy_primitives::keccak256; + use alloy_primitives::{keccak256, B256}; + use alloy_provider::layers::CacheLayer; use super::*; @@ -392,4 +515,135 @@ mod tests { let msg = format!("{err}"); assert!(msg.contains("parse"), "error should mention parse: {msg}"); } + + /// Interleaving: A holds only key A in memory; B persists key B; A then + /// persists — on-disk file must contain the union (B's entries survive). + #[test] + fn test_provider_cache_persist_merges_interleaved_disk_entries() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-1.json"); + + let key_a = B256::repeat_byte(0xaa); + let key_b = B256::repeat_byte(0xbb); + let val_a = r#"{"result":"a"}"#.to_string(); + let val_b = r#"{"result":"b"}"#.to_string(); + + // Process B persists first. + let cache_b = CacheLayer::new(64).cache(); + cache_b.put(key_b, val_b.clone()).expect("put b"); + RpcCacheStore::new(cache_b, path.clone()).persist().expect("persist b"); + + // Process A never loaded B's write; only has key_a in memory. + let cache_a = CacheLayer::new(64).cache(); + cache_a.put(key_a, val_a.clone()).expect("put a"); + RpcCacheStore::new(cache_a, path.clone()).persist().expect("persist a"); + + let loaded = CacheLayer::new(64).cache(); + loaded.load_cache(path).expect("load"); + assert_eq!(loaded.get(&key_a).as_deref(), Some(val_a.as_str())); + assert_eq!(loaded.get(&key_b).as_deref(), Some(val_b.as_str())); + } + + /// On collision, the process that persists last wins for that key. + #[test] + fn test_provider_cache_persist_ours_wins_on_collision() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-1.json"); + let key = B256::repeat_byte(0x01); + + let cache_b = CacheLayer::new(64).cache(); + cache_b.put(key, "from-b".into()).expect("put"); + RpcCacheStore::new(cache_b, path.clone()).persist().expect("persist b"); + + let cache_a = CacheLayer::new(64).cache(); + cache_a.put(key, "from-a".into()).expect("put"); + RpcCacheStore::new(cache_a, path.clone()).persist().expect("persist a"); + + let loaded = CacheLayer::new(64).cache(); + loaded.load_cache(path).expect("load"); + assert_eq!(loaded.get(&key).as_deref(), Some("from-a")); + } + + /// Lock sidecar `.lock` is created on persist and left in place. + #[test] + fn test_provider_cache_persist_creates_lock_sidecar_left_in_place() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-9.json"); + let lock = lock_sidecar_path(&path); + assert!(!lock.exists()); + + let cache = CacheLayer::new(16).cache(); + cache.put(B256::repeat_byte(1), "v".into()).expect("put"); + RpcCacheStore::new(cache, path.clone()).persist().expect("persist"); + + assert!(path.exists(), "cache file written"); + assert!(lock.exists(), "lock sidecar left in place"); + // Sidecar is an empty (or near-empty) lock file, not the cache payload. + let lock_meta = fs::metadata(&lock).expect("lock meta"); + assert!(lock_meta.len() == 0 || lock_meta.is_file()); + } + + /// Envelope persist merges on-disk entries the same way, with `chain_id` check. + #[test] + fn test_envelope_persist_merges_interleaved_disk_entries() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + let key_a = keccak256("a"); + let key_b = keccak256("b"); + + let cache_b = TransportCache::new(); + cache_b + .merge(&serde_json::json!([{ + "key": key_b, + "value": r#"{"result":"b"}"#, + }])) + .expect("seed b"); + CacheFileEnvelope::new(&cache_b, 99, None).save(&path).expect("save b"); + + let cache_a = TransportCache::new(); + cache_a + .merge(&serde_json::json!([{ + "key": key_a, + "value": r#"{"result":"a"}"#, + }])) + .expect("seed a"); + CacheFileEnvelope::new(&cache_a, 99, None).save(&path).expect("save a"); + + let env = CacheFileEnvelope::load(&path).expect("load"); + let loaded = TransportCache::from_value(&env.cache).expect("from_value"); + assert_eq!(loaded.len(), 2); + assert_eq!(env.chain_id, 99); + } + + /// Envelope persist hard-errors on `chain_id` mismatch with on-disk file. + #[test] + fn test_envelope_persist_rejects_chain_id_mismatch() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + let cache_b = TransportCache::new(); + CacheFileEnvelope::new(&cache_b, 1, None).save(&path).expect("save b"); + + let cache_a = TransportCache::new(); + let err = CacheFileEnvelope::new(&cache_a, 2, None).save(&path).expect_err("mismatch"); + assert!(err.to_string().contains("chain_id")); + } + + /// Corrupt on-disk provider cache during re-read does not abort; ours are written. + #[test] + fn test_provider_cache_persist_degrades_on_corrupt_disk() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-1.json"); + fs::write(&path, "not-json{{{").expect("corrupt"); + + let key = B256::repeat_byte(0xcc); + let cache = CacheLayer::new(16).cache(); + cache.put(key, "ok".into()).expect("put"); + RpcCacheStore::new(cache, path.clone()).persist().expect("persist"); + + let loaded = CacheLayer::new(16).cache(); + loaded.load_cache(path).expect("load"); + assert_eq!(loaded.get(&key).as_deref(), Some("ok")); + } } diff --git a/bin/mega-evme/src/lib.rs b/bin/mega-evme/src/lib.rs index 02c41cb5..be4e21bc 100644 --- a/bin/mega-evme/src/lib.rs +++ b/bin/mega-evme/src/lib.rs @@ -6,6 +6,8 @@ //! the library directly and exercise the public API the same way an external //! consumer would. +/// Offline RPC cache utilities (`cache merge`, …). +pub mod cache; /// Top-level CLI command parser and dispatch (`MainCmd`, `Commands`, `Error`). pub mod cmd; /// Shared building blocks: RPC provider/session, state, env, error, output diff --git a/docs/mega-evme/SUMMARY.md b/docs/mega-evme/SUMMARY.md index 7c817797..995a2c09 100644 --- a/docs/mega-evme/SUMMARY.md +++ b/docs/mega-evme/SUMMARY.md @@ -8,6 +8,7 @@ - [run](commands/run.md) - [tx](commands/tx.md) - [replay](commands/replay.md) +- [cache](commands/cache.md) ## Configuration diff --git a/docs/mega-evme/commands/cache.md b/docs/mega-evme/commands/cache.md new file mode 100644 index 00000000..7b09cce3 --- /dev/null +++ b/docs/mega-evme/commands/cache.md @@ -0,0 +1,76 @@ +--- +description: Merge provider-cache files or capture envelopes offline. +--- + +# cache + +Offline utilities for RPC cache files produced by `mega-evme`. + +Today the only subcommand is `merge`, which consolidates multiple cache files into one without contacting a network. + +## Usage + +``` +mega-evme cache merge ... --output +``` + +## `cache merge` + +Union one or more input files into a single output file. + +Inputs are auto-detected by JSON shape: + +| Shape | On-disk form | Produced by | +| ---------------- | ------------------------------------------- | --------------------------------------- | +| Provider cache | JSON array of `{key, value}` | `--rpc.cache-dir` per-chain files | +| Capture envelope | `{version, chain_id, cache, external_env?}` | `--rpc.capture-file` / offline fixtures | + +All inputs in one invocation must share the same shape. +Mixing a provider-cache file with a capture envelope is a hard error that names the offending path. + +### Provider-cache merge + +- Union entries by `key`. +- Later inputs win on collision. +- Output is a provider-cache-shaped JSON array, written atomically (temp file + rename). + +### Envelope merge + +- Every input must use the current envelope `version` and the same `chain_id` (else hard error naming the mismatch). +- Union the `cache` arrays by key; later inputs win on collision. +- `external_env`: if two inputs carry non-identical snapshots, hard error; otherwise propagate the non-null snapshot. +- Output is a pretty-printed envelope, written atomically. + +### Summary + +On success, `cache merge` prints one line and exits 0: + +``` +Merged 3 inputs (120 entries in) → 95 unique entries out +``` + +### Examples + +Merge sharded worker provider caches after a multi-process campaign: + +```bash +mega-evme cache merge \ + worker0/rpc-cache-4326.json \ + worker1/rpc-cache-4326.json \ + worker2/rpc-cache-4326.json \ + --output ./rpc-cache-4326.json +``` + +Merge two capture envelopes for the same chain: + +```bash +mega-evme cache merge \ + capture-a.json \ + capture-b.json \ + -o merged-capture.json +``` + +## See also + +- [State Management](../configuration/state-management.md#rpc-cache-and-retry) — live `--rpc.cache-dir` behavior and concurrent sharing +- [replay](replay.md#rpc-cache-file) — capture and offline replay fixtures diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index d1db51e8..ca735b7a 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -217,6 +217,25 @@ The default cache directory is the platform cache directory: - **Linux**: `$XDG_CACHE_HOME/mega-evme/rpc` - **macOS**: `~/Library/Caches/mega-evme/rpc` +### Concurrent cache-dir sharing + +Multiple `mega-evme` processes may share the same `--rpc.cache-dir` safely. +On clean-exit persist, each process: + +1. Takes an exclusive advisory lock on a sidecar file next to the cache (`rpc-cache-{chain_id}.json.lock`). +2. Re-reads the on-disk cache (a sibling process may have written since this process loaded). +3. Merges its in-memory entries over the on-disk ones (same key → this process's value wins). +4. Writes the result via a temp file and atomic rename, then releases the lock. + +The lock sidecar is left in place after the process exits; only the flock is released when the handle closes. +Lock contention blocks for a short critical section rather than failing the finished run. +If the lock cannot be acquired at all (for example the directory is not writable), persist logs a warning and falls back to an unlocked write. +A missing or corrupt on-disk file during the re-read degrades to writing this process's entries only (also warned). + +Capture envelopes (`--rpc.capture-file`) use the same lock + re-read-merge path, with an additional check that the on-disk `chain_id` matches before merging. + +To consolidate historical per-worker cache directories offline, use [`cache merge`](../commands/cache.md). + ### Cache Flags | Flag | Type | Default | Description | diff --git a/docs/mega-evme/overview.md b/docs/mega-evme/overview.md index 51207926..314f4e2e 100644 --- a/docs/mega-evme/overview.md +++ b/docs/mega-evme/overview.md @@ -21,6 +21,7 @@ cargo build --release -p mega-evme | [`run`](commands/run.md) | Execute arbitrary EVM bytecode directly | | [`tx`](commands/tx.md) | Run a transaction with full transaction context and optional RPC state forking | | [`replay`](commands/replay.md) | Replay an existing on-chain transaction from RPC | +| [`cache`](commands/cache.md) | Offline RPC cache utilities (merge provider caches or capture envelopes) | ## Quick Start From 903091e11e87aada4b1176cb8805db70e9d75c10 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 17:27:11 +0800 Subject: [PATCH 06/64] feat(mega-evme): verify replays against on-chain receipts Add `replay --verify-receipt`, which fetches each replayed transaction's on-chain receipt and compares it against the receipt the replay produced: success status, gas used, and the emitted logs (count plus each log's address, topics, and data). Works in single-transaction and batch mode. The comparison lives in a new `replay/verify` module as a pure function over the consensus facts of both receipts, so it is independent of how either receipt was obtained. The verdict is reported as a `verification` object in JSON (absent without the flag) and as one verdict line in human-readable output; a mismatch fails the run through a dedicated `VerificationMismatch` error. Anything that prevents the comparison from running is reported as an infrastructure failure rather than a mismatch: a receipt the endpoint cannot serve or has pruned, a receipt describing a different inclusion than the replayed block (reorg or divergent endpoint), and pending transactions. --- bin/mega-evme/src/common/error.rs | 17 + bin/mega-evme/src/common/outcome.rs | 4 + bin/mega-evme/src/replay/batch.rs | 130 +++++- bin/mega-evme/src/replay/cmd.rs | 100 ++++- bin/mega-evme/src/replay/mod.rs | 1 + bin/mega-evme/src/replay/verify.rs | 613 +++++++++++++++++++++++++++ bin/mega-evme/tests/replay_batch.rs | 23 + bin/mega-evme/tests/replay_verify.rs | 381 +++++++++++++++++ docs/mega-evme/commands/replay.md | 124 +++++- 9 files changed, 1383 insertions(+), 10 deletions(-) create mode 100644 bin/mega-evme/src/replay/verify.rs create mode 100644 bin/mega-evme/tests/replay_verify.rs diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index 69ca5bc0..ce14f008 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -56,6 +56,23 @@ pub enum EvmeError { #[error("Unsupported transaction type: {0}")] UnsupportedTxType(u8), + /// A `replay --verify-receipt` run found at least one local replay that did + /// not reproduce the on-chain receipt. + /// + /// Distinct from the infrastructure error variants so a verification + /// mismatch can be told apart from a target that could not be replayed or + /// verified at all. + #[error( + "Receipt verification mismatch: {mismatched} of {total} verified transaction(s) did \ + not reproduce the on-chain receipt" + )] + VerificationMismatch { + /// Number of verified transactions whose replay diverged. + mismatched: usize, + /// Number of transactions that were verified. + total: usize, + }, + /// Code hash mismatch #[error("Code hash mismatch: expected {expected}, computed {computed}")] CodeHashMismatch { diff --git a/bin/mega-evme/src/common/outcome.rs b/bin/mega-evme/src/common/outcome.rs index 71b37c41..d6eccc36 100644 --- a/bin/mega-evme/src/common/outcome.rs +++ b/bin/mega-evme/src/common/outcome.rs @@ -275,6 +275,10 @@ pub struct ExecutionSummary { /// Transaction receipt (present only for `tx` command) #[serde(skip_serializing_if = "Option::is_none")] pub receipt: Option, + /// On-chain receipt verification verdict (present only for `replay + /// --verify-receipt`) + #[serde(skip_serializing_if = "Option::is_none")] + pub verification: Option, } impl ExecutionSummary { diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 19e98779..d28039ff 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -20,6 +20,7 @@ use std::{ }; use alloy_consensus::{BlockHeader, Transaction as _}; +use alloy_network::ReceiptResponse; use alloy_primitives::{Address, B256}; use alloy_provider::Provider; use alloy_rpc_types_eth::Block; @@ -47,7 +48,20 @@ use crate::{ ChainArgs, EvmeState, }; -use super::{cmd::retrieve_block_env, ReplayError, Result}; +use super::{ + cmd::retrieve_block_env, + verify::{self, ReceiptFacts, VerificationOutcome}, + ReplayError, Result, +}; + +/// How a batch run reports its targets. +#[derive(Debug, Clone, Copy)] +pub(super) struct ReportArgs { + /// Emit one NDJSON line per target instead of the human-readable summary. + pub json: bool, + /// Verify every target against its on-chain receipt. + pub verify_receipt: bool, +} /// What a batch run was asked to replay. #[derive(Debug)] @@ -113,6 +127,8 @@ struct ExecutedTx { contract_address: Option

, exec_time: Duration, receipt: OpTxReceipt, + /// On-chain receipt verdict, present iff `--verify-receipt` was given. + verification: Option, } /// A target transaction that hit an infrastructure failure. @@ -175,12 +191,16 @@ struct BatchErrorBody<'a> { /// /// Returns an error when at least one target produced an infrastructure error /// entry, so the process exits non-zero; execution outcomes never fail the run. +/// With `--verify-receipt`, a run in which every target replayed but some +/// diverged from its on-chain receipt fails with +/// [`ReplayError::VerificationMismatch`] instead — a distinct variant, so a +/// divergence is never confused with a target that could not be replayed. pub(super) async fn run

( provider: &P, chain_id: u64, mode: &BatchMode, external_envs: EvmeExternalEnvs, - json: bool, + report: ReportArgs, ) -> Result<()> where P: Provider + Clone + std::fmt::Debug, @@ -188,6 +208,8 @@ where let start = Instant::now(); let mut replayed = 0usize; let mut failed = 0usize; + let mut verified = 0usize; + let mut mismatched = 0usize; let jobs = match mode { BatchMode::Block(number) => { @@ -206,30 +228,50 @@ where ); for failure in failures { failed += 1; - emit(&BatchEntry::Failed(failure), json); + emit(&BatchEntry::Failed(failure), report.json); } jobs } }; for job in jobs { - for entry in replay_block(provider, chain_id, job, external_envs.clone()).await { - match entry { - BatchEntry::Executed(_) => replayed += 1, + for entry in + replay_block(provider, chain_id, job, external_envs.clone(), report.verify_receipt) + .await + { + match &entry { + BatchEntry::Executed(tx) => { + replayed += 1; + if let Some(verification) = &tx.verification { + verified += 1; + if !verification.matched { + mismatched += 1; + } + } + } BatchEntry::Failed(_) => failed += 1, } - emit(&entry, json); + emit(&entry, report.json); } } info!(replayed, failed, elapsed = ?start.elapsed(), "Batch replay finished"); + if report.verify_receipt { + info!(verified, mismatched, "On-chain receipt verification finished"); + } + // Infrastructure failures keep their own error: a target that never replayed + // was also never verified, so reporting it as a mismatch would overstate + // what the run actually found. if failed > 0 { return Err(ReplayError::Other(format!( "{failed} of {} target transaction(s) failed to replay", replayed + failed ))); } + if mismatched > 0 { + return Err(ReplayError::VerificationMismatch { mismatched, total: verified }); + } Ok(()) } @@ -285,6 +327,7 @@ async fn replay_block

( chain_id: u64, job: BlockJob, external_envs: EvmeExternalEnvs, + verify_receipt: bool, ) -> Vec where P: Provider + Clone + std::fmt::Debug, @@ -307,6 +350,17 @@ where Err(e) => return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()), }; + // Fetch the on-chain receipts before the block runs, so the comparison + // afterwards is a pure function over the two receipts. A receipt that cannot + // be fetched, or that describes a different inclusion than this block, is + // recorded as a per-target failure here and reported as an `rpc` error entry + // below: such a target is unverified, never mismatched. + let onchain_receipts = if verify_receipt { + fetch_target_receipts(provider, &targets, block.hash()).await + } else { + BTreeMap::new() + }; + let hardforks = get_hardfork_config(chain_id); let timestamp = block.header.timestamp(); let spec = hardforks.spec_id(timestamp); @@ -457,6 +511,26 @@ where Some(block_hash), target.tx_index, ); + let verification = if verify_receipt { + match onchain_receipts.get(&target.tx_hash) { + Some(Ok(onchain)) => { + Some(verify::compare(onchain, &ReceiptFacts::from_receipt(&receipt))) + } + // Without an on-chain receipt there is nothing to compare + // against: report the target as unverified. + unverified => { + let message = match unverified { + Some(Err(message)) => message.clone(), + _ => "No on-chain receipt was fetched for this transaction" + .to_string(), + }; + entries.push(failure(target.tx_hash, BatchErrorKind::Rpc, message)); + continue; + } + } + } else { + None + }; entries.push(BatchEntry::Executed(Box::new(ExecutedTx { tx_hash: target.tx_hash, block_number: number, @@ -465,6 +539,7 @@ where contract_address, exec_time: target.exec_time, receipt, + verification, }))); } } @@ -495,6 +570,40 @@ where entries } +/// Fetch the on-chain receipt of every target of a block. +/// +/// Each target maps either to the consensus facts its receipt reports, or to the +/// message explaining why it could not be verified (the endpoint failed the +/// call or pruned the receipt, or the receipt describes a different inclusion +/// than the block being replayed). +async fn fetch_target_receipts

( + provider: &P, + targets: &[B256], + block_hash: B256, +) -> BTreeMap> +where + P: Provider, +{ + let mut receipts = BTreeMap::new(); + for tx_hash in targets { + let fetched = match verify::fetch_receipt(provider, *tx_hash).await { + Ok(receipt) => match verify::check_inclusion(receipt.block_hash(), block_hash) { + Ok(()) => Ok(ReceiptFacts::from_receipt(&receipt.inner)), + Err(message) => Err(message), + }, + // The reported entry already carries the `rpc` kind, so the error's + // own "RPC error" prefix would only repeat it. + Err(ReplayError::RpcError(message)) => Err(message), + Err(e) => Err(e.to_string()), + }; + if let Err(message) = &fetched { + warn!(tx_hash = %tx_hash, %message, "Could not fetch the on-chain receipt"); + } + receipts.insert(*tx_hash, fetched); + } + receipts +} + /// Fetch a block by number, using the same call shape as the single-transaction path. async fn fetch_block

(provider: &P, number: u64) -> Result> where @@ -536,6 +645,9 @@ fn emit(entry: &BatchEntry, json: bool) { ExecutionSummary::from_result(&tx.exec_result, tx.contract_address); summary.receipt = Some(serde_json::to_value(&tx.receipt).expect("failed to serialize receipt")); + summary.verification = tx.verification.as_ref().map(|verification| { + serde_json::to_value(verification).expect("failed to serialize verification") + }); serde_json::to_string(&BatchResultLine { tx_hash: tx.tx_hash, block_number: tx.block_number, @@ -561,6 +673,10 @@ fn emit(entry: &BatchEntry, json: bool) { ); print_execution_summary(&tx.exec_result, tx.contract_address, tx.exec_time); print_receipt(&tx.receipt); + if let Some(verification) = &tx.verification { + println!(); + println!("{}", verification.verdict_line()); + } } BatchEntry::Failed(tx) => { println!(); diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 04d3f676..c4cc8d79 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -33,7 +33,11 @@ use crate::{ run, ChainArgs, EvmeState, }; -use super::{batch, ReplayError, Result}; +use super::{ + batch, + verify::{self, VerificationOutcome}, + ReplayError, Result, +}; /// Replay a transaction from RPC #[derive(Parser, Debug)] @@ -101,6 +105,18 @@ pub struct Cmd { /// status. Incompatible with transaction overrides and `--override.spec`. #[arg(long = "dump-fixture", value_name = "FILE")] pub dump_fixture: Option, + + /// Verify every replayed transaction against its on-chain receipt. + /// + /// Fetches the receipt of each target and compares the success status, the + /// gas used, and the emitted logs (count plus each log's address, topics, + /// and data). The verdict is reported per transaction, and a mismatch makes + /// the run exit non-zero. A target whose receipt cannot be fetched, or whose + /// receipt describes a different inclusion than the replayed block, is + /// reported as an infrastructure failure rather than a mismatch. Supported + /// in both single-transaction and batch mode. + #[arg(long = "verify-receipt")] + pub verify_receipt: bool, } /// Resolved provider and associated metadata from `--rpc` / `--rpc.capture-file` / @@ -120,6 +136,8 @@ pub(super) struct ReplayOutcome { pub receipt: OpTxReceipt, /// Self-validating fixture draft, present iff `--dump-fixture` was given. pub fixture: Option, + /// On-chain receipt verdict, present iff `--verify-receipt` was given. + pub verification: Option, } /// Intermediate context fetched from RPC before execution. @@ -289,6 +307,16 @@ impl Cmd { /// Replay the single transaction named by the positional argument. async fn run_single(&self, pctx: &mut ProviderContext, tx_hash: B256) -> Result<()> { let rctx = self.fetch_replay_context(&pctx.provider, tx_hash, pctx.chain_id).await?; + // A pending transaction has no receipt to verify against; fail clearly + // instead of replaying it and then surfacing the receipt lookup's + // confusing "transaction is unknown to the endpoint". + if self.verify_receipt && rctx.target_tx.block_number.is_none() { + return Err(ReplayError::Other( + "--verify-receipt does not support pending transactions: the comparison needs \ + the on-chain receipt, which does not exist yet" + .to_string(), + )); + } let external_envs = self.apply_external_envs(pctx)?; self.execute_and_report(&pctx.provider, &rctx, external_envs).await } @@ -296,7 +324,14 @@ impl Cmd { /// Replay a batch of transactions through the shared per-block driver. async fn run_batch(&self, pctx: &mut ProviderContext, mode: &batch::BatchMode) -> Result<()> { let external_envs = self.apply_external_envs(pctx)?; - batch::run(&pctx.provider, pctx.chain_id, mode, external_envs, self.output_args.json).await + batch::run( + &pctx.provider, + pctx.chain_id, + mode, + external_envs, + batch::ReportArgs { json: self.output_args.json, verify_receipt: self.verify_receipt }, + ) + .await } /// Resolve the external environment and hand the capture snapshot to the @@ -324,6 +359,10 @@ impl Cmd { P: Provider + Clone + std::fmt::Debug, { let result = self.execute(provider, rctx, external_envs).await?; + // Read the verdict before `result.fixture` is moved below; the mismatch + // is reported after every artifact has been written, so a failing + // verification never costs the user the output it was derived from. + let mismatched = result.verification.as_ref().is_some_and(|v| !v.matched); self.output_results(&result)?; // Write the self-validating fixture (re-executes the isolated unit through // state-test and cross-checks it against the replay before writing). @@ -331,6 +370,9 @@ impl Cmd { super::fixture::finalize_and_write(draft, path)?; info!(path = %path.display(), "Wrote self-validating fixture"); } + if mismatched { + return Err(ReplayError::VerificationMismatch { mismatched: 1, total: 1 }); + } Ok(()) } @@ -582,6 +624,23 @@ impl Cmd { None }; + // For `--verify-receipt`, fetch the on-chain receipt here — before the + // executor borrows the database, and with the same call shape the + // fixture path uses — so `--rpc.capture-file` records it and a later + // offline run verifies without network access. It is compared against + // the replay's own receipt once the block is finished. + let onchain_receipt = if self.verify_receipt { + let receipt = verify::fetch_receipt(provider, ctx.tx_hash).await?; + // A receipt describing a different inclusion than the replayed block + // would compare the replay against the wrong on-chain execution: + // that is an infrastructure failure, not a verification mismatch. + verify::check_inclusion(receipt.block_hash(), ctx.block.hash()) + .map_err(ReplayError::RpcError)?; + Some(receipt) + } else { + None + }; + let evm_factory = MegaEvmFactory::new().with_external_env_factory(external_envs); let block_executor_factory = MegaBlockExecutorFactory::new( &hardforks, @@ -763,6 +822,16 @@ impl Cmd { ctx.preceding_tx_hashes.len() as u64, ); + let verification = onchain_receipt.as_ref().map(|onchain| { + verify::compare( + &verify::ReceiptFacts::from_receipt(&onchain.inner), + &verify::ReceiptFacts::from_receipt(&receipt), + ) + }); + if let Some(verification) = &verification { + debug!(matched = verification.matched, "On-chain receipt verified"); + } + Ok(ReplayOutcome { outcome: EvmeOutcome { pre_execution_nonce, @@ -773,6 +842,7 @@ impl Cmd { }, receipt, fixture, + verification, }) } @@ -787,6 +857,9 @@ impl Cmd { summary.fill_trace_and_dump(&result.outcome, &self.trace_args, &self.dump_args)?; summary.receipt = Some(serde_json::to_value(&result.receipt).expect("failed to serialize receipt")); + summary.verification = result.verification.as_ref().map(|verification| { + serde_json::to_value(verification).expect("failed to serialize verification") + }); println!( "{}", serde_json::to_string_pretty(&summary).expect("failed to serialize output") @@ -798,6 +871,10 @@ impl Cmd { result.outcome.exec_time, ); print_receipt(&result.receipt); + if let Some(verification) = &result.verification { + println!(); + println!("{}", verification.verdict_line()); + } print_execution_trace( result.outcome.trace_data.as_deref(), self.trace_args.trace_output_file.as_deref(), @@ -958,6 +1035,25 @@ mod tests { } } + /// `--verify-receipt` is a whole-corpus flag: both replay modes take it. + #[test] + fn test_verify_receipt_is_accepted_in_both_modes() { + for extra in [vec![TX], vec!["--block", "1"], vec!["--tx-file", "/tmp/list.txt"]] { + let mut argv = extra.clone(); + argv.push("--verify-receipt"); + let cmd = parse(&argv).expect("--verify-receipt should parse"); + assert!(cmd.verify_receipt, "the flag must be recorded for {extra:?}"); + cmd.validate() + .unwrap_or_else(|e| panic!("--verify-receipt must be accepted for {extra:?}: {e}")); + } + } + + /// The flag defaults to off, so a replay without it does no receipt fetch. + #[test] + fn test_verify_receipt_defaults_to_off() { + assert!(!parse(&[TX]).expect("parse").verify_receipt); + } + #[test] fn test_batch_accepts_the_flags_it_supports() { parse(&["--block", "1", "--json"]).expect("parse").validate().expect("--json is allowed"); diff --git a/bin/mega-evme/src/replay/mod.rs b/bin/mega-evme/src/replay/mod.rs index ab8d0522..fdcc03f0 100644 --- a/bin/mega-evme/src/replay/mod.rs +++ b/bin/mega-evme/src/replay/mod.rs @@ -7,6 +7,7 @@ mod batch; mod cmd; mod fixture; mod hardforks; +mod verify; pub use cmd::Cmd; pub use hardforks::*; diff --git a/bin/mega-evme/src/replay/verify.rs b/bin/mega-evme/src/replay/verify.rs new file mode 100644 index 00000000..4c920219 --- /dev/null +++ b/bin/mega-evme/src/replay/verify.rs @@ -0,0 +1,613 @@ +//! Compare a local replay against the transaction's on-chain receipt. +//! +//! `mega-evme replay --verify-receipt` fetches the on-chain receipt of every +//! replayed target and checks that the local execution reproduces it. The +//! comparison is a pure function over [`ReceiptFacts`] — the consensus facts +//! both sides carry — so it is independent of how either receipt was obtained +//! and testable without a provider. +//! +//! Anything that prevents the comparison from running at all (a receipt the +//! endpoint cannot serve, or a receipt describing a different inclusion than the +//! replayed block) is an infrastructure failure, never a mismatch: a target that +//! could not be verified must not be reported as a divergence. + +use core::fmt; + +use alloy_consensus::TxReceipt; +use alloy_primitives::{Address, Bytes, Log, B256}; +use alloy_provider::Provider; +use alloy_rpc_types_eth::{Log as RpcLog, TransactionReceipt}; +use op_alloy_rpc_types::OpTransactionReceipt; +use serde::Serialize; + +use super::{ReplayError, Result}; + +/// The consensus facts compared between the on-chain receipt and the receipt +/// the local replay produced. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct ReceiptFacts { + /// Whether the transaction succeeded. + pub status: bool, + /// Gas the transaction used. + pub gas_used: u64, + /// The consensus logs the transaction emitted, in order. + pub logs: Vec, +} + +impl ReceiptFacts { + /// Extract the compared facts from a receipt envelope. + /// + /// Both sides go through this one accessor set — the on-chain side is the + /// RPC receipt's inner envelope, the local side the envelope the replay + /// built — so neither side can be read with different semantics. + pub(super) fn from_receipt(receipt: &TransactionReceipt) -> Self + where + T: TxReceipt, + { + Self { + status: receipt.inner.status(), + gas_used: receipt.gas_used, + logs: receipt.logs().iter().map(|log| log.inner.clone()).collect(), + } + } +} + +/// The verdict for one verified transaction. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(super) struct VerificationOutcome { + /// Whether the local replay reproduced the on-chain receipt. + #[serde(rename = "match")] + pub matched: bool, + /// The mismatched dimensions; absent when the replay matched. + #[serde(skip_serializing_if = "Option::is_none")] + pub diff: Option, +} + +impl VerificationOutcome { + /// The one-line human verdict printed for a verified transaction. + pub(super) fn verdict_line(&self) -> String { + match &self.diff { + None => "verification: MATCH".to_string(), + Some(diff) => format!("verification: MISMATCH ({})", diff.describe()), + } + } +} + +/// The mismatched dimensions of a verification. Dimensions that agree are +/// absent, so a diff never has to be scanned for "everything equal" entries. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub(super) struct VerificationDiff { + /// Present when the success flags differ. + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option>, + /// Present when the gas used differs. + #[serde(skip_serializing_if = "Option::is_none")] + pub gas_used: Option>, + /// Present when the emitted logs differ. + #[serde(skip_serializing_if = "Option::is_none")] + pub logs: Option, +} + +impl VerificationDiff { + /// Whether every compared dimension agreed. + fn is_empty(&self) -> bool { + self.status.is_none() && self.gas_used.is_none() && self.logs.is_none() + } + + /// Render every mismatched dimension as one comma-separated line. + fn describe(&self) -> String { + let mut parts = Vec::new(); + if let Some(m) = &self.status { + parts.push(format!("status: onchain {} vs replay {}", m.onchain, m.replay)); + } + if let Some(m) = &self.gas_used { + parts.push(format!("gas_used: onchain {} vs replay {}", m.onchain, m.replay)); + } + if let Some(logs) = &self.logs { + if let Some(m) = &logs.count { + parts.push(format!("logs_count: onchain {} vs replay {}", m.onchain, m.replay)); + } + if let Some(m) = &logs.first_mismatch { + parts.push(format!( + "logs[{}].{}: onchain {} vs replay {}", + m.index, + m.field.as_str(), + m.onchain, + m.replay, + )); + } + } + parts.join(", ") + } +} + +/// One dimension's two values. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(super) struct Mismatch { + /// The value the on-chain receipt reports. + pub onchain: T, + /// The value the local replay produced. + pub replay: T, +} + +/// How the emitted logs differ. +/// +/// A differing log count and a differing log field are independent findings: +/// both are reported when both apply, so truncated logs and rewritten logs are +/// distinguishable. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +pub(super) struct LogsDiff { + /// Present when the two sides emitted a different number of logs. + #[serde(skip_serializing_if = "Option::is_none")] + pub count: Option>, + /// The first log both sides emitted whose contents differ, if any. + #[serde(skip_serializing_if = "Option::is_none")] + pub first_mismatch: Option, +} + +impl LogsDiff { + /// Whether the logs agreed. + fn is_empty(&self) -> bool { + self.count.is_none() && self.first_mismatch.is_none() + } +} + +/// The first differing field of the first differing log. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(super) struct LogFieldMismatch { + /// Position of the log in the transaction's log list. + pub index: usize, + /// Which field of the log differs. + pub field: LogField, + /// That field's value in the on-chain receipt. + pub onchain: LogFieldValue, + /// That field's value in the local replay. + pub replay: LogFieldValue, +} + +/// The log field a [`LogFieldMismatch`] reports on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub(super) enum LogField { + /// The emitting contract's address. + Address, + /// The indexed topics. + Topics, + /// The unindexed data payload. + Data, +} + +impl LogField { + /// Wire name, shared by the JSON diff and the human verdict line. + const fn as_str(self) -> &'static str { + match self { + Self::Address => "address", + Self::Topics => "topics", + Self::Data => "data", + } + } +} + +/// The value of the log field named by a [`LogFieldMismatch`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(untagged)] +pub(super) enum LogFieldValue { + /// An emitting contract address. + Address(Address), + /// A topic list. + Topics(Vec), + /// A data payload. + Data(Bytes), +} + +impl fmt::Display for LogFieldValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Address(address) => write!(f, "{address}"), + Self::Topics(topics) => { + write!(f, "[")?; + for (index, topic) in topics.iter().enumerate() { + if index > 0 { + write!(f, ", ")?; + } + write!(f, "{topic}")?; + } + write!(f, "]") + } + Self::Data(data) => write!(f, "{data}"), + } + } +} + +/// Compare the on-chain receipt against the local replay's receipt. +pub(super) fn compare(onchain: &ReceiptFacts, replay: &ReceiptFacts) -> VerificationOutcome { + let mut diff = VerificationDiff::default(); + + if onchain.status != replay.status { + diff.status = Some(Mismatch { onchain: onchain.status, replay: replay.status }); + } + if onchain.gas_used != replay.gas_used { + diff.gas_used = Some(Mismatch { onchain: onchain.gas_used, replay: replay.gas_used }); + } + let logs = compare_logs(&onchain.logs, &replay.logs); + if !logs.is_empty() { + diff.logs = Some(logs); + } + + if diff.is_empty() { + VerificationOutcome { matched: true, diff: None } + } else { + VerificationOutcome { matched: false, diff: Some(diff) } + } +} + +/// Compare two log lists: their length, and the contents of the logs both sides +/// emitted. +fn compare_logs(onchain: &[Log], replay: &[Log]) -> LogsDiff { + let count = (onchain.len() != replay.len()) + .then_some(Mismatch { onchain: onchain.len(), replay: replay.len() }); + // Only the logs both sides emitted can be compared field by field; a length + // difference is already reported by `count`. + let first_mismatch = onchain + .iter() + .zip(replay) + .enumerate() + .find_map(|(index, (onchain, replay))| compare_log(index, onchain, replay)); + LogsDiff { count, first_mismatch } +} + +/// Report the first differing field of one log, if any. +fn compare_log(index: usize, onchain: &Log, replay: &Log) -> Option { + if onchain.address != replay.address { + return Some(LogFieldMismatch { + index, + field: LogField::Address, + onchain: LogFieldValue::Address(onchain.address), + replay: LogFieldValue::Address(replay.address), + }); + } + if onchain.topics() != replay.topics() { + return Some(LogFieldMismatch { + index, + field: LogField::Topics, + onchain: LogFieldValue::Topics(onchain.topics().to_vec()), + replay: LogFieldValue::Topics(replay.topics().to_vec()), + }); + } + if onchain.data.data != replay.data.data { + return Some(LogFieldMismatch { + index, + field: LogField::Data, + onchain: LogFieldValue::Data(onchain.data.data.clone()), + replay: LogFieldValue::Data(replay.data.data.clone()), + }); + } + None +} + +/// Fetch a transaction's on-chain receipt. +/// +/// Uses the same call shape as the `--dump-fixture` path, so a run with +/// `--rpc.capture-file` records the receipt and a later offline run verifies +/// without network access. +/// +/// A receipt the endpoint cannot serve — a transport failure, or a receipt +/// pruned below the endpoint's retention height — is an [`ReplayError::RpcError`] +/// so the target is reported as unverified rather than as a mismatch. +pub(super) async fn fetch_receipt

(provider: &P, tx_hash: B256) -> Result +where + P: Provider, +{ + provider + .get_transaction_receipt(tx_hash) + .await + .map_err(|e| ReplayError::RpcError(format!("Failed to fetch receipt: {e}")))? + .ok_or_else(|| { + ReplayError::RpcError(format!( + "No on-chain receipt for transaction {tx_hash}: the transaction is unknown to \ + the endpoint, or the endpoint has pruned its receipt" + )) + }) +} + +/// Check that a fetched receipt describes the block the replay executed. +/// +/// Across a reorg, or against a load-balanced endpoint serving divergent views, +/// the receipt can describe a different inclusion than the block the replay ran, +/// which would compare the replay against the wrong on-chain execution. Returns +/// the explanatory message so each mode can wrap it in the error shape it +/// reports — a hard error in single-transaction mode, an `rpc` error entry in +/// batch mode. +pub(super) fn check_inclusion( + receipt_block_hash: Option, + replayed_block_hash: B256, +) -> std::result::Result<(), String> { + match receipt_block_hash { + Some(hash) if hash != replayed_block_hash => Err(format!( + "receipt block hash {hash} != replayed block hash {replayed_block_hash}: the receipt \ + describes a different inclusion than the replayed block (reorg in progress, or a \ + load-balanced endpoint serving divergent views); the transaction is unverified, \ + retry once the chain settles" + )), + _ => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{address, b256, LogData}; + + const ADDR_A: Address = address!("0x00000000000000000000000000000000000000aa"); + const ADDR_B: Address = address!("0x00000000000000000000000000000000000000bb"); + const TOPIC_A: B256 = + b256!("0x000000000000000000000000000000000000000000000000000000000000000a"); + const TOPIC_B: B256 = + b256!("0x000000000000000000000000000000000000000000000000000000000000000b"); + + /// Parse a log's hex-encoded data payload. + fn data(hex: &str) -> Bytes { + hex.parse().expect("valid hex payload") + } + + /// Build a log from its three compared fields. + fn log(address: Address, topics: &[B256], data: Bytes) -> Log { + Log { + address, + data: LogData::new(topics.to_vec(), data).expect("topic count within bounds"), + } + } + + /// A successful 21,000-gas receipt emitting the given logs. + fn facts(logs: Vec) -> ReceiptFacts { + ReceiptFacts { status: true, gas_used: 21_000, logs } + } + + /// The `diff` of an outcome that must be a mismatch. + fn diff_of(outcome: &VerificationOutcome) -> &VerificationDiff { + assert!(!outcome.matched, "expected a mismatch, got {outcome:?}"); + outcome.diff.as_ref().expect("a mismatch always carries a diff") + } + + /// Serialize an outcome the way the JSON output does. + fn json(outcome: &VerificationOutcome) -> serde_json::Value { + serde_json::to_value(outcome).expect("outcome is serializable") + } + + #[test] + fn test_compare_equal_receipts_match() { + let onchain = facts(vec![log(ADDR_A, &[TOPIC_A], data("0xdeadbeef"))]); + let replay = onchain.clone(); + + let outcome = compare(&onchain, &replay); + + assert!(outcome.matched); + assert!(outcome.diff.is_none(), "a match carries no diff"); + assert_eq!(json(&outcome), serde_json::json!({ "match": true })); + assert_eq!(outcome.verdict_line(), "verification: MATCH"); + } + + #[test] + fn test_compare_empty_logs_on_both_sides_match() { + let outcome = compare(&facts(vec![]), &facts(vec![])); + + assert!(outcome.matched); + assert_eq!(json(&outcome), serde_json::json!({ "match": true })); + } + + #[test] + fn test_compare_reports_status_flip() { + let onchain = facts(vec![]); + let replay = ReceiptFacts { status: false, ..facts(vec![]) }; + + let outcome = compare(&onchain, &replay); + + let diff = diff_of(&outcome); + assert_eq!(diff.status, Some(Mismatch { onchain: true, replay: false })); + assert!(diff.gas_used.is_none(), "gas agreed, so it must be absent: {diff:?}"); + assert!(diff.logs.is_none(), "logs agreed, so they must be absent: {diff:?}"); + assert_eq!( + json(&outcome), + serde_json::json!({ + "match": false, + "diff": { "status": { "onchain": true, "replay": false } }, + }) + ); + assert_eq!( + outcome.verdict_line(), + "verification: MISMATCH (status: onchain true vs replay false)" + ); + } + + #[test] + fn test_compare_reports_gas_delta() { + let onchain = facts(vec![]); + let replay = ReceiptFacts { gas_used: 22_000, ..facts(vec![]) }; + + let outcome = compare(&onchain, &replay); + + let diff = diff_of(&outcome); + assert_eq!(diff.gas_used, Some(Mismatch { onchain: 21_000, replay: 22_000 })); + assert!(diff.status.is_none(), "status agreed, so it must be absent: {diff:?}"); + assert_eq!( + json(&outcome), + serde_json::json!({ + "match": false, + "diff": { "gas_used": { "onchain": 21000, "replay": 22000 } }, + }) + ); + assert_eq!( + outcome.verdict_line(), + "verification: MISMATCH (gas_used: onchain 21000 vs replay 22000)" + ); + } + + #[test] + fn test_compare_reports_log_count_delta() { + let entry = log(ADDR_A, &[TOPIC_A], data("0x")); + let onchain = facts(vec![entry.clone(), entry.clone()]); + let replay = facts(vec![entry]); + + let outcome = compare(&onchain, &replay); + + let logs = diff_of(&outcome).logs.as_ref().expect("logs differ"); + assert_eq!(logs.count, Some(Mismatch { onchain: 2, replay: 1 })); + assert!( + logs.first_mismatch.is_none(), + "the shared prefix is identical, so no field mismatch: {logs:?}" + ); + assert_eq!( + json(&outcome), + serde_json::json!({ + "match": false, + "diff": { "logs": { "count": { "onchain": 2, "replay": 1 } } }, + }) + ); + } + + #[test] + fn test_compare_reports_log_address_delta() { + let onchain = facts(vec![log(ADDR_A, &[TOPIC_A], data("0x"))]); + let replay = facts(vec![log(ADDR_B, &[TOPIC_A], data("0x"))]); + + let outcome = compare(&onchain, &replay); + + let logs = diff_of(&outcome).logs.as_ref().expect("logs differ"); + assert!(logs.count.is_none(), "both sides emitted one log: {logs:?}"); + assert_eq!( + logs.first_mismatch, + Some(LogFieldMismatch { + index: 0, + field: LogField::Address, + onchain: LogFieldValue::Address(ADDR_A), + replay: LogFieldValue::Address(ADDR_B), + }) + ); + assert_eq!( + json(&outcome)["diff"]["logs"]["first_mismatch"], + serde_json::json!({ + "index": 0, + "field": "address", + "onchain": "0x00000000000000000000000000000000000000aa", + "replay": "0x00000000000000000000000000000000000000bb", + }) + ); + } + + #[test] + fn test_compare_reports_log_topics_delta() { + let onchain = facts(vec![log(ADDR_A, &[TOPIC_A], data("0x"))]); + let replay = facts(vec![log(ADDR_A, &[TOPIC_A, TOPIC_B], data("0x"))]); + + let outcome = compare(&onchain, &replay); + + let first = diff_of(&outcome).logs.as_ref().and_then(|l| l.first_mismatch.clone()); + assert_eq!( + first, + Some(LogFieldMismatch { + index: 0, + field: LogField::Topics, + onchain: LogFieldValue::Topics(vec![TOPIC_A]), + replay: LogFieldValue::Topics(vec![TOPIC_A, TOPIC_B]), + }) + ); + assert_eq!(json(&outcome)["diff"]["logs"]["first_mismatch"]["field"], "topics"); + } + + #[test] + fn test_compare_reports_log_data_delta() { + let onchain = facts(vec![log(ADDR_A, &[TOPIC_A], data("0xdeadbeef"))]); + let replay = facts(vec![log(ADDR_A, &[TOPIC_A], data("0xfeedface"))]); + + let outcome = compare(&onchain, &replay); + + let first = diff_of(&outcome).logs.as_ref().and_then(|l| l.first_mismatch.clone()); + assert_eq!( + first, + Some(LogFieldMismatch { + index: 0, + field: LogField::Data, + onchain: LogFieldValue::Data(data("0xdeadbeef")), + replay: LogFieldValue::Data(data("0xfeedface")), + }) + ); + assert_eq!( + json(&outcome)["diff"]["logs"]["first_mismatch"], + serde_json::json!({ + "index": 0, + "field": "data", + "onchain": "0xdeadbeef", + "replay": "0xfeedface", + }) + ); + } + + /// The reported log mismatch is the first differing one, and a later + /// difference does not displace it. + #[test] + fn test_compare_reports_the_first_differing_log() { + let same = log(ADDR_A, &[TOPIC_A], data("0x")); + let onchain = facts(vec![same.clone(), same.clone(), same.clone()]); + let replay = facts(vec![ + same, + log(ADDR_B, &[TOPIC_A], data("0x")), + log(ADDR_A, &[TOPIC_A], data("0xff")), + ]); + + let outcome = compare(&onchain, &replay); + + let first = diff_of(&outcome).logs.as_ref().and_then(|l| l.first_mismatch.clone()); + assert_eq!(first.map(|m| (m.index, m.field)), Some((1, LogField::Address))); + } + + /// Every mismatched dimension is reported at once — a status flip does not + /// hide the gas delta or the log difference behind it. + #[test] + fn test_compare_reports_all_mismatched_dimensions() { + let onchain = facts(vec![log(ADDR_A, &[TOPIC_A], data("0x"))]); + let replay = ReceiptFacts { + status: false, + gas_used: 30_000, + logs: vec![log(ADDR_B, &[TOPIC_A], data("0x")), log(ADDR_A, &[], data("0x"))], + }; + + let outcome = compare(&onchain, &replay); + + let diff = diff_of(&outcome); + assert!(diff.status.is_some() && diff.gas_used.is_some()); + let logs = diff.logs.as_ref().expect("logs differ"); + assert_eq!(logs.count, Some(Mismatch { onchain: 1, replay: 2 })); + assert_eq!(logs.first_mismatch.as_ref().map(|m| m.field), Some(LogField::Address)); + assert_eq!( + outcome.verdict_line(), + format!( + "verification: MISMATCH (status: onchain true vs replay false, \ + gas_used: onchain 21000 vs replay 30000, logs_count: onchain 1 vs replay 2, \ + logs[0].address: onchain {ADDR_A} vs replay {ADDR_B})" + ) + ); + } + + #[test] + fn test_check_inclusion_accepts_the_replayed_block() { + let hash = b256!("0x1111111111111111111111111111111111111111111111111111111111111111"); + + assert!(check_inclusion(Some(hash), hash).is_ok()); + // A receipt without a block hash cannot contradict the replayed block. + assert!(check_inclusion(None, hash).is_ok()); + } + + #[test] + fn test_check_inclusion_rejects_a_different_inclusion() { + let message = check_inclusion( + Some(b256!("0x1111111111111111111111111111111111111111111111111111111111111111")), + b256!("0x2222222222222222222222222222222222222222222222222222222222222222"), + ) + .expect_err("a receipt from another block must be rejected"); + + assert!( + message.contains("different inclusion") && message.contains("unverified"), + "message must explain the reorg and that the target is unverified: {message}" + ); + } +} diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 35425b2b..e76643ab 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -182,6 +182,29 @@ fn test_replay_tx_file_reports_unresolved_targets_and_exits_nonzero() { assert_eq!(lines[1]["success"].as_bool(), Some(true), "the resolvable target still replays"); } +/// `--verify-receipt` against an envelope that carries no receipts: every target +/// becomes an `rpc` error entry (unverified), never a mismatch, and the run +/// exits non-zero. +/// +/// The development envelope is captured by replays, which do not fetch receipts, +/// so this pins the endpoint-cannot-serve-the-receipt path end to end. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_block_verify_receipt_without_receipts_reports_rpc_errors() { + let stdout = replay(&["--block", &BLOCK.to_string(), "--verify-receipt", "--json"], false); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + for line in &lines { + assert_eq!( + line["error"]["kind"].as_str(), + Some("rpc"), + "a receipt the envelope cannot serve is an infrastructure error: {line}" + ); + assert!(line.get("verification").is_none(), "an unverified target carries no verdict"); + } +} + /// Batch mode rejects the single-transaction-only flags before doing any work. #[test] #[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs new file mode 100644 index 00000000..310aec8f --- /dev/null +++ b/bin/mega-evme/tests/replay_verify.rs @@ -0,0 +1,381 @@ +//! Integration tests for `mega-evme replay --verify-receipt`: the end-to-end +//! comparison against the on-chain receipt in single-transaction and batch mode. +//! +//! They run fully offline against the committed RPC capture that carries the +//! on-chain receipt (`fixtures/replay_offline.cache.json`), so they are +//! deterministic. Mismatch and infrastructure cases are produced by doctoring a +//! copy of that capture: its entries are keyed by the request, not the response, +//! so a doctored response still resolves. + +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +/// Offline RPC capture, including the transaction's on-chain receipt. +const CACHE: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/replay_offline.cache.json"); + +/// The transaction captured in `CACHE` (a 75,514-gas Rex5 mainnet call). +const TX: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; + +/// Gas the transaction used on-chain, which a faithful replay reproduces. +const GAS_USED: u64 = 75_514; + +/// Outcome of one `mega-evme replay` invocation. +struct Run { + success: bool, + stdout: String, + stderr: String, +} + +impl Run { + /// Parse the stdout of a `--json` single-transaction run. + fn json(&self) -> serde_json::Value { + serde_json::from_str(self.stdout.trim()) + .unwrap_or_else(|e| panic!("stdout is not JSON ({e}):\n{}", self.stdout)) + } + + /// Parse the stdout of a `--json` batch run as one value per NDJSON line. + fn ndjson(&self) -> Vec { + self.stdout + .lines() + .map(|line| { + serde_json::from_str(line) + .unwrap_or_else(|e| panic!("stdout line is not compact JSON ({e}): {line}")) + }) + .collect() + } +} + +/// Run `replay` offline against `cache`. +fn replay(cache: &Path, args: &[&str]) -> Run { + let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args(["replay", "--rpc.replay-file", cache.to_str().expect("cache path is utf-8")]) + .args(args) + .output() + .expect("failed to run mega-evme"); + Run { + success: output.status.success(), + stdout: String::from_utf8(output.stdout).expect("stdout is utf-8"), + stderr: String::from_utf8(output.stderr).expect("stderr is utf-8"), + } +} + +/// The committed capture, unmodified. +fn cache() -> PathBuf { + PathBuf::from(CACHE) +} + +/// A temp path unique to this process and this test. +fn temp_path(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("mega_evme_verify_{name}_{}.json", std::process::id())) +} + +/// Write a copy of the committed capture whose receipt response is rewritten by +/// `doctor`, and return its path. +fn doctored_cache(name: &str, doctor: impl Fn(&mut serde_json::Value)) -> PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + .expect("parse offline cache"); + let mut doctored = false; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + // The receipt is the only cached response carrying cumulativeGasUsed. + if !value.contains("cumulativeGasUsed") { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse receipt response"); + doctor(&mut response["result"]); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored = true; + } + assert!(doctored, "offline cache should contain the receipt entry"); + + let path = temp_path(name); + std::fs::write(&path, envelope.to_string()).expect("write doctored cache"); + path +} + +/// Write a copy of the committed capture with the receipt dropped entirely, +/// modelling an endpoint that has pruned it. +fn cache_without_receipt(name: &str) -> PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + .expect("parse offline cache"); + let entries = envelope["cache"].as_array_mut().expect("cache entries"); + let before = entries.len(); + entries.retain(|entry| { + !entry["value"].as_str().expect("entry value is a string").contains("cumulativeGasUsed") + }); + assert!(entries.len() < before, "offline cache should contain the receipt entry"); + + let path = temp_path(name); + std::fs::write(&path, envelope.to_string()).expect("write pruned cache"); + path +} + +/// Write a `--tx-file` holding the single captured transaction. +fn tx_file(name: &str) -> PathBuf { + let path = + std::env::temp_dir().join(format!("mega_evme_verify_{name}_{}.txt", std::process::id())); + std::fs::write(&path, format!("{TX}\n")).expect("write tx list"); + path +} + +/// A faithful replay reproduces the on-chain receipt, reports a match, and exits 0. +#[test] +fn test_verify_receipt_reports_a_match() { + let run = replay(&cache(), &["--verify-receipt", "--json", TX]); + + assert!(run.success, "a matching verification must exit 0.\nstderr: {}", run.stderr); + assert_eq!(run.json()["verification"], serde_json::json!({ "match": true })); +} + +/// Human-readable output carries one verdict line per transaction. +#[test] +fn test_verify_receipt_prints_a_human_verdict_line() { + let run = replay(&cache(), &["--verify-receipt", TX]); + + assert!(run.success, "a matching verification must exit 0.\nstderr: {}", run.stderr); + assert!( + run.stdout.contains("verification: MATCH"), + "expected a verdict line, got stdout:\n{}", + run.stdout + ); +} + +/// Without the flag the single-transaction JSON is unchanged: no `verification` +/// key, and the flag adds that key and nothing else. +#[test] +fn test_single_transaction_json_is_unchanged_without_the_flag() { + let plain = replay(&cache(), &["--json", TX]); + let verified = replay(&cache(), &["--verify-receipt", "--json", TX]); + + assert!(plain.success && verified.success, "both runs must exit 0"); + assert!( + !plain.stdout.contains("verification"), + "output without the flag must not mention verification:\n{}", + plain.stdout + ); + assert!(plain.json().get("verification").is_none(), "the key must be absent without the flag"); + + let mut stripped = verified.json(); + stripped.as_object_mut().expect("summary is an object").remove("verification"); + assert_eq!(stripped, plain.json(), "--verify-receipt must add the verdict and nothing else"); +} + +/// A gas divergence is reported as a `gas_used` diff and fails the run. +#[test] +fn test_verify_receipt_reports_a_gas_mismatch() { + let path = doctored_cache("gas", |receipt| receipt["gasUsed"] = "0x1".into()); + + let run = replay(&path, &["--verify-receipt", "--json", TX]); + let _ = std::fs::remove_file(&path); + + assert!(!run.success, "a mismatch must exit non-zero"); + assert_eq!( + run.json()["verification"], + serde_json::json!({ + "match": false, + "diff": { "gas_used": { "onchain": 1, "replay": GAS_USED } }, + }) + ); + assert!( + run.stderr.contains("Receipt verification mismatch"), + "expected the mismatch error, got stderr:\n{}", + run.stderr + ); +} + +/// A status divergence is reported on its own, without dragging in the +/// dimensions that agreed. +#[test] +fn test_verify_receipt_reports_a_status_mismatch() { + let path = doctored_cache("status", |receipt| receipt["status"] = "0x0".into()); + + let run = replay(&path, &["--verify-receipt", "--json", TX]); + let _ = std::fs::remove_file(&path); + + assert!(!run.success, "a mismatch must exit non-zero"); + assert_eq!( + run.json()["verification"], + serde_json::json!({ + "match": false, + "diff": { "status": { "onchain": false, "replay": true } }, + }) + ); +} + +/// A log divergence is reported under `logs`. +#[test] +fn test_verify_receipt_reports_a_log_mismatch() { + let path = doctored_cache("logs", |receipt| { + receipt["logs"] = serde_json::json!([{ + "address": "0x00000000000000000000000000000000000000aa", + "topics": ["0x000000000000000000000000000000000000000000000000000000000000000a"], + "data": "0xdeadbeef", + "blockHash": receipt["blockHash"], + "blockNumber": receipt["blockNumber"], + "transactionHash": receipt["transactionHash"], + "transactionIndex": receipt["transactionIndex"], + "logIndex": "0x0", + "removed": false, + }]); + }); + + let run = replay(&path, &["--verify-receipt", "--json", TX]); + let _ = std::fs::remove_file(&path); + + assert!(!run.success, "a mismatch must exit non-zero"); + assert_eq!( + run.json()["verification"], + serde_json::json!({ + "match": false, + "diff": { "logs": { "count": { "onchain": 1, "replay": 0 } } }, + }) + ); +} + +/// A receipt describing a different inclusion than the replayed block is an +/// infrastructure failure: the transaction is unverified, never mismatched. +#[test] +fn test_verify_receipt_reorg_is_an_infrastructure_error() { + let path = doctored_cache("reorg", |receipt| { + receipt["blockHash"] = + "0x1111111111111111111111111111111111111111111111111111111111111111".into(); + }); + + let run = replay(&path, &["--verify-receipt", "--json", TX]); + let _ = std::fs::remove_file(&path); + + assert!(!run.success, "a receipt from another block must fail the run"); + assert!( + run.stderr.contains("different inclusion"), + "expected the reorg/divergent-endpoint hint, got stderr:\n{}", + run.stderr + ); + assert!( + !run.stderr.contains("verification mismatch") && !run.stdout.contains("MISMATCH"), + "an unverifiable transaction must not be reported as a mismatch:\n{}\n{}", + run.stdout, + run.stderr, + ); +} + +/// A receipt the endpoint cannot serve (e.g. pruned below its retention height) +/// is an infrastructure failure, not a mismatch. +#[test] +fn test_verify_receipt_missing_receipt_is_an_infrastructure_error() { + let path = cache_without_receipt("pruned"); + + let run = replay(&path, &["--verify-receipt", "--json", TX]); + let _ = std::fs::remove_file(&path); + + assert!(!run.success, "an unavailable receipt must fail the run"); + assert!( + run.stderr.contains("receipt"), + "expected an error naming the receipt, got stderr:\n{}", + run.stderr + ); + assert!( + !run.stderr.contains("verification mismatch") && !run.stdout.contains("MISMATCH"), + "an unverifiable transaction must not be reported as a mismatch:\n{}\n{}", + run.stdout, + run.stderr, + ); +} + +/// Batch mode carries the verdict on the transaction's NDJSON line. +#[test] +fn test_batch_verify_receipt_reports_a_match() { + let list = tx_file("batch_match"); + + let run = + replay(&cache(), &["--tx-file", list.to_str().unwrap(), "--verify-receipt", "--json"]); + let _ = std::fs::remove_file(&list); + + assert!(run.success, "a matching verification must exit 0.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "one line per requested transaction"); + assert_eq!(lines[0]["tx_hash"].as_str(), Some(TX)); + assert_eq!(lines[0]["verification"], serde_json::json!({ "match": true })); +} + +/// A batch mismatch keeps the result line (with its diff) and fails the run +/// through the dedicated verification error. +#[test] +fn test_batch_verify_receipt_reports_a_mismatch_and_exits_nonzero() { + let path = doctored_cache("batch_gas", |receipt| receipt["gasUsed"] = "0x1".into()); + let list = tx_file("batch_gas"); + + let run = replay(&path, &["--tx-file", list.to_str().unwrap(), "--verify-receipt", "--json"]); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + + assert!(!run.success, "a mismatch must exit non-zero"); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "a mismatch is still a result line, not an error entry"); + assert!(lines[0].get("error").is_none(), "a mismatch is not an infrastructure error"); + assert_eq!( + lines[0]["verification"], + serde_json::json!({ + "match": false, + "diff": { "gas_used": { "onchain": 1, "replay": GAS_USED } }, + }) + ); + assert!( + run.stderr.contains("Receipt verification mismatch"), + "expected the mismatch error, got stderr:\n{}", + run.stderr + ); +} + +/// In batch mode an unavailable receipt turns the target into an `rpc` error +/// entry — reported as unverified, never as a mismatch. +#[test] +fn test_batch_verify_receipt_missing_receipt_is_an_rpc_error_entry() { + let path = cache_without_receipt("batch_pruned"); + let list = tx_file("batch_pruned"); + + let run = replay(&path, &["--tx-file", list.to_str().unwrap(), "--verify-receipt", "--json"]); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + + assert!(!run.success, "an unverified target must exit non-zero"); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "one line per requested transaction"); + assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); + assert!(lines[0].get("verification").is_none(), "an unverified target carries no verdict"); + assert!( + !run.stderr.contains("verification mismatch"), + "an unverifiable target must not fail as a mismatch:\n{}", + run.stderr + ); +} + +/// The reorg guard applies in batch mode too, as an `rpc` error entry. +#[test] +fn test_batch_verify_receipt_reorg_is_an_rpc_error_entry() { + let path = doctored_cache("batch_reorg", |receipt| { + receipt["blockHash"] = + "0x1111111111111111111111111111111111111111111111111111111111111111".into(); + }); + let list = tx_file("batch_reorg"); + + let run = replay(&path, &["--tx-file", list.to_str().unwrap(), "--verify-receipt", "--json"]); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + + assert!(!run.success, "an unverified target must exit non-zero"); + let lines = run.ndjson(); + assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); + assert!( + lines[0]["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("different inclusion")), + "expected the reorg/divergent-endpoint hint: {}", + lines[0] + ); +} diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 56575c17..50bfec5b 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -1,5 +1,5 @@ --- -description: Fetch and re-execute one or many on-chain transactions with optional overrides and tracing. +description: Fetch and re-execute one or many on-chain transactions with optional overrides, tracing, and on-chain receipt verification. --- # replay @@ -111,6 +111,8 @@ Execution outcomes are not errors: a reverted or halted transaction is a normal Without `--json`, each transaction is printed with a header naming its hash, block, and index, followed by the same summary and receipt the single-transaction mode prints. A final one-line summary (transactions replayed, transactions failed, elapsed time) is logged at `INFO` level, so pass `-vvv` to see it. +With [`--verify-receipt`](#receipt-verification), each result line additionally carries a `verification` object. + ### Exit Status A batch run exits `0` when every requested transaction produced an execution result, and `1` when any of them produced an error entry. @@ -144,6 +146,118 @@ Count the transactions that did not succeed: jq -c 'select(.error != null or .success == false)' results.ndjson | wc -l ``` +## Receipt Verification + +Replaying a transaction only proves that the local EVM produced _some_ result; equivalence verification needs that result checked against what the chain recorded. +`--verify-receipt` builds that check into the tool: it fetches the on-chain receipt of every replayed transaction and compares it against the receipt the replay produced, so verifying an upgrade is one command over one transaction list instead of a replay run plus a separate receipt-diffing pipeline. + +### `--verify-receipt` + +Verify every replayed transaction against its on-chain receipt. +Supported in both single-transaction and [batch](#batch-replay) mode. + +Three dimensions are compared: + +- **Status** — the success flag. +- **Gas used** — the transaction's gas, not the block's cumulative gas. +- **Logs** — the number of logs, and each log's `address`, `topics`, and `data`. + +Logs are compared explicitly rather than inferred from gas: `LOG` gas depends on topic count and data length, never on content, so two executions can burn identical gas yet emit different log payloads. + +The receipt is fetched with the same call the [fixture dump](#self-validating-fixture-dump) uses, so a run with `--rpc.capture-file` records it and a later `--rpc.replay-file` run verifies the same transaction offline. +An envelope captured without `--verify-receipt` (or by any earlier run that never needed a receipt) holds no receipts, so verifying against it fails the receipt fetch — capture once online with the flag, then re-verify offline as often as you like. + +### Verified, Unverified, and Mismatched + +A transaction is only reported as mismatched when both receipts were compared and disagreed. +Anything that prevents the comparison from running is an infrastructure failure — the transaction is _unverified_, which is a different finding from a divergence: + +- The endpoint fails the receipt call, or has pruned the receipt below its retention height (common on non-archive endpoints): reported as an `rpc` failure. +- The receipt describes a different inclusion than the replayed block (its `blockHash` differs — a reorg in progress, or a load-balanced endpoint serving divergent views): reported as an `rpc` failure, because comparing against it would compare the replay to the wrong on-chain execution. +- The target is a pending transaction, which has no receipt yet: rejected up front in single-transaction mode, and reported as a `pending` error entry in batch mode. + +In batch mode each of these becomes an error entry for that transaction, exactly like any other infrastructure failure. + +Transaction overrides and `--override.spec` are still accepted with `--verify-receipt`, but they make the replay a what-if that the chain never executed, so the comparison will normally report a mismatch. + +### Output + +With `--json`, the verdict is a `verification` object — added to the single-transaction summary, and to each batch result line. +The field is absent entirely without the flag. + +A match carries nothing else: + +```json +{ "match": true } +``` + +A mismatch carries a `diff` holding only the dimensions that disagreed, each as `{"onchain": …, "replay": …}`: + +```json +{ + "match": false, + "diff": { + "status": { "onchain": true, "replay": false }, + "gas_used": { "onchain": 75514, "replay": 75500 }, + "logs": { + "count": { "onchain": 2, "replay": 1 }, + "first_mismatch": { + "index": 0, + "field": "address", + "onchain": "0x00000000000000000000000000000000000000aa", + "replay": "0x00000000000000000000000000000000000000bb" + } + } + } +} +``` + +Under `logs`, `count` is present when the two sides emitted a different number of logs, and `first_mismatch` names the first log both sides emitted whose contents differ — its `field` is `address`, `topics`, or `data`, and the two values are that field's contents on each side. +Both can appear at once, which distinguishes truncated logs from rewritten ones. + +Without `--json`, each transaction gets one verdict line after its usual output: + +``` +verification: MATCH +verification: MISMATCH (gas_used: onchain 75514 vs replay 75500) +``` + +The mismatch line names every dimension that disagreed, comma-separated. + +### Exit Status + +A run in which every target replayed and every verification matched exits `0`. +A verification mismatch exits non-zero through a dedicated error (`Receipt verification mismatch: N of M verified transaction(s) did not reproduce the on-chain receipt`), reported after every result line has been written. +Infrastructure failures keep their own non-zero exit and take precedence in a batch run: a target that never replayed was also never verified, so reporting it as a mismatch would overstate what the run found. + +### Examples + +Verify one transaction against a live RPC: + +```bash +mega-evme replay --rpc https://mainnet.megaeth.com/rpc --verify-receipt 0xabc123... +``` + +Verify a whole corpus in one process and collect the divergences: + +```bash +mega-evme replay --rpc https://mainnet.megaeth.com/rpc \ + --tx-file ./corpus.txt --verify-receipt --json > results.ndjson + +jq -c 'select(.verification.match == false)' results.ndjson # mismatched +jq -c 'select(.error != null)' results.ndjson # unverified +``` + +Capture once online, then re-verify the same corpus offline: + +```bash +mega-evme replay --rpc https://mainnet.megaeth.com/rpc \ + --rpc.capture-file ./corpus.cache.json --tx-file ./corpus.txt --verify-receipt --json + +mega-evme replay --rpc.replay-file ./corpus.cache.json \ + --tx-file ./corpus.txt --verify-receipt --json +``` + ## RPC Cache File `mega-evme replay` supports a transport-level JSON-RPC fixture mechanism that records every request/response pair to a single file and serves them back on later runs without touching the network. @@ -294,6 +408,8 @@ Options marked _(single transaction only)_ are rejected in [batch mode](#batch-r - **Batch replay** — Replay many transactions in one process via `--tx-file` / `--block`. See [Batch Replay](#batch-replay) above. +- **Receipt verification** — Check every replayed transaction against its on-chain receipt via `--verify-receipt`. + See [Receipt Verification](#receipt-verification) above. - **SALT buckets** — Configure SALT bucket capacity for dynamic storage gas pricing. See [SALT Buckets](../configuration/salt-buckets.md). - **State dump** _(single transaction only)_ — Dump or load pre/post-state snapshots. @@ -350,6 +466,12 @@ mega-evme replay --rpc https://mainnet.megaeth.com/rpc --override.spec Rex2 0xab mega-evme replay --rpc https://mainnet.megaeth.com/rpc --block 22945844 --json ``` +**Verify a whole block against its on-chain receipts** + +```bash +mega-evme replay --rpc https://mainnet.megaeth.com/rpc --block 22945844 --verify-receipt --json +``` + ## See Also - [`run`](./run.md) — Execute raw EVM bytecode locally without fetching from RPC From 31d0f76fb37425692168f2f57c8937e5c80bceb6 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 18:37:45 +0800 Subject: [PATCH 07/64] feat(mega-evme): batch fixture dump via --dump-fixture-dir Add batch sedimentation of self-validating EEST fixtures so a tx list or block can be swept into

/.json in one run. Per-target fidelity/BLOCKHASH/unsupported-shape gates skip with a recorded reason instead of failing the run; write failures stay infrastructure errors. --- bin/mega-evme/src/replay/batch.rs | 358 +++++++++++++++++++++++++-- bin/mega-evme/src/replay/cmd.rs | 112 ++++++++- bin/mega-evme/src/replay/fixture.rs | 102 ++++---- bin/mega-evme/tests/replay_batch.rs | 42 ++++ bin/mega-evme/tests/replay_verify.rs | 100 ++++++++ docs/mega-evme/commands/replay.md | 50 +++- 6 files changed, 683 insertions(+), 81 deletions(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index d28039ff..df398043 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -15,6 +15,7 @@ use std::{ collections::{BTreeMap, HashSet}, + path::{Path, PathBuf}, str::FromStr, time::{Duration, Instant}, }; @@ -33,10 +34,11 @@ use mega_evm::{ DatabaseRef, }, BlockLimits, MegaBlockExecutionCtx, MegaBlockExecutorFactory, MegaEvmFactory, MegaHaltReason, - MegaHardforks, + MegaHardforks, MegaSpecId, }; use op_alloy_rpc_types::Transaction; use serde::Serialize; +use state_test::types::MegaEnv; use tracing::{debug, info, warn}; use crate::{ @@ -50,17 +52,63 @@ use crate::{ use super::{ cmd::retrieve_block_env, + fixture, verify::{self, ReceiptFacts, VerificationOutcome}, ReplayError, Result, }; /// How a batch run reports its targets. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub(super) struct ReportArgs { /// Emit one NDJSON line per target instead of the human-readable summary. pub json: bool, /// Verify every target against its on-chain receipt. pub verify_receipt: bool, + /// When set, dump a self-validating fixture for every successful target into + /// this directory as `/.json`. + pub dump_fixture_dir: Option, + /// Replace existing fixture files under [`Self::dump_fixture_dir`]. + pub overwrite: bool, +} + +/// Per-target fixture dump outcome reported on the NDJSON / human result line. +#[derive(Debug, Clone, Serialize)] +struct FixtureReport { + /// Absolute or as-written path of a successfully written fixture. + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + /// Why the fixture was not written for this target. + #[serde(skip_serializing_if = "Option::is_none")] + skipped: Option, +} + +impl FixtureReport { + fn written(path: &Path) -> Self { + Self { path: Some(path.display().to_string()), skipped: None } + } + + fn skipped(reason: impl Into) -> Self { + Self { path: None, skipped: Some(reason.into()) } + } + + /// One-line human summary printed under the transaction header. + fn human_line(&self) -> String { + if let Some(path) = &self.path { + format!("fixture: written to {path}") + } else if let Some(reason) = &self.skipped { + format!("fixture: skipped ({reason})") + } else { + "fixture: (no report)".to_string() + } + } +} + +/// Result of attempting to dump a fixture for one target during the block loop. +enum FixtureAttempt { + /// Wrote a fixture or recorded an expected skip (fidelity / BLOCKHASH / unsupported). + Report(FixtureReport), + /// Finalize/write failed; the target becomes an infrastructure error entry. + WriteError(String), } /// What a batch run was asked to replay. @@ -129,6 +177,8 @@ struct ExecutedTx { receipt: OpTxReceipt, /// On-chain receipt verdict, present iff `--verify-receipt` was given. verification: Option, + /// Fixture dump outcome, present iff `--dump-fixture-dir` was given. + fixture: Option, } /// A target transaction that hit an infrastructure failure. @@ -161,6 +211,11 @@ struct PendingTarget { from: Address, to: Option
, effective_gas_price: u128, + /// Fixture dump outcome, present iff `--dump-fixture-dir` was given. + /// + /// `Ok` is a written or skipped report; `Err` is a finalize/write failure + /// that turns this target into an infrastructure error entry. + fixture: Option>, } /// NDJSON line for a target that produced an execution result. @@ -171,6 +226,8 @@ struct BatchResultLine<'a> { tx_index: u64, #[serde(flatten)] summary: &'a ExecutionSummary, + #[serde(skip_serializing_if = "Option::is_none")] + fixture: Option<&'a FixtureReport>, } /// NDJSON line for a target that produced an infrastructure error. @@ -191,8 +248,9 @@ struct BatchErrorBody<'a> { /// /// Returns an error when at least one target produced an infrastructure error /// entry, so the process exits non-zero; execution outcomes never fail the run. -/// With `--verify-receipt`, a run in which every target replayed but some -/// diverged from its on-chain receipt fails with +/// Fixture skips (fidelity gate, BLOCKHASH readers, unsupported tx shapes) do +/// not fail the run. With `--verify-receipt`, a run in which every target +/// replayed but some diverged from its on-chain receipt fails with /// [`ReplayError::VerificationMismatch`] instead — a distinct variant, so a /// divergence is never confused with a target that could not be replayed. pub(super) async fn run

( @@ -210,6 +268,18 @@ where let mut failed = 0usize; let mut verified = 0usize; let mut mismatched = 0usize; + let mut fixtures_written = 0usize; + let mut fixtures_skipped = 0usize; + let mut fixtures_failed = 0usize; + + if let Some(dir) = &report.dump_fixture_dir { + std::fs::create_dir_all(dir).map_err(|e| { + ReplayError::Other(format!( + "failed to create --dump-fixture-dir '{}': {e}", + dir.display() + )) + })?; + } let jobs = match mode { BatchMode::Block(number) => { @@ -235,10 +305,10 @@ where }; for job in jobs { - for entry in - replay_block(provider, chain_id, job, external_envs.clone(), report.verify_receipt) - .await - { + let block_result = + replay_block(provider, chain_id, job, external_envs.clone(), &report).await; + fixtures_failed += block_result.fixture_write_failures; + for entry in block_result.entries { match &entry { BatchEntry::Executed(tx) => { replayed += 1; @@ -248,6 +318,13 @@ where mismatched += 1; } } + if let Some(fixture) = &tx.fixture { + if fixture.path.is_some() { + fixtures_written += 1; + } else { + fixtures_skipped += 1; + } + } } BatchEntry::Failed(_) => failed += 1, } @@ -259,10 +336,18 @@ where if report.verify_receipt { info!(verified, mismatched, "On-chain receipt verification finished"); } + if report.dump_fixture_dir.is_some() { + info!( + written = fixtures_written, + skipped = fixtures_skipped, + failed = fixtures_failed, + "Fixture dump finished" + ); + } // Infrastructure failures keep their own error: a target that never replayed // was also never verified, so reporting it as a mismatch would overstate - // what the run actually found. + // what the run actually found. Fixture skips never contribute to `failed`. if failed > 0 { return Err(ReplayError::Other(format!( "{failed} of {} target transaction(s) failed to replay", @@ -316,51 +401,95 @@ where (jobs, failures) } +/// Entries produced by replaying one block, plus how many fixture write failures +/// it contributed (for the end-of-run fixture summary). +struct BlockReplayResult { + entries: Vec, + /// Finalize/write errors that became `execution` infrastructure entries. + fixture_write_failures: usize, +} + +impl BlockReplayResult { + fn from_entries(entries: Vec) -> Self { + Self { entries, fixture_write_failures: 0 } + } + + fn fail_all(targets: &[B256], kind: BatchErrorKind, message: &str) -> Self { + Self::from_entries(fail_all(targets, kind, message)) + } +} + /// Replay one block, reporting an entry for every target it was asked about. /// /// The block is executed exactly once: every transaction runs in order, and each /// target's result is recorded before the transaction is committed. Receipts are /// harvested from the finished block, which is why the block's entries are only /// produced once the block is done. +/// +/// When `--dump-fixture-dir` is set, each target's fixture draft is built from +/// the pre-commit state (same moment as the single-transaction dump), gated per +/// target for fidelity and BLOCKHASH, and written before the transaction commits. async fn replay_block

( provider: &P, chain_id: u64, job: BlockJob, external_envs: EvmeExternalEnvs, - verify_receipt: bool, -) -> Vec + report: &ReportArgs, +) -> BlockReplayResult where P: Provider + Clone + std::fmt::Debug, { let BlockJob { number, block, targets } = job; + let verify_receipt = report.verify_receipt; + let dump_dir = report.dump_fixture_dir.as_deref(); + let overwrite = report.overwrite; if number == 0 { - return fail_all(&targets, BatchErrorKind::Rpc, "Block 0 has no parent block to fork from"); + return BlockReplayResult::fail_all( + &targets, + BatchErrorKind::Rpc, + "Block 0 has no parent block to fork from", + ); } let block = match block { Some(block) => block, None => match fetch_block(provider, number).await { Ok(block) => block, - Err(e) => return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()), + Err(e) => { + return BlockReplayResult::fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + } }, }; let parent_block = match fetch_block(provider, number - 1).await { Ok(block) => block, - Err(e) => return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()), + Err(e) => { + return BlockReplayResult::fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + } }; - // Fetch the on-chain receipts before the block runs, so the comparison - // afterwards is a pure function over the two receipts. A receipt that cannot - // be fetched, or that describes a different inclusion than this block, is - // recorded as a per-target failure here and reported as an `rpc` error entry - // below: such a target is unverified, never mismatched. - let onchain_receipts = if verify_receipt { + // Fetch the on-chain receipts before the block runs. Needed for + // `--verify-receipt` (mismatch vs unverified) and for `--dump-fixture-dir` + // (fidelity gate). A receipt that cannot be fetched, or that describes a + // different inclusion than this block, is recorded here and interpreted by + // each feature below. + let need_receipts = verify_receipt || dump_dir.is_some(); + let onchain_receipts = if need_receipts { fetch_target_receipts(provider, &targets, block.hash()).await } else { BTreeMap::new() }; + // Sorted once so every fixture of this block is byte-reproducible for the + // same megaEnv (hash-map iteration order is otherwise non-deterministic). + let mega_env = dump_dir.map(|_| { + let mut bucket_capacities = external_envs.bucket_capacities(); + bucket_capacities.sort_unstable(); + let mut oracle_storage = external_envs.oracle_storage(); + oracle_storage.sort_unstable(); + MegaEnv { bucket_capacities, oracle_storage } + }); + let hardforks = get_hardfork_config(chain_id); let timestamp = block.header.timestamp(); let spec = hardforks.spec_id(timestamp); @@ -369,17 +498,22 @@ where let cfg_env = match chain_args.create_cfg_env() { Ok(cfg) => cfg, - Err(e) => return fail_all(&targets, BatchErrorKind::Execution, &e.to_string()), + Err(e) => { + return BlockReplayResult::fail_all(&targets, BatchErrorKind::Execution, &e.to_string()); + } }; let block_env = match retrieve_block_env(&block) { Ok(env) => env, - Err(e) => return fail_all(&targets, BatchErrorKind::Execution, &e.to_string()), + Err(e) => { + return BlockReplayResult::fail_all(&targets, BatchErrorKind::Execution, &e.to_string()); + } }; + let executed_spec = cfg_env.spec; let evm_env = EvmEnv::new(cfg_env, block_env); let Some(hardfork) = hardforks.hardfork(timestamp) else { let message = format!("No `MegaHardfork` active at block timestamp: {timestamp}"); - return fail_all(&targets, BatchErrorKind::Execution, &message); + return BlockReplayResult::fail_all(&targets, BatchErrorKind::Execution, &message); }; let block_limits = BlockLimits::from_hardfork_and_block_gas_limit(hardfork, block.header.gas_limit()); @@ -400,7 +534,9 @@ where .await { Ok(database) => database, - Err(e) => return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()), + Err(e) => { + return BlockReplayResult::fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + } }; let evm_factory = MegaEvmFactory::new().with_external_env_factory(external_envs); @@ -411,7 +547,7 @@ where if let Err(e) = block_executor.apply_pre_execution_changes() { let message = format!("Block execution error: {e}"); - return fail_all(&targets, BatchErrorKind::Execution, &message); + return BlockReplayResult::fail_all(&targets, BatchErrorKind::Execution, &message); } let target_set: HashSet = targets.iter().copied().collect(); @@ -424,6 +560,11 @@ where // cannot be replayed faithfully. let loop_result: Result<()> = async { for (tx_index, tx_hash) in tx_hashes.iter().enumerate() { + // Isolate BLOCKHASH reads per transaction so a fixture dump sees only + // the target's own accesses (mirrors the single-tx clear after + // preceding transactions). + block_executor.clear_accessed_block_hashes(); + let tx = provider .get_transaction_by_hash(*tx_hash) .await @@ -446,6 +587,36 @@ where let outcome = block_executor .run_transaction(tx.as_recovered()) .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + + // Fixture draft must be built before commit: the pre-state closure + // is the database after preceding txs, with the target's result + // state still uncommitted — same moment as the single-tx dump. + let fixture = if is_target { + if let (Some(dir), Some(mega_env)) = (dump_dir, mega_env.as_ref()) { + let accessed_block_hashes = block_executor.get_accessed_block_hashes(); + Some(dump_target_fixture( + block_executor.evm().db_ref(), + DumpFixtureArgs { + accessed_block_hash_count: accessed_block_hashes.len(), + exec_result: &outcome.inner.result, + evm_state: &outcome.inner.state, + chain_id, + executed_spec, + block: &block, + target_tx: &tx, + mega_env: mega_env.clone(), + onchain: onchain_receipts.get(tx_hash), + dir, + overwrite, + }, + )) + } else { + None + } + } else { + None + }; + // Record the target's result before committing, mirroring the // single-transaction path. let exec_result = is_target.then(|| outcome.inner.result.clone()); @@ -456,6 +627,11 @@ where committed += 1; if let Some(exec_result) = exec_result { + let fixture = match fixture { + Some(FixtureAttempt::Report(report)) => Some(Ok(report)), + Some(FixtureAttempt::WriteError(message)) => Some(Err(message)), + None => None, + }; pending.push(PendingTarget { tx_hash: *tx_hash, tx_index: tx_index as u64, @@ -467,6 +643,7 @@ where from: tx.inner.inner.signer(), to: tx.inner.inner.to(), effective_gas_price: tx.inner.effective_gas_price.unwrap_or(0), + fixture, }); } } @@ -477,6 +654,7 @@ where // Finish the block even when it aborted midway: targets that already ran // still have a receipt worth reporting. let mut entries = Vec::with_capacity(targets.len()); + let mut fixture_write_failures = 0usize; match block_executor.finish() { Ok((evm, block_result)) => { let (db, _) = evm.finish(); @@ -488,6 +666,15 @@ where let offset = receipts.len().saturating_sub(committed); let block_hash = block.hash(); for target in pending { + // Fixture finalize/write failure: infrastructure error entry. + // The target did replay, but the dump request failed. + if let Some(Err(message)) = target.fixture { + fixture_write_failures += 1; + entries.push(failure(target.tx_hash, BatchErrorKind::Execution, message)); + continue; + } + let fixture = target.fixture.and_then(|report| report.ok()); + let Some(envelope) = receipts.get(offset + target.commit_index) else { entries.push(failure( target.tx_hash, @@ -540,12 +727,16 @@ where exec_time: target.exec_time, receipt, verification, + fixture, }))); } } Err(e) => { let message = format!("Block execution error: {e}"); for target in pending { + if matches!(target.fixture, Some(Err(_))) { + fixture_write_failures += 1; + } entries.push(failure(target.tx_hash, BatchErrorKind::Execution, message.clone())); } } @@ -567,7 +758,117 @@ where } } - entries + BlockReplayResult { entries, fixture_write_failures } +} + +/// Inputs for [`dump_target_fixture`], grouped so the dump path stays a single +/// call site without a long positional argument list. +struct DumpFixtureArgs<'a> { + accessed_block_hash_count: usize, + exec_result: &'a ExecutionResult, + evm_state: &'a mega_evm::revm::state::EvmState, + chain_id: u64, + executed_spec: MegaSpecId, + block: &'a Block, + target_tx: &'a Transaction, + mega_env: MegaEnv, + onchain: Option<&'a std::result::Result>, + dir: &'a Path, + overwrite: bool, +} + +/// Attempt to build and write a fixture for one successfully executed target. +/// +/// Expected skips (missing receipt, fidelity mismatch, BLOCKHASH, unsupported +/// transaction shapes) return [`FixtureAttempt::Report`] with a skip reason and +/// never fail the batch run. Finalize/write errors return +/// [`FixtureAttempt::WriteError`] and become infrastructure error entries. +/// +/// `db` must reflect the pre-target-commit state (preceding txs committed, target +/// not yet), matching the single-transaction dump. +fn dump_target_fixture(db: &DB, args: DumpFixtureArgs<'_>) -> FixtureAttempt +where + DB: DatabaseRef, + DB::Error: core::fmt::Display, +{ + let DumpFixtureArgs { + accessed_block_hash_count, + exec_result, + evm_state, + chain_id, + executed_spec, + block, + target_tx, + mega_env, + onchain, + dir, + overwrite, + } = args; + + // Fidelity gate needs the on-chain receipt; without it the dump is skipped, + // not failed — the envelope may simply lack receipts (expected in sweeps + // over captures that never fetched them). + let facts = match onchain { + Some(Ok(facts)) => facts, + Some(Err(message)) => { + return FixtureAttempt::Report(FixtureReport::skipped(format!( + "fidelity-gate-unavailable: {message}" + ))); + } + None => { + return FixtureAttempt::Report(FixtureReport::skipped( + "fidelity-gate-unavailable: no on-chain receipt was fetched for this transaction", + )); + } + }; + + if accessed_block_hash_count > 0 { + return FixtureAttempt::Report(FixtureReport::skipped(format!( + "transaction reads block hashes (BLOCKHASH): {accessed_block_hash_count} block \ + hash(es) were accessed and the fixture cannot faithfully reproduce them" + ))); + } + + let anchor = fixture::anchor_from_receipt_facts(facts); + if let Err(reason) = fixture::check_fidelity(exec_result, &anchor, chain_id) { + return FixtureAttempt::Report(FixtureReport::skipped(format!( + "fidelity gate failed: {reason}" + ))); + } + + let draft = match fixture::build_draft( + db, + evm_state, + chain_id, + executed_spec, + block, + target_tx, + fixture::FixtureInputs { mega_env, result: exec_result, anchor }, + ) { + Ok(draft) => draft, + // Unsupported shapes (deposit, EIP-7702, unknown spec) are expected in + // whole-block sweeps: skip rather than fail the run. + Err(e) => { + return FixtureAttempt::Report(FixtureReport::skipped(e.to_string())); + } + }; + + let tx_hash = target_tx.inner.inner.tx_hash(); + let path = dir.join(format!("{tx_hash:#x}.json")); + if path.exists() && !overwrite { + return FixtureAttempt::WriteError(format!( + "fixture already exists at {} (pass --overwrite to replace)", + path.display() + )); + } + + match fixture::finalize_and_write(draft, &path) { + Ok(()) => { + info!(path = %path.display(), tx_hash = %tx_hash, "Wrote self-validating fixture"); + FixtureAttempt::Report(FixtureReport::written(&path)) + } + Err(e) => FixtureAttempt::WriteError(format!("fixture write failed: {e}")), + } } /// Fetch the on-chain receipt of every target of a block. @@ -653,6 +954,7 @@ fn emit(entry: &BatchEntry, json: bool) { block_number: tx.block_number, tx_index: tx.tx_index, summary: &summary, + fixture: tx.fixture.as_ref(), }) } BatchEntry::Failed(tx) => serde_json::to_string(&BatchErrorLine { @@ -677,6 +979,10 @@ fn emit(entry: &BatchEntry, json: bool) { println!(); println!("{}", verification.verdict_line()); } + if let Some(fixture) = &tx.fixture { + println!(); + println!("{}", fixture.human_line()); + } } BatchEntry::Failed(tx) => { println!(); diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index c4cc8d79..95c76341 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -54,8 +54,8 @@ pub struct Cmd { /// Blank lines and `#`-prefixed comment lines are ignored, and duplicates are /// replayed once. All hashes are replayed in a single process: transactions /// are grouped by their containing block and each block is executed once. - /// Batch mode does not support `--dump-fixture`, transaction overrides, - /// `--override.spec`, tracing, or state dumps. + /// Batch mode does not support `--dump-fixture` (use `--dump-fixture-dir`), + /// transaction overrides, `--override.spec`, tracing, or state dumps. #[arg(long = "tx-file", value_name = "PATH")] pub tx_file: Option, @@ -103,9 +103,29 @@ pub struct Cmd { /// replay, and `state-test --bench` benchmarks it. The dump is rejected /// unless the local replay reproduces the on-chain receipt's gas and success /// status. Incompatible with transaction overrides and `--override.spec`. + /// Single-transaction only; for batch mode use `--dump-fixture-dir`. #[arg(long = "dump-fixture", value_name = "FILE")] pub dump_fixture: Option, + /// Dump a self-validating EEST state-test fixture for every successfully + /// replayed target into `

/.json`. + /// + /// Batch mode only (`--tx-file` / `--block`). Per-target gating mirrors the + /// single-transaction dump: fidelity-gate failures and BLOCKHASH readers are + /// skipped with a recorded reason instead of failing the run; pending or + /// unresolvable targets stay error entries. Existing files are refused unless + /// `--overwrite` is set. Registration into `bench/replay/manifest.json` is + /// not performed — corpus curation stays manual. + #[arg(long = "dump-fixture-dir", value_name = "DIR")] + pub dump_fixture_dir: Option, + + /// Replace existing files when writing fixtures with `--dump-fixture-dir`. + /// + /// Without this flag, a target whose `/.json` already exists is + /// reported as an infrastructure error for that target. + #[arg(long = "overwrite")] + pub overwrite: bool, + /// Verify every replayed transaction against its on-chain receipt. /// /// Fetches the receipt of each target and compares the success status, the @@ -196,6 +216,8 @@ impl Cmd { fn validate(&self) -> Result<()> { if self.is_batch() { self.validate_batch_args()?; + } else { + self.validate_single_args()?; } // A dumped fixture must represent the on-chain transaction, so it can @@ -219,6 +241,26 @@ impl Cmd { } } + if self.dump_fixture.is_some() && self.dump_fixture_dir.is_some() { + return Err(ReplayError::Other( + "--dump-fixture and --dump-fixture-dir are mutually exclusive: dump one \ + transaction with --dump-fixture, or a batch with --dump-fixture-dir" + .to_string(), + )); + } + + Ok(()) + } + + /// Reject batch-only flags in single-transaction mode. + fn validate_single_args(&self) -> Result<()> { + if self.dump_fixture_dir.is_some() { + return Err(ReplayError::Other( + "--dump-fixture-dir is only supported by batch replay (--tx-file / --block); \ + dump a single transaction with --dump-fixture " + .to_string(), + )); + } Ok(()) } @@ -229,17 +271,18 @@ impl Cmd { /// Reject the single-transaction-only flags in batch mode. /// - /// Batch mode reports one summary per transaction; per-transaction artifacts - /// (fixture, trace, state dump) and what-if knobs (overrides, forced spec) - /// have no meaningful batch semantics, so they are rejected up front rather - /// than silently ignored. + /// Batch mode reports one summary per transaction; single-file fixture dumps, + /// tracing, state dumps, and what-if knobs (overrides, forced spec) have no + /// meaningful batch semantics, so they are rejected up front rather than + /// silently ignored. Per-target fixture sedimentation uses + /// `--dump-fixture-dir` instead of `--dump-fixture`. fn validate_batch_args(&self) -> Result<()> { const MODE: &str = "batch replay (--tx-file / --block)"; if self.dump_fixture.is_some() { return Err(ReplayError::Other(format!( - "--dump-fixture is not supported by {MODE}; dump a fixture by replaying a \ - single transaction" + "--dump-fixture is not supported by {MODE}; dump fixtures for a batch \ + with --dump-fixture-dir " ))); } if self.tx_override_args.has_overrides() { @@ -329,7 +372,12 @@ impl Cmd { pctx.chain_id, mode, external_envs, - batch::ReportArgs { json: self.output_args.json, verify_receipt: self.verify_receipt }, + batch::ReportArgs { + json: self.output_args.json, + verify_receipt: self.verify_receipt, + dump_fixture_dir: self.dump_fixture_dir.clone(), + overwrite: self.overwrite, + }, ) .await } @@ -1054,6 +1102,52 @@ mod tests { assert!(!parse(&[TX]).expect("parse").verify_receipt); } + /// `--dump-fixture-dir` is batch-only; single-transaction mode keeps + /// `--dump-fixture` and must reject the dir flag. + #[test] + fn test_dump_fixture_dir_rejected_in_single_transaction_mode() { + let cmd = parse(&["--dump-fixture-dir", "/tmp/fixtures", TX]).expect("parse"); + let message = + cmd.validate().expect_err("single-tx must reject --dump-fixture-dir").to_string(); + assert!( + message.contains("--dump-fixture-dir") && message.contains("batch"), + "unexpected rejection: {message}" + ); + } + + /// Batch mode accepts `--dump-fixture-dir` and still rejects the single-file + /// dump flag (use the dir form for sedimentation sweeps). + #[test] + fn test_dump_fixture_dir_accepted_in_batch_mode() { + let cmd = parse(&["--block", "1", "--dump-fixture-dir", "/tmp/fixtures"]).expect("parse"); + cmd.validate().expect("--dump-fixture-dir must be accepted in batch mode"); + assert_eq!(cmd.dump_fixture_dir, Some(PathBuf::from("/tmp/fixtures"))); + + let with_overwrite = + parse(&["--tx-file", "/tmp/list.txt", "--dump-fixture-dir", "/tmp/f", "--overwrite"]) + .expect("parse"); + with_overwrite.validate().expect("--overwrite is allowed with --dump-fixture-dir"); + assert!(with_overwrite.overwrite); + } + + /// The two dump forms are mutually exclusive even when one would otherwise + /// be valid for the selected mode. + #[test] + fn test_dump_fixture_forms_are_mutually_exclusive() { + let cmd = parse(&[ + "--block", + "1", + "--dump-fixture", + "/tmp/f.json", + "--dump-fixture-dir", + "/tmp/fixtures", + ]) + .expect("parse"); + // Batch validation rejects --dump-fixture first; either way both must not run. + let message = cmd.validate().expect_err("both dump forms must be rejected").to_string(); + assert!(message.contains("--dump-fixture"), "unexpected rejection: {message}"); + } + #[test] fn test_batch_accepts_the_flags_it_supports() { parse(&["--block", "1", "--json"]).expect("parse").validate().expect("--json is allowed"); diff --git a/bin/mega-evme/src/replay/fixture.rs b/bin/mega-evme/src/replay/fixture.rs index cee5afdf..f97e250b 100644 --- a/bin/mega-evme/src/replay/fixture.rs +++ b/bin/mega-evme/src/replay/fixture.rs @@ -79,6 +79,63 @@ const DEPOSIT_TX_TYPE: u8 = 0x7e; /// than emit a fixture whose isolated run diverges from the chain. const EIP7702_TX_TYPE: u8 = 0x04; +/// Check that a local replay reproduces the on-chain receipt's gas, success +/// status, and logs root. +/// +/// A mismatch means the replay executed under the wrong spec / hardfork config +/// for this chain and block; self-validation cannot catch this, because the +/// fixture is validated under the same spec it was dumped with. +/// +/// Logs are checked, not just inferred from gas: LOG gas depends on topic count +/// and data length, never content, so two executions can burn identical gas yet +/// emit different log payloads (e.g. a preceding-tx divergence that changes a +/// value the target re-emits). +/// +/// Returns the explanatory reason on failure so batch dump can record a skip +/// without treating it as an infrastructure error. +pub(crate) fn check_fidelity( + result: &ExecutionResult, + anchor: &OnchainAnchor, + chain_id: u64, +) -> std::result::Result<(), String> { + let actual_gas = result.tx_gas_used(); + if actual_gas != anchor.gas_used { + return Err(format!( + "replay gas {actual_gas} != on-chain receipt gas {}: the local replay does \ + not reproduce on-chain execution (likely a wrong spec or hardfork config \ + for chain {chain_id} at this block)", + anchor.gas_used + )); + } + if result.is_success() != anchor.success { + return Err(format!( + "replay status (success={}) != on-chain receipt status (success={}): the \ + local replay does not reproduce on-chain execution for chain {chain_id}", + result.is_success(), + anchor.success + )); + } + let actual_logs_root = state_test::utils::log_rlp_hash(result.logs()); + if actual_logs_root != anchor.logs_root { + return Err(format!( + "replay logs root {actual_logs_root} != on-chain receipt logs root {}: the \ + local replay emits different logs than the chain for chain {chain_id} \ + (same gas/status, different log contents)", + anchor.logs_root + )); + } + Ok(()) +} + +/// Build an [`OnchainAnchor`] from the consensus facts of an on-chain receipt. +pub(crate) fn anchor_from_receipt_facts(facts: &super::verify::ReceiptFacts) -> OnchainAnchor { + OnchainAnchor { + gas_used: facts.gas_used, + success: facts.status, + logs_root: state_test::utils::log_rlp_hash(&facts.logs), + } +} + /// A fixture built from a replay, awaiting its `post` expectation. /// /// The `post` map is filled by [`finalize_and_write`] after re-executing the @@ -144,48 +201,9 @@ where let actual_output = inputs.result.output().cloned(); let actual_logs_root = state_test::utils::log_rlp_hash(inputs.result.logs()); - // Fidelity gate: the local replay must reproduce the on-chain receipt's gas, - // success status, and logs. A mismatch means the replay executed under the - // wrong spec / hardfork config for this chain and block; self-validation - // cannot catch this, because the fixture is validated under the same spec it - // was dumped with. Refuse to build a fixture that does not match the chain. - // - // Logs are checked, not just inferred from gas: LOG gas depends on topic count - // and data length, never content, so two executions can burn identical gas yet - // emit different log payloads (e.g. a preceding-tx divergence that changes a - // value the target re-emits). The receipt's logs are already fetched, so the - // comparison is a single root equality. `finalize_and_write` then re-checks the - // isolated run's logs root against this same value, so any gas-, output-, or - // log-visible divergence from the zeroed L1 data fee aborts the dump. One - // channel stays open by construction: the isolated run's sender balance is - // shifted by the zeroed fee, so a contract that stores a balance-derived value - // bakes that shifted value into `post` (gas, status, output, and logs all - // still match). The fixture still self-validates and reproduces gas exactly. - let anchor = &inputs.anchor; - if actual_gas != anchor.gas_used { - return Err(ReplayError::Other(format!( - "replay gas {actual_gas} != on-chain receipt gas {}: the local replay does \ - not reproduce on-chain execution (likely a wrong spec or hardfork config \ - for chain {chain_id} at this block)", - anchor.gas_used - ))); - } - if inputs.result.is_success() != anchor.success { - return Err(ReplayError::Other(format!( - "replay status (success={}) != on-chain receipt status (success={}): the \ - local replay does not reproduce on-chain execution for chain {chain_id}", - inputs.result.is_success(), - anchor.success - ))); - } - if actual_logs_root != anchor.logs_root { - return Err(ReplayError::Other(format!( - "replay logs root {actual_logs_root} != on-chain receipt logs root {}: the \ - local replay emits different logs than the chain for chain {chain_id} \ - (same gas/status, different log contents)", - anchor.logs_root - ))); - } + // Fidelity gate: refuse to dump a fixture that does not match the chain. + // See [`check_fidelity`] for the rationale and the dimensions checked. + check_fidelity(inputs.result, &inputs.anchor, chain_id).map_err(ReplayError::Other)?; let pre = build_pre_state(db, evm_state)?; let env = build_env(chain_id, block); diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index e76643ab..770ab806 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -231,3 +231,45 @@ fn test_replay_batch_rejects_single_transaction_flags() { assert!(output.stdout.is_empty(), "a rejected batch run must print nothing on stdout"); } } + +/// Sweeping a block with `--dump-fixture-dir` against an envelope that carries +/// no receipts skips every target on the fidelity gate and still exits 0. +/// +/// Fixture skips are expected (not infrastructure failures); the development +/// envelope is captured without receipts, so this pins the skip path end to end. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_block_dump_fixture_dir_skips_without_receipts() { + let dir = std::env::temp_dir().join(format!( + "mega_evme_dump_dir_skip_{}_{}", + std::process::id(), + BLOCK + )); + let _ = std::fs::remove_dir_all(&dir); + + let stdout = replay( + &["--block", &BLOCK.to_string(), "--dump-fixture-dir", dir.to_str().unwrap(), "--json"], + true, + ); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + for line in &lines { + assert!(line.get("error").is_none(), "skips must not turn into error entries: {line}"); + let skipped = line["fixture"]["skipped"] + .as_str() + .unwrap_or_else(|| panic!("every line must carry a fixture skip reason: {line}")); + assert!( + skipped.contains("fidelity-gate-unavailable"), + "expected fidelity-gate-unavailable skip, got: {skipped}" + ); + assert!(line["fixture"].get("path").is_none(), "a skip must not report a path: {line}"); + } + + // No fixtures written: the directory may exist (create_dir_all) but be empty. + if dir.exists() { + let entries: Vec<_> = std::fs::read_dir(&dir).expect("read dump dir").collect(); + assert!(entries.is_empty(), "fidelity skips must write no fixture files"); + } + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index 310aec8f..4b1bd80e 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -379,3 +379,103 @@ fn test_batch_verify_receipt_reorg_is_an_rpc_error_entry() { lines[0] ); } + +/// Batch `--dump-fixture-dir` writes a self-validating fixture for a target +/// whose capture includes the on-chain receipt, and exits 0. +#[test] +fn test_batch_dump_fixture_dir_writes_validatable_file() { + let list = tx_file("batch_dump"); + let dir = std::env::temp_dir().join(format!("mega_evme_batch_dump_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let run = replay( + &cache(), + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + let _ = std::fs::remove_file(&list); + + assert!(run.success, "a successful dump must exit 0.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "one line per requested transaction"); + assert_eq!(lines[0]["tx_hash"].as_str(), Some(TX)); + let path = lines[0]["fixture"]["path"] + .as_str() + .unwrap_or_else(|| panic!("expected a written fixture path: {}", lines[0])); + assert!( + path.ends_with(&format!("{TX}.json")), + "fixture path should be /.json, got: {path}" + ); + assert!(std::path::Path::new(path).exists(), "fixture file must exist at {path}"); + + let elapsed = std::sync::Arc::new(std::sync::Mutex::new(std::time::Duration::ZERO)); + let result = state_test::runner::execute_test_suite(Path::new(path), &elapsed, false, false); + let _ = std::fs::remove_dir_all(&dir); + result.unwrap_or_else(|e| panic!("dumped fixture failed to validate: {e}")); +} + +/// Without `--overwrite`, a second dump into a directory that already holds the +/// fixture fails that target as an infrastructure error. +#[test] +fn test_batch_dump_fixture_dir_refuses_overwrite_without_flag() { + let list = tx_file("batch_dump_ow"); + let dir = std::env::temp_dir().join(format!("mega_evme_batch_dump_ow_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let first = replay( + &cache(), + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + assert!(first.success, "first dump must succeed.\nstderr: {}", first.stderr); + + let second = replay( + &cache(), + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + assert!(!second.success, "overwrite without --overwrite must fail the run"); + let lines = second.ndjson(); + assert_eq!(lines.len(), 1); + assert_eq!(lines[0]["error"]["kind"].as_str(), Some("execution")); + assert!( + lines[0]["error"]["message"] + .as_str() + .is_some_and(|m| m.contains("already exists") && m.contains("--overwrite")), + "expected overwrite refusal: {}", + lines[0] + ); + + // With --overwrite the second dump succeeds and replaces the file. + let third = replay( + &cache(), + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--overwrite", + "--json", + ], + ); + let _ = std::fs::remove_file(&list); + assert!(third.success, "dump with --overwrite must succeed.\nstderr: {}", third.stderr); + let lines = third.ndjson(); + assert!(lines[0]["fixture"]["path"].is_string(), "overwrite must report a written path"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 50bfec5b..2c6f5044 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -67,15 +67,16 @@ Replay every transaction of block `N`, given in decimal or `0x`-prefixed hex. ### Restrictions -Batch mode reports one summary per transaction and has no meaningful semantics for per-transaction artifacts or what-if knobs, so the following are rejected up front with an explanatory error rather than silently ignored: +Batch mode reports one summary per transaction and has no meaningful semantics for single-file fixture dumps, tracing, state dumps, or what-if knobs, so the following are rejected up front with an explanatory error rather than silently ignored: -- `--dump-fixture` +- `--dump-fixture` — use [`--dump-fixture-dir`](#dump-fixture-dir-dir) for batch sedimentation - Transaction overrides (`--override.gas-limit`, `--override.value`, `--override.input`, `--override.input-file`) - `--override.spec` — each block's spec is auto-detected from its timestamp - All trace options (`--trace`, `--trace.output`, `--tracer`, `--trace.*`) - All state dump options (`--dump`, `--dump.output`) Single-transaction replay keeps accepting all of them. +Batch mode additionally accepts [`--dump-fixture-dir`](#dump-fixture-dir-dir) for per-target fixture sedimentation. ### Output @@ -112,10 +113,12 @@ Without `--json`, each transaction is printed with a header naming its hash, blo A final one-line summary (transactions replayed, transactions failed, elapsed time) is logged at `INFO` level, so pass `-vvv` to see it. With [`--verify-receipt`](#receipt-verification), each result line additionally carries a `verification` object. +With [`--dump-fixture-dir`](#dump-fixture-dir-dir), each result line additionally carries a `fixture` object (`path` or `skipped`). ### Exit Status A batch run exits `0` when every requested transaction produced an execution result, and `1` when any of them produced an error entry. +Fixture skips (fidelity gate, BLOCKHASH readers, unsupported shapes) are not error entries and do not fail the run. The NDJSON stream is written to stdout in both cases; diagnostics go to stderr. ### Examples @@ -353,6 +356,45 @@ mega-evme replay --rpc.replay-file ./cap.json --dump-fixture ./fixtures/0xabc123 state-test ./fixtures/0xabc123.json ``` +### `--dump-fixture-dir ` + +Batch-only. +Dump a self-validating fixture for every successfully replayed target into `/.json`. +The fixture content and format match the single-transaction [`--dump-fixture`](#dump-fixture-file) path (same EEST schema, same sorted `megaEnv`, same self-validation via `state-test`). +The directory is created if it does not exist. +Existing files are refused unless `--overwrite` is also set — a refused overwrite is an infrastructure error for that target (`execution`), not a skip. + +Per-target gating mirrors the single-transaction rules, but records a skip instead of failing the run: + +| Gate | Outcome | +| -------------------------------------------------------------------------------- | ------------------------------------------------------------- | +| On-chain receipt unavailable (not in capture, pruned, reorg/divergent inclusion) | `fixture.skipped` with `fidelity-gate-unavailable: …` | +| Fidelity mismatch (gas / status / logs root) | `fixture.skipped` with `fidelity gate failed: …` | +| Target reads `BLOCKHASH` | `fixture.skipped` (fixtures carry no historical block hashes) | +| Unsupported shape (deposit, EIP-7702, unknown spec mapping) | `fixture.skipped` | +| Finalize / write / self-validation failure | infrastructure error entry (`kind: execution`) | +| Pending / unresolvable target | already an error entry; no fixture report | + +`BLOCKHASH` access is isolated per transaction: the access record is cleared before each transaction of the block, so preceding readers do not poison a later target's dump. + +NDJSON result lines gain `"fixture": {"path": "…"}` or `"fixture": {"skipped": ""}`. +Human mode prints one fixture line per target. +An end-of-run `INFO` summary reports written / skipped / failed counts. +Fixture skips do not fail the run; infrastructure failures keep the usual batch exit semantics. + +Registration into `bench/replay/manifest.json` is not performed — corpus curation stays manual. +`--dump-fixture-dir` cannot be combined with `--dump-fixture`, and is rejected in single-transaction mode. + +```bash +# Sweep a whole block offline into per-tx fixtures (skips targets without receipts): +mega-evme replay --rpc.replay-file ./fixtures/blocks.json \ + --block 22945844 --dump-fixture-dir ./fixtures/out --json + +# Sediment a curated list, replacing any previously written files: +mega-evme replay --rpc https://mainnet.megaeth.com/rpc \ + --tx-file ./corpus.txt --dump-fixture-dir ./fixtures/out --overwrite +``` + ## Throughput Benchmark To benchmark a replayed transaction, dump it to a fixture and time the fixture with the `state-test` runner — there is no `replay`-side benchmark flag: @@ -420,9 +462,9 @@ Options marked _(single transaction only)_ are rejected in [batch mode](#batch-r See [RPC Cache and Retry](../configuration/state-management.md#rpc-cache-and-retry). - **Tracing** _(single transaction only)_ — Emit execution traces (call traces, opcode traces, gas profiles, etc.). See [Tracing Overview](../tracing/overview.md). -- **Fixture dump** _(single transaction only)_ — Write a self-validating EEST state-test fixture via `--dump-fixture`. +- **Fixture dump** — Write a self-validating EEST state-test fixture via `--dump-fixture` (single transaction) or `--dump-fixture-dir` (batch). See [Self-Validating Fixture Dump](#self-validating-fixture-dump) above. -- **Throughput benchmark** — Dump a fixture (`--dump-fixture`) and time it with `state-test --bench`. +- **Throughput benchmark** — Dump a fixture (`--dump-fixture` / `--dump-fixture-dir`) and time it with `state-test --bench`. See [Throughput Benchmark](#throughput-benchmark) above. ## Examples From 169b415e2665b555a676a6e69badca6fa57202d8 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 22:48:03 +0800 Subject: [PATCH 08/64] feat(mega-evme): centralized exit-code taxonomy and structured failure output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every failure now maps onto one of four documented exit codes through a single module: 0 success, 1 execution/internal error, 2 receipt verification mismatch, 3 RPC/transport failure. The mapping matches the error enums exhaustively, and `main` is the only place that turns a command result into a process status. Batch replay aggregates its per-target failures into a structured error carrying the counts by class, so the run-level precedence (execution before rpc before mismatch) is resolved from data instead of a formatted message. On failure a run reports once: an `error: ` line on stderr, plus — with `--json` — a compact `{"error":{"code","kind","message"}}` object as the last stdout line, so a machine-readable run never ends with empty stdout. Per-target NDJSON lines and all success-path output are unchanged. --- bin/mega-evme/src/cmd.rs | 29 ++- bin/mega-evme/src/common/error.rs | 47 ++++ bin/mega-evme/src/common/exit.rs | 326 +++++++++++++++++++++++++++ bin/mega-evme/src/common/mod.rs | 2 + bin/mega-evme/src/main.rs | 33 ++- bin/mega-evme/src/replay/batch.rs | 212 +++++++++++++---- bin/mega-evme/tests/common/mod.rs | 21 ++ bin/mega-evme/tests/exit_codes.rs | 212 +++++++++++++++++ bin/mega-evme/tests/replay_batch.rs | 48 +++- bin/mega-evme/tests/replay_verify.rs | 74 ++++-- docs/mega-evme/commands/replay.md | 6 +- docs/mega-evme/overview.md | 26 +++ 12 files changed, 949 insertions(+), 87 deletions(-) create mode 100644 bin/mega-evme/src/common/exit.rs create mode 100644 bin/mega-evme/tests/exit_codes.rs diff --git a/bin/mega-evme/src/cmd.rs b/bin/mega-evme/src/cmd.rs index 1fcd1eb2..f33fc473 100644 --- a/bin/mega-evme/src/cmd.rs +++ b/bin/mega-evme/src/cmd.rs @@ -1,5 +1,4 @@ use clap::{Parser, Subcommand}; -use tracing::error; use crate::common::LogArgs; @@ -42,25 +41,33 @@ pub enum Error { } impl MainCmd { - /// Execute the main command + /// Execute the main command. + /// + /// Failures are returned, never reported here: the binary hands the result + /// to [`crate::common::report_command_result`], which owns the single + /// failure report and the process exit code. pub async fn run(self) -> Result<(), Error> { // Initialize logging first self.log.init(); - // Map instead of `?`: `?` inside an arm returns from `run` directly and - // skips the handler below, leaving the error to be printed on stdout by - // the binary's fallback — which corrupts machine-readable output such as - // the batch replay NDJSON stream. match self.command { Commands::Run(cmd) => cmd.run().await.map_err(Error::from), Commands::Tx(cmd) => cmd.run().await.map_err(Error::from), Commands::Replay(cmd) => cmd.run().await.map_err(Error::from), Commands::Cache(cmd) => cmd.run().map_err(Error::from), } - .inspect_err(|e| { - error!(err = ?e, "Error executing command"); - eprintln!("{e}"); - std::process::exit(1); - }) + } + + /// Whether the selected subcommand was asked for machine-readable output. + /// + /// Read before [`Self::run`] consumes the command, so a failure is reported + /// in the output mode the user asked for. `cache` has no `--json` mode. + pub const fn json_output(&self) -> bool { + match &self.command { + Commands::Run(cmd) => cmd.output_args.json, + Commands::Tx(cmd) => cmd.output_args.json, + Commands::Replay(cmd) => cmd.output_args.json, + Commands::Cache(_) => false, + } } } diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index ce14f008..5c33657d 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -73,6 +73,13 @@ pub enum EvmeError { total: usize, }, + /// A batch replay in which at least one target did not come out clean. + /// + /// Carries the counts by failure class so the exit-code mapping resolves the + /// batch precedence from data instead of parsing this message. + #[error("{0}")] + BatchFailed(BatchFailureCounts), + /// Code hash mismatch #[error("Code hash mismatch: expected {expected}, computed {computed}")] CodeHashMismatch { @@ -87,6 +94,46 @@ pub enum EvmeError { Other(String), } +/// How many targets of a batch replay failed, by failure class. +/// +/// A batch run reports every target on its own output line and then fails once +/// with this summary, so the exit-code mapping can apply the batch precedence +/// (execution before RPC before mismatch) without re-reading the per-target +/// lines or parsing an error message. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct BatchFailureCounts { + /// Targets that failed for an execution, setup, or definitive-answer reason + /// (unknown or pending transaction, block executor rejection). + pub execution: usize, + /// Targets whose question went unanswered because an RPC call failed. + pub rpc: usize, + /// Targets that replayed but did not reproduce their on-chain receipt. + pub mismatched: usize, + /// Targets the run reported on. + pub total: usize, +} + +impl core::fmt::Display for BatchFailureCounts { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "{} of {} target transaction(s) failed to replay ({} execution, {} rpc)", + self.execution + self.rpc, + self.total, + self.execution, + self.rpc, + )?; + if self.mismatched > 0 { + write!( + f, + "; {} replayed transaction(s) did not reproduce the on-chain receipt", + self.mismatched, + )?; + } + Ok(()) + } +} + // Implement DBErrorMarker to allow EvmeError to be used as Database error type impl mega_evm::revm::database::DBErrorMarker for EvmeError {} diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs new file mode 100644 index 00000000..56fa3478 --- /dev/null +++ b/bin/mega-evme/src/common/exit.rs @@ -0,0 +1,326 @@ +//! Central exit-code taxonomy for the `mega-evme` CLI. +//! +//! Verification pipelines branch on the process status, so every failure the +//! CLI can reach maps onto exactly one documented code, and every exit flows +//! through this module: the binary hands its top-level result to +//! [`report_command_result`] and returns the code it produces. No other code +//! path calls `std::process::exit`. +//! +//! | Code | Class | Meaning | +//! | ---- | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +//! | `0` | success | The command completed; with `--verify-receipt`, every verification matched. | +//! | `1` | `execution-error` | Execution or internal error: an EVM/setup failure, bad input, or a definitive negative answer such as an unknown transaction or block. | +//! | `2` | `verification-mismatch` | The run completed, but at least one replay did not reproduce its on-chain receipt. | +//! | `3` | `rpc-failure` | An RPC/transport call failed (endpoint unreachable, transport error, offline replay cache miss): the question went unanswered rather than answered no. | +//! +//! A batch run reports every target individually and then fails once with the +//! counts by failure class ([`BatchFailureCounts`]), which +//! [`ExitCode::from_batch_failures`] resolves by precedence: any +//! execution/internal failure yields `1`, else any RPC failure yields `3`, else +//! any mismatch yields `2`. +//! +//! Extension rule: a new failure class gets a new discriminant. The meaning of +//! an existing code never changes and a retired code is never reused, because +//! callers pin these numbers in scripts. The mapping matches the error enums +//! exhaustively, so a new error variant does not compile until it has been +//! assigned a class. + +use serde::Serialize; +use tracing::error; + +use crate::{ + cmd::Error, + common::{BatchFailureCounts, EvmeError}, +}; + +/// Process exit status of a `mega-evme` run. +/// +/// The discriminants are the wire contract with calling scripts; see the module +/// documentation for the taxonomy and the rule for extending it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum ExitCode { + /// The command completed and every check it ran passed. + Success = 0, + /// Execution or internal error, bad input, or a definitive negative answer. + ExecutionError = 1, + /// The run completed but at least one receipt verification mismatched. + VerificationMismatch = 2, + /// An RPC or transport call failed, so the question went unanswered. + RpcFailure = 3, +} + +impl ExitCode { + /// The numeric status this class exits with. + pub const fn code(self) -> u8 { + self as u8 + } + + /// Kebab-case name of the class, used as the `kind` of the structured error + /// object printed in `--json` mode. + /// + /// This namespace describes the run as a whole and is distinct from the + /// per-target `kind` on a batch NDJSON error line (`not_found`, `pending`, + /// `rpc`, `execution`), which reports why one target failed. + pub const fn kind(self) -> &'static str { + match self { + Self::Success => "success", + Self::ExecutionError => "execution-error", + Self::VerificationMismatch => "verification-mismatch", + Self::RpcFailure => "rpc-failure", + } + } + + /// Map a top-level command error onto its class. + pub fn from_command_error(err: &Error) -> Self { + match err { + Error::Custom(_) => Self::ExecutionError, + Error::Evme(err) => Self::from_evme_error(err), + } + } + + /// Map a command failure onto its class. + /// + /// The match is exhaustive on purpose: a new [`EvmeError`] variant must + /// pick a class here rather than inherit one by default. + pub fn from_evme_error(err: &EvmeError) -> Self { + match err { + // The endpoint never answered: unreachable, transport-level + // failure, or an offline replay file without the response. + EvmeError::RpcTransportError(_) | EvmeError::RpcError(_) => Self::RpcFailure, + // Answered, definitively negative. + EvmeError::TransactionNotFound(_) | + EvmeError::BlockNotFound(_) | + // Execution, setup, input, and internal failures. + EvmeError::BlockExecutionError(_) | + EvmeError::InvalidBytecode(_) | + EvmeError::FileRead(_) | + EvmeError::InvalidHex(_) | + EvmeError::ExecutionError(_) | + EvmeError::InvalidInput(_) | + EvmeError::FixtureError(_) | + EvmeError::UnsupportedTxType(_) | + EvmeError::CodeHashMismatch { .. } | + EvmeError::Other(_) => Self::ExecutionError, + // The run completed; the replay diverged from the chain. + EvmeError::VerificationMismatch { .. } => Self::VerificationMismatch, + EvmeError::BatchFailed(counts) => Self::from_batch_failures(counts), + } + } + + /// Resolve the class of a batch run from its failure counts. + /// + /// Precedence: an execution/internal failure outranks an RPC failure, which + /// outranks a mismatch. A target that never replayed was also never + /// verified, so reporting such a run as a mismatch would overstate what it + /// found. + pub const fn from_batch_failures(counts: &BatchFailureCounts) -> Self { + if counts.execution > 0 { + Self::ExecutionError + } else if counts.rpc > 0 { + Self::RpcFailure + } else if counts.mismatched > 0 { + Self::VerificationMismatch + } else { + Self::Success + } + } +} + +impl From for std::process::ExitCode { + fn from(code: ExitCode) -> Self { + Self::from(code.code()) + } +} + +/// The structured failure object `--json` runs print as their last stdout line. +#[derive(Debug, Serialize)] +struct ErrorEnvelope<'a> { + error: ErrorBody<'a>, +} + +/// Payload of an [`ErrorEnvelope`]. +#[derive(Debug, Serialize)] +struct ErrorBody<'a> { + /// The process exit code this failure produces. + code: u8, + /// Kebab-case failure class, see [`ExitCode::kind`]. + kind: &'static str, + /// The error's `Display` text, never its `Debug` form. + message: &'a str, +} + +/// Report a finished command and return the code the process exits with. +/// +/// A successful run prints nothing. A failure is reported exactly once on +/// stderr as `error: `, always `Display`-formatted (the message itself +/// may carry continuation lines, such as an RPC error's re-capture hint), and +/// in `--json` mode additionally as the structured object on stdout — the last +/// line, so a machine-readable run never ends with empty stdout, and in batch +/// mode the object follows the per-target lines. The `tracing` event carries +/// the same text and stays silent unless `-v` was given. +pub fn report_command_result(result: Result<(), Error>, json: bool) -> ExitCode { + let Err(err) = result else { + return ExitCode::Success; + }; + + let code = ExitCode::from_command_error(&err); + error!(err = %err, exit_code = code.code(), "Command failed"); + + let message = err.to_string(); + eprintln!("error: {message}"); + if json { + let envelope = ErrorEnvelope { + error: ErrorBody { code: code.code(), kind: code.kind(), message: &message }, + }; + println!("{}", serde_json::to_string(&envelope).expect("failed to serialize the error")); + } + + code +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + + /// Every failure class the taxonomy defines keeps its documented code. + #[test] + fn test_exit_codes_are_stable() { + assert_eq!(ExitCode::Success.code(), 0); + assert_eq!(ExitCode::ExecutionError.code(), 1); + assert_eq!(ExitCode::VerificationMismatch.code(), 2); + assert_eq!(ExitCode::RpcFailure.code(), 3); + } + + /// The `kind` namespace is kebab-case and one name per class. + #[test] + fn test_exit_code_kinds_are_kebab_case() { + for code in [ + ExitCode::Success, + ExitCode::ExecutionError, + ExitCode::VerificationMismatch, + ExitCode::RpcFailure, + ] { + let kind = code.kind(); + assert!( + kind.chars().all(|c| c.is_ascii_lowercase() || c == '-'), + "kind must be kebab-case, got: {kind}" + ); + } + } + + /// An unanswered question is an RPC failure, whatever shape it arrived in. + #[test] + fn test_rpc_class_errors_map_to_three() { + for err in [ + EvmeError::RpcError("cache miss in offline replay file".to_string()), + EvmeError::RpcTransportError( + alloy_provider::transport::TransportErrorKind::custom_str("connection refused"), + ), + ] { + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::RpcFailure, + "unexpected class for {err}" + ); + } + } + + /// Definitive answers, bad input, and internal failures all exit 1. + #[test] + fn test_execution_class_errors_map_to_one() { + for err in [ + EvmeError::TransactionNotFound(B256::ZERO), + EvmeError::BlockNotFound(7), + EvmeError::ExecutionError("halted".to_string()), + EvmeError::InvalidInput("no transaction hashes".to_string()), + EvmeError::FixtureError("unsupported transaction".to_string()), + EvmeError::UnsupportedTxType(0x7e), + EvmeError::CodeHashMismatch { expected: B256::ZERO, computed: B256::ZERO }, + EvmeError::FileRead(std::io::Error::other("boom")), + EvmeError::InvalidHex(alloy_primitives::hex::FromHexError::OddLength), + EvmeError::Other("something else".to_string()), + ] { + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::ExecutionError, + "unexpected class for {err}" + ); + } + } + + /// A completed run whose replay diverged from the chain exits 2. + #[test] + fn test_verification_mismatch_maps_to_two() { + let err = EvmeError::VerificationMismatch { mismatched: 1, total: 3 }; + assert_eq!(ExitCode::from_evme_error(&err), ExitCode::VerificationMismatch); + } + + /// The top-level wrapper adds no class of its own beyond the internal one. + #[test] + fn test_command_error_classes() { + assert_eq!( + ExitCode::from_command_error(&Error::Custom("bad state")), + ExitCode::ExecutionError + ); + assert_eq!( + ExitCode::from_command_error(&Error::Evme(EvmeError::RpcError("down".to_string()))), + ExitCode::RpcFailure + ); + } + + /// Batch precedence: execution beats rpc beats mismatch. + #[test] + fn test_batch_failure_precedence() { + let mixed = BatchFailureCounts { execution: 1, rpc: 2, mismatched: 3, total: 6 }; + assert_eq!(ExitCode::from_batch_failures(&mixed), ExitCode::ExecutionError); + + let rpc_only = BatchFailureCounts { execution: 0, rpc: 2, mismatched: 3, total: 6 }; + assert_eq!(ExitCode::from_batch_failures(&rpc_only), ExitCode::RpcFailure); + + let mismatch_only = BatchFailureCounts { execution: 0, rpc: 0, mismatched: 3, total: 6 }; + assert_eq!(ExitCode::from_batch_failures(&mismatch_only), ExitCode::VerificationMismatch); + + let clean = BatchFailureCounts::default(); + assert_eq!(ExitCode::from_batch_failures(&clean), ExitCode::Success); + } + + /// The aggregate error carries its counts through the top-level mapping. + #[test] + fn test_batch_failed_error_maps_by_counts() { + let rpc_only = EvmeError::BatchFailed(BatchFailureCounts { + execution: 0, + rpc: 1, + ..Default::default() + }); + assert_eq!(ExitCode::from_evme_error(&rpc_only), ExitCode::RpcFailure); + + let with_execution = EvmeError::BatchFailed(BatchFailureCounts { + execution: 1, + rpc: 1, + mismatched: 1, + total: 3, + }); + assert_eq!(ExitCode::from_evme_error(&with_execution), ExitCode::ExecutionError); + } + + /// The serialized object is a single compact line with the documented shape. + #[test] + fn test_error_envelope_shape() { + let code = ExitCode::RpcFailure; + let envelope = ErrorEnvelope { + error: ErrorBody { code: code.code(), kind: code.kind(), message: "endpoint down" }, + }; + let line = serde_json::to_string(&envelope).expect("serialize"); + + assert!(!line.contains('\n'), "the object must be a single line: {line}"); + let value: serde_json::Value = serde_json::from_str(&line).expect("valid JSON"); + assert_eq!( + value, + serde_json::json!({ + "error": { "code": 3, "kind": "rpc-failure", "message": "endpoint down" } + }) + ); + } +} diff --git a/bin/mega-evme/src/common/mod.rs b/bin/mega-evme/src/common/mod.rs index 96ddf3e2..7555f366 100644 --- a/bin/mega-evme/src/common/mod.rs +++ b/bin/mega-evme/src/common/mod.rs @@ -1,5 +1,6 @@ mod env; mod error; +mod exit; mod hardfork; mod hex; mod logging; @@ -12,6 +13,7 @@ mod tx_override; pub use env::*; pub use error::*; +pub use exit::*; pub use hardfork::*; pub use hex::*; pub use logging::*; diff --git a/bin/mega-evme/src/main.rs b/bin/mega-evme/src/main.rs index da36f3de..d357a6af 100644 --- a/bin/mega-evme/src/main.rs +++ b/bin/mega-evme/src/main.rs @@ -3,15 +3,36 @@ //! All business logic lives in the `mega_evme` library crate (`src/lib.rs`). //! This binary is intentionally minimal: parse CLI arguments, install the panic //! hook, dispatch to the parsed command, and exit. +//! +//! This is the only place a finished command becomes a process status; the +//! taxonomy of statuses lives in `common::exit`. + +use std::process::ExitCode; use clap::Parser; -use mega_evme::{ - cmd::{Error, MainCmd}, - set_thread_panic_hook, -}; +use mega_evme::{cmd::MainCmd, report_command_result, set_thread_panic_hook}; #[tokio::main] -async fn main() -> std::result::Result<(), Error> { +async fn main() -> ExitCode { set_thread_panic_hook(); - MainCmd::parse().run().await.inspect_err(|e| println!("{e:?}")) + + let cmd = match MainCmd::try_parse() { + Ok(cmd) => cmd, + Err(err) => { + // `--help` / `--version` are not failures: clap writes them to + // stdout and the run exits 0. A usage error is bad input, which the + // taxonomy classifies with the other input errors. + let _ = err.print(); + return if err.use_stderr() { + ExitCode::from(mega_evme::ExitCode::ExecutionError) + } else { + ExitCode::SUCCESS + }; + } + }; + + // Read the output mode before `run` consumes the command. + let json = cmd.json_output(); + let result = cmd.run().await; + ExitCode::from(report_command_result(result, json)) } diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index df398043..6ab55dec 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -43,8 +43,8 @@ use tracing::{debug, info, warn}; use crate::{ common::{ - op_receipt_to_tx_receipt, print_execution_summary, print_receipt, EvmeExternalEnvs, - ExecutionSummary, OpTxReceipt, + op_receipt_to_tx_receipt, print_execution_summary, print_receipt, BatchFailureCounts, + EvmeExternalEnvs, ExecutionSummary, OpTxReceipt, }, replay::get_hardfork_config, ChainArgs, EvmeState, @@ -188,6 +188,74 @@ struct FailedTx { message: String, } +/// Running tally of a batch run's per-target outcomes. +/// +/// A batch reports each target as it goes and fails once at the end, so the +/// outcome classes are counted here rather than recovered from the emitted +/// lines. +#[derive(Debug, Default)] +struct BatchTally { + /// Targets that produced an execution result. + replayed: usize, + /// Targets compared against an on-chain receipt. + verified: usize, + /// Failed and mismatched targets, by class. + counts: BatchFailureCounts, +} + +impl BatchTally { + /// Count one reported target. + fn record(&mut self, entry: &BatchEntry) { + match entry { + BatchEntry::Executed(tx) => { + self.replayed += 1; + if let Some(verification) = &tx.verification { + self.verified += 1; + if !verification.matched { + self.counts.mismatched += 1; + } + } + } + // A transaction the endpoint does not know, or that is not mined + // yet, is a definitive answer about the target rather than an + // unanswered question, so it counts as an execution failure. + BatchEntry::Failed(tx) => match tx.kind { + BatchErrorKind::Rpc => self.counts.rpc += 1, + BatchErrorKind::NotFound | BatchErrorKind::Pending | BatchErrorKind::Execution => { + self.counts.execution += 1 + } + }, + } + } + + /// Targets that produced no execution result. + const fn failed(&self) -> usize { + self.counts.execution + self.counts.rpc + } + + /// The run's terminal error, or `None` when every target came out clean. + /// + /// Infrastructure failures are reported with their counts by class so the + /// exit-code mapping resolves the precedence between them and a mismatch; a + /// run whose only finding is divergence fails as the mismatch it is. + /// Fixture skips never count as failures. + fn into_error(self) -> Option { + if self.failed() > 0 { + return Some(ReplayError::BatchFailed(BatchFailureCounts { + total: self.replayed + self.failed(), + ..self.counts + })); + } + if self.counts.mismatched > 0 { + return Some(ReplayError::VerificationMismatch { + mismatched: self.counts.mismatched, + total: self.verified, + }); + } + None + } +} + /// One block's worth of work. struct BlockJob { /// Number of the block holding the targets. @@ -249,10 +317,12 @@ struct BatchErrorBody<'a> { /// Returns an error when at least one target produced an infrastructure error /// entry, so the process exits non-zero; execution outcomes never fail the run. /// Fixture skips (fidelity gate, BLOCKHASH readers, unsupported tx shapes) do -/// not fail the run. With `--verify-receipt`, a run in which every target -/// replayed but some diverged from its on-chain receipt fails with -/// [`ReplayError::VerificationMismatch`] instead — a distinct variant, so a -/// divergence is never confused with a target that could not be replayed. +/// not fail the run. The failure carries the counts by class +/// ([`ReplayError::BatchFailed`]), which decide the exit code. With +/// `--verify-receipt`, a run in which every target replayed but some diverged +/// from its on-chain receipt fails with [`ReplayError::VerificationMismatch`] +/// instead — a distinct variant, so a divergence is never confused with a +/// target that could not be replayed. pub(super) async fn run

( provider: &P, chain_id: u64, @@ -264,10 +334,7 @@ where P: Provider + Clone + std::fmt::Debug, { let start = Instant::now(); - let mut replayed = 0usize; - let mut failed = 0usize; - let mut verified = 0usize; - let mut mismatched = 0usize; + let mut tally = BatchTally::default(); let mut fixtures_written = 0usize; let mut fixtures_skipped = 0usize; let mut fixtures_failed = 0usize; @@ -297,8 +364,9 @@ where "Batch replay of a transaction list", ); for failure in failures { - failed += 1; - emit(&BatchEntry::Failed(failure), report.json); + let entry = BatchEntry::Failed(failure); + tally.record(&entry); + emit(&entry, report.json); } jobs } @@ -309,32 +377,32 @@ where replay_block(provider, chain_id, job, external_envs.clone(), &report).await; fixtures_failed += block_result.fixture_write_failures; for entry in block_result.entries { - match &entry { - BatchEntry::Executed(tx) => { - replayed += 1; - if let Some(verification) = &tx.verification { - verified += 1; - if !verification.matched { - mismatched += 1; - } - } - if let Some(fixture) = &tx.fixture { - if fixture.path.is_some() { - fixtures_written += 1; - } else { - fixtures_skipped += 1; - } + if let BatchEntry::Executed(tx) = &entry { + if let Some(fixture) = &tx.fixture { + if fixture.path.is_some() { + fixtures_written += 1; + } else { + fixtures_skipped += 1; } } - BatchEntry::Failed(_) => failed += 1, } + tally.record(&entry); emit(&entry, report.json); } } - info!(replayed, failed, elapsed = ?start.elapsed(), "Batch replay finished"); + info!( + replayed = tally.replayed, + failed = tally.failed(), + elapsed = ?start.elapsed(), + "Batch replay finished", + ); if report.verify_receipt { - info!(verified, mismatched, "On-chain receipt verification finished"); + info!( + verified = tally.verified, + mismatched = tally.counts.mismatched, + "On-chain receipt verification finished", + ); } if report.dump_fixture_dir.is_some() { info!( @@ -345,19 +413,7 @@ where ); } - // Infrastructure failures keep their own error: a target that never replayed - // was also never verified, so reporting it as a mismatch would overstate - // what the run actually found. Fixture skips never contribute to `failed`. - if failed > 0 { - return Err(ReplayError::Other(format!( - "{failed} of {} target transaction(s) failed to replay", - replayed + failed - ))); - } - if mismatched > 0 { - return Err(ReplayError::VerificationMismatch { mismatched, total: verified }); - } - Ok(()) + tally.into_error().map_or(Ok(()), Err) } /// Resolve each requested hash to its containing block. @@ -1038,10 +1094,80 @@ pub(super) fn parse_block_number(value: &str) -> std::result::Result BatchTally { + let mut tally = BatchTally::default(); + for kind in failures { + tally.record(&failure(B256::ZERO, *kind, String::new())); + } + // Verified targets are counted through the executed entries, which need + // a full `ExecutedTx`; the tally fields they feed are set directly. + tally.replayed = replayed; + tally.verified = replayed; + tally.counts.mismatched = mismatched; + tally + } + + /// The exit code a batch run with these outcomes ends with. + fn exit_code(failures: &[BatchErrorKind], replayed: usize, mismatched: usize) -> ExitCode { + tally(failures, replayed, mismatched) + .into_error() + .map_or(ExitCode::Success, |err| ExitCode::from_evme_error(&err)) + } + + /// A clean run has nothing to report and exits 0. + #[test] + fn test_batch_tally_clean_run_has_no_error() { + assert!(tally(&[], 3, 0).into_error().is_none()); + assert_eq!(exit_code(&[], 3, 0), ExitCode::Success); + } + + /// Mixed failures are ranked by class: an execution failure outranks the + /// rest, an RPC failure outranks a mismatch. + #[test] + fn test_batch_tally_failure_precedence() { + use BatchErrorKind::{Execution, NotFound, Pending, Rpc}; + + assert_eq!(exit_code(&[Execution, Rpc], 1, 1), ExitCode::ExecutionError); + assert_eq!(exit_code(&[Rpc, Rpc], 1, 1), ExitCode::RpcFailure); + assert_eq!(exit_code(&[], 2, 1), ExitCode::VerificationMismatch); + // A definitive answer about a target is an execution-class failure. + assert_eq!(exit_code(&[NotFound], 1, 0), ExitCode::ExecutionError); + assert_eq!(exit_code(&[Pending], 1, 0), ExitCode::ExecutionError); + } + + /// The aggregate error carries the counts by class, not a formatted string + /// the exit mapping would have to parse. + #[test] + fn test_batch_tally_aggregate_carries_counts() { + use BatchErrorKind::{Execution, NotFound, Rpc}; + + let err = tally(&[Execution, NotFound, Rpc], 2, 1).into_error().expect("run failed"); + let ReplayError::BatchFailed(counts) = err else { + panic!("infrastructure failures must aggregate: {err:?}"); + }; + assert_eq!(counts, BatchFailureCounts { execution: 2, rpc: 1, mismatched: 1, total: 5 }); + assert!( + counts.to_string().contains("3 of 5 target transaction(s) failed to replay"), + "unexpected message: {counts}" + ); + } + + /// A run whose only finding is divergence fails as the mismatch it is. + #[test] + fn test_batch_tally_mismatch_only_reports_the_verification_error() { + let err = tally(&[], 4, 2).into_error().expect("run failed"); + assert!( + matches!(err, ReplayError::VerificationMismatch { mismatched: 2, total: 4 }), + "unexpected error: {err:?}" + ); + } + #[test] fn test_parse_tx_hash_list_skips_blanks_and_comments() { let contents = diff --git a/bin/mega-evme/tests/common/mod.rs b/bin/mega-evme/tests/common/mod.rs index c0cd0626..df3fca73 100644 --- a/bin/mega-evme/tests/common/mod.rs +++ b/bin/mega-evme/tests/common/mod.rs @@ -107,6 +107,27 @@ impl MockRpcServer { } } +/// Parse every top-level JSON value a run printed on stdout. +/// +/// Streaming parse, so it covers both the pretty-printed single-transaction +/// summary and the compact NDJSON of a batch run — in either case followed by +/// the structured error object a failing `--json` run ends with. +pub(crate) fn json_values(stdout: &str) -> Vec { + serde_json::Deserializer::from_str(stdout) + .into_iter::() + .collect::>() + .unwrap_or_else(|e| panic!("stdout is not a JSON stream ({e}):\n{stdout}")) +} + +/// Whether a printed value is the run-level error object of a failing `--json` +/// run (`{"error":{"code":…,"kind":…,"message":…}}`). +/// +/// A per-target NDJSON error line carries its transaction hash alongside the +/// `error` key, so the single-key shape identifies the run-level object. +pub(crate) fn is_run_error(value: &serde_json::Value) -> bool { + value.as_object().is_some_and(|obj| obj.len() == 1 && obj.contains_key("error")) +} + /// Build [`RpcArgs`] for a test pointed at `url` with the on-disk cache disabled. /// /// Defaults: `--rpc.no-cache-file` (in-memory LRU still applies; no disk persistence), diff --git a/bin/mega-evme/tests/exit_codes.rs b/bin/mega-evme/tests/exit_codes.rs new file mode 100644 index 00000000..d3deee79 --- /dev/null +++ b/bin/mega-evme/tests/exit_codes.rs @@ -0,0 +1,212 @@ +//! Integration tests for the CLI's exit-code taxonomy and its failure output. +//! +//! They run fully offline against the committed RPC capture +//! (`fixtures/replay_offline.cache.json`), so they are deterministic: a hash the +//! capture cannot answer models an endpoint that never answers, and the +//! validation paths need no provider at all. The mismatch class (exit 2) is +//! covered by `replay_verify.rs`, which doctors a copy of the same capture. + +use std::process::{Command, Output}; + +mod common; + +/// Offline RPC capture used as the replay file. +const CACHE: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/replay_offline.cache.json"); + +/// The transaction the committed capture can replay. +const TX_OK: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; + +/// A hash the capture holds no response for: the question goes unanswered. +const UNANSWERABLE_TX: &str = "0x0000000000000000000000000000000000000000000000000000000000000001"; + +/// Outcome of one `mega-evme` invocation. +struct Run { + code: Option, + stdout: String, + stderr: String, +} + +impl Run { + /// The process exit code the run ended with. + fn code(&self) -> i32 { + self.code.expect("mega-evme was killed by a signal") + } + + /// The structured error object a failing `--json` run ends with. + fn error_object(&self) -> serde_json::Value { + let values = common::json_values(&self.stdout); + let last = values + .last() + .unwrap_or_else(|| panic!("a failing --json run must not leave stdout empty")); + assert!( + common::is_run_error(last), + "the last stdout value must be the error object, got: {last}" + ); + last.clone() + } + + /// How many failure reports stderr carries. + /// + /// Counted by the report prefix: a message may itself span lines (an RPC + /// error appends a re-capture hint), and only the report opens one. + fn error_lines(&self) -> usize { + self.stderr.lines().filter(|line| line.starts_with("error: ")).count() + } +} + +fn run(args: &[&str]) -> Run { + let output: Output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args(args) + .output() + .expect("failed to run mega-evme"); + Run { + code: output.status.code(), + stdout: String::from_utf8(output.stdout).expect("stdout is utf-8"), + stderr: String::from_utf8(output.stderr).expect("stderr is utf-8"), + } +} + +/// Run `replay` against the committed offline capture. +fn replay(args: &[&str]) -> Run { + let mut argv = vec!["replay", "--rpc.replay-file", CACHE]; + argv.extend_from_slice(args); + run(&argv) +} + +/// Write a `--tx-file` holding `contents`, and return its path. +fn tx_file(name: &str, contents: &str) -> std::path::PathBuf { + let path = + std::env::temp_dir().join(format!("mega_evme_exit_{name}_{}.txt", std::process::id())); + std::fs::write(&path, contents).expect("write tx list"); + path +} + +/// Bad input is an execution-class failure: exit 1, with the structured object +/// as the last stdout line. +#[test] +fn test_invalid_input_exits_one_with_a_json_error_object() { + let list = tx_file("bad_hash", "not-a-hash\n"); + + let run = replay(&["--tx-file", list.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&list); + + assert_eq!(run.code(), 1, "bad input exits 1.\nstderr: {}", run.stderr); + let error = run.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(1)); + assert_eq!(error["error"]["kind"].as_str(), Some("execution-error")); + assert!( + error["error"]["message"].as_str().is_some_and(|m| m.contains("not-a-hash")), + "the message must name the offending input: {error}" + ); +} + +/// A rejected flag combination is bad input too, and still ends `--json` stdout +/// with the error object rather than nothing at all. +#[test] +fn test_rejected_flag_combination_exits_one_with_a_json_error_object() { + let run = replay(&["--dump-fixture-dir", "/tmp/mega-evme-should-not-exist", "--json", TX_OK]); + + assert_eq!(run.code(), 1, "a rejected flag combination exits 1.\nstderr: {}", run.stderr); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("execution-error")); +} + +/// A transaction the offline capture cannot answer is an RPC failure: the +/// question went unanswered, which is distinct from a definitive "no". +#[test] +fn test_offline_cache_miss_exits_rpc_failure_with_a_json_error_object() { + let run = replay(&["--json", UNANSWERABLE_TX]); + + assert_eq!(run.code(), 3, "a cache miss exits 3.\nstderr: {}", run.stderr); + let error = run.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + assert!( + error["error"]["message"].as_str().is_some_and(|m| m.contains("cache miss")), + "the message must explain the miss: {error}" + ); +} + +/// A batch run's error object follows the per-target lines, so a parser reading +/// the stream sees every target before the run-level verdict. +#[test] +fn test_batch_error_object_follows_the_per_target_lines() { + let list = tx_file("batch_miss", &format!("{UNANSWERABLE_TX}\n")); + + let run = replay(&["--tx-file", list.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&list); + + assert_eq!(run.code(), 3, "an unanswered target exits 3.\nstderr: {}", run.stderr); + let values = common::json_values(&run.stdout); + assert_eq!(values.len(), 2, "one per-target line plus the error object:\n{}", run.stdout); + assert_eq!( + values[0]["tx_hash"].as_str(), + Some(UNANSWERABLE_TX), + "the per-target line comes first: {}", + values[0] + ); + assert!(common::is_run_error(&values[1]), "the error object comes last: {}", values[1]); +} + +/// Human mode reports the failure once, as `Display` text, and leaves stdout +/// untouched. +#[test] +fn test_human_failure_prints_exactly_one_error_line() { + let list = tx_file("human", "not-a-hash\n"); + + let run = replay(&["--tx-file", list.to_str().unwrap()]); + let _ = std::fs::remove_file(&list); + + assert_eq!(run.code(), 1); + assert_eq!(run.stderr.lines().count(), 1, "exactly one line on stderr:\n{}", run.stderr); + assert!(run.stderr.starts_with("error: "), "the report is prefixed:\n{}", run.stderr); + assert!( + !run.stderr.contains("Evme(") && !run.stderr.contains("InvalidInput("), + "the report must be Display-formatted, not Debug:\n{}", + run.stderr + ); + assert!(run.stdout.is_empty(), "human mode prints no failure on stdout:\n{}", run.stdout); +} + +/// A message that carries its own extra lines (the RPC hint) is still reported +/// exactly once, and never in `Debug` form. +#[test] +fn test_human_failure_reports_a_multi_line_message_once() { + let run = replay(&[UNANSWERABLE_TX]); + + assert_eq!(run.code(), 3); + assert_eq!(run.error_lines(), 1, "exactly one report on stderr:\n{}", run.stderr); + assert!( + !run.stderr.contains("Evme(") && !run.stderr.contains("RpcError("), + "the report must be Display-formatted, not Debug:\n{}", + run.stderr + ); + assert!(run.stdout.is_empty(), "human mode prints no failure on stdout:\n{}", run.stdout); +} + +/// A successful run exits 0 and prints no error object: the failure surface +/// leaves the success output untouched. +#[test] +fn test_successful_run_exits_zero_without_an_error_object() { + let run = replay(&["--json", TX_OK]); + + assert_eq!(run.code(), 0, "a faithful replay exits 0.\nstderr: {}", run.stderr); + let values = common::json_values(&run.stdout); + assert_eq!(values.len(), 1, "only the summary is printed:\n{}", run.stdout); + assert!(!common::is_run_error(&values[0]), "a successful run prints no error object"); + assert_eq!(run.error_lines(), 0, "a successful run reports nothing on stderr"); +} + +/// A usage error is bad input, so it joins the execution class instead of +/// colliding with the mismatch code; `--help` stays a successful run. +#[test] +fn test_usage_errors_exit_one_and_help_exits_zero() { + // No replay target: rejected by argument parsing. + let usage = run(&["replay"]); + assert_eq!(usage.code(), 1, "a usage error exits 1.\nstderr: {}", usage.stderr); + assert!(usage.stdout.is_empty(), "a usage error prints nothing on stdout"); + + let help = run(&["--help"]); + assert_eq!(help.code(), 0, "--help exits 0"); + assert!(help.stdout.contains("mega-evme"), "--help prints usage on stdout"); +} diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 770ab806..26603a9a 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -12,6 +12,8 @@ use std::process::Command; +mod common; + /// Block fully covered by the envelope, and its transaction count. const BLOCK: u64 = 22_945_844; const BLOCK_TX_COUNT: usize = 23; @@ -57,16 +59,40 @@ fn replay(args: &[&str], expect_success: bool) -> String { String::from_utf8(output.stdout).expect("stdout is utf-8") } -/// Parse NDJSON stdout into one JSON value per line. +/// Run `replay` offline and return its stdout plus its exit code. +fn replay_with_code(args: &[&str]) -> (String, Option) { + let envelope = envelope(); + let mut cmd = mega_evme(); + cmd.args(["replay", "--rpc.replay-file", &envelope]); + cmd.args(args); + let output = cmd.output().expect("failed to run mega-evme"); + (String::from_utf8(output.stdout).expect("stdout is utf-8"), output.status.code()) +} + +/// Parse NDJSON stdout into one JSON value per line, dropping the structured +/// error object a failing run ends with. fn ndjson(stdout: &str) -> Vec { - stdout + let mut lines: Vec = stdout .lines() .map(|line| { assert!(!line.trim().is_empty(), "NDJSON output must not contain blank lines"); serde_json::from_str(line) .unwrap_or_else(|e| panic!("stdout line is not compact JSON ({e}): {line}")) }) - .collect() + .collect(); + if lines.last().is_some_and(common::is_run_error) { + lines.pop(); + } + lines +} + +/// The structured error object a failing `--json` run ends with. +fn run_error(stdout: &str) -> serde_json::Value { + let last = stdout.lines().last().unwrap_or_else(|| panic!("stdout must not be empty")); + let value: serde_json::Value = serde_json::from_str(last) + .unwrap_or_else(|e| panic!("last stdout line is not compact JSON ({e}): {last}")); + assert!(common::is_run_error(&value), "the last line must be the error object: {value}"); + value } /// `--block N --json` emits exactly one NDJSON line per transaction of the @@ -161,7 +187,8 @@ fn test_replay_tx_file_spans_blocks_in_order() { } /// A hash that cannot be resolved is reported as an error entry, the remaining -/// targets still replay, and the process exits non-zero. +/// targets still replay, and the process exits non-zero with the class of the +/// failure — here an unanswered lookup against the offline envelope. #[test] #[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_tx_file_reports_unresolved_targets_and_exits_nonzero() { @@ -170,7 +197,7 @@ fn test_replay_tx_file_reports_unresolved_targets_and_exits_nonzero() { std::env::temp_dir().join(format!("mega_evme_tx_list_bad_{}.txt", std::process::id())); std::fs::write(&path, format!("{unknown}\n{}\n", BLOCK_TXS[1].0)).expect("write tx list"); - let stdout = replay(&["--tx-file", path.to_str().unwrap(), "--json"], false); + let (stdout, code) = replay_with_code(&["--tx-file", path.to_str().unwrap(), "--json"]); let _ = std::fs::remove_file(&path); let lines = ndjson(&stdout); @@ -180,6 +207,10 @@ fn test_replay_tx_file_reports_unresolved_targets_and_exits_nonzero() { assert!(lines[0]["error"]["message"].is_string(), "failure line carries a message"); assert_eq!(lines[1]["tx_hash"].as_str(), Some(BLOCK_TXS[1].0)); assert_eq!(lines[1]["success"].as_bool(), Some(true), "the resolvable target still replays"); + + // A hash the envelope cannot answer is an RPC-class failure for the run. + assert_eq!(code, Some(3), "an unanswered target exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); } /// `--verify-receipt` against an envelope that carries no receipts: every target @@ -191,7 +222,8 @@ fn test_replay_tx_file_reports_unresolved_targets_and_exits_nonzero() { #[test] #[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_block_verify_receipt_without_receipts_reports_rpc_errors() { - let stdout = replay(&["--block", &BLOCK.to_string(), "--verify-receipt", "--json"], false); + let (stdout, code) = + replay_with_code(&["--block", &BLOCK.to_string(), "--verify-receipt", "--json"]); let lines = ndjson(&stdout); assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); @@ -203,6 +235,10 @@ fn test_replay_block_verify_receipt_without_receipts_reports_rpc_errors() { ); assert!(line.get("verification").is_none(), "an unverified target carries no verdict"); } + + // Unverified targets are RPC-class failures, never mismatches. + assert_eq!(code, Some(3), "a run of unverified targets exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); } /// Batch mode rejects the single-transaction-only flags before doing any work. diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index 4b1bd80e..28a199d1 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -12,6 +12,8 @@ use std::{ process::Command, }; +mod common; + /// Offline RPC capture, including the transaction's on-chain receipt. const CACHE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/replay_offline.cache.json"); @@ -25,26 +27,50 @@ const GAS_USED: u64 = 75_514; /// Outcome of one `mega-evme replay` invocation. struct Run { success: bool, + code: Option, stdout: String, stderr: String, } impl Run { + /// The process exit code the run ended with. + fn code(&self) -> i32 { + self.code.expect("mega-evme was killed by a signal") + } + + /// The results printed on stdout, without the structured error object a + /// failing `--json` run ends with. + fn results(&self) -> Vec { + let mut values = common::json_values(&self.stdout); + if values.last().is_some_and(common::is_run_error) { + values.pop(); + } + values + } + /// Parse the stdout of a `--json` single-transaction run. fn json(&self) -> serde_json::Value { - serde_json::from_str(self.stdout.trim()) - .unwrap_or_else(|e| panic!("stdout is not JSON ({e}):\n{}", self.stdout)) + let mut results = self.results(); + assert_eq!(results.len(), 1, "expected one summary on stdout:\n{}", self.stdout); + results.pop().expect("checked above") } /// Parse the stdout of a `--json` batch run as one value per NDJSON line. fn ndjson(&self) -> Vec { - self.stdout - .lines() - .map(|line| { - serde_json::from_str(line) - .unwrap_or_else(|e| panic!("stdout line is not compact JSON ({e}): {line}")) - }) - .collect() + self.results() + } + + /// The structured error object a failing `--json` run ends with. + fn error_object(&self) -> serde_json::Value { + let values = common::json_values(&self.stdout); + let last = values + .last() + .unwrap_or_else(|| panic!("a failing --json run must not leave stdout empty")); + assert!( + common::is_run_error(last), + "the last stdout value must be the error object, got: {last}" + ); + last.clone() } } @@ -57,6 +83,7 @@ fn replay(cache: &Path, args: &[&str]) -> Run { .expect("failed to run mega-evme"); Run { success: output.status.success(), + code: output.status.code(), stdout: String::from_utf8(output.stdout).expect("stdout is utf-8"), stderr: String::from_utf8(output.stderr).expect("stderr is utf-8"), } @@ -166,7 +193,8 @@ fn test_single_transaction_json_is_unchanged_without_the_flag() { assert_eq!(stripped, plain.json(), "--verify-receipt must add the verdict and nothing else"); } -/// A gas divergence is reported as a `gas_used` diff and fails the run. +/// A gas divergence is reported as a `gas_used` diff and fails the run with the +/// dedicated mismatch exit code. #[test] fn test_verify_receipt_reports_a_gas_mismatch() { let path = doctored_cache("gas", |receipt| receipt["gasUsed"] = "0x1".into()); @@ -174,7 +202,7 @@ fn test_verify_receipt_reports_a_gas_mismatch() { let run = replay(&path, &["--verify-receipt", "--json", TX]); let _ = std::fs::remove_file(&path); - assert!(!run.success, "a mismatch must exit non-zero"); + assert_eq!(run.code(), 2, "a mismatch exits 2.\nstderr: {}", run.stderr); assert_eq!( run.json()["verification"], serde_json::json!({ @@ -182,6 +210,8 @@ fn test_verify_receipt_reports_a_gas_mismatch() { "diff": { "gas_used": { "onchain": 1, "replay": GAS_USED } }, }) ); + assert_eq!(run.error_object()["error"]["code"].as_u64(), Some(2)); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("verification-mismatch")); assert!( run.stderr.contains("Receipt verification mismatch"), "expected the mismatch error, got stderr:\n{}", @@ -198,7 +228,7 @@ fn test_verify_receipt_reports_a_status_mismatch() { let run = replay(&path, &["--verify-receipt", "--json", TX]); let _ = std::fs::remove_file(&path); - assert!(!run.success, "a mismatch must exit non-zero"); + assert_eq!(run.code(), 2, "a mismatch exits 2.\nstderr: {}", run.stderr); assert_eq!( run.json()["verification"], serde_json::json!({ @@ -228,7 +258,7 @@ fn test_verify_receipt_reports_a_log_mismatch() { let run = replay(&path, &["--verify-receipt", "--json", TX]); let _ = std::fs::remove_file(&path); - assert!(!run.success, "a mismatch must exit non-zero"); + assert_eq!(run.code(), 2, "a mismatch exits 2.\nstderr: {}", run.stderr); assert_eq!( run.json()["verification"], serde_json::json!({ @@ -250,7 +280,10 @@ fn test_verify_receipt_reorg_is_an_infrastructure_error() { let run = replay(&path, &["--verify-receipt", "--json", TX]); let _ = std::fs::remove_file(&path); - assert!(!run.success, "a receipt from another block must fail the run"); + // The comparison never ran, so the run fails as an RPC-class failure, not + // as a mismatch. + assert_eq!(run.code(), 3, "an unverifiable target exits 3.\nstderr: {}", run.stderr); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("rpc-failure")); assert!( run.stderr.contains("different inclusion"), "expected the reorg/divergent-endpoint hint, got stderr:\n{}", @@ -273,7 +306,7 @@ fn test_verify_receipt_missing_receipt_is_an_infrastructure_error() { let run = replay(&path, &["--verify-receipt", "--json", TX]); let _ = std::fs::remove_file(&path); - assert!(!run.success, "an unavailable receipt must fail the run"); + assert_eq!(run.code(), 3, "an unavailable receipt exits 3.\nstderr: {}", run.stderr); assert!( run.stderr.contains("receipt"), "expected an error naming the receipt, got stderr:\n{}", @@ -314,7 +347,8 @@ fn test_batch_verify_receipt_reports_a_mismatch_and_exits_nonzero() { let _ = std::fs::remove_file(&path); let _ = std::fs::remove_file(&list); - assert!(!run.success, "a mismatch must exit non-zero"); + assert_eq!(run.code(), 2, "a mismatch exits 2.\nstderr: {}", run.stderr); + assert_eq!(run.error_object()["error"]["code"].as_u64(), Some(2)); let lines = run.ndjson(); assert_eq!(lines.len(), 1, "a mismatch is still a result line, not an error entry"); assert!(lines[0].get("error").is_none(), "a mismatch is not an infrastructure error"); @@ -343,11 +377,12 @@ fn test_batch_verify_receipt_missing_receipt_is_an_rpc_error_entry() { let _ = std::fs::remove_file(&path); let _ = std::fs::remove_file(&list); - assert!(!run.success, "an unverified target must exit non-zero"); + assert_eq!(run.code(), 3, "an unverified target exits 3.\nstderr: {}", run.stderr); let lines = run.ndjson(); assert_eq!(lines.len(), 1, "one line per requested transaction"); assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); assert!(lines[0].get("verification").is_none(), "an unverified target carries no verdict"); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("rpc-failure")); assert!( !run.stderr.contains("verification mismatch"), "an unverifiable target must not fail as a mismatch:\n{}", @@ -368,7 +403,7 @@ fn test_batch_verify_receipt_reorg_is_an_rpc_error_entry() { let _ = std::fs::remove_file(&path); let _ = std::fs::remove_file(&list); - assert!(!run.success, "an unverified target must exit non-zero"); + assert_eq!(run.code(), 3, "an unverified target exits 3.\nstderr: {}", run.stderr); let lines = run.ndjson(); assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); assert!( @@ -449,7 +484,8 @@ fn test_batch_dump_fixture_dir_refuses_overwrite_without_flag() { "--json", ], ); - assert!(!second.success, "overwrite without --overwrite must fail the run"); + // The dump failed for the target, which is an execution-class failure. + assert_eq!(second.code(), 1, "a failed dump exits 1.\nstderr: {}", second.stderr); let lines = second.ndjson(); assert_eq!(lines.len(), 1); assert_eq!(lines[0]["error"]["kind"].as_str(), Some("execution")); diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 2c6f5044..34bfb783 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -230,8 +230,10 @@ The mismatch line names every dimension that disagreed, comma-separated. ### Exit Status A run in which every target replayed and every verification matched exits `0`. -A verification mismatch exits non-zero through a dedicated error (`Receipt verification mismatch: N of M verified transaction(s) did not reproduce the on-chain receipt`), reported after every result line has been written. -Infrastructure failures keep their own non-zero exit and take precedence in a batch run: a target that never replayed was also never verified, so reporting it as a mismatch would overstate what the run found. +A verification mismatch exits `2` through a dedicated error (`Receipt verification mismatch: N of M verified transaction(s) did not reproduce the on-chain receipt`), reported after every result line has been written. +Infrastructure failures keep their own exit code and take precedence in a batch run: a target that never replayed was also never verified, so reporting it as a mismatch would overstate what the run found. +An execution or input failure exits `1`, an RPC failure (including a receipt the endpoint cannot serve) exits `3`. +See [Exit codes](../overview.md#exit-codes) for the full taxonomy and the batch precedence rule. ### Examples diff --git a/docs/mega-evme/overview.md b/docs/mega-evme/overview.md index 314f4e2e..2ffde16b 100644 --- a/docs/mega-evme/overview.md +++ b/docs/mega-evme/overview.md @@ -67,6 +67,32 @@ These flags apply to all commands. | `--log.file ` | stderr | `--log-file` | Write logs to a file instead of stderr | | `--log.no-color` | `false` | `--log-no-color` | Disable colored console output | +## Exit codes + +Every command reports its outcome through the same set of exit codes, so a pipeline can branch on the process status without parsing output. + +| Code | Class | Meaning | +| ---- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `0` | success | The command completed; with [`--verify-receipt`](commands/replay.md#receipt-verification), every verification matched. | +| `1` | `execution-error` | Execution or internal error: an EVM or setup failure, bad input (including a usage error), or a definitive negative answer such as an unknown transaction. | +| `2` | `verification-mismatch` | The run completed, but at least one replay did not reproduce its on-chain receipt. | +| `3` | `rpc-failure` | An RPC or transport call failed — endpoint unreachable, transport error, or an offline replay file that holds no response for a request the run had to make. | + +Codes `1` and `3` separate the two ways a question can go wrong: `1` means the tool answered, and the answer is negative; `3` means the question went unanswered, so retrying against a healthy endpoint may still produce a result. + +A batch run (`--tx-file` / `--block`) reports every target on its own line and then exits once for the run as a whole, ranking the failure classes it saw: any execution or internal failure exits `1`, otherwise any RPC failure exits `3`, otherwise any verification mismatch exits `2`. +A target that never replayed was also never verified, which is why an infrastructure failure outranks a mismatch. + +On failure the run also prints a report: one `error: ` line on stderr, plus — with `--json` — a structured object as the last line of stdout, so a machine-readable run never ends with empty output. + +```json +{ "error": { "code": 3, "kind": "rpc-failure", "message": "RPC error: …" } } +``` + +In batch mode that object follows the per-target lines, whose own `error.kind` (`not_found`, `pending`, `rpc`, `execution`) describes why one target failed and is independent of the run-level class above. + +New failure classes are added as new codes; the meaning of an existing code does not change. + ## Read more - **[Cookbook](cookbook.md)** — Real-world recipes and worked examples. From 038d546e5d698ade9fae5fe74458f3c59cf75ef1 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Mon, 3 Aug 2026 23:27:05 +0800 Subject: [PATCH 09/64] fix(mega-evme): cache persist external_env conflict and provider chain checks Reject conflicting non-null external_env on envelope persist (same rule as cache merge), validate rpc-cache-{id}.json chain identity on provider merge, classify malformed --rpc URLs as InvalidInput (exit 1), and type envelope re-read hard vs degradable without substring matching. --- bin/mega-evme/src/cache/merge.rs | 209 ++++++++++++++++-- bin/mega-evme/src/cache/mod.rs | 134 ++++++++++- .../src/common/provider/cache_store.rs | 110 +++++++-- bin/mega-evme/src/common/provider/mod.rs | 4 +- bin/mega-evme/tests/provider.rs | 16 +- docs/mega-evme/commands/cache.md | 3 + .../configuration/state-management.md | 10 +- 7 files changed, 434 insertions(+), 52 deletions(-) diff --git a/bin/mega-evme/src/cache/merge.rs b/bin/mega-evme/src/cache/merge.rs index 1d7137ad..0e7ce5f9 100644 --- a/bin/mega-evme/src/cache/merge.rs +++ b/bin/mega-evme/src/cache/merge.rs @@ -65,6 +65,24 @@ pub(crate) fn lock_sidecar_path(target: &Path) -> PathBuf { PathBuf::from(os) } +/// Parse `rpc-cache-{chain_id}.json` from a path's file name. +/// +/// Returns `None` when the name does not match the per-chain provider-cache +/// convention (so callers can warn that chain identity cannot be validated). +pub(crate) fn parse_rpc_cache_filename_chain_id(path: &Path) -> Option { + let name = path.file_name()?.to_str()?; + let rest = name.strip_prefix("rpc-cache-")?.strip_suffix(".json")?; + if rest.is_empty() || !rest.chars().all(|c| c.is_ascii_digit()) { + return None; + } + // Reject leading zeros (except the single digit `0`) so `rpc-cache-01.json` + // is not treated as chain 1 under a different spelling. + if rest.len() > 1 && rest.starts_with('0') { + return None; + } + rest.parse().ok() +} + /// Union `base` with `overlay` by key; overlay wins on collision. /// /// Output is sorted by key for deterministic files. @@ -135,33 +153,73 @@ pub(crate) fn read_provider_cache(path: &Path) -> Result> { } } -/// Read and parse a capture envelope. Missing file is an error for callers that -/// require a document; use [`try_read_envelope`] for best-effort re-read. -pub(crate) fn read_envelope(path: &Path) -> Result { - let content = fs::read_to_string(path).map_err(|e| { - EvmeError::FixtureError(format!("Failed to read envelope {}: {e}", path.display())) - })?; - let value: serde_json::Value = serde_json::from_str(&content).map_err(|e| { - EvmeError::FixtureError(format!("Failed to parse envelope {}: {e}", path.display())) - })?; - match detect_shape(&value, path).map_err(|e| EvmeError::FixtureError(e.to_string()))? { +/// Classification of an envelope re-read during concurrent persist merge. +/// +/// Typed so hard identity failures (version / `chain_id` / wrong shape) are not +/// confused with corrupt JSON merely because a path or message contains those +/// substrings. +#[derive(Debug)] +pub(crate) enum EnvelopeReread { + /// Successfully parsed and version-validated envelope. + Ok(EnvelopeDoc), + /// Corrupt, unreadable, or undecodable content — safe to warn and replace. + Degradable(String), + /// Schema / identity failure that must abort the capture persist. + Hard(EvmeError), +} + +/// Re-read an on-disk envelope for the lock-protected merge-on-persist path. +/// +/// Distinguishes hard identity failures from degradable corrupt content without +/// substring-searching formatted messages. +pub(crate) fn reread_envelope_for_merge(path: &Path) -> EnvelopeReread { + let content = match fs::read_to_string(path) { + Ok(c) => c, + Err(e) => { + return EnvelopeReread::Degradable(format!( + "Failed to read envelope {}: {e}", + path.display() + )); + } + }; + let value: serde_json::Value = match serde_json::from_str(&content) { + Ok(v) => v, + Err(e) => { + return EnvelopeReread::Degradable(format!( + "Failed to parse envelope {}: {e}", + path.display() + )); + } + }; + let shape = match detect_shape(&value, path) { + Ok(s) => s, + Err(e) => { + // Unrecognized shape is a hard identity/schema failure: the on-disk + // file is not a capture envelope this build can merge into. + return EnvelopeReread::Hard(EvmeError::FixtureError(e.to_string())); + } + }; + match shape { CacheShape::Envelope => { - let doc: EnvelopeDoc = serde_json::from_value(value).map_err(|e| { - EvmeError::FixtureError(format!( - "Failed to decode envelope {}: {e}", - path.display() - )) - })?; + let doc: EnvelopeDoc = match serde_json::from_value(value) { + Ok(d) => d, + Err(e) => { + return EnvelopeReread::Degradable(format!( + "Failed to decode envelope {}: {e}", + path.display() + )); + } + }; if doc.version != ENVELOPE_VERSION { - return Err(EvmeError::FixtureError(format!( + return EnvelopeReread::Hard(EvmeError::FixtureError(format!( "Unsupported cache file version {} in '{}'; expected {ENVELOPE_VERSION}", doc.version, path.display(), ))); } - Ok(doc) + EnvelopeReread::Ok(doc) } - CacheShape::Provider => Err(EvmeError::FixtureError(format!( + CacheShape::Provider => EnvelopeReread::Hard(EvmeError::FixtureError(format!( "Expected capture envelope in '{}', found provider-cache array", path.display() ))), @@ -173,10 +231,12 @@ pub(crate) fn merge_provider_lists(base: Vec, overlay: Vec) -> merge_kv_entries(base, overlay) } -/// Merge `ours` over `on_disk` for envelope persist (ours wins on key collision; -/// `external_env`: keep ours if set, else on-disk). +/// Merge `ours` over `on_disk` for envelope persist (ours wins on key collision). /// -/// Returns an error if `chain_id` or `version` disagree. +/// Returns an error if `chain_id` or `version` disagree, or if both sides carry +/// non-null `external_env` snapshots that are not identical (same rule as +/// [`merge_envelopes_cli`]). One-sided or identical snapshots are accepted: +/// ours is kept when set, otherwise the on-disk snapshot is propagated. pub(crate) fn merge_envelope_for_persist( on_disk: &EnvelopeDoc, ours: &EnvelopeDoc, @@ -198,11 +258,24 @@ pub(crate) fn merge_envelope_for_persist( ours.chain_id, ))); } + let external_env = match (&ours.external_env, &on_disk.external_env) { + (Some(a), Some(b)) if a != b => { + return Err(EvmeError::FixtureError(format!( + "Conflicting external_env snapshots when merging '{}': \ + on-disk {on_disk_env:?}, ours {ours_env:?}", + path.display(), + on_disk_env = on_disk.external_env, + ours_env = ours.external_env, + ))); + } + (Some(a), _) => Some(a.clone()), + (None, disk) => disk.clone(), + }; Ok(EnvelopeDoc { version: ours.version, chain_id: ours.chain_id, cache: merge_kv_entries(on_disk.cache.clone(), ours.cache.clone()), - external_env: ours.external_env.clone().or_else(|| on_disk.external_env.clone()), + external_env, }) } @@ -417,6 +490,77 @@ mod tests { assert!(err.to_string().contains("chain_id")); } + /// Conflicting non-null `external_env` snapshots hard-error and name both values. + #[test] + fn test_merge_envelope_for_persist_rejects_conflicting_external_env() { + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "disk")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10)] }), + }; + let ours = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(2, "ours")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 99)] }), + }; + let err = + merge_envelope_for_persist(&on_disk, &ours, Path::new("capture.json")).unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("external_env"), "msg={msg}"); + assert!(msg.contains("on-disk"), "msg={msg}"); + assert!(msg.contains("ours"), "msg={msg}"); + // Both snapshots named in the message (Debug form of bucket capacities). + assert!(msg.contains("10") && msg.contains("99"), "msg={msg}"); + } + + /// Identical non-null `external_env` snapshots merge successfully. + #[test] + fn test_merge_envelope_for_persist_identical_external_env() { + let ext = ExternalEnvDoc { bucket_capacities: vec![(1, 10), (2, 20)] }; + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "disk")], + external_env: Some(ext.clone()), + }; + let ours = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(2, "ours")], + external_env: Some(ext.clone()), + }; + let merged = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap(); + assert_eq!(merged.cache, vec![kv(1, "disk"), kv(2, "ours")]); + assert_eq!(merged.external_env, Some(ext)); + } + + /// One-sided `external_env` propagates the non-null snapshot (either side). + #[test] + fn test_merge_envelope_for_persist_one_sided_external_env() { + let ext = ExternalEnvDoc { bucket_capacities: vec![(3, 30)] }; + // Ours None, disk Some → disk propagates (covered by existing union test + // for the reverse orientation; re-assert disk-propagates here). + let on_disk = + EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: Some(ext.clone()) }; + let ours = + EnvelopeDoc { version: 1, chain_id: 1, cache: vec![kv(1, "a")], external_env: None }; + let merged = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap(); + assert_eq!(merged.external_env, Some(ext.clone())); + + // Ours Some, disk None → ours kept. + let on_disk = EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: None }; + let ours = EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(1, "a")], + external_env: Some(ext.clone()), + }; + let merged = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap(); + assert_eq!(merged.external_env, Some(ext)); + } + #[test] fn test_merge_envelopes_cli_conflict_external_env() { let a = EnvelopeDoc { @@ -441,4 +585,23 @@ mod tests { let p = Path::new("/tmp/rpc-cache-1.json"); assert_eq!(lock_sidecar_path(p), PathBuf::from("/tmp/rpc-cache-1.json.lock")); } + + /// Filename-derived chain id for the standard provider-cache naming scheme. + #[test] + fn test_parse_rpc_cache_filename_chain_id() { + assert_eq!(parse_rpc_cache_filename_chain_id(Path::new("rpc-cache-1.json")), Some(1)); + assert_eq!( + parse_rpc_cache_filename_chain_id(Path::new("/tmp/rpc-cache-4326.json")), + Some(4326) + ); + assert_eq!( + parse_rpc_cache_filename_chain_id(Path::new("worker/rpc-cache-11155420.json")), + Some(11_155_420) + ); + // Non-matching names cannot be validated from the filename alone. + assert_eq!(parse_rpc_cache_filename_chain_id(Path::new("out.json")), None); + assert_eq!(parse_rpc_cache_filename_chain_id(Path::new("rpc-cache.json")), None); + assert_eq!(parse_rpc_cache_filename_chain_id(Path::new("rpc-cache-abc.json")), None); + assert_eq!(parse_rpc_cache_filename_chain_id(Path::new("cache-4326.json")), None); + } } diff --git a/bin/mega-evme/src/cache/mod.rs b/bin/mega-evme/src/cache/mod.rs index a00ff861..f2853d0e 100644 --- a/bin/mega-evme/src/cache/mod.rs +++ b/bin/mega-evme/src/cache/mod.rs @@ -13,11 +13,13 @@ use crate::common::{EvmeError, Result}; pub(crate) use merge::{ lock_sidecar_path, merge_envelope_for_persist, merge_kv_entries, merge_provider_lists, - read_envelope, read_provider_cache, write_bytes_atomic, write_envelope_atomic, - write_provider_cache_atomic, CacheKv, EnvelopeDoc, ExternalEnvDoc, ENVELOPE_VERSION, + parse_rpc_cache_filename_chain_id, read_provider_cache, reread_envelope_for_merge, + write_bytes_atomic, write_envelope_atomic, write_provider_cache_atomic, CacheKv, EnvelopeDoc, + EnvelopeReread, ExternalEnvDoc, ENVELOPE_VERSION, }; use merge::{load_cache_file, merge_envelopes_cli, CacheShape, LoadedCache}; +use tracing::warn; /// `mega-evme cache` — offline cache-file utilities. #[derive(Parser, Debug)] @@ -55,6 +57,42 @@ impl Cmd { } } +/// Validate that provider-cache paths agreeing with `rpc-cache-{id}.json` all +/// name the same chain id. +/// +/// Paths that do not match the pattern emit a `warn!` (chain identity cannot be +/// validated for them) and are otherwise ignored. Two or more matching paths +/// with different ids are a hard error naming the conflicting files. +pub(crate) fn check_provider_cache_chain_identity<'a>( + paths: impl IntoIterator, +) -> Result<()> { + let mut seen: Option<(u64, PathBuf)> = None; + for path in paths { + match parse_rpc_cache_filename_chain_id(path) { + None => { + warn!( + path = %path.display(), + "Provider-cache path does not match rpc-cache-{{id}}.json; \ + chain identity cannot be validated for this file", + ); + } + Some(id) => match &seen { + None => seen = Some((id, path.to_path_buf())), + Some((prev_id, prev_path)) if *prev_id != id => { + return Err(EvmeError::InvalidInput(format!( + "Provider-cache chain identity mismatch: '{}' is chain {prev_id}, \ + but '{}' is chain {id}. Merge only caches from the same chain.", + prev_path.display(), + path.display(), + ))); + } + Some(_) => {} + }, + } + } + Ok(()) +} + impl MergeArgs { /// Merge inputs into `--output` and print a one-line summary. pub fn run(self) -> Result<()> { @@ -89,6 +127,16 @@ impl MergeArgs { let unique_out = match first_shape { CacheShape::Provider => { + // Provider-cache files carry chain identity only in the + // `rpc-cache-{id}.json` filename. Reject merges that would + // union different chains; warn when a path cannot be checked. + check_provider_cache_chain_identity( + loaded + .iter() + .map(|(p, _, _)| p.as_path()) + .chain(std::iter::once(self.output.as_path())), + )?; + let mut acc = Vec::new(); for (_, _, data) in loaded { let LoadedCache::Provider(entries) = data else { unreachable!() }; @@ -259,4 +307,86 @@ mod tests { let err = merge_envelopes_cli(&docs).unwrap_err(); assert!(err.to_string().contains("version")); } + + /// Mismatched `rpc-cache-{id}.json` filenames hard-error naming both files. + #[test] + fn test_cache_merge_rejects_provider_chain_id_filename_mismatch() { + let dir = tempdir().unwrap(); + let a = dir.path().join("rpc-cache-1.json"); + let b = dir.path().join("rpc-cache-4326.json"); + let out = dir.path().join("rpc-cache-4326-out.json"); + + write(&a, &serde_json::to_string(&vec![kv(1, "a")]).unwrap()); + write(&b, &serde_json::to_string(&vec![kv(2, "b")]).unwrap()); + + let err = MergeArgs { inputs: vec![a.clone(), b.clone()], output: out }.run().unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("chain identity") || msg.contains("chain"), "msg={msg}"); + assert!(msg.contains("chain 1") && msg.contains("chain 4326"), "msg={msg}"); + assert!( + msg.contains(a.file_name().unwrap().to_str().unwrap()) || + msg.contains("rpc-cache-1.json"), + "msg={msg}" + ); + assert!( + msg.contains(b.file_name().unwrap().to_str().unwrap()) || + msg.contains("rpc-cache-4326.json"), + "msg={msg}" + ); + } + + /// Output path is included in the chain-identity check. + #[test] + fn test_cache_merge_rejects_provider_output_chain_id_mismatch() { + let dir = tempdir().unwrap(); + let a = dir.path().join("rpc-cache-1.json"); + let out = dir.path().join("rpc-cache-4326.json"); + write(&a, &serde_json::to_string(&vec![kv(1, "a")]).unwrap()); + + let err = MergeArgs { inputs: vec![a], output: out }.run().unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("chain 1") && msg.contains("chain 4326"), "msg={msg}"); + } + + /// Unit-testable predicate: non-matching filename cannot supply chain identity. + #[test] + fn test_provider_cache_chain_identity_non_matching_filename_is_none() { + assert_eq!(parse_rpc_cache_filename_chain_id(std::path::Path::new("merged.json")), None); + assert_eq!( + parse_rpc_cache_filename_chain_id(std::path::Path::new("rpc-cache-1.json")), + Some(1) + ); + } + + /// Same chain id across matching names is accepted (including output). + #[test] + fn test_check_provider_cache_chain_identity_same_id_ok() { + let paths = [ + std::path::Path::new("worker0/rpc-cache-4326.json"), + std::path::Path::new("worker1/rpc-cache-4326.json"), + std::path::Path::new("rpc-cache-4326.json"), + ]; + check_provider_cache_chain_identity(paths).expect("same chain ok"); + } + + /// Different ids hard-error; non-matching names alone do not. + #[test] + fn test_check_provider_cache_chain_identity_mismatch_and_non_matching() { + let paths = [ + std::path::Path::new("rpc-cache-1.json"), + std::path::Path::new("out.json"), // non-matching → warn only + std::path::Path::new("rpc-cache-4326.json"), + ]; + let err = check_provider_cache_chain_identity(paths).unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("rpc-cache-1.json") && msg.contains("rpc-cache-4326.json"), "{msg}"); + + // Only non-matching names: no chain id to disagree on → ok (with warns). + let only_free = [ + std::path::Path::new("a.json"), + std::path::Path::new("b.json"), + std::path::Path::new("merged.json"), + ]; + check_provider_cache_chain_identity(only_free).expect("no ids to conflict"); + } } diff --git a/bin/mega-evme/src/common/provider/cache_store.rs b/bin/mega-evme/src/common/provider/cache_store.rs index 692a263c..6799bde3 100644 --- a/bin/mega-evme/src/common/provider/cache_store.rs +++ b/bin/mega-evme/src/common/provider/cache_store.rs @@ -32,9 +32,9 @@ use tracing::{info, warn}; use super::transport::TransportCache; use crate::{ cache::{ - lock_sidecar_path, merge_envelope_for_persist, merge_kv_entries, read_envelope, - read_provider_cache, write_bytes_atomic, write_envelope_atomic, CacheKv, EnvelopeDoc, - ExternalEnvDoc, ENVELOPE_VERSION, + lock_sidecar_path, merge_envelope_for_persist, merge_kv_entries, read_provider_cache, + reread_envelope_for_merge, write_bytes_atomic, write_envelope_atomic, CacheKv, EnvelopeDoc, + EnvelopeReread, ExternalEnvDoc, ENVELOPE_VERSION, }, common::{EvmeError, Result}, }; @@ -363,8 +363,12 @@ impl CacheFileEnvelope { } /// Atomically write this envelope to `path` under a lock, merging with any - /// on-disk envelope already present (ours win on cache key collision; - /// `external_env` keeps ours if set, else the on-disk one). + /// on-disk envelope already present (ours win on cache key collision). + /// + /// `external_env` keeps ours if set, else the on-disk one. Both sides non-null + /// and not identical is a hard error (same rule as offline `cache merge`). + /// On-disk re-read failures are typed: identity/schema mismatches hard-fail; + /// corrupt JSON degrades to ours-only with a warning. /// /// Lock contention blocks until the lock is free. Failure to create/acquire /// the lock degrades to an unlocked write with a `warn!`. Write failures @@ -384,23 +388,14 @@ impl CacheFileEnvelope { let ours = self.to_merge_doc()?; let to_write = if path.exists() { - match read_envelope(path) { - Ok(on_disk) => merge_envelope_for_persist(&on_disk, &ours, path)?, - // Version / chain_id / shape mismatches are hard errors (primary capture output). - // Corrupt or unreadable JSON degrades to ours-only with a warning. - Err(err) => { - let msg = err.to_string(); - let hard = msg.contains("chain_id") || - msg.contains("version") || - msg.contains("Unsupported") || - msg.contains("Expected capture envelope") || - msg.contains("provider-cache"); - if hard { - return Err(err); - } + // Typed hard vs degradable: no substring matching on formatted messages. + match reread_envelope_for_merge(path) { + EnvelopeReread::Ok(on_disk) => merge_envelope_for_persist(&on_disk, &ours, path)?, + EnvelopeReread::Hard(err) => return Err(err), + EnvelopeReread::Degradable(msg) => { warn!( path = %path.display(), - error = %err, + error = %msg, "Failed to re-read on-disk envelope during merge; persisting our entries only", ); ours @@ -630,6 +625,81 @@ mod tests { assert!(err.to_string().contains("chain_id")); } + /// Corrupt on-disk envelope under a path whose name contains `chain_id` is + /// degradable (warn + replace), not a hard error — classification is typed, + /// not substring-based on the formatted message / path. + #[test] + fn test_envelope_persist_degrades_on_corrupt_disk_path_containing_chain_id() { + let dir = tempfile::tempdir().expect("tempdir"); + // Path deliberately contains the substrings the old classifier matched. + let path = dir.path().join("chain_id_version_capture.json"); + fs::write(&path, "not-json{{{").expect("corrupt"); + + let cache = TransportCache::new(); + cache + .merge(&serde_json::json!([{ + "key": keccak256("eth_blockNumber"), + "value": r#"{"id":0,"jsonrpc":"2.0","result":"0x1"}"#, + }])) + .expect("seed"); + CacheFileEnvelope::new(&cache, 7, None) + .save(&path) + .expect("corrupt disk with chain_id in path must degrade, not hard-fail"); + + let env = CacheFileEnvelope::load(&path).expect("ours written"); + assert_eq!(env.chain_id, 7); + assert_eq!(TransportCache::from_value(&env.cache).expect("from_value").len(), 1); + } + + /// Genuine `chain_id` mismatch remains a hard error (typed path via merge). + #[test] + fn test_envelope_persist_hard_errors_on_genuine_chain_id_mismatch() { + let dir = tempfile::tempdir().expect("tempdir"); + // Same path naming trap as the corrupt-file test: must not flip classification. + let path = dir.path().join("chain_id_version_capture.json"); + + let cache_b = TransportCache::new(); + CacheFileEnvelope::new(&cache_b, 1, None).save(&path).expect("save b"); + + let cache_a = TransportCache::new(); + let err = CacheFileEnvelope::new(&cache_a, 2, None) + .save(&path) + .expect_err("chain_id mismatch must hard-fail"); + let msg = err.to_string(); + assert!(msg.contains("chain_id"), "msg={msg}"); + } + + /// Typed re-read: corrupt content is Degradable even when path mentions `chain_id`. + #[test] + fn test_reread_envelope_classifies_corrupt_vs_identity() { + let dir = tempfile::tempdir().expect("tempdir"); + + let corrupt = dir.path().join("chain_id_and_version.json"); + fs::write(&corrupt, "{not valid").expect("write"); + match reread_envelope_for_merge(&corrupt) { + EnvelopeReread::Degradable(msg) => { + assert!(msg.contains("parse") || msg.contains("Failed"), "{msg}"); + } + other => panic!("corrupt must be Degradable, got {other:?}"), + } + + let bad_version = dir.path().join("env.json"); + fs::write(&bad_version, r#"{"version":99,"chain_id":1,"cache":[]}"#).unwrap(); + match reread_envelope_for_merge(&bad_version) { + EnvelopeReread::Hard(err) => { + assert!(err.to_string().contains("Unsupported") || err.to_string().contains("99")); + } + other => panic!("unsupported version must be Hard, got {other:?}"), + } + + let ok_path = dir.path().join("ok.json"); + fs::write(&ok_path, r#"{"version":1,"chain_id":5,"cache":[]}"#).unwrap(); + match reread_envelope_for_merge(&ok_path) { + EnvelopeReread::Ok(doc) => assert_eq!(doc.chain_id, 5), + other => panic!("valid envelope must be Ok, got {other:?}"), + } + } + /// Corrupt on-disk provider cache during re-read does not abort; ours are written. #[test] fn test_provider_cache_persist_degrades_on_corrupt_disk() { diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 0ebac3ff..34260e1a 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -156,7 +156,7 @@ impl RpcArgs { })?; let url: reqwest::Url = rpc_url_str.parse().map_err(|e| { - EvmeError::RpcError(format!("Invalid RPC URL '{}': {}", rpc_url_str, e)) + EvmeError::InvalidInput(format!("Invalid RPC URL '{}': {}", rpc_url_str, e)) })?; // Once per provider build (not per client: resolve_chain_id also builds a client). @@ -299,7 +299,7 @@ impl RpcArgs { let rpc_url_str = self.rpc_url.as_ref().expect("capture mode requires --rpc"); let url: reqwest::Url = rpc_url_str.parse().map_err(|e| { - EvmeError::RpcError(format!("Invalid RPC URL '{}': {}", rpc_url_str, e)) + EvmeError::InvalidInput(format!("Invalid RPC URL '{}': {}", rpc_url_str, e)) })?; // Once per provider build (capture builds a single client; keep the same entry point). diff --git a/bin/mega-evme/tests/provider.rs b/bin/mega-evme/tests/provider.rs index 2a475c91..836e6346 100644 --- a/bin/mega-evme/tests/provider.rs +++ b/bin/mega-evme/tests/provider.rs @@ -14,7 +14,7 @@ use std::path::PathBuf; use alloy_primitives::B256; use alloy_provider::Provider; use clap::Parser; -use mega_evme::common::{BuildProviderOutput, EvmeError, RpcArgs}; +use mega_evme::common::{BuildProviderOutput, EvmeError, ExitCode, RpcArgs}; use tempfile::tempdir; mod common; @@ -179,16 +179,24 @@ async fn test_build_provider_with_cache_names_file_from_fetched_chain_id() { assert_eq!(cache_store.cache_path(), Some(dir.path().join("rpc-cache-4326.json").as_path())); } +/// Malformed `--rpc` is bad input (exit 1), not an RPC transport failure (exit 3). #[tokio::test(flavor = "multi_thread")] async fn test_build_provider_invalid_url() { let args = RpcArgs::parse_from(["mega-evme", "--rpc", "not a url", "--rpc.no-cache-file"]); let err = args.build_provider().await.expect_err("build_provider should fail"); - match err { - EvmeError::RpcError(msg) => { + match &err { + EvmeError::InvalidInput(msg) => { assert!(msg.contains("not a url"), "error must echo the original input, got: {msg}"); + assert!(msg.contains("Invalid RPC URL"), "msg={msg}"); } - other => panic!("expected EvmeError::RpcError, got {other:?}"), + other => panic!("expected EvmeError::InvalidInput, got {other:?}"), } + // Exit-code taxonomy: InvalidInput → execution-error (code 1), never rpc-failure (3). + assert_eq!( + ExitCode::from_evme_error(&err).code(), + 1, + "malformed --rpc must exit 1 (bad input), not 3 (rpc-failure)", + ); } // ─── Chain-id resolution ───────────────────────────────────────────────────── diff --git a/docs/mega-evme/commands/cache.md b/docs/mega-evme/commands/cache.md index 7b09cce3..2328171a 100644 --- a/docs/mega-evme/commands/cache.md +++ b/docs/mega-evme/commands/cache.md @@ -33,6 +33,9 @@ Mixing a provider-cache file with a capture envelope is a hard error that names - Union entries by `key`. - Later inputs win on collision. - Output is a provider-cache-shaped JSON array, written atomically (temp file + rename). +- Chain identity is taken only from the standard filename `rpc-cache-{chain_id}.json` (provider-cache bodies have no chain field). + Every input path and `--output` that matches that pattern must name the same chain id; a mismatch is a hard error that names the conflicting files. + Paths that do not match the pattern emit a warning that chain identity cannot be validated for them, and the merge proceeds for those paths without a filename-based check. ### Envelope merge diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index ca735b7a..1a04fe2b 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -232,9 +232,17 @@ Lock contention blocks for a short critical section rather than failing the fini If the lock cannot be acquired at all (for example the directory is not writable), persist logs a warning and falls back to an unlocked write. A missing or corrupt on-disk file during the re-read degrades to writing this process's entries only (also warned). -Capture envelopes (`--rpc.capture-file`) use the same lock + re-read-merge path, with an additional check that the on-disk `chain_id` matches before merging. +Capture envelopes (`--rpc.capture-file`) use the same lock + re-read-merge path, with additional hard-error checks before writing: + +- The on-disk envelope `version` and `chain_id` must match this process's capture. +- If both the on-disk envelope and this process carry a non-null `external_env` snapshot and those snapshots are not identical, persist hard-errors and names both values. + One-sided or identical snapshots still merge: this process's snapshot is kept when set, otherwise the on-disk snapshot is propagated. + This matches the offline [`cache merge`](../commands/cache.md) rule so concurrent captures cannot silently union RPC entries under a single wrong environment. + +A corrupt or unreadable on-disk envelope during re-read degrades to writing this process's entries only (warned), while identity/schema failures remain hard errors. To consolidate historical per-worker cache directories offline, use [`cache merge`](../commands/cache.md). +Provider-cache merge also rejects inputs (and `--output`) whose `rpc-cache-{chain_id}.json` filenames disagree on chain id. ### Cache Flags From a964c1d11eff1ba0bb7ce19a277b2c9b55fc4aa0 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 00:08:46 +0800 Subject: [PATCH 10/64] fix(mega-evme): batch reporting and exit-path defects Report a failed fixture dump on the target's own result line instead of replacing it, so the receipt verification still runs and its mismatch is counted; the failed dump remains an execution-class failure. Classify targets swept up by a mid-block abort as unanswered (`rpc`) with a message naming the aborting cause, reserving `not_found` for the target the endpoint actually denied, and emit them in block transaction-index order so the stream stays ascending by (block, index). Route argument-parsing failures through the structured output, report a capture-persist failure on stderr next to the run error that owns the exit code, and stop mapping a `BatchFailed` with no counts to success. Classify a block execution error by the database failure behind it, so a state read that fails mid-execution exits 3 instead of 1; the pre-block system calls and the keyless-deploy sandbox render their cause into a message before it reaches the mapping and stay execution-class. --- bin/mega-evme/src/common/error.rs | 5 +- bin/mega-evme/src/common/exit.rs | 125 ++++++++- bin/mega-evme/src/main.rs | 44 +++- bin/mega-evme/src/replay/batch.rs | 368 +++++++++++++++++---------- bin/mega-evme/src/replay/cmd.rs | 32 +-- bin/mega-evme/tests/exit_codes.rs | 155 +++++++++++ bin/mega-evme/tests/replay_batch.rs | 130 ++++++++++ bin/mega-evme/tests/replay_verify.rs | 55 +++- docs/mega-evme/commands/replay.md | 24 +- docs/mega-evme/overview.md | 7 +- 10 files changed, 768 insertions(+), 177 deletions(-) diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index 5c33657d..d3ac1e6a 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -103,7 +103,8 @@ pub enum EvmeError { #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct BatchFailureCounts { /// Targets that failed for an execution, setup, or definitive-answer reason - /// (unknown or pending transaction, block executor rejection). + /// (unknown or pending transaction, block executor rejection, a fixture the + /// run was asked to write and could not). pub execution: usize, /// Targets whose question went unanswered because an RPC call failed. pub rpc: usize, @@ -117,7 +118,7 @@ impl core::fmt::Display for BatchFailureCounts { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!( f, - "{} of {} target transaction(s) failed to replay ({} execution, {} rpc)", + "{} of {} target transaction(s) failed ({} execution, {} rpc)", self.execution + self.rpc, self.total, self.execution, diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs index 56fa3478..6cd46326 100644 --- a/bin/mega-evme/src/common/exit.rs +++ b/bin/mega-evme/src/common/exit.rs @@ -25,6 +25,11 @@ //! exhaustively, so a new error variant does not compile until it has been //! assigned a class. +use mega_evm::{ + alloy_evm::block::BlockExecutionError, + alloy_op_evm::OpTxError, + revm::{context::result::EVMError, database_interface::bal::EvmDatabaseError}, +}; use serde::Serialize; use tracing::error; @@ -33,6 +38,55 @@ use crate::{ common::{BatchFailureCounts, EvmeError}, }; +/// The concrete EVM error a `mega-evme` block executor produces. +/// +/// The executor runs the `MegaEvmFactory` EVM (transaction error [`OpTxError`]) +/// over revm's `State<_>` wrapper around [`crate::EvmeState`], whose database +/// error is `EvmDatabaseError`. A fatal EVM error is boxed as +/// `dyn Error` inside the block error, so recovering the cause needs this exact +/// type. +type BlockEvmError = EVMError, OpTxError>; + +/// The database failure behind a block execution error, if it has one. +/// +/// A state read that fails mid-execution (offline cache miss, transport error) +/// is fatal, so the block executor keeps it in one of its internal branches as +/// a boxed `dyn Error`. Which wrapper it arrives in depends on where the read +/// happened — inside the EVM, or in the executor around it — so the cause is +/// recovered by downcasting to each concrete type it can be boxed as. Every +/// step is typed: the rendered message is never inspected. +/// +/// Not every block error carries its cause this way. A read that fails inside +/// the pre-block system calls (EIP-4788 beacon root, EIP-2935 block hashes) or +/// inside the sandboxed execution used by the keyless-deploy system contract +/// has its cause rendered into a message string by the layer that raised it, so +/// the type is gone before the error arrives here and the failure stays +/// execution-class. +fn database_cause(err: &BlockExecutionError) -> Option<&EvmeError> { + let internal = err.as_internal()?; + let boxed = internal.as_evm().map(|(_, error)| error).or_else(|| internal.as_other())?; + + if let Some(evm_error) = boxed.downcast_ref::() { + return match evm_error { + EVMError::Database(database_error) => external_cause(database_error), + _ => None, + }; + } + if let Some(database_error) = boxed.downcast_ref::>() { + return external_cause(database_error); + } + boxed.downcast_ref::() +} + +/// The external database error behind a revm database error, if it is one. +const fn external_cause(err: &EvmDatabaseError) -> Option<&EvmeError> { + match err { + EvmDatabaseError::Database(cause) => Some(cause), + // A block access list error is the executor's own bookkeeping. + EvmDatabaseError::Bal(_) => None, + } +} + /// Process exit status of a `mega-evme` run. /// /// The discriminants are the wire contract with calling scripts; see the module @@ -88,11 +142,16 @@ impl ExitCode { // The endpoint never answered: unreachable, transport-level // failure, or an offline replay file without the response. EvmeError::RpcTransportError(_) | EvmeError::RpcError(_) => Self::RpcFailure, + // A block error the EVM raised because a state read failed is that + // read's failure, not an execution result: classify it by its + // cause, so an endpoint that died mid-execution still reports the + // question as unanswered. + EvmeError::BlockExecutionError(err) => database_cause(err) + .map_or(Self::ExecutionError, Self::from_evme_error), // Answered, definitively negative. EvmeError::TransactionNotFound(_) | EvmeError::BlockNotFound(_) | // Execution, setup, input, and internal failures. - EvmeError::BlockExecutionError(_) | EvmeError::InvalidBytecode(_) | EvmeError::FileRead(_) | EvmeError::InvalidHex(_) | @@ -114,6 +173,11 @@ impl ExitCode { /// outranks a mismatch. A target that never replayed was also never /// verified, so reporting such a run as a mismatch would overstate what it /// found. + /// + /// Counts that record no failure at all reach this mapping only through a + /// batch aggregation bug, since the run reports a failure precisely when it + /// counted one. That is an internal error, not a success: a failure can + /// never produce exit `0`. pub const fn from_batch_failures(counts: &BatchFailureCounts) -> Self { if counts.execution > 0 { Self::ExecutionError @@ -122,7 +186,7 @@ impl ExitCode { } else if counts.mismatched > 0 { Self::VerificationMismatch } else { - Self::Success + Self::ExecutionError } } } @@ -170,15 +234,23 @@ pub fn report_command_result(result: Result<(), Error>, json: bool) -> ExitCode let message = err.to_string(); eprintln!("error: {message}"); if json { - let envelope = ErrorEnvelope { - error: ErrorBody { code: code.code(), kind: code.kind(), message: &message }, - }; - println!("{}", serde_json::to_string(&envelope).expect("failed to serialize the error")); + print_json_error(code, &message); } code } +/// Print the structured failure object of a `--json` run on stdout. +/// +/// Also used by failures that never become a command result — an argument +/// parsing error is reported by `clap` itself, but a machine-readable run must +/// still end its stdout with the object the taxonomy promises. +pub fn print_json_error(code: ExitCode, message: &str) { + let envelope = + ErrorEnvelope { error: ErrorBody { code: code.code(), kind: code.kind(), message } }; + println!("{}", serde_json::to_string(&envelope).expect("failed to serialize the error")); +} + #[cfg(test)] mod tests { use super::*; @@ -250,6 +322,34 @@ mod tests { } } + /// A state read that failed mid-execution is an unanswered question, even + /// though it surfaced as a block execution error. + #[test] + fn test_block_execution_error_caused_by_a_database_failure_maps_to_three() { + let evm_err: BlockEvmError = EVMError::Database(EvmDatabaseError::Database( + EvmeError::RpcError("cache miss in offline replay file".to_string()), + )); + let err = EvmeError::BlockExecutionError(BlockExecutionError::evm(evm_err, B256::ZERO)); + + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::RpcFailure, + "unexpected class: {err}" + ); + } + + /// A block error with no database cause stays an execution failure. + #[test] + fn test_block_execution_error_without_a_database_cause_maps_to_one() { + let err = EvmeError::BlockExecutionError(BlockExecutionError::msg("gas limit reached")); + + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::ExecutionError, + "unexpected class: {err}" + ); + } + /// A completed run whose replay diverged from the chain exits 2. #[test] fn test_verification_mismatch_maps_to_two() { @@ -281,9 +381,18 @@ mod tests { let mismatch_only = BatchFailureCounts { execution: 0, rpc: 0, mismatched: 3, total: 6 }; assert_eq!(ExitCode::from_batch_failures(&mismatch_only), ExitCode::VerificationMismatch); + } - let clean = BatchFailureCounts::default(); - assert_eq!(ExitCode::from_batch_failures(&clean), ExitCode::Success); + /// A batch failure that counted nothing is an internal error, never a + /// success: an error variant must not be able to produce exit 0. + #[test] + fn test_batch_failure_without_counts_is_not_a_success() { + let counts = BatchFailureCounts::default(); + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::ExecutionError); + assert_eq!( + ExitCode::from_evme_error(&EvmeError::BatchFailed(counts)), + ExitCode::ExecutionError, + ); } /// The aggregate error carries its counts through the top-level mapping. diff --git a/bin/mega-evme/src/main.rs b/bin/mega-evme/src/main.rs index d357a6af..b4ebb4c6 100644 --- a/bin/mega-evme/src/main.rs +++ b/bin/mega-evme/src/main.rs @@ -10,7 +10,7 @@ use std::process::ExitCode; use clap::Parser; -use mega_evme::{cmd::MainCmd, report_command_result, set_thread_panic_hook}; +use mega_evme::{cmd::MainCmd, print_json_error, report_command_result, set_thread_panic_hook}; #[tokio::main] async fn main() -> ExitCode { @@ -23,11 +23,17 @@ async fn main() -> ExitCode { // stdout and the run exits 0. A usage error is bad input, which the // taxonomy classifies with the other input errors. let _ = err.print(); - return if err.use_stderr() { - ExitCode::from(mega_evme::ExitCode::ExecutionError) - } else { - ExitCode::SUCCESS - }; + if !err.use_stderr() { + return ExitCode::SUCCESS; + } + let code = mega_evme::ExitCode::ExecutionError; + // The parsed command does not exist yet, so the output mode is read + // off the raw arguments: a `--json` run must end its stdout with the + // structured error object even when it never got as far as running. + if wants_json_output() { + print_json_error(code, &parse_error_summary(&err)); + } + return ExitCode::from(code); } }; @@ -36,3 +42,29 @@ async fn main() -> ExitCode { let result = cmd.run().await; ExitCode::from(report_command_result(result, json)) } + +/// Whether the raw arguments ask for machine-readable output. +fn wants_json_output() -> bool { + std::env::args_os().any(|arg| arg == "--json") +} + +/// One-line summary of an argument parsing failure. +/// +/// `clap` renders a multi-line report — the message, the usage block, and the +/// help hint — of which only the leading message paragraph describes what went +/// wrong; it is joined into the single line the structured object carries. +fn parse_error_summary(err: &clap::Error) -> String { + let rendered = err.to_string(); + let summary = rendered + .lines() + .map(str::trim) + .take_while(|line| !line.is_empty()) + .collect::>() + .join(" "); + let summary = summary.strip_prefix("error: ").unwrap_or(&summary).trim(); + if summary.is_empty() { + "invalid command-line arguments".to_string() + } else { + summary.to_string() + } +} diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 6ab55dec..6c8c5f77 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -26,7 +26,10 @@ use alloy_primitives::{Address, B256}; use alloy_provider::Provider; use alloy_rpc_types_eth::Block; use mega_evm::{ - alloy_evm::{block::BlockExecutor, Evm, EvmEnv}, + alloy_evm::{ + block::{BlockExecutionError, BlockExecutor, BlockValidationError}, + Evm, EvmEnv, + }, alloy_op_evm::block::OpAlloyReceiptBuilder, revm::{ context::{result::ExecutionResult, ContextTr}, @@ -44,7 +47,7 @@ use tracing::{debug, info, warn}; use crate::{ common::{ op_receipt_to_tx_receipt, print_execution_summary, print_receipt, BatchFailureCounts, - EvmeExternalEnvs, ExecutionSummary, OpTxReceipt, + EvmeExternalEnvs, ExecutionSummary, ExitCode, OpTxReceipt, }, replay::get_hardfork_config, ChainArgs, EvmeState, @@ -72,6 +75,12 @@ pub(super) struct ReportArgs { } /// Per-target fixture dump outcome reported on the NDJSON / human result line. +/// +/// Exactly one field is set: the fixture was written, expectedly skipped +/// (fidelity gate, BLOCKHASH, unsupported shape), or could not be written. A +/// write failure is reported here rather than replacing the target's result, +/// so a target that did replay keeps its result — including its receipt +/// verification verdict — and still fails the run. #[derive(Debug, Clone, Serialize)] struct FixtureReport { /// Absolute or as-written path of a successfully written fixture. @@ -80,15 +89,27 @@ struct FixtureReport { /// Why the fixture was not written for this target. #[serde(skip_serializing_if = "Option::is_none")] skipped: Option, + /// Why writing the fixture failed. Counts as an execution-class failure. + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, } impl FixtureReport { fn written(path: &Path) -> Self { - Self { path: Some(path.display().to_string()), skipped: None } + Self { path: Some(path.display().to_string()), skipped: None, error: None } } fn skipped(reason: impl Into) -> Self { - Self { path: None, skipped: Some(reason.into()) } + Self { path: None, skipped: Some(reason.into()), error: None } + } + + fn error(message: impl Into) -> Self { + Self { path: None, skipped: None, error: Some(message.into()) } + } + + /// Whether the fixture the run was asked to write could not be written. + const fn is_error(&self) -> bool { + self.error.is_some() } /// One-line human summary printed under the transaction header. @@ -97,20 +118,14 @@ impl FixtureReport { format!("fixture: written to {path}") } else if let Some(reason) = &self.skipped { format!("fixture: skipped ({reason})") + } else if let Some(message) = &self.error { + format!("fixture: FAILED ({message})") } else { "fixture: (no report)".to_string() } } } -/// Result of attempting to dump a fixture for one target during the block loop. -enum FixtureAttempt { - /// Wrote a fixture or recorded an expected skip (fidelity / BLOCKHASH / unsupported). - Report(FixtureReport), - /// Finalize/write failed; the target becomes an infrastructure error entry. - WriteError(String), -} - /// What a batch run was asked to replay. #[derive(Debug)] pub(super) enum BatchMode { @@ -195,6 +210,8 @@ struct FailedTx { /// lines. #[derive(Debug, Default)] struct BatchTally { + /// Targets the run reported on, one per emitted entry. + reported: usize, /// Targets that produced an execution result. replayed: usize, /// Targets compared against an on-chain receipt. @@ -208,27 +225,49 @@ impl BatchTally { fn record(&mut self, entry: &BatchEntry) { match entry { BatchEntry::Executed(tx) => { - self.replayed += 1; - if let Some(verification) = &tx.verification { - self.verified += 1; - if !verification.matched { - self.counts.mismatched += 1; - } - } + self.record_executed(tx.verification.as_ref(), tx.fixture.as_ref()); } // A transaction the endpoint does not know, or that is not mined // yet, is a definitive answer about the target rather than an // unanswered question, so it counts as an execution failure. - BatchEntry::Failed(tx) => match tx.kind { - BatchErrorKind::Rpc => self.counts.rpc += 1, - BatchErrorKind::NotFound | BatchErrorKind::Pending | BatchErrorKind::Execution => { - self.counts.execution += 1 + BatchEntry::Failed(tx) => { + self.reported += 1; + match tx.kind { + BatchErrorKind::Rpc => self.counts.rpc += 1, + BatchErrorKind::NotFound | + BatchErrorKind::Pending | + BatchErrorKind::Execution => self.counts.execution += 1, } - }, + } + } + } + + /// Count the findings of one target that produced an execution result. + /// + /// A verdict and a fixture failure are independent findings about the same + /// target: a replay that diverged from its receipt is counted as a mismatch + /// whether or not its fixture could be written. + fn record_executed( + &mut self, + verification: Option<&VerificationOutcome>, + fixture: Option<&FixtureReport>, + ) { + self.reported += 1; + self.replayed += 1; + if let Some(verification) = verification { + self.verified += 1; + if !verification.matched { + self.counts.mismatched += 1; + } + } + // A fixture the run was asked to write and could not is a failure of + // that target, even though its replay produced a result. + if fixture.is_some_and(FixtureReport::is_error) { + self.counts.execution += 1; } } - /// Targets that produced no execution result. + /// Targets that failed, by any class other than a receipt mismatch. const fn failed(&self) -> usize { self.counts.execution + self.counts.rpc } @@ -238,11 +277,12 @@ impl BatchTally { /// Infrastructure failures are reported with their counts by class so the /// exit-code mapping resolves the precedence between them and a mismatch; a /// run whose only finding is divergence fails as the mismatch it is. - /// Fixture skips never count as failures. + /// Fixture skips never count as failures; a fixture that could not be + /// written does, as an execution-class failure of its target. fn into_error(self) -> Option { if self.failed() > 0 { return Some(ReplayError::BatchFailed(BatchFailureCounts { - total: self.replayed + self.failed(), + total: self.reported, ..self.counts })); } @@ -280,10 +320,7 @@ struct PendingTarget { to: Option

, effective_gas_price: u128, /// Fixture dump outcome, present iff `--dump-fixture-dir` was given. - /// - /// `Ok` is a written or skipped report; `Err` is a finalize/write failure - /// that turns this target into an infrastructure error entry. - fixture: Option>, + fixture: Option, } /// NDJSON line for a target that produced an execution result. @@ -373,17 +410,14 @@ where }; for job in jobs { - let block_result = - replay_block(provider, chain_id, job, external_envs.clone(), &report).await; - fixtures_failed += block_result.fixture_write_failures; - for entry in block_result.entries { + let entries = replay_block(provider, chain_id, job, external_envs.clone(), &report).await; + for entry in entries { if let BatchEntry::Executed(tx) = &entry { - if let Some(fixture) = &tx.fixture { - if fixture.path.is_some() { - fixtures_written += 1; - } else { - fixtures_skipped += 1; - } + match &tx.fixture { + Some(fixture) if fixture.path.is_some() => fixtures_written += 1, + Some(fixture) if fixture.is_error() => fixtures_failed += 1, + Some(_) => fixtures_skipped += 1, + None => {} } } tally.record(&entry); @@ -457,24 +491,6 @@ where (jobs, failures) } -/// Entries produced by replaying one block, plus how many fixture write failures -/// it contributed (for the end-of-run fixture summary). -struct BlockReplayResult { - entries: Vec, - /// Finalize/write errors that became `execution` infrastructure entries. - fixture_write_failures: usize, -} - -impl BlockReplayResult { - fn from_entries(entries: Vec) -> Self { - Self { entries, fixture_write_failures: 0 } - } - - fn fail_all(targets: &[B256], kind: BatchErrorKind, message: &str) -> Self { - Self::from_entries(fail_all(targets, kind, message)) - } -} - /// Replay one block, reporting an entry for every target it was asked about. /// /// The block is executed exactly once: every transaction runs in order, and each @@ -491,7 +507,7 @@ async fn replay_block

( job: BlockJob, external_envs: EvmeExternalEnvs, report: &ReportArgs, -) -> BlockReplayResult +) -> Vec where P: Provider + Clone + std::fmt::Debug, { @@ -501,11 +517,7 @@ where let overwrite = report.overwrite; if number == 0 { - return BlockReplayResult::fail_all( - &targets, - BatchErrorKind::Rpc, - "Block 0 has no parent block to fork from", - ); + return fail_all(&targets, BatchErrorKind::Rpc, "Block 0 has no parent block to fork from"); } let block = match block { @@ -513,14 +525,14 @@ where None => match fetch_block(provider, number).await { Ok(block) => block, Err(e) => { - return BlockReplayResult::fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); } }, }; let parent_block = match fetch_block(provider, number - 1).await { Ok(block) => block, Err(e) => { - return BlockReplayResult::fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); } }; @@ -555,13 +567,13 @@ where let cfg_env = match chain_args.create_cfg_env() { Ok(cfg) => cfg, Err(e) => { - return BlockReplayResult::fail_all(&targets, BatchErrorKind::Execution, &e.to_string()); + return fail_all(&targets, BatchErrorKind::Execution, &e.to_string()); } }; let block_env = match retrieve_block_env(&block) { Ok(env) => env, Err(e) => { - return BlockReplayResult::fail_all(&targets, BatchErrorKind::Execution, &e.to_string()); + return fail_all(&targets, BatchErrorKind::Execution, &e.to_string()); } }; let executed_spec = cfg_env.spec; @@ -569,7 +581,7 @@ where let Some(hardfork) = hardforks.hardfork(timestamp) else { let message = format!("No `MegaHardfork` active at block timestamp: {timestamp}"); - return BlockReplayResult::fail_all(&targets, BatchErrorKind::Execution, &message); + return fail_all(&targets, BatchErrorKind::Execution, &message); }; let block_limits = BlockLimits::from_hardfork_and_block_gas_limit(hardfork, block.header.gas_limit()); @@ -591,7 +603,7 @@ where { Ok(database) => database, Err(e) => { - return BlockReplayResult::fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); } }; @@ -602,8 +614,8 @@ where let mut block_executor = block_executor_factory.create_executor(&mut state, block_ctx, evm_env); if let Err(e) = block_executor.apply_pre_execution_changes() { - let message = format!("Block execution error: {e}"); - return BlockReplayResult::fail_all(&targets, BatchErrorKind::Execution, &message); + let error = ReplayError::BlockExecutionError(e); + return fail_all(&targets, classify(&error), &error.to_string()); } let target_set: HashSet = targets.iter().copied().collect(); @@ -642,7 +654,7 @@ where let outcome = block_executor .run_transaction(tx.as_recovered()) - .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + .map_err(ReplayError::BlockExecutionError)?; // Fixture draft must be built before commit: the pre-state closure // is the database after preceding txs, with the target's result @@ -678,16 +690,11 @@ where let exec_result = is_target.then(|| outcome.inner.result.clone()); let gas_used = block_executor .commit_transaction_outcome(outcome) - .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + .map_err(ReplayError::BlockExecutionError)?; let commit_index = committed; committed += 1; if let Some(exec_result) = exec_result { - let fixture = match fixture { - Some(FixtureAttempt::Report(report)) => Some(Ok(report)), - Some(FixtureAttempt::WriteError(message)) => Some(Err(message)), - None => None, - }; pending.push(PendingTarget { tx_hash: *tx_hash, tx_index: tx_index as u64, @@ -710,7 +717,6 @@ where // Finish the block even when it aborted midway: targets that already ran // still have a receipt worth reporting. let mut entries = Vec::with_capacity(targets.len()); - let mut fixture_write_failures = 0usize; match block_executor.finish() { Ok((evm, block_result)) => { let (db, _) = evm.finish(); @@ -721,16 +727,12 @@ where // later) cannot shift the mapping. let offset = receipts.len().saturating_sub(committed); let block_hash = block.hash(); + // A fixture that could not be written stays on the target's own + // result line below: the target did replay, so its receipt and its + // verification verdict are still what the run was asked for. The + // failed dump fails the run through the tally, not by replacing the + // result with an error entry. for target in pending { - // Fixture finalize/write failure: infrastructure error entry. - // The target did replay, but the dump request failed. - if let Some(Err(message)) = target.fixture { - fixture_write_failures += 1; - entries.push(failure(target.tx_hash, BatchErrorKind::Execution, message)); - continue; - } - let fixture = target.fixture.and_then(|report| report.ok()); - let Some(envelope) = receipts.get(offset + target.commit_index) else { entries.push(failure( target.tx_hash, @@ -783,38 +785,63 @@ where exec_time: target.exec_time, receipt, verification, - fixture, + fixture: target.fixture, }))); } } + // The block itself failed to finish, so no target of it has a receipt: + // that failure outranks whatever each target's fixture dump reported. Err(e) => { - let message = format!("Block execution error: {e}"); + let error = ReplayError::BlockExecutionError(e); + let kind = classify(&error); + let message = error.to_string(); for target in pending { - if matches!(target.fixture, Some(Err(_))) { - fixture_write_failures += 1; - } - entries.push(failure(target.tx_hash, BatchErrorKind::Execution, message.clone())); + entries.push(failure(target.tx_hash, kind, message.clone())); } } } // Any target that produced no entry either sat behind the abort or is not - // part of this block at all. - let (kind, message) = match loop_result { - Ok(()) => (BatchErrorKind::NotFound, format!("Transaction is not part of block {number}")), + // part of this block at all. They are appended in block transaction-index + // order, keeping the run's ascending (block, index) order; a target the + // block does not contain has no index and keeps its input position. + let reported: HashSet = entries.iter().map(BatchEntry::tx_hash).collect(); + let block_txs: HashSet = tx_hashes.iter().copied().collect(); + let unreported = tx_hashes + .iter() + .filter(|hash| target_set.contains(*hash)) + .chain(targets.iter().filter(|hash| !block_txs.contains(*hash))) + .filter(|hash| !reported.contains(*hash)); + + match &loop_result { + Ok(()) => { + let message = format!("Transaction is not part of block {number}"); + for tx_hash in unreported { + entries.push(failure(*tx_hash, BatchErrorKind::NotFound, message.clone())); + } + } Err(e) => { warn!(block = number, error = %e, "Aborted block replay; skipping its remaining targets"); - (classify(&e), e.to_string()) - } - }; - let reported: HashSet = entries.iter().map(BatchEntry::tx_hash).collect(); - for tx_hash in &targets { - if !reported.contains(tx_hash) { - entries.push(failure(*tx_hash, kind, message.clone())); + let aborting = aborting_tx_hash(e); + for tx_hash in unreported { + if aborting == Some(*tx_hash) { + // The abort is this target's own answer. + entries.push(failure(*tx_hash, classify(e), e.to_string())); + } else { + // The abort belongs to another transaction of the block, so + // nothing was established about this target: it went + // unanswered rather than being unknown or invalid. + entries.push(failure( + *tx_hash, + swept_kind(e), + format!("Block replay aborted before this transaction: {e}"), + )); + } + } } } - BlockReplayResult { entries, fixture_write_failures } + entries } /// Inputs for [`dump_target_fixture`], grouped so the dump path stays a single @@ -836,13 +863,13 @@ struct DumpFixtureArgs<'a> { /// Attempt to build and write a fixture for one successfully executed target. /// /// Expected skips (missing receipt, fidelity mismatch, BLOCKHASH, unsupported -/// transaction shapes) return [`FixtureAttempt::Report`] with a skip reason and -/// never fail the batch run. Finalize/write errors return -/// [`FixtureAttempt::WriteError`] and become infrastructure error entries. +/// transaction shapes) report a skip reason and never fail the batch run. +/// Finalize/write errors report a fixture error, which fails the run as an +/// execution-class failure of this target without discarding its result. /// /// `db` must reflect the pre-target-commit state (preceding txs committed, target /// not yet), matching the single-transaction dump. -fn dump_target_fixture(db: &DB, args: DumpFixtureArgs<'_>) -> FixtureAttempt +fn dump_target_fixture(db: &DB, args: DumpFixtureArgs<'_>) -> FixtureReport where DB: DatabaseRef, DB::Error: core::fmt::Display, @@ -867,29 +894,25 @@ where let facts = match onchain { Some(Ok(facts)) => facts, Some(Err(message)) => { - return FixtureAttempt::Report(FixtureReport::skipped(format!( - "fidelity-gate-unavailable: {message}" - ))); + return FixtureReport::skipped(format!("fidelity-gate-unavailable: {message}")); } None => { - return FixtureAttempt::Report(FixtureReport::skipped( + return FixtureReport::skipped( "fidelity-gate-unavailable: no on-chain receipt was fetched for this transaction", - )); + ); } }; if accessed_block_hash_count > 0 { - return FixtureAttempt::Report(FixtureReport::skipped(format!( + return FixtureReport::skipped(format!( "transaction reads block hashes (BLOCKHASH): {accessed_block_hash_count} block \ hash(es) were accessed and the fixture cannot faithfully reproduce them" - ))); + )); } let anchor = fixture::anchor_from_receipt_facts(facts); if let Err(reason) = fixture::check_fidelity(exec_result, &anchor, chain_id) { - return FixtureAttempt::Report(FixtureReport::skipped(format!( - "fidelity gate failed: {reason}" - ))); + return FixtureReport::skipped(format!("fidelity gate failed: {reason}")); } let draft = match fixture::build_draft( @@ -905,14 +928,14 @@ where // Unsupported shapes (deposit, EIP-7702, unknown spec) are expected in // whole-block sweeps: skip rather than fail the run. Err(e) => { - return FixtureAttempt::Report(FixtureReport::skipped(e.to_string())); + return FixtureReport::skipped(e.to_string()); } }; let tx_hash = target_tx.inner.inner.tx_hash(); let path = dir.join(format!("{tx_hash:#x}.json")); if path.exists() && !overwrite { - return FixtureAttempt::WriteError(format!( + return FixtureReport::error(format!( "fixture already exists at {} (pass --overwrite to replace)", path.display() )); @@ -921,9 +944,9 @@ where match fixture::finalize_and_write(draft, &path) { Ok(()) => { info!(path = %path.display(), tx_hash = %tx_hash, "Wrote self-validating fixture"); - FixtureAttempt::Report(FixtureReport::written(&path)) + FixtureReport::written(&path) } - Err(e) => FixtureAttempt::WriteError(format!("fixture write failed: {e}")), + Err(e) => FixtureReport::error(format!("fixture write failed: {e}")), } } @@ -973,15 +996,55 @@ where .ok_or(ReplayError::BlockNotFound(number)) } -/// Map an error raised while replaying a block onto a reported error kind. -const fn classify(err: &ReplayError) -> BatchErrorKind { +/// Map an error raised while replaying a block onto the kind reported for the +/// target the error is about. +fn classify(err: &ReplayError) -> BatchErrorKind { match err { ReplayError::TransactionNotFound(_) => BatchErrorKind::NotFound, ReplayError::RpcError(_) | ReplayError::RpcTransportError(_) => BatchErrorKind::Rpc, + // A block error the EVM raised because a state read failed is that + // read's failure: the same classification the run-level exit code uses. + ReplayError::BlockExecutionError(_) + if ExitCode::from_evme_error(err) == ExitCode::RpcFailure => + { + BatchErrorKind::Rpc + } _ => BatchErrorKind::Execution, } } +/// The kind reported for a target swept up by an abort caused elsewhere. +/// +/// The abort says nothing about this target, so a definitive answer about +/// another transaction (an unknown hash) becomes an unanswered question here. +fn swept_kind(err: &ReplayError) -> BatchErrorKind { + match classify(err) { + BatchErrorKind::NotFound | BatchErrorKind::Rpc => BatchErrorKind::Rpc, + kind => kind, + } +} + +/// The transaction an aborting error is about, when it names one. +fn aborting_tx_hash(err: &ReplayError) -> Option { + match err { + ReplayError::TransactionNotFound(hash) => Some(*hash), + ReplayError::BlockExecutionError(err) => block_error_tx_hash(err), + _ => None, + } +} + +/// The transaction a block execution error names, when it carries one. +fn block_error_tx_hash(err: &BlockExecutionError) -> Option { + if let Some(validation) = err.as_validation() { + return match validation { + BlockValidationError::InvalidTx { hash, .. } | + BlockValidationError::EVM { hash, .. } => Some(*hash), + _ => None, + }; + } + err.as_internal()?.as_evm().map(|(hash, _)| *hash) +} + /// Build a failure entry. fn failure(tx_hash: B256, kind: BatchErrorKind, message: String) -> BatchEntry { BatchEntry::Failed(FailedTx { tx_hash, kind, message }) @@ -1099,17 +1162,22 @@ mod tests { const HASH_A: &str = "0xde3d56dc739484166b8af1bea757bf7e3e9a4b9a0fb62d722703345570dfc1d6"; const HASH_B: &str = "0x323ddc8e67dfc134284d78c65f3c1dc7ff45ba1db02eeaf62e211ae3253478ef"; - /// Build a tally from the outcomes a run would have reported. + /// A verification verdict as a run would have reported it. + fn verdict(matched: bool) -> VerificationOutcome { + VerificationOutcome { matched, diff: None } + } + + /// Build a tally from the outcomes a run would have reported: `failures` + /// error entries, plus `replayed` verified result lines of which + /// `mismatched` diverged from their receipt. fn tally(failures: &[BatchErrorKind], replayed: usize, mismatched: usize) -> BatchTally { let mut tally = BatchTally::default(); for kind in failures { tally.record(&failure(B256::ZERO, *kind, String::new())); } - // Verified targets are counted through the executed entries, which need - // a full `ExecutedTx`; the tally fields they feed are set directly. - tally.replayed = replayed; - tally.verified = replayed; - tally.counts.mismatched = mismatched; + for index in 0..replayed { + tally.record_executed(Some(&verdict(index >= mismatched)), None); + } tally } @@ -1153,11 +1221,43 @@ mod tests { }; assert_eq!(counts, BatchFailureCounts { execution: 2, rpc: 1, mismatched: 1, total: 5 }); assert!( - counts.to_string().contains("3 of 5 target transaction(s) failed to replay"), + counts.to_string().contains("3 of 5 target transaction(s) failed"), "unexpected message: {counts}" ); } + /// A target whose fixture could not be written keeps its result line and + /// its verdict, and still fails the run as an execution-class failure — + /// including when that same target diverged from its on-chain receipt. + #[test] + fn test_batch_tally_counts_a_fixture_failure_and_its_mismatch() { + let mut tally = BatchTally::default(); + tally.record_executed(Some(&verdict(false)), Some(&FixtureReport::error("disk full"))); + + assert_eq!(tally.replayed, 1, "the target replayed"); + assert_eq!(tally.verified, 1, "the target was verified"); + assert_eq!(tally.counts.mismatched, 1, "its divergence is counted"); + assert_eq!(tally.counts.execution, 1, "its failed fixture is counted"); + + let err = tally.into_error().expect("run failed"); + let ReplayError::BatchFailed(counts) = err else { + panic!("a failed fixture must fail the run: {err:?}"); + }; + assert_eq!(counts, BatchFailureCounts { execution: 1, rpc: 0, mismatched: 1, total: 1 }); + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::ExecutionError); + } + + /// A written or skipped fixture is not a failure. + #[test] + fn test_batch_tally_ignores_written_and_skipped_fixtures() { + let mut tally = BatchTally::default(); + tally.record_executed(None, Some(&FixtureReport::written(Path::new("/tmp/tx.json")))); + tally.record_executed(None, Some(&FixtureReport::skipped("fidelity gate failed"))); + + assert_eq!(tally.counts.execution, 0); + assert!(tally.into_error().is_none(), "fixture skips never fail the run"); + } + /// A run whose only finding is divergence fails as the mismatch it is. #[test] fn test_batch_tally_mismatch_only_reports_the_verification_error() { diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 95c76341..0746cf79 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -17,7 +17,7 @@ use mega_evm::{ BlockLimits, EvmTxRuntimeLimits, MegaBlockExecutionCtx, MegaBlockExecutorFactory, MegaEvmFactory, MegaHardforks, MegaSpecId, }; -use tracing::{debug, info, trace, warn}; +use tracing::{debug, error, info, trace, warn}; use alloy_network::ReceiptResponse; use op_alloy_rpc_types::Transaction; @@ -199,13 +199,17 @@ impl Cmd { match run_result { Ok(()) => Ok(persist_result?), Err(run_err) => { - // Surface the original error; a persist failure on top of it is - // logged, not propagated, so it cannot mask the root cause. + // The run error is the root cause and keeps the exit code, so + // the persist failure is not propagated. It is still reported on + // stderr the way the central reporter reports a failure — a + // capture file that never reached disk must not be silent just + // because the run it captured also failed. if let Err(persist_err) = persist_result { - warn!( + error!( error = %persist_err, "Failed to persist RPC cache while handling an earlier error", ); + eprintln!("error: {persist_err}"); } Err(run_err) } @@ -733,9 +737,7 @@ impl Cmd { &mut inspector, ); - block_executor - .apply_pre_execution_changes() - .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + block_executor.apply_pre_execution_changes().map_err(ReplayError::BlockExecutionError)?; // Execute preceding transactions info!(preceding_count = ctx.preceding_tx_hashes.len(), "Executing preceding transactions",); @@ -748,11 +750,11 @@ impl Cmd { .ok_or(ReplayError::TransactionNotFound(*tx_hash))?; let outcome = block_executor .run_transaction(tx.as_recovered()) - .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + .map_err(ReplayError::BlockExecutionError)?; trace!(tx_hash = %tx_hash, ?outcome, "Preceding transaction executed"); block_executor .commit_transaction_outcome(outcome) - .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + .map_err(ReplayError::BlockExecutionError)?; } // Clear block hash reads accumulated by the preceding transactions so the @@ -774,9 +776,8 @@ impl Cmd { .unwrap_or(0); block_executor.inspector_mut().fuse(); - let outcome = block_executor - .run_transaction(wrapped_tx) - .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + let outcome = + block_executor.run_transaction(wrapped_tx).map_err(ReplayError::BlockExecutionError)?; trace!(tx_hash = %ctx.target_tx.inner.inner.tx_hash(), ?outcome, "Target transaction executed"); let exec_result = outcome.inner.result.clone(); let evm_state = outcome.inner.state.clone(); @@ -841,12 +842,11 @@ impl Cmd { let gas_used = block_executor .commit_transaction_outcome(outcome) - .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + .map_err(ReplayError::BlockExecutionError)?; let duration = start.elapsed(); - let (evm, block_result) = block_executor - .finish() - .map_err(|e| ReplayError::Other(format!("Block execution error: {e}")))?; + let (evm, block_result) = + block_executor.finish().map_err(ReplayError::BlockExecutionError)?; let (db, _) = evm.finish(); db.merge_transitions(BundleRetention::Reverts); let receipt_envelope = block_result.receipts.last().unwrap().clone(); diff --git a/bin/mega-evme/tests/exit_codes.rs b/bin/mega-evme/tests/exit_codes.rs index d3deee79..57f5b064 100644 --- a/bin/mega-evme/tests/exit_codes.rs +++ b/bin/mega-evme/tests/exit_codes.rs @@ -20,6 +20,14 @@ const TX_OK: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2 /// A hash the capture holds no response for: the question goes unanswered. const UNANSWERABLE_TX: &str = "0x0000000000000000000000000000000000000000000000000000000000000001"; +/// Request fingerprint of a state read `TX_OK` performs while it executes. +/// +/// Entries are keyed by the request, so dropping this one from a copy of the +/// capture models an endpoint that stops answering mid-execution — the read +/// then fails inside the EVM and surfaces as a block execution error. +const IN_EXECUTION_STATE_READ: &str = + "0x0d9aee1b171e0c4a2be0107def891d838cc94d71e4046cb95b00a1c2a61cffed"; + /// Outcome of one `mega-evme` invocation. struct Run { code: Option, @@ -74,6 +82,23 @@ fn replay(args: &[&str]) -> Run { run(&argv) } +/// Write a copy of the committed capture without the entry `key` answers, and +/// return its path. +fn cache_without_entry(name: &str, key: &str) -> std::path::PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + .expect("parse offline cache"); + let entries = envelope["cache"].as_array_mut().expect("cache entries"); + let before = entries.len(); + entries.retain(|entry| entry["key"].as_str() != Some(key)); + assert_eq!(entries.len() + 1, before, "the capture must hold exactly one entry for {key}"); + + let path = + std::env::temp_dir().join(format!("mega_evme_exit_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write pruned cache"); + path +} + /// Write a `--tx-file` holding `contents`, and return its path. fn tx_file(name: &str, contents: &str) -> std::path::PathBuf { let path = @@ -127,6 +152,38 @@ fn test_offline_cache_miss_exits_rpc_failure_with_a_json_error_object() { ); } +/// A state read that fails while the EVM is executing arrives as a block +/// execution error, but it is still an unanswered question: the run exits 3, and +/// a batch reports the target as an `rpc` failure rather than an execution one. +#[test] +fn test_state_read_failure_during_execution_is_an_rpc_failure() { + let path = cache_without_entry("state_read", IN_EXECUTION_STATE_READ); + let cache = path.to_str().unwrap(); + + let single = run(&["replay", "--rpc.replay-file", cache, "--json", TX_OK]); + assert_eq!(single.code(), 3, "an unanswered state read exits 3.\nstderr: {}", single.stderr); + let error = single.error_object(); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + assert!( + error["error"]["message"] + .as_str() + .is_some_and(|m| m.contains("Block execution error") && m.contains("cache miss")), + "the failure must be the block error carrying the missed read: {error}" + ); + + let list = tx_file("state_read", &format!("{TX_OK}\n")); + let batch = run(&["replay", "--rpc.replay-file", cache, "--tx-file", list.to_str().unwrap()]); + let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_file(&path); + + assert_eq!(batch.code(), 3, "the batch run exits 3 too.\nstderr: {}", batch.stderr); + assert!( + batch.stdout.contains("Error (rpc):"), + "the target is reported as unanswered:\n{}", + batch.stdout + ); +} + /// A batch run's error object follows the per-target lines, so a parser reading /// the stream sees every target before the run-level verdict. #[test] @@ -197,6 +254,69 @@ fn test_successful_run_exits_zero_without_an_error_object() { assert_eq!(run.error_lines(), 0, "a successful run reports nothing on stderr"); } +/// A capture that could not be persisted is reported even when the run it was +/// capturing also failed: the run error keeps the exit code (it is the root +/// cause), and both failures are named on stderr without `-v`, so a stale or +/// missing capture file cannot go unnoticed. +#[tokio::test(flavor = "multi_thread")] +async fn test_capture_persist_failure_is_reported_next_to_the_run_error() { + let server = common::MockRpcServer::start().await; + // Chain id resolves, every other call fails: the replay itself goes + // unanswered while the capture store still has entries to write. + server.respond_eth_chain_id(6342, 1).await; + server.respond_status_always(500).await; + let url = server.uri(); + + // A capture path whose parent is a regular file: persisting cannot succeed. + let blocker = + std::env::temp_dir().join(format!("mega_evme_capture_blocker_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&blocker); + std::fs::write(&blocker, b"not a directory").expect("write blocker file"); + let capture = blocker.join("capture.json"); + + let mut argv = vec![ + "replay", + "--rpc", + &url, + "--rpc.capture-file", + capture.to_str().unwrap(), + "--rpc.max-retries", + "0", + "--rpc.backoff-ms", + "1", + TX_OK, + ]; + let human = run(&argv); + argv.push("--json"); + let json = run(&argv); + let _ = std::fs::remove_file(&blocker); + + for run in [&human, &json] { + assert_eq!(run.code(), 3, "the run error keeps the exit code.\nstderr: {}", run.stderr); + assert_eq!(run.error_lines(), 2, "both failures are reported:\n{}", run.stderr); + assert!( + run.stderr.contains("Failed to fetch transaction"), + "the run error must be reported:\n{}", + run.stderr + ); + assert!( + run.stderr.contains(blocker.to_str().expect("blocker path is utf-8")), + "the persist failure must name where the capture could not be written:\n{}", + run.stderr + ); + } + + // The structured object still reports the run error, which owns the code. + let error = json.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert!( + error["error"]["message"] + .as_str() + .is_some_and(|m| m.contains("Failed to fetch transaction")), + "the object carries the run error, not the persist failure: {error}" + ); +} + /// A usage error is bad input, so it joins the execution class instead of /// colliding with the mismatch code; `--help` stays a successful run. #[test] @@ -210,3 +330,38 @@ fn test_usage_errors_exit_one_and_help_exits_zero() { assert_eq!(help.code(), 0, "--help exits 0"); assert!(help.stdout.contains("mega-evme"), "--help prints usage on stdout"); } + +/// A usage error of a `--json` run still ends stdout with the structured error +/// object: argument parsing fails before the command exists, but a +/// machine-readable run must never end with empty stdout. +#[test] +fn test_usage_error_in_json_mode_ends_stdout_with_the_error_object() { + // No replay target: rejected by argument parsing. + let usage = run(&["replay", "--json"]); + + assert_eq!(usage.code(), 1, "a usage error exits 1.\nstderr: {}", usage.stderr); + let error = usage.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(1)); + assert_eq!(error["error"]["kind"].as_str(), Some("execution-error")); + let message = error["error"]["message"].as_str().expect("the object carries a message"); + assert!(!message.contains('\n'), "the message is a single line: {message}"); + assert!( + message.contains("required arguments"), + "the message must summarize the usage error: {message}" + ); + // clap keeps rendering its own report, including the usage block. + assert!(usage.stderr.contains("Usage:"), "clap still reports on stderr:\n{}", usage.stderr); +} + +/// `--help` in a `--json` run is still not a failure: no error object, exit 0. +#[test] +fn test_help_in_json_mode_prints_no_error_object() { + let help = run(&["--help", "--json"]); + + assert_eq!(help.code(), 0, "--help exits 0"); + assert!( + !help.stdout.lines().any(|line| line.trim_start().starts_with(r#"{"error""#)), + "--help prints no error object:\n{}", + help.stdout + ); +} diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 26603a9a..8393f218 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -69,6 +69,51 @@ fn replay_with_code(args: &[&str]) -> (String, Option) { (String::from_utf8(output.stdout).expect("stdout is utf-8"), output.status.code()) } +/// Write a copy of the envelope whose `eth_getTransactionByHash` response for +/// `tx_hash` answers "unknown transaction", and return its path. +/// +/// Entries are keyed by the request, not the response, so the doctored answer +/// still resolves. This models the endpoint losing one transaction of a block it +/// still serves — the block body lists the hash, the lookup denies it. +fn envelope_without_transaction(name: &str, tx_hash: &str) -> std::path::PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + // Only the transaction's own response carries it as the `hash` field; the + // block body lists bare hashes and a receipt names it `transactionHash`. + let marker = format!("\"hash\":\"{tx_hash}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + response["result"] = serde_json::Value::Null; + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "the envelope must hold exactly one response for {tx_hash}"); + + let path = + std::env::temp_dir().join(format!("mega_evme_batch_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored envelope"); + path +} + +/// Run `replay` against `envelope_path` and return its stdout plus its exit code. +fn replay_envelope_with_code( + envelope_path: &std::path::Path, + args: &[&str], +) -> (String, Option) { + let mut cmd = mega_evme(); + cmd.args(["replay", "--rpc.replay-file", envelope_path.to_str().expect("path is utf-8")]); + cmd.args(args); + let output = cmd.output().expect("failed to run mega-evme"); + (String::from_utf8(output.stdout).expect("stdout is utf-8"), output.status.code()) +} + /// Parse NDJSON stdout into one JSON value per line, dropping the structured /// error object a failing run ends with. fn ndjson(stdout: &str) -> Vec { @@ -241,6 +286,91 @@ fn test_replay_block_verify_receipt_without_receipts_reports_rpc_errors() { assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); } +/// An abort caused by one transaction of the block is not an answer about the +/// targets behind it: only the transaction the endpoint denied is reported as +/// `not_found`, and every target swept up behind it is reported as unanswered +/// (`rpc`) with a message naming the transaction that aborted the block. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_block_sweeps_targets_behind_an_abort_as_unanswered() { + let (missing, missing_index) = BLOCK_TXS[1]; + let path = envelope_without_transaction("abort_block", missing); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--block", &BLOCK.to_string(), "--json"]); + let _ = std::fs::remove_file(&path); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + for (index, line) in lines.iter().enumerate() { + let index = index as u64; + if index < missing_index { + assert!(line.get("error").is_none(), "targets before the abort replay: {line}"); + continue; + } + if index == missing_index { + assert_eq!( + line["error"]["kind"].as_str(), + Some("not_found"), + "only the denied transaction is unknown: {line}" + ); + continue; + } + assert_eq!( + line["error"]["kind"].as_str(), + Some("rpc"), + "a target swept up behind the abort went unanswered: {line}" + ); + assert!( + line["error"]["message"].as_str().is_some_and(|m| m.contains(missing)), + "the message must name the transaction that aborted the block: {line}" + ); + } + + // The denied transaction is an execution-class failure, which outranks the + // unanswered ones. + assert_eq!(code, Some(1), "a definitive negative answer exits 1"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("execution-error")); +} + +/// Targets swept up by an abort are reported in block transaction-index order, +/// whatever order `--tx-file` listed them in. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_tx_file_sweeps_targets_in_block_order() { + let missing = BLOCK_TXS[0].0; + let path = envelope_without_transaction("abort_order", missing); + // Deliberately reversed: the last transaction of the block first. + let list = format!("{}\n{}\n", BLOCK_TXS[2].0, BLOCK_TXS[1].0); + let list_path = + std::env::temp_dir().join(format!("mega_evme_tx_list_order_{}.txt", std::process::id())); + std::fs::write(&list_path, list).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--tx-file", list_path.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list_path); + let lines = ndjson(&stdout); + + let observed: Vec<&str> = lines.iter().map(|line| line["tx_hash"].as_str().unwrap()).collect(); + assert_eq!( + observed, + vec![BLOCK_TXS[1].0, BLOCK_TXS[2].0], + "swept targets must follow the block's transaction order, not the input order", + ); + for line in &lines { + assert_eq!(line["error"]["kind"].as_str(), Some("rpc"), "swept target: {line}"); + assert!( + line["error"]["message"].as_str().is_some_and(|m| m.contains(missing)), + "the message must name the transaction that aborted the block: {line}" + ); + } + + // No target was answered definitively, so the run is an RPC failure. + assert_eq!(code, Some(3), "targets that went unanswered exit 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); +} + /// Batch mode rejects the single-transaction-only flags before doing any work. #[test] #[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index 28a199d1..d19f0731 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -454,6 +454,51 @@ fn test_batch_dump_fixture_dir_writes_validatable_file() { result.unwrap_or_else(|e| panic!("dumped fixture failed to validate: {e}")); } +/// A target whose fixture could not be written is still verified against its +/// on-chain receipt: the failed dump is reported on the target's result line +/// alongside the verdict, and fails the run as an execution-class failure. +#[test] +fn test_batch_fixture_write_failure_keeps_the_receipt_verification() { + let list = tx_file("batch_dump_verify"); + let dir = + std::env::temp_dir().join(format!("mega_evme_batch_dump_verify_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + // Pre-create the fixture path so the dump is refused without --overwrite. + std::fs::create_dir_all(&dir).expect("create dump dir"); + std::fs::write(dir.join(format!("{TX}.json")), "{}").expect("pre-create fixture file"); + + let run = replay( + &cache(), + &[ + "--tx-file", + list.to_str().unwrap(), + "--verify-receipt", + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(run.code(), 1, "a failed dump exits 1.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "one line per requested transaction"); + assert!(lines[0].get("error").is_none(), "a failed dump is not an error entry: {}", lines[0]); + assert_eq!( + lines[0]["verification"], + serde_json::json!({ "match": true }), + "the verification still ran: {}", + lines[0] + ); + assert!( + lines[0]["fixture"]["error"].as_str().is_some_and(|m| m.contains("already exists")), + "the failed dump is reported on the result line: {}", + lines[0] + ); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("execution-error")); +} + /// Without `--overwrite`, a second dump into a directory that already holds the /// fixture fails that target as an infrastructure error. #[test] @@ -484,18 +529,22 @@ fn test_batch_dump_fixture_dir_refuses_overwrite_without_flag() { "--json", ], ); - // The dump failed for the target, which is an execution-class failure. + // The dump failed for the target, which is an execution-class failure. The + // target still replayed, so it keeps its result line and reports the failed + // dump on it. assert_eq!(second.code(), 1, "a failed dump exits 1.\nstderr: {}", second.stderr); let lines = second.ndjson(); assert_eq!(lines.len(), 1); - assert_eq!(lines[0]["error"]["kind"].as_str(), Some("execution")); + assert!(lines[0].get("error").is_none(), "a failed dump is not an error entry: {}", lines[0]); + assert!(lines[0]["receipt"].is_object(), "the replayed target keeps its receipt: {}", lines[0]); assert!( - lines[0]["error"]["message"] + lines[0]["fixture"]["error"] .as_str() .is_some_and(|m| m.contains("already exists") && m.contains("--overwrite")), "expected overwrite refusal: {}", lines[0] ); + assert_eq!(second.error_object()["error"]["kind"].as_str(), Some("execution-error")); // With --overwrite the second dump succeeds and replaces the file. let third = replay( diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 34bfb783..e28e0d95 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -109,16 +109,21 @@ A transaction that could not be executed is reported as an error entry instead: `kind` is one of `not_found` (unknown hash), `pending` (mined into no block yet), `rpc` (an RPC call failed), or `execution` (block setup or the block executor rejected the transaction). Execution outcomes are not errors: a reverted or halted transaction is a normal result line with `success: false`. +A failure while running the block aborts it, because the executor state no longer matches the chain. +The transaction the failure is about — the hash the endpoint denied, or the one the executor rejected — is reported with that failure's own kind. +Every target behind it is reported as `rpc` with a message naming the aborting cause: nothing was established about those transactions, so they went unanswered rather than being unknown. +Targets that never ran are still emitted in the block's transaction-index order, keeping the whole stream in ascending `(block, tx_index)` order; a hash the block does not contain is reported last, in input order, as `not_found`. + Without `--json`, each transaction is printed with a header naming its hash, block, and index, followed by the same summary and receipt the single-transaction mode prints. A final one-line summary (transactions replayed, transactions failed, elapsed time) is logged at `INFO` level, so pass `-vvv` to see it. With [`--verify-receipt`](#receipt-verification), each result line additionally carries a `verification` object. -With [`--dump-fixture-dir`](#dump-fixture-dir-dir), each result line additionally carries a `fixture` object (`path` or `skipped`). +With [`--dump-fixture-dir`](#dump-fixture-dir-dir), each result line additionally carries a `fixture` object (`path`, `skipped`, or `error`). ### Exit Status -A batch run exits `0` when every requested transaction produced an execution result, and `1` when any of them produced an error entry. -Fixture skips (fidelity gate, BLOCKHASH readers, unsupported shapes) are not error entries and do not fail the run. +A batch run exits `0` when every requested transaction produced an execution result and nothing the run was asked to do failed, and non-zero otherwise — see [Exit codes](../overview.md#exit-codes) for how the failure classes are ranked. +Fixture skips (fidelity gate, BLOCKHASH readers, unsupported shapes) are not failures and do not fail the run; a fixture the run was asked to write and could not is an execution-class failure of its target. The NDJSON stream is written to stdout in both cases; diagnostics go to stderr. ### Examples @@ -283,6 +288,9 @@ The updated set of entries is persisted back to the same file on clean exit. The file also embeds an external-environment snapshot — currently the set of `--bucket-capacity` values in effect — so the captured fixture is self-contained. If `--bucket-capacity` is not passed on a subsequent run, the previous envelope's values are reused; passing `--bucket-capacity` overrides them. +The capture is written even when the replay itself failed — an execution or verification failure is exactly the case you want to debug offline. +If the write fails, it is reported on stderr like any other failure, next to the run's own error; the run error keeps the exit code, since it is the root cause. + `--rpc.capture-file` is mutually exclusive with `--rpc.replay-file`, `--rpc.cache-dir`, `--rpc.clear-cache`, `--rpc.no-cache-file`, and `--rpc.cache-max-entries`. ### `--rpc.replay-file ` @@ -364,7 +372,7 @@ Batch-only. Dump a self-validating fixture for every successfully replayed target into `

/.json`. The fixture content and format match the single-transaction [`--dump-fixture`](#dump-fixture-file) path (same EEST schema, same sorted `megaEnv`, same self-validation via `state-test`). The directory is created if it does not exist. -Existing files are refused unless `--overwrite` is also set — a refused overwrite is an infrastructure error for that target (`execution`), not a skip. +Existing files are refused unless `--overwrite` is also set — a refused overwrite is a failed dump for that target, not a skip. Per-target gating mirrors the single-transaction rules, but records a skip instead of failing the run: @@ -374,15 +382,17 @@ Per-target gating mirrors the single-transaction rules, but records a skip inste | Fidelity mismatch (gas / status / logs root) | `fixture.skipped` with `fidelity gate failed: …` | | Target reads `BLOCKHASH` | `fixture.skipped` (fixtures carry no historical block hashes) | | Unsupported shape (deposit, EIP-7702, unknown spec mapping) | `fixture.skipped` | -| Finalize / write / self-validation failure | infrastructure error entry (`kind: execution`) | +| Finalize / write / self-validation failure, refused overwrite | `fixture.error` with the reason; execution-class failure | | Pending / unresolvable target | already an error entry; no fixture report | `BLOCKHASH` access is isolated per transaction: the access record is cleared before each transaction of the block, so preceding readers do not poison a later target's dump. -NDJSON result lines gain `"fixture": {"path": "…"}` or `"fixture": {"skipped": ""}`. +NDJSON result lines gain `"fixture": {"path": "…"}`, `"fixture": {"skipped": ""}`, or `"fixture": {"error": ""}`. Human mode prints one fixture line per target. An end-of-run `INFO` summary reports written / skipped / failed counts. -Fixture skips do not fail the run; infrastructure failures keep the usual batch exit semantics. + +A failed dump is reported on the target's own result line rather than replacing it: the transaction did replay, so its receipt — and, with [`--verify-receipt`](#receipt-verification), its verdict — is still what the run was asked for, and a divergence found on such a target is still counted as a mismatch. +Fixture skips do not fail the run; a failed dump does, as an execution-class failure of that target. Registration into `bench/replay/manifest.json` is not performed — corpus curation stays manual. `--dump-fixture-dir` cannot be combined with `--dump-fixture`, and is rejected in single-transaction mode. diff --git a/docs/mega-evme/overview.md b/docs/mega-evme/overview.md index 2ffde16b..3557f436 100644 --- a/docs/mega-evme/overview.md +++ b/docs/mega-evme/overview.md @@ -79,17 +79,22 @@ Every command reports its outcome through the same set of exit codes, so a pipel | `3` | `rpc-failure` | An RPC or transport call failed — endpoint unreachable, transport error, or an offline replay file that holds no response for a request the run had to make. | Codes `1` and `3` separate the two ways a question can go wrong: `1` means the tool answered, and the answer is negative; `3` means the question went unanswered, so retrying against a healthy endpoint may still produce a result. +A state read that fails while the EVM is executing — an offline replay file without the response, or an endpoint that dies mid-transaction — belongs to `3` as well, even though it surfaces as a block execution error. +Two paths cannot be classified that way: a read that fails inside the pre-block system calls (EIP-4788 beacon root, EIP-2935 block hashes) or inside the sandboxed execution of the keyless-deploy system contract has its cause rendered into a message by the layer that raises it, so `mega-evme` cannot tell it from an execution failure and reports `1`. A batch run (`--tx-file` / `--block`) reports every target on its own line and then exits once for the run as a whole, ranking the failure classes it saw: any execution or internal failure exits `1`, otherwise any RPC failure exits `3`, otherwise any verification mismatch exits `2`. A target that never replayed was also never verified, which is why an infrastructure failure outranks a mismatch. -On failure the run also prints a report: one `error: ` line on stderr, plus — with `--json` — a structured object as the last line of stdout, so a machine-readable run never ends with empty output. +On failure the run also prints a report: one `error: ` line per failure on stderr, plus — with `--json` — a structured object as the last line of stdout, so a machine-readable run never ends with empty output. +A run reports more than one line when a secondary failure must not go unnoticed but does not own the exit code — an unwritable [`--rpc.capture-file`](commands/replay.md#rpccapture-file-path) behind an earlier replay failure, for instance. +The structured object always carries the failure the exit code came from. ```json { "error": { "code": 3, "kind": "rpc-failure", "message": "RPC error: …" } } ``` In batch mode that object follows the per-target lines, whose own `error.kind` (`not_found`, `pending`, `rpc`, `execution`) describes why one target failed and is independent of the run-level class above. +A usage error is reported the same way: the argument parser prints its own report and usage block on stderr, and a `--json` run still ends its stdout with the object, whose `message` is a one-line summary of the parse failure. New failure classes are added as new codes; the meaning of an existing code does not change. From 53e876834ccbaf78e4e4edff8ba779b9de644817 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 02:19:01 +0800 Subject: [PATCH 11/64] fix(mega-evme): optimistic external_env concurrency, swept kind, panic JSON Carry load-time external_env through capture persist so intentional --bucket-capacity refreshes win while true concurrent conflicts hard-error; canonicalize bucket lists; always report non-aborting swept targets as rpc; emit structured JSON on panic under --json. --- bin/mega-evme/src/cache/merge.rs | 243 +++++++++++++++--- .../src/common/provider/cache_store.rs | 206 +++++++++++++-- bin/mega-evme/src/lib.rs | 51 ++++ bin/mega-evme/src/replay/batch.rs | 39 ++- bin/mega-evme/tests/replay_batch.rs | 90 +++++++ docs/mega-evme/commands/replay.md | 2 +- .../configuration/state-management.md | 10 +- 7 files changed, 575 insertions(+), 66 deletions(-) diff --git a/bin/mega-evme/src/cache/merge.rs b/bin/mega-evme/src/cache/merge.rs index 0e7ce5f9..ba4260c0 100644 --- a/bin/mega-evme/src/cache/merge.rs +++ b/bin/mega-evme/src/cache/merge.rs @@ -58,6 +58,27 @@ pub(crate) struct ExternalEnvDoc { pub bucket_capacities: Vec<(u32, u64)>, } +impl ExternalEnvDoc { + /// Canonical form used for equality and on-disk writes. + /// + /// Deduplicates by bucket id with last-wins (matching runtime map-insert + /// semantics when applying `--bucket-capacity`), then sorts by bucket id so + /// two workers with the same effective capacities never conflict solely + /// because of CLI order. + pub(crate) fn canonicalized(&self) -> Self { + Self { bucket_capacities: canonicalize_bucket_capacities(&self.bucket_capacities) } + } +} + +/// Deduplicate by bucket id (last-wins), then sort by bucket id. +pub(crate) fn canonicalize_bucket_capacities(caps: &[(u32, u64)]) -> Vec<(u32, u64)> { + let mut map = BTreeMap::new(); + for &(id, capacity) in caps { + map.insert(id, capacity); + } + map.into_iter().collect() +} + /// Path of the advisory lock sidecar for `target` (`.lock`). pub(crate) fn lock_sidecar_path(target: &Path) -> PathBuf { let mut os = target.as_os_str().to_owned(); @@ -233,13 +254,24 @@ pub(crate) fn merge_provider_lists(base: Vec, overlay: Vec) -> /// Merge `ours` over `on_disk` for envelope persist (ours wins on key collision). /// -/// Returns an error if `chain_id` or `version` disagree, or if both sides carry -/// non-null `external_env` snapshots that are not identical (same rule as -/// [`merge_envelopes_cli`]). One-sided or identical snapshots are accepted: -/// ours is kept when set, otherwise the on-disk snapshot is propagated. +/// Returns an error if `chain_id` or `version` disagree. +/// +/// `external_env` uses optimistic concurrency against `loaded_external_env` — the +/// snapshot observed when this process opened the capture file (or `None` when +/// the file was absent / had no snapshot): +/// +/// - If the locked on-disk snapshot is absent, or still equals the loaded one, nobody else changed +/// it: the caller's intentional update wins (`ours`). +/// - Hard-error only when the on-disk snapshot changed since load **and** differs from ours (true +/// concurrent conflict). The error names all three values (loaded, ours, on-disk). +/// +/// Snapshots are canonicalized (last-wins per bucket id, then sorted) before +/// comparison and before writing, so CLI order alone cannot spuriously conflict. +/// One-sided snapshots still merge: ours is kept when set, otherwise on-disk. pub(crate) fn merge_envelope_for_persist( on_disk: &EnvelopeDoc, ours: &EnvelopeDoc, + loaded_external_env: Option<&ExternalEnvDoc>, path: &Path, ) -> Result { if on_disk.version != ours.version { @@ -258,19 +290,12 @@ pub(crate) fn merge_envelope_for_persist( ours.chain_id, ))); } - let external_env = match (&ours.external_env, &on_disk.external_env) { - (Some(a), Some(b)) if a != b => { - return Err(EvmeError::FixtureError(format!( - "Conflicting external_env snapshots when merging '{}': \ - on-disk {on_disk_env:?}, ours {ours_env:?}", - path.display(), - on_disk_env = on_disk.external_env, - ours_env = ours.external_env, - ))); - } - (Some(a), _) => Some(a.clone()), - (None, disk) => disk.clone(), - }; + let external_env = resolve_external_env_for_persist( + &ours.external_env, + &on_disk.external_env, + loaded_external_env, + path, + )?; Ok(EnvelopeDoc { version: ours.version, chain_id: ours.chain_id, @@ -279,11 +304,50 @@ pub(crate) fn merge_envelope_for_persist( }) } +/// Resolve the envelope `external_env` under optimistic concurrency. +/// +/// See [`merge_envelope_for_persist`] for the accept / hard-error rules. +fn resolve_external_env_for_persist( + ours: &Option, + on_disk: &Option, + loaded: Option<&ExternalEnvDoc>, + path: &Path, +) -> Result> { + let ours_c = ours.as_ref().map(ExternalEnvDoc::canonicalized); + let disk_c = on_disk.as_ref().map(ExternalEnvDoc::canonicalized); + let loaded_c = loaded.map(ExternalEnvDoc::canonicalized); + + match (&ours_c, &disk_c) { + (Some(o), Some(d)) if o != d => { + // Differing non-null snapshots: accept only when the on-disk value + // is still the one this process observed at load (intentional A→B + // refresh). A third concurrent writer that changed the file since + // load is a true conflict. + let disk_unchanged = disk_c == loaded_c; + if disk_unchanged { + Ok(ours_c) + } else { + Err(EvmeError::FixtureError(format!( + "Conflicting external_env snapshots when merging '{}': \ + loaded {loaded_env:?}, ours {ours_env:?}, on-disk {on_disk_env:?}", + path.display(), + loaded_env = loaded_c, + ours_env = ours_c, + on_disk_env = disk_c, + ))) + } + } + (Some(_), _) => Ok(ours_c), + (None, disk) => Ok(disk.clone()), + } +} + /// Merge multiple envelope inputs for the `cache merge` subcommand. /// /// All inputs must share `version` and `chain_id`. Later inputs win on cache -/// key collision. Non-null `external_env` values must be identical when more -/// than one is present. +/// key collision. Non-null `external_env` values must be identical (after +/// canonicalization) when more than one is present; the written snapshot is +/// always the canonical form. pub(crate) fn merge_envelopes_cli(docs: &[(PathBuf, EnvelopeDoc)]) -> Result { let Some((_, first)) = docs.first() else { return Err(EvmeError::InvalidInput("cache merge requires at least one input file".into())); @@ -317,9 +381,10 @@ pub(crate) fn merge_envelopes_cli(docs: &[(PathBuf, EnvelopeDoc)]) -> Result external_env = Some(ext.clone()), - Some(prev) if prev != ext => { + None => external_env = Some(canon), + Some(prev) if prev != &canon => { return Err(EvmeError::InvalidInput(format!( "Conflicting external_env snapshots while merging '{}'", path.display(), @@ -477,7 +542,8 @@ mod tests { cache: vec![kv(1, "ours"), kv(2, "new")], external_env: None, }; - let merged = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap(); + let merged = + merge_envelope_for_persist(&on_disk, &ours, None, Path::new("x.json")).unwrap(); assert_eq!(merged.cache, vec![kv(1, "ours"), kv(2, "new")]); assert_eq!(merged.external_env, Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10)] })); } @@ -486,33 +552,80 @@ mod tests { fn test_merge_envelope_for_persist_chain_id_mismatch() { let on_disk = EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: None }; let ours = EnvelopeDoc { version: 1, chain_id: 2, cache: vec![], external_env: None }; - let err = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap_err(); + let err = + merge_envelope_for_persist(&on_disk, &ours, None, Path::new("x.json")).unwrap_err(); assert!(err.to_string().contains("chain_id")); } - /// Conflicting non-null `external_env` snapshots hard-error and name both values. + /// Intentional refresh: loaded A, ours B, disk still A → B wins. #[test] - fn test_merge_envelope_for_persist_rejects_conflicting_external_env() { + fn test_merge_envelope_for_persist_intentional_refresh_wins() { + let loaded = ExternalEnvDoc { bucket_capacities: vec![(1, 10)] }; + let ours_ext = ExternalEnvDoc { bucket_capacities: vec![(1, 99)] }; let on_disk = EnvelopeDoc { version: 1, chain_id: 7, cache: vec![kv(1, "disk")], - external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10)] }), + external_env: Some(loaded.clone()), + }; + let ours = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(2, "ours")], + external_env: Some(ours_ext.clone()), + }; + let merged = + merge_envelope_for_persist(&on_disk, &ours, Some(&loaded), Path::new("capture.json")) + .expect("intentional A→B refresh must succeed"); + assert_eq!(merged.external_env, Some(ours_ext.canonicalized())); + assert_eq!(merged.cache, vec![kv(1, "disk"), kv(2, "ours")]); + } + + /// True concurrent conflict: loaded A, ours B, disk now C≠B → hard error naming A/B/C. + #[test] + fn test_merge_envelope_for_persist_rejects_true_concurrent_conflict() { + let loaded = ExternalEnvDoc { bucket_capacities: vec![(1, 10)] }; + let ours_ext = ExternalEnvDoc { bucket_capacities: vec![(1, 99)] }; + let disk_ext = ExternalEnvDoc { bucket_capacities: vec![(1, 42)] }; + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "disk")], + external_env: Some(disk_ext), }; let ours = EnvelopeDoc { version: 1, chain_id: 7, cache: vec![kv(2, "ours")], - external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 99)] }), + external_env: Some(ours_ext), }; let err = - merge_envelope_for_persist(&on_disk, &ours, Path::new("capture.json")).unwrap_err(); + merge_envelope_for_persist(&on_disk, &ours, Some(&loaded), Path::new("capture.json")) + .unwrap_err(); let msg = err.to_string(); assert!(msg.contains("external_env"), "msg={msg}"); - assert!(msg.contains("on-disk"), "msg={msg}"); + assert!(msg.contains("loaded"), "msg={msg}"); assert!(msg.contains("ours"), "msg={msg}"); - // Both snapshots named in the message (Debug form of bucket capacities). - assert!(msg.contains("10") && msg.contains("99"), "msg={msg}"); + assert!(msg.contains("on-disk"), "msg={msg}"); + // All three snapshots named (Debug form of bucket capacities). + assert!(msg.contains("10") && msg.contains("99") && msg.contains("42"), "msg={msg}"); + } + + /// Loaded none, ours B, disk now C≠B → hard error (file gained a foreign snapshot). + #[test] + fn test_merge_envelope_for_persist_rejects_conflict_when_loaded_none() { + let ours_ext = ExternalEnvDoc { bucket_capacities: vec![(1, 99)] }; + let disk_ext = ExternalEnvDoc { bucket_capacities: vec![(1, 42)] }; + let on_disk = + EnvelopeDoc { version: 1, chain_id: 7, cache: vec![], external_env: Some(disk_ext) }; + let ours = + EnvelopeDoc { version: 1, chain_id: 7, cache: vec![], external_env: Some(ours_ext) }; + let err = merge_envelope_for_persist(&on_disk, &ours, None, Path::new("capture.json")) + .unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("external_env"), "msg={msg}"); + assert!(msg.contains("loaded"), "msg={msg}"); + assert!(msg.contains("99") && msg.contains("42"), "msg={msg}"); } /// Identical non-null `external_env` snapshots merge successfully. @@ -531,9 +644,40 @@ mod tests { cache: vec![kv(2, "ours")], external_env: Some(ext.clone()), }; - let merged = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap(); + let merged = + merge_envelope_for_persist(&on_disk, &ours, Some(&ext), Path::new("x.json")).unwrap(); assert_eq!(merged.cache, vec![kv(1, "disk"), kv(2, "ours")]); - assert_eq!(merged.external_env, Some(ext)); + assert_eq!(merged.external_env, Some(ext.canonicalized())); + } + + /// Same capacities in different order are not a conflict (canonical equality). + #[test] + fn test_merge_envelope_for_persist_order_insensitive_external_env() { + let a = ExternalEnvDoc { bucket_capacities: vec![(1, 10), (2, 20)] }; + let b = ExternalEnvDoc { bucket_capacities: vec![(2, 20), (1, 10)] }; + let on_disk = EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: Some(a) }; + let ours = EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: Some(b) }; + // Concurrent writer used the same effective capacities in different CLI order. + let merged = merge_envelope_for_persist( + &on_disk, + &ours, + Some(&ExternalEnvDoc { bucket_capacities: vec![(9, 9)] }), + Path::new("x.json"), + ) + .expect("order-only difference must not conflict"); + assert_eq!( + merged.external_env, + Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10), (2, 20)] }) + ); + } + + /// Duplicate bucket ids collapse with last-wins before sort. + #[test] + fn test_canonicalize_bucket_capacities_last_wins_and_sorts() { + let caps = canonicalize_bucket_capacities(&[(2, 20), (1, 10), (2, 99), (1, 11)]); + assert_eq!(caps, vec![(1, 11), (2, 99)]); + let doc = ExternalEnvDoc { bucket_capacities: vec![(3, 1), (1, 2), (3, 9)] }; + assert_eq!(doc.canonicalized().bucket_capacities, vec![(1, 2), (3, 9)]); } /// One-sided `external_env` propagates the non-null snapshot (either side). @@ -546,8 +690,9 @@ mod tests { EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: Some(ext.clone()) }; let ours = EnvelopeDoc { version: 1, chain_id: 1, cache: vec![kv(1, "a")], external_env: None }; - let merged = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap(); - assert_eq!(merged.external_env, Some(ext.clone())); + let merged = + merge_envelope_for_persist(&on_disk, &ours, None, Path::new("x.json")).unwrap(); + assert_eq!(merged.external_env, Some(ext.canonicalized())); // Ours Some, disk None → ours kept. let on_disk = EnvelopeDoc { version: 1, chain_id: 1, cache: vec![], external_env: None }; @@ -557,8 +702,9 @@ mod tests { cache: vec![kv(1, "a")], external_env: Some(ext.clone()), }; - let merged = merge_envelope_for_persist(&on_disk, &ours, Path::new("x.json")).unwrap(); - assert_eq!(merged.external_env, Some(ext)); + let merged = + merge_envelope_for_persist(&on_disk, &ours, None, Path::new("x.json")).unwrap(); + assert_eq!(merged.external_env, Some(ext.canonicalized())); } #[test] @@ -580,6 +726,29 @@ mod tests { assert!(err.to_string().contains("external_env")); } + /// CLI merge treats equal capacities in different order as identical. + #[test] + fn test_merge_envelopes_cli_order_insensitive_external_env() { + let a = EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(1, "a")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(2, 20), (1, 10)] }), + }; + let b = EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(2, "b")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10), (2, 20)] }), + }; + let docs = vec![(PathBuf::from("a.json"), a), (PathBuf::from("b.json"), b)]; + let merged = merge_envelopes_cli(&docs).expect("order-only difference must merge"); + assert_eq!( + merged.external_env, + Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10), (2, 20)] }) + ); + } + #[test] fn test_lock_sidecar_path_suffix() { let p = Path::new("/tmp/rpc-cache-1.json"); diff --git a/bin/mega-evme/src/common/provider/cache_store.rs b/bin/mega-evme/src/common/provider/cache_store.rs index 6799bde3..0940ca22 100644 --- a/bin/mega-evme/src/common/provider/cache_store.rs +++ b/bin/mega-evme/src/common/provider/cache_store.rs @@ -76,6 +76,11 @@ enum RpcCacheStoreInner { /// by the command layer once it has computed the effective value /// from CLI + prior envelope. external_env: Option, + /// Snapshot observed when the capture file was loaded (or `None` when + /// the file was absent / carried no snapshot). Used for optimistic + /// concurrency at persist: intentional A→B refreshes are accepted when + /// the locked re-read is still A; only a true concurrent change conflicts. + loaded_external_env: Option, }, } @@ -91,15 +96,28 @@ impl RpcCacheStore { /// Construct a store backed by a transport-level fixture envelope file. /// /// `pub(super)` to keep the `TransportCache` parameter from leaking out of - /// this module. The snapshot field starts empty; callers inject it later + /// this module. The write snapshot starts empty; callers inject it later /// via [`Self::set_external_env`]. + /// + /// The on-disk snapshot at `path` (if any) is captured here as the load-time + /// baseline for optimistic concurrency at persist — intentional A→B + /// refreshes are accepted when the locked re-read still matches this value. pub(super) fn new_envelope(cache: TransportCache, path: PathBuf, chain_id: u64) -> Self { + // Observe the load-time external_env once, at store construction (the + // same moment capture mode opens the file). Re-reading at persist is + // compared against this baseline, not against a second open-time read. + let loaded_external_env = if path.exists() { + CacheFileEnvelope::load(&path).ok().and_then(|e| e.external_env) + } else { + None + }; Self { inner: Some(RpcCacheStoreInner::FixtureCapture { cache, path, chain_id, external_env: None, + loaded_external_env, }), } } @@ -184,9 +202,16 @@ impl RpcCacheStore { } Ok(()) } - RpcCacheStoreInner::FixtureCapture { cache, path, chain_id, external_env } => { + RpcCacheStoreInner::FixtureCapture { + cache, + path, + chain_id, + external_env, + loaded_external_env, + } => { let entry_count = cache.len(); - CacheFileEnvelope::new(&cache, chain_id, external_env.as_ref()).save(&path)?; + CacheFileEnvelope::new(&cache, chain_id, external_env.as_ref()) + .save(&path, loaded_external_env.as_ref())?; info!( path = %path.display(), entries = entry_count, @@ -365,15 +390,23 @@ impl CacheFileEnvelope { /// Atomically write this envelope to `path` under a lock, merging with any /// on-disk envelope already present (ours win on cache key collision). /// - /// `external_env` keeps ours if set, else the on-disk one. Both sides non-null - /// and not identical is a hard error (same rule as offline `cache merge`). + /// `loaded_external_env` is the snapshot observed when this process opened + /// the capture file. Persist accepts an intentional A→B refresh when the + /// locked re-read is still A (or absent); a concurrent change to a third + /// snapshot C hard-errors and names loaded/ours/on-disk. See + /// [`merge_envelope_for_persist`]. + /// /// On-disk re-read failures are typed: identity/schema mismatches hard-fail; /// corrupt JSON degrades to ours-only with a warning. /// /// Lock contention blocks until the lock is free. Failure to create/acquire /// the lock degrades to an unlocked write with a `warn!`. Write failures /// remain hard errors. - pub(super) fn save(&self, path: &Path) -> Result<()> { + pub(super) fn save( + &self, + path: &Path, + loaded_external_env: Option<&ExternalEnvSnapshot>, + ) -> Result<()> { let _guard = match acquire_exclusive_lock(path) { Ok(g) => Some(g), Err(err) => { @@ -387,10 +420,14 @@ impl CacheFileEnvelope { }; let ours = self.to_merge_doc()?; + let loaded_doc = loaded_external_env + .map(|e| ExternalEnvDoc { bucket_capacities: e.bucket_capacities.clone() }); let to_write = if path.exists() { // Typed hard vs degradable: no substring matching on formatted messages. match reread_envelope_for_merge(path) { - EnvelopeReread::Ok(on_disk) => merge_envelope_for_persist(&on_disk, &ours, path)?, + EnvelopeReread::Ok(on_disk) => { + merge_envelope_for_persist(&on_disk, &ours, loaded_doc.as_ref(), path)? + } EnvelopeReread::Hard(err) => return Err(err), EnvelopeReread::Degradable(msg) => { warn!( @@ -398,11 +435,12 @@ impl CacheFileEnvelope { error = %msg, "Failed to re-read on-disk envelope during merge; persisting our entries only", ); - ours + // Still write the canonical form when replacing corrupt content. + canonicalize_envelope_external_env(ours) } } } else { - ours + canonicalize_envelope_external_env(ours) }; write_envelope_atomic(path, &to_write) @@ -424,6 +462,15 @@ impl CacheFileEnvelope { } } +/// Canonicalize `external_env` on a merge doc about to be written alone (no +/// on-disk merge). Merge path already returns a canonical snapshot. +fn canonicalize_envelope_external_env(mut doc: EnvelopeDoc) -> EnvelopeDoc { + if let Some(ext) = doc.external_env.take() { + doc.external_env = Some(ext.canonicalized()); + } + doc +} + /// Snapshot of mega-evm external environment inputs not derivable from RPC. #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct ExternalEnvSnapshot { @@ -457,7 +504,7 @@ mod tests { .expect("seed cache"); let ext = ExternalEnvSnapshot { bucket_capacities: vec![(1, 100), (2, 200)] }; - CacheFileEnvelope::new(&cache, 4326, Some(&ext)).save(&path).expect("save envelope"); + CacheFileEnvelope::new(&cache, 4326, Some(&ext)).save(&path, None).expect("save envelope"); let envelope = CacheFileEnvelope::load(&path).expect("load envelope"); assert_eq!(envelope.version, 1); @@ -594,7 +641,7 @@ mod tests { "value": r#"{"result":"b"}"#, }])) .expect("seed b"); - CacheFileEnvelope::new(&cache_b, 99, None).save(&path).expect("save b"); + CacheFileEnvelope::new(&cache_b, 99, None).save(&path, None).expect("save b"); let cache_a = TransportCache::new(); cache_a @@ -603,7 +650,7 @@ mod tests { "value": r#"{"result":"a"}"#, }])) .expect("seed a"); - CacheFileEnvelope::new(&cache_a, 99, None).save(&path).expect("save a"); + CacheFileEnvelope::new(&cache_a, 99, None).save(&path, None).expect("save a"); let env = CacheFileEnvelope::load(&path).expect("load"); let loaded = TransportCache::from_value(&env.cache).expect("from_value"); @@ -618,10 +665,11 @@ mod tests { let path = dir.path().join("capture.json"); let cache_b = TransportCache::new(); - CacheFileEnvelope::new(&cache_b, 1, None).save(&path).expect("save b"); + CacheFileEnvelope::new(&cache_b, 1, None).save(&path, None).expect("save b"); let cache_a = TransportCache::new(); - let err = CacheFileEnvelope::new(&cache_a, 2, None).save(&path).expect_err("mismatch"); + let err = + CacheFileEnvelope::new(&cache_a, 2, None).save(&path, None).expect_err("mismatch"); assert!(err.to_string().contains("chain_id")); } @@ -643,7 +691,7 @@ mod tests { }])) .expect("seed"); CacheFileEnvelope::new(&cache, 7, None) - .save(&path) + .save(&path, None) .expect("corrupt disk with chain_id in path must degrade, not hard-fail"); let env = CacheFileEnvelope::load(&path).expect("ours written"); @@ -659,16 +707,140 @@ mod tests { let path = dir.path().join("chain_id_version_capture.json"); let cache_b = TransportCache::new(); - CacheFileEnvelope::new(&cache_b, 1, None).save(&path).expect("save b"); + CacheFileEnvelope::new(&cache_b, 1, None).save(&path, None).expect("save b"); let cache_a = TransportCache::new(); let err = CacheFileEnvelope::new(&cache_a, 2, None) - .save(&path) + .save(&path, None) .expect_err("chain_id mismatch must hard-fail"); let msg = err.to_string(); assert!(msg.contains("chain_id"), "msg={msg}"); } + /// Sequential capture refresh: loaded A, ours B, disk still A → B is written. + #[test] + fn test_envelope_persist_intentional_external_env_refresh() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + let loaded = ExternalEnvSnapshot { bucket_capacities: vec![(1, 10)] }; + let cache_a = TransportCache::new(); + CacheFileEnvelope::new(&cache_a, 7, Some(&loaded)).save(&path, None).expect("seed A"); + + let ours = ExternalEnvSnapshot { bucket_capacities: vec![(1, 99)] }; + let cache_b = TransportCache::new(); + CacheFileEnvelope::new(&cache_b, 7, Some(&ours)) + .save(&path, Some(&loaded)) + .expect("intentional A→B refresh"); + + let env = CacheFileEnvelope::load(&path).expect("load"); + let written = env.external_env.expect("external_env written"); + assert_eq!(written.bucket_capacities, vec![(1, 99)]); + } + + /// Store construction observes the on-disk snapshot so intentional A→B + /// refresh works through the public `set_external_env` + `persist` path + /// without callers plumbing the load-time baseline themselves. + #[test] + fn test_store_persist_intentional_external_env_refresh() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + let loaded = ExternalEnvSnapshot { bucket_capacities: vec![(1, 10)] }; + CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&loaded)) + .save(&path, None) + .expect("seed A"); + + let mut store = RpcCacheStore::new_envelope(TransportCache::new(), path.clone(), 7); + store.set_external_env(ExternalEnvSnapshot { bucket_capacities: vec![(1, 99)] }); + store.persist().expect("store-level intentional A→B refresh must succeed"); + + let env = CacheFileEnvelope::load(&path).expect("load"); + assert_eq!( + env.external_env.expect("external_env written").bucket_capacities, + vec![(1, 99)] + ); + } + + /// After store construction loads A, a concurrent writer changing the file + /// to C causes persist with ours B to hard-error (true concurrent conflict). + #[test] + fn test_store_persist_rejects_concurrent_external_env_conflict() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + let a = ExternalEnvSnapshot { bucket_capacities: vec![(1, 10)] }; + CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&a)) + .save(&path, None) + .expect("seed A"); + + // Observe A at construction, then a concurrent writer intentionally + // refreshes A→C (its own loaded baseline is A). + let mut store = RpcCacheStore::new_envelope(TransportCache::new(), path.clone(), 7); + let c = ExternalEnvSnapshot { bucket_capacities: vec![(1, 42)] }; + CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&c)) + .save(&path, Some(&a)) + .expect("concurrent C"); + + store.set_external_env(ExternalEnvSnapshot { bucket_capacities: vec![(1, 99)] }); + let err = store.persist().expect_err("true concurrent conflict via store"); + let msg = err.to_string(); + assert!(msg.contains("external_env"), "msg={msg}"); + assert!( + msg.contains("loaded") && msg.contains("ours") && msg.contains("on-disk"), + "msg={msg}" + ); + assert!(msg.contains("10") && msg.contains("99") && msg.contains("42"), "msg={msg}"); + } + + /// Concurrent conflict through save: loaded A, ours B, disk C → hard error. + #[test] + fn test_envelope_persist_rejects_concurrent_external_env_conflict() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + let c = ExternalEnvSnapshot { bucket_capacities: vec![(1, 42)] }; + CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&c)) + .save(&path, None) + .expect("seed C"); + + let loaded = ExternalEnvSnapshot { bucket_capacities: vec![(1, 10)] }; + let ours = ExternalEnvSnapshot { bucket_capacities: vec![(1, 99)] }; + let err = CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&ours)) + .save(&path, Some(&loaded)) + .expect_err("true concurrent conflict"); + let msg = err.to_string(); + assert!(msg.contains("external_env"), "msg={msg}"); + assert!( + msg.contains("loaded") && msg.contains("ours") && msg.contains("on-disk"), + "msg={msg}" + ); + assert!(msg.contains("10") && msg.contains("99") && msg.contains("42"), "msg={msg}"); + } + + /// Same effective capacities in different order do not conflict at save. + #[test] + fn test_envelope_persist_order_insensitive_external_env() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + let a = ExternalEnvSnapshot { bucket_capacities: vec![(1, 10), (2, 20)] }; + CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&a)) + .save(&path, None) + .expect("seed"); + + let b = ExternalEnvSnapshot { bucket_capacities: vec![(2, 20), (1, 10)] }; + // Pretend we loaded something else so equality is the only thing that + // would save us from a false concurrent-conflict report. + let foreign = ExternalEnvSnapshot { bucket_capacities: vec![(9, 9)] }; + CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&b)) + .save(&path, Some(&foreign)) + .expect("order-only difference must not conflict"); + + let env = CacheFileEnvelope::load(&path).expect("load"); + assert_eq!(env.external_env.expect("present").bucket_capacities, vec![(1, 10), (2, 20)]); + } + /// Typed re-read: corrupt content is Degradable even when path mentions `chain_id`. #[test] fn test_reread_envelope_classifies_corrupt_vs_identity() { diff --git a/bin/mega-evme/src/lib.rs b/bin/mega-evme/src/lib.rs index be4e21bc..8eb99601 100644 --- a/bin/mega-evme/src/lib.rs +++ b/bin/mega-evme/src/lib.rs @@ -28,6 +28,10 @@ pub use common::*; /// Install a thread panic hook that prints a custom backtrace and exits with a /// non-zero status. Lets failing tests and CLI runs surface a useful trace /// without relying on `RUST_BACKTRACE`. +/// +/// When the raw process argv contains `--json`, the hook also prints the +/// standard structured error object on stdout before exiting so a machine- +/// readable run never ends with empty stdout on panic. pub fn set_thread_panic_hook() { use std::{ backtrace::Backtrace, @@ -40,6 +44,53 @@ pub fn set_thread_panic_hook() { // installed yet when a panic fires during CLI startup. eprintln!("Custom backtrace: {}", Backtrace::capture()); orig_hook(panic_info); + if raw_argv_wants_json() { + // Keep the panic text on stderr (via `orig_hook`); the structured + // object is the machine-readable final stdout line. + let message = format!("panic: {panic_info}"); + print_json_error(ExitCode::ExecutionError, &message); + } exit(1); })); } + +/// Whether the raw process argv contains `--json`. +/// +/// Used by the panic hook when the parsed command is not available (and kept +/// public so unit tests can document the same decision as production). +pub fn raw_argv_wants_json() -> bool { + std::env::args_os().any(|arg| arg == "--json") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The panic-hook JSON decision is driven by raw argv, not the parsed CLI. + #[test] + fn test_raw_argv_wants_json_detects_flag() { + // Unit-test the predicate shape by scanning a synthetic argv slice + // (the production helper reads process args; this mirrors its logic). + fn wants_json(args: &[&str]) -> bool { + args.contains(&"--json") + } + assert!(!wants_json(&["mega-evme", "replay", "0xabc"])); + assert!(wants_json(&["mega-evme", "replay", "--json", "0xabc"])); + assert!(wants_json(&["mega-evme", "--json"])); + // Only the exact flag; a value containing the substring is not enough. + assert!(!wants_json(&["mega-evme", "--json-pretty"])); + } + + /// The structured panic object uses the standard error envelope shape. + #[test] + fn test_panic_json_error_object_shape() { + let code = ExitCode::ExecutionError; + assert_eq!(code.code(), 1); + assert_eq!(code.kind(), "execution-error"); + // Message prefix matches the hook's `panic: …` form; printing itself is + // covered by `print_json_error` and cannot be unit-tested without + // capturing stdout, so a deterministic binary panic trigger is not used. + let message = format!("panic: {}", "explicit test panic"); + assert!(message.starts_with("panic: ")); + } +} diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 6c8c5f77..1a38d471 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -139,7 +139,7 @@ pub(super) enum BatchMode { /// /// Execution outcomes (success, revert, halt) are normal results and never map /// to one of these kinds. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] enum BatchErrorKind { /// The transaction hash is unknown to the endpoint. NotFound, @@ -1015,13 +1015,12 @@ fn classify(err: &ReplayError) -> BatchErrorKind { /// The kind reported for a target swept up by an abort caused elsewhere. /// -/// The abort says nothing about this target, so a definitive answer about -/// another transaction (an unknown hash) becomes an unanswered question here. -fn swept_kind(err: &ReplayError) -> BatchErrorKind { - match classify(err) { - BatchErrorKind::NotFound | BatchErrorKind::Rpc => BatchErrorKind::Rpc, - kind => kind, - } +/// The abort says nothing about this target: whatever class caused the block to +/// stop (unknown hash, RPC failure, executor/setup error on another +/// transaction), a non-aborting swept target is unanswered (`rpc`). Only the +/// transaction that caused the abort keeps its own classified kind. +fn swept_kind(_err: &ReplayError) -> BatchErrorKind { + BatchErrorKind::Rpc } /// The transaction an aborting error is about, when it names one. @@ -1258,6 +1257,30 @@ mod tests { assert!(tally.into_error().is_none(), "fixture skips never fail the run"); } + /// Every non-aborting swept target is unanswered (`rpc`), even when the + /// abort itself is an execution-class failure of another transaction. + #[test] + fn test_swept_kind_always_rpc_regardless_of_abort_class() { + // Unknown hash: already unanswered for the cause, and for swept peers. + assert_eq!(swept_kind(&ReplayError::TransactionNotFound(B256::ZERO)), BatchErrorKind::Rpc); + // Transport/RPC failure. + assert_eq!(swept_kind(&ReplayError::RpcError("endpoint down".into())), BatchErrorKind::Rpc); + // Execution-class aborts (other, setup, internal) must not blame swept targets. + assert_eq!( + swept_kind(&ReplayError::Other("executor setup failed".into())), + BatchErrorKind::Rpc + ); + assert_eq!( + swept_kind(&ReplayError::InvalidInput("bad hardfork schedule".into())), + BatchErrorKind::Rpc + ); + // classify itself still distinguishes execution for the aborting target. + assert_eq!( + classify(&ReplayError::Other("executor setup failed".into())), + BatchErrorKind::Execution + ); + } + /// A run whose only finding is divergence fails as the mismatch it is. #[test] fn test_batch_tally_mismatch_only_reports_the_verification_error() { diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 8393f218..5d67b979 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -26,6 +26,12 @@ const BLOCK_TXS: [(&str, u64); 3] = [ ("0xb6a0b7a302c741f64b8e46861a3dcb2d5c1047f6f2cb89a35b5c2183c96296b7", 22), ]; +/// A mid-block type-0x2 call (not a deposit): zeroing its gas makes the block +/// executor reject the transaction as invalid and abort — unlike deposits, +/// which can still halt as `FailedDeposit` without aborting the block. +const EXEC_ABORT_TX: (&str, u64) = + ("0xa637d68cda9423d67826e008b1c90295193f30f19cd74a6f4acf54022d56cae2", 2); + /// Last transaction of the envelope's second block. const OTHER_BLOCK: u64 = 22_945_853; const OTHER_BLOCK_TX: &str = "0x18302160f2395069a44e1654d173fa9eed95ead8f922f12bfe07b6bdcc0a14f2"; @@ -102,6 +108,37 @@ fn envelope_without_transaction(name: &str, tx_hash: &str) -> std::path::PathBuf path } +/// Write a copy of the envelope whose `eth_getTransactionByHash` response for +/// `tx_hash` still returns the transaction object, but with `gas` set to `0x0` +/// so execution/setup fails (intrinsic gas / validation) rather than a missing +/// lookup. Models an executor abort mid-block. +fn envelope_with_zero_gas_transaction(name: &str, tx_hash: &str) -> std::path::PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let marker = format!("\"hash\":\"{tx_hash}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object for {tx_hash}"); + result["gas"] = serde_json::Value::String("0x0".into()); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "the envelope must hold exactly one response for {tx_hash}"); + + let path = + std::env::temp_dir().join(format!("mega_evme_batch_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored envelope"); + path +} + /// Run `replay` against `envelope_path` and return its stdout plus its exit code. fn replay_envelope_with_code( envelope_path: &std::path::Path, @@ -333,6 +370,59 @@ fn test_replay_block_sweeps_targets_behind_an_abort_as_unanswered() { assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("execution-error")); } +/// An executor/setup abort on a mid-block transaction is still an execution +/// failure for that transaction only: every target behind it is unanswered +/// (`rpc`), not blamed as execution. +/// +/// Doctors the envelope so a mid-block type-0x2 call has gas `0x0` — the lookup +/// succeeds, but the block executor rejects it as an invalid transaction +/// (intrinsic/call gas) and aborts the block — an execution-class error, not +/// `TransactionNotFound`. +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_block_sweeps_targets_behind_execution_abort_as_rpc() { + let (aborting, aborting_index) = EXEC_ABORT_TX; + let path = envelope_with_zero_gas_transaction("exec_abort_block", aborting); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--block", &BLOCK.to_string(), "--json"]); + let _ = std::fs::remove_file(&path); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + for (index, line) in lines.iter().enumerate() { + let index = index as u64; + if index < aborting_index { + assert!(line.get("error").is_none(), "targets before the abort replay: {line}"); + continue; + } + if index == aborting_index { + assert_eq!( + line["error"]["kind"].as_str(), + Some("execution"), + "the aborting transaction keeps its own execution kind: {line}" + ); + continue; + } + assert_eq!( + line["error"]["kind"].as_str(), + Some("rpc"), + "a target swept up behind an execution abort went unanswered: {line}" + ); + assert!( + line["error"]["message"].as_str().is_some_and(|m| { + m.contains(aborting) || m.contains("aborted") || m.contains("Block replay") + }), + "the message must name the abort cause: {line}" + ); + } + + // The aborting transaction is an execution-class failure, which outranks + // the unanswered ones. + assert_eq!(code, Some(1), "a definitive execution failure exits 1"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("execution-error")); +} + /// Targets swept up by an abort are reported in block transaction-index order, /// whatever order `--tx-file` listed them in. #[test] diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index e28e0d95..34472121 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -286,7 +286,7 @@ On subsequent runs the existing file is loaded, its entries are merged into the The updated set of entries is persisted back to the same file on clean exit. The file also embeds an external-environment snapshot — currently the set of `--bucket-capacity` values in effect — so the captured fixture is self-contained. -If `--bucket-capacity` is not passed on a subsequent run, the previous envelope's values are reused; passing `--bucket-capacity` overrides them. +If `--bucket-capacity` is not passed on a subsequent run, the previous envelope's values are reused; passing `--bucket-capacity` overrides them (an intentional A→B refresh of an existing capture is accepted at persist when no concurrent writer changed the on-disk snapshot; a true concurrent conflict hard-errors and names the load-time, caller, and on-disk values — see [state management](../configuration/state-management.md#rpc-cache)). The capture is written even when the replay itself failed — an execution or verification failure is exactly the case you want to debug offline. If the write fails, it is reported on stderr like any other failure, next to the run's own error; the run error keeps the exit code, since it is the root cause. diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index 1a04fe2b..91258449 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -235,9 +235,13 @@ A missing or corrupt on-disk file during the re-read degrades to writing this pr Capture envelopes (`--rpc.capture-file`) use the same lock + re-read-merge path, with additional hard-error checks before writing: - The on-disk envelope `version` and `chain_id` must match this process's capture. -- If both the on-disk envelope and this process carry a non-null `external_env` snapshot and those snapshots are not identical, persist hard-errors and names both values. - One-sided or identical snapshots still merge: this process's snapshot is kept when set, otherwise the on-disk snapshot is propagated. - This matches the offline [`cache merge`](../commands/cache.md) rule so concurrent captures cannot silently union RPC entries under a single wrong environment. +- `external_env` uses optimistic concurrency against the snapshot observed when this process opened the capture file: + - If the locked on-disk snapshot is absent, or still equals the load-time snapshot, the caller's intentional update wins (so a sequential refresh with `--bucket-capacity` on an existing capture is accepted). + - Persist hard-errors only when the on-disk snapshot changed since load **and** differs from this process's snapshot (true concurrent conflict). + The error names all three values: loaded, ours, and on-disk. + - Snapshots are canonicalized before comparison and write (deduplicate by bucket id with last-wins, then sort by id), so two workers with the same effective capacities in different CLI order do not conflict. + - One-sided snapshots still merge: this process's snapshot is kept when set, otherwise the on-disk snapshot is propagated. +- Offline [`cache merge`](../commands/cache.md) still rejects non-identical non-null `external_env` snapshots across inputs (no load-time baseline to compare against). A corrupt or unreadable on-disk envelope during re-read degrades to writing this process's entries only (warned), while identity/schema failures remain hard errors. From cd3326dd5aea7930dcc454b5c34b0cd6e47d5d53 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 02:31:18 +0800 Subject: [PATCH 12/64] fix(mega-evme): pass loaded external_env baseline into capture store Stop RpcCacheStore::new_envelope from re-reading the capture file after build_capture_provider already loaded it. A concurrent writer between the two reads could make C look like the load-time baseline so an A-derived persist silently overwrote C. Carry the first-load snapshot as a parameter. --- .../src/common/provider/cache_store.rs | 82 +++++++++++++------ bin/mega-evme/src/common/provider/mod.rs | 11 ++- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/bin/mega-evme/src/common/provider/cache_store.rs b/bin/mega-evme/src/common/provider/cache_store.rs index 0940ca22..837df319 100644 --- a/bin/mega-evme/src/common/provider/cache_store.rs +++ b/bin/mega-evme/src/common/provider/cache_store.rs @@ -99,18 +99,17 @@ impl RpcCacheStore { /// this module. The write snapshot starts empty; callers inject it later /// via [`Self::set_external_env`]. /// - /// The on-disk snapshot at `path` (if any) is captured here as the load-time - /// baseline for optimistic concurrency at persist — intentional A→B - /// refreshes are accepted when the locked re-read still matches this value. - pub(super) fn new_envelope(cache: TransportCache, path: PathBuf, chain_id: u64) -> Self { - // Observe the load-time external_env once, at store construction (the - // same moment capture mode opens the file). Re-reading at persist is - // compared against this baseline, not against a second open-time read. - let loaded_external_env = if path.exists() { - CacheFileEnvelope::load(&path).ok().and_then(|e| e.external_env) - } else { - None - }; + /// `loaded_external_env` is the load-time baseline already observed by the + /// caller when it opened the capture file (if any). Persist compares the + /// locked re-read against this value so intentional A→B refreshes are + /// accepted when the on-disk snapshot is still A. The baseline must come + /// from that first load — this constructor must not re-read the file. + pub(super) fn new_envelope( + cache: TransportCache, + path: PathBuf, + chain_id: u64, + loaded_external_env: Option, + ) -> Self { Self { inner: Some(RpcCacheStoreInner::FixtureCapture { cache, @@ -738,20 +737,21 @@ mod tests { assert_eq!(written.bucket_capacities, vec![(1, 99)]); } - /// Store construction observes the on-disk snapshot so intentional A→B - /// refresh works through the public `set_external_env` + `persist` path - /// without callers plumbing the load-time baseline themselves. + /// Store constructed with baseline A; disk still A; ours B → B wins + /// (intentional refresh through `set_external_env` + `persist`). #[test] fn test_store_persist_intentional_external_env_refresh() { let dir = tempfile::tempdir().expect("tempdir"); let path = dir.path().join("capture.json"); - let loaded = ExternalEnvSnapshot { bucket_capacities: vec![(1, 10)] }; - CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&loaded)) + let a = ExternalEnvSnapshot { bucket_capacities: vec![(1, 10)] }; + CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&a)) .save(&path, None) .expect("seed A"); - let mut store = RpcCacheStore::new_envelope(TransportCache::new(), path.clone(), 7); + // Baseline A is passed in (same object the caller loaded); no re-read. + let mut store = + RpcCacheStore::new_envelope(TransportCache::new(), path.clone(), 7, Some(a)); store.set_external_env(ExternalEnvSnapshot { bucket_capacities: vec![(1, 99)] }); store.persist().expect("store-level intentional A→B refresh must succeed"); @@ -762,8 +762,12 @@ mod tests { ); } - /// After store construction loads A, a concurrent writer changing the file - /// to C causes persist with ours B to hard-error (true concurrent conflict). + /// Store constructed with baseline A; on-disk mutated to C before persist; + /// ours B derived from A → hard conflict naming loaded/ours/on-disk. + /// + /// Regression for the double-read defect: the store must use the caller's + /// loaded baseline, not re-read the file at construction (which would + /// observe C and treat ours-from-A as an intentional refresh of C). #[test] fn test_store_persist_rejects_concurrent_external_env_conflict() { let dir = tempfile::tempdir().expect("tempdir"); @@ -774,9 +778,11 @@ mod tests { .save(&path, None) .expect("seed A"); - // Observe A at construction, then a concurrent writer intentionally - // refreshes A→C (its own loaded baseline is A). - let mut store = RpcCacheStore::new_envelope(TransportCache::new(), path.clone(), 7); + // Baseline A from the first load — not re-read from disk at construction. + let mut store = + RpcCacheStore::new_envelope(TransportCache::new(), path.clone(), 7, Some(a.clone())); + + // Concurrent writer lands C (≠A, ≠B) after our load, before our persist. let c = ExternalEnvSnapshot { bucket_capacities: vec![(1, 42)] }; CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&c)) .save(&path, Some(&a)) @@ -793,6 +799,36 @@ mod tests { assert!(msg.contains("10") && msg.contains("99") && msg.contains("42"), "msg={msg}"); } + /// If the store re-read the file at construction, a concurrent C would be + /// mistaken for the baseline and an A-derived B would silently overwrite C. + /// Passing baseline A while disk is already C must still conflict. + #[test] + fn test_store_persist_uses_passed_baseline_not_disk_at_construction() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + // Disk already holds C when the store is constructed (simulates a + // writer that landed between the caller's first load and store build). + let a = ExternalEnvSnapshot { bucket_capacities: vec![(1, 10)] }; + let c = ExternalEnvSnapshot { bucket_capacities: vec![(1, 42)] }; + CacheFileEnvelope::new(&TransportCache::new(), 7, Some(&c)) + .save(&path, None) + .expect("disk is C"); + + let mut store = RpcCacheStore::new_envelope(TransportCache::new(), path, 7, Some(a)); + store.set_external_env(ExternalEnvSnapshot { bucket_capacities: vec![(1, 99)] }); + let err = store + .persist() + .expect_err("passed baseline A must not be replaced by on-disk C at construction"); + let msg = err.to_string(); + assert!(msg.contains("external_env"), "msg={msg}"); + assert!( + msg.contains("loaded") && msg.contains("ours") && msg.contains("on-disk"), + "msg={msg}" + ); + assert!(msg.contains("10") && msg.contains("99") && msg.contains("42"), "msg={msg}"); + } + /// Concurrent conflict through save: loaded A, ours B, disk C → hard error. #[test] fn test_envelope_persist_rejects_concurrent_external_env_conflict() { diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 34260e1a..6405af50 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -362,11 +362,20 @@ impl RpcArgs { "Built RPC provider (capture to cache file)", ); + // Same snapshot whose entries were merged above — carry it into the + // store as the OCC load-time baseline. Do not re-read the file: a + // concurrent writer between this load and store construction would + // make the later write treat C as baseline and silently overwrite it. let prev_external_env = existing_envelope.and_then(|e| e.external_env); Ok(BuildProviderOutput { provider: DynProvider::new(provider), - cache_store: RpcCacheStore::new_envelope(transport_cache, path.clone(), chain_id), + cache_store: RpcCacheStore::new_envelope( + transport_cache, + path.clone(), + chain_id, + prev_external_env.clone(), + ), chain_id, external_env: prev_external_env, }) From d8c47fd128287fc24b9f0eeee12a2f046feb05c8 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 11:36:08 +0800 Subject: [PATCH 13/64] feat(mega-evme): add --rpc.request-timeout per-HTTP request timeout Default 30s bounds hung endpoints so they surface as retryable transport errors (exit 3 after retries) instead of hanging the process forever. --- bin/mega-evme/src/common/provider/mod.rs | 51 ++++++- bin/mega-evme/tests/common/mod.rs | 15 ++ bin/mega-evme/tests/provider.rs | 143 ++++++++++++++++++ docs/mega-evme/commands/run.md | 9 +- docs/mega-evme/commands/tx.md | 1 + .../configuration/state-management.md | 11 +- 6 files changed, 220 insertions(+), 10 deletions(-) diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 6405af50..e1116a17 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -18,6 +18,7 @@ mod transport; use std::{ fs, path::{Path, PathBuf}, + time::Duration, }; use alloy_provider::{ @@ -127,7 +128,8 @@ pub struct RpcArgs { /// Maximum number of times the transport layer will retry a failing RPC request. /// Retries trigger on HTTP 429 / 503, JSON-RPC rate-limit error responses, and /// transport failures surfaced as `TransportErrorKind::Custom` (connection refused, - /// DNS failure, TLS handshake, etc.). Set to 0 to disable retries entirely. + /// DNS failure, TLS handshake, request timeout, etc.). Set to 0 to disable retries + /// entirely. #[arg(long = "rpc.max-retries", default_value_t = 5)] pub max_retries: u32, @@ -142,6 +144,12 @@ pub struct RpcArgs { /// public-endpoint budgets. #[arg(long = "rpc.cu-per-sec", visible_alias = "rpc.rate-limit", default_value_t = 660)] pub compute_units_per_sec: u64, + + /// Total per-HTTP-request timeout in seconds (connect + response). + /// `0` disables the timeout (previous behavior: a hung endpoint can block forever). + /// A non-zero timeout surfaces a hung endpoint as a retryable transport error. + #[arg(long = "rpc.request-timeout", default_value_t = 30)] + pub request_timeout: u64, } impl RpcArgs { @@ -325,7 +333,7 @@ impl RpcArgs { // envelope first, a stale eth_chainId entry would short-circuit the // cross-chain validation. let transport_cache = TransportCache::new(); - let http = alloy_transport_http::Http::new(url.clone()); + let http = self.build_http_transport(url.clone()); let caching = CachingTransport::new(http, transport_cache.clone()); let client = self.build_client(caching, &url); let provider = ProviderBuilder::new() @@ -381,14 +389,49 @@ impl RpcArgs { }) } + /// Build the HTTP transport, applying [`Self::request_timeout`] when non-zero. + /// + /// All networked `Http` construction (standard provider, capture provider, and + /// the throwaway chain-id probe) must go through this helper so the timeout is + /// applied uniformly. Offline replay (`--rpc.replay-file`) never constructs + /// an HTTP transport. + /// + /// When `request_timeout == 0`, uses the default reqwest client (no total + /// request timeout). When non-zero, builds a client with both `.timeout` and + /// `.connect_timeout` set to the same duration so a hung endpoint surfaces as + /// a `TransportErrorKind::Custom` error (retryable under the policy below) + /// instead of hanging the process forever. + fn build_http_transport( + &self, + url: reqwest::Url, + ) -> alloy_transport_http::Http { + if self.request_timeout == 0 { + return alloy_transport_http::Http::new(url); + } + let timeout = Duration::from_secs(self.request_timeout); + // `Client::builder` with only timeout settings cannot fail under normal + // conditions (failure requires a broken TLS backend). + let client = reqwest::Client::builder() + .timeout(timeout) + .connect_timeout(timeout) + .build() + .expect("reqwest Client with timeout settings must build"); + alloy_transport_http::Http::with_client(client, url) + } + /// Build an `RpcClient` over HTTP, wired with the configured retry layer. fn build_retry_client(&self, url: reqwest::Url) -> RpcClient { - self.build_client(alloy_transport_http::Http::new(url.clone()), &url) + self.build_client(self.build_http_transport(url.clone()), &url) } /// Build an `RpcClient` over an arbitrary transport, wired with the configured /// retry layer (or bare when `max_retries == 0`). `url` is used only to detect /// whether the endpoint is local. + /// + /// Retry coverage: `RateLimitRetryPolicy` handles HTTP 429/503 and JSON-RPC + /// rate-limit bodies. The `TransportErrorKind::Custom` predicate additionally + /// retries transport failures, including reqwest request timeouts (mapped by + /// `alloy-transport-http` via `TransportErrorKind::custom`). fn build_client( &self, transport: T, @@ -397,6 +440,8 @@ impl RpcArgs { let is_local = url.host_str().is_some_and(|h| h == "localhost" || h == "127.0.0.1" || h == "::1"); if self.max_retries > 0 { + // Reqwest timeouts, connection refused, DNS failure, TLS handshake, + // etc. all arrive as `TransportErrorKind::Custom` and are retryable. let policy = RateLimitRetryPolicy::default().or(|err: &TransportError| { matches!(err, RpcError::Transport(TransportErrorKind::Custom(_))) }); diff --git a/bin/mega-evme/tests/common/mod.rs b/bin/mega-evme/tests/common/mod.rs index df3fca73..63385990 100644 --- a/bin/mega-evme/tests/common/mod.rs +++ b/bin/mega-evme/tests/common/mod.rs @@ -105,6 +105,21 @@ impl MockRpcServer { pub(crate) async fn received_request_count(&self) -> usize { self.server.received_requests().await.expect("received_requests").len() } + + /// Accept every POST and delay the response far beyond any test timeout. + /// + /// Models a black-hole endpoint that accepts the TCP connection (and the + /// HTTP request) but never answers in time — the failure mode that + /// `--rpc.request-timeout` is meant to bound. The delay is 5 minutes so a + /// client with a 1s timeout fires first while the mock still records the hit. + pub(crate) async fn respond_black_hole(&self) { + use std::time::Duration; + + Mock::given(matchers::method("POST")) + .respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(300))) + .mount(&self.server) + .await; + } } /// Parse every top-level JSON value a run printed on stdout. diff --git a/bin/mega-evme/tests/provider.rs b/bin/mega-evme/tests/provider.rs index 836e6346..3b3c8b85 100644 --- a/bin/mega-evme/tests/provider.rs +++ b/bin/mega-evme/tests/provider.rs @@ -41,6 +41,8 @@ fn test_rpc_args_parses_all_new_flags() { "250", "--rpc.rate-limit", "1234", + "--rpc.request-timeout", + "45", ]); assert_eq!(args.rpc_url, Some("https://example.test/rpc".to_string())); assert_eq!(args.cache_max_entries, 256); @@ -50,6 +52,20 @@ fn test_rpc_args_parses_all_new_flags() { assert_eq!(args.max_retries, 7); assert_eq!(args.backoff_ms, 250); assert_eq!(args.compute_units_per_sec, 1234); + assert_eq!(args.request_timeout, 45); +} + +/// Explicit `--rpc.request-timeout` values parse, including the disable sentinel `0`. +#[test] +fn test_rpc_args_parses_request_timeout() { + let defaulted = RpcArgs::parse_from(["mega-evme"]); + assert_eq!(defaulted.request_timeout, 30, "default request timeout is 30s"); + + let explicit = RpcArgs::parse_from(["mega-evme", "--rpc.request-timeout", "12"]); + assert_eq!(explicit.request_timeout, 12); + + let disabled = RpcArgs::parse_from(["mega-evme", "--rpc.request-timeout", "0"]); + assert_eq!(disabled.request_timeout, 0, "0 disables the per-request timeout"); } /// The removed `--rpc.cache-size` flag must fail to parse (pin the deletion). @@ -121,6 +137,7 @@ fn test_rpc_args_default_values() { assert_eq!(args.max_retries, 5); assert_eq!(args.backoff_ms, 1_000); assert_eq!(args.compute_units_per_sec, 660); + assert_eq!(args.request_timeout, 30); } // ─── build_provider shape variants ─────────────────────────────────────────── @@ -575,6 +592,132 @@ async fn test_retry_layer_retries_on_unreachable_endpoint() { ); } +// ─── Request-timeout behavior ──────────────────────────────────────────────── + +/// A hung endpoint (accepts TCP, never responds) with +/// `--rpc.request-timeout 1` and `--rpc.max-retries 0` fails quickly at +/// chain-id resolution as `EvmeError::RpcError` (exit 3), instead of hanging. +#[tokio::test(flavor = "multi_thread")] +async fn test_request_timeout_fails_black_hole_within_bound() { + let server = MockRpcServer::start().await; + server.respond_black_hole().await; + + let args = RpcArgs::parse_from([ + "mega-evme", + "--rpc", + &server.uri(), + "--rpc.no-cache-file", + "--rpc.request-timeout", + "1", + "--rpc.max-retries", + "0", + "--rpc.backoff-ms", + "1", + ]); + + let started = std::time::Instant::now(); + let err = args.build_provider().await.expect_err("black-hole must time out"); + let elapsed = started.elapsed(); + + assert!( + elapsed < std::time::Duration::from_secs(15), + "must fail within a few seconds (timeout=1s, retries=0), took {elapsed:?}", + ); + // Floor: a real 1s timeout should not return in sub-millisecond time. + assert!( + elapsed >= std::time::Duration::from_millis(500), + "must wait for the request timeout, took {elapsed:?}", + ); + + match &err { + EvmeError::RpcError(msg) => { + assert!( + msg.contains("Failed to fetch chain ID"), + "timeout must surface via chain-id resolution, got: {msg}", + ); + } + other => panic!("expected EvmeError::RpcError, got {other:?}"), + } + assert_eq!( + ExitCode::from_evme_error(&err).code(), + 3, + "exhausted timeout must exit 3 (rpc-failure)", + ); + + // Spawned-binary assertion: the same flags on `replay` must exit 3 within + // the wall-time bound (chain-id fetch is the first networked call). + let started = std::time::Instant::now(); + let output = std::process::Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args([ + "replay", + "--rpc", + &server.uri(), + "--rpc.no-cache-file", + "--rpc.request-timeout", + "1", + "--rpc.max-retries", + "0", + "--rpc.backoff-ms", + "1", + "0x0000000000000000000000000000000000000000000000000000000000000001", + ]) + .output() + .expect("spawn mega-evme"); + let elapsed = started.elapsed(); + assert!( + elapsed < std::time::Duration::from_secs(15), + "spawned binary must fail within a few seconds, took {elapsed:?}", + ); + assert_eq!( + output.status.code(), + Some(3), + "spawned binary must exit 3 on timeout.\nstderr: {}", + String::from_utf8_lossy(&output.stderr), + ); +} + +/// A black-hole endpoint with `--rpc.max-retries 2` makes three attempts +/// (1 initial + 2 retries) before giving up — proving reqwest timeouts are +/// classified as retryable `TransportErrorKind::Custom` errors. +#[tokio::test(flavor = "multi_thread")] +async fn test_request_timeout_is_retried_up_to_max_retries() { + let server = MockRpcServer::start().await; + server.respond_black_hole().await; + + let args = RpcArgs::parse_from([ + "mega-evme", + "--rpc", + &server.uri(), + "--rpc.no-cache-file", + "--rpc.request-timeout", + "1", + "--rpc.max-retries", + "2", + "--rpc.backoff-ms", + "1", + ]); + + let started = std::time::Instant::now(); + let err = args.build_provider().await.expect_err("black-hole must exhaust retries"); + let elapsed = started.elapsed(); + + // 3 attempts × ~1s timeout + small backoffs; keep a generous upper bound. + assert!( + elapsed < std::time::Duration::from_secs(30), + "3×1s timeouts must finish well under 30s, took {elapsed:?}", + ); + assert_eq!( + server.received_request_count().await, + 3, + "max-retries=2 → 1 initial + 2 retries against the black-hole", + ); + assert_eq!( + ExitCode::from_evme_error(&err).code(), + 3, + "exhausted timeout retries must exit 3 (rpc-failure)", + ); +} + // ─── Contract regression guards (fixture-file modes) ─────────────────────── /// The `env = "RPC_URL"` attribute was removed from `--rpc`, so parsing diff --git a/docs/mega-evme/commands/run.md b/docs/mega-evme/commands/run.md index 91172155..8a873a91 100644 --- a/docs/mega-evme/commands/run.md +++ b/docs/mega-evme/commands/run.md @@ -54,7 +54,7 @@ Each group is documented on its own page. | Chain and spec | `--spec`, `--chain-id` | [Chain and Spec](../configuration/chain-and-spec.md) | | Block environment | `--block.number`, `--block.coinbase`, `--block.timestamp`, `--block.gaslimit`, `--block.basefee`, `--block.difficulty`, `--block.prevrandao`, `--block.blobexcessgas` | [Block Environment](../configuration/block-environment.md) | | SALT buckets | `--bucket-capacity` | [SALT Buckets](../configuration/salt-buckets.md) | -| RPC cache / retry | `--rpc.cache-max-entries`, `--rpc.cache-dir`, `--rpc.no-cache-file`, `--rpc.clear-cache`, `--rpc.max-retries`, `--rpc.backoff-ms`, `--rpc.cu-per-sec` | [RPC Cache and Retry](../configuration/state-management.md#rpc-cache-and-retry) | +| RPC cache / retry | `--rpc.cache-max-entries`, `--rpc.cache-dir`, `--rpc.no-cache-file`, `--rpc.clear-cache`, `--rpc.max-retries`, `--rpc.backoff-ms`, `--rpc.cu-per-sec`, `--rpc.request-timeout` | [RPC Cache and Retry](../configuration/state-management.md#rpc-cache-and-retry) | | Tracing | `--trace`, `--tracer`, `--trace.output`, and tracer-specific flags | [Tracing Overview](../tracing/overview.md) | | Output | `--json` | See [JSON output](#json-output) below | @@ -317,7 +317,7 @@ RPC Options: Delete the current chain's cache file before loading it --rpc.max-retries - Max transport retries on 429/503, rate-limit, and transport failures; 0 disables [default: 5] + Max transport retries on 429/503, rate-limit, transport failures, and request timeouts; 0 disables [default: 5] --rpc.backoff-ms Fixed sleep (ms) between retries; no exponential backoff [default: 1000] @@ -328,6 +328,11 @@ RPC Options: [default: 660] [alias: --rpc.rate-limit] + --rpc.request-timeout + Total per-HTTP-request timeout in seconds (connect + response). `0` disables the timeout (previous behavior: a hung endpoint can block forever). A non-zero timeout surfaces a hung endpoint as a retryable transport error + + [default: 30] + Chain Options: --spec Name of spec to use, possible values: `MiniRex`, `Equivalence`, `Rex`, `Rex1`, `Rex2`, `Rex3`, `Rex4`, `Rex5`, `Rex6`, `Rex7` diff --git a/docs/mega-evme/commands/tx.md b/docs/mega-evme/commands/tx.md index 08685d9f..b7f06942 100644 --- a/docs/mega-evme/commands/tx.md +++ b/docs/mega-evme/commands/tx.md @@ -213,6 +213,7 @@ RPC Options: --rpc.cu-per-sec Compute-unit budget (CU/s) for the retry layer's rate-limit accounting (NOT requests/s) [default: 660] [alias: --rpc.rate-limit] + --rpc.request-timeout Total per-HTTP-request timeout (connect + response); 0 disables [default: 30] Chain Options: --spec Spec [default: Rex7] diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index 91258449..71dac18c 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -259,11 +259,12 @@ Provider-cache merge also rejects inputs (and `--output`) whose `rpc-cache-{chai ### Retry Flags -| Flag | Type | Default | Description | -| ------------------------- | ----- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--rpc.max-retries ` | `u32` | `5` | Maximum retry attempts for failing RPC requests. Retries on HTTP 429/503, rate-limit errors, and transport failures. `0` to disable. | -| `--rpc.backoff-ms ` | `u64` | `1000` | Fixed sleep duration in milliseconds between retry attempts (no exponential backoff). | -| `--rpc.cu-per-sec ` | `u64` | `660` | Compute-unit budget (CU/s) for the retry layer's rate-limit accounting — not requests per second. Alias: `--rpc.rate-limit`. Values below 100 with retries enabled emit a warning. | +| Flag | Type | Default | Description | +| --------------------------- | ----- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--rpc.max-retries ` | `u32` | `5` | Maximum retry attempts for failing RPC requests. Retries on HTTP 429/503, rate-limit errors, and transport failures. `0` to disable. | +| `--rpc.backoff-ms ` | `u64` | `1000` | Fixed sleep duration in milliseconds between retry attempts (no exponential backoff). | +| `--rpc.cu-per-sec ` | `u64` | `660` | Compute-unit budget (CU/s) for the retry layer's rate-limit accounting — not requests per second. Alias: `--rpc.rate-limit`. Values below 100 with retries enabled emit a warning. | +| `--rpc.request-timeout ` | `u64` | `30` | Total per-HTTP-request timeout in seconds (connect + response). `0` disables. A hung endpoint then surfaces as a retryable transport error instead of hanging the process. | ### Examples From 303bb65e95fc84f1c73698e10ccc045e522b9137 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 15:09:08 +0800 Subject: [PATCH 14/64] docs(mega-evme): fix intra-doc anchor fragments flagged by lychee --- docs/mega-evme/commands/replay.md | 10 +++++----- docs/mega-evme/overview.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 34472121..176e7f29 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -69,14 +69,14 @@ Replay every transaction of block `N`, given in decimal or `0x`-prefixed hex. Batch mode reports one summary per transaction and has no meaningful semantics for single-file fixture dumps, tracing, state dumps, or what-if knobs, so the following are rejected up front with an explanatory error rather than silently ignored: -- `--dump-fixture` — use [`--dump-fixture-dir`](#dump-fixture-dir-dir) for batch sedimentation +- `--dump-fixture` — use [`--dump-fixture-dir`](#--dump-fixture-dir-dir) for batch sedimentation - Transaction overrides (`--override.gas-limit`, `--override.value`, `--override.input`, `--override.input-file`) - `--override.spec` — each block's spec is auto-detected from its timestamp - All trace options (`--trace`, `--trace.output`, `--tracer`, `--trace.*`) - All state dump options (`--dump`, `--dump.output`) Single-transaction replay keeps accepting all of them. -Batch mode additionally accepts [`--dump-fixture-dir`](#dump-fixture-dir-dir) for per-target fixture sedimentation. +Batch mode additionally accepts [`--dump-fixture-dir`](#--dump-fixture-dir-dir) for per-target fixture sedimentation. ### Output @@ -118,7 +118,7 @@ Without `--json`, each transaction is printed with a header naming its hash, blo A final one-line summary (transactions replayed, transactions failed, elapsed time) is logged at `INFO` level, so pass `-vvv` to see it. With [`--verify-receipt`](#receipt-verification), each result line additionally carries a `verification` object. -With [`--dump-fixture-dir`](#dump-fixture-dir-dir), each result line additionally carries a `fixture` object (`path`, `skipped`, or `error`). +With [`--dump-fixture-dir`](#--dump-fixture-dir-dir), each result line additionally carries a `fixture` object (`path`, `skipped`, or `error`). ### Exit Status @@ -286,7 +286,7 @@ On subsequent runs the existing file is loaded, its entries are merged into the The updated set of entries is persisted back to the same file on clean exit. The file also embeds an external-environment snapshot — currently the set of `--bucket-capacity` values in effect — so the captured fixture is self-contained. -If `--bucket-capacity` is not passed on a subsequent run, the previous envelope's values are reused; passing `--bucket-capacity` overrides them (an intentional A→B refresh of an existing capture is accepted at persist when no concurrent writer changed the on-disk snapshot; a true concurrent conflict hard-errors and names the load-time, caller, and on-disk values — see [state management](../configuration/state-management.md#rpc-cache)). +If `--bucket-capacity` is not passed on a subsequent run, the previous envelope's values are reused; passing `--bucket-capacity` overrides them (an intentional A→B refresh of an existing capture is accepted at persist when no concurrent writer changed the on-disk snapshot; a true concurrent conflict hard-errors and names the load-time, caller, and on-disk values — see [state management](../configuration/state-management.md#rpc-cache-and-retry)). The capture is written even when the replay itself failed — an execution or verification failure is exactly the case you want to debug offline. If the write fails, it is reported on stderr like any other failure, next to the run's own error; the run error keeps the exit code, since it is the root cause. @@ -370,7 +370,7 @@ state-test ./fixtures/0xabc123.json Batch-only. Dump a self-validating fixture for every successfully replayed target into `/.json`. -The fixture content and format match the single-transaction [`--dump-fixture`](#dump-fixture-file) path (same EEST schema, same sorted `megaEnv`, same self-validation via `state-test`). +The fixture content and format match the single-transaction [`--dump-fixture`](#--dump-fixture-file) path (same EEST schema, same sorted `megaEnv`, same self-validation via `state-test`). The directory is created if it does not exist. Existing files are refused unless `--overwrite` is also set — a refused overwrite is a failed dump for that target, not a skip. diff --git a/docs/mega-evme/overview.md b/docs/mega-evme/overview.md index 3557f436..fc909249 100644 --- a/docs/mega-evme/overview.md +++ b/docs/mega-evme/overview.md @@ -86,7 +86,7 @@ A batch run (`--tx-file` / `--block`) reports every target on its own line and t A target that never replayed was also never verified, which is why an infrastructure failure outranks a mismatch. On failure the run also prints a report: one `error: ` line per failure on stderr, plus — with `--json` — a structured object as the last line of stdout, so a machine-readable run never ends with empty output. -A run reports more than one line when a secondary failure must not go unnoticed but does not own the exit code — an unwritable [`--rpc.capture-file`](commands/replay.md#rpccapture-file-path) behind an earlier replay failure, for instance. +A run reports more than one line when a secondary failure must not go unnoticed but does not own the exit code — an unwritable [`--rpc.capture-file`](commands/replay.md#--rpccapture-file-path) behind an earlier replay failure, for instance. The structured object always carries the failure the exit code came from. ```json From a7b3de64d77e6970f5ed5b2ca25329936f45f6f8 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 16:34:04 +0800 Subject: [PATCH 15/64] fix(mega-evme): address five PR #366 review findings on batch replay Defer fixture writes until block finish succeeds; classify construction failures as fixture errors rather than skips; validate parent-block hash linkage; reject receipts with a null blockHash as infrastructure errors; stamp receipt inner logs with block/tx identity and block-global indices. --- bin/mega-evme/src/common/outcome.rs | 72 ++++- bin/mega-evme/src/replay/batch.rs | 213 +++++++++++--- bin/mega-evme/src/replay/cmd.rs | 11 + bin/mega-evme/src/replay/verify.rs | 27 +- bin/mega-evme/src/tx/cmd.rs | 1 + .../test_replay_offline_bucket_cap.json | 54 ++-- bin/mega-evme/tests/replay_batch.rs | 65 +++++ bin/mega-evme/tests/replay_verify.rs | 262 ++++++++++++++++++ docs/mega-evme/commands/replay.md | 6 +- 9 files changed, 640 insertions(+), 71 deletions(-) diff --git a/bin/mega-evme/src/common/outcome.rs b/bin/mega-evme/src/common/outcome.rs index d6eccc36..f7c5661b 100644 --- a/bin/mega-evme/src/common/outcome.rs +++ b/bin/mega-evme/src/common/outcome.rs @@ -5,7 +5,7 @@ use std::{path::Path, time::Duration}; use super::{EvmeError, StateDumpArgs, TraceArgs}; use alloy_consensus::{Eip658Value, Receipt}; -use alloy_primitives::{hex, Address, BlockHash, Bytes, TxHash, B256}; +use alloy_primitives::{hex, Address, BlockHash, Bytes, TxHash}; use alloy_rpc_types_eth::TransactionReceipt; use alloy_sol_types::{Panic, Revert, SolError}; use clap::Parser; @@ -68,6 +68,11 @@ impl EvmeOutcome { } /// Convert an [`OpReceiptEnvelope`] to an OP transaction receipt. +/// +/// `first_log_index` is the block-global log index of this receipt's first log +/// (the cumulative log count of all preceding receipts in the block). Each +/// inner log is stamped with the same block/tx identity as the outer receipt so +/// the JSON is self-consistent. #[allow(clippy::too_many_arguments)] pub fn op_receipt_to_tx_receipt( receipt: &OpReceiptEnvelope, @@ -81,17 +86,18 @@ pub fn op_receipt_to_tx_receipt( transaction_hash: Option, // only used for replay command where tx hash is known block_hash: Option, // only used for replay command where block hash is known transaction_index: u64, + first_log_index: u64, ) -> OpTxReceipt { - // Map logs to include block/tx metadata - let mut log_index = 0; + // Map logs to include block/tx metadata matching the outer receipt. + let mut log_index = first_log_index; let inner = receipt.clone().map_logs(|log| { let log = alloy_rpc_types_eth::Log { inner: log, - block_hash: None, + block_hash, block_number: Some(block_number), block_timestamp: Some(block_timestamp), - transaction_hash: Some(B256::ZERO), - transaction_index: Some(0), + transaction_hash, + transaction_index: Some(transaction_index), log_index: Some(log_index), removed: false, }; @@ -357,7 +363,7 @@ impl ExecutionSummary { #[cfg(test)] mod tests { use super::*; - use alloy_primitives::Bytes; + use alloy_primitives::{address, b256, Bytes, Log as PrimitiveLog, LogData}; use alloy_sol_types::SolError; #[test] @@ -383,4 +389,56 @@ mod tests { let raw = Bytes::from(vec![0xde, 0xad]); assert_eq!(decode_revert_reason(&raw), "0xdead"); } + + /// Inner logs carry the same block/tx identity as the outer receipt, and + /// `log_index` is the block-global index starting at `first_log_index`. + #[test] + fn test_op_receipt_to_tx_receipt_stamps_inner_log_metadata() { + let addr = address!("0x00000000000000000000000000000000000000aa"); + let topic = b256!("0x000000000000000000000000000000000000000000000000000000000000000a"); + let log = PrimitiveLog { + address: addr, + data: LogData::new(vec![topic], Bytes::from(vec![0xde, 0xad])).expect("topics"), + }; + let receipt = OpReceiptEnvelope::Legacy( + alloy_consensus::Receipt { + status: Eip658Value::Eip658(true), + cumulative_gas_used: 21_000, + logs: vec![log.clone(), log], + } + .with_bloom(), + ); + let tx_hash = b256!("0x1111111111111111111111111111111111111111111111111111111111111111"); + let block_hash = + b256!("0x2222222222222222222222222222222222222222222222222222222222222222"); + let from = address!("0x00000000000000000000000000000000000000bb"); + + let tx_receipt = op_receipt_to_tx_receipt( + &receipt, + 42, + 1_700_000_000, + from, + Some(addr), + None, + 1, + 21_000, + Some(tx_hash), + Some(block_hash), + 3, + 7, // two preceding receipts already emitted 7 logs in this block + ); + + assert_eq!(tx_receipt.transaction_hash, tx_hash); + assert_eq!(tx_receipt.block_hash, Some(block_hash)); + assert_eq!(tx_receipt.transaction_index, Some(3)); + let logs = tx_receipt.inner.logs(); + assert_eq!(logs.len(), 2); + for (i, log) in logs.iter().enumerate() { + assert_eq!(log.block_hash, Some(block_hash), "log {i} block_hash"); + assert_eq!(log.transaction_hash, Some(tx_hash), "log {i} transaction_hash"); + assert_eq!(log.transaction_index, Some(3), "log {i} transaction_index"); + assert_eq!(log.log_index, Some(7 + i as u64), "log {i} block-global log_index"); + assert_eq!(log.block_number, Some(42)); + } + } } diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 1a38d471..c3eb1caf 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -306,6 +306,21 @@ struct BlockJob { targets: Vec, } +/// Fixture work for one target, held until `finish()` succeeds. +/// +/// Skips and construction failures are decided against the pre-commit state and +/// carried as a final report. A successfully built draft is written only after +/// the transaction commits and the block finishes — a commit-time rejection or +/// finish failure must not leave a fixture file on disk (and must not clobber a +/// pre-existing file under `--overwrite`). +enum DeferredFixture { + /// Already decided (skip, construction error, or refused overwrite). + Report(FixtureReport), + /// Draft built against pre-commit state; write after `finish()` succeeds. + /// Boxed so the enum is not dominated by the draft's size on the skip path. + Ready { draft: Box, path: PathBuf }, +} + /// A target that executed, awaiting the receipt harvested by `finish()`. struct PendingTarget { tx_hash: B256, @@ -319,8 +334,8 @@ struct PendingTarget { from: Address, to: Option
, effective_gas_price: u128, - /// Fixture dump outcome, present iff `--dump-fixture-dir` was given. - fixture: Option, + /// Fixture dump work, present iff `--dump-fixture-dir` was given. + fixture: Option, } /// NDJSON line for a target that produced an execution result. @@ -500,7 +515,9 @@ where /// /// When `--dump-fixture-dir` is set, each target's fixture draft is built from /// the pre-commit state (same moment as the single-transaction dump), gated per -/// target for fidelity and BLOCKHASH, and written before the transaction commits. +/// target for fidelity and BLOCKHASH, and written only after the block +/// `finish()` succeeds — so a commit-time rejection or finish failure cannot +/// leave a fixture file on disk. async fn replay_block

( provider: &P, chain_id: u64, @@ -536,6 +553,21 @@ where } }; + // Parent/block linkage guard: across a reorg or a load-balanced endpoint + // serving divergent views, `eth_getBlockByNumber(N-1)` can return a block + // that is not the parent of the block being replayed. Forking from that + // state would silently execute against the wrong pre-state. + let parent_hash = parent_block.hash(); + let expected_parent = block.header.parent_hash(); + if parent_hash != expected_parent { + let message = format!( + "parent block hash {parent_hash} != block parent_hash {expected_parent}: the parent \ + block describes a different chain than the block being replayed (reorg in progress, \ + or a load-balanced endpoint serving divergent views); retry once the chain settles" + ); + return fail_all(&targets, BatchErrorKind::Rpc, &message); + } + // Fetch the on-chain receipts before the block runs. Needed for // `--verify-receipt` (mismatch vs unverified) and for `--dump-fixture-dir` // (fidelity gate). A receipt that cannot be fetched, or that describes a @@ -659,10 +691,11 @@ where // Fixture draft must be built before commit: the pre-state closure // is the database after preceding txs, with the target's result // state still uncommitted — same moment as the single-tx dump. + // The draft is only written after `finish()` succeeds (see harvest). let fixture = if is_target { if let (Some(dir), Some(mega_env)) = (dump_dir, mega_env.as_ref()) { let accessed_block_hashes = block_executor.get_accessed_block_hashes(); - Some(dump_target_fixture( + Some(prepare_target_fixture( block_executor.evm().db_ref(), DumpFixtureArgs { accessed_block_hash_count: accessed_block_hashes.len(), @@ -732,6 +765,10 @@ where // verification verdict are still what the run was asked for. The // failed dump fails the run through the tally, not by replacing the // result with an error entry. + // + // Finalize+write runs only here, after finish succeeded: a + // commit-time rejection never reaches pending, and a finish failure + // drops ready drafts unwritten (see the Err arm). for target in pending { let Some(envelope) = receipts.get(offset + target.commit_index) else { entries.push(failure( @@ -743,6 +780,12 @@ where }; let contract_address = (target.to.is_none() && envelope.is_success()) .then(|| target.from.create(target.pre_execution_nonce)); + // Block-global log index: cumulative log count of all committed + // receipts that precede this target in the block. + let first_log_index: u64 = receipts[offset..offset + target.commit_index] + .iter() + .map(|r| r.logs().len() as u64) + .sum(); let receipt = op_receipt_to_tx_receipt( envelope, number, @@ -755,6 +798,7 @@ where Some(target.tx_hash), Some(block_hash), target.tx_index, + first_log_index, ); let verification = if verify_receipt { match onchain_receipts.get(&target.tx_hash) { @@ -776,6 +820,7 @@ where } else { None }; + let fixture = target.fixture.map(materialize_deferred_fixture); entries.push(BatchEntry::Executed(Box::new(ExecutedTx { tx_hash: target.tx_hash, block_number: number, @@ -785,12 +830,12 @@ where exec_time: target.exec_time, receipt, verification, - fixture: target.fixture, + fixture, }))); } } - // The block itself failed to finish, so no target of it has a receipt: - // that failure outranks whatever each target's fixture dump reported. + // The block itself failed to finish, so no target of it has a receipt + // and no deferred fixture is written or replaced. Err(e) => { let error = ReplayError::BlockExecutionError(e); let kind = classify(&error); @@ -844,7 +889,7 @@ where entries } -/// Inputs for [`dump_target_fixture`], grouped so the dump path stays a single +/// Inputs for [`prepare_target_fixture`], grouped so the dump path stays a single /// call site without a long positional argument list. struct DumpFixtureArgs<'a> { accessed_block_hash_count: usize, @@ -860,16 +905,17 @@ struct DumpFixtureArgs<'a> { overwrite: bool, } -/// Attempt to build and write a fixture for one successfully executed target. +/// Prepare a fixture for one successfully executed target against pre-commit state. /// /// Expected skips (missing receipt, fidelity mismatch, BLOCKHASH, unsupported -/// transaction shapes) report a skip reason and never fail the batch run. -/// Finalize/write errors report a fixture error, which fails the run as an -/// execution-class failure of this target without discarding its result. +/// transaction shapes) become a final [`FixtureReport`] and never fail the run. +/// Database and other construction failures become a fixture error (execution-class). +/// A successfully built draft is carried as [`DeferredFixture::Ready`] and only +/// written by [`materialize_deferred_fixture`] after `finish()` succeeds. /// /// `db` must reflect the pre-target-commit state (preceding txs committed, target /// not yet), matching the single-transaction dump. -fn dump_target_fixture(db: &DB, args: DumpFixtureArgs<'_>) -> FixtureReport +fn prepare_target_fixture(db: &DB, args: DumpFixtureArgs<'_>) -> DeferredFixture where DB: DatabaseRef, DB::Error: core::fmt::Display, @@ -894,25 +940,29 @@ where let facts = match onchain { Some(Ok(facts)) => facts, Some(Err(message)) => { - return FixtureReport::skipped(format!("fidelity-gate-unavailable: {message}")); + return DeferredFixture::Report(FixtureReport::skipped(format!( + "fidelity-gate-unavailable: {message}" + ))); } None => { - return FixtureReport::skipped( + return DeferredFixture::Report(FixtureReport::skipped( "fidelity-gate-unavailable: no on-chain receipt was fetched for this transaction", - ); + )); } }; if accessed_block_hash_count > 0 { - return FixtureReport::skipped(format!( + return DeferredFixture::Report(FixtureReport::skipped(format!( "transaction reads block hashes (BLOCKHASH): {accessed_block_hash_count} block \ hash(es) were accessed and the fixture cannot faithfully reproduce them" - )); + ))); } let anchor = fixture::anchor_from_receipt_facts(facts); if let Err(reason) = fixture::check_fidelity(exec_result, &anchor, chain_id) { - return FixtureReport::skipped(format!("fidelity gate failed: {reason}")); + return DeferredFixture::Report(FixtureReport::skipped(format!( + "fidelity gate failed: {reason}" + ))); } let draft = match fixture::build_draft( @@ -925,31 +975,71 @@ where fixture::FixtureInputs { mega_env, result: exec_result, anchor }, ) { Ok(draft) => draft, - // Unsupported shapes (deposit, EIP-7702, unknown spec) are expected in - // whole-block sweeps: skip rather than fail the run. - Err(e) => { - return FixtureReport::skipped(e.to_string()); - } + Err(e) => return DeferredFixture::Report(fixture_report_from_build_err(e)), }; let tx_hash = target_tx.inner.inner.tx_hash(); let path = dir.join(format!("{tx_hash:#x}.json")); + // Refuse overwrite without the flag before carrying a ready draft, so the + // harvest path never has to re-check and a finish failure cannot be confused + // with an overwrite refusal. if path.exists() && !overwrite { - return FixtureReport::error(format!( + return DeferredFixture::Report(FixtureReport::error(format!( "fixture already exists at {} (pass --overwrite to replace)", path.display() - )); + ))); } - match fixture::finalize_and_write(draft, &path) { - Ok(()) => { - info!(path = %path.display(), tx_hash = %tx_hash, "Wrote self-validating fixture"); - FixtureReport::written(&path) + DeferredFixture::Ready { draft: Box::new(draft), path } +} + +/// Finalize a deferred fixture after the block `finish()` succeeded. +/// +/// Ready drafts are self-validated and written here. Pre-decided reports pass +/// through unchanged. On finish failure the caller drops the deferred value +/// without calling this, so no file is written or replaced. +fn materialize_deferred_fixture(deferred: DeferredFixture) -> FixtureReport { + match deferred { + DeferredFixture::Report(report) => report, + DeferredFixture::Ready { draft, path } => { + match fixture::finalize_and_write(*draft, &path) { + Ok(()) => { + info!(path = %path.display(), "Wrote self-validating fixture"); + FixtureReport::written(&path) + } + Err(e) => FixtureReport::error(format!("fixture write failed: {e}")), + } } - Err(e) => FixtureReport::error(format!("fixture write failed: {e}")), } } +/// Classify a [`fixture::build_draft`] error as a skip (unsupported shape) or a +/// fixture construction error (database / other failures). +/// +/// Unsupported shapes are expected in whole-block sweeps and must not fail the +/// run. Endpoint/DB failures during construction mean the requested artifact +/// could not be produced and fail the run as an execution-class fixture error. +fn fixture_report_from_build_err(err: ReplayError) -> FixtureReport { + let message = err.to_string(); + if is_unsupported_fixture_shape(&message) { + FixtureReport::skipped(message) + } else { + FixtureReport::error(format!("fixture construction failed: {message}")) + } +} + +/// Whether a `build_draft` error is an expected unsupported shape (skip) rather +/// than a construction failure. +fn is_unsupported_fixture_shape(message: &str) -> bool { + message.contains("does not support deposit") + || message.contains("does not support EIP-7702") + || message.contains("has no fixture mapping") + || message.contains("reports no gas price") + // Fidelity is normally gated before `build_draft`; keep the classification + // if a fidelity check inside the builder ever surfaces here. + || message.contains("does not reproduce on-chain execution") +} + /// Fetch the on-chain receipt of every target of a block. /// /// Each target maps either to the consensus facts its receipt reports, or to the @@ -1348,4 +1438,65 @@ mod tests { assert!(parse_block_number("-1").is_err()); assert!(parse_block_number("12.5").is_err()); } + + /// Unsupported shapes remain skips; database/construction failures become + /// fixture errors so the run exits non-zero. + #[test] + fn test_fixture_build_err_classifies_skips_vs_construction_errors() { + let deposit = fixture_report_from_build_err(ReplayError::Other( + "--dump-fixture does not support deposit transactions".into(), + )); + assert!(deposit.skipped.is_some(), "deposit is a skip: {deposit:?}"); + assert!(deposit.error.is_none()); + + let eip7702 = fixture_report_from_build_err(ReplayError::Other( + "--dump-fixture does not support EIP-7702 (set-code) transactions: the \ + fixture builder does not serialize the authorization list" + .into(), + )); + assert!(eip7702.skipped.is_some(), "EIP-7702 is a skip: {eip7702:?}"); + + let unknown_spec = fixture_report_from_build_err(ReplayError::Other( + "--dump-fixture: spec Rex99 has no fixture mapping".into(), + )); + assert!(unknown_spec.skipped.is_some(), "unknown spec is a skip: {unknown_spec:?}"); + + let pre_state = fixture_report_from_build_err(ReplayError::Other( + "pre-state read for 0x00000000000000000000000000000000000000aa: database unavailable" + .into(), + )); + assert!( + pre_state.error.as_ref().is_some_and(|m| m.contains("construction failed")), + "DB pre-state failure is a fixture error: {pre_state:?}" + ); + assert!(pre_state.skipped.is_none()); + + let code_fetch = fixture_report_from_build_err(ReplayError::Other( + "code fetch for 0x1111: endpoint timeout".into(), + )); + assert!( + code_fetch.error.as_ref().is_some_and(|m| m.contains("construction failed")), + "code fetch failure is a fixture error: {code_fetch:?}" + ); + assert!(code_fetch.skipped.is_none()); + } + + /// A pre-decided fixture report is never rewritten by materialization, so a + /// finish failure that drops a Ready draft (without calling materialize) + /// cannot leave a file and a Report never touches the filesystem. + #[test] + fn test_materialize_deferred_fixture_passes_reports_through() { + let skipped = materialize_deferred_fixture(DeferredFixture::Report( + FixtureReport::skipped("fidelity-gate-unavailable: no receipt"), + )); + assert_eq!(skipped.skipped.as_deref(), Some("fidelity-gate-unavailable: no receipt")); + assert!(skipped.path.is_none()); + assert!(skipped.error.is_none()); + + let err = materialize_deferred_fixture(DeferredFixture::Report(FixtureReport::error( + "fixture construction failed: code fetch failed", + ))); + assert!(err.error.as_ref().is_some_and(|m| m.contains("construction failed"))); + assert!(err.path.is_none()); + } } diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 0746cf79..689a8f19 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -852,6 +852,16 @@ impl Cmd { let receipt_envelope = block_result.receipts.last().unwrap().clone(); trace!(?receipt_envelope, "Receipt envelope obtained"); + // Block-global log index: cumulative log count of every preceding + // receipt in this block (same data `finish()` harvested). + let first_log_index: u64 = block_result + .receipts + .iter() + .rev() + .skip(1) + .map(|envelope| envelope.logs().len() as u64) + .sum(); + let from = ctx.target_tx.inner.inner.signer(); let to = ctx.target_tx.inner.inner.to(); let contract_address = (to.is_none() && receipt_envelope.is_success()) @@ -868,6 +878,7 @@ impl Cmd { Some(ctx.target_tx.inner.inner.tx_hash()), Some(ctx.block.hash()), ctx.preceding_tx_hashes.len() as u64, + first_log_index, ); let verification = onchain_receipt.as_ref().map(|onchain| { diff --git a/bin/mega-evme/src/replay/verify.rs b/bin/mega-evme/src/replay/verify.rs index 4c920219..36e65156 100644 --- a/bin/mega-evme/src/replay/verify.rs +++ b/bin/mega-evme/src/replay/verify.rs @@ -323,13 +323,20 @@ pub(super) fn check_inclusion( replayed_block_hash: B256, ) -> std::result::Result<(), String> { match receipt_block_hash { - Some(hash) if hash != replayed_block_hash => Err(format!( + Some(hash) if hash == replayed_block_hash => Ok(()), + Some(hash) => Err(format!( "receipt block hash {hash} != replayed block hash {replayed_block_hash}: the receipt \ describes a different inclusion than the replayed block (reorg in progress, or a \ load-balanced endpoint serving divergent views); the transaction is unverified, \ retry once the chain settles" )), - _ => Ok(()), + // A receipt with no inclusion hash cannot be anchored to the replayed + // block, so it is the same class of failure as a mismatched hash. + None => Err(format!( + "receipt has no block hash: cannot anchor the receipt to the replayed block \ + {replayed_block_hash} (reorg in progress, or a load-balanced endpoint serving \ + divergent views); the transaction is unverified, retry once the chain settles" + )), } } @@ -593,8 +600,6 @@ mod tests { let hash = b256!("0x1111111111111111111111111111111111111111111111111111111111111111"); assert!(check_inclusion(Some(hash), hash).is_ok()); - // A receipt without a block hash cannot contradict the replayed block. - assert!(check_inclusion(None, hash).is_ok()); } #[test] @@ -610,4 +615,18 @@ mod tests { "message must explain the reorg and that the target is unverified: {message}" ); } + + #[test] + fn test_check_inclusion_rejects_a_missing_block_hash() { + let replayed = b256!("0x2222222222222222222222222222222222222222222222222222222222222222"); + let message = check_inclusion(None, replayed) + .expect_err("a receipt without a block hash must be rejected"); + + assert!( + message.contains("no block hash") && + message.contains("unverified") && + message.contains(&format!("{replayed}")), + "message must explain the missing anchor and name the replayed block: {message}" + ); + } } diff --git a/bin/mega-evme/src/tx/cmd.rs b/bin/mega-evme/src/tx/cmd.rs index 732adbb7..d9d59494 100644 --- a/bin/mega-evme/src/tx/cmd.rs +++ b/bin/mega-evme/src/tx/cmd.rs @@ -168,6 +168,7 @@ impl Cmd { None, None, 0, + 0, ); if self.output_args.json { diff --git a/bin/mega-evme/tests/fixtures/test_replay_offline_bucket_cap.json b/bin/mega-evme/tests/fixtures/test_replay_offline_bucket_cap.json index ea805833..1ae2a337 100644 --- a/bin/mega-evme/tests/fixtures/test_replay_offline_bucket_cap.json +++ b/bin/mega-evme/tests/fixtures/test_replay_offline_bucket_cap.json @@ -21,11 +21,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x0000000000000000000000000000000000000000000000000000000005f6080c555344540000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x0", "removed": false }, @@ -35,11 +35,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x0000000000000000000000000000000000000000000000000000000007e7c208585250000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x1", "removed": false }, @@ -49,11 +49,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x00000000000000000000000000000000000000000000000000000000008acf28444f47450000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x2", "removed": false }, @@ -63,11 +63,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x000000000000000000000000000000000000000000000000000006716f482ed2425443000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x3", "removed": false }, @@ -77,11 +77,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x00000000000000000000000000000000000000000000000000000000016c0b50414441000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x4", "removed": false }, @@ -91,11 +91,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x0000000000000000000000000000000000000000000000000000000005f592dd555344430000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x5", "removed": false }, @@ -105,11 +105,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x0000000000000000000000000000000000000000000000000000000deaecfcd1424e42000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x6", "removed": false }, @@ -119,11 +119,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x00000000000000000000000000000000000000000000000000000032f3a28db8455448000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x7", "removed": false }, @@ -133,11 +133,11 @@ "0xffcbf7f11b241556bfaa9228d6d51124009fd1f67009431663b49b6c87da0dd3" ], "data": "0x00000000000000000000000000000000000000000000000000000001e8730400534f4c000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000064f54ad9fbea900000000000000000000000000000000000000000000000000064f54ad9f28400000000000000000000000000000000000000000000000000000000000000000", - "blockHash": null, + "blockHash": "0x3f77d9845128bfbaa01176be2caf5e2db891e27e44c4503ba0fbd52e0ffb3ac6", "blockNumber": "0xf9b3e2", "blockTimestamp": "0x69dcc0da", - "transactionHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "transactionIndex": "0x0", + "transactionHash": "0x346c9ecd95ea9502e62534b68d8797f592d5f20bf7ed2d7b88f6ca7970e46919", + "transactionIndex": "0x4", "logIndex": "0x8", "removed": false } diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 5d67b979..bc9568b0 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -488,6 +488,71 @@ fn test_replay_batch_rejects_single_transaction_flags() { } } +/// A parent block whose hash does not match the child block's `parentHash` is an +/// infrastructure failure for every target of that block (reorg / divergent views). +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_block_rejects_mismatched_parent_hash() { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let wrong_parent = "0x1111111111111111111111111111111111111111111111111111111111111111"; + let mut expected_parent = None; + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + let Ok(mut response) = serde_json::from_str::(value) else { + continue; + }; + let Some(result) = response.get_mut("result") else { + continue; + }; + if !result.is_object() { + continue; + } + // Doctor the parent block (number == BLOCK - 1), not the target block. + let number = result.get("number").and_then(|n| { + n.as_str().and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + }); + if number != Some(BLOCK - 1) { + continue; + } + let original = result.get("hash").and_then(|h| h.as_str()).map(str::to_string); + assert!(original.is_some(), "parent block must report a hash"); + expected_parent = original; + result["hash"] = serde_json::Value::String(wrong_parent.into()); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "the envelope must hold exactly one parent-block body for {BLOCK}"); + let expected_parent = expected_parent.expect("parent hash"); + + let path = std::env::temp_dir() + .join(format!("mega_evme_batch_parent_mismatch_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored envelope"); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--block", &BLOCK.to_string(), "--json"]); + let _ = std::fs::remove_file(&path); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + for line in &lines { + assert_eq!( + line["error"]["kind"].as_str(), + Some("rpc"), + "a parent/block linkage failure is an infrastructure error: {line}" + ); + let message = line["error"]["message"].as_str().unwrap_or(""); + assert!( + message.contains(wrong_parent) && message.contains(&expected_parent), + "the message must name both hashes (got parent {expected_parent}, wrong {wrong_parent}): {line}" + ); + } + assert_eq!(code, Some(3), "an infrastructure failure exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); +} + /// Sweeping a block with `--dump-fixture-dir` against an envelope that carries /// no receipts skips every target on the fidelity gate and still exits 0. /// diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index d19f0731..633329d3 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -415,6 +415,57 @@ fn test_batch_verify_receipt_reorg_is_an_rpc_error_entry() { ); } +/// A receipt with a null `blockHash` cannot be anchored to the replayed block: +/// infrastructure failure, never a match/mismatch verdict. +#[test] +fn test_verify_receipt_null_block_hash_is_an_infrastructure_error() { + let path = doctored_cache("null_block_hash", |receipt| { + receipt["blockHash"] = serde_json::Value::Null; + }); + + let run = replay(&path, &["--verify-receipt", "--json", TX]); + let _ = std::fs::remove_file(&path); + + assert_eq!(run.code(), 3, "an unverifiable target exits 3.\nstderr: {}", run.stderr); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("rpc-failure")); + assert!( + run.stderr.contains("no block hash") || run.stderr.contains("block hash"), + "expected a missing-inclusion-hash hint, got stderr:\n{}", + run.stderr + ); + assert!( + !run.stderr.contains("verification mismatch") && !run.stdout.contains("MISMATCH"), + "an unanchorable receipt must not be reported as a mismatch:\n{}\n{}", + run.stdout, + run.stderr, + ); +} + +/// Batch mode reports a null `blockHash` as an `rpc` error entry. +#[test] +fn test_batch_verify_receipt_null_block_hash_is_an_rpc_error_entry() { + let path = doctored_cache("batch_null_block_hash", |receipt| { + receipt["blockHash"] = serde_json::Value::Null; + }); + let list = tx_file("batch_null_block_hash"); + + let run = replay(&path, &["--tx-file", list.to_str().unwrap(), "--verify-receipt", "--json"]); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + + assert_eq!(run.code(), 3, "an unverified target exits 3.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); + assert!( + lines[0]["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("no block hash")), + "expected a missing-inclusion-hash hint: {}", + lines[0] + ); + assert!(lines[0].get("verification").is_none(), "an unverified target carries no verdict"); +} + /// Batch `--dump-fixture-dir` writes a self-validating fixture for a target /// whose capture includes the on-chain receipt, and exits 0. #[test] @@ -564,3 +615,214 @@ fn test_batch_dump_fixture_dir_refuses_overwrite_without_flag() { assert!(lines[0]["fixture"]["path"].is_string(), "overwrite must report a written path"); let _ = std::fs::remove_dir_all(&dir); } + +/// A fixture-construction failure (database / pre-state reads) is reported as +/// `fixture.error` and fails the run — never as a silent skip that exits 0. +/// +/// Classification of construction vs unsupported-shape errors is unit-tested in +/// `batch::tests::test_fixture_build_err_classifies_skips_vs_construction_errors`. +/// This end-to-end check forces a construction-time failure by removing every +/// non-empty bytecode response from the capture so a pre-state `code` fetch +/// fails after the transaction has already executed against cached account +/// state (execution may still succeed from in-memory code; draft-time re-fetch +/// does not). +#[test] +fn test_batch_fixture_construction_failure_is_fixture_error_not_skip() { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + .expect("parse offline cache"); + let mut nulled = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + let Ok(mut response) = serde_json::from_str::(value) else { + continue; + }; + let Some(result) = response.get("result").cloned() else { + continue; + }; + // Non-empty bytecode hex responses (eth_getCode / code-by-hash). + let Some(hex) = result.as_str() else { + continue; + }; + if hex == "0x" || hex == "0x0" || hex.len() <= 4 { + continue; + } + // Only rewrite pure hex bytecode payloads, not block/tx objects. + if !hex.starts_with("0x") || hex.len() < 100 { + continue; + } + // Turn the code response into a JSON-RPC error so a draft-time re-fetch fails. + response.as_object_mut().expect("response object").remove("result"); + response["error"] = serde_json::json!({ + "code": -32000, + "message": "code unavailable at draft time", + }); + entry["value"] = serde_json::Value::String(response.to_string()); + nulled += 1; + } + // If the capture has no long bytecode entries the construction path cannot + // be forced this way; fail loudly so the fixture is refreshed. + assert!(nulled > 0, "offline cache should contain bytecode responses to doctor"); + + let cache_path = temp_path("fixture_construction"); + std::fs::write(&cache_path, envelope.to_string()).expect("write doctored cache"); + let list = tx_file("fixture_construction"); + let dir = + std::env::temp_dir().join(format!("mega_evme_fixture_construction_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let run = replay( + &cache_path, + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + let _ = std::fs::remove_file(&cache_path); + let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_dir_all(&dir); + + // Either the run fails during execution (code needed to execute) or during + // fixture construction. The construction path is the one we want: a result + // line with fixture.error. If execution fails first, the target is an error + // entry — also non-zero, never a silent skip. + assert!(!run.success, "missing code must not exit 0.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "one line per requested transaction"); + if let Some(error) = lines[0].get("error") { + // Execution failed before draft: still non-zero, not a skip. + assert!(error.get("kind").is_some(), "execution failure entry: {}", lines[0]); + } else { + assert!( + lines[0]["fixture"]["error"].as_str().is_some_and(|m| { + m.contains("construction failed") || + m.contains("code") || + m.contains("pre-state") || + m.contains("fixture") + }), + "expected fixture.error for a construction failure: {}", + lines[0] + ); + assert!( + lines[0]["fixture"].get("skipped").is_none(), + "construction failure must not be reported as a skip: {}", + lines[0] + ); + assert_eq!(run.code(), 1, "fixture construction failure exits 1"); + } +} + +/// When the block aborts before a dump target can finish, no fixture file is +/// written and `--overwrite` does not clobber a pre-existing file. +/// +/// Doctors the capture so a preceding transaction is unknown to the endpoint: +/// the dump target never reaches commit/finish, so deferred finalize+write never +/// runs. Happy-path writes are covered by +/// [`test_batch_dump_fixture_dir_writes_validatable_file`]. +#[test] +fn test_batch_dump_does_not_write_or_clobber_when_block_aborts_before_finish() { + // The captured transaction is at index 1; deny the index-0 hash so the block + // aborts before the dump target runs. + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + .expect("parse offline cache"); + let preceding = "0x14ca11aad284153b6e0460428dadf5d0dcab76a5b129ee1f6c9e07f535620a24"; + let marker = format!("\"hash\":\"{preceding}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + response["result"] = serde_json::Value::Null; + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "the offline cache must hold the preceding transaction"); + let cache_path = temp_path("dump_abort_before"); + std::fs::write(&cache_path, envelope.to_string()).expect("write doctored cache"); + + let list = tx_file("dump_abort_before"); + let dir = + std::env::temp_dir().join(format!("mega_evme_batch_dump_abort_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create dump dir"); + let fixture_path = dir.join(format!("{TX}.json")); + let sentinel = br#"{"pre-existing":"must-not-be-clobbered"}"#; + std::fs::write(&fixture_path, sentinel).expect("seed pre-existing fixture"); + + let run = replay( + &cache_path, + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--overwrite", + "--json", + ], + ); + let _ = std::fs::remove_file(&cache_path); + let _ = std::fs::remove_file(&list); + + assert!(!run.success, "an aborted block must exit non-zero.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "one line for the requested target"); + assert!( + lines[0].get("error").is_some(), + "the target must be reported as an error entry, not a dump: {}", + lines[0] + ); + assert!( + lines[0].get("fixture").is_none(), + "an unfinished target carries no fixture report: {}", + lines[0] + ); + let kept = std::fs::read(&fixture_path).expect("pre-existing fixture must still exist"); + assert_eq!(kept, sentinel, "--overwrite must not clobber when the dump never finalizes"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Replay receipts stamp each inner log with the outer receipt's block/tx +/// identity. The offline capture emits no logs, so this asserts the empty-log +/// path stays consistent; non-empty log metadata is covered by the unit test on +/// `op_receipt_to_tx_receipt`. +#[test] +fn test_replay_receipt_inner_log_metadata_matches_outer_receipt() { + let run = replay(&cache(), &["--json", TX]); + assert!(run.success, "replay must exit 0.\nstderr: {}", run.stderr); + let summary = run.json(); + let receipt = &summary["receipt"]; + let block_hash = receipt["blockHash"].as_str().expect("receipt blockHash"); + let tx_hash = receipt["transactionHash"].as_str().expect("receipt transactionHash"); + let tx_index = receipt["transactionIndex"].as_str().expect("receipt transactionIndex"); + assert_eq!(tx_hash, TX); + let logs = receipt["logs"].as_array().expect("receipt logs array"); + for (i, log) in logs.iter().enumerate() { + assert_eq!(log["blockHash"].as_str(), Some(block_hash), "log {i} blockHash"); + assert_eq!(log["transactionHash"].as_str(), Some(tx_hash), "log {i} transactionHash"); + assert_eq!(log["transactionIndex"].as_str(), Some(tx_index), "log {i} transactionIndex"); + assert!(log["logIndex"].is_string(), "log {i} must carry logIndex: {log}"); + } + + // Batch path stamps the same fields. + let list = tx_file("batch_log_meta"); + let batch = replay(&cache(), &["--tx-file", list.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&list); + assert!(batch.success, "batch replay must exit 0.\nstderr: {}", batch.stderr); + let line = &batch.ndjson()[0]; + let batch_receipt = &line["receipt"]; + assert_eq!(batch_receipt["blockHash"].as_str(), Some(block_hash)); + assert_eq!(batch_receipt["transactionHash"].as_str(), Some(tx_hash)); + let batch_logs = batch_receipt["logs"].as_array().expect("batch receipt logs"); + for (i, log) in batch_logs.iter().enumerate() { + assert_eq!(log["blockHash"].as_str(), Some(block_hash), "batch log {i} blockHash"); + assert_eq!(log["transactionHash"].as_str(), Some(tx_hash), "batch log {i} transactionHash"); + assert!(log["logIndex"].is_string(), "batch log {i} must carry logIndex: {log}"); + } +} diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 176e7f29..8f238910 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -181,7 +181,7 @@ A transaction is only reported as mismatched when both receipts were compared an Anything that prevents the comparison from running is an infrastructure failure — the transaction is _unverified_, which is a different finding from a divergence: - The endpoint fails the receipt call, or has pruned the receipt below its retention height (common on non-archive endpoints): reported as an `rpc` failure. -- The receipt describes a different inclusion than the replayed block (its `blockHash` differs — a reorg in progress, or a load-balanced endpoint serving divergent views): reported as an `rpc` failure, because comparing against it would compare the replay to the wrong on-chain execution. +- The receipt describes a different inclusion than the replayed block (its `blockHash` differs from the replayed block, or is null — a reorg in progress, or a load-balanced endpoint serving divergent views): reported as an `rpc` failure, because comparing against it would compare the replay to the wrong on-chain execution, and a receipt with no inclusion hash cannot be anchored at all. - The target is a pending transaction, which has no receipt yet: rejected up front in single-transaction mode, and reported as a `pending` error entry in batch mode. In batch mode each of these becomes an error entry for that transaction, exactly like any other infrastructure failure. @@ -374,7 +374,8 @@ The fixture content and format match the single-transaction [`--dump-fixture`](# The directory is created if it does not exist. Existing files are refused unless `--overwrite` is also set — a refused overwrite is a failed dump for that target, not a skip. -Per-target gating mirrors the single-transaction rules, but records a skip instead of failing the run: +Per-target gating mirrors the single-transaction rules, but records a skip instead of failing the run. +The fixture draft is built against the pre-commit state (same moment as the single-transaction dump) and only written after the block finishes successfully — a commit-time rejection or finish failure never creates or replaces a fixture file. | Gate | Outcome | | -------------------------------------------------------------------------------- | ------------------------------------------------------------- | @@ -382,6 +383,7 @@ Per-target gating mirrors the single-transaction rules, but records a skip inste | Fidelity mismatch (gas / status / logs root) | `fixture.skipped` with `fidelity gate failed: …` | | Target reads `BLOCKHASH` | `fixture.skipped` (fixtures carry no historical block hashes) | | Unsupported shape (deposit, EIP-7702, unknown spec mapping) | `fixture.skipped` | +| Fixture construction failure (database / pre-state reads) | `fixture.error` with the reason; execution-class failure | | Finalize / write / self-validation failure, refused overwrite | `fixture.error` with the reason; execution-class failure | | Pending / unresolvable target | already an error entry; no fixture report | From 6df81ed93f6eb5d9e707b50449da445743e78f99 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 17:15:36 +0800 Subject: [PATCH 16/64] fix(mega-evme): close four pre-push review findings on fixture dump Enforce no-replace at materialization with NamedTempFile + persist_noclobber, route single-tx dump-fixture through verify::check_inclusion, stamp missing receipt tx hashes consistently on outer and inner logs, and rework the three integration tests so each path is actually reached (construction after execution, deferred draft discarded on mid-block abort, multi-log metadata). --- bin/mega-evme/src/common/outcome.rs | 57 +++++- bin/mega-evme/src/replay/batch.rs | 61 +++++- bin/mega-evme/src/replay/cmd.rs | 24 +-- bin/mega-evme/src/replay/fixture.rs | 75 +++++-- bin/mega-evme/tests/replay_batch.rs | 32 +++ bin/mega-evme/tests/replay_dump.rs | 26 ++- bin/mega-evme/tests/replay_verify.rs | 284 ++++++++++++++++----------- 7 files changed, 399 insertions(+), 160 deletions(-) diff --git a/bin/mega-evme/src/common/outcome.rs b/bin/mega-evme/src/common/outcome.rs index f7c5661b..2fd1ab09 100644 --- a/bin/mega-evme/src/common/outcome.rs +++ b/bin/mega-evme/src/common/outcome.rs @@ -88,6 +88,12 @@ pub fn op_receipt_to_tx_receipt( transaction_index: u64, first_log_index: u64, ) -> OpTxReceipt { + // Resolve the effective tx hash once so the outer receipt and every inner + // log agree: a missing hash becomes `B256::ZERO` on both sides (the outer + // field is non-optional on `TransactionReceipt`). + let effective_tx_hash = transaction_hash.unwrap_or_default(); + let stamped_tx_hash = Some(effective_tx_hash); + // Map logs to include block/tx metadata matching the outer receipt. let mut log_index = first_log_index; let inner = receipt.clone().map_logs(|log| { @@ -96,7 +102,7 @@ pub fn op_receipt_to_tx_receipt( block_hash, block_number: Some(block_number), block_timestamp: Some(block_timestamp), - transaction_hash, + transaction_hash: stamped_tx_hash, transaction_index: Some(transaction_index), log_index: Some(log_index), removed: false, @@ -107,7 +113,7 @@ pub fn op_receipt_to_tx_receipt( TransactionReceipt { inner, - transaction_hash: transaction_hash.unwrap_or_default(), + transaction_hash: effective_tx_hash, transaction_index: Some(transaction_index), block_hash, block_number: Some(block_number), @@ -363,7 +369,7 @@ impl ExecutionSummary { #[cfg(test)] mod tests { use super::*; - use alloy_primitives::{address, b256, Bytes, Log as PrimitiveLog, LogData}; + use alloy_primitives::{address, b256, Bytes, Log as PrimitiveLog, LogData, B256}; use alloy_sol_types::SolError; #[test] @@ -441,4 +447,49 @@ mod tests { assert_eq!(log.block_number, Some(42)); } } + + /// A missing transaction hash becomes `B256::ZERO` on the outer receipt and + /// the same value on every inner log (not `None` on logs / zero only outside). + #[test] + fn test_op_receipt_to_tx_receipt_missing_tx_hash_stamps_outer_and_inner_consistently() { + let addr = address!("0x00000000000000000000000000000000000000aa"); + let topic = b256!("0x000000000000000000000000000000000000000000000000000000000000000a"); + let log = PrimitiveLog { + address: addr, + data: LogData::new(vec![topic], Bytes::from(vec![0xbe, 0xef])).expect("topics"), + }; + let receipt = OpReceiptEnvelope::Legacy( + alloy_consensus::Receipt { + status: Eip658Value::Eip658(true), + cumulative_gas_used: 21_000, + logs: vec![log], + } + .with_bloom(), + ); + let from = address!("0x00000000000000000000000000000000000000bb"); + + let tx_receipt = op_receipt_to_tx_receipt( + &receipt, + 1, + 1, + from, + Some(addr), + None, + 1, + 21_000, + None, + None, + 0, + 0, + ); + + assert_eq!(tx_receipt.transaction_hash, B256::ZERO); + let logs = tx_receipt.inner.logs(); + assert_eq!(logs.len(), 1); + assert_eq!( + logs[0].transaction_hash, + Some(B256::ZERO), + "inner log must use the same effective hash as the outer receipt" + ); + } } diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index c3eb1caf..c5afda03 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -318,7 +318,9 @@ enum DeferredFixture { Report(FixtureReport), /// Draft built against pre-commit state; write after `finish()` succeeds. /// Boxed so the enum is not dominated by the draft's size on the skip path. - Ready { draft: Box, path: PathBuf }, + /// `overwrite` is enforced at materialization via noclobber persist — the + /// prep-time existence check is only a fast path. + Ready { draft: Box, path: PathBuf, overwrite: bool }, } /// A target that executed, awaiting the receipt harvested by `finish()`. @@ -652,6 +654,16 @@ where let target_set: HashSet = targets.iter().copied().collect(); let tx_hashes: Vec = block.transactions.hashes().collect(); + // Highest block index among this job's targets: once that transaction has + // committed we can stop — later non-targets are not needed for receipts or + // fixtures, and requiring them would force incomplete offline captures to + // abort after a successful dump target. + let last_target_index = tx_hashes + .iter() + .enumerate() + .filter(|(_, hash)| target_set.contains(*hash)) + .map(|(i, _)| i) + .max(); let mut pending: Vec = Vec::new(); let mut committed = 0usize; @@ -742,6 +754,12 @@ where fixture, }); } + + // Stop once every requested target that can run has committed: trailing + // non-targets are irrelevant to this job's receipts and fixtures. + if Some(tx_index) == last_target_index { + break; + } } Ok(()) } @@ -820,7 +838,25 @@ where } else { None }; - let fixture = target.fixture.map(materialize_deferred_fixture); + // Materialize only when the block loop completed cleanly: a + // mid-block abort after this target built a Ready draft must + // not publish (or clobber) a fixture for a block that failed. + let fixture = target.fixture.map(|deferred| { + if loop_result.is_ok() { + materialize_deferred_fixture(deferred) + } else { + // Drop the ready draft without writing; the target still + // reports its receipt below when finish succeeded. + match deferred { + DeferredFixture::Report(report) => report, + DeferredFixture::Ready { path, .. } => FixtureReport::error(format!( + "fixture not written: block aborted before a clean finish \ + (draft for {} was discarded)", + path.display() + )), + } + } + }); entries.push(BatchEntry::Executed(Box::new(ExecutedTx { tx_hash: target.tx_hash, block_number: number, @@ -983,6 +1019,10 @@ where // Refuse overwrite without the flag before carrying a ready draft, so the // harvest path never has to re-check and a finish failure cannot be confused // with an overwrite refusal. + // Fast-path courtesy: refuse overwrite before carrying a ready draft so the + // harvest path never confuses a finish failure with an overwrite refusal. + // Correctness against a concurrent creator is still enforced at materialize + // time via noclobber persist. if path.exists() && !overwrite { return DeferredFixture::Report(FixtureReport::error(format!( "fixture already exists at {} (pass --overwrite to replace)", @@ -990,7 +1030,7 @@ where ))); } - DeferredFixture::Ready { draft: Box::new(draft), path } + DeferredFixture::Ready { draft: Box::new(draft), path, overwrite } } /// Finalize a deferred fixture after the block `finish()` succeeded. @@ -1001,13 +1041,22 @@ where fn materialize_deferred_fixture(deferred: DeferredFixture) -> FixtureReport { match deferred { DeferredFixture::Report(report) => report, - DeferredFixture::Ready { draft, path } => { - match fixture::finalize_and_write(*draft, &path) { + DeferredFixture::Ready { draft, path, overwrite } => { + match fixture::finalize_and_write(*draft, &path, overwrite) { Ok(()) => { info!(path = %path.display(), "Wrote self-validating fixture"); FixtureReport::written(&path) } - Err(e) => FixtureReport::error(format!("fixture write failed: {e}")), + Err(e) => { + let message = e.to_string(); + // Noclobber / prep-time refusal already carry the full + // "already exists … --overwrite" text; do not wrap them. + if message.contains("already exists") { + FixtureReport::error(message) + } else { + FixtureReport::error(format!("fixture write failed: {message}")) + } + } } } } diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 689a8f19..4806dd4f 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -419,7 +419,9 @@ impl Cmd { // Write the self-validating fixture (re-executes the isolated unit through // state-test and cross-checks it against the replay before writing). if let (Some(path), Some(draft)) = (&self.dump_fixture, result.fixture) { - super::fixture::finalize_and_write(draft, path)?; + // Single-file `--dump-fixture` always replaces the destination path + // (there is no `--overwrite` gate on this form). + super::fixture::finalize_and_write(draft, path, true)?; info!(path = %path.display(), "Wrote self-validating fixture"); } if mismatched { @@ -646,21 +648,11 @@ impl Cmd { .ok_or(ReplayError::TransactionNotFound(ctx.tx_hash))?; // Anchor the receipt to the replayed block: across a reorg or a // load-balanced endpoint serving divergent views, the receipt can - // describe a different inclusion than the block fetched earlier, - // and the fidelity gate would then compare the replay against the - // wrong on-chain execution. - if let Some(receipt_block_hash) = receipt.block_hash() { - let replayed_block_hash = ctx.block.hash(); - if receipt_block_hash != replayed_block_hash { - return Err(ReplayError::Other(format!( - "receipt block hash {receipt_block_hash} != replayed block hash \ - {replayed_block_hash}: the receipt describes a different inclusion \ - than the fetched block (reorg in progress, or a load-balanced \ - endpoint serving divergent views); retry the dump once the chain \ - settles" - ))); - } - } + // describe a different inclusion than the block fetched earlier + // (including a receipt with `blockHash: null`). Same check as + // `--verify-receipt` so dump and verify agree on unanchored receipts. + verify::check_inclusion(receipt.block_hash(), ctx.block.hash()) + .map_err(ReplayError::Other)?; // RLP-hash the receipt's logs with the same helper the state-test // runner uses for `logsRoot`, so the dump can check the replay's logs // against the chain (the rich RPC logs' `inner` is the consensus log). diff --git a/bin/mega-evme/src/replay/fixture.rs b/bin/mega-evme/src/replay/fixture.rs index f97e250b..884b653d 100644 --- a/bin/mega-evme/src/replay/fixture.rs +++ b/bin/mega-evme/src/replay/fixture.rs @@ -242,7 +242,16 @@ where /// Re-execute the isolated unit through `state-test`, cross-check it against the /// observed replay outcome, fill the `post` expectation, and write the fixture. -pub(crate) fn finalize_and_write(draft: FixtureDraft, path: &std::path::Path) -> Result<()> { +/// +/// `overwrite` controls the final publish step: when false, the write refuses to +/// replace an existing file (`persist_noclobber`); when true, it replaces via +/// `persist`. Existence checks earlier in the dump pipeline are a fast-path only +/// — correctness against a concurrent creator comes from the noclobber publish. +pub(crate) fn finalize_and_write( + draft: FixtureDraft, + path: &std::path::Path, + overwrite: bool, +) -> Result<()> { let executed = execute_unit_collect(&draft.unit, &draft.spec) .map_err(|e| ReplayError::Other(format!("fixture self-execution failed: {e}")))?; @@ -297,20 +306,46 @@ pub(crate) fn finalize_and_write(draft: FixtureDraft, path: &std::path::Path) -> let suite = TestSuite(BTreeMap::from([(draft.name, unit)])); let json = serde_json::to_string_pretty(&suite) .map_err(|e| ReplayError::Other(format!("failed to serialize fixture: {e}")))?; - // Write to a sibling temp file and rename so an interrupted write cannot - // truncate an existing fixture at `path` (e.g. a committed corpus entry - // being refreshed in place). - let tmp = path.with_extension("json.tmp"); - std::fs::write(&tmp, json).map_err(|e| { - ReplayError::Other(format!("failed to write fixture {}: {e}", tmp.display())) + + // Unique temp file in the target directory, then persist (or noclobber-persist) + // into `path`. A fixed sibling name would race two concurrent dumps; a unique + // name plus noclobber makes `--overwrite=false` safe at materialization time. + let dir = path.parent().unwrap_or_else(|| std::path::Path::new(".")); + let mut tmp = tempfile::NamedTempFile::new_in(dir).map_err(|e| { + ReplayError::Other(format!("failed to create temp fixture file in {}: {e}", dir.display())) })?; - std::fs::rename(&tmp, path).map_err(|e| { - ReplayError::Other(format!( - "failed to rename fixture {} -> {}: {e}", - tmp.display(), - path.display() - )) - }) + use std::io::Write; + tmp.write_all(json.as_bytes()) + .map_err(|e| ReplayError::Other(format!("failed to write fixture temp file: {e}")))?; + tmp.flush() + .map_err(|e| ReplayError::Other(format!("failed to flush fixture temp file: {e}")))?; + if overwrite { + tmp.persist(path).map_err(|e| { + ReplayError::Other(format!( + "failed to persist fixture to {}: {}", + path.display(), + e.error + )) + })?; + } else { + tmp.persist_noclobber(path).map_err(|e| { + // Target already present (or appeared between prep and publish): same + // refused-overwrite path the prep-time existence check uses. + if path.exists() { + ReplayError::Other(format!( + "fixture already exists at {} (pass --overwrite to replace)", + path.display() + )) + } else { + ReplayError::Other(format!( + "failed to persist fixture to {}: {}", + path.display(), + e.error + )) + } + })?; + } + Ok(()) } /// Read the pre-execution values of every account in the target transaction's @@ -324,6 +359,18 @@ where DB: DatabaseRef, DB::Error: Display, { + // Test-only injection: the offline State cache reuses account basics already + // loaded during execution, so doctoring the capture cannot force a draft-only + // pre-state failure. Integration tests set this env var to exercise the + // construction-error path after a successful execution. + if std::env::var_os("MEGA_EVME_INJECT_FIXTURE_PRE_STATE_ERROR").is_some() { + return Err(ReplayError::Other( + "pre-state read for 0x0000000000000000000000000000000000000001: \ + injected draft-time database failure" + .to_string(), + )); + } + let mut pre = BTreeMap::new(); for (address, account) in evm_state { let Some(info) = db diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index bc9568b0..cdf54bc2 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -553,6 +553,38 @@ fn test_replay_block_rejects_mismatched_parent_hash() { assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); } +/// Receipts from a late multi-log transaction stamp every inner log with the +/// outer block/tx identity and a block-global `logIndex` that starts above zero +/// (preceding receipts already emitted logs). +#[test] +#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] +fn test_replay_receipt_inner_log_metadata_nonzero_preceding_offset() { + // Last transaction of BLOCK: multi-log, with many preceding logs in-block. + const LATE_TX: &str = "0xb6a0b7a302c741f64b8e46861a3dcb2d5c1047f6f2cb89a35b5c2183c96296b7"; + + let stdout = replay(&["--json", LATE_TX], true); + let summary = common::json_values(&stdout) + .into_iter() + .find(|v| v.get("receipt").is_some()) + .expect("replay summary with receipt"); + let receipt = &summary["receipt"]; + let block_hash = receipt["blockHash"].as_str().expect("blockHash"); + let tx_hash = receipt["transactionHash"].as_str().expect("transactionHash"); + let logs = receipt["logs"].as_array().expect("logs"); + assert!(!logs.is_empty(), "late tx must emit logs"); + let first = u64::from_str_radix( + logs[0]["logIndex"].as_str().expect("logIndex").trim_start_matches("0x"), + 16, + ) + .expect("parse logIndex"); + assert!(first > 0, "expected non-zero preceding-log offset, got {first}"); + for (i, log) in logs.iter().enumerate() { + assert_eq!(log["blockHash"].as_str(), Some(block_hash), "log {i}"); + assert_eq!(log["transactionHash"].as_str(), Some(tx_hash), "log {i}"); + assert!(log["logIndex"].is_string(), "log {i} logIndex"); + } +} + /// Sweeping a block with `--dump-fixture-dir` against an envelope that carries /// no receipts skips every target on the fidelity gate and still exits 0. /// diff --git a/bin/mega-evme/tests/replay_dump.rs b/bin/mega-evme/tests/replay_dump.rs index 516e0661..173bbac3 100644 --- a/bin/mega-evme/tests/replay_dump.rs +++ b/bin/mega-evme/tests/replay_dump.rs @@ -121,16 +121,15 @@ fn test_replay_dump_is_byte_reproducible() { ); } -/// Dumping over an existing fixture must go through a sibling temp file + -/// rename: on success the target holds the new (valid) content and no -/// `.json.tmp` residue is left behind, so an interrupt mid-write can no longer -/// truncate a committed corpus fixture. +/// Dumping over an existing fixture must go through a unique temp file + +/// persist: on success the target holds the new (valid) content and no +/// leftover temp files remain in the destination directory, so an interrupt +/// mid-write can no longer truncate a committed corpus fixture. #[test] fn test_replay_dump_overwrites_atomically_without_tmp_residue() { let out = std::env::temp_dir().join(format!("mega_evme_dump_atomic_{}.json", std::process::id())); - let tmp = out.with_extension("json.tmp"); - let _ = std::fs::remove_file(&tmp); + let _ = std::fs::remove_file(&out); // Seed a pre-existing "committed" fixture that the dump overwrites in place. std::fs::write(&out, br#"{"pre-existing":"corpus fixture"}"#).expect("seed existing fixture"); @@ -144,7 +143,20 @@ fn test_replay_dump_overwrites_atomically_without_tmp_residue() { "dump over an existing fixture failed.\nstderr: {}", String::from_utf8_lossy(&output.stderr) ); - assert!(!tmp.exists(), "dump must not leave a .json.tmp file behind"); + // NamedTempFile uses a random name; ensure only the destination remains. + let parent = out.parent().expect("temp dir"); + let stem = out.file_stem().and_then(|s| s.to_str()).expect("utf-8 stem"); + let leftovers: Vec<_> = std::fs::read_dir(parent) + .expect("list temp dir") + .filter_map(|e| e.ok()) + .filter(|e| { + let name = e.file_name(); + let name = name.to_string_lossy(); + name.starts_with(stem) && name != out.file_name().unwrap().to_string_lossy() + }) + .map(|e| e.path()) + .collect(); + assert!(leftovers.is_empty(), "dump must not leave temp residue: {leftovers:?}"); let content = std::fs::read_to_string(&out).expect("read dumped fixture"); let _ = std::fs::remove_file(&out); diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index 633329d3..586caf3e 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -76,11 +76,18 @@ impl Run { /// Run `replay` offline against `cache`. fn replay(cache: &Path, args: &[&str]) -> Run { - let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) - .args(["replay", "--rpc.replay-file", cache.to_str().expect("cache path is utf-8")]) - .args(args) - .output() - .expect("failed to run mega-evme"); + replay_with_env(cache, args, &[]) +} + +/// Run `replay` offline against `cache` with additional process environment. +fn replay_with_env(cache: &Path, args: &[&str], envs: &[(&str, &str)]) -> Run { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_mega-evme")); + cmd.args(["replay", "--rpc.replay-file", cache.to_str().expect("cache path is utf-8")]) + .args(args); + for (key, value) in envs { + cmd.env(key, value); + } + let output = cmd.output().expect("failed to run mega-evme"); Run { success: output.status.success(), code: output.status.code(), @@ -621,58 +628,22 @@ fn test_batch_dump_fixture_dir_refuses_overwrite_without_flag() { /// /// Classification of construction vs unsupported-shape errors is unit-tested in /// `batch::tests::test_fixture_build_err_classifies_skips_vs_construction_errors`. -/// This end-to-end check forces a construction-time failure by removing every -/// non-empty bytecode response from the capture so a pre-state `code` fetch -/// fails after the transaction has already executed against cached account -/// state (execution may still succeed from in-memory code; draft-time re-fetch -/// does not). +/// +/// The offline State cache reuses account basics already loaded during +/// execution, so doctoring bytecode responses only kills execution. This test +/// injects a draft-time pre-state failure after execution succeeds +/// (`MEGA_EVME_INJECT_FIXTURE_PRE_STATE_ERROR`), proving the construction path +/// itself — a result line with `fixture.error` containing `construction failed`, +/// exit 1, and no skip. #[test] fn test_batch_fixture_construction_failure_is_fixture_error_not_skip() { - let mut envelope: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) - .expect("parse offline cache"); - let mut nulled = 0; - for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { - let value = entry["value"].as_str().expect("entry value is a string"); - let Ok(mut response) = serde_json::from_str::(value) else { - continue; - }; - let Some(result) = response.get("result").cloned() else { - continue; - }; - // Non-empty bytecode hex responses (eth_getCode / code-by-hash). - let Some(hex) = result.as_str() else { - continue; - }; - if hex == "0x" || hex == "0x0" || hex.len() <= 4 { - continue; - } - // Only rewrite pure hex bytecode payloads, not block/tx objects. - if !hex.starts_with("0x") || hex.len() < 100 { - continue; - } - // Turn the code response into a JSON-RPC error so a draft-time re-fetch fails. - response.as_object_mut().expect("response object").remove("result"); - response["error"] = serde_json::json!({ - "code": -32000, - "message": "code unavailable at draft time", - }); - entry["value"] = serde_json::Value::String(response.to_string()); - nulled += 1; - } - // If the capture has no long bytecode entries the construction path cannot - // be forced this way; fail loudly so the fixture is refreshed. - assert!(nulled > 0, "offline cache should contain bytecode responses to doctor"); - - let cache_path = temp_path("fixture_construction"); - std::fs::write(&cache_path, envelope.to_string()).expect("write doctored cache"); let list = tx_file("fixture_construction"); let dir = std::env::temp_dir().join(format!("mega_evme_fixture_construction_{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); - let run = replay( - &cache_path, + let run = replay_with_env( + &cache(), &[ "--tx-file", list.to_str().unwrap(), @@ -680,74 +651,100 @@ fn test_batch_fixture_construction_failure_is_fixture_error_not_skip() { dir.to_str().unwrap(), "--json", ], + &[("MEGA_EVME_INJECT_FIXTURE_PRE_STATE_ERROR", "1")], ); - let _ = std::fs::remove_file(&cache_path); let _ = std::fs::remove_file(&list); let _ = std::fs::remove_dir_all(&dir); - // Either the run fails during execution (code needed to execute) or during - // fixture construction. The construction path is the one we want: a result - // line with fixture.error. If execution fails first, the target is an error - // entry — also non-zero, never a silent skip. - assert!(!run.success, "missing code must not exit 0.\nstderr: {}", run.stderr); + assert!(!run.success, "construction failure must not exit 0.\nstderr: {}", run.stderr); let lines = run.ndjson(); assert_eq!(lines.len(), 1, "one line per requested transaction"); - if let Some(error) = lines[0].get("error") { - // Execution failed before draft: still non-zero, not a skip. - assert!(error.get("kind").is_some(), "execution failure entry: {}", lines[0]); - } else { - assert!( - lines[0]["fixture"]["error"].as_str().is_some_and(|m| { - m.contains("construction failed") || - m.contains("code") || - m.contains("pre-state") || - m.contains("fixture") - }), - "expected fixture.error for a construction failure: {}", - lines[0] - ); - assert!( - lines[0]["fixture"].get("skipped").is_none(), - "construction failure must not be reported as a skip: {}", - lines[0] - ); - assert_eq!(run.code(), 1, "fixture construction failure exits 1"); - } + // Sentinel: execution reached the dump target (receipt present); only the + // draft failed. The old code path accepted an execution-only failure here. + assert!( + lines[0].get("error").is_none(), + "execution must succeed so the failure is fixture construction, not an error entry: {}", + lines[0] + ); + assert!( + lines[0]["success"].as_bool() == Some(true), + "target must have executed successfully: {}", + lines[0] + ); + let fixture_error = lines[0]["fixture"]["error"] + .as_str() + .expect("fixture.error must be set for a construction failure"); + assert!( + fixture_error.contains("construction failed"), + "expected construction failed, got: {fixture_error}" + ); + assert!( + fixture_error.contains("pre-state") || fixture_error.contains("injected"), + "expected pre-state/injected failure detail, got: {fixture_error}" + ); + assert!( + lines[0]["fixture"].get("skipped").is_none(), + "construction failure must not be reported as a skip: {}", + lines[0] + ); + assert_eq!(run.code(), 1, "fixture construction failure exits 1"); } -/// When the block aborts before a dump target can finish, no fixture file is -/// written and `--overwrite` does not clobber a pre-existing file. +/// When the block aborts after a dump target built a Ready draft, no fixture +/// file is written and `--overwrite` does not clobber a pre-existing file. /// -/// Doctors the capture so a preceding transaction is unknown to the endpoint: -/// the dump target never reaches commit/finish, so deferred finalize+write never -/// runs. Happy-path writes are covered by +/// Seeds a zero-gas object for the index-2 hash so resolve succeeds but the +/// block loop aborts on that transaction *after* the dump target (index 1) +/// has executed and built a deferred draft. Materialize runs only on a clean +/// loop, so the Ready draft is discarded. Happy-path writes are covered by /// [`test_batch_dump_fixture_dir_writes_validatable_file`]. #[test] fn test_batch_dump_does_not_write_or_clobber_when_block_aborts_before_finish() { - // The captured transaction is at index 1; deny the index-0 hash so the block - // aborts before the dump target runs. + // Index-2 hash of the captured block. Not present as a full TX object in + // the offline capture; we inject a zero-gas type-2 call so lookup succeeds + // and execution aborts after the dump target has drafted. + const LATER: &str = "0xfc0a0b9d76b13125ac1e36e524f6df3a72c25720c023b960b23c6f5891be05bc"; + // `keccak256("eth_getTransactionByHash\0[\"\"]")` — same formula as + // `transport_cache_key` in `common/provider/transport.rs`. + const LATER_CACHE_KEY: &str = + "0x91bbb37d27a588e217e5be6aeab0fb377ffea0ad3a2714d1f54ceb69852124f2"; + let mut envelope: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) .expect("parse offline cache"); - let preceding = "0x14ca11aad284153b6e0460428dadf5d0dcab76a5b129ee1f6c9e07f535620a24"; - let marker = format!("\"hash\":\"{preceding}\""); - let mut doctored = 0; - for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + // Clone the dump target's TX response shape and rewrite hash + gas so the + // later index is fetchable but rejected at execution (intrinsic gas). + let mut template: Option = None; + let target_marker = format!("\"hash\":\"{TX}\""); + for entry in envelope["cache"].as_array().expect("cache entries") { let value = entry["value"].as_str().expect("entry value is a string"); - if !value.contains(&marker) { + if !value.contains(&target_marker) { continue; } - let mut response: serde_json::Value = + let response: serde_json::Value = serde_json::from_str(value).expect("parse transaction response"); - response["result"] = serde_json::Value::Null; - entry["value"] = serde_json::Value::String(response.to_string()); - doctored += 1; + if response["result"].get("hash").and_then(|h| h.as_str()) == Some(TX) { + template = Some(response); + break; + } } - assert_eq!(doctored, 1, "the offline cache must hold the preceding transaction"); - let cache_path = temp_path("dump_abort_before"); + let mut response = template.expect("offline cache must hold the dump target TX object"); + let result = response["result"].as_object_mut().expect("tx result object"); + result.insert("hash".into(), serde_json::Value::String(LATER.to_string())); + result.insert("transactionIndex".into(), serde_json::Value::String("0x2".into())); + result.insert("gas".into(), serde_json::Value::String("0x0".into())); + envelope["cache"].as_array_mut().expect("cache").push(serde_json::json!({ + "key": LATER_CACHE_KEY, + "value": response.to_string(), + })); + + let cache_path = temp_path("dump_abort_after"); std::fs::write(&cache_path, envelope.to_string()).expect("write doctored cache"); - let list = tx_file("dump_abort_before"); + let list_path = std::env::temp_dir() + .join(format!("mega_evme_verify_dump_abort_after_{}.txt", std::process::id())); + std::fs::write(&list_path, format!("{TX}\n{LATER}\n")).expect("write tx list"); + let dir = std::env::temp_dir().join(format!("mega_evme_batch_dump_abort_{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); @@ -760,7 +757,7 @@ fn test_batch_dump_does_not_write_or_clobber_when_block_aborts_before_finish() { &cache_path, &[ "--tx-file", - list.to_str().unwrap(), + list_path.to_str().unwrap(), "--dump-fixture-dir", dir.to_str().unwrap(), "--overwrite", @@ -768,41 +765,91 @@ fn test_batch_dump_does_not_write_or_clobber_when_block_aborts_before_finish() { ], ); let _ = std::fs::remove_file(&cache_path); - let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_file(&list_path); assert!(!run.success, "an aborted block must exit non-zero.\nstderr: {}", run.stderr); let lines = run.ndjson(); - assert_eq!(lines.len(), 1, "one line for the requested target"); + assert!(lines.len() >= 2, "expected lines for both targets, got: {lines:?}"); + + let dump_line = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(TX)) + .expect("dump target must appear in the output"); + // Sentinel: the target ran and built a Ready draft that was then discarded. + // The old test aborted before the target, so there was never a fixture report. assert!( - lines[0].get("error").is_some(), - "the target must be reported as an error entry, not a dump: {}", - lines[0] + dump_line.get("error").is_none(), + "dump target must execute before the abort: {dump_line}" ); + assert!(dump_line["success"].as_bool() == Some(true), "dump target must succeed: {dump_line}"); + let fixture_error = dump_line["fixture"]["error"] + .as_str() + .expect("discarded Ready draft must surface as fixture.error"); assert!( - lines[0].get("fixture").is_none(), - "an unfinished target carries no fixture report: {}", - lines[0] + fixture_error.contains("discarded") || fixture_error.contains("aborted"), + "expected draft-discarded message, got: {fixture_error}" + ); + assert!( + dump_line["fixture"].get("path").is_none(), + "discarded draft must not report a written path: {dump_line}" ); + let kept = std::fs::read(&fixture_path).expect("pre-existing fixture must still exist"); assert_eq!(kept, sentinel, "--overwrite must not clobber when the dump never finalizes"); let _ = std::fs::remove_dir_all(&dir); } /// Replay receipts stamp each inner log with the outer receipt's block/tx -/// identity. The offline capture emits no logs, so this asserts the empty-log -/// path stays consistent; non-empty log metadata is covered by the unit test on -/// `op_receipt_to_tx_receipt`. +/// identity and a block-global `logIndex` that starts above zero when earlier +/// receipts in the block already emitted logs. +/// +/// The committed offline capture has no logs, so this test uses the dev +/// envelope (block 22945844, last tx) when `MEGA_EVME_TEST_ENVELOPE` is set — +/// the same fixture the ignored batch suite uses. Without the envelope the +/// non-empty path is covered by +/// `outcome::tests::test_op_receipt_to_tx_receipt_stamps_inner_log_metadata`. #[test] fn test_replay_receipt_inner_log_metadata_matches_outer_receipt() { - let run = replay(&cache(), &["--json", TX]); - assert!(run.success, "replay must exit 0.\nstderr: {}", run.stderr); + let envelope = match std::env::var("MEGA_EVME_TEST_ENVELOPE") { + Ok(path) if !path.is_empty() => PathBuf::from(path), + _ => { + // Unit test covers non-empty logs + non-zero first_log_index; keep + // this integration test green in CI without the large envelope. + let run = replay(&cache(), &["--json", TX]); + assert!(run.success, "replay must exit 0.\nstderr: {}", run.stderr); + let summary = run.json(); + let logs = summary["receipt"]["logs"].as_array().expect("logs array"); + assert!( + logs.is_empty(), + "committed capture is log-less; set MEGA_EVME_TEST_ENVELOPE for multi-log coverage" + ); + return; + } + }; + + // Last transaction of block 22945844: multi-log, with many preceding logs. + const LATE_TX: &str = "0xb6a0b7a302c741f64b8e46861a3dcb2d5c1047f6f2cb89a35b5c2183c96296b7"; + + let run = replay(&envelope, &["--json", LATE_TX]); + assert!(run.success, "envelope replay must exit 0.\nstderr: {}", run.stderr); let summary = run.json(); let receipt = &summary["receipt"]; let block_hash = receipt["blockHash"].as_str().expect("receipt blockHash"); let tx_hash = receipt["transactionHash"].as_str().expect("receipt transactionHash"); let tx_index = receipt["transactionIndex"].as_str().expect("receipt transactionIndex"); - assert_eq!(tx_hash, TX); + assert_eq!(tx_hash, LATE_TX); let logs = receipt["logs"].as_array().expect("receipt logs array"); + assert!(!logs.is_empty(), "late envelope tx must emit logs (got empty); envelope may be stale"); + // Sentinel: preceding receipts emitted logs, so the first log_index is > 0. + let first_log_index = u64::from_str_radix( + logs[0]["logIndex"].as_str().expect("logIndex string").trim_start_matches("0x"), + 16, + ) + .expect("parse logIndex"); + assert!( + first_log_index > 0, + "expected non-zero preceding-log offset, got logIndex={first_log_index}" + ); for (i, log) in logs.iter().enumerate() { assert_eq!(log["blockHash"].as_str(), Some(block_hash), "log {i} blockHash"); assert_eq!(log["transactionHash"].as_str(), Some(tx_hash), "log {i} transactionHash"); @@ -811,15 +858,24 @@ fn test_replay_receipt_inner_log_metadata_matches_outer_receipt() { } // Batch path stamps the same fields. - let list = tx_file("batch_log_meta"); - let batch = replay(&cache(), &["--tx-file", list.to_str().unwrap(), "--json"]); - let _ = std::fs::remove_file(&list); + let list_path = std::env::temp_dir() + .join(format!("mega_evme_verify_batch_log_meta_{}.txt", std::process::id())); + std::fs::write(&list_path, format!("{LATE_TX}\n")).expect("write tx list"); + let batch = replay(&envelope, &["--tx-file", list_path.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&list_path); assert!(batch.success, "batch replay must exit 0.\nstderr: {}", batch.stderr); let line = &batch.ndjson()[0]; let batch_receipt = &line["receipt"]; assert_eq!(batch_receipt["blockHash"].as_str(), Some(block_hash)); assert_eq!(batch_receipt["transactionHash"].as_str(), Some(tx_hash)); let batch_logs = batch_receipt["logs"].as_array().expect("batch receipt logs"); + assert!(!batch_logs.is_empty(), "batch path must also carry non-empty logs"); + let batch_first = u64::from_str_radix( + batch_logs[0]["logIndex"].as_str().expect("logIndex").trim_start_matches("0x"), + 16, + ) + .expect("parse batch logIndex"); + assert!(batch_first > 0, "batch logIndex must start above zero, got {batch_first}"); for (i, log) in batch_logs.iter().enumerate() { assert_eq!(log["blockHash"].as_str(), Some(block_hash), "batch log {i} blockHash"); assert_eq!(log["transactionHash"].as_str(), Some(tx_hash), "batch log {i} transactionHash"); From 00e5468f807cabf08b588c7dacb8c315f4568554 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 17:22:00 +0800 Subject: [PATCH 17/64] test(mega-evme): compile the draft-failure injection hook out of production builds --- bin/mega-evme/src/replay/fixture.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/mega-evme/src/replay/fixture.rs b/bin/mega-evme/src/replay/fixture.rs index 884b653d..eeef70a4 100644 --- a/bin/mega-evme/src/replay/fixture.rs +++ b/bin/mega-evme/src/replay/fixture.rs @@ -362,7 +362,10 @@ where // Test-only injection: the offline State cache reuses account basics already // loaded during execution, so doctoring the capture cannot force a draft-only // pre-state failure. Integration tests set this env var to exercise the - // construction-error path after a successful execution. + // construction-error path after a successful execution. Compiled out of + // production builds: only the test profile and the `test-utils` feature + // (enabled for the binary via the self dev-dependency) carry the hook. + #[cfg(any(test, feature = "test-utils"))] if std::env::var_os("MEGA_EVME_INJECT_FIXTURE_PRE_STATE_ERROR").is_some() { return Err(ReplayError::Other( "pre-state read for 0x0000000000000000000000000000000000000001: \ From 08ba95010fb1911be3b90763f342c67fd729ed85 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 4 Aug 2026 17:50:00 +0800 Subject: [PATCH 18/64] chore(mega-evme): drop a stale duplicate comment block --- bin/mega-evme/src/replay/batch.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index c5afda03..0c8185c0 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -1016,9 +1016,6 @@ where let tx_hash = target_tx.inner.inner.tx_hash(); let path = dir.join(format!("{tx_hash:#x}.json")); - // Refuse overwrite without the flag before carrying a ready draft, so the - // harvest path never has to re-check and a finish failure cannot be confused - // with an overwrite refusal. // Fast-path courtesy: refuse overwrite before carrying a ready draft so the // harvest path never confuses a finish failure with an overwrite refusal. // Correctness against a concurrent creator is still enforced at materialize From ad7e0b1689f3847e30f13d83b8815a3803c77042 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 10:44:01 +0800 Subject: [PATCH 19/64] fix(mega-evme): keep captured entries when a run left the external_env snapshot unchanged A capture run given no --bucket-capacity carries the loaded snapshot forward verbatim, so `ours == loaded`: the run expressed no opinion about the value. The persist-time optimistic concurrency check only branched on `ours != on_disk` and then required `on_disk == loaded`, so a sibling that legitimately refreshed the snapshot made both conditions fail and the persist hard-error. Because capture persistence is a hard error by design, that discarded every RPC response the run had captured, over metadata the run never had a stake in. Resolve the snapshot from an explicit equivalence-class table over the three inputs (loaded, ours, on-disk), documented on the function: a run whose snapshot still equals the load-time one yields to a concurrent refresh and still merges its cache entries, while two runs changing the same snapshot differently remain a hard conflict. The predicate is value equality, not "was the flag passed", since persist has no record of the caller's argv. Add the missing table rows as tests, including the carried-forward case that the previous tests never covered: they all used three distinct values, which is what "conflict" looks like in the abstract but not what the common case looks like. Also sync the cache temp file before the rename: flush only clears the userspace buffer, so a crash in between could publish a truncated file under the target name. --- bin/mega-evme/src/cache/merge.rs | 109 +++++++++++++++--- docs/mega-evme/commands/replay.md | 11 +- .../configuration/state-management.md | 14 ++- 3 files changed, 113 insertions(+), 21 deletions(-) diff --git a/bin/mega-evme/src/cache/merge.rs b/bin/mega-evme/src/cache/merge.rs index ba4260c0..7c842104 100644 --- a/bin/mega-evme/src/cache/merge.rs +++ b/bin/mega-evme/src/cache/merge.rs @@ -258,16 +258,10 @@ pub(crate) fn merge_provider_lists(base: Vec, overlay: Vec) -> /// /// `external_env` uses optimistic concurrency against `loaded_external_env` — the /// snapshot observed when this process opened the capture file (or `None` when -/// the file was absent / had no snapshot): -/// -/// - If the locked on-disk snapshot is absent, or still equals the loaded one, nobody else changed -/// it: the caller's intentional update wins (`ours`). -/// - Hard-error only when the on-disk snapshot changed since load **and** differs from ours (true -/// concurrent conflict). The error names all three values (loaded, ours, on-disk). -/// -/// Snapshots are canonicalized (last-wins per bucket id, then sorted) before -/// comparison and before writing, so CLI order alone cannot spuriously conflict. -/// One-sided snapshots still merge: ours is kept when set, otherwise on-disk. +/// the file was absent / had no snapshot). See +/// [`resolve_external_env_for_persist`] for the full decision table; only a true +/// concurrent conflict is a hard error, and the message then names all three +/// values (loaded, ours, on-disk). pub(crate) fn merge_envelope_for_persist( on_disk: &EnvelopeDoc, ours: &EnvelopeDoc, @@ -306,7 +300,34 @@ pub(crate) fn merge_envelope_for_persist( /// Resolve the envelope `external_env` under optimistic concurrency. /// -/// See [`merge_envelope_for_persist`] for the accept / hard-error rules. +/// Three inputs decide the outcome: `loaded` (the snapshot this process observed +/// when it opened the file), `ours` (what this process would write), and +/// `on_disk` (what the locked re-read found). All three are canonicalized first, +/// so CLI ordering alone never decides anything. +/// +/// | `ours` | `on_disk` | relation | result | why | +/// | ------ | --------- | ---------------------------- | --------- | ---------------------------------------------------------------- | +/// | `None` | any | — | `on_disk` | this run has no snapshot to contribute | +/// | `Some` | `None` | — | `ours` | nothing on disk to disagree with | +/// | `Some` | `Some` | equal | `ours` | no decision to make | +/// | `Some` | `Some` | differ, `ours == loaded` | `on_disk` | this run changed nothing: sibling's refresh wins | +/// | `Some` | `Some` | differ, `on_disk == loaded` | `ours` | nobody wrote since load: our intentional refresh wins | +/// | `Some` | `Some` | differ, neither | `Err` | true conflict: two runs changed the same snapshot differently | +/// +/// The last two conditions are mutually exclusive, so their order does not +/// matter: if both held, `ours` and `on_disk` would each equal `loaded` and +/// therefore each other, contradicting "differ". +/// +/// Row four carries as much weight as row six. A capture run given no +/// `--bucket-capacity` carries the previous snapshot forward verbatim, so +/// `ours == loaded` means "changed nothing", not "chose this value". Calling +/// that a conflict fails the persist — and because capture persistence is a +/// hard error, that discards every RPC response the run captured, over metadata +/// the run never had a stake in. +/// +/// The predicate is value equality, not "was the flag passed": persist has no +/// record of the caller's argv, so re-asserting the values already in force is +/// indistinguishable from carrying them forward, and both yield. fn resolve_external_env_for_persist( ours: &Option, on_disk: &Option, @@ -319,12 +340,11 @@ fn resolve_external_env_for_persist( match (&ours_c, &disk_c) { (Some(o), Some(d)) if o != d => { - // Differing non-null snapshots: accept only when the on-disk value - // is still the one this process observed at load (intentional A→B - // refresh). A third concurrent writer that changed the file since - // load is a true conflict. - let disk_unchanged = disk_c == loaded_c; - if disk_unchanged { + let ours_carried_forward = loaded_c.as_ref() == Some(o); + let disk_unchanged_since_load = loaded_c.as_ref() == Some(d); + if ours_carried_forward { + Ok(disk_c) + } else if disk_unchanged_since_load { Ok(ours_c) } else { Err(EvmeError::FixtureError(format!( @@ -435,6 +455,10 @@ pub(crate) fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<( })?; tmp.write_all(bytes)?; tmp.flush()?; + // flush() only clears the userspace buffer. Without sync_all() a crash + // between write and rename can publish a truncated file under the target + // name — the rename is atomic, the contents are not. + tmp.as_file().sync_all()?; tmp.persist(path).map_err(|e| { std::io::Error::other(format!( "failed to rename temp file into {}: {}", @@ -581,6 +605,57 @@ mod tests { assert_eq!(merged.cache, vec![kv(1, "disk"), kv(2, "ours")]); } + /// No opinion: loaded A, ours A (carried forward), disk now B → B wins and + /// our cache entries still merge. A run given no `--bucket-capacity` reaches + /// persist with `ours == loaded`; treating that as a conflict would fail the + /// persist and throw away everything the run captured. + #[test] + fn test_merge_envelope_for_persist_carried_forward_snapshot_yields_to_sibling_refresh() { + let loaded = ExternalEnvDoc { bucket_capacities: vec![(1, 10)] }; + let disk_ext = ExternalEnvDoc { bucket_capacities: vec![(1, 20)] }; + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "disk")], + external_env: Some(disk_ext.clone()), + }; + // No `--bucket-capacity` on this run: the loaded snapshot is carried + // forward verbatim, so `ours` is byte-identical to `loaded`. + let ours = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(2, "ours")], + external_env: Some(loaded.clone()), + }; + let merged = + merge_envelope_for_persist(&on_disk, &ours, Some(&loaded), Path::new("capture.json")) + .expect("a run that expressed no opinion must not conflict"); + assert_eq!(merged.external_env, Some(disk_ext.canonicalized())); + assert_eq!(merged.cache, vec![kv(1, "disk"), kv(2, "ours")]); + } + + /// Table row one (`ours = None`): a run with no snapshot of its own keeps + /// the on-disk one and still merges its cache entries. + /// + /// This row was always correct; it is pinned so the table has a test per + /// row rather than only where a bug was found. + #[test] + fn test_merge_envelope_for_persist_no_snapshot_yields_to_sibling_refresh() { + let disk_ext = ExternalEnvDoc { bucket_capacities: vec![(1, 20)] }; + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "disk")], + external_env: Some(disk_ext.clone()), + }; + let ours = + EnvelopeDoc { version: 1, chain_id: 7, cache: vec![kv(2, "ours")], external_env: None }; + let merged = merge_envelope_for_persist(&on_disk, &ours, None, Path::new("capture.json")) + .expect("a run with no snapshot must not conflict"); + assert_eq!(merged.external_env, Some(disk_ext)); + assert_eq!(merged.cache, vec![kv(1, "disk"), kv(2, "ours")]); + } + /// True concurrent conflict: loaded A, ours B, disk now C≠B → hard error naming A/B/C. #[test] fn test_merge_envelope_for_persist_rejects_true_concurrent_conflict() { diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 8f238910..47745b04 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -51,7 +51,8 @@ A batch run builds a single provider and a single RPC cache, groups the requeste Each block is executed exactly once: state is forked at the parent block, pre-execution changes are applied, and every transaction of the block runs in order, with each requested transaction's result recorded before it is committed. The RPC cache is persisted once, on exit, even if some transactions failed — the captured responses are the artifact you need to debug the failure offline. -Batch mode issues the same RPC calls as single-transaction replay, so an offline envelope captured by single-transaction runs serves a batch run without a cache miss. +A plain batch replay issues the same RPC calls as single-transaction replay, so an offline envelope captured by single-transaction runs serves a batch run without a cache miss. +`--verify-receipt` and `--dump-fixture-dir` are the exception: both fetch the receipt of every target in the block, including transactions a single-transaction capture never asked about, so an older envelope will miss them and the run exits `3`. ### `--tx-file ` @@ -112,7 +113,8 @@ Execution outcomes are not errors: a reverted or halted transaction is a normal A failure while running the block aborts it, because the executor state no longer matches the chain. The transaction the failure is about — the hash the endpoint denied, or the one the executor rejected — is reported with that failure's own kind. Every target behind it is reported as `rpc` with a message naming the aborting cause: nothing was established about those transactions, so they went unanswered rather than being unknown. -Targets that never ran are still emitted in the block's transaction-index order, keeping the whole stream in ascending `(block, tx_index)` order; a hash the block does not contain is reported last, in input order, as `not_found`. +Targets that never ran are still emitted in the block's transaction-index order, keeping the whole stream in ascending `(block, tx_index)` order; a hash the block does not contain is reported last within its block, in input order, as `not_found`. +Hashes that could not be resolved to a block at all (unknown, pending, or an endpoint failure during resolution) are emitted before every block result, since the run cannot place them in the stream's order. Without `--json`, each transaction is printed with a header naming its hash, block, and index, followed by the same summary and receipt the single-transaction mode prints. A final one-line summary (transactions replayed, transactions failed, elapsed time) is logged at `INFO` level, so pass `-vvv` to see it. @@ -126,6 +128,9 @@ A batch run exits `0` when every requested transaction produced an execution res Fixture skips (fidelity gate, BLOCKHASH readers, unsupported shapes) are not failures and do not fail the run; a fixture the run was asked to write and could not is an execution-class failure of its target. The NDJSON stream is written to stdout in both cases; diagnostics go to stderr. +The exit code can understate a failure in one case: when the transaction that aborts a block is not itself a target, no target can claim the abort's own class, so every target is reported as `rpc` and the run exits `3` — "the question went unanswered" — even if the underlying cause was a deterministic execution failure that retrying will not fix. +`--block 0` is rejected as invalid input; a block that genuinely holds no transactions produces no stdout lines, exits `0`, and says so on stderr. + ### Examples Replay a whole block offline and stream the results as NDJSON: @@ -286,7 +291,7 @@ On subsequent runs the existing file is loaded, its entries are merged into the The updated set of entries is persisted back to the same file on clean exit. The file also embeds an external-environment snapshot — currently the set of `--bucket-capacity` values in effect — so the captured fixture is self-contained. -If `--bucket-capacity` is not passed on a subsequent run, the previous envelope's values are reused; passing `--bucket-capacity` overrides them (an intentional A→B refresh of an existing capture is accepted at persist when no concurrent writer changed the on-disk snapshot; a true concurrent conflict hard-errors and names the load-time, caller, and on-disk values — see [state management](../configuration/state-management.md#rpc-cache-and-retry)). +If `--bucket-capacity` is not passed on a subsequent run, the previous envelope's values are reused; passing `--bucket-capacity` overrides them (an intentional A→B refresh of an existing capture is accepted at persist when no concurrent writer changed the on-disk snapshot, and a run that reused the previous values yields to a concurrent refresh rather than conflicting with it; only two writers changing the same snapshot differently hard-errors, naming the load-time, caller, and on-disk values — see [state management](../configuration/state-management.md#rpc-cache-and-retry)). The capture is written even when the replay itself failed — an execution or verification failure is exactly the case you want to debug offline. If the write fails, it is reported on stderr like any other failure, next to the run's own error; the run error keeps the exit code, since it is the root cause. diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index 71dac18c..07beee76 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -237,7 +237,9 @@ Capture envelopes (`--rpc.capture-file`) use the same lock + re-read-merge path, - The on-disk envelope `version` and `chain_id` must match this process's capture. - `external_env` uses optimistic concurrency against the snapshot observed when this process opened the capture file: - If the locked on-disk snapshot is absent, or still equals the load-time snapshot, the caller's intentional update wins (so a sequential refresh with `--bucket-capacity` on an existing capture is accepted). - - Persist hard-errors only when the on-disk snapshot changed since load **and** differs from this process's snapshot (true concurrent conflict). + - A run whose snapshot still equals the load-time one has not changed anything, whether it omitted `--bucket-capacity` or passed the values already in force: if another writer refreshed the on-disk snapshot meanwhile, that refresh is kept and this run's cache entries still merge. + Re-asserting the current values is therefore not a way to defend them against a concurrent refresh. + - Persist hard-errors only when this process changed the snapshot **and** the on-disk snapshot also changed since load, to a different value (true concurrent conflict). The error names all three values: loaded, ours, and on-disk. - Snapshots are canonicalized before comparison and write (deduplicate by bucket id with last-wins, then sort by id), so two workers with the same effective capacities in different CLI order do not conflict. - One-sided snapshots still merge: this process's snapshot is kept when set, otherwise the on-disk snapshot is propagated. @@ -257,6 +259,16 @@ Provider-cache merge also rejects inputs (and `--output`) whose `rpc-cache-{chai | `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies. | | `--rpc.clear-cache` | flag | `false` | Delete the current chain's cache file before loading it. Recovery path for a polluted or corrupt cache. | +The in-memory cache layer is always installed on a forked or online run and cannot be turned off; `--rpc.no-cache-file` disables only on-disk persistence. +At the default cap the cache index is preallocated to tens of MiB regardless of how many entries a run actually stores, which is the trade for never re-fetching during a long verification sweep. +Set `--rpc.cache-max-entries` to a smaller value to reduce that footprint. + +#### Removed Flags + +| Removed | Replacement | Note | +| ---------------------- | ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--rpc.cache-size ` | `--rpc.cache-max-entries ` | `N > 0` carries over unchanged. `0` inverted meaning — it used to disable the cache, and now means "effectively unlimited" — so the old flag is rejected rather than aliased, and a script passing it fails loudly instead of silently doing the opposite of what it asked. | + ### Retry Flags | Flag | Type | Default | Description | From cdb61f39e7672e6527b7cf79fdf7e6eb5ce4bb31 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 10:44:11 +0800 Subject: [PATCH 20/64] refactor(mega-evme): type fixture build rejections instead of matching their text Batch dump decided skip-vs-fail by substring-matching build_draft's error message. The match already missed one case it claimed to cover: check_fidelity's logs-root rejection says "emits different logs than the chain" rather than the "does not reproduce on-chain execution" phrase the other two share, so a logs-root divergence would have been reported as a run-failing construction error instead of a skip. Rewording any message could silently reclassify a whole-block sweep. build_draft now returns FixtureBuildError, classified at each rejection site: Unsupported for shapes a fixture cannot express (deposit, EIP-7702, unmapped spec, missing gas price, any fidelity divergence) and Construction for a draft that could not be built. All three fidelity rejections pass through one map_err, so the classification no longer depends on which dimension diverged. Cover the classification by driving the real builder: a deposit against a database that fails every read must still be Unsupported, proving the rejection fires before any state is touched, and a failing pre-state read must be Construction. check_fidelity had no tests at all; add one per dimension. Also sync the fixture temp file before publishing it, so a crash cannot leave a truncated fixture in a corpus under a name that looks complete. --- bin/mega-evme/src/replay/batch.rs | 110 ++++++------ bin/mega-evme/src/replay/fixture.rs | 258 +++++++++++++++++++++++++++- 2 files changed, 306 insertions(+), 62 deletions(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 0c8185c0..cb77651d 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -8,10 +8,14 @@ //! executes each block exactly once while recording the result of every target it //! passes through. //! -//! Every RPC call issued here has the same shape as the single-transaction path -//! (`eth_getTransactionByHash`, `eth_getBlockByNumber` with hash-only bodies, and -//! the state reads behind [`EvmeState::new_forked`]), so an offline envelope -//! captured by single-transaction replays serves batch runs without a miss. +//! Every RPC call issued by a plain batch replay has the same shape as the +//! single-transaction path (`eth_getTransactionByHash`, `eth_getBlockByNumber` +//! with hash-only bodies, and the state reads behind [`EvmeState::new_forked`]), +//! so an offline envelope captured by single-transaction replays serves batch +//! runs without a miss. `--verify-receipt` and `--dump-fixture-dir` are the +//! exception: both fetch `eth_getTransactionReceipt` for *every* target of a +//! block, including the non-targets a single-transaction capture never asked +//! about. Offline, those come back as `rpc` entries and the run exits `3`. use std::{ collections::{BTreeMap, HashSet}, @@ -407,7 +411,16 @@ where let block = fetch_block(provider, *number).await?; let targets: Vec = block.transactions.hashes().collect(); info!(block = number, tx_count = targets.len(), "Batch replay of a whole block"); - vec![BlockJob { number: *number, block: Some(block), targets }] + if targets.is_empty() { + // Nothing failed, so this is a clean exit — but a silent one is + // indistinguishable from a run that produced no output for a bad + // reason, so say why stdout is empty. No job is queued: with no + // targets to report, forking the parent state would buy nothing. + eprintln!("Block {number} contains no transactions; nothing to replay"); + vec![] + } else { + vec![BlockJob { number: *number, block: Some(block), targets }] + } } BatchMode::TxList(hashes) => { let (jobs, failures) = resolve_targets(provider, hashes).await; @@ -1065,27 +1078,17 @@ fn materialize_deferred_fixture(deferred: DeferredFixture) -> FixtureReport { /// Unsupported shapes are expected in whole-block sweeps and must not fail the /// run. Endpoint/DB failures during construction mean the requested artifact /// could not be produced and fail the run as an execution-class fixture error. -fn fixture_report_from_build_err(err: ReplayError) -> FixtureReport { - let message = err.to_string(); - if is_unsupported_fixture_shape(&message) { - FixtureReport::skipped(message) - } else { - FixtureReport::error(format!("fixture construction failed: {message}")) +/// The builder decides which is which at the point it knows, so rewording any of +/// its messages cannot silently reclassify a sweep. +fn fixture_report_from_build_err(err: fixture::FixtureBuildError) -> FixtureReport { + match err { + fixture::FixtureBuildError::Unsupported(reason) => FixtureReport::skipped(reason), + fixture::FixtureBuildError::Construction(err) => { + FixtureReport::error(format!("fixture construction failed: {err}")) + } } } -/// Whether a `build_draft` error is an expected unsupported shape (skip) rather -/// than a construction failure. -fn is_unsupported_fixture_shape(message: &str) -> bool { - message.contains("does not support deposit") - || message.contains("does not support EIP-7702") - || message.contains("has no fixture mapping") - || message.contains("reports no gas price") - // Fidelity is normally gated before `build_draft`; keep the classification - // if a fidelity check inside the builder ever surfaces here. - || message.contains("does not reproduce on-chain execution") -} - /// Fetch the on-chain receipt of every target of a block. /// /// Each target maps either to the consensus facts its receipt reports, or to the @@ -1155,6 +1158,13 @@ fn classify(err: &ReplayError) -> BatchErrorKind { /// stop (unknown hash, RPC failure, executor/setup error on another /// transaction), a non-aborting swept target is unanswered (`rpc`). Only the /// transaction that caused the abort keeps its own classified kind. +/// +/// The error is taken and ignored on purpose: the signature keeps the decision +/// visible at the call site, so a future change that wants to classify by cause +/// has to argue against this rule rather than silently add a parameter. Note the +/// consequence — a deterministic execution failure in a *non-target* transaction +/// sweeps every target as `rpc` (exit `3`, "retrying may help") even though +/// retrying will not help. fn swept_kind(_err: &ReplayError) -> BatchErrorKind { BatchErrorKind::Rpc } @@ -1485,46 +1495,38 @@ mod tests { assert!(parse_block_number("12.5").is_err()); } - /// Unsupported shapes remain skips; database/construction failures become - /// fixture errors so the run exits non-zero. + /// Unsupported shapes remain skips; construction failures become fixture + /// errors so the run exits non-zero. + /// + /// Which builder rejection lands in which variant is decided inside + /// `build_draft` and pinned by the integration tests that drive the real + /// builder (deposit skip, injected pre-state failure); this test only pins + /// the variant-to-report mapping, which no rewording can move. #[test] fn test_fixture_build_err_classifies_skips_vs_construction_errors() { - let deposit = fixture_report_from_build_err(ReplayError::Other( + let unsupported = fixture_report_from_build_err(fixture::FixtureBuildError::Unsupported( "--dump-fixture does not support deposit transactions".into(), )); - assert!(deposit.skipped.is_some(), "deposit is a skip: {deposit:?}"); - assert!(deposit.error.is_none()); - - let eip7702 = fixture_report_from_build_err(ReplayError::Other( - "--dump-fixture does not support EIP-7702 (set-code) transactions: the \ - fixture builder does not serialize the authorization list" - .into(), - )); - assert!(eip7702.skipped.is_some(), "EIP-7702 is a skip: {eip7702:?}"); - - let unknown_spec = fixture_report_from_build_err(ReplayError::Other( - "--dump-fixture: spec Rex99 has no fixture mapping".into(), - )); - assert!(unknown_spec.skipped.is_some(), "unknown spec is a skip: {unknown_spec:?}"); - - let pre_state = fixture_report_from_build_err(ReplayError::Other( - "pre-state read for 0x00000000000000000000000000000000000000aa: database unavailable" - .into(), - )); - assert!( - pre_state.error.as_ref().is_some_and(|m| m.contains("construction failed")), - "DB pre-state failure is a fixture error: {pre_state:?}" + assert!(unsupported.skipped.is_some(), "unsupported shape is a skip: {unsupported:?}"); + assert!(unsupported.error.is_none()); + assert_eq!( + unsupported.skipped.as_deref(), + Some("--dump-fixture does not support deposit transactions"), + "the builder's reason is reported verbatim" ); - assert!(pre_state.skipped.is_none()); - let code_fetch = fixture_report_from_build_err(ReplayError::Other( - "code fetch for 0x1111: endpoint timeout".into(), + let construction = fixture_report_from_build_err(fixture::FixtureBuildError::Construction( + ReplayError::Other( + "pre-state read for 0x00000000000000000000000000000000000000aa: \ + database unavailable" + .into(), + ), )); assert!( - code_fetch.error.as_ref().is_some_and(|m| m.contains("construction failed")), - "code fetch failure is a fixture error: {code_fetch:?}" + construction.error.as_ref().is_some_and(|m| m.contains("construction failed")), + "construction failure is a fixture error: {construction:?}" ); - assert!(code_fetch.skipped.is_none()); + assert!(construction.skipped.is_none()); } /// A pre-decided fixture report is never rewritten by materialization, so a diff --git a/bin/mega-evme/src/replay/fixture.rs b/bin/mega-evme/src/replay/fixture.rs index eeef70a4..19e8aa69 100644 --- a/bin/mega-evme/src/replay/fixture.rs +++ b/bin/mega-evme/src/replay/fixture.rs @@ -39,6 +39,40 @@ use state_test::{ use super::{ReplayError, Result}; +/// Why [`build_draft`] refused to produce a fixture. +/// +/// The two variants carry different consequences, so the distinction is typed +/// rather than recovered from the message: a whole-block sweep always meets some +/// transactions the fixture format cannot express, and those must not fail the +/// run, whereas a failure to construct a draft the caller asked for must. +pub(crate) enum FixtureBuildError { + /// The transaction, spec, or replay is outside what a fixture can express + /// (deposit and set-code transactions, specs with no fixture mapping, a + /// replay that does not reproduce the chain). Reported as a skip. + Unsupported(String), + /// The draft could not be built (pre-state or code read failed). The + /// requested artifact was not produced; reported as an error. + Construction(ReplayError), +} + +impl Display for FixtureBuildError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Unsupported(reason) => f.write_str(reason), + Self::Construction(err) => write!(f, "{err}"), + } + } +} + +impl From for ReplayError { + fn from(err: FixtureBuildError) -> Self { + match err { + FixtureBuildError::Unsupported(reason) => Self::Other(reason), + FixtureBuildError::Construction(err) => err, + } + } +} + /// The on-chain receipt values a dumped fixture is anchored to: a replay that /// does not reproduce all of these did not reproduce the on-chain transaction. pub(crate) struct OnchainAnchor { @@ -176,19 +210,19 @@ pub(crate) fn build_draft( block: &Block, target_tx: &Transaction, inputs: FixtureInputs<'_>, -) -> Result +) -> std::result::Result where DB: DatabaseRef, DB::Error: Display, { let envelope: &OpTxEnvelope = &target_tx.inner.inner; if envelope.ty() == DEPOSIT_TX_TYPE { - return Err(ReplayError::Other( + return Err(FixtureBuildError::Unsupported( "--dump-fixture does not support deposit transactions".to_string(), )); } if envelope.ty() == EIP7702_TX_TYPE { - return Err(ReplayError::Other( + return Err(FixtureBuildError::Unsupported( "--dump-fixture does not support EIP-7702 (set-code) transactions: the \ fixture builder does not serialize the authorization list" .to_string(), @@ -203,14 +237,17 @@ where // Fidelity gate: refuse to dump a fixture that does not match the chain. // See [`check_fidelity`] for the rationale and the dimensions checked. - check_fidelity(inputs.result, &inputs.anchor, chain_id).map_err(ReplayError::Other)?; + // Every rejection it can return is an unsupported replay, not a construction + // failure, so the classification does not depend on which one fired. + check_fidelity(inputs.result, &inputs.anchor, chain_id) + .map_err(FixtureBuildError::Unsupported)?; - let pre = build_pre_state(db, evm_state)?; + let pre = build_pre_state(db, evm_state).map_err(FixtureBuildError::Construction)?; let env = build_env(chain_id, block); let transaction = build_transaction(target_tx)?; let spec_name = SpecName::from_mega_spec(spec); if spec_name == SpecName::Unknown { - return Err(ReplayError::Other(format!( + return Err(FixtureBuildError::Unsupported(format!( "--dump-fixture: spec {spec:?} has no fixture mapping" ))); } @@ -319,6 +356,12 @@ pub(crate) fn finalize_and_write( .map_err(|e| ReplayError::Other(format!("failed to write fixture temp file: {e}")))?; tmp.flush() .map_err(|e| ReplayError::Other(format!("failed to flush fixture temp file: {e}")))?; + // flush() only clears the userspace buffer; the rename below is atomic but + // the contents are not. A benchmark corpus that a crash left holding a + // truncated fixture would fail in a way that looks like a replay bug. + tmp.as_file() + .sync_all() + .map_err(|e| ReplayError::Other(format!("failed to sync fixture temp file: {e}")))?; if overwrite { tmp.persist(path).map_err(|e| { ReplayError::Other(format!( @@ -466,7 +509,9 @@ fn build_env(chain_id: u64, block: &Block) -> Env { } /// Build the EEST `transaction` (single-element index arrays) from the target tx. -fn build_transaction(target_tx: &Transaction) -> Result { +fn build_transaction( + target_tx: &Transaction, +) -> std::result::Result { let sender = target_tx.inner.inner.signer(); let tx: &OpTxEnvelope = &target_tx.inner.inner; let tx_type = tx.ty(); @@ -478,7 +523,7 @@ fn build_transaction(target_tx: &Transaction) -> Result { let (gas_price, max_fee_per_gas) = match tx_type { 0 | 1 => { let gas_price = tx.gas_price().ok_or_else(|| { - ReplayError::Other(format!( + FixtureBuildError::Unsupported(format!( "--dump-fixture: transaction type {tx_type} reports no gas price; \ refusing to record a guessed price in the fixture" )) @@ -507,3 +552,200 @@ fn build_transaction(target_tx: &Transaction) -> Result { max_fee_per_blob_gas: tx.max_fee_per_blob_gas().map(U256::from), }) } + +#[cfg(test)] +mod tests { + use alloy_consensus::transaction::Recovered; + use alloy_primitives::Sealed; + use mega_evm::revm::{ + context::result::{Output, ResultGas, SuccessReason}, + primitives::{StorageKey, StorageValue}, + state::{AccountInfo as RevmAccountInfo, Bytecode}, + }; + use op_alloy_consensus::TxDeposit; + + use super::*; + + fn success_result(gas_used: u64) -> ExecutionResult { + ExecutionResult::Success { + reason: SuccessReason::Stop, + gas: ResultGas::default().with_total_gas_spent(gas_used), + logs: Vec::new(), + output: Output::Call(Bytes::new()), + } + } + + /// A database that fails every read. + /// + /// Any `build_draft` rejection that fires *before* the pre-state closure is + /// read must be reachable with this: if a rejection ever moved behind a + /// database read, the test would surface a `Construction` error instead. + struct UnreadableDb; + + impl DatabaseRef for UnreadableDb { + type Error = crate::common::EvmeError; + + fn basic_ref( + &self, + _: Address, + ) -> std::result::Result, Self::Error> { + Err(unavailable()) + } + + fn code_by_hash_ref(&self, _: B256) -> std::result::Result { + Err(unavailable()) + } + + fn storage_ref( + &self, + _: Address, + _: StorageKey, + ) -> std::result::Result { + Err(unavailable()) + } + + fn block_hash_ref(&self, _: u64) -> std::result::Result { + Err(unavailable()) + } + } + + fn unavailable() -> crate::common::EvmeError { + crate::common::EvmeError::InvalidInput("database unavailable".to_string()) + } + + fn deposit_transaction() -> Transaction { + let envelope = OpTxEnvelope::Deposit(Sealed::new(TxDeposit::default())); + let inner = alloy_rpc_types_eth::Transaction { + inner: Recovered::new_unchecked(envelope, Address::ZERO), + block_hash: None, + block_number: None, + block_timestamp: None, + transaction_index: None, + effective_gas_price: None, + }; + Transaction { inner, deposit_nonce: None, deposit_receipt_version: None } + } + + /// A deposit transaction is an unsupported shape, not a construction failure. + /// + /// Every OP-stack block opens with one, so misclassifying this would make + /// `--block N --dump-fixture-dir` exit non-zero on every block instead of + /// skipping the transaction the fixture format cannot express. The database + /// here fails every read, which proves the rejection is reached without + /// touching state — a `Construction` verdict would mean the check moved. + #[test] + fn test_build_draft_rejects_a_deposit_as_unsupported() { + let result = success_result(21_000); + let anchor = OnchainAnchor { + gas_used: 21_000, + success: true, + logs_root: state_test::utils::log_rlp_hash(&[]), + }; + let err = build_draft( + &UnreadableDb, + &EvmState::default(), + 4326, + MegaSpecId::REX6, + &Block::default(), + &deposit_transaction(), + FixtureInputs { mega_env: MegaEnv::default(), result: &result, anchor }, + ) + .err() + .expect("a deposit cannot be dumped"); + match err { + FixtureBuildError::Unsupported(reason) => { + assert!(reason.contains("deposit"), "reason={reason}"); + } + FixtureBuildError::Construction(err) => { + panic!("a deposit is an unsupported shape, not a construction failure: {err}") + } + } + } + + /// A failing pre-state read is a construction error, not a skip. + /// + /// The counterpart to the deposit case: this rejection means the artifact + /// the caller asked for was not produced, so the run must fail rather than + /// report a skip and exit 0. + #[test] + fn test_build_draft_reports_a_failed_pre_state_read_as_construction() { + let result = success_result(21_000); + let anchor = OnchainAnchor { + gas_used: 21_000, + success: true, + logs_root: state_test::utils::log_rlp_hash(&[]), + }; + // One touched account is enough to force a `basic_ref` during the + // pre-state closure; the transaction itself is a plain legacy call. + let mut evm_state = EvmState::default(); + evm_state.insert(Address::repeat_byte(0x11), Default::default()); + let envelope = OpTxEnvelope::Eip1559(alloy_consensus::Signed::new_unchecked( + alloy_consensus::TxEip1559::default(), + alloy_primitives::Signature::new(U256::ONE, U256::ONE, false), + B256::ZERO, + )); + let inner = alloy_rpc_types_eth::Transaction { + inner: Recovered::new_unchecked(envelope, Address::ZERO), + block_hash: None, + block_number: None, + block_timestamp: None, + transaction_index: None, + effective_gas_price: None, + }; + let tx = Transaction { inner, deposit_nonce: None, deposit_receipt_version: None }; + let err = build_draft( + &UnreadableDb, + &evm_state, + 4326, + MegaSpecId::REX6, + &Block::default(), + &tx, + FixtureInputs { mega_env: MegaEnv::default(), result: &result, anchor }, + ) + .err() + .expect("the pre-state read fails"); + match err { + FixtureBuildError::Construction(err) => { + let message = err.to_string(); + assert!(message.contains("pre-state read"), "message={message}"); + } + FixtureBuildError::Unsupported(reason) => { + panic!("a failed database read is not an unsupported shape: {reason}") + } + } + } + + /// Each of the three fidelity dimensions rejects on its own. + /// + /// [`build_draft`] wraps every rejection at one `map_err` site, so all three + /// are reported as [`FixtureBuildError::Unsupported`] — a whole-block sweep + /// skips a diverging replay rather than failing the run, whichever dimension + /// diverged. Only the gas and status messages share a phrase; a classifier + /// keyed on message text would have had to enumerate the third separately. + #[test] + fn test_check_fidelity_rejects_each_dimension() { + let logs_root = state_test::utils::log_rlp_hash(&[]); + let matching = OnchainAnchor { gas_used: 21_000, success: true, logs_root }; + let result = success_result(21_000); + check_fidelity(&result, &matching, 4326).expect("a faithful replay must pass the gate"); + + let cases = [ + ("gas", OnchainAnchor { gas_used: 42_000, ..matching }), + ("status", OnchainAnchor { success: false, ..matching }), + ("logs root", OnchainAnchor { logs_root: B256::repeat_byte(0xab), ..matching }), + ]; + let mut reasons = Vec::new(); + for (dimension, anchor) in cases { + let Err(reason) = check_fidelity(&result, &anchor, 4326) else { + panic!("a {dimension} divergence must be rejected"); + }; + assert!(!reason.is_empty(), "{dimension} rejection must explain itself"); + reasons.push(reason); + } + assert_eq!( + reasons.iter().collect::>().len(), + 3, + "each dimension explains its own divergence: {reasons:?}" + ); + } +} From eb3e4f7ef1f37c7fe55de4e11a99cf112dacbf5b Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 10:44:22 +0800 Subject: [PATCH 21/64] fix(mega-evme): reject --block 0 and explain an empty block --block 0 collected no targets, so the "no parent block to fork from" rejection raised during replay was reported for zero transactions: the run emitted nothing and exited 0, and the message never reached the user. Reject it as input instead. A block that genuinely holds no transactions is a clean exit, but a silent one is indistinguishable from a run that produced no output for a bad reason, so say why stdout is empty and skip the job rather than forking a parent state no target will read. --- bin/mega-evme/src/replay/cmd.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 4806dd4f..fb1cc57d 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -283,6 +283,16 @@ impl Cmd { fn validate_batch_args(&self) -> Result<()> { const MODE: &str = "batch replay (--tx-file / --block)"; + // Genesis has no transactions and no parent to fork from. Without this + // the run would collect zero targets, emit nothing, and exit 0 — the + // block-0 rejection raised during replay never reaching the user. + if self.block == Some(0) { + return Err(ReplayError::Other( + "--block 0 cannot be replayed: the genesis block has no transactions \ + and no parent block to fork from" + .to_string(), + )); + } if self.dump_fixture.is_some() { return Err(ReplayError::Other(format!( "--dump-fixture is not supported by {MODE}; dump fixtures for a batch \ @@ -1151,6 +1161,19 @@ mod tests { assert!(message.contains("--dump-fixture"), "unexpected rejection: {message}"); } + /// `--block 0` is rejected as input rather than replayed into an empty, + /// silent, exit-0 run. + #[test] + fn test_batch_rejects_block_zero() { + let err = parse(&["--block", "0"]) + .expect("parse") + .validate() + .expect_err("genesis cannot be replayed"); + let message = err.to_string(); + assert!(message.contains("--block 0"), "message={message}"); + assert!(message.contains("genesis"), "message={message}"); + } + #[test] fn test_batch_accepts_the_flags_it_supports() { parse(&["--block", "1", "--json"]).expect("parse").validate().expect("--json is allowed"); From 0ee09b863fecfd212273a8c150b7d2211983e7ad Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 11:34:07 +0800 Subject: [PATCH 22/64] chore(mega-evme): use std file locking instead of fs2 fs2 was last released in 2018 and its repository is archived. It was pulled in for one call and dragged winapi and its two windows-gnu stubs into the lock file. std::fs::File::lock has been stable since 1.89 with the same blocking exclusive advisory semantics and the workspace requires 1.94, so the dependency buys nothing. --- Cargo.lock | 33 ------------------- bin/mega-evme/Cargo.toml | 1 - .../src/common/provider/cache_store.rs | 10 +++--- 3 files changed, 5 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b0aba7a7..67952e73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,16 +2192,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -3120,7 +3110,6 @@ dependencies = [ "alloy-transport-http", "clap", "dirs", - "fs2", "mega-evm", "mega-evme", "mega-state-test", @@ -5737,22 +5726,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.11" @@ -5762,12 +5735,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.62.2" diff --git a/bin/mega-evme/Cargo.toml b/bin/mega-evme/Cargo.toml index d90c939a..1f893ce0 100644 --- a/bin/mega-evme/Cargo.toml +++ b/bin/mega-evme/Cargo.toml @@ -48,7 +48,6 @@ op-alloy-rpc-types.workspace = true # misc clap = { workspace = true, features = ["default", "env"] } dirs.workspace = true -fs2 = { version = "0.4", default-features = false } reqwest = "0.13" serde.workspace = true serde_json.workspace = true diff --git a/bin/mega-evme/src/common/provider/cache_store.rs b/bin/mega-evme/src/common/provider/cache_store.rs index 837df319..4dcc9e85 100644 --- a/bin/mega-evme/src/common/provider/cache_store.rs +++ b/bin/mega-evme/src/common/provider/cache_store.rs @@ -25,7 +25,6 @@ use std::{ }; use alloy_provider::layers::SharedCache; -use fs2::FileExt as _; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; @@ -41,9 +40,9 @@ use crate::{ /// Clean-exit cache persistence handle. /// -/// An `RpcCacheStore` may internally have nothing to persist — non-fork run, -/// `--rpc.cache-size 0`, or `--rpc.no-cache-file`. In any of those cases -/// `persist()` is a no-op. Callers do not and must not branch on whether +/// An `RpcCacheStore` may internally have nothing to persist — a non-fork run, +/// or `--rpc.no-cache-file`. In either case `persist()` is a no-op. Callers do +/// not and must not branch on whether /// a given store is real or no-op; the whole point of this type is a single /// uniform persistence entry point. /// @@ -254,7 +253,8 @@ fn acquire_exclusive_lock(target: &Path) -> std::io::Result { // truncate(false): the sidecar is only a flock target; keep any existing bytes. let file = OpenOptions::new().create(true).read(true).write(true).truncate(false).open(&lock_path)?; - file.lock_exclusive()?; + // Blocking exclusive advisory lock. + file.lock()?; Ok(ExclusiveFileLock { _file: file }) } From 1f6f83f183bfc9ce4de780f5342c76b7276f134e Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 11:34:07 +0800 Subject: [PATCH 23/64] docs(mega-evme): correct the exit-site claim and record the cache module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exit.rs claimed no other code path calls std::process::exit; the panic hook does. The invariant it is really asserting — every command result becomes a status there — still holds, so state that instead. AGENTS.md predated src/cache/, so an agent looking for cache-merge behavior had no pointer to it. --- bin/mega-evme/AGENTS.md | 7 +++++-- bin/mega-evme/src/common/exit.rs | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/mega-evme/AGENTS.md b/bin/mega-evme/AGENTS.md index 7dc0e946..f7485d60 100644 --- a/bin/mega-evme/AGENTS.md +++ b/bin/mega-evme/AGENTS.md @@ -1,7 +1,7 @@ # AGENTS.md ## OVERVIEW -CLI toolbox for direct MegaEVM execution (`run`, `tx`, `replay`) with optional forking, tracing, and state dump workflows. +CLI toolbox for direct MegaEVM execution (`run`, `tx`, `replay`, `cache`) with optional forking, tracing, and state dump workflows. ## STRUCTURE - `src/main.rs`: CLI bootstrap and panic hook. @@ -9,7 +9,8 @@ CLI toolbox for direct MegaEVM execution (`run`, `tx`, `replay`) with optional f - `src/common/`: shared CLI args, state loading, tracing, tx parsing, output printers. - `src/run/`: bytecode execution command. - `src/tx/`: full transaction execution command with raw-tx override support. -- `src/replay/`: RPC-backed historical transaction replay through block executor. +- `src/replay/`: RPC-backed historical transaction replay through block executor, plus the batch driver. +- `src/cache/`: cache-file merge utilities (provider-cache and capture-envelope JSON shapes) backing the `cache merge` subcommand and the lock-protected merge-on-persist. ## KEY PATTERNS - Shared argument groups are flattened from `run` argument structs into sibling commands. @@ -31,3 +32,5 @@ CLI toolbox for direct MegaEVM execution (`run`, `tx`, `replay`) with optional f - Change state-forking or prestate merge semantics: `src/common/state.rs`. - Change replay hardfork/spec selection: `src/replay/{cmd.rs,hardforks.rs}`. - Change receipt/summary formatting: `src/common/outcome.rs` and printer helpers. +- Change cache merge behavior (CLI or merge-on-persist): `src/cache/{mod.rs,merge.rs}`. +- Change process exit classification: `src/common/exit.rs` — the single exit site for command results. diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs index 6cd46326..61ca6f60 100644 --- a/bin/mega-evme/src/common/exit.rs +++ b/bin/mega-evme/src/common/exit.rs @@ -3,8 +3,9 @@ //! Verification pipelines branch on the process status, so every failure the //! CLI can reach maps onto exactly one documented code, and every exit flows //! through this module: the binary hands its top-level result to -//! [`report_command_result`] and returns the code it produces. No other code -//! path calls `std::process::exit`. +//! [`report_command_result`] and returns the code it produces. Every *command +//! result* becomes a status here; the only other exit is the panic hook, which +//! reports an execution error for a failure no command result can describe. //! //! | Code | Class | Meaning | //! | ---- | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | From d9eb30b28d19dd883af6d15083ffbcb87acd8229 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 14:25:09 +0800 Subject: [PATCH 24/64] test(mega-evme): run the batch driver's multi-target tests in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch driver is built for replaying many transactions in one process, but every CI-visible test of it was a single-target run: replaying a whole block needs a capture covering all of the block's transactions, which no committed fixture had, so the twelve tests that do exercise it were #[ignore]d behind an environment variable and never ran. Commit that capture. Two blocks, gzipped to about an eighth of their size, and extracted into a temp dir once per test binary by shelling out to tar — the same shape kona uses for its block fixtures, and it keeps a decompressor out of the dependency tree. The tests now run by default; MEGA_EVME_TEST_ENVELOPE still overrides the capture. This puts five paths under CI that had none: more than one target in a block with per-target commit_index fan-out, whole-block mode, grouping targets across blocks, sweeping targets on both sides of a mid-block abort, and block-global log indexing against transactions that actually emit logs. Two tests asserted the absence of receipts, which the capture now carries, so they assert the positive behavior instead: every target of the block reproduces its on-chain receipt, and a whole-block fixture sweep writes every transaction except the deposit that opens every OP-stack block, which is skipped with its reason rather than failing the run. --- .../tests/fixtures/replay_batch_blocks.tar.gz | Bin 0 -> 112068 bytes bin/mega-evme/tests/replay_batch.rs | 164 ++++++++++++------ 2 files changed, 108 insertions(+), 56 deletions(-) create mode 100644 bin/mega-evme/tests/fixtures/replay_batch_blocks.tar.gz diff --git a/bin/mega-evme/tests/fixtures/replay_batch_blocks.tar.gz b/bin/mega-evme/tests/fixtures/replay_batch_blocks.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..cf87d534ed37f8554d4c6910fb94ffef144429c0 GIT binary patch literal 112068 zcmV)AK*YZviwFQk)^ch91MI!Yt{q95CYIfRAgHvK)`8e6k+OMcpcj=bbyWeS?&=x@ z36+I{jg?eIkWoQKc4jx~J+#+STfK#rdIi0NcKSXub3cBDdwCAYU?h_~#O1xm+}+%U zf3|=4#ovG5zWXNq?(cJYQ-1yT`I}eeA7B5Xr1IL)ARTJ$6vSh%`?}}C-v2b z9O94q`7i$4|MF-5@@GH$pHlhi-~ZKD|I1hF!?vIOSNKu*`6vAF`~PuYmw)`%fBl#1 zPd@R#f2wW@4ow{{o=dtU;U=NOfO6O*}wRgKl}gwU;Tgn-~ZSD z^Y1>uB)@!~`us)u?Vq+(+xP$dk4FdIivPl&9)xWSx1QU(_BkRBV^FRCLAOGdQ`O`o9pZ~{S{5HLL^ZhSwVe7yCSVI;!Q0+#a;_L-Lo%gD(NwCTb(M@SHicGGk}gM`@OMtu)n3il zKi|6iP5R~szUk}#{Hw1Cgx`Nxe)aXg`qkI!7vfi6|NK{9zo_^soxz{qx7R;>^JZMd z2(CZQ2H8`@*sRuv@1kb4-BUv`D*KRr_4Pl0J;w0QKYwzLu7ngiX`*tzmEcWJxwK-- zgM&2ooyORsZOZ!|Blsznt-4T(4#GBPn(JP47h2KyxR{zOF6&}^)53P;<%464SFqwl zz!n)n!iu%AP6`3)>OqQerB|D|g~DiD;I-W5HEo?}LKZx&BVeHw3{LV65N|(P6NYVN zpRMTUycK(a$MtN@9;mLYoUJv?Rsd9MS?Af>@^MsW?;Y2pSZ@Hl#Uk`M2ptj ze0q*;Kw17Twd0CA2|24+H?GOv#$Vhi%rt-zItaknu+#4@Z%*T=rPztNYHa_beq z#sAaku4K2nl-b@G*gtm1`F`=?NTkEHF`57X#)xpQbhx7Nw~8P!Y>Oh!R(`|Xc;8t6 zCbqFyJN_59MvOH)j3w$|)*B7>U9oSx>6yNR9y`@eeRsF6X|SSwW#>~7qs2Y4s&&s? zO&Qybv6&dRx8!kGmN(vFQhnkrO6BeO`If`}tNkt31pY-lI^EgajW@X>&sP*A{d&~i zuiX{IY9#=dr&bHfh!qv%-{CG``{Ax=qF(oi{!5F4anf% z#9I|CA=-CWs+Q~|XKx~Cw<|!=4ymG8(~8WtG$U)X#x@@!fD|C-QdhmJYO>>df(D;M zGof;Rk9;$TPLyL*fE{bDVVPr=qX!2`Nkdo%>;bN^K3F4+SX|$E`GFi^a5tZo zWH#LQh}mKV6@DMqn3Q<@52&;S6MK&poj8IO)etc7UH0x(GZoMVp&&yDtKECAyaGMT zprgSzqm9>1dZm+=y?UF#h}6X4v4(2m-WYiQrHhKh7z!`pk;@I*c0S}}}>P%g(h zO)4XAQmILi9N=1x_Zz(4t+mf_?xx!I!I8S&=mk&8!5{VDDdRVBs`xX?EC$?(L7f+k z=GZ2WT^v?-ayYqDaI`eL(eXvU*8O9*cWQt88|F2}e!3DzA6PX_3eEmO6?`1?9e0m@ z@dS1h^O~a~K673MTnhWnRp2$Y$TF`$`~dVwwXOd)!?6oHoel>74o~YmpN8!B^wZpC z39PlpC-K$K-_Jcc*4W|S;Yn}jlh~ZIC%GPA!vhD!4m}4hGO%Z0nH*e1PtI!xy*{Id zCYcMWc||wh3QMNMInEN_5td9>Cs-Odp@gC`5w!L~-nr)x|1S32QoWwPpWpK-P2kzH zr}bQa{C>;&pH>_S~V+%Ipry5aI|mh}A=$GO$}+unA~d!i+`m=dh?lTz%z*&^&($H-L!tr9u8 zNfXB2Px!W>Z*7NvJHGo4uq&b#^VT(c;rM*oSHVG~3o9~tYY?bf^K#bqe#jS%F ziSN;M*Koyp;UFub#hUpW3JWI#{T;AsPJMwcxIwVIRurP6YaV){HmXT`lLdFX&>$$hlQ@ppA#s?kwHIzGGITP^P$ ze<`O#H_DJQ>~{tYDm$^2rUAb}jOkj&-XB`4Ylxv#M&26-{ZsRsncvL(rsp>|zp43+ zk|4+I@2LqiwXCfp->1?va8kYb~*@{$B#*{SFLO|*0fjnUJpie#EW{*)9l^aCz z)RQj?Qb`BQL2R%XjxP|x-Rz;{)q{y=Gy(Fv3RTzl*njc7eM`z0Gx02KK z0X^0LZm1Y7EnJjNAFk4RlsEXB{BOfq@G$i*=iSP5adRH=ky^ z>@xCy&X_z>$j!!OPdqjP!WWN?ZR6QXjuulM#nmhJf&S#!u~^fFhnobFs%yCxY7<>S z5C}SVC{{@TOSKM#z}EqD5IL#_rh{FZo;ALhrYRMP+cq66!bZ1p?7Ob5Zu4HO?M7>s zY%%1t*vmHG>~G5y(k)3R>2f)@t_@8?GUilct~yw4fTN|k=sko$Rq+^JnF_7EG+tVh zN*BO&l^}d=bK^)#8t>oeoSG8&7-cg3?V}88pVzH9OvyUM_7UblXfz*A{@g5kW@z|B z27?wqba21|wS6#ANxC~MYVwxMLtXhPiHuf=VfV~KC!4EdKJ4mTL+`+)p>3Zam@^sk zG`IKhKC3{aML`|s731Z+s_L!kA?1{8fNh{VR@T}zh;xcHM(1jmQV~v~sSsphsS(45 zKGL#1?0Qyl?Yi8p2j?0eCBH^Z&b4!^&iLEzx&&UZtj6+3gr2lRk33m$rdB%bU{O3{ z#Sz2GddrTZ*Q%Rgf2DQH+-t{)a#(WIpuIF#bUh}2plj+t*NO}G(X}N% zl{(G|Zqp9hAGUqsy>Z$f@cU8+rk!otUM$CRlTX}1r3&@`uwl;9Ij1ZWX=haH{AFqo zIWJ2CB@OJrgr%ymW*BI1rYC?p;5rvbl$i>BDY8Sp_6{Nh04XU*O)(lT6?v-zQ1#|> zF0O`1ezEM+75gEN_N0?cvg5V7)?)^bA|RU~T?5F^wD+ep;G3(ry*u21(6j5|sq4Em zUwsLI5D^FmZ3K-<5CLU`RIFEtj{UoPWkM#T?9YZ1n}^b?Lr50qxtG{2d&ejbkkaJ zlT%B+X`D*nj`z2N+-Gr_*6u-r^=65q4Fj~s@|0xSea2O+(m0(N=L+*=ZPOxxIPGQ} zP||yWxC=&_mEqA-?kPnD<;7KOAod#~XqBB~j7AUr0F2I+;A;}K>Nrj=hBLyTI7!~X z%x?RnEi{=g&a`me1aL3r{G9rhe+j+R+!UCi&c3GS8mO z)w64and8@2B>zr?mvkhzHS#Lb$F!NpAKD}0w_~q9%t5XOWaoP1atG5%@AAkP|4>2Xb+u5r|t=ZWK-3$QnNC7+xYl{k4W zTl{=>SR{z^dDw8&$x^>@oOFJsvZT7Bt$bMcFHYpB0Q%irJ|LVz05v*Hw{E8`?p(8pZE-EleBMemq}|kU3s6=`3NzsGam*8 zW_4RCoIAnCj`z!Po`)GwdV<~!$|1Qwa-0Gj?$dVI%~vn9{mkE|vyZtua_2qo_i5-& z&+)iHuZGp0XTx}{pQ8wAjv}Os6PO`2TQx0RK5j$lx zDGR+~0QN)bUUzKgpren01(0w3ZMj5kr$Icu89o*BoOkM0QA~^f+KHhza-?GnOXt9* z4LiRbY6j0}gPZ54;E8UW98B*GJAMypze%A{%zigXRLa)(Q!!7v@%V_>x{6IYVatG} zeY}pIGvr8mJKH9Hj^A=0i86jaeRHHj=i?xz-8_H8+~xGK@MpVsh0RlLB~9M(MWr+N zA1!H6OXbG83_7{wueacmD*ghO+`=VyhD)mEXK+cibavHG%H_x=GFMmZcY-Y($G*;u z$Qgn-QQ%2O$Il0)N?JWbm5!8FTYbL9v>Xhozt;G}jY;01l*`{@-=`t7`}uo}t~|*j zpnqM^dwGDW8?ZQ_k-X7~b?UqJO4p%(^7rYTgTKiWd=DOm`v;DJ-%BI?bK@Or$TJ%M z0ax(GuG(eG?oc|NanbX%2|%lqR7#|7b91z$_gX+R9Rzx5YzPSoX=({bO=z?&*X*PA zP{~xRJ(re~%y^WiQQZ}K+i_A9Xtch=;Zi|&kS#@!OJ7nb88WW3Ca2z-Z854u2juEy z%gQ!mDhNxD4N?kmvOef=Y26+E$K#U%1YxX1wOz;7eT1SzXH{Vn6(8YC&$-Ha&lGDB z)x{tb1VqT(g>so~0z_N7=37f4Ly~N*D$Lz=Y?r}njfot>qkgYSAE$^r-~8^ocJhG5 z#n*4rn;%|p&dH;x{P67$-=sG$e$)OWz5bimt#0n9fR8qB`R3I>@R`5;yY=@kzj^iQ z+qW#@m(S<%836L;`}Fd4DsNuAdikg6^{+Sha>nGQ3aD{}vW2f;8#xDHl<8i#Zmbb0 zCg7U4$#<(X#3I#vfDRgjx$OGSUe@;8t?zkt1i&0#nMh$0J@NI72Z*Q|r2|cg=!x@$Uc}*GZ4Ki+7X!IA~ zzbNO3*T3H}=6>?R>A%|0@xF7xh10KIV%^G{Kdu$zfVlhehi|?i_R5RVMW-ql)>_d?fVk1GQDoN_*W%;leWdOdv~A{dEK$DX)Tnbzhs8+qpxo}>xfSK zZ>6x%*C?rB1!=znkM&u`EVJ#I{G~Vn0t>ye_v%%Nfo6LeSX%(ws`^0+tNpt~7HClU z;++Tossa4e(wLo!4c_G1uozO88cRtwHC+nC;97Hy0pcRI-9O9)#N{A)$nC^E3`=M5 zVgMmzgeIDlH75GeCJ8$t>cd$MB1{4D$7Ud`;F~oy)(SyO6sQM}ZuxcGQtjG%EKCSl0jiA1*{0~E z_ohX|^B%}v45W~*sY&Q8(fMYQs#<_6t4ArQ)cS#K@D&pN@q2LXcvltK zxO?W962MEDo*h&&s9e5v?2Gf#U>05nS zz)(&1?@;%4K+8fU$X)lK6RqcXwLy_GT_iEf|py}8pK7TLG7>#J}1$l z0J#NOi;t2Fdj)H6CmCdQt{zB++en7*>7F@ag@)IDgzTAoQwFjgBQl}>HSqfa!f2rH zdu+7L85op-WHE5+Pz3wKbG)M6S}DqRZy#tW!0iTQ8e%B)`{E-+qGqi0!Gq^_>=jm^ zK_>-X0rYfC$;B3|;!(k)`u7!1-3oXVbkzh)yAF3O=>*}0L;;<0Q38Z&fYqnaVx~qLtBoutzgQu`mb%o8Lw6z zJ{)^eSpk2WUEz;w{Cn$oMz1kxBPE#V3Z*o83uLU`7#nA+ZRh<%ehYaSSB0o6_z^QM zL9N+|y1mu;pp@~mCrKyDy!AEiI|L%G&^OV?zO=jrwTT+dIwk{v9FDE;c2%6gS#Jrc zBv&^Nxj&Av?g4}&AkVndsH@&W17$tG%f=gbxzf+>0@56V?v}Z72Cl4NIxIQ>kU#JC zSqEe@gkbp4NVyyYYR~6m&UznnGi_ai7(5J?{e4z$>^FqY8~dHK-Q2O>`3)ew@A2$| zFroFr3&kMm?T8EeMR&9Oj@u-TPP8|i-W6T#?jI!&xt?=>xO-HB&J95g*8*eha9N`J zBEB({kH*~<=p<)&1rKrYUN8vocIbGeLTRT^(_%HYxl&Yhj(##wf|RYMFpi*U)j-BN zWG^2ZIHs_`=%hCTx$Q>pMtn`lZCm%6n+h?#muvMi>kb8QchSpc-Bb^rV z)6mYtPxdxK-Hj_xyQa`(^$|R}bQQ2GXttXXZqb^YK^@EGR8LyDf^qC)ID6(2z~D%( zDr6{rH^h|@x@HgiTOf1+s@^k<=l1@-?}0dvn}(YhXbyCk`>6s0okqZ5)dxt9sww?` z!3obmavYTF$uZsg7E&8)2M(lh1Z^MZ7!TdseJYSIiE%afLeH)OjU7t`2~u_o;siFp z^dzn54&lxvsJS3o3G!ymJ-+`g?~a_11>Wk&xzoz0v3Uri;_m^L|ea8x;(Is=q;Fy%e7q zjSa(2^f-<6!G1JI_8*+ahDl=|!;nWQYz;bQ-+C^osazpxC7+TkBaw*(pg90efheG% z(`pEeJ*ljtP0)uZ82soI_U)83`0#s5NkjSA@b}YG(y&v~sLG=n>$}j=_l4j)=xFpa zAgS#7IH^)eLpt8u>WS8FI@%-J;*YW)3|AD7>*nvW56wc)1n)9kmRJgS5606>5j|V2 zx@f9`wQ8$Of%d4A%NnBuZwn%KWy{)yYl-So_e@^=B)Vrx<)i7I`AqlBIxXFESZj)X zM)yp0CJdn;?1$4mv+^HF_sm+|)IC#{2i?=Whwk|>Ht1=$1=gST-{+o8j(20`gZu9v z8{u5JQt8&e<`}G*65oWNTC6`=avq~iZwutPcg{0?6sc2KHmj4xb*A!q8WN)cB<{B5 znf_~8X;%oWkTa!%wREzcw$?bqdzpIY8OUX3f>VHz?-aVTPY%;o-V5CYsEp4!W{TZD z;T6(LpS5W|cC9lrP>4`vPeRrlPtM*91=@J?N?kU!W(;ucQ+$V=E=3)7I-Tt_i12Br z-NN%fn7uAzB@G?1aoyTJ1JvGquWNS^4@4?OvgpbfVH>5R*S-bbN045qVJ6Z3YgFgv zLB<5L>n4TbS8^b`>9S|Lob0mOU%EDnn#}Q}>=^t)NBallhTQ>Q)N(jMI`zkv%TOA@ zb3%VOA*!tonWm<}Pof!|LQhD>=(If#NMKNC9{~xwd;XSSi+i&NO1AZGHC^XtDB*_) zMv+@EiV<=PB0p038*AQB1(Y~raaz)@r0A+jnqk|5?qbS=%UAud3y;6@X*->BAJ4Pxjr6^gVQZ3ko(P+D zV!OAF6I-VJ9jR|S^J)5Zj@xgYT}wHG=7p;(pi66KW9iM!r~EpnHgs1~VYNnkd`i3Z z_iP(yjH({bfsbvcll&mhLofC_A?NKUh^}rYu3M5@o`i?-|LL63t#ouu%;HW~)3F+M zor8LxukEQkFFuW^r0Wq?JPy0 zLr9>8hG-HawCRf}8Rbz(z@mao5o$@AX~kSAJcdb8F}qB%G$aS$L9r|05$XP8{DV6q zneVlpJ~okA~nb4-DajWb8)oG zWBN2a7d^AYy`v5Dls;`H;<}acZ0t3IP1kO&@5Ux5rh6)-Fq8YXExpp>?elLb2_^gp#FdsRhK*~g$Sm3laHU0KXSJ;bNqg25zu+I zL?0?l{xlLK5_Z#e3b?R394H7KuGm(@hnVf-c8GVIbN3f>Hex2D)M5m0Z*yLu)DOsK zTPl=A@wDoDatPn{)HW9hyDZ^q@_${o)U|`UYquxMnxXD9MVIcnZYjE!PgbSGJYjDS zplqXuzU|X^L!LqNBT#aj!2Z~jK*wh&0h1Pz4)~eq&}pVdbY&!3?1L77RGv|cBPF;+ zJt%dc4R4W5&&X7bjya6WHO0831cd6zyH-j9BToVG#M3qS~g0*D_zrRrycVy zeYjP6-=0R2zTM*Tz&eRxbz!0bx6&X_pG7s2vIu%m`{wAh8jVt2`XVIs(r9R&?=}Re zTe9Vz(1F#}sMLa1wJ?+~l;~sVw3S?^WVG*V`Ed0-am0Co@k={^Yv+OcgOU$RfXn8@ zb{5S~e)&YtzOL7bkwKVboc^z)J`Pb4e=>$^y7a7!q z!M=847Jr!5yDg=~NPvDyf9{#OBiNbBXeGZ`vdFLushOl|md!YhJ}<3qS*ZN95>4-M zNlWa#;PAw!iOt%#jpm$F2c{rWQ?8FArRlIIE(p41xe&C0-eb7{eM$7cH_(V<@6Znt z@klxbC}hW=cr-R)G-Qzm0_3y93V|qFM(R8oNuaJG-E6h3fNWk$Wz2Kfdo|)ZBz46~ z1lx)r-WZ1|23>Lm>vF+W>>K3BkR;pIDOb{wVp1zrdL|u9A{NmA>{W#-m5%Ws^N`y5*a ze`A86&LHY$dxok68OJeo9^a5Vm78LN*4R?fG^8S}8Qh;*dvK0tIXTV~R;Wi4R)F*A zV$Tv*pvSa_zz>4vo%gvG)0L^Vh)`=`_CgAhq9}mAD5;3NV(-6uQAR&}w6orXs}Yp8 z;ojgh^Y^flKP=_0mHcZ*Dt*_(7!FoND|E3i5v*{@G26o5B2}R$xZ(sdp0BKL(yl(&e^FKVG1Qm@H z@+L$(qj;A5fg$dW&ljM)6DZ5)0+a&UBc!w@qB9mic_qaD_@6I8c_&cT&ju*@nMo;3 zr3;#Q=M(vN{d@t+JAtx&K0qn6&~k*6Xw%I*XBU6>&ljM)6Da%V1C&E+g`&gCW!n4% zQsoz*yb~zbXg(XL)Xi!!q0+u{+}pc;zCh)@P$@qrs9aWR&`G8ivL6oK?tJ?fsJs&@ z)#n41SXiBz3flx5t@r|!zrRqaKOd+R%<;ysdCyv?#$xwvH;^(-1ND?cFK?EA71bw~ zckdh>c0{2Aa0viEkAqV5nFZm$a)TH`&fpkw3%(K1?cdqqMxrMP6exR#x+S zlZ(6yLQ$4AW-Os+%G4|w;ui|5gNB%$$D&2w0CqYz@2hjn?dp0n4=2(!>dHOf?dSWF zg&0G#DO-`e&5}hvbnsrM(yD4zvP8I74DYFcU0I@T+-dyZLwPiLw!D!r{rYo9LSvPh z>plB>{B5LES+;2C%YaSJCa2_jinTDysVYEto9AA|fC)y`d`4$gXEEVp1x>~5= zHuSu!MTkCWoj}rz%{nLMzyQ55EOJ$u2HmRWQmMBj#rLdV_=j*-GjRaOT1t2q@nV18 zy>o_v{Ubj$Ni{f#az%^8TBBfOfW2k|z@-%^6BeVWttFv-4poEnX8M#Xm3(nsrBI$@ z`QetNs_wOb{y|{H@+SKU?DAi*;*-LPdo8ek5Lkh|RLwHuxcbP3t}j^eNnypk7T`Ys ztf--2#e7IVfwSrttoWp`;$92%Cyy1-tdr`dYo-JG=-#qlu;P=#irW_OKR~P)r-dp? z$vCKwep>hiD?TZ#u%8cB{FMGiU$Ekn!V32}VZ|zMXe7_<-y6Cf`Fz2OPYNsi=Yti) zjlVP7#>aQ#|AG~t6jp@K1}ju*NqHx8`3b@RzhK2Dg%$C$!HNM~F{xI50*~=8Sn)|= zMf$j2nEd<%X)S)*&tq!6y7l$rYC$&#{F8$1Pws^o72fc?7p4WtyYj-!xd!GWXEA~3 zvaG(^nuU`lbz8E}x}r_?b1%$?c+iLO!mMjij!u{=TS!-3{k}M1N(DGe(q(6ANWEuM zt1yk;{He*J9kxG*C`3p~Dy*8;!C@3KgwTYl4<~8*Fo~%6xjRXdRNe$r6if74U7GZw zCZ|n}QfjYUawW3VMK!9`8Ug9}!!#YO%$($dbBtG&+vOc&WA1Vxu*x zZv^OwykcwI4dptLyk0nP161x|uv0V|Gdt`5#mn!0*cTZ)*@}G6GtBojYQMKq_tCR( ze1P|FQgCMSz$v0_U=u(eSka4lJC(_iiRyIJ04G*g%bJ>SDxel%8X}8M3Bp!91)7=r z+~p01jL={zDd`Y{jSLR367+@00Rh~n91{Li71wL-r86C65DOc~m zKJaO6qAp&k)@%doOUZzfhU}APy6jq7g&^Q`)4~TOT=&e2szJGxuWcd1Pr&@25JvhX zdrtujx#U=F*O7xzUIn~P*pzGc!j}#SK-PPh8_+r0>Z7wF)yyL6Jr|d#Cyq?A-DodB zXhp`-y9x=bHkllwJCalnXUml8X?msZrB_<%<=k-~OeQWs_%{6(Z2m9W_kVJ_C`4BK zRnWIr-@kbCyUV*qqRI=p7Qkgh07Pcypp3b>aNMQ=?hMPo6Vh)>Y9R0?FHKY#I> zll|MjyP$MaH3KKWK8i~Idj7d1Cu)px$;dlAImjUivM;evIn^_G1VEcfkQsi|Sdu6L z%>kugkPuab0nAudRO&&7Gfb65)>LaYw$%{`E{*pwXRosb3PCm~8L~7Oo9>PUajM6= zobonD*C$Z_?k^G-`^4uD$SW@~7c#v~Et(c1%LD|F1>sfoA=PA+jvX)66~b1N-Gj0S z7An@!l&HW!%k2P2t3z*EVJ&e#utg|sLlH%A6_U@QbO-skdjpgPI+w1BFh+t1wG^~| z7IU8s=Bk&N3$e6d1bAyN4A21K13^=zU_UKrNjIQ#UEXBH7#KvYdvMyZ)RYbyqyYah zt!N!(nte))%c@C0Pbwf`VC1@CD}4jhXkRQIiI3JNf|PYBD6u}#WrdD> za)5Wc<&@_5BId_r6`}7v_ zvN$V5jC%{Kr~$j}%x4}b4qrb%#?ipzgCDDY(HF+UA*^-R%&a|n@8J2+I&fbZxyDzG zp~!vN)*kDgx2k!}vA%j?PH(QW-YSWWoykwtD09$5EQl6$oF=#X#8x#-teTus%*sw$ zI|Vrcka6f$?0sRs(Q&wWb!JOvy<8bt9{>u(7_XDFXMmyt(cw7E)w6xoRoWqq(28ed z$5JKF-0;w&S?P`#qM2abLx&peG@AO^8lvjP%~rj&R%rmEjjgg3v06cWQ%ilergg6} zA;psPY~{+0Rt619TW70YwYoIOc!S_ITV-0U*2D}&ceK_Pw$@(p&NN%&>Q_7Kz#`|! zs#*2KbVx>LG5amxa)&Q`bPW;Ap@+&NlD z<=s)!Of;}FG1npKH8QI+WFbHMCFPl|DFaf&sA&cwvRZK&{IHkVn${S6D2?R~W~Z2$$M zf-yCvK5L|HJc)FD)YAe5SQQUeJFwB7c#_$8lHBiLwTr}L-J>?gsMv!k`BiB~M?3h9 zJ@FqPgai%AK`m~8y_(#Be`h=XA5{(lpAvfYrF_{cL!5I+R&#KyqWBa~Wd%)Ed;(vY z`({`H5a_zTrNQ+`ski#qtKunlfL=1RR}8v2>N)9C?WFM+e~)Tavknq@qMJ2GSxEvr zg-&KQ7tg9>aaJ+v<4%0vIhRp&@mD=L>+EILkXp4gM+K>k>#QBZSVvt~@&bLfcVbk5 z7<`@*gwTgw8^2jU^m^5TWW|YVS%lmTSnXG3RmcI{OAl?>JK%RNO&M82^kCOUWg5*= z)hw9_shr1Dv{`{&K*)9hdjakbu*^88)e}%|#=BS?XTqv5tpWCwo^`tLmu;)Yw_No&SffQ*dAUYorQN7?GHT+^N*t_` zvRWu_NzN}U*(l6PWYC!7-pUpz24n80DvWQRhj=@-Vs^K&HHtH=w3IamR$k1I*G8RM zVky+A;7(Xd=|{6l{ElU}A?&uS%eqOkf6VtCVDl0#S(*oG5ICH$P;8v{U=>g@OUw$o z4|O3xO{|PsSK$1yv=T$Kxj;qfP_R#MY2BnfT1C@ClKuo~T3cj+cwUA5jH)Y!by$|>;{2#RHJ=>x$=SWe z=cto2;PetQpWq|pbYjOY2W1ztS=Hr7{idA9|Fjl&)-~RCeAO!IgO=?T%&@Ajl||lyq^U*2=2mzCo$Gf^TqrYEBrfxQ6vv zH%TvEeCWsC2klpsXH<%2r5*6KL8DnODvl%#ES`u3C?uHIREaN>dZV36&Teu@T-t99~5<#yiabH2$NM_r^j`E&9Rm9q`Xlk#d&$E52qG$#ej-!V9c~2s*{>4VKvHmPwBIlFj z8obZOL8VM!v9AlgbaIBMu2*ny5ldNP-nVdFEa(x1D67KL@SYJMbaDG)8(!5V2CU>2N(eJocE7+ zBu|QRXq~lyeGOZ+^8pzl$J3k8kAhGk=iJ8b*Gs9^Jqx3E$hM(}Cm?myzkU9Bz8eab z;=4~0TUdRKwTnEuyMT2f@hw=F6V{d4B*6hCr15IhU&lB%yU8ZI4=GlSwK4-n6X4iy zwZ`}^%1$TQe8`dJBqwS;cm<){EGUmYJ}DPOQjaAUgmbyzE%Ld%Ng>slw60Fl`lXz1 zM$J4ecF0$(-{!Wz^~62?O>UjUCcP)Ir1#i>lSN!e@95ummT2>M0hBLn5~fozW`KnA ztc2=DZWFX?&pN5zSvnMGt#=#tvhS4fo-}HKCrbLdkgJSuP!ow&JM<6v4+?XdG;vg` zzhoJ_(97H#e(0N8)?TRyTl2 zXgm;+x|b3yy+Ea5?KE8*6oCqY*R&^WC5=CP^SaQVM)at6NgLznfm+XI?cfx+qlOA06P8hYZ8-{$-@D|vsY{Puum-yvXXcel2@d+-r36PEpU*;QGgBMl$cGL!>RHa6 z7<8`Gt(N$N2tdQs=0Q!RXsXi93uM^qtQ^#b+N+no0qm40-Iz?PUQro0s2Y38O=Xrx zESNNzA_L_i}HGwI`E=-T)ODZN-3#~qQ31J zLar3o&po3TH$|#qOJ4APL21oKX)mQpeaRy)pV*|#izi3%&0w~xVpyugQEJs!7FIQD zRP$|8Q~sM4NT6amzIi*bBoBF;yeHk{8S_Mw=VZqd&pXc`TT%C)LE1hf;a&P(5^-yC zokemyt#eO0^dZ`~HaVSgh9#zH+3K$MbSWRfp|B*uNvla4*oP~tO7xTkyX9L)rQb2b zt{0tDef4pLCClPyC&w8wf?4ynn{K2iz7853rUg=+ti$6xxlG<6jbut!UPyD)F${hq zjvQz0Qy8*>oaZsB3%6vUo~FKwzhVY&_h;)EkLx)9?sa=Ihh>Z9lhibiPp0FO>$F%6 zJBZmhztpo%Gc$#5Ty|9-muF|nY)$K=c~oB?2gc(ReD=ZBz&&oO&Mvji+jYfC&UmBX z8O4wn`_gZD{0vz>pW4XvBu8I48$vJk2pk2?dGzk)X{#c%Ebf8XWXc{q4X25K=hB$C zp1g_fIV0t%V$Tk8bBLkwZ#(}7PVnFK6OcTsrl$j;r{LeQT@}As+qaf!R|`IrK_^JA z26v`BLOp?4E7drQwV?WgC(frAbms-#a?Moi*o!7Qz76Rnj+y%*y?E*7**SQv*(9IM z%E=j(8piKD)`KIPuuW<^t#HICN(Zy{wN*zw4%FZb?Q)}PQiMO-p2VQG$iYz8WdT^6mu-GMi;ebI@X+8Q;q(GBVtL_ zYD)!a>OR|IjLa&u-c#hcY}QQoo^nY%GRM9HJJAo{5qorY!i3Cyj)#^mNn4Yq)QEL2 zlcR>r{hF&B1~i@RV7jJZ>@`jJUp4Iq1X@DysMS5-bnJ^?=N@Lf$!sPfWuIUDTigvJGI8Ob#q7T>EoQ3%KL{vf=m`UA@*_gB5%VOW zEIHcFk<;&jE696>4N-0PaW9Y3xl|C@{IoYsGQF(bPUWXWbkkeL zTzSi|UOJ(NI@0qLX{Sf-NQ*W~1XwwvL(nSH)BUZU^z*cg^kc#-fuz+~3TLAZCzW$1 zoY|TWbQ!QZfOpQ9XP$>y51$(HiL$&RdSB;M0#a71pCOef7@&54t;05jz(x1SNr zZ?9E%W#>(Dp7*Geygcc+TjL&N<(8K6)XL%wmo_boG3@8% z0P7{gNt5S_)8j%9joWOa7&b7yd*JQXudrKE&MjY;)-8Fk>E3@LP9MV)R17PL+MpuH z32pOS|Ju4Wp9#Z04ag{iOGgieC+N`ItSk&8i-BkBwlq_L z-VE62HiaYucBw&8(WM(olsz{}L5c;|Eg-Zg>HXdBe9TuHdoG5Tr5(PPYI#+U6D#&~ z%Odi2+t&y3ViA*LPkS8nxbl$2@Yz#R=|g_nrwH=g)F6p>9!Rl#{Un41`kRc?6(JdDjN zH%^q#JQ3etlMb;ysYyr1YdoV#M`$@ilZLEmcpR9yDIs(wFzd+{-!Zn7t5Pm?tVK?k zSukVSg*1t^ONwT6Ds{43;$5SDzhDJMNKVV2d z(^JcYo0cBZQ{#vr5VuASIW$wl`~WMHXhz}Ne0oAV4ZYr_otCAYMpqxwPC?Btv{NWy z4``=x_z&KvoyO*N?UZl1t)0f{iLz^_u}q8eN6~@6@a)~1Lo)f+dBU`n9{mKiQ*}r} z_FO92ZR~PVgNII)(O)P>-R0a%<`C61Y(2!QCF9oKx1fQ1Si3Qgo(BOZ#t>R@uKR*2S8FRm!% z?L$7D_?-y(#g}b;RnPIZHn}YRQ+q0zXAx@Ji$8=7fPFT5x_4-iZfYAJ=nd1qSH64(u~#&-TP6qxB3+Th#vAPIC$GbW;W;E_Odzp^giHiU$5kkr`BXx zS6VE$Y?e2Be^)MIPO{+LFy25BFJz<4Y99DQy(xgi_L z8J#vYPo)`IdDJ|?W%L79iCxCp;>W=j1kaeyeS4WRao5fvq|zrl42xtf%j;q8FHfC# z{_juM#(Iovu{^?e`9!vlNPL%m6m+R_ZCX0{HxDHhUB_vGqupIVV)E=|cuo!fJTYRo z#>pwz=a}Q$?R)NqhlRWLu-N?pFV3Aqn6?NdlpQ0;PuL=2NQ`J(0(Rl2pygBQfXA7c4( ziC)e;5Bjy|!S40>&Nh)5|EHvinBh&kh@#H=j9o-&Mfxyo7x-IkTKhcl1fCBqJ&L8F z5B>bo%L2ls+UM%=@77%1``dq%eA@n_m^bgac`Ewu?4A@krnt`Ap?gyV2^H$J^|c$w zxw(1+@UA*C0g9`p8XucM2O<`tBQ$bAM99L^LvVRgD5c6~2U3a|1D>@5mCX+HPJR*c zPS1#G73nhub+aMaVS7}vwMT47O69ouI(*nQTUMzSYb=ozi8iTD9d+()l27?MoP0Q* zvnJW)>!4ILk64p{58~XKq~s&kB&B7#*P5iXsGBuuyyd1fNvYnC)+BtF*Iq1iOk2;} zDf!Z(z<9E?({`-OX=k+$V`ry#CoS3|s%u_z;`ZvR2R%)I68fRl*E7QXxcbahF?3yV z{~ojfJu6NP_icAklEE%WAI(LtrS9Gz(XH4~{Dg_2Lkd*pejQ6po31|DTxMvyh|9mJJxL2Ji!SURo(n-CPk1V1JjI#JbBJsTkS4w;pevUx8J+`Zb)s)u+2?9 z%_%#5B-J?Y95M@C2>jEYZ0gD{ixcpDdSK9jX!FnwOS6zSB`kVIE*XpXaqU^K0i9PxXdv2dK9itg%f43taukRa~Xr`S0mYjRj9s|XbGdS!D#EyfFpSt*z z^=DLt%$p7%4`=Y(tOp%b=PRegmfPb*=jJIhKr5X^lXG7IvL|1x!(Phiw>UE=KuGZe zqi$wSP%SU48Mf155z6BvT5p~gmytS~|JN7+w4k#nohY}cj5&tB>zacajJO@reyyiJ z#r~FX{+4s8MKgu#lRGr;I`@j8_Xc?Q1miKQ*a6FCYH{Oa2im4y5e{eaWz^~_do-yQ+B^fD-v%!nnNX%xaf`W`S7%Pik>Ns| z#p;8k582IaG8_I*jy{+xQzZ9_I5SQ~#%h2c?)kRtj1|UQ3ch)J9kD;IHR1tKdU{e` z&5QT6>DM|}g&sKdv?|*y^psXd-l_dw+7-*`-ltEX@6OyJ;)v57lwga?@K^MS?@z>L zH~unZjU&xBt~Q>xmCOB|-V2nSQ7C@(+CY<+sb^PxzzrLHs|lmEY1hp<4F|{GjsdKu zV_mM|kuMPsQ46pSs24;QJmAy&ba@6R_hQt6lQSr1@`9K%Sw(8jQx4O^Jt=|6NXyc4 zJFZ+@St(QP&BK&kgN*52UbD<DUPfco6(JGuqMY_IQK<$ zF&Q&C!QQxf?~B->*oMW|&zgklNyXG6lbI`Oci)x4K_{wNDHv{`J=WY~s5KUI-2`ex zt0P#t(=Wp=V~@p-1|LpyD`D8phW&c!66wt;54bfNGskEyp+hk=DL%x9dz*&a7T2Aw zvcdE1A!@H&bRtV6{vEuF7N_PNy=uY-WUX2Q4fg@^f@gQoM`j$;H{;+T$b&Evx?vg) z9mB4TU5}5=5&IVA-}`I5-WP-1!(=wnecrd+J08^r6*sh9z(wT5&b0UbIszUZl+>?6~Y?m0;f zDc?m&jV9kg$qXgWaWU;AuW?C@qmr2P#+ zF?>y1IZKMN`g5V+5Bym}3JpPnHntl^4netk*>G_X0nmA8fA47%s9dYrPo5 z&dmkxng=SR%F$kFUQ`Zp+aWo*E|(a**YnCG;E3NtGA#^{xu+t#3-seQkCR%>oR8xY z8(Zuyr=lBoqiwE4q^mnvltxsUTD0#iCTg1M!7Ieu0U}91yJP7?{l>@NXs^7!Ibv}5 zzh!m_^Oia6G*%^@uf?1i(WjWZ(+MpEFL+&Yc^y(8s{^8)8lI>5cf6fSTbSV268LrL zLn$<|YTGC34mCC2fR$?xA_a6?J3IyXWPYh-660TT%Qfu=klLFuXw$sBzF!7sh+fu> z71Z1V*Finj^1~tw<`IUQ5O!M1u3r81RCD3jpZqx{ zzx;VgIvt2`FQDme{tMd7Oh;U=cE={|(>?sFz*vGjC%bz^gxfXOU;Aa#pJ{ukK6wr+ zMlGZfU`}N*+UDtQENw1RDd4u3sRz!To|8af3cKS#WuS9ou0lC3rO^(XP8&)V%xJ z#xit5)(FkluEShlEN7|WUU-E-q)cuwCg zSlJw7;6IR%P)>QfmnhY~kY#1FN%t}|*ZuDs6s!B|E!@bnN58+HbM{=lJ3401s<7C% zP7O}R(&>5b#L2#x?L2r3ufkAjW6utPz|IrSj31UttDjZg$R8c?kBiQB-9+tF(R!{B zVe-uUqQBj}>=H0DZss@F%QAX$4@Erps)LYkn{yeNdZ|B+7jSmGhjduW!2km0r(}&s zI=}oa$f%Ri9agiL_v?z}A+2m;DB__dclLUJ-s-}d`FA6;>qN~K;%%Xmj#1I8BuCqh zyT?Zb@)$NO5gw?tOxB~BX{jM$cgUHT>7PzCVJav7d@`q>Sa)(rCSlB}z=7bwE9!NT z4!}s(Gr0?a;TG0 z!o4Xgo)1Ix@gnY3fHP zKDmwJsn|TG&H`bzC?`uRf+;Z{7$r;hKzD}aC#~?v)5s2}MJZd=diIE3W}7{p#ZrvY z#Fv8%gSGg>|4kCGFx+DS2!pJ{`61?m;=Oj%wb?u7p>wyVu_5Llt&WlNDSAETa5rWy zrjxA0$Mst~+ne{3X;kYfnXq}{I=RG6+TO*RcY><2^~|NYKtbNA8@7GLMfz;@;c(2l zb3B%mHiGzQy2{acJfp;P({SNCR~c|8E89BCj`_k&!H8yay!==>)085WD?w9 ze^+p*+3>`h5j-5qF;}=iWfo_VE7?K=QdF+OBg!-CkftdynvjGWWlmaef{vS00F^X~j)A(-}M&jDz_9Eww{$;Q71l1Qnk4aCf)&;D>udAJ@zJ3@sUL+*&QCW>u5;2 zOfM(@0H*7X3){4$mXocG$RC|G)6ENS>!DVD`+9svN5Q|fqz)+6J*dgH)!QzW-R=95 zdHa@Y{Y&H2Q|miN9;qZ=L^l2`bHgq9$d-T8G8f~u1(lKk4*=gOq|U%ptwX8q0Fm$l zBPPl6<#vg$W-CYAnm6rSKl8DQ=ensDc>y&w76?P*bnrz0_haz2)!J(}y)kLEHp34JEMIe3ZkOAYj%0UEo{3i);YJ)7H*G_|ZYrTYr)+ZLh zV<_iJ0APvskPDzI{}37406-8!aUFyzi!T^60Q}8g8#J!HH70+q$tT|?!H|UIBcSrM zJg*Q9RTZW*Md0wftWJ=rTd6V}6ESqJe&dWxaP2Y5!^eDp09CjT^)MY?CtfC2@@tEj~| z>0t*Fl$fUW$LHC(H#Q?HVS-ykdD@k96b#B61&{!M0zmeXbUVac>3(7RfVv7hw2kWm zz9qvD-T=Rm-k@XS+%KwhOjzAS}T7umlh+cy2&OK~qo7!Yd5O zig5N2bVYE>^9#rgSQ@*#UBC`xIk|J8FR%882%9#)(zMXW0AX( z_@a@_>d9OJ5WK>JHd5&|LVUn70M{d`Z#B3`(yuuRXI4#r1PmrH(@>&h2YUyQ;6IEa zVkZNaNx1})<)E%jAWumL_Y5`8fb}ROF%&mHOVS+PlQcWx;*J8d^)%3TvvQro@f zxQsi>y<@I2g>Ac?6xPS!o3-R6R_EhogpV|q>=l9Cne5!@=6SBmBbYHf7NlxC_ul`! zm$S%H0%9rAIh^#nb4=uzJ7@^2`n(b@%8}F=%=5AJnwrc^=Q&R|XDlTN6o1dJTmzKy|(@9ljxjz?@Q78#U!6DF57OALCmNvOhTs5E z!wY(6=>C&KV2;Cw<{Emea5)x)2=dRk0YzpvY2wX|;)c7o(K?l+q|}LtT{* zk|kKK5WE_NQFoqHtc}brRMP1VlohNo=C1 z9g0>IRqwE<|1b_Z;y?9t&3O-2TaRzj?%#V))`x?Gi%(xZ)9XclXR03K{4#ZaJ!viJI#lxv zDdurASD7|fc~5w$X8zM!=a^PcI$3I$=YKaslLaciEMN3}9c*p)?h(Luf)5ugt@Qam z*Z^Dme(iJ)2qqsN+<;`?ElsUebbCBpo*0;8H99X9GLiJDj2E?HStZ%^glx9}Iz;Ll zH)}XoRlQXj{`F_`r@*UE)A`+)+Vt)Ew8dV>$DSEZyW483^JT6tF0<2Z^ZWV0CsK`W z8lDxIQvXAxg8%UJAL}ch2di2RVuAfAM5nGOoHQi<_kV!zvsn--aJ!*m4F<68!AzIH z8F$}1w5qqHx&a@0b-f13icDEv3Q*nr9ke#QPw#)Q@2k^5B&3!KE4B5$!(Z|5|K-^} z3q~Dcky7(BA{3l>1FZw+`yW;O`=KAQ9G#u2IuDHd>$gDtF9otXA|HOW{&1SDDzmo; zw&#&w_uPN{GLf&WI$;StBL;EINBQH@zq#RhA|F|GgRU3c&ewfZ{1vE||6XRCz2WdK z^=a&XYWlZT2YNpIwY9^y{ipQuQd^(z_UxNee?P_gZ%F%h{Mqip!byw7q*X8KDVt0B zUO$D+pK4l@TE$=Au}p%*It`Q*t$4>lNG6I9gn9m zZfviQA@)6lppy}VA|t`z3YM6tFi2HW)vrFAt~+(rS@6BJ5WUhhSoGh@V!OqE@X{Sz zGf^EkTm8HcBi4$^N>^|1O*OISO3zLW9h!vOKG5pL3xz^e@WAeSJo7H9@OY=YFudc2 zP~yUuqG9vkit=*NPsF7~8`nd*R*n+f=2nK*PMoAkpEuvc9J4YT^Up3Du2Yf<8NF5F z^HLkvx?y2ese74H@(M7hsmd)cG~_2$a%%wtNJ*14 z%eiKlxmFi@eqXfA1&j#;lkuA3{HF$!m2JF6MHl5<;s!NT7Ld8&M8jq+afbgRmH#$$ zmVkQCeaV@&32#}w?B2-}B|kxk%W3AIByv8`@coytfgl|+FjKIxeg+mI#lSI&mObT3 zc_ggv30CwP%p_d3+k!HCv^+b!qB^yo@5Qlin3Ib$XA}hJ5W9(^LOhjvK|9Hcv|Hmy zI_q&ldSDt%ggjM*YT^Cj%VRzH4=M8d3nZ`c0+Hz*^N>1_&v;g%Ao)zSqNZ~ZVCo2Y zl9j8p7rB4`!h3KR6q9Gb_}Qj8qS-SzuzN^Jx2O5lz8}e9B|&O7no6 z^`*uaf~b)Sf?amP(&y9Xn#AwS)T7K+hZ60`GMXjzu8^}NC&NNd*r!HiLWO1OOZ>D+ zayKAeBuUKWO1NdA?iu`EuX_jagAd=5;%#)}_equ77TeoxdaR;;lBUfhvx2+K1XCvk zvt6w#824mmd`Tmi_RuXDnRu0=+5x@A#+C5i%<{keAGyLaQrQ6_@Wjn1YPvv@Ysa=)_pMuvb0211;Bo)G+d6v_sp|B@I9;s)rnrpS)kBd_-^*d_ zNL%Akvg3K0)ARl^1eP|58IQ*98IXtc!@ldXh(kc$-Leai`5kIGsVI91(+1gZ1D9Me zRGOYoEU;@}!cV3<4SwzSCt(tCDG%O6#tj(R5ddZ>rB<`H2(ZvOrP1r*VSLl7N+PHK zihAXJzYyb(*4%#P?{$B=`ttNQ#O9ZFj+pM+Hf>qB2VvcnGJmZ)tRyH`udo>a_q2clbfcA zPioCPz+IL@%m=z&rWhif(Vo;i(=oD}GX8$x1=TWj%JyTDjD6{H@N2-ES zwR@T&m^Svc0%P?kCq_u<<1~blDUs58TVf%4RaUZn%6)3d2#aXSfC+rBh6A&xiWsk@ z{Y-fXWrO7X63l?uYd?idQw*462v}K745|d`(@8l58HXk?|P7!%kytaN33{m6<+ONkYYsIlWz?Ar!YFq! zUgllRw&ogG5*2cByn^kdAdpp^wFy=L!9e3p0BdPmzy-ogYiC4_3uDq{VEePqftaAi zMeP1ik9^NcRA>N^>S$K@{$3nM#flTj1?bH9lm*Ep+CS3z&By;LN&)Z)i;VI?TZ+Jm zRynNx^+I(6k_U>Wu=zq9X*~?%nlS=SZU3sxGK-2Mo~D>;SgI*j)KQTmQvBwhcNE8E3M&X_aQu%y9%G3@u{`Sg8q~rR7feBnd*;pWI z=DL{T9|-9Hu>FLPccGFhlj*{=4IH!qNb)d3@zOSfBF(K6uzv|w8w{yd zd&UBF9sm-mtt=5iL#ch7SjLPxFGu^ZC))tEUprMVwpFgJ!&bnFw$}lw-|E6 zJ7~+!6G_{$8Jr1hDXMLSUR?^<4}@V+VAt0OR)Y1OL#cF&Zd3zPG1DTCP6hAyyD0kZ z!!_M@GYzM?P&+4}#@KSp8!BhgewNQ09HLIJRck|R{_Tro12PYy5dMRVw?XLpwnL-C~3J8MWAgp z{$7n8Q{I0-b47p$rkunf?WRJs+pL@o?kKFH3PbYt&V(O?C|1?kg z+rol256gY5WK1ip;?Q(ZqzftMYO;ahHi2JdN&qZkiJnAoyTD0E2pU4G)i)^2;=I48 z{=pg-AOQ{j;DF*8=%SCll;qOAA2%EE4K^RYW_L2FB{l~5Sg_HM%N4A}_>Vq(FVFklqQchzsXQyq+R!|;jH z4Hn}j9EUyj0l{Qlg7%)E+%UDIxe%fLqnag94y&*))M7x9+)*I;dT{ffu53ckdEOZ$ zV*oZbe%~)`dA`4O;M6bXk#%*yU(V>icdu=G`UE;Vx6FfIA6&h<5%+_~)vEyqFXxWe zIMUU0PC>4Z`+9%sAX>1IX*kOW@KMw?h11;?3K%QBXa}i(NC0p|v$8FITT+>;Be36VDpkX6XQ$y0O!$G>h=t2VxN{#Ij zI@)4%L8Z@F?s4bDWM*{C)%XxMXROnUB!m56#vn(#!U7GgN&$%E?zar93frdhbqJ=R zjBGpnLAYXin}U*0qrf1KxcY^HIPu!QX;8CaIEd>Po#=;ElkOEj4HVBq!@-)uQn9+U z2l6>$T#s%8kKgKGKSq>Y3pl-B9YpPC3wDe#)WLj1f~x>CKs<`MvL)q|<JYOU=~i>SNB4DAkCVzK`OETrKj^zq5O4V0i`Omw6w91ji(j11X?>G^ z-)(UqRqxZi+6|od{oIu8{l2`?kS{UC_gRPeP7kCA{0Si}{=F*O^WD3S{+H!yuk9to zjxPyywy*V>l4JBv)Xv>4)Zxe9u{sH$;%9+d~KiklYnFrKogaz1ZKK2*hcK zih=!8U_VK(6V%1=tURn7B3ua*zD zbyL_s%utz9L;`-4TFXb3t}~u^SJgE$I-!}2cDasnRAi!uOb5sBRY4<|-reX@$0P9f znd9l+2H#)HF+Mi>qVqP8Y|(hFC!#uHdeR@=R!0zh6naVP(2=QP_KI%F0!G*+Z`a{h zGu6_L3~|8iSKDAkZ&nlQ1ahjy|F-^~DSg#@lB()Fn+~?MV2Kh{#|nxtbJ05?HVC!+fGmTjLp1xXGuPsOwdw5{VfIYL$S4T{rsmgwWJy)Vh*Z~ z|M}$~D2A@Jj2}Y#gR`%dXZ_$)FZ!jhemEDP_xbM@xYjd%DD7|_VDfDeZ7+0d{h_#F zRNUChw{g^65Up?VDuf@9dZ%3LJKy@@*hS2J*?G~^e_?+GW)Od1Vx0Ll4!`uH@JqP| z-F-L?U&yt*@vR?(Tg2bp?f3o{=HM4BH#}PR@tLF_#q-M*wf+u!{wfEsfpHSd&I`Ul z;iDuwOeP#x;W3tIbXRCXSW6=VZWhB_(^o`r3?68GO^32j6JDuR24$e1N(_c{@Ijza3~@qGi8eU3u~0j%<#HAsqiyie^g zz8`3?b7EPa-9#ht7A+f_0dnRo8TFu5IiROlicKoMnUp)ayx+vzi0hJP812JsF4PO-zH8Z3i0bd0XZIi;C4qr6^7;+;8 z8ec7e!wQ7S`oqkutj zl;FEob-E1NK`}MM1ffrDw8k@2l~G+Ph3D`@)10OmQ=--LOl|tq_pMdZ)Df$?g0z~z zo;n8+_VUS)a}OOPHoJRzwLCadgY<=HgTps67y7~g;{5y~lxQUSg7B()4U(&~Yj7cN z%v%+e@S<}*7=$CxHp=Z!!qKn*O!eSMuB_;<2qck3RT4vP&N$YJN}JOe^ps%AjZ8zt zd}!9v+T!s0Z-xNE%w2c>Taqi4o&~sM$WqZ|wls`O z_4C0YC16AmC16tT^(4f249kin%_AD}kcf794dunot@^qvMbeN33l zaf6h@y!W&q5ltC;|1+@B^WgR~>9lU&=EAypP}{6|ds07jS$PkAoxNd%&63l^gLUN7 zL+IJG9`PkJW~ikU>t9-VpcXggYubfGi#fFO;e?iJnT@pVKC+JGoMa0OxsP36|6&iw z_qM<7;)^XW{z6ot3kyJQ|6RTMIi^0OTRh-ws2$$QKI+H>)&2_6h*#uK2TdAzxmyX^ zi(@p(cbcX#8=ly8`-3sQNOFKQSxl&(^AvLXN10VPbI>8w_P!Q{Y7~SrjpbBi9f~!Z zS=kQ9ZEWxGLjximxHcN&ir2J0N+5ge*}o&t%f@N)1G1ZmKR|Bz6GT|h{iG9~?=tv_ zpu5L^Jalpd|6C88Z+PMMez^8Pnt9feF80V?e#VzJ($gIOs&`()Y4D%zFAa2G$skv| zd3B*~)j#eUnp}6yr5d!B6s6?lXHaXh!L@ zrFF_*)Ikth219RAg@ggDJZv+Jhqag+<{(DygSWInYL#@z8!g#1|EQwso_nP4Y-J_i zdYY|nzRd;8t)J^r?+3HW>c?CHKScbP!p*IEbkr&OqNWwLPC?9cc zeecebKx2>NtBX>H8PD{V597TcZIGyLLi8Mpu$DG4UvVpV;m4AeU(30BTdF`~zJ@gxEOa1UJXSL z!ZU1z^_zNAy#_(6YeN^b-xou=+6zSRh2tDA%62*>ABk8EgmdPr4b<-(HTMt9&6N7^ zRLyxHk2y(9U=3oM@#+D#XQ@bK|A<3;+=&O7Qz$HOH*Sv z)dX+F?f)238^nR|U=JYIyPv_Oz?f@}In`7$qCucrswHuBD*x#lj=P2FUX)-=D6*RR)DFpLKL#C>&OA-$@UaivMF>Q%#BO)?82br;4^Q^QZG`%na`=-YvEft*WJ zp{E<5kBH3K;=1lY30JVRat;Wff9kD~Us|7!jA7EbyN3=MbsC$^kvJ6J)P&8R%C)Q| zfW?o>4>K2quY7i7i@8G`e_EO;p*8~6YJE4eyO>)WTBH8hAHJ+6WjWiB5JARkAFOg> zK$}2t{D^Kly=O)bwuYjLZZnhr}vgNQdBEf_T4MyaZZ z9_*fG)&bULZ!?2*sv-A%tXAt*@|~s4ui&NC6(qn z?*uZ_@)h;~ilvnMP|u;tjZ6!~BTE{Kt9%dPM=w02eIMa;LhPRi+0jgv=Td6Y^!3cv z4y=@>;4jMgq(M#U3YwdvWvkHT3mHxAy_0*qhs;LOq-{veG}6ocLdKsPoTG5SQ>Ng^ z@Tb@!nUb3v=}v{N7><$+$c0=x1t%3Q?qI;5Fck)XP6m0ldw~H;2i$4v2B_7O*Stwh zskLngNfZ+-=~W7(=mKE4Qm$BMnmO*HYf#j(``XlUUX2XFrh!4d{vRjaV3a50aQ&>8 zfhEu~+$JLZZlG#yjrA#Ilx7yNAUa^&Fsw1oSFpT{>k=SGN=C*(QxFG*yuSl__VUf3I;ola z92thqZW3*i$6~HxxArOBl#>Xtzs{$+@6(oF|DVf?tIFouOmoM$LbcpDk~TMc%5;l? z+9jq%gl(H{6?=v+gclRjbB0LGjnCM#)F!tMRp{ZKv%PB7pSa*6!_0US5T-I4#X((B zs$gJh4xu@|0F*y;T)!sr$Jj zPgNMsfS1C*b|j!Q3jLcP83j7XEVCOf;cg2bQt{Sc*~PhlWtdP|1F^LGBqyJ3O9<0i z1R}Hi7C=2rt1fQRnV6~$!v0TvH_>c!;mjoB>H`b!zUw&ii5ZcGp$XeiaT;90A{d?N zZ;Lz8CS8{hiRLH5VV7EJ+qLN12cE+`-X~CoqTlx0+O_e2AD|9a7$L%u-`RL>>C`9= z+(U45y9Uu>v&Y(~o87wyq1I)}R4`vV0*E(g&K`CPm@g*_vT9YIu;4O>zYY=&7DJwn zusdDasc`GD&*Ju0sKe;!(Bam$h6HRogKasS>FjC?_Uc$;?`MDZ=Or)=@FHs8j}7p> zJPG%M&XTh-jn6}u56lGg%j<>9g~d*BFK)1~RT9zPA(fqA7~w>#y_4;ONVQ6Kx#JYN z`w{g`E#B<<-x}><#Y5RG)(IL zS8>fN#RrNg#sbpMO}jDkvOi8VJsvsp-8KXW~Bf=#EE zH(=}mLsnOxLG-<@&32iG@@9?TYt!4`u>t(B{by77@!wI-8K>x=jY3fGw4-Qk$FSn9 zZm*9laU<4YAtLMxR!bqN{M@|rn}G}%^_5M%BeYcJ_oQtIsZCZBn{cZXQM6kWmUY%` zc}Kh37H3Pfy$ZMa$JUvupTPWafU4G(9P!`h>uV}E)-H@-K-8;o+C#=Jt$6oMK{Ayp*c!8Dh^6cTgl?Xcjp?XbE+zaaPg0OF3b)UNYE*&P`0NpaOI z9GZZ)(A9TsRykhdGAlSp1B))03$B*k`sm@QE0R~tO_G*H`kK#usWe`;A~^WB`xK9)YIKj?`>Tu*AQ|2U0qN&XZtVqZ%?ZY zI2kx8#aR6?JT6Pn6}*{WG%l-8>GSF`>>AN4x8T`H7+)*k@53fN-!TZ31@Zmw8Ff*o zgdszZnCY#);C1B=*ai!3o&Xx97B<+kyCK`4g|H!P*0YE!cfS!#rfjQ*iAmn; zw!M-zaIB7gx950sJp#(dk0UpV3x^s*W{m|E}00Ii{BpG9%*oyjze?0+YIhBH1 zE~K-E*3=Wo!m}xd{8892Lg+_P9GDP}^CCF;yklemED0cI(}I^N82O&UK`M<>1Mh$) z2ieO5qDdt{(f+_I{K2pyHAxTIAt(EIHV?AX?a#IOaOSy_&MbAJY&kls8eBa)s;qCq za6})0Qq-GLaaJp`n>VlriL_m?%rmCQUHC9G_;A^ z#E&%`$yyf;nF2JETe+2rljoRE7}Rf}x-O`!ylGfn>L`ANJHK}@%{`Pgq;->;k9$UcSSR9E&s{a zzg^z4+1VoKplM zOEvte1=_f))lh|i;b0`a<-5V4GW_+H?Rqr$FuQb&VdXs&I#F>UE-)V6b133$gACAstCHrs^Z+=gXPZj)Kzu+ z`}FP!OrrA5Hk?MRjdg;eS}fX{JlUmg7txZ68`iR4yFXEhnUxk;jo6BV?4Gqb4~(16 zmGw-Tj$@OVnaXqq*FX*U7;lV0!AlOEG<3VLj&~q0r}Eg|Nz={9(3>Tqxy=?tCrG^2 z2;=N!IY+OFF!w7tjGRuN8W@;0d|NdAdnaTkV2s3$M3kEVkIkgX*M@H{!-9myzcn3Kb)HfupH%X|=G=VJ5S)ZV{)zFU?fUqd~z`*`*o& zaG+2Hhc=igP4AcO@evP8_?B{tqVFDK@J>hbu{mk0%EO85GqhUBr%cF`%xkb3!DYr( z+tzKc(93XL;a{miiP0NiEYuA#q}ihyeXbo)?J+HDIWoJ!p)d`qAZF@G*v`ffsJ3%* zxtc)gHIi%9tapEPUVcWsQyY zp#4zHw>VU-B->_0cF5g;1FdNSNHzve>)TjDj+_{t7d$v|Cp4SH!YPBLks}y%UOCyO<%3o;S$;9wT$Hu{D)O3 zW8FK=DZiReesVemyF=IM_a@x}Y50aWGV?@{GXzkTvvsQ^o70Y2FFhSv8b{e{7kf=w z$MNVj@S*%G(@d)scZ_vZ|E+pJ@GR{iW;zd?@WqE6x%C=H{AiC+N2wEIgwaYmDU;5_ zNNQ2|!r}KBDbo!c4s(McQOm=@^cI0-Ij(1&r!o69yTWHS`m|L9o`bQyKYxMED06Sx zPcMftU6Y-;^=xUQF|BY^F%IxE?eW5Tukqq=!kSC>BV7tzZ@MH4Qzo&4+|Z=(y!y1v zM%OI;H;!5%WernoJC2e#$oS@0_sqUWTfP%y&F3eT$@LM!y+0Z!tD{m(=5weoP37eL zaDUoH-0Qk=j`vOCU)6+cM(?IXX)$*DI0I4j)q7fr2N2plc4CVD7!nztr@=Yd@CUEk znf^H8r|qw9?1z|dIw8NUF-psVZX(NBU`Q0WhI4iTTo2B%p4Ssb5$ zdv;f|Lr&J#`c&l()GG!4%Q1mwOn$aeMxK$-$ZOUW4?fqwpG+#23IXPS4$1-AUZv zlUJUFC>p2(g1&p>qimnu>vVD-@69glrT9YyT}!WN8I`P)0hBleU7UB?)3gO7Obf<1>K1czA=OD|IF)dlZ)x@jg?cky@0V>XX^!>gGFV67o{=N)Ff4sJee0L z=fmhwa7)LR2-mvq-A{NSf@nV*nbs)yGhX zcPYI*y=Z+WUNg~3u*gpA@nwwF?>j~%(eVb=$1W8k>Xm(-C$y5B7dl21+UgW<4xejh zBbHa&0DbbLZC03AZaczPX|}anCDU545MW4Tg0srCQ=ppguz6&~+HC9^iV3);nB;=I zGOLzX#mn#IitA9Z8P`9}{nWNkn>BJk4ZwB*4Se#FT&D%L3J>+Du+Oe~BG~U_cizLd zv)*?7ioO@d|561NnZJL44*r9Y0z07NtL=Hjqd!-Sp?HcRx6)GMq8TR?t z1Zs%Op9;2ECYWjp?AcUk2}(p+WvN3*rGAcPkwqj4Q-C6$X5Q00i~o; z|FMu4S<@JRm{Kn>sP(yC3ud+M-8$*57XkwBsK?r#cn(duJ7vwMP)9QKcwX&B(SD%W zO`jtt<@>A7@4F)UDmu7Y*BzLW8Bn!)inpX3C90H08@ap@qVNvx83EP-)|qWuzCR?X z92M9I%}u7S?5*YmuK5RWqDe~>ksa&M%a{JLOQr3jJhD*Sl)w{hfM1m!XO2nzMg_g2}4AJfR&Fv_L4 zVWi5}BVY@Ap7aARW5|y}dLU>-#qelNv7H{m#{|kAvQj#J2ql@t4AqLA9_iPAGwb-F zl@8SMrvVM)ajA z#TX~Y_wRoEFU>H17^N#mCA|O8=YJ2kC|vqJmh$ib&;Q$Lrq|;}`?}U9ZuYgm#>p+i z`}uC;rh*6eE?n^ntp1)?%T!P}X#ynJva6+0p%{lyQ0s&i^rzR5gQ|_dXz)DpaTSRG zE6KR4x%YT;$3^pj5>?sv8`Me7_cyK!?d#-@|5q9=m0dVmR>h*sfuom)M?ZynLF%Wh z^(@Mv8@u;x?h${{4p#3cF40$(dw9c_b3C$pcD>cBSw3-vL*s=u=GuL8B_wxXaZ#*o zKXYJMpD(eg)l@ICg%dO_gHJCSeZGVZDTdfINeNY5DMu9Qm4+bB(e8ASHzZ;5Gs!he z?uXd0Sd7f6SM9GKX!LU2AO!6-MIoP|)IAjHr4b|23c1)hxKXe#fmpTUL4-{5OxOY#X)GwBsxn5Xy-o^TOwGKs@rrAh(X}Bg z3wlt6mcCmXs>~wDUGL8p7Lr_V9Bct0V~N!b7)+WZ=@WBSU|0mCA>F%;e=<&(zK{>n#;Sv zg)KE)s5)3(#A=!#zRxQy@0)owktsjD)u$PPo@`GcG=Vlk>)Q)QJVfnEQ z$yEBSEMXcdHA81l?g;lU8-khc?Bn7@mW9)%foynuWkhi4Y;$Yu9r0FRHVxvDvMs|n z&@Ob& zpow^Je5$pBztsQmQx{0>SvgZ?D=#Y=#3Q#EW;B!J81yZmfGK$W z1Maqq^14~pofwaD zJjBBvSr0jbMoS2xEQcaj0QCx;Z*#EOV`iLAG?Fb#?()%dF^vEx z+;QT%E4JoY@Z+I|BgL0Y`a=(X(5MW~92Nz#eS-7VBRKRurik zzUkHw%`sAe+d~WDSxVLE95Z$9D?NG0mov?U*ZhDYmcKl6#oHX{G|wmCUMF4gUujZP zpQ`U+E6KH!!Uj=ciE+S;duhobTNWVCM}@GL+^=VX2ZgK`=)CFR`1C26sNLJWi#rX2w|G2=pR+cml1tJu# zc;O_i6W*d0&}R*t_FrHF{6wa-yF4nWvm@_~&-aQml8y!NmE_x6#7YGZ5 zjf8*Y?Z|8FL3FM}eMXs`8N+QsBXbNN>@1elt5SOpj}7C3J*Drd_ZgOTiVf>8>5{v+ zNI&dmLz>2K!?BN|E*FpeJ12P$c`s9Egpx%`Ler)l?JW3}LNoO&2&cZ7wbekL=FIs`izw zM?Zvn7Q3_9YzSvM6CH5YTc~UyxB`yN8XYX|c;h&0x+st@McPe{|N$1b?bP!Xd-D8n@U9e{!;+ds(2y%0QdFIq?^zsczcZ+NL!WonscSmW4KR znt?3r;SrC;QfATI-|K6kIZ)3b_71W zb|In^U_9}z@@z-UvJCw=6uF73-H5oL`4M#@JM)}ZKVf<-ZAr7?HbTZ>>UW({>peoXK?GlT0g z%9^IbU5i??yqY%>8l@rr0m@3x-YWh1{FV>mHCxN>34x;}nyQw$xg}zChr6H;yN7G6 zBrA81R%_x?47tzElyM9tsIF@K1L5|i0ck~jt)octAHKzme#pr;-*I;&<-BrAMrvE) zezM$jwVg(shk%Hpq!He11qd2w$6U#?V$duV4u}1m?r-Tc&GB>Us>D4i-o%QE;mHA$ z&A5zv$ToMP;GZ=*(~lO<6cF>-sTU9>8;B=^%upANM2Pn{TA3}4tB3RiBX1Xz#Vno|uN9&N)`cYbiTj_@ zGCT{<6=wmA?rrfd(Y92=3ZN8wad3@v%h2#o(?Tm=sP9%s*~aSws|mH zc}EK7^Q^t7pNnn?Y<4?Ix1GTe9bH$E%nM$r7KqL)CJE~-V`fZ5gM3=6IgJ1~yhyvT z3gkVko%fO@CZVT3P$3v~jjV_65z7ZR4I8zhORk4kS?XJp)AkSMI1T-Bz-=!V%lZC_ z$AEW!p(KVFEi;wH1w%`V7*|MJ&+mdzFz`^`+)l4 z(+=p#>mGAiR+X?)`AC|4Pk`%IABubC?UWy9MID0M^K6jfu<{p+S**`}c|}_qmb}jW zeAP2*GT>)m(bB8RQ21q{J)z0pZCj0(cDag3S-YR3%G~!zuSkfF8s5$;2>&X+N4&&A zvv`=FTRuRtdA|u+zZ{$9Fu!xP1TNOit=mt4=K9j8S|LO@o@P@gkqoTL^yS52Kt>|#}>6V_(jb5r)T|_>DD%_Z#PIqY$ z#a;JjoU7B9L}pa8jib?kvOtAg)z@&yqzJF6e+G}8w5L;XPXrB%2u7$lk5WY!6M9B< zo(^);Pz@UbO5SZL|@OgtffuoX?CR!Mbi159+?n9soO{d+w=#aGg@py(DgS z-tG?C1YWE|o@t!`R`^rb`-sCK@@OfVx~|(`c;_`S=H)8?WcTz)92*4-o#UlE314#c z)=aY|HyNMZ`@|l-g3cvVJ?qSdrj_aCR7>5iAmt}_$|}G|vn4A{!aBo=;A{9*lyS0} zLp{8>+;|g7DST?I*SoaS;{!s2#sapZsG|#bC&-Pj4SF@)!v|^ZK_B*V>9IakGr^KRe$PD1a&8BSaj0qa zq>-71Q+iy*wYx;pJvgMV{(DI!K_{|3f~v~py*a$gIDpFKAFk}meSAWBjpe%iXgJ4O znSF4S-jXA5l3jsCH{=H*Gk6)@>`q;dR`Erw2)P7{SKMK#A=Zls9_y4w34;|R=cA!g zjy8z%kmX&wBO=Y4x8{(M;K;sf3y|U3tjYqUKQSlpO^fc<0t{LV{Fx|zE@Jb!7zl1P zSMM-Y64p7V{Q4r{o(!gDWQ!c8d+LBPg*a*zf95H_ec%P;?&?-2AoS~i#=8$5n5~|@ zI7O<4m&WpL%*8rETF`5bATrA5trctK!sH^}D;B9|EWpkW6gyoUr0NozK92^B(gZ%N z;sB8^u(~tLw_>(?b%F)zsH-j;48)+{BhdSxM_7!&uS&^Qtd-Uo&wsI-5XHwc+K9$#Am^-pzwpY074u zC0|KfD$_PI9K_d|hT8+U?eRBB9*#kUuxYP&NF>^z*2t1sEzyzwJb9r*k4bgJfb?n= zhK+Fs9aia=oH+|SXd}`psSArAT=-|Z%UvPZuJyNycsdc>7j85$UJ_}>X=9x!_L|vr zIZzc`W%jeBvi1=y;wO{gcTv&)`uom$YkTA*P|8vtgiZqDyjE8M8(*_yZn1%W<)cj-@(o!5>v@;8`U!%kmI6G-Ot7!rhksVYO((8) zI~_RGK9=HW8;Mf5O6K&))TA!gj7O}tmmq?jiu&G`^_Rw|lpLcNQ;kPAJMK>awtqx# z#yAWK9l2|b7dX=?-zhp&0jy?iET{nL)A56>dl|EUpE>W1p#&!5>yI-7FB%F+-i6C# zb96E5@)*|fFpUmmA=}D@DK#zCg}N!miG&Fz;-pnW3QsJjwThfMfwIl_-V+AQj()s$Avb5lWQ8Lkyg7>WM96 zoy*&Ma?&LoW6qG;!UPuY!7$G?%}_eazOjIgkLI&d%rDI)3HS(eSOR=sH|+p7U$FPJ z;9a%X1sR|aS|3e1WFM7Abn+bhCsrFA#HHNKP3$7JFw!3SUc4UsIy3%Htd2aJl}saO zSruV4(DN)=lf0t_#U-ANEL%HC5*V~kUxqRdUkRd1-l5D#Beb~Cq$%U+?LmqT4_PjBMhj`TI z(CCf^I@djR5VytzSd}-PpOTI`uD<%2os$c5A{S(vKj=r{OeYxsZEpgo@0V<)TO+W4 zbftSL7D5oJu3Jao)|3Vtz4iyZdHq4`L^rr+N5W^gkP%hTGusX8?MU7}70*Qg1v-+v|H=9MzUk{-?=7SZLJex@4jAW-5RPkm z7-VGD>Oow$nP<8t{Dc2{+iqIQ+pn(W!>dgj?tIB3eJh=|x&p7HuqPi`u7Q_)3VcP6 z#f0c9hhPc>&3QX+21j|pJG{pN(-y^ zEUB9ay=A}l&%FZ8XMb_kvp#*1Nd&jwo3)+Q1RvI@Rt-T85&?>Uu~wbmqFV=Cl817O zEC#&$6?+$7T(uJU=Gn6=QxnDQ?7aNc)E83CfRv> znscwVqKSP?XPkTkOCx{(ToD6p1==3;u3`wnj3yQA66-Nx@;9$wLp zo-)PPjIwv}MF)}F(8m44x-v&QVcq!^OO=(o+!6%Wp_uhIHP@9-MRqLXT4ICnF*j<0#p5-3(fvMo_qMLA-hpu&m}5DVI*~er6k+g3o=ZO zw#>F8h^3_rA$O=o;Sb;b|7q1_nKEw~V*hE?K3`gO9eW$`$NZO8?TKPB6wxj6!*AK@ z!QtYHc;k+(YYyMUry&zy*xK(y`%zvA^O)?}OuS?GjzwlDvYULszpGlAnAt^7J$QF< zkNB$YeNd-+funO(rMrN;_#Cm|(*e!%;A7TV<#o|&>SU|FB4cSS;Mxd#VTmETfE(3K zXwpUqs^D}Srg0@=r?)?BL)zLd!+RtODnqd3JixA^aJoz|u{}ih3zacO$E!crXU;gY z@?6Dp^pG_IE_1)`6>(zECSi;hq#pEF_?BTXVrpKQFiI%;aSV;Sk5Vba~{FI7`0G}N9T2pTydnu!(Ps*~;sy7C@{Nsi$R7`mddA`UM z$jypL%N*|P?c1Fq(K?>Hot!Nb^`2Ix*9MQb$7A)4*FAoTF1%0q@2BqPoam0O4^rV$ zp@6n6#H|c+u~NH|s+x_SsmVp$Sl7ikEb-{yd3rlHaIoqzqD#n88bY_ zG8sbWYU_CV11>EG4xI?{)ujKlYSVo2ZD@OZX@wgp7IE{m>?m?d+EWZ^99MTNX@&Af zLl)Sa18)rYpYuzX=Z#+2@_M|h8s^-G2le%9A$9f4fCQ9t`9;=l$K7jCMM4_69;~;; z!+oV4{}f*xRd5Y;wr&7hsp(?bPRBWo29p{j<^DaFlDj%|mCJOdmPw1&MH5c@amC_{ zI=ywieq3bMOkNP^k8PSe2zTh)t5S$3|%s?gx>o`4R#m4&$Vj#ZEMO#ysZtqSXUj11-HF zdE1gv_b_{a~`{pnzb8!VdV@jBG z@B6SD;m#rfT$M>(tuI#18gh|#v@iY^=$&t*)Z5^ucv71SM=|2^)3?vLK&72U39TnITL`<#j6)U-$+QgY9K5D9rA)rbd1wmHi1(WkG*#r|MiLz;xM9%Ag(;ysWCF_2VfFk6LKm~H>$h$-Cu)-%HIJ<~*B>4J!p!G)>FBJ_H0fvHmjJ3%nZQ9uPP^H=_zN(Q5}am zL`zE3&0S#`zqnm!QC{KywCZZ?x_K0oF+&Os+ceCUBreU5FRi*Gw}R?Rt1kaftA0%v z!ij9hAM>d)u=ki2+!(VS7aX1Nge^ufj4G?uXuv@i3pe-EEiEWT{7TY(}T< zF%D|mHDG;pi_lWDwI|&}xh5`BX0uWQALqMsMl_IITOK0G7X>psb8qKof*@dxMbKN) zGoH@}67gU#cgs}>iM>~BY#LUt_&5*s=nBQE(cMiz;gPF;eCyZEpi1qJeu3E#bTPilot|s)tzFvK;inBIJ6!b`{?d~!J2sd_*8Jqz8 z%Rk_S$I+;I;|XoNzKdtol>xlKW6LGQ1c}>q62-ENH16&L0Mmo>2q;_3mYuPd_i)fq zJu-EbfSWA>v0BwtI3RhWqGYl45++9Cyhp2-pf6cS053GXy#n=*hZnJUuU4A-%}# zH=Z=9+9;QUU&gk_%t@LvG627ct~}6mCACe|Zoa&vYo>D5PEO&RRkr)A77aX;&q~8_ zfEhCH=tEfLG4Y{LC0pzJ!LaP zt@Zm}U}U&qk_))x{tyqhcS^T;{ks2qH23NHO|B))@b1F|BTUwgKf-trR=0F%4?D~Z z{KW=CG-xnrH-wCCz=m_(t^;`S@&u6HvhutlY4AN*3TcZ_Tx;}m1@UgRRT)rgexrcG z3%9Im-VG;^1Cfht!QfD<;U|0N?M>^XEedqBYA|4qMw(z+W{WJC&*gqYM}lSNG#0+T zB}jqu9xVDz_^*441$mymfK|YXklz5kW5|Y5s+-EHd6@i~*V9+qajHnfNF(31Db-s} zn@*K8lKP9Q#yIez`;V(;{Ew?9`r@j&&`GLCv%(dY+;B;X&%N0sRvS6di>$^#@)Dbu zO}~v3E~q3{1A){0#@=R~%0*>`eeswG55BF{RZ5JNa6GhBWbNH$>cEN-f&s~5P|OG9 ztK~5b_f+a#c`xffyOtEs^*yTi@*&?0Upg1*_3)f@w`VL~(4s0lMEK_*yav{fnY(=3 zQ;dVQwxeo|oU?rsD=Vu`$HEJH~5tj!)zfd+Zl3l~2{Q9p45o#GU{%Dd#p9ax)3tedee)&-l*N1j67c zt_6^FCBxl51MUM~W4ot<6sgPG%4}4`$P+=tdQ_((dX-5OO{!^H!01=R9T8gj39OyH z&@loeEcpX<`~J5Q>buV~(|UN^of%!fR*1^O^d#)ZCSQn@9Sl$3322^wZoQjZ6-4VK zcizW}`5y-@Gmq-3ye66oX2{&udU-o-1@0hCI=t2s&EWBiQ%Fd}0x!QWt@=^-!w#)A z=#9;8BG^>lmD{S{%X2$~kXSLskPfqJ8Px4s%Y8&Q*BI8e6mu~;9g7l${_6OpRcDr`xpd#Uu(V`yH7F1a?@~xlbmDeZIKO#|8Wtq=mL|3CO`*Wf z+x(@tSJ~j;x3=HcM&#dSWk2<&j~iI+-Ic$yODNfa-!Jq0U12)74ex-v4J=N0d|Ji91b>TX<@ZA|5ZCK!*hsQI#MjzE-YpZ8cAd8$rL7_@V` zMKE2=#ew-QLQhXjeYgS5V^S~Z?lNN#zyR^ig_GpOILk2^Rd``L{ds#!bWX_rwbo>I zm8B3U7~8RHpgj7NoQb)L_4Jcm6M;y_xPm6bzSwy1B7sjn%{|Tb$Usb5WlndNCZ{-} zfHmxTUH_MzXY9us3A~Bezo2bcOs@f%GtdCqdL-uBbC!g@nDfMF$I#11y#82&QZfvSV3x@nCt>~+oBtNPV{rH@8#iefuK-lju?eg}s zy}4@tsi^m>Z)37nejah(yJ15FpJJgY($;%9qwJE24fXC=Bc48g7J^nS+q@CmkeqP?cPd==-8nOkS?%-q1= z(Xc-Y6uZmbIjY~WCXXZPfuhSfi;G9&0M0{;67(`{8_dj2^IqPpXPV2xwZmn`a1rj= zdoGa9vU%PXFa~1TsezDQ9Yb_1qtsq%Gy`C?W;pvEUG6_EBhx7xm^fQEq1kY%H79`6 zRgZF62}E&Eik0cRY#zrt56z!;%UPicw7chMfK3_9G4XjmXJ2mZTb?rJpzviW=iEf>TZ@){ziS^k5)%DQ{v#;FH~|13OX+ZWprqy;GKuS8L_%3}+s1&PfSZG2lg{brs(NhY-_LgZi#e zjndBFt_$r&Q>AQ`gF4ma!Nl}j3I8n3F(s0$xEsd|G(W23;cmGjt85?eDN zvEdr^^bXU?6?IGlw%0=sXww1bq%3~0>RTR_IbJ^L)^Oe-hlU!xQJa>`mh4M z%c@)_!=q}+qrtzbFM_H7=xnHJ8cWxhSM~VQuzIj19)C+BiiIa?)*}5j!Q$T#y%0C@ z$)gAwl+dc10i-;h;vt2^Kj8RlqAksJk0q{oC*aqJ{)JU~!C-}1&A_Gw@%| z*uqK5=FDc2to?@)F5nWO%XCW*aTzrw4gC~a`1cmgR3pdyt`zW4h%@tKV&xdw4m+^_ zF?Q|Wx!ZFO!Io!#&kETbbPVNmnct86%`dUK{Y$L25r8T95~~B1c4qeAuv5lbEr?Pp z00Xez)U67}hK5?O9s){QWjhkI^^G_(j1fKdMTpEf*F@>3RuiFm`_51jttFty(7U<&L@UTF=&i5!tB&hkCvQT@BzW;JvqW1*GUk5mO;(+ zIcYOtmM96|U!;o#h5R+kD@5G8al&#*cj2~?d^WDeWk`#d*yhmeZONK)dvk`s#gXA) zep-$>?W!+!Xj`A6KOhaR5|~0~N;5@sWKf|hmNRwH2a|W|wfk8-b?v-fnD97sul?eo zfgz>fMhbulm?4ppThUWBGYS4@p*OXM)D>cP^ntVh%i*jwiC--uD9ypW2a0TfpQ`2n z?1F}rY`wOAhNgbnC$Rga>Q6T0wRe@*4=qYj{?cb{`OA2wV=u`eUEjY*+{&tNfoeu; z7{q*-at{`ed$=-pV1e-H7wb3pKejrJa-KA!PF+h+j#38k4Q_=2=Rr`MKEbecXFU`PF5Zk0#puF`nIt^2Fg{ zVy5#M(i)WzNhhvz4uaX&Rb##p^_c7gmHydPF=C_*K7vziQMCYLXY3U9Cf*kKm4UJK zmPSy^Q;>QkZUW6 zP*@>K29K^R%Q3&&*f#fLq1KGdD95kF13QdN`jd2upwz_LD$1^>Xew#zwuGurMoD{n zBC~(GLWgVm@loSzU18shQ}X`c`SO0*T_xmzN9FrDzU3>^{dq@!&R??iv20sqlhqvM zf6XNG{JiYDYx_Z{#kd%bYo+?dISG;|k>H1Rb zalM&elJ$3isZ-%>(>*5Y<@FK_q+v3Pp$>-Wr-ggGLhD|$zOMH#)^)Z$zWJn5UV2I!jPQSId~|;@zduY{Dmae* znnx4zFI#9#o^8G;+dL)-l#?#sboMw;0y;c_7g`%*4s^=jJIsE*Ja+Ftv_#$LS^Rm? z?RwAQe=qW9`@C6>ddlT&Q)PucvOP)JQh0tZ(e>s0QMJ&AkQ)DMQ!$|B`zTz|l9y?$ zM$E6+fg7l4#PO#wQxyts`UuGWXQ1R~imt`$7Vzm#S!yoWA>GTrVNp=cDQ! z===0IjKLQ1oj>1ERal)laI?kJyMFL-5MxJFKjQk|Dz^i2B;KtZiG(4vt@c9$G7!bb>n+d0|5rQzgTX3KK@xPB5mC2fr+JIp^`FFL5b@AoJD6LY zszl9@+kQRNmC`k}{`4h#Q8yp+BWj9CJAs))Re%)97AiUMiKukXdX?1xzx?THX z%RcOXSH#sE39-74p+?##^!d17?(P6OwYhr(8zcC8&(@*BJnpW6!3l(jPlhKC5Uu}B z&@Rjm--z0)dQ?$?Tl9Se$`+k>Hx*5opZWq<$&YY^hgQ65W{uUo$t)@pK)UMn)9HSL z_4M91WE!P!FYx3wZy$uNe;>YD=tmk2jcqU%e%dmmm0!_;^BaE?Zw!t5ktW+& zgs)gPtJgEX#0#f2v&rN5AGLkwp%)p;JhjlH`cv(#92aMziS_QzECb94nRGjRMxcqX zjGEk*@b52c)RXBCD;Q)x9tP0Npc=l|o~_8^j6xAq=*u}MlPO)tJf?h5tpNIvV|;yn zq0_AiRi^;daD}3JvJbMu-`1^}L@P)Cn5ryU2K2(q$O>o4OknER2SvmS12s;#MUe9jp`)T zyMnX*sal9<{HZN^`t3m3uMv~YI8y3qjwrym6Qfp!Kk0+AJKxV)WF1z`yq1eGm_NpE zlE|;Qe32K`eEciD8f+6~g=M7NpG{KBsNcaO3Si-z0zl}Rfa=Li^jazcG7$WY?aOz6 z;gy5yh7IC@5VRqRn?`_Ca1sm#RbCLDC{?NMQb@aH!m)qYwS7!wUNPn8n%)UWH;ylR1*H4j2DKVK4DEDcL zI612_TP7dchOWThoPUk-FW1x#Hc6fx3uN$V!8+2ETmF1dD`fq6KYohRF`NsoVhYa| zRaNe03sy#8n<{|Fc(U`JErljG$wF41-a;1PD7|;ew{dmm`)&NzF$en5plzASPW8S3 zG9x*x<69~^n4R?Ru9&rA^?#qSA3I}*qo!ZnG7uWoR=y0O!`7sLSE10jePb2f607t0 zW-I$|xFfQ?y6bn}h=(^uhWS=j6Y}+AgG$uJ5~aD2FwSCzi;L{+fE?P}t#OT{@?K9L zzBd^;TYD+5ZcLgq-EwM%I?OOJ6ksN5lhM(z(|8_fiQ62YjOBZczirXI*Fs?~cs7VR z1_+WKd=bR8kYe)PVX{_p6&b;8-#9HBOp~8VuS}_k&mmYmo??9ki7=mT-Jg!~w7dMZ zeylG1S(7oSk&Qe@X86w{BzwM61)WyK5Z5WHe2P-C)Jo1Hi0-?6ubefqA%j|aOjWD8 zck-QzuMHqARWI=_UF;UvvAnJnTB-Cdbhx43@F=o~T$S>QYVb6k@>=zZu_lUJ-!bE@ zw|>pRFHL&K@WgT$gj>s{vWw$$$Gtc}7#Pax_`!w-)l`c~*V&Xut$VS%O&(p5x&a+U zO&`)goPGF%5aqx{bO={DM1Q>u$w_2Z#Z`rKmND?*#jn|T`Wl4N89k?AuW}1YFkq{s zGekPmRI14>rj-isvpiKCivwSGq$af$C6~aiY8guv# zq|L@+xEl(3BBo#+Dsu$L1|>ahx8itRy9MS$p7E;B0$fQRu67_7C z*7sgj-TQBfXps*Fn>lx>ts1=JlIR_8gHuUC_V0%GYauU#qggcyFO+MAL4^bC#S=%O zW?jJ|8z7~V$IXW@uKDN-AcW`xsj{raQ*$H!9d2WoY1xN=WKqtu_vng%lz{Q<|%d zel)x%wqz$fk8F7gse~R{9CY7$lU&MEe@dItE*?jf)YnoTFbO|itrxOZL@!s?G=}RC z&C*)JB6HG2Zll&@Te#znI9+;_((wwtOmo2l@ozp(sCJnynLlTfXeffHyG+X=XyZS9 z{%$c-e@1ytS1t-|F;CJZjz6+oUh&F024vdw*-dG2<{pI! z{0`R^J=CBR`}aM7%hCHm>xpVkyqb;%%#+yW<lI`8?&Qr`afa=Q!Wb%0oj7>0J0dC+7&?4enj2jna~d*qehfa?i_LX*dDfQ`a$ zp=r8>Ik%Rv(dyRsbugHLkPKdF3#Q>LriYSf1EJKQRQFArSWR6L`Eh;d69--bb$l1l z!Ncy+XG|uxIcjZFW!uu$7zI~}L5 zawFFkEKl^zz(4#29~vyqFiT|+Pm2!8%$p%6G?k}x)rRv~S+!tZ4}y%giA$0s(NDW|NpwbXDcWGuuo)V!pa2iH_u;NgU3Z+m8Dwl#?E}>h zPnruXZyT;iKO`DFq^3lAl!MnxZ+CXH*q6<4?iE&sF&!htaueKKxpY-n5nadP)aZaw z6S*Ii&KYQXjGb=ht0cfg0-f;LigRSrTmb=WRDr!`3T_WqKfFlpgASQzu!W@LL{Rvj84MZBs4Z#0TEYP|U4Fi>?j- z2jr(l7*%d;6wV84tU#G67;k=|ZCn+WUMLmazLaR6tWRN)V+^lj9QIcX@ux3a72W2+ z2XEZV@WkZW-w=dtubC-qTxW3Wn~dFm*BF)r`c$penbFOG#SSDhwova`_yvrv&tYE* z^ht$M&DeFrHFu5VBshoV%g<@E6+1vUT~f1pS)V_=ohnkcWvhTwOVrF4|4?V9wxyd~FRR-0Ew&z|P3=jWJ*r(dmy zFD6eVeC;rUPi#jmi14?BNEa~lpLc@IlfEwxEw<3TPQ4k$K7bWY2eHVO6^CHHQJ9JH z)feu@6MkI>{@;5qwHaVf?*j_v@=wVA%?LSZ4)LP*BzQC1C$~1(KS)PomLv%0`yYGo z05GD3wU#DLO8oC{^FkUkB_btNA1~b}^gTbICyq8m2tM9{wx3KsWuJ(LcKH>x;gGc3 z_FhlADs6}ac@U&7jU%WCj2&Vj^Q0`WKlb$yC)@&NgP2 z{5&H}9nh7f__J-=;R=bkm>*Hp7xQylZ_m0`cL4?D>~bF3D?8Mw&j z`?@8%&cY`478CHQ#)OxH1MCm)m-9ymp+|K?ur;(wBZAUU6tkWi!w@Oyt7X&c-q&Ix z=KJ2N-{@RfVC7S@!3+~^j)v>VY?1>e`)_%_qNNiaS=rJ=4zix`>dCk@QPPNNII(Ys zDK3@o(LaV8TT(5lZ-^DRlO|dFmwde>h-SCX9(=er5jQp%8N0DtTjXj3ObIQmej@yc zR`CMm@oT6yU3hRY+9cB=Yqpmg2#1BdP1Z1V&LmY0JBce0$_VEB#F=Y{u~HeCdChvZ z#{la_U^93oc8cY;Hymj8ZP6+PEX|C7rvJ5+vRN;2b-7Gz_KMgpAJU^F4_vcX$bCzj zl@n3FI%&3Ld?JD)IcZLB@ zKjxyf0uc`OJ8A}Xvf7lEnUM+7p+8C2yMB7s2H^d%aM}TO7=F?H$RTTJeWe0J%xSy& zYneMrN*Oev&``x`aGdJVT_##3jCE#s%8chG0IB0dHh*z(L)g@)R*TC8NA{oF#i+}_ zP3!e8xH9ij@$9x}Ox_vTMg>JCreQ?6lQN&zabNL^J`Jh{gAR#3#ZqnvXKxRhM8Z<+EA@Mt-8yU{aDahq4d|_0n zIm$aIQlbMlh7*LnH3djW(F}b}ukkAZ&NV6sT`Vw15cXf#hh#}(27QXr6S98~EsfQG zbDkaoaKRy-o86W^?{4{hxDLN1LvX8az1BE>l#6otvF&{r-#UK2DZZZ*cH?DgKcv#G z=xjeTgpXcDB}mbEz$8=t>>4i$hW%lLn)+*tNLp_pYYk=*FKVZ-LiVo<3nSS@)UFDpw51hWE?K7bN=$p@y(K79jp#$ady)tYL$v|cA6ozOzqcPRq z$PT~*>f0VupnWP_D{8>d!VkrhU*}+foFSc7YA*XP!Qe4!GC8Im`v+j}Q*Hh-+Hq&N z&09izej(&$`DH=W?*2-^m8G`SpbKTJzbmJkJ-5baT*B?fn#pY!`0D4I&99lCbY<`!lYOrmxR1r~{Wt!cul|UQoCEXwPP{ z!0C>7T~m^yfjFTxFDhVLzGCK-0BKkDtr!4X0u#!%^VPGhal}#4vjB{LF~Uhl*#Ql0 zdL3heb;&x}yDKfvAoL&%o1OQD9!z<&tY;1(As0Y|PFV`Gf^&iCG)*+D*p0$gP6q1% zF4w|dwZVwS7}C?4`B)KV<|qSa86G?hQ|T8OQ-6@MF!(W$2;0=KtMBxX;H~)C2S*(D zD!Htf%-2)%;+Xx^>O1c2#Vmqb-e_Zj48m(?4+79GtHmZjv!k8RBsU2))TyvG&wQ@r zdqnM%2{5z{CC6K{`a?tJg4oos!BUjuAXbob@qu37cS`eMQ~#mCd+iw2Bu)6bcZW0U zSzHwj4;Fm7*Kj!c@?5*=2TTNk(r&;oCcwAA&soj4h3yjLsBopl2;^4l-Uh6G9v26!!NGN@;h)-+wk;RFaX|6p@PWI0 z0lpZi>zXe|0EHbtTZTAcLj>GN@`(Ps?^4`=JqJ2c&OhqRw?fV{j$pH;xg?C7*F%?8 zJ>MUj`{{ax$vs~(|0WO+{CL51l_vaYW=rvYkOm>wr_x^!Px}ka9LHgRAzsFm(-Z^$ zJ3FOOI(YfLiF%s}fq?Q)_#I?uwVAThVkZtS4n*=jK}m2aWBqa~NSe@Jq|D!s&{>v~ zHkz41`Yopw%LzwclOA>yHIKJhy{&j4Cp;*mMnfsnD9xbozgIFk|Cm8AE^pNrfCCE< zct=av!CSsCs;O*a20jlpO3LdKonenY{UnfPB_u*tGL_v*sJVZ7N8QS#KQ&o&>wYvV zFaDRUDkeTNgkJDlrQLu5?V^MgvqUgL@LZcX5Kbn?LRC-Ts;bI-dXXGaHcL9{Vmxz& zjgTmMry3(ih<2a4Z15bS@Y3R%bD__dtf|%eun3^xh1dk6D@w~Jcr*6D;7O2En_uwI z@Bab?ac$cilf(bj?^01&&N$-gsu(x4!un=x*PXf+?HIpY?CFq(k__HQCFU zr8sBu zYP0N#b42m#MX#h~u4fWv1sYN1vwxE~(hvrEh0V_ETNFy9F@K_{7iW=;rx?^%Vn=od z7dg8Js`Q}}LGebQda#&DG|(_c!~{D%P-)kjw|AwNjQ9NtK3Gg!R=w`7+N`P3$Vee> zf^xBRy4hyl!@ztO~-Cl_<-m3kU;@nLKZB9*Qu zvjVmRZa#`9Esxc!vZ{?L=CI(kZx;Gs*XU4}V$14t*w)qQfy^l@_{t#`G8o$vz#%+L z%rMg>Z427V;)FT`a|0f`sTi$8AN*b`R(qvW%rU=>JzxnDX{wMx4$LB~er_*vIYAWF(pE51|5Mjp1Y_27ZHkQ6 z)vz)*4x_fYQq#(M&Ksbb*2G6HP5EI;w0~)-N%`pPs;_R55%?lMv0geMM^>qp4rh8u zKn^;^W}{ zC!$Zi>MooLT)iqF+Fips#NEi}OZ`OEXCWgf8J7VRPBMpDGzbYij>xAYcAp+@U4{gY zu|+#sBjnVPF!aSOIpV?(8%r8qUE6mz;KegS6nj7Djcd-$wHU`a$6KH|8-%1Ap~9SV zwO@Aner3E4X*<$Zgfj3F+G@~s$%=w1aNza7d!;|hIxqht6G1^IKZeE`mz+3R!nzVu%IFCA=fqQSS0iC1Z z#QAYHip(eg5*V&wN+fp6@%L`rpJ;@&ydv#*?`5{g-MeWYjgj1vi>dc{X6o#j-Rg@M zU>TcQebD||x)kJU;*Tptbg~43Io~zf?wAsy0cAM0AYOXE+ONJ6B5RF3rh9S^d>`ujqYNv`(hx&ec*}z-pn`AlqwxP^|3ziAp(foQzHXbDgYVUfv3YvvZWTbyL=% zIWo7{9J|0s!I-Y}A^Uw^-PYuM5OC=mm?|524+xBOq< zgyMe7G-+Hs#K98sn@hILJYFE`6djzk&KYZ6^8BIp$#TC zbMFb8@xGdj$$-&>2y@=B`e+UoUS*^@mEOGfkgk38l-$-!tQBge*fv-#NEo*HyHuL$ zNjfHM1z|>!M_Tf7Sr3)?IROwifGBj@%+=M#gGjw;pgGqJ^o{Mi}_HO|U zUm{dTVjs{*KTNmQm?Y2yWX9)cYEmaP!rFl^)$CsKLAar2&K5F{oSh{7?O#7Xk+&ct z*~?2W-_q|`$0i{X#omKck~JxBcpwZU++3~UK-$RPaA1&un1Z(y5T6w(K`IiTnhIDN z@>TB{i5Yy}`a1q9rweSWr`z8prQAs5EG%FCrOLhctq7(sW zjBv`_&ZczuY=7OWk3?$9YjJhGb1-ZXhSs@n*Zs1+uPJ7_+(4;UJvGNNmDTQsXl(|4 zOwKx42@(&|heDAQo&e5M*c4FI6U^II$9bES6vgBBGD=Oj2ANSbMJbCJ;`X|%qod~-J!Tn z2;3irL#T>pl)8bfZTICLK>d8~_iw1CUJQLCKb z{I9trD<8Ug;wW7URfy3M^dRlVwWc{Ql+Bb=uiN~2-?vb;2-^(AT&U#&BXx8Uv)Nc> zQOG%%W*tO(Sq%wep4=AxqeX%Ng%G+s_i-@lcYiWalB{_qQ*ULc3R_s$tQ|OsI~VaqCSP~% z&sfJ0gH}RXpQ0GF>#W?dl|(Hi{U`2S9KNS@fr z{za}R(xrv60YUa#`utnEs*6yP_y5IY`aC+Yjdq$pCT_GXEG%U_ds#k6k-Uupv?pW8 zqlh_H!67o^y#yKF>R|`t@`#$*u&dj>TyQ{rUeDJ+HhH-~tR3O**C2+pp@!@}-WWRX zcY@I0t7lu&y3OYsxLUpj8tLR~uu+f8>Vuz)!I&%zC+u1rp-Mm+X~WJNc-U*Y^VQ_M z3_V!X${j^xNBe|FLy1se(A3i!|3|T^pM-*GDhQ}YvIX*Imi#2m9SV6u!GzpyMN86J z5R6wuL5h%PdM<%I%iSr!slQ}H?+W*pF=KrCK)TO!gQ<$KVCq#Obb2tHyunK2%9#4| zU31Y7z#NUoY=?}tt!pvmUGIxBO_fj+uW^h-#6?*S8!1SsV~u0{%qdH4K)H7I<=ND_#!#E_eO_TNyq?H95Y(XEaJHHRWFKEj z44t8Y?~3xGa$)2*-K**$x%3Z^N(iUHk=@ruuQtE6nEX#FQNo-hWR7OUKb%L5Uv*ojvECfp*eAS zZeY07PRbFvbqL~rea9(F?II-5@|fh;*O2nyi+7VFokh7O*PK$dR-$24u#ANRXXG8M zNU}7l272TSa1gUXj%6yROUrFUYvjN`wUoquIp+FJe~bwr9rCySV=oiKI6qIZztW2Z zud!hcca7;9BEybd-}!bA>K3FvIFMBvc1d5lukF(@bF~l5oMxDjoT&YdR~_;)u-`Ye zq9j1{%~#6}oW3d|rd2SX%w>VRRy$ITN4}S4M$sZ||A){XN$l{jPDHA8=WEmrdihB9 zv6<8OfEjJy-5tM8!1MbK=5y4r&G;kY<`?~wcg*#(>$qr8ha?Nl7V=@a$fbYDd0Ra3 zCIr7n&|teI#AvAl0jswea!XKIAIwPlw=+whGLGj?Xi+A4ecidC!+}6#csE`lPiqO2o|c!4mIh?$ zO2|Jj3{WOQdC@(ddQt3-%sRq##8{Y&diJ+ZuhD<+*APr~z7h6l)$&INr^i&`>+H;{ zA(oI)dlqeiZE)sM5*2XLK!j)W$b1cT83ARt0iW}qYVi~1uG=z`Vtg|2ORKB{G$pX| z937#`H~Wwk-0P~h!cMUJvy(fM0E~w-t^RJmw-b1XYCiEcC-+2#kAG-OU+^KXAq~VC znGKH>YQLG|Z!DIA$)4X6l>FrdhBajb4#Jr22UZQM<*0EV16P0E6pB#HZ-dx}3}leBNwi{8+j zjVW9oi@lWp+1BOrIAuNlS&r>{S4}xdBli!G-^RK9l}FWg?p;4aLy)J`)nvsiUH-UmhE1?m6^5OQy5>{h-GqPTQ zv=L;;^!#X4M|CW)j<7}ZI3Pcsv-Kc}>@L_Qk8>s6Z;F3nb})v$-R0_^GZM{TuI_@wTrl!|X{l^MZKgwMqJ=9^~<=4Uq2p-2Z zWNkiFtax1M=v;Z!NLRUuF%Uiu!w z6^ZaGc_GFaDQuEt$kzuI8o+3U37~KIc7`A@R-ekXX96cLu4<2B^o=E^x&R-gv4jn& z5B@YbPl&_Wt)$=7K20PTWRowQ9{B{*2ZOQ=iCJrp%2Rhi>vLTWZ~T*vYh~E63=2Mf zDVWvp^4jAWp@0i$u-i(j#t546C7p(!8W zP$2#^>LG_tw;n8}CK1m&)cLl8^$}wx3*ood@qd5d)9U>IHU0 z|5+wR4Jcv(giSf+PTgb)p)Ko7)G5i@D@s zL%#tb{LjwY_kT*9<9t!CSox)IL&DFA9s zsO&FT^~fi9Sj)#uF>x~JMof%Y1uzO?H=j8avB4_Pq_0iSFaJ}T?7xqL=qs;2mUsdd z`ZMvL_Bq!N-9BGRU;nW*d8!0TAxdDxVWHuH{V@DEO!>k~J@=k?1oE`_@I~3zCpE&T z9g4)RhQ%zFd3>8}U#G#zI*`zJ)&4bR9bczS;Z)}l%&!@J4=yXh6* zKPpDii*U$-G{no;(onFwP-D=$$o@9R6F*5p+Jk~`?y4r z0?wb|LL10B_lJHPN$aGBKz?_3#QIP|c~vvby9i_oRG%X3=N2Jsp)DaWjB8dz*lUml16y4&*_*nee!xazR?M5A>V-eKi zMEQ=w=Cq8E<$orN*E|L7q}u!QpJ6c0M&2-NRJ=zHDANPHw2c;~a|ZfBI$jDLYB*Eo z*ppR404RKFn73Ni5FK+oHRqD(Wx2cc`=CdmV4o)X@IY;2lF|LmxNzTXl@CW%uPHr0 zv2e=?kfbbFo1jjHwhOoFr2Rd}{IBBj9}s{D45>586==8UGz{)g>9k*Gucf}RYXW2b zZG)~t7HvEoF*W!+o)H$tD&#ELpO$?$XipQ+Y>;ZfKy)7t!9b)e##JqMO%`+Y(;&#BT;mPZ>eU`OuKRs= zAe(3_m1xV`X@Ak`t3wC3+u2~NP6=%+$58zsXq{?_q=G9}CndP-i7cyd8i+Ad$Q9@& zmJj5o$Wn>M-$uR%dZ!J!n;&g|5o3YkLMM?ZlMdFFyAG5!glN&iC$qJ67xA>Uo%&Uf zIx{0{;dFL8a2qI~LsAo;2De#UatEhP(lS!l@f&#O!Hm}Pl@d;PqHG9;9Ib`C{wM?uR{= zJRZJ6K!x2D8;VFKw)5U?AW-WJU`*!Yew*xmV@&=Yc+TjaW5_YC1=ZK$Pb|qu;`dVt>&rgi-2=%9c`RZ?cX*_-jZiJUr)nM{0G$%}Ou$!60I*NMOrS(nge4js6BlrBm^9>=aN8|H9^h8&pHqqF}`?FugKb426 zF`>E$M~7;M(DrI$Vb`TB^pno|OUCFamQeGuTjxK4M>+ahZQeMg>H#6Z;)@uV(552BI~hsK=?z*;g#L3Btz9h6h% zMSZalRB#G|P{MP{oUlD6dBvorfY%bP?oHKPOml(!2?L)oS)%)7uI=`hHS~Tw{-=&Z ziktIpYlINN{y-P|F~UQ70v{+c#h% z>xzEsK2%iD!ptdSijLP@Rt?!E7Q`Z3&R zo=2`#>OuWR!pd@DAV@`*%dIt5Cvx7$|!&QHbT1# zg=c^`DVkNsdtgqspir6q_?#_hRHEvnkd2A9ij3h09nlVVN5F8Dyu?V9PFN7nTzE#65x%x0iwHC+8J}!9u9FA`RqYlbIw`X z&Q?vrrzx?9rRn}?9y-`%LK$m)B{Aj&72-0W#GzVK39=-z`w|%>=SL8m+p*Ms7>2QZzf#Y_FpQsD1XM~TnT}a=z zBX9yNut+ywTIc(%g2smSHPw~z``G?iWzgRz9uox$>FZ!@oG5NWzenb;YHrb z7WkfSFe@3}8bk-s!nsnm@V`Rs+R2-?!eSD}9fyQ=k25}luT9CoSk}wgmhdTMxiBbh z_2kDnSZdeog&<6dA>7J_0F!KZBOta>$q-K*F8m5CUC9tpve1gM*b!7&y|iM#Vvjup zGwehO{urxW3z8%%PWfsM_*iZOkl@@PbnTZD^im^hn&~_Lu<3D-9^O<_A@nGM$q7mS zxNlV$W=s@lQS*jvx2Uu8WVWHRgtjyUZ--(cE*X9L(p#H`9j4yz!343^U+n+&Vf%c( zqRqxKySIcrLzOl6p2o37xwB#6k?ZI?YqR9TOA2O;T9{r165MpXdYVV1D zVARkn=8x`2$2HzL8t@C7kD_G^{Djm;(-49Dy+pz|?$g96 z!Yx)|vcHjVnE{1xrTdCaCnUU8d{5Bkt&%54{t^@U8=>F|0($;5P-cGMb`lTGP(Y4B zB^2%Aa*yv-Iw(=ftsQ>XMh4xCO9IbqE?m9y2H>Y<38Wu7l1;YK=<9c- zvy9^V&=x+M+VD(ei&KIKfw&XXC6%F~HK8xg8k<1E1}oa^v?$01`7T6hoPh!7)3v-gz#wEYXF3Zau}jB+A6@ybUIJen`GM&tz5s{@ugsO zS0YznEm=6YpFpGTx29@&t4nWB0S}^Wn%6U-Uu5TC-^4=2A!=UQY?Tty9#JOLwldHW z`^~lX7pX8_j?T$ld}J@$C@isyT?uP3-MIUby)h}{5E-8EaN<;PpX(1JRp{`AhV3-5 z=DE;R(@)rmhu=)h4wg*#Q~0^AFrJeF3S1feRFH~|4l2XPKX=jdldYO~@grDJhI*v0 zy5!YUF-aIm+R+HZh%-aUIw(@-j(BpuC+q8>(~C;K9rK9eFWA(AGfm$^LYxP+@VBWk z3x2R%j+t4i$yT%e!zOn=fk$AjBB=>`^f0r&57h_$EAt9yf6o*RAr52IH(2|d`@Gb2 z&^a=>c2HVX{PEQ$S-^j?FZ`D8%U+*idbq;zYh)ktPT<%VoO~VvcwO;IyA}JbL8)p+ zq?l(2p`DFr6!X^0!X~1sD2Kf$rh^0cVei5I2WS3P60vfFmLL$BIc;RhGf9==+zIw= zv>kB!w_s{9*+)ONw_vB}II|wP;}OxLm*0mY3}dMVcnuqv%gH-Eshfb0LL@okw#R)d z$XCz5Cr!~T5Zv;*Utso4v=6-d6u1TAzhPj!^6UuuJrtJ?^fW8U>X9g$^o%b$@nlSm z8anGc+d(^L$`KC}-bSmYnlVn|FZBO&qdiXF!DhBU^J9D!R^|cXo}P3~zq2+x$6Tf2 z%TO!K<6K>?7FkK%2)!@%U$Y_Km`Pqir=J-GT2hNkHbg~(UsbaQ3bF?JnX)NyH<(aWzg*i2W}k02^R%OftB=Plbfjbg zv-KIR3d01{UP$`)R&X%2k}8};ep&%HpoPtBpk^DdI8c8xLz8%Gk|_Sb&t~kE@UrW{ z+G``$mUL(=I~KcP6r;fYDSryFINv~UpM0;E#^({nW6&LM7CLv}L z*4Dj;3TohTm)AsxUf<@<2zLBrq}7;NvMM7@M|~KHBK-G*%8AQ{-e&%j%D6XP>z&us z#6{1nE<1z=CC9S4IkDM@;NOb9_Q$lnyISZ#0y|}@)5Yxfzz3;LKmK&S+SO!}?k%+4oR!H5F-9k2*aQ?v z5GVPnzE**1FB=A@F%4-sRa3n#Lzgb7jq?#8o)cC0o$(K3>!zv9H=UuJwWYC@dv~Tk zL>0D`{t2W&n69~3kynjKod(#aRJ--2qa2s6DO`P(ksvJzcz&Ux2RH5Oi{{!X!_)^h zV#nCRi7g#H^LEAK zW-fyzi-dG+G4KX?t;g|RC%-au+xKC+fHi&Akc_pPE-S(MB#6>Sp_ z2`0NYyj%yTo9DU!Jbg~7^44yir}+KCUng-OrIkekr*KFc8x)z|>4pRDqLmKrc}tfa z?gEbCP5)GSWIy~|`ua)*DSVac5E&jI&1lyVoU6wTd3BYhb&ux0Y+Lc=198|V@Ls45 zPQtF%YE3}>v(p>*o^etVEhcYGf;RN-W{-7t_cM}y8{`N>-PN-{a5udZheOezM^*VA zbx!n2z|)x7GPb)DK@+8Qj!$$LM2qu1%}S;fhuU@a?4ew%T4S>R;TA%Lcg%l^{&8e< z+a}z-!Fh2^T#S`*(un;Jl=Y6w4~C9Qd|0jEBzX(3pY%2w)d+vYD=pU2opyft(BsIP zq|96d z3`gF1dWlc<88M+YIg68)dZfRGNM3HoA>f#U)M^N>Cy`W&v8AIw;7k2|jPlLAmNdc@ zQolEbFLPTE!=gl?1Y&>4DvOrsCMV1$)!Ix4k@PcbpICZ~Vqa6F@07ZSe?gKp3NZ@n<3u(y#1`)7i2p-C zO<@@1ekIW!ZAoC6K&nP_wrj@J+qJomm;{!o=)$f_%3KHaN@jA-uiLr_Zx@PybOI}0 zv>JZwBT69YZg282K4t*&`ZEfjcaKJ0xTYU17dz{1UY9#kv!aM=-ug2UHYst^pv{W} zf{`vN`&x9um-yDrPoa6CI;NUCZ4#Vmy)+tX5!c&;jvOh-cJLdP()Ho)E8Qg$u}XeA zhHAOIhGe;ITh->|%h);4;cIs3fi&~K(Ah{kf&Ol#F~PCokgN6g^++Q(;T(r$%eq6m zq3M-CNr6&IxNr@r`qx!4*@_TeYduJ^I>GQhaui8Bad?%+7pDu8sym&DkYsDr#1nn) z8Zrf_hkStsEk@+mZ2jQP5cb>zsl++agH0K5R+UfrjmT_!~~;;)#AW6}QyB z4{=wpPf^TcRhb*27?es+>{^ zZVOU)YdH61544170%V(H;CtRHgQ>`%m;u7}wT0nH_fx07zTliLDw-zFpzpoj(AlX| zkO}yY(Iecy`dD;C45&4RVcF!xYAo=}{A>E}Pu_xU4TM;PCPMUv0dRaYhqF*KPMKCL zGLQnuh`9!^%ea_3;)-86dJJ6ud zbNYn+9K-^FIb^k^wIxy!?gozyH2yGbKZ%)}eIE+^T+5t1GOj)~wc2!;$9qPS_X%qi z{kOpbW3WS<&u4Ln-PauF@Z;L$86w~;?MdqeqPj0%2AQIwI%tSe*Q$NjD3XjhA$gp( zMQjAg7S~<_|F7@Xp6RHy9u6ek+DMq&U)Owloe4c$2f3fc*=6u53r?Zyfy9#NzrI6zSL2;?2kW~mWaFoTvQk|i!G`jYu z;L@G)OsIL?H^Wsc^e6L$a`_xQM^2Bjx-f) z-l)XjBx0T3WHo}Rf7=8a1z%Kxb!aC3<)rFRo%dv?AMctYxHKWJHp}7dPho~(HAtD) zIW7vyfUR%v?L7dTkH&-hr?O^=Q|G%xWnLPD&rIGSKI9w$q@dPDi2U=U^&Sva6oWELXe|{i{ZO{C$vNWJW@oml6uQSOyMjYmOE}v6J;qwuVeTRdb!ZNpX zucXyn=B8qVHvdCbMoynw1f|f-paqLxMwCWQmS2Qa2c!f4M>oeOlwkSqDTf~`1PnZF zO>EQMeQvM}0A|ufV$Le!u14d$jv9Ofl8mbr$pNZ^!t*h3uWjOM2BRWP?M|82$4CAx z+QnGY6{s&a%{G#tj|N-vC~#U!;%oPZ5T|jol`Ue{aO-n`p@loqJ+<2p1 zHde6e*t_}ON@KS$L5a}v$-h$>no&NX9Mo;bC+q%{Jgv?kJ*@Q7J`P~}-J6<{@*-=Y znl#sR{)q9Y(2_VS;hBbrrC;1*9jcp1FosOJ@aGj0EuD7@RXNgvn$xRb;PP@T?f!Uw za`0X)G@e&h6)I%1LCINP$LJgs0Cb%3B)8u>Pp* zvb%9H<#x08-Guii+4cWZ*`$`{@j2YwO>4U65FjM`*QzjQ{Ca)dwaGIn5qZmH9AO)! z4Q|$5MEP3Qlrsn_V*9UlkcCx~FkKc{RbMvY5Y^e^^|YzvA^PvdQGm{X>teu%+UK;v z-xyXtNJq<53RT7^--YEmM?-|K_~$ z8b^lT;Us0YaP9>MJIa%a0saiUGa>m?zAXk9Rji>ob>u2;4myWxKGF={qW$)Ep(OCZ z$@-X@-Xc{HbUz0QKFqF-T}+N|vNu;ybvaJ-@e17?Xdw+{59o@P3$`EHA-Uk?A~BGh z>!)ygtNcFVJ8JLfdCO`H%)Z18XH@OWq8f^KKK$xLU(CiJ)6PtLK;18{tK&@%n!PX? zJKvRj#55_ba<7UVR!7ob>8^9FeTSjVF;9+mfkM62r>M8mwVX;_De8mEv{9etKMPMK zr#=!i1y{l{mcK)8riq4@y@1GJo@HleGLO!)GlOk6;ZDh+ZRyM0p0@t9kLX=?eRX@A z(i)VQo1aIR7sJSHVeX_Xk)Z6@x~t<_lhku{YN*h#qV9EFA$CLl3n1%9Zn8mY-`jinZi^7eK|Q$)8DXW3^>6XI}qWgl#Xh#W?et-tuU*9l2Yj~ zr!e($W!rDeeK=qFttVR<4?wAsuE#;5FEF-PqE$4+r|zYx)fda1p^^1h-)p189FyEW z+bAw$VvL}LjIieY`O0BNF>88(`6E~FACt5Z?|qX@T4^ZoCs=|<2qrboB1ui6DPP;z zhVNWIc$-=K9sRc>LDKf^9vFHrHG!yGf;6OUh%!r(&NOG=UTC-DubA%OiPi%ZBQ#X3 zmo{JacRbxJTm#XzJ+TaMsWBeOB?N^#PWugo31b~m!B<(m>8H<#8@~uXPa=%A50VL% zZ?3U%+bb^|QAP-Y{#(cJpQ^3{I`W&PMKFJ?{Zeo&2x|E*OCh`fMh&yt6BEBvKnzjp zEuLr8=afA)qSb4g7vGPVn=peRJgWQEuiZy>vE{-Tk8MM-m6#jT$>PWlqHiJ^h=XV4 zaDlTj zM>iB`bMWt|h0qq6_)fX}{Uy#p0jA|M_M#d!ZA*Ll4y0~Q|886h4J@YxWUlRSSjM8W z&-ut<&Kc>*q?F!M;)juX&QzciM$l`j7u;}|F{`{(nE+Al^o0N3F>Kf*&SavDCWx@z zp=<0H~(48vhp70vMFI*gGUDzvT=Y>YDH#q{a81BVKtns6oI3Ar$OP?B#K) ze%Ii~y>JFKqB1qh0@J|PNYfie5xzzXo$SJ(u#O{C?ZBW=!t=cB12xb7C&@;KSYwvk zBDIem9V&paoYqR)E8-f$`@S$whf~!*7t+D94HTiVH6TA_QSno!Lp5_j`#evn8^ zUBSThaSWwH2&B#*h4PwTEA3JFtg8J_uT)3p;v{uIuXS6BQYgm0HEReevq~dUc!6~? z{a;FVlsdz~@&)qZ`q#GSD(u`J^nDt$QB=o8D<_&!AK`G>>y5iAR^sox+D3$mst$WN z>3EmgH)Rc9{fH|)N2YJ~?t=fDG1z$XGXSx2`RiYYgvOKq4pfCQ{jOp;=nRxxM{!2j z_|G_ljkEaysFlmCF(1!)2kq*hyK7Uf zc?4dMw3T6h_<{=qDq%gxF1e-gGKVtUI;~DqsG^5$u?va3EOQPTcOn*aGG%2<+P*w3 zriY^&WPSKt)c$1u`|9^Fsj_6KTVq)}ZiO=qtwn8>`UP{Kub}?eDHalqhd9)7U|nT$ zrikS+=ckABmq<6c28d)ssj2LgvdZ~o9ZlyaO$RggwfvoliC>(YdmMaQVN*Zbkc^5t z_*Q$aMl+w3^LB6Mx6b@R5?`$al1dxZB_3*#ak@t<*a-K4lT?{`eJ1Xh#S_ak3`nXR za&A@*Ez6i*cX9ab26pAd3xh;!*NtwO5=JLl>Eal{!M2?GvUSQpho;POyP5$7=9)N2 zz6KrUsUV>)Hqut^rPtm+BwSJBzJmCzy$RhJG3$q#q*N5~E7mNn2dn>Jr>^i8Kg<#L z6#TCA=2_#9ZYw)SM>H6xPf%ui8*%Tx;a`L#?(-Ta?N0Ql2B|>9z1RRft_#TsEElBc zl*0)9;Kk1Dz`TS!nu3`SzTIri6l6@ z8%h*GhPde)T*9vqq%F)x9K1S4G&?^GS=le+6YFvD{k!KJ())RrIQ+64r4-#m6&#b) zI9vj0osK)~sOkwwx4HbAIaYhW$t@As=G$1StkHAb_^?8Y*~?_+gAwq19nnHzI905m zit4fMpevmG0Prd1elCclSvIs1gHdbQat^*@5V~6r1SJG1vzj*mFA@_>mc4^$aRk#) zBkNI77423-#=dQ4e$0{pka;^boe*VoyH6FBV-v;~E2t(RV;BkmS{gCe4D#XJO^oJa zVo3YbA0iP|G#!h==ETtUd_vM`4_83vnV0DR|E*mLza(ovy_y}3-NBw&YXiVZ?=Z~5 zzt8FiwkbdUAPoplT*pWBR_V(4M2bLZNY-yzGHiBe5^Q?!URGKpZMMPPEIxNMmwfHe zN=apudje6LJ1#v`RA@{*=zWkygOQ=F*+^2bD~Y0}Rybikk~Mk5(P z>I{aY$pw8tGc#;sSxpp6gv3%Zn_2cO0DQHusKG0hN*+7u+0Rp zg|Yyr;qcnjh~3a7|6G~X(x7)qkZmJS#7oTV;OQF!6%jk+TPDxI6XY(U>)2}`l)jum zPwv^I#wgwoPnkK!`rsnitJUlBq3fT+L-Cs|^=x%H&=OWX*ANhtWEyZ=PzvYH7aW>b zZIghpR_ZV{57#N~1kVzFBW;esBi5Sw@eIv2kk&<1%;GRnm`R3TYWl`VUT{J!rEC#H z(?-qhe4@$wORdPAo-%iEpP5FqRRsn2Vv3?v*wq4+2`1QRDl`i^!zf>=B0y$yIsF;d zWKYJ}oEJaEstW)Oz|^I;pTfh?tl@UV;5C?pis6>v_CX^>3wd`M4m+vu!Gx-mVYH5d z>I+orIkzSR*2(fG%VGwOrZz%$vyc2hV!;IzL;C$%5nX&<_(L<)mdaC5P5b@c_prhA z8S1#)3g8X{1sx1J=QT_aY*+{KY}G6}ada5{0g>d%D1m4qk1@8tG0@=|$xMu~bI6J@ z4xp#NzG#o(n8<275=H_(FTK(fDF5j?~h{YI@W6$8xl=!B^bW+@R0vsIvv@ zcN;pi?5p7OMM^+cW-dcC6y?dVa9=T;htQ&PZS;&Gs4XH8Q``uU^q zPE4T&DB=lP!8qj9LE)eRBQOlCr~HT-V=%2$`q}YboZN(>XS9W`oMFybmW9>z5gsKv z1$bZA4~?nT6J@A|pekEB!gH2-qMJLk-JP1l5^jE6_)`KCnbA`37H^dm+%#^hG^U1i3RBzv%-Bylio$iDB-K2iO0WJFM5!;@ex`J;%}L! zUs~`pDGGmT4yO#uYA6Ri** zfJs+6n~XqTlr&{P6c#~Pvx3rU!QiIvaxDlLlE^T@fXpc?W`R!&fnq4Q9B3d1R%Ei+ zDP^6e3N+ta3+$uD)+9%2$F!(fbq0M;@AI~SH$)S7;W#|cAG8LY7u3;Hl0c8tUjr0ZhP13j}XpcF@XBa!E7UXX)7&A9ApdQaY5qqw2H`KBTo>4Y{xy@zbxg9bm6$g02L*FEgi&Kk>-HV!u!B0qb9tS zhq&k%$Z;x@>*a|G$K6gNNWtr6!?<1?h{@zrHCD>}s549{;LA4@iC^+(^!!z^sPahb z^P;JSUGy;+x>!B2;kU6;?OXT)s4~ITB7Q^@t}*DG+-%i}AX=jk#=8mcL)Cd-m)3lK zAudi{^PnUUO8Q{hCOImF>@)31K#uGsw1EmDAYh`rfdPsiQ9olFRutP%-|0~*_Im$LeXf@n zmrC{W>pN-w$t0&!?Kpb*zskLZ}a8YQumliXz3m z)D)^y%VrNeB0W8AWOSywWHxw|*HJXhe02i#u@ZyqBesaD*rNR=gdA-1|htuHT=XUs@M&eceAH=|FFje)n;a|Nj zW&qTG2m^zx$-dZ*s!pvJ3mEdo+W^6)}6$IEiIv@f-F>r@qUja z-OQsWC+fpS0gJ*w8zDBt#WBpn?ex&w`?t%;D`qX?q0q2+E(Sw-#<7OV?_sX z9n5fK>-XKwFqZ2AtNBR6SoWVPc#5u(ymlJREuHSE0^<^tww+0n91yc~HlAkcTjMcG zpYF7(&BjWE*-{qM!CyQohjl-(H(w_wR*PmAK%-Q1cg6t1NF0VbnpGRSDc`M(-KC#% z<1H>Y_??}Nlg|%)COi|8qS?ZO*1uZAT1Zx?>-Y@r(23_l|poKx#J$8Gn5)-p|-I znFnS2!nDC*dk!$xRy_6nUn~X0NZfb$(E>G>VzXbQ!PO-fuKy|#t(?zk2<;_|4<9!(ec0uBM-8N$KlvKcwlnm(@ZkNP3LMe8U%jYoZK*Cpigd zCP(|_@tZwS@=}U+qs0&rNFZ(s$TH=%9!q=G90kpV%U-$(@Uws?JyHami~I*8GjHwXkc3gF`3uojRXOUY zJ>U-_a$Ke=-2|0Z7`k<+1vMF#)t;L))MXug%Bd8S_0gXtXkhKUp@$AX2kYqjb7Uyn zu#~)n;wvaONPQXoe}55(XkIW)GkjbAMM-oskVGxKi5EY-o zJ+wiCpDf0mQMq~y9Px^{5R%9!zX-VX(Kl*zSNiN3N%tKW!gBi#Z=BBZ=^B2aA3J&U zLG?r5h@28}AnGC&PK75!MuH~Fv$C@BA5mcYVikYIv)8*hyD;&^xU#D(7J z>h_k3I6vF`YXyMCxisCek za<4lrXTZWZv^$*Bx=R|_c=o5&)BOE0;n|2q`JV>)&Q$A%&q1WvbT%O6L!udTBT^)L zbOjpE8!;Q)%Ek+2d}v&eXI5AbK`}3g+D^+TOp5SX>yJ-2P01V$P5z`&kWgxVaoJ6IHHD{CKxN`pNF~BTeWjFW#@ORdm+R7oV;YN2^+5mkY3bUo)FJ zR{o>@huQSQ;)8MIX%4@0mXuiY5jp-`O7$VZslrTV;lY^=dLsF&(ODHw1kaZf;|{Po5*ijKg>>NUzfmE-Mp&wAu zvo^Z@JmDV_n+TPxEZT7UO3;jO2d8TSVLwXYT#^a-0y7HAq9#NZ@sTmkRqHEXz|S8V z46je^5K>$$(TwfHZMXVLj3)y$D&M*!Z$IqcUnF=M)|1 ziadk_i0D|qgl*24&@Q2Zn3iMzPM`mJg$4&~+P;H;+uvKYSoj-D~4&49aqHMt=nm5^NzQ}z`1X@pKR3=^bJk>|HKk+GP>7y zG~NHfb94H!YmF8S)%Y3OhX0%GN?|9Wwi$qN3Fq zbkX?~G^)~vm>PYAxUwQm%kFj;R$t;|(vtVv#or@E-NzUZMLnR-={(kw4Z9CY=Af$jSyS)n;07UAyHBBLRJ=MU$Sm3EWA47-m1A0Bg(ZZl zbp(Og(VVW3m3(+aQIJbu$w1g{D5uFlsQM~A!m%%mJ>G|`PqKZkcaCD*jTKkSa z^&(~fDpOrPXx%5mXpZ;3d(weck<>hk)O}H^C>tC^Q}+tZBMh**WCYVreOgt% z`1@az5ff$~4Kb(m91pM980)Zv0~c0zc~QMNz?o?!MHf2K7q{gp2?yH{@gk(7!Wldx z8D$6*-K|9}SF>4?PK7$!%H5uHwkQXH3Fm=Vo@e=M;j{1bL0o-wp-s;E^@viogVGeI z#JfngL;G&HgcKgY6)B1+d1LOAJgCqmqa%*QZa5YZk7~t&j*x3ebQE}DGNXlCUhl?V zK8RS#(Mv|MATBj(0HZ_T;3Z9Gf80hEaT!A1Tl`$BD858a_qllE{SA@`>#rMrwS;-Z zXFW_i9uoJR-5Ey+4TrFyY!627+M@K9CSrUtH1rq~jKDLXS&E6kRZA!t7){znu;S2IFtpQ7V+z$ z-jQY%0Ir@J**rQ&kEQkIs||x|zJBb7m$B*xW76PLbu1{ajvHZ~s3Y6z>2i^B@vpwC zy^4MFWH*b4V+M9q5`Be(0Q*NLci+od9KZ{}cD@ems8FVX6rOs;V)nsepVQe8>iA*? z-9t0bP3RGTFb^||PIxY*G!HW;ACV9`-qRB=H zlCBo^+C5U*a`he4B7GuAk_Y+5)sCDBl#J;f3A)%$2cljzG<9AI(Oo>#+4M|MYl+t< z>K`pPn-Opc4USxRdt;NzQ#=n(T%8wrRFI-lhTUH|+*JyEL@;-WD{t}v%P^s>3(soty(a>(ic>FINDUeKcXF^rzR zy2@;1st()!Be~D3T`Xap-FiZ_Wh{!+py&UUBP}MWrY%*lR6}U!nk_8;C{v=Z)Tzm! z@T;7(h7#VRkd$S(^GTVzK7;QpeO`VxhFfd}dS8XDp#&^10sL9nX5=}-GA#JF(y1Fe z5m`l8lUO?HerVl-U>X6VJCcvberVh0b7#wT3jc(8%l04EiSqs`FP3Zo@N3#xpJP&orIdK zvZ4z@4N?vr-l&B;J1cH|<;CSzJEH|*3EI6Bsvdtg7|@T>iDOkA#2QwloLTRx4)D+B zis;gPEjkPPilghxH2j!o(?-dXBxhD(Rlu_@i{CT+L`1@IV{q=bdH*BUs~NB zx>8tT@U)s!_&P>BcP{j|{%Zjo<6+u;aI8lM81Ij@%Fsz}=^rt+Flz(u2o;fgMrbXa5)HP(s{*<1(p6Naih#M1uL zOYq<;_1(I%0`v4^i>mIW#?WMo^Gte&^9-Ms1w6ybZl5$}!+c3`&PaPIaNaq+~7Mbd7&bNRGn@|@mp^ZvRbe0;{PVSJq6W*&Q7 z*;A)^Y$+2N$A+wshBN_1J)}M1fh2}p!v3a-H9)xMdQ=^9Q-~xoB|&K@*7d&HpyuyB zesA~p><%qf?7(;`Q_-v+U2JtZe@_TC4LGcC>p}ximWWew7z07@JNGTJiRT z5_mMfwAdq(jFIo2{Lt&!Z4XkYOs3cs_{e$)Y4BCLrG>=dF~}D_U>L&9X&KzGy5$XO zC*cvkK8**S{2Ej4U~@`sR)BbT!(rgDsL+VOPOer?0e-P3$>QgR~nStAJ$A-CqJQS|ylEzJF*7a>D4VtvNFWi1{Bb-h6RAh=1=*E#g z;xh`()w%v9KG`)|yZqJ(4x!^vC2Eg1$AOj?N;Zm;l6RWzKUQIvz<^{m{?%YElA$`W zaH6O;UdE-K=4 z^4oMRr7LUv`41DJb#92MC-hX*ZL{>%ep4$w+OjFU=maY8O5`}!+)u({r7mqbG~=v_P{$&Jwj_c z$Fpo6Rajr_jH<=Pxtr8?PVJkRdtIZ_nSzPRkrzlv$HSz`c=)J<`iTWyh$|N(HH2U+ z$qaCTukI0|*vQ1XDHwJZ4FNLsXA_=2E)&m2-|@FADD&!ZYgt;6DTuiESy-T?_#_Ej zVlIQ}#&JD62=uygsB zu4ku|e*5F+oITGeT3*QcP^s1A+(Si@_V}{Z?)+Ccv(`tC`UMIYjeEvx$RISYDm= z+P-UFRGm}=rGm%C~I0>0`r4oOi#-? zl#C(r!mBbOme$b5PA%Evi5=S+`4u^|Bm47Gh~Xu=h~CL8CUJcC9@7*D)S_v+FHZ8> zbmt~U@ogyv9mA}fRvLL3h9ad*I-Ywk%+ES@eli-*~66-95@<@6q+SR@`UbbBpW*m-jnPa74juhjC^C8FWKgw<-2LrmH z*(xrN?4uN#>z*+KT~wrq7B_s;cQc4xbcbL@m6(R^->10TbXAhj0@#K)3Y8V$ePGL} z2sCgdox>dd795BHY74rtskB_lLP+`R;%#^SL^tyRg>H4N{bwkJdocR{a&~g>5^niF zwv2Q>W6=?xl*cb9z^+571O{vwEXvvyxC<1Vw4F-SFv#?v8+tsR5U#uW|m4%C4T=m6c7EMSG2W&N8{Z?EBei^anQ`IvCK!t>ZFaJ;-YMi3 zu}SKSLefNMI^ws=_p_-PtS)6OE_mU_nM6SNRL!TePuI3B(|COGbu3h?v*B;yDbB(+ z+Kf-@qw|2g8GHZ34@ZnOV(7~AXe9@&tWA5nHKCQW`wnQkDp_j^6s_AmVA8z?DC zR}a)PUOhO!&3)sBTDxp9c5M_Iy%6`8Isc}SpT}toWsx#)ZF!kMFr_n4@@kxS zwAnQ!@bhk2My7PT^PvyE>&}y-JGNbH{5~;p<<3^Y*5c)72r=eP z*%Kn;+K29S+`9}6)S2DwHT7ch+D;)bKdBlBzq}h^CicrZ`pEwqoe^VY-i7|HIgPP} zG+H9$-<J{)C^rG`*rI_)qG4@Qx+k7>#r9VcUiZ~D~N~Ant!IY*PhRI z1P<$j+~tRgx?Z`{aVH1OBdZ*14Scl0OzPgdNZRna49uY(alDHsAJT>>aVm1+ZTdLQ zH0W9XjGMI@T1vuQYIo6>RoQ(s_rL6i+jg2QUg{FA5_Xtpc&(Fkw3bY{OK+}FR8aa& zQFsS;G}NvrNe7lx{QEB1p3jx(90KLUnVb#rgkQ@EzBn?-q|rPui97C9Pi*Zm?r6Ab z`U1`HJ?h-mk(NFjtv*$|sh*q0^67 z(w&f*)szRDHTu@T>;h21a3f^Y0z%RO#0(!77kjFnESHg@&nM4DOQN4wlOvp-#j-+SBh0z{&z@O_2Ja zRuUllk$DddeqxLKm|>++)41!(>>cN63C>e(I4}`o?&vSzdx>sQ0bWn@Ro@`>WR4mtoVjv=l0O6smF;aXX_0Csd4zJV^K35uS19^t2@C zQjB#TxirQQkctXV8i*^O;OZE?ghf@mfjmfFcP+$-EE6Fqm;7lO(M&AOF=%@W!rckiHC?Al7M6ex zLvEdkpX!R`7_ch-!p^0VE)1t|ri=~6F3D?wL@I+i9tM5)f@eEQ-0-|jw608Pz_I}#N)@W&M!SvD6y;AGl0OB9R zAoc%MtUL}x_IRX{EvBy35?zTtC>`zHP_!lsS5)$>J25cS2Li0;0|2s6`gFfTCu0%Ha@;H$&W`5gK+oGi!_2&vyXE)(B;`4;b!n2$bb3*^iT z9o1O6idnM3=UOIOY(goz7W^Xf#|v*|hcSRpJBoMPe4}p>jotLfzGrJvk5-K|uSZEB zeS%q-AG*zsZUUjd1^x7F)o*T;L4-&?Bsiv8zD>}S*a`DL?>@9^rh zTWru_+bD$q%@1`+sI z4XijVF*sA6Y?zA{lPQlX0S)aG%|-kTNe>yIOX^DF)ft+==i>5E>pp0%6BJloz*H2 zSSrgHZBZ?&Da>%C`(s*=d{Ly*CnlV&-gtf}XN=gCnv|@20TPmK8g)y{(HI~6TY_EZ z={Uf(9kDuK^HHIDz31f5RQ2>-SIG}WBdQQwukl(&B<)6PhxhIa*0dAx;M^>#E?{%3 z950GM>nEbuF(mYbSJ$2K!_crGLQHhV$32N%*Sp$@xa`MPf2XX{R39#>a>!ZZa5=i& ziLf8Qei?EIkztToF4oOfH~Syr_-l*)MI09~;DJ`?q_R7_o^yZdUgKyi<@90;mYcMm zg5uadVh0qyK4Yf4{R=^ZMy-@Pk%G!x*V~3~Kh^gOT-n^axfikNDz#1mjP2?^Oha5+ zd%2ChPg}RSQW<$T<~vyu2Cl6u!3OA$E;RCfgEux(RSG3Lr$UjzRvz(6mK|jj_fKi1 zkF!}lK{?L-x2&!)ao@-bUlxw_`me5%5qApq<{g9ln^yjN0!hEdpIoYMRK++GODKl@>%~Ox)>{slG+hadL@ua3cMP zw?WTx*>PYGWz`L%6quEySu%-SOmCPOkzCBTtdj1HwzIlR;@Fi&%&=WhKgQr&#uZBf z#-bEEM||4erlDAQuHd9*(k5?1?cc|ZT;&4NZyTGhJh)u&?Ix+x>g zL(bXiK21whMbNw!dVg>~zFf-fjOB{I3L}e(@#{n*_VqGl6*4^Cd!yIk%BhyIeWuUc z`*=DR9y7O-^6DK#d>$S|Ej8TEuI?8Q8pC+S#KAl?*BbZft(|xqu2m{Ofk$z2J;?ps z)r-I3Y55ZWluU)p9iwvn+ow42y)gv0q4T#I1+p_QgT0B>n3!5wbtoBPR@x^~eQE?| zH_d`GJx_R@WH%p=8lEh4<#boL*5K#{iiHQY zO6U>wL0w5o_k|tU#AX1;vg)w5ekAzJ@&xQkVde%nFupzs6ZD`VkqyKY{Z+W6pbkl; zm!`KgL~^ja$+8L?NN`d}=*nyPU1hp3yZuDc$qGSiVSHctVuPQWKf=0S)slh(PDd)< zZJwlB73fQaXCi}Zm_>S;1^xiHeZ+l2r|nr74bE1Mc&1q5As5MXggj;<8j5#PEmGbpPeuT`tLGSAr6=?9 z%Gy?puRuDhc_JM6W9oq=e5(b=9c!l6?)MH&;;Lmi)cGq#?@p~}Sc1_usroysTa%ZBtc!X6y^ZVGQY+{4L0mqGwIE_S(PTV>z$g(7sbD^9cfuR`Tbn?!#SVrQd*jA$evA=wF~gb zHxAOmYZxb|3&q-wx4I_>f4qNPqknLqcf1O()!@zao0ze41x2*u+A@W!DX3u!C689P zNmIwdDsl*9K=6z)6hWXJw{W8co}%e0Sw}#)R6bCt!!1}LH#w2;P0S`Q%w|1O7HP` z(TNzYoscn!dsvAu*BKPO%f9Q7dZej=t;|Rdo*-4BLhU$^VOfv?a;h=`X$;0<;bE@T zoEk6ZVz*$J(ij#T4zCox!GXoolqD=S3sGjVXW}3z8Y1dMbTYjV2Czx*5HuL!3T#YF ziBXb4JPlu;!sr?5O<6EbSX$Thr+!{&8T8^>rbho6QiQ<0?*V#o5V?fRgn+Ga-qlhh;b%dEz;QBqm3+~RJ;pTcFLGN z#Eb@5ByvpuD#p_Fj{ASC~ zPksw@fz=mi39y8Pxk&*~mQXd&$Mma$HPidUqbazCEnaR+2);N(2V_RpaakczYDKgz z%|4g0I6Oa3a=<7vFXz~H8C-*O)Qq58kp)I3N5L3=7BzZ7ck!HpW$;LhqT4JYGAdFf zW07H}q*dCA88KFIEEZ6%R>4a~A##lLf-#GXDid{!o?HG6U9pYqzLb!?i zs$hwuWau~(-8QRt)Z^eshmhrxww{XdaYXuU#;1Wf7)q+Ch^`k{QGb4vZ`ZCp|8Apx z2`gS3?eC5hUBY7Cs-?y-ZIG%5+Nm(VtL7{rOG4EH*rbk~;kV*Hp0&}PBlIW0i3cQ+}e1mSypa)mU8 z;~gn6iC;^T@C{>afn1mNdQxmy73nFVm_1MOyu-xMm;O{hHkndCo3`CUOAC-}2<7g& zl8{)K91Xhfx7nP##|1&}A`;6NdHyS#QBy}H)AIq4VoOOxz!GT86v4X^GC94DzbfwZKE<=@3IYh*f<-* z&gLh_^5-GbzF)nqm>$Exf@Fh8aQ+Mcv)}4Bp1PhZuNvJCM;c@)0B^K83cYGpLZv)# z^wsb6oGDY)r&_kRA2Z>j={>dU!!^rHYkdMa)YuRao77sf1!6AZe!5gmr#u|nOlpc5 zmO-p@A1dvHbr8qlQ5omc_wkH|mT8fm>cT*nQqKy+m`LB$I4ukpz|0`Qc>|Tk-(Y-0 zvx|_{_@oUnD@pwJ?kpdUUz~GhFc$YKx1+xSLIiYx2Wi@lRWYjW;O6L`go6c-u1^-E za%P6?_xh5{S@-4#htU$W_-U1+Px<%!+N@b86(#>~*Zqf1Krqi{*-I*4yy53@li#Z? zh1{j8-4EN)ZpJbMP86tllT`2R+;gFB`OibwEqi_6aW8bR&zeM4f!rzfLugx4e(s4M`&j6jC(bRL}E21)th;9~}6kn8MAZ^-0HnK8A`4ibUS z{F1grbm5iHsh$sK<4OCU5NdaeU2l2Rey#!JZh)lf(%kuXzMW;;o!DSl253 z7J31e)(Ce@zar_q-4_2Kd~OP{g!^qO1WuH3o2Plr9Fdn~x6$?UmmP6a(1>%v`amF` z%Q~ImHS;CFir!+%e7-k*c1#BJ8AKa@>Mjhyk*pxDezpc@`pMv7`Cta2>YX>*(x=H!(tJy9hFAEF&fIRC zBCOAx)Ea5@Q78^f>WXvf^=(l@LaG|;rpkkTgNQl+*UhM85H%h;642^FD-sh*#RwPG zq*3(9{1djF3k>@O!#<~B{e75*ia2 z{fFWuAMDmZR@2-zVZFY!w?4>G6!`_WIJB~KSQ4%D8=`!A>X46&DfhUl{~JT#pN_@J zubzCjSlGk``M3Vk4Go341Ms7B>GYkJ#>({mzrsQ3FA2Q(`t{c&KAL~6r17T(&4O5b zLb_XV%#^8NpHip6G&`zH$L%3T2rz4}A~37;_!dOkNXPOsSVOA;6HhDq;gfdK$&0I}Qk%xkcba4EAXh=Z1?fBc74LWV=0btBD7s5b^8+AXXDf zoK%@1|B0yMxR8x(Fu>uB!)Rt!+9a{Ykg-6C;XDY)UM>}xT_4SQ5)rWbBjX!UQfioM zjZ`o<(S$mZ2l3ZJ8xMm7l#ocBqsZq|WwMx|#*FLxA2QIC;i~UMERqKlANAIXuv2J@ zN2wgzY%)$+of%CO$(aSD8CUmNig2W@Av2uQ^3xy+n@i&y1jhXR*-aP?xwRqpgsC833tRBiV0gpYysoSs)LyWNZA0bNCQ%eVuk88 z3*>cH@E)M1DDV)XzqXbLWyF$TU(TrVKBF@I^n;NYa{6(6!TcAj<@BNBpfZmmj=5GW z(txcAiN=E@QaSiGZVe^j>PN^I1dE`~l|51AxK=v0-&6QbDz3)M!0|k1`97MmTAu_; zG&mfM-l)0?@3ghdK#@elli&B`aj#ujV!>*eJ#o*rG)JGi|324FnxLETd2ZJ}*IzeR zy)atop=yCiv-Y3wOvfajZ>tAc@^M7*N%plv)wHHxe_m@;oI;c_rstnM{px`MzI{+U z$fF_BeG5TMh9h~qXNF~5Jn+pW14N7x;WO6ZB!D0Dg}q6ttZ|^~Lj)76P_?QO|BoTR zKOA^3;F58y+vwCPN&bXORZ82g(B}-0?WW-ywaQxZT$+eRs|bB+ns!F_VsIGT!LeRk zQ9P@aFEjpUq^})Za&L8zwRxpN4v))+x8TcDD8GMM30LeX`ML?eBtnYj5Tm{B`EQYs z#C@~}gB4)(&q2=S{E6GPk(ds*R2$0X!=&IDfLG@xWse#}7=~EtuPy<5FmY`Q?yhE) zy-Fa135^xfL5HdjbG1$k1xVi1Jcxh@dxAJm*VU zuigbQz}o3|yX7ibL^oX1PUIiWbZF4nnHLRG9GmoHE@`)@@WKnjvYVtZJqJ*eyt_U< z3nQmxpWp&M}5{U9|A!a!tjb) zCDZeb1`qn5^;E(+dP%{9W)rdV(fXI)ns&@rckIfnu8TOWoi4R3k1R|pO7cnH+iYZp z%Us4&pA_=?^16Ur5{9^duoMuoynX4(I-PxqjGBfF*`qZh%PHgPaCAITN^v72Ut#1A zTJ1{H#gWffbl={; zIFnxN_b<-GM8&ex|8KTB&$X+J(*<5UMMg^W;3)`Vj0z>U>HuHVL-64AeT6+dq~K*XZWVvLA@Z##L;jRd2T}mTRpZx zvI+RNPQkGUTK^;y20!~0B*P9GWIrrJTX^+#$vqy{O$o$U7l%JCF)LS=kvVeUkk`pg z#-Mt}Y3ZNfyR>P%x5#QgWhADX$F4$fjh9KM1fJ2CDy8n=oz7_S8v}yK08-B5rZ`h^ zP!~?@VoZZe2Mt>-zJ^r8-x46m-8Kq(l|%bDS&m+mYbma^$d~~!0p5auHv^6fSwkBc zd!|U`cD0xCvL>A?ny&xRG}q!;%3oAsN9UC6;BtoD|A}m?rC2%nk|eJ@`Y-6%)>6c` z=_4jt{PQI_{#Asus6K3fb)QgoyUHyz_na*o5>2_udv|S1CAApezm}83t52cbShQ@xQv}m9r*E&y6o96f+Tob*lYadg&S~Fg$k-puVQXpJ##}Ycx3q zn&ox01*W{Uso5-EtzT=BC(`(x%ZGQ=YMo`M zJ9fWx%_B3Z$|)tmd;g!Vc?mVBBsV0@N2?5Xt+V~T)pdt=!?|IDYJ*dRmI)XQ2Igp7 z&oT-(BF%}R6FnBE1b&t&nnKBt#KOz$V@S@8+&!lqLlzhjuxM=;?^B>-3I*NLg;;=K zaOjcB3Gjr^F@3{2iU+@}+do4>vLA6PZe!|50O-y{Oc9g8$1(D z!^|*`?xjfpvik$M`AsmPieSAiXz9XX#3%LDj?4k<<3uuHsIET?#ARg5<{&)W*;Qxw zrw4`vGs$l3z6Cf&1}qfOeFv1$lcg`Xj1VmmEl`T#ic)=NP?{k#A^9u&16R4SxIhrw z72A>473-?e2r5RJGwU&PP0eCYo+3>_p0&3J6vf;9P(RWL!-F6g&WM}dDXP{$&ylCa z+DOFYaxSQ7z~h>n*?-2X7J-srgjZ)&OGe;Z0ZKWdk5cVfkaPrA*ZBnKB^NH~@t>6O zVhww98Q1Wl4b$Nn7rkXKEK^6SA1>h!c8d0@VVIFlDg%lCF)L5v5xM@UoYrLvi z%rb-Y45(l^R10LJpl{pY>rl7@qk*7cHmR!|N9331wkO<1x9juwvCZ~P2qNEB7A~T4 zKVjM=KI`e1?96B2Y;xze*~aik=;gE2**G^9oi~WF^DD(;R>28{(KTY(DXjjx`!9_P z0lp2TXkKfzaDS23^36N2`#Vdsm4XbX-T!6%w_II99Sl&3DUrv2YY0hFZ;=DHf zRKaI9y1s2y!KWN2nethe?43U6NOF=KbHG>4evxz(E{w{^Aa=N(eT%UPkp;8k`2bo~ z(oWb);B1Ut#r-9S!SgR#QC(*jf>KNcAn(e2;PV?+zK=RtKL_wr1<_FZwn8*Y=RAf) z=l8S=1gc;S+Wv6s-I-|86n$`6G?uR4*Gf}@oo`|gXOdIXS!H9wYPo2A&x+#6sw$jSQ~ip|Wp@y- zY}CSE6HQS7XDDt~(QI^#aBL@ddkE@$-Z)t)QgXRmfbeCHsTCs{dtGwt<-)g0-Lt&R zgz&`s#N;5SAt>Y@@F z9=Ca-Nz0P6jE}U>J~r{ZyEs8;wY~a0e?j26brRH&za}<7Fn1dAB;jzgGZEf{ov^1d zdlMt$4{9+Nk8c%QGh!G%Yem7YSCa~*yCt9Y)Ct{oH zg4%${xmipB7J{cphE8amV3H&hT>;;|&94YiXKfPpqr@aqpA7i%6~pSJ<@Di9DXd)^ zMPByeJW$sR+?MNgE$w($hr9x8YBpW2Dk!19=4G_G4!YsbdLS2|%Qzaq*$iR$rJP?< zQ8CI=R#F7`-(Rtsp8Yc(^J`}^C-vhE+~xHyL&heTS`dwQo4oW7vbGuSjUbR)i)RQmB(BGyM>rxYJ?d71+6SmtrE7beHZQB{ zWF6*gNHYBBOHJ@mqr3E@7ltlKS9rGrmQ&AJwsa9hyCDkAjvlQMZ5z3vrBMdnm0T!_ z_+mhE3~9MI(=SOmz$5nvKYqyv7FK}9GM(_sla5cRFT~xOXh~|+*9o(jc`ZmRFJm{J z1&4EQ0rk~M+xEVx(W7jv8pj@dO}F{)H_yKr%dVQEf^ZVY3EDF`^?O36>>upCUXo9J z=9^vQa*4CreZG)l@={`P6x!}tqPw$+Oaccx+#H!!i$xE;ya#e3o`A-DDL}B@i*mPf z5t)9ARw>oIuZvO%adOBSC%2rs64MM1#N)})@Y9m>43UY>AZY$<%H4*bqaSq6X$$+e z!%7DG0_?fQ)4}w9;@d7peMA`w#E?e2IA-4u^#tz?%28oZ+{yd5)%tZt?89AbSQ147 z4oyZZpIqGBLI%@v~;&`h& z=QTM7h02PTXorMcIyZNwn^w?z|o*~&x{C_7=+>r_QE2Wn$q>jvIt{&JAtLEl( ze^9Ah`gPXH5$zvMnbPjf(wkzI;X&IWoq-E zRCJlEOmj$!fhJpwzY8w3qm&N@A%1JP{;|RY(NMMOt z#kXO>&d7@q7a7TvTb^5X zuPmS;q3*{BADm3QNSCaOh~wk5E8{i9X)fbbFyi7)F@s? zs$zx}g^+pCWI38tA*sJbe%GKlDl=I^<`t$;9wm~lPNI!z7;YiJjFrMHy8;&yu23|C z-r3OjEB39vas*T?P-2upUaytHg(XLvX`0=`Hko8$eW@GMh~E0{&j)kTc>2EiVe{`r zj^SRI1kVQ6Sy3ba@lASJwsd9MDeCAaQ29W)MoF2xLOe$lLi|d-tqg2AcE_oeap7H< zfQ?OV2ubvJH9QzgJr?dHg3(A^JAP8w&ah>r2f@JX?5C={>)^;`J7M*YnJ{~8@S z7n652($C3g@cf8*MhcQ^odXlH``~X@A$pkwQkNqx!B}Nm%Pi zOzjHag955FA9g^?&{_qq=0SBS!f(}!1_>8RrUZ?Qk{Jsbx0b!snBd-U(U9BO{#Wm~ zD~R9XVDJxNmu}dlPv^h%F>Q&TO;`U;IacV>y1*srZ6<%~595ZemlD{>bW4=V+8E$A ze1|awr=`QAy!1jeQpjFz!9mg&Kr+zx37SCCVej_00fal#z&MJnNKPnMrX!XTk~>h( z3n9^KYF@6ipomIf+k0_VCZ31l)eZzQAx_h-oPsS469}WC%#)MmLm$X7c0YhrvqKa~ zCLgboL+ua;0qHJssj1K?NNf&*iS(;AlT{dI>0m?3;g!&EOz4|4MJ9$_ey|;LMwQbr z$Z$Qd{<{}m^&*Kkxu{3DF&pC4$-9}`5RMAUC*KyA^Zn0aTBZKsEJz3OAXOAAlBVjS zLZcPnu3lx*c_coPj)H)T`1ARHP?Y}?T%{H$pXV{U83dUSg4KDXCN!YHKrq*Fhw!*K z=E>F^e_#G^+1*G;&8U}2$Ro^_Xf4aP6;;f=gVjdJWK9HjhUgE=rk`f-7Bcu@KuY}g z8*@9=IMA#ttGwGD_M~p&wLh$`?^p`q*HkP$zZk9SMc#3*h?YtgEG2j|N*uKXd^<-2ZTwpPcmhcL|y4s%J#p-@dysWn{6M^{yJyfTfP zn7ff6)>VRPdB4d;<)*mpE9uBUcQ`D-bDE?htGW6fk zCSR9fzcu0y)R~s7YY7VW0m>_|7mT=Lp@BI=$_2v&ja+d;YhVkQ!5LByV-affR4gEG z&IYze*0mnCBHXpb`c@y`H3lCw%Ap%N!~=#1LW9{@862xRI#FDPV$vnB%~460tPUE$ zNQIhIhe)^Os4-~#ySbJgN}(w#AYU2`KQV zS_;3l+`+R`7(OY5=(-u~nJFZtxZ`tExSD4aB9_lq2>ID5M4z5Q_E$=w34s*6Eo7>9 zMc`zv4K|MLp;{R5$^`~QHWtk`4N|jNMGX7Q07zR`&s``Xn@E}*EPa4emMvN(CzBI6 zPPQCyXr)94sM@uxpuk>_xB%~Q8X#!yVK~pBdA@x>UHVjP9!;vH6R_4u^oj9>DFc+8 zyI4&bpfY;Ve*l>=nrf{?ia;;p2LFrsRIavT5g>NWssJ~W<}}ijqc;H3n6d!S?=gV2 zhg(k4hf`=4;2jFaxdh~*qBzwQx{CujRh@ugqOBI_1CLXfU&{b|2KIT47BuY-;=1^e zjanE1G2T*!!O$VdK$t-4Fn~Z~Ha0~iB@j|CuqwK^ReDj}vy098-15;jYW-FCdQo(L1QP++QCBF903s7^b)}-$35eUUQ z4B`Q`rWh!s1_RiXl3Gtv{epDy_Y~9Z0zf|~y7L!E*81X#@4*$fgFF9}T=5H}0(f!7 z_uvZpF)EaD{q)4Tu+4d=IWDKZz@zp1t_R72ks^epDH)rzbCdamDxGiXT)(Yumc|ylj*& zuJ|5Y@nec;Nk;EfxOFu8X&Jp=T=6Zq!rZQ*^-9tDm7hO-iq_wsY=40at<2M3dn`k1 z&Xtq|2*zGHWL+bgl~vF#bzgGKw&K{LoL(7P-`iCeTQ;d`3++7O8Op7#Sx>;u(p2eV2*+v~WltNv_N)#LDG;->kT#nIG z^`?UFTG6?*5{`ztokfv;a(kmrHsyY{z44Ww>-D+(+|hPQrQBL>l^|~-^qbKEnBGFF zdk=BfY;DnKjN4kNl6%&n3a(_qClW)ULIi)g_n~37Uqx zt>kit{bAX&k<3QT_Xb38nZJZ|V?~)#_OUoF(bX7Lw}GQ~r4-w=NSZIycHbdMR|yvL zLpn=tQxFwM$}}Alpp((1Ym)u<-5cn+2ryK7%|IKKn6rVoc{Bw2M*gb@6hUwjIz%+ZA#EUlDw&5BJeM5^ldL zaO>Fh-QT94%>6AC?V&z4kd5SP35HrL%?pgeyh={52!sWU#VG(lCtQjjtOG%p1Ksw- zhhx|2U6uY&%(L&lzha)h%3tyYG0#6Z=8z0bjAzAsultjq6?47r1bueQ_48tWXBq0J z#ynf|?3m+c#+;9}(Wm7+n3nUiV*aV=?)9SAR*cy?pjV1LG)Ynd^TFUx=&J;af!R`% z(8Qd30mC%wRrf)=4lIe^i(iTcZAP_*Q4;|%okTQOrI>00F!wI>ngT8-Km-^;e_$Q|+wemLeiMH@1vH^OEJu(2W~*IROmU}kAw zv}7j(u5;Bo56$JlO1+mtH1dKTjYGYv_oE-co_S=gf(Y-8xmrq)pfO--0M=5`U@v1q znIq48^Q2tVF2GU)vw`x}Q9I{DaoJfCLD>qNmQ7Md(j-&R0DwtsvzUXAsj9A%j#O|Q z<6aG{T}UL`&GDG!BR0TS#VltS*(n-Zta3Ji;TcS(f{V7v7RBDd`Nc~;C=jND0a_CW z5RSG9RCU+}u#Y<~@5Y`VvgLAzomOECLjINJDy*ZBV7z9D!e*DyWrE!rEJP#v$>^-d z+{%bnmyX6|YkoWd-uzT5p*BC~Z&@+yN^>uPHE8XTKEjyOwt74$kmXvbYba*rNE2yL zZw8}5%5ctP3Bpx_b}36zJR7`Ahco>iu*oAvoc;dq$m!Jghco?_0`oe-3bU|5eexND zmTW2s)?el1q40Wv8Jk5tL85~LVob)9llnLPaE`7s-^eE`M0k)DzdX>d&;8FG5448J zZ6-i(hYEcJM(0o=nmhP8*5r}_P6H_Okj<^TZmN@tQ`tj_cDp15ekX`vCo#fWWPTUTUdyXFzG43b1n4QoJZMUk#91CUdw3k2-*OdUMI` zmN9!Rwe7DO3CJGMBGurZ&b2`GJIOv{BDEyd&vY|U!#?oBJXT7B79c)VSRA4ndM3HU z=-8-7GSQq7U4~U*xYW_DkD-AYO2iBnQ>xRs85jdSl5lOzT72^rjG=n)P)ZEkuL z=#p1fT-ZmKYL5b*cRvqwu{Q&`lx3>|uwXAyB{Ckk*N9;wPHk!?8Ka>6r3j*BGJvlx zLUMV>NAoW~l6UpJ&l#?Lzl&8_E6WMaQ_J0|ig^T*X|9uRBgq%bW(Z7Y%CgD`{_qJH z&_Ov+JzOZnT(3dEcm&Nj<&ETc)jnw1mE>gy0nYU5Gmx}`;eZqPPfOiNIcI%Pu%rv_ zwya=>!bsOz?yls84s9{f$wKKoN@X7HtRPq%b$M2f z37FaId9?9z6biMfFk5T-)f=|48|Sl{cX8C23tB&pLg8NA<$;Y|l-;K?CpOKyVwJKb zI5duGcX4z)EBbkqJV6NAI*-yawiICUMPuesR&Ty{oHxsy25q_zdCn8)XnC*U+<{u= zQ5P?t1@W6m@??$HhCJ`uj>=uSEo1AT77Vz19_`E6O2rQl%ko}TF5iIhWGQAd@1i*G zf@z^08a$5{&265iQ6;f&P>bIOY6Jao>bL(P8@*ytBi`buF`$^ z&FDUUrqflR#pNn%cC8vMVDuE%v|!HO0=RnqR%f>7HNe?X!7z}&>sxc) zQ(UjKJbtJ@- zfbmRe9!iyJhGnd(rbK@DujU&CJ4_^XdgHVmk1TKW0jG{-Ox&P z()Q2>5oX`U7hbW2j8%T?!@SC*{Z;JPF4rtq1!<+0@?00K(fA1J^iHtR_!fRN`b>W+ zscs@j75u+Q0crn4t_yDUQ8c_>WQ6DhP#9tOA?>4@qb}`Nlb^g)OwqEYHSaEUzCRJk zy04z+4Gw6{!T{ZjAHN7dhVYn1N=89+f!?T1I$ak#QB_X@eSr88~!OAIMaE3m+!GDVOmu zeVc|bBBxP~g1na&2M0cs7ALpw%jhM<;_yF38MCM_`Wgbp7*bh_-e1pQroHrfWES3i z2DLB!PU+NdK!94fev0ef>i5`zy}YwP{$00;MH*l{)Sw57WG}B6_*BksEv}MZ769mX znJIZ!B?l|>G~!|7Vd22hUfI$qSMa1iNr*mX#Og`0@{jkD0iroLC2M+62WW#>3t*LX z)`U{$Q#LBsL!NctBm;8EQtAw2NH$8wZp#jKve+6DjF(F0Y?HFCN7&0!08o6SVD~;G zfb986O9+}g@17%twQ*t=z{Xm~% z1V=l|+i#R-Me~@XM-TKO^MI5GHhxqZsNTo?C6($-_b7%em%o; zSBUOi`blQe557N0KgrcEkPa!fi+N5_FK6`Kd0x2YLOP`3@fzO`ONZp`UfVe}QCOuj z)RuA*&8J$ktKMAXekyU$_iC&9n z9R%R9C>15=({!-Av$95z_ir?6eJgMlW`lxHJ!!wlg#vC8#3=1hw$0z>D#ZLZ@0R1b zYnGovJs3^=Tvsx@aO?0(2|rE`l~HZ=G!~i~rPp;9Pn@a-7Z;xMzGwqH!TEunmiZ~k z(ee*000Fysh0KmiyVVf3AFfz-#-)Dc((#v@f4TXWoqvThex@^Cy2O90tO~>PHzd>M zo0C-;Wg7Si<2qV@r3emw#04VkMdIprLmt-9Pb1Z046g8}N9rw#7KkqqGlRlv+RL!W zp+`#ZzTyLK_MaZN8t1r;IRW&S);B-AnTrKLP-Z84ZJh5by`QXRv6NNq!b@h_U-R5D zB2Y=E5h)hBGuPimx?FpX7OS$kA$OrNFWSB7zcys`A24h3PC=cUF9Gl7G)8VW(nA)E zzAT#jQi8N>rfkS*_GXhFO8i?&Vj6~)Ih(y!Bw*5=Eaa&R&S&%-X~>rBlZKe@?!(H~ z#-%;hc%!9VMd{fl>z1BP|B;x{<1LO?|Gut>c7*gMY^DdPx_O?ky=d~5`^wkQG-;23?Q>vk5Xgthn$8$Z| z+<`=sg2cC$3KcfpDLbnF#A>Y4u4jykb9;yx1MWr(<=t_Hgv1Fy`?9%$nUZNWy*r-L7RLg&iE6b<&ZHG;lnPnYnYkCyTdUvQIpef?9G%G-WZtrHCy8B*Cea%kGw_H~VP&z1Z0WjpUS1UC9$ z3@tkv&TL$OGCN1D*}Ha&%#d)B?!1oPLNBLh!L%LEGU}dxLPU4I#uZaLE%b1j2Sa~{$vhJWyUaxJ$Tx{#gnYPUX zWu);><~8(X6>LcHi`72u%nskzo{dlOcaep}G_eDSMj4qcPl5AE;CgQu0E=v4az^vjm0Ex9sRlBi^4fnRP z-kVY`tRny=hy^|9vKk><8+4ZtX`g%nHKDufy$EBa@-*HxXGZ!o-Cs32=y&};eIq&sxNqUTf&Hex<$3RcT5SX zN_Yagt-HG$ot_5^Ji93IPDG`a((U&+`MK^(1{i4%|rThE9O<@ z;9HAIa>8KJy(e*RQ+2vEE(taUtc(IPQjBGAa&7?tG|GtUMb0joFl}quq`W>!pGEtK zUvl(iFWaYI8*Sa~l+Z9S$FYf}#Ohd&j~&gj?~w*g&)%;Vr4A*m{I(v+8!jbv`iV19 z`C(h$@KVgHMAwh2t6Z(y=*PK*5)Fadc<-}-fHD_oQExghIz1zc&Y$A)xlOaRLiESj zp$prnpWhSwm2DJ5zwsQQMp@GnlpbkFezC~J9>I0sCNnfA>FM>6tLix=*%Ku}Iq2zM z=DEr?@PK$G$k_OKU#ZgPezF^TN78I#*A&)q(NQ+oU5o|{lF`?&Twi3R4O0BT3ge!+7OIB7v^I1#CUfK^d+C(mj_LaSrzAiLqm$k9^ zj=DpQEUWia&+H2d)4rf^$G)Jkn^JS+#iA-_n2)H;XV9JX-G!3SQM{igw`+m?AGx>7Jb7e%Mo2qUN+kKZ3eRT*Si|0jJj)san7@)Ie*ZS;pPt5 zgsZ*fcbt>*&?ex^{T#`z@;JZm(cyf;!1V5n_mWxeH+SwQvm$*M#WuD675mqQPmULB zY@$B1?fM_PA$HzTo!H0@yM4>a&z4VL-JAOamL2uelX|{>tW%2f87aPJDefqYTbUFl zsO{R{hB-BYCx%Z?t=MJ$F8#6q$(W0=s;WoC*5Do))**2Q-3dS}T5{D-36c$;EI{x} zjwOG3^<-eGcx(4S?C)toFW$l?axHx|GW?M(U-kWdm(sX; z*a$@Pp1FsbCkhy|JY!AvYqP7(!Hd-?^Ow|q1$=q79Wua;Z@*s)*ID-dzqJ;=j>+fI zhSyWd9oW6#WYrNG+x?Mqv&iAg-J)X!QI`OD_tuW!9TyE;VK7UBgG=78e9GvS&-)?H z*X#E^n0Rx@XA++&@IzTVpJT=C1CaHH@(mz6NR?485C7|J&TbOku}zT zhwrYGEvDt_PMH7aF`?G)A}6|Qj$)1GWyI@zT^8sZ<}*J#*JsK@~h7MfAW2L#?S^{=hcULHtOP!^rvyJ;cUia)#h!a zQQP_aS~bUQP^1;aGp=t$720Xb#Qm=HlO!-Jk}|{Ym-gvJ z_6~$e!>=>7s911vmM z1>R)S9ZG05AE1QR%{!FPmAyd;{T3x0od14g{{MgW-ekv>EKLu5ms;22%)?T(D4~H~ z6k6(DwIH0a6zIr|L}p|`TIk*T`_3^V%{V#f^tim9?u@uj(rCjO{&~Ez+kNi#+xxm& z`<6X*yaN-jHkR`~c6L}%EPqpBPykkVrS4BN=lEvFyISmadKr;jy&PlzyUWY|%WIH$ z+&w#dd>sFJ=Y2Me?3Hr8osz;bhDG)t93+St9k^0u`i@8B=ZJt)Ebq8L`G`oAx6Yb&>t|nK_|p~8 z%=-C$e8Q?LED`aG-B8*A{RSAsDtb3Ycgnk!=wZ+Ghh}ra^*?Y2F6{4k`tT8(;UDEY z|E`i9Gj0y_bDU@GJdY7QoAX9qeQ!xQyW*mhqF^N@-Aa%vcjMm zI4@vU+U8;o38d1b9uu_q1C zX(mOXMXMfcua)k!N=y=@s~~anRDKk4>_-&1*}d+VqAfZ55U_8R97blgey|PrFKxHn zVo{sPEWd8CG0JHuz%$S2);9=@wEluUsr%#HT)~DZh-LW1^~MP^gR{~rOBEZBSE*h zflyaWC{?onJG5(`tTcgDmNInT;02`lS~^?GRktfZubTsQL2w^rPI=ALAlA3f6(Y6m z&3Gp4KyZLCWO@vh&P$w9Bc1k-*Po9NBApKzLk3^P)3hHoI;65JK*~)-`H}+k)wT*P24Y$-qFa+$!G>nh)*yXHz(x?Nu`iQ`)`^*` z*h(4(o*F}cv}GeS$y3?&4%yR*1_dXFoY>~5=FVFAF%)QMkgBj5M)um16N#o%Ra}oP zMU0nK#O0$Zggq;ge~*0Z6y3woLb$1(5wZ#(S1sdd0E;x2Rfx-wL3EKIhQ$Gn08Y}? zp0QbvsyXtJqTaIfVGcNahYGemZIGsW=IEgi4<}23uA>agv#P|j8xw)a#b}dR$~0mJ zs&`%{W*&wTfbQ0+r%IJxD(ZcQ3Q({mx?IE&ajF+sjj>u6o~S>3F4u3Rcp(eqc=;s% zdzN`TEq=a2etP!#E9ggne%x_E*836;d@G1lA)ojvd(NOILN;KqAVsVd(yKSV0>zE< z*+a8$6-=V^eH)e(6X=xpwQG=3@SK>nh88@Uvawnd0`!`PCMrRW4^mqjJN2|!O(B1V|=KQ0c1WFacL7wdR9|eD$~q(wPq; zu||tKNo=IG_L&mz-z4@2yxvb@qX6iF?{2Lt1Jo?EEzq~d;8zGYy~>oO0~ahxxu60& zAecv8SJqwUZGMs3e*61hOWR;+kQvIaC{;mVaJib`0U|X^=mo4j8*7PlSeB?j0+h-H z!h^R3A|W*5;!iQHwp8ExCsxLGV2Ae^sO2P!fBNOmzwfx=Ie$FzU)yb4+Ys)!W(+aL z#XkTHx%|XKcjnpZ0;2M9Fu~^%-<^yomToMMU%2n9l3S}vWZJ(>#E{tpp*N$%RGZ_gT^PO`#_im`7pZ4E>r1AI-MPuPM`te%IlQH z)TXNickw!!5rmmT0_|@ML}m#J-C{C1CefiZRIik0A#RHWubtiNcHC9$*yFvO9Dg1y zV2-D@zx?*MUw;3;T4}%h<@f)m{q=7+ZKo$(lr6V{=zNyGhiyfM`exN+oQax%AMcC? z;ekZb0E?Y<6-fKQ;S~0wQl@~c(jl6Zhg<214{D_+9%!C3=GcAcifPPJGr1_K$n5@R znTa4mf^q_>EEOLi>_`*95g`>o64Uxowuh@f%p!F#R=+vN1WgEnlCV(QMth)n<3PorQDXf{Dk8tjDZMsId7Db#dOhupP0u=YpLI$- z>(qVLDf+Ba{aL5vvresNopR4QwY_)j+yuqUvW$}jG?~R!-7G&9)#n^9mr#0HPyu}+ z5I|q$RAm+Oz{aIqqxZ{xPmBTE0$TdQpo+pEw$}zVDAJE^o{jB!*Gsa9seWNlkM!VP zuMDaH{<&8>^$!gPU0!}zh*8umHWO_vMMFyo6eg99t@<2d)zE06eM7BDP5GiA^HvGz zpao2&X?7{lLi(Cw^#JtJI!G)ewblhLj|R*int*2ztZ^Z9sKye)rl`OSM&7C9Wcj9z z-?Z_YHvagBf78adXd|=zpaKC-g2!TIrIU^&86TnKS7juKn9tJNo|&7i@QcC!-Z~Af z&N-z=JbQs^?UItE*a|T+1cu6VqdRoW1kIhj4HeWkR%l68X$CxjD(5QmD`}$_jDva^ zZ4|5s^-UYUY2!C-{26H@vv7=QY>3xv3j#u6Hib1UYt=nO7AH5QDUdh=OmZPxqh$h} z#DqdE!igqRE2jJI7@f)VOtx?}qDLB`oQC`ub7fRKKXoTfW;MwWlZzd+@s)wPMyRW6 zBWXr7)u!NV?Nu=j$dJNYkr~*UF%zr0Nv3#P1>|W^`ke3+*1^YIyzVTM4-(*G>^X%T zqhv~rmqm6FLTf!kt_}*^siz!@777A;z%mD2x_oUA^X;`UHd|>E8#MxQETD{)Kt#4( zM3CB8RRir&nq9GL@0FTD&@n(AiIT+^OOMR{O}M+#GFB{gYqZ|h2sSJUYq~W+pU;|s zpMfE=09aM|_h(+4F1(?qNJdaUEs_RAC{t4{P6nu6#_MW5_E)izrb^isrZ5IfTvX;l z`^*3V!)B%|C(=aGSTSFVvXC?=tq74uTV!ZGSjZAmG7TV!p*f$hHZM#5&`dNp*VhKH z0EJOHRW$QHg$J-?=#!bqHI+vFXnOpm$&;Y#2nL71@NiHX57(I*Cx-GGu8 zS=^{$)FmYL5>tbC;Cr`CcL``=F~ph*U{V7CebhzQ+)Bo`je0A+%@-~bJ|il{>ZkpWKPpx3HnArn+Y3sg~{>!n2BdQ;$AAVWDf*8p~1 zb_TSi~>4BvKF_UH2uA-Y@)Xz~~1QT1kzd0FAw-ihw27kQcNY ze?4n+LLO>5;-66387Do9FIMaq_SHfD!sEFs)q4*T=LL7tkLjtYZSI;P?IRT>^>vXu zCKp?bAfVl4&qfhg6Z*)bWyuo<>8;ivA(FW=y0eESfJnPg46rQPUyTNyG{%F2ZwsI4 z`yNT>5dLy^o{XspmX_=}gEv)n3KZD($8)0C9fi4;Q38l*?v0s^>Ux4a<)SbcJ=4>> zXyAXkdI|Xe;u2Oor4*Jhjh`oUQJ=+A-C3;rVoKEFLw`Fs1h~yqHEcK_jGbOMRa3MQ zT?>l-RRwH6NMvm`Xu(vgSRrPyWk`NF7Vo^o!Ue2d10^7slX-+Im`oH367xfMo96E9^AeN9iC;EXfn|#kc)=}}3p{;>*Fo397TrZ$RNH%k-rUmdENN^QYK=c6W1tGAq zZe1@A2l+e#tG!gO%J#eQWB|?=tcsBjOE^OLuVu5 z!w1O_aPZ;BOHarfl_!=^yWi9Ae?Nrg%&?!o|LO0)rr&@0kM^I_-~R8vwKV_;BnOJ# zIOW&h{{3%j6b(QB%dfv3=-S(W^8NZpew{MGE%((Bv5Hp^ooF+$(CMR!&}fo3Qbp#W zX~^oDS}~L*NL4Hbso6ck*QWX{{@P!~@Be*=B+%H*l$;urma<^cd93HVQW)NyXHb|4dZIb)TIb|KH_r|oWZm%vkPO9`@VjjcdzXMQ6@ED-OG2Y3ihtP8;Q zPY|>dt#;ryxhNW@04I;9%6&u>rE*TCC2;Yp*Y?3CJnct(zP8WH+-V^|%aJdyZSrb& z&Z}$Nysoy3>-5b*3QH^+ti54SpO?;bts7MTsQWlI#o*sy-@3F4J5Fz1d3=jIt!kuAGc#coE1>ZYsb923(E8g{a99}r-aZQTWnUr z7FWeL)2eSQQL0Vts>Q5+w%R`CRD6{fuoJP-Je+8HhGa#9a4lZQ(sXdvIMbdi?NjL; zGqSLMN`Pd}gr(c+0p#$@iNMZcd$Fq7^&x-EYZDy>-($h(JpO3#BdM4`ah)Oyi zQRQ|R88;mq@9tJGJM101htBcrRsr|EvI^|7dAdfCn8yRp!vjy+M>`vY2STl=Ie zy(#8r&8pU5Oj3atm}FhZ8qk^=HazygCOVj{(T$n7EkGAwlPNL`A1nZKbX13EcK6N> z|KI~>&)%Ry`8xY-uKjw1qPcav=P>$*q_qReg4JO|K}HuU94q1@W*Z;uK=R%>cYiWx zBSs%Lp@n7F>UGXbj~DyMGpN!XC?4d2s8*WS+ObOdgt(wb&3AU*F|*peu1lB1*;hr6 z{&KTh*XSN2yNvWF={hRrA9{7+I>o<-8ti~p&~XmvF-YHC#r{V$xZn}-Ry6Y)&(huvT0|}T&-}NRNl$?D&AljRKZ{o3vCb+}3UeuFo zJw{=U*B&~&nbCsryeHD8?~C5|Jd*6AGx)GO^uST0r;0!AleX;(WA{C|{4(FsJ7+z| z?bc_U!Hz=gIxLP|A7|gU@$bRjLBgl-cJCnJLs@&9EGm zN%1h6lYYi>c1m75C&wYPtPs>&lBbq7qdqvS(RyRSehcvMAGicEsk) zlv6FcQE}A!upud0IJApE-3Kif!G4TCO0*un{ZXUJs;JL8fOMel~~H zSxML+lCOB6$}Py4@chDBYF#Kc5Vc)e50=*5a}KGNXlo@txMPmPxY;x$e0=gys{QDxVl>k)Q!JVN zN3o--LgkLCb%0~($*`cMFc0se&>s{$eh|&xo?{cq$lRg$_yE)ZD`&$y1)a5bRWTPj zX+!U&Kznbc+Ty_Np{I10OOEl;IVyku9D6fqUYj11vN|f7@tne_8z&`lQ!vKUtxNx%M!u`m@$A*I3BCkNG~fY>m-*c917)S+AzFB$J@iwCvKAw%W%vOsXo=HWaCaYk~EKJcSIR*Dnf z=EggoFL=kJ(7u6E!Lv;Ri2^Cw(o|IFHuVXP!p~)9GKAZQqxA z``qz+6w4DlD|>?}kE2PSUIjEsQkFEw#gewWqtk0&=26c^78R&A?!}VA!^7_qkWQ(2 z)Qi(67?8?PQl0l&=V*WvDu#KLXtX)!oxMra;nA;h^*>87tt!J?(T<+*D2~4EOP=-P zy_orRb}uF6dRNiM-*h}H*%QD2Vr^t!>~otJM=~voi#=&}xK^~w4r>v6ZqwGtaE^IY zt-Y_OU%sv0gVBK(f5oK{7rh-z1ePpMT>5!OcK5z<@1Vy{wd1|JTYl5Q?;{u5?Ds9D ztZI#X7^^8`yD&Br{!kOfjEDXD{{Z0 zJzf~K_Xm6Eh!KdxO1d1i;-b$>$+A5wnxlPjyE|O}r4zQE_QYvtTp)yf)9Gtly0OnL z>$ERUkAKypeR|H2Dzi$CxqrPR@G;Yh%(gTmYqKot5g?J`me!@N$n{16UD`8)tFVJv}Uqw5}?DM=Odrx=u49h1u$~1M?d# zq1(RaN6Yl^+GtyXeP%@`j$lQ#?f@SJxxlMtDxgi$O=Fmc;Jxw+^elspvPFvq18UN< z)RgS>s1z!z1Bb^Ns)>7Z95>d`zOjbG*1Y2Fy=!=+9%~P8jEGP!XRrD}W#mOFH7Sy_ zqh0_$LFwmijJ2mvL64D$)q5T;-sND~)`O>vcj8p>JIX8u+ztD>SZMUocfW90J)*+o zfr8^mvnw56=+|=oj1V|PGF|*eAI`LRufGhB7XFeW3=gITRq(^|oyXZ?vGZd`50-x7 zGv{T%rPvTv&fkU?rOaPs!Hl(3>-w)V6tma+LZ^elf19gyo>xOMx_&jcc?8LCue38x zj$PAlTshX*1-HG@&saIuL)%@+^$=>}pg2I!hKmgB8CWI<7txdRT5;lgB270ZRP&5( zz6ndF#5qn@r+gPInXZnowD|TEu-P-dhP2k$y)q1W! z-rrdN<4T()nsBZzX$Xspm^Je4aGbF&P;CxWQ-2Ka;OICS&=T+ujF3A?SYFr4Wrhd2 z0ckoMW}0R^k!6a~I@38`BSyiUb?g`M898cEah?Eb03tj&temZ$0gt_e4u>F(p{(C#5*N^N6s^I8unLXf zl@e0Zz@RCw`7~mPNRx*SNdI>qM}~Jiv;S&{jvCVT+V$KzB67wj<&@|~(Pw@%KZfFK0sBUXN@nc;zH=VY|}~d zApPXn54yIx%zLr6E3H{%iy^1QUe@_$e_5s&VX<86q|52tx@Op*lQE|nbJf9W0~{?3 z-1n^OQ*l|KbJ;7%Uxx6t&W$4}X*_?Wb83kt;%~|Hw?`S&VP2Q!FePi+ zj{o>M5E{+L$)Brb&kPNJ$Y6#sR+Qo*tr{2$q%pHt)Z`_Zhr04Li7bxdScIdYlg-sJ zAGI|zoBbtL)VH6_tCXQeqzf{KPdbq$}%!K=05S(lvm5 zr@dd(fG@7z`s{E4LU+H1Yrh|)`RYpul=uNYQZ%D3=waI#KsH#plVKH7U60lVQZqdy zfSN1TT}=zrLdS@Ec7e>A-612WTYXpwaQZy+KIM>o#=V-laW7)5+uT(j?>jY=aVI4Zl|dOXWs81ytm00~(5selDuo{S=#v)fJtaI)K09p}!Ot}T zJY2I~Xtuok5W9h1Bb24T$f+ftG`tcx<9R#CeHNE#?H)8(ua-F4FhFbUhy@?GGk#@W zHgR!_f-xQY$LenSMUvU7~l=%F8g(YeB= z)uL7gR}G6bH26Vr4*3IH%E>)xf=xmvGem=g3zcxA^&%akgQN+}({{ z4l|#{AiHa8#WZ%R&S%%^`754&bry6EUt1sAVTLd!^x)*f+KBN_D+YPixJr-T)bTft z`qZB|{9J&&$t!u~w5Y_%b9uziYllUGIG<;h!_iW|a-6jHsVq|6)>b|&{3jl{(+m@^ zhPIE5(01<^2y4EBJzkT;{u+hSi?FVabLDV-z9HQG|5jff-V>RnxLaxS}2@mbrp|=ye`xpQEJ58G86c?3B^u zsNx*}_CxAk&RG4R;~q0>GDLq_E>WvDh~AqKQ!)L#Q@4s@TKt!t7xZ*9CGBZ z4<8M2!ZI6d`cJ_V-S8aPCBMh6)P9vhqnP(yB~d9`KX}DF>Bi$DUe;Bt(g}}jGmij4 zjGQ4y(yMQq_&MI?VI<11OuOiRTQbZfP1jd5%r57ONtJf+KaQk9E!E7y zzCTw?s&bM8b~#rJ>UX|@ORDW^Xso$nj%*@tGsYnI?{nnt${71Ns-)Eosx5hixkUkp!fh%}nSM6fU?oc|NanaM;1fW$)DkW04 zxj9~ zheYZWZUAYlWz{vtt2|_xP(^B>IEHFb(9k$!3@TwogUCu&^=NSLE^PKv>fQszdgUMp zyJEe#V>;CTd^q#&_uu~VOBweWRkWUz-~Zo#Vy0eAzo)0F25%B{etd$^8kL>t!FuBQ z4=8xKBwoHCDP#QlUw`>i18n~3FNd6#QUBZDCRncyiCno9YN6rTwx>Q_RFHNX7F=eK_& z#b-i__EnH#Q}o~kMHjx2;xi#d|0+naNjH8$C7N%f_)JJ4#n-_Kai~x9Xi_cFgxc7eG1dHN3Bz5L#wR1{j8wIzT}0gB(NK_zV^NrNgS_F`id-3Y0=JfX6nA*`a4L(0|{6N@)&*4nj;-kMU1 zZZb6yKak2gBhIL-Gy!Tr3_lN*%|hkkSt2$$D?$(r3zBTb10>jzNzB61Gmp<%Ws(B- zFDbRG0gew+StaFLWcKc!BXNbP_2`%EITA!_@9(xZU(fd(`LaDPnrir0Ufpxx)jc;} z*mH*2-1)2bocZ!SKVH@IB}yG3BfYTaO916r9L8&?QKC21Kn*Qeo!V0h#re`jha|^d zI!km7A0HrP4^sFXkI8& zs{^Zyp&Bqzkj1WLnu6a$Jx8F0%Fa%hO0h|ts<;tk$Rc&UQ*@?H7p@!IHaoVRbZpzU zZCh__+g8Wv*tTuk*}cAh?Xi#cSyjzBtB#&gV?1@;_e{FgYTU$pXzze)5W-m!Oe0vM zrNe?wC<#%QY*R13a-#12j?X2Hy&@*I4jXo{MPwQ>QAdAmt}F7+p(S$w>KB?qs78ex zTLTF~x9)yHT+AIBSZ4tvS?FmwxmlT7fxlXNSEw9UJl3kIjA{|UvCvN=QLm>Sy%6M4 z?rUpl#x9GOYD5nM=YW59yy61&uasv(8#*r03W^DsRQXHZOCL|JcEep>S68ZJmFOz< zT_Mi?dW3Zl3q?|jHTU8%P<^ZN4GOfDY{q1ZHE&jHI zW{xVAT+ukC28!BH`_g}m#s!l2+dAotjWSQBRVj&&{eaZvSfR;=nNPytO3q#n@@*Nb z##pL3(gqR72yv}xM?7(elhJY65EvEwcR5;txL_3IWmToI(U9(2Ikakq1N$gP068pO zctO{>dYDG7%m}e`*J|j}#?i**lBnoc>N<5T(2H#pdB$QNG^#>; zch%JZM2rXb5M-u#QvqD5g*Y7^}iP~V82W&cF0m^^s^jlS4?o0lF{?OmF$IWhy_ON)!t4BQSwbSge!6Iw#g)FX0H>So4Y#iHedFz%}Iho&3uD!Z2Ln_qufB=o>&-9FvpKw1+4TRWp;3~N zD``UiPMHbB_`Xm{GCH)3qdLRHVEOs);GLslztIa^I(|$oomz?~UHCmx{h!+TJcp-a+~rsY`V`n^+Bs=f3q3+}=XGeiZ+1n(+9V z`_|ozkMv=yN|JOL22L_e9E7I#x*i)!rl_{JEAJNBRVwFz z+sUy;67K}d?l*d??cMwS#2YA5xNY>jER~$@t!4;v&m8AeQPJvFXCRSjIX;&zeV3V<+0#|qYGCRcp=lxcL@2WU5}`un&ewu;gD0VNap+p=E9K!>yI7guvM zzAvs8rL|dv{O}$_vLmQutT5?(TGp*J5&7c^YEM*>0#yCKhwE0IRFkGemBX;TtG0m1S>#|Ee=GJ5rs6h=mQ~t;{>qN zlBXUB4>sYpeo(FpSOyZ^kmiZ=^7mqJYXkq#KW6mWu#W>&73%=ZmRs+<2vpq&V(HLO zbF=}i3lwbZOI4frK>YlF>(Em#(fl4bDe*!ib}PZ;-s9NTs9}@o$E1X zRiv}Ujr9Q_(v0gz7VzqM6Fa8*&x-*&s4FL`R-W{^(3KVQPR-fd(!akO#^{P%J@s5U zxpU)Fy(J;D>HIwHGOn!Qqy*V_4AO7Ldv<~#A2+ktUDi7QkY_U z)p&&X`GboYbVpoSJ_k7$1~O7>3Qk@IgT%)(iQ;lv*8)9Cc?@~OuQ!lij9TwK-0F>K zmqZ-5VN-NH<>_N6c^K*6eF{H)}t~Ye)3jc=R9WmFRT)?`++B54;vj>dv#% zn1LT)PIP#jT%S@r&p(bi-r1wq*th`BAxG^5<6&0o74BiKUG?&2`>z4ILs1F+mAUMiYW+CIjE8M=xOMaX-JGT}UaaQl zyRW76Wku(h37CL{rkA_Vu$It8>tveR;9k=l2RnCoB`X!9T5bU20NMzjbQX3O&AnIl zOwTY9*NV5HVrIBn*ui|@!>-T7jSmEXAU8{^6{pT?~avUp9YWor{0tZspFluSoTvj$r3yL`I`m7+t^*b4&psV{jI%9Yb>y zZd7C5>lFc-EY(84*czkD2YEHv$v# zhHc&ZjMKY1;pb4z)!LOr&7nIp(g5pEMocppzjA#B60|6};~t?+e9s5yxe0*PGyf?5 zZ8-~g;xG>nAq=ab(eiV?;9#i@?wCm7Y)3c*Dl`buv%J+jcv!76hz@Uh4P#kOcgRi)^b^jb}BcmK&)0w4d_aNT9+R$xBe zkSE_y>v}6BS389)bvhnr-2V;EHzG@`Egrn2U<_zV>*T%aqK}Y1LwI7yxYnMl@i0}n z8Ai*T!YfC%wl&ymoXP>`=x~yCPbm0;&LgHX4qhKX2YSr-c8^6MvzQ3tmsjmH4mGia z)hz}$wkWk;hz7RXGRBsrN`*iVu2{lQ7aR)J0~@qNiNa~6WMb)U)C-WD-YBeE(!(j$ zxd>O)!JeiqdNBz+(3CU}K1x|#*D#eQDJhH&>o;IZzI2YE%Bt=h(^H+R?6*(BDf3e_-l?;$!v*vqS+N6o855}nHb)$527y&ZqtZqpOmRabS=6eO zpdnJ1^g-q)1tVo?*=fFB&g?;c(sMl;VG<3-XE#eX`_t?Yi3Dc^%Iaip{uq) zWm0QTZ$~KMPVF;WkM<$5bg*FV;`8Rm_m1iG*~@a^)I;TlkSS1e%g#FW0J9SZsMHmz zHf}81-b#{NHhLy2zBQebi5p)IAHrzfY#v$>*DnnQMeu?$SM1=k4d|?4448x;qJObg zL=vd6RHm%P4$YEVo)-zW$`xSrH#aprvdhLN2bU?cJ1pDx>+Av)>j_s2J25sR@M-j4| zxN;|+m>BJpZ$04qAZXZwwHlCXUsLqrMNGB7CJnuMx_|s)utE7E_R(UxZ!+(DC6$EN zCAAboo-?qwi9 zga z+W}a^*&^!b5DPDTG;|RPdULMUwAK+0UU_K2(hJ(M7>6=$who0oNRuLWuNixWw#h8b`E5p$OY31+`=u-^DFSy za3?Q^9=Euuj8F`QDmKt*^e?Hby6VaxEjR$1b?;O8mr>IdfxaxH@Mo#Awe9W;-rWeU zL7Eu1c!u{}FL(&$mI8m{c3OX1{Ko?r1dRO01MtX14y72zTn1wVs9aMHr=sW-60idA zV&#JDV!4D2k8sxV6=0b|NOUYf+~8Qt$`v7m72aJzy=Wf{1lHmf2_R!&<;gV=X%f6~ zZnIu%a#XRKKVzSgC>+9foQkYncC+21Uh&!bs6T%X9N)5{&2?E^6*=i)0Rz&-??Tb3 zQY_#H3x`#-*A(fpS75tS=v?)vaUW{cc*9xi0c~2eCBy$Y7Bqqgc=bPM( zhz4~^ZdbVj>k@)BtWd5`SwVXX-!$q2D($D!&x;f*rS(H>Ab2uuITYH3@U4vG{{${A zl`jUVwbc7LQqGOc7V)}X{PHZ$WZe)TZ>QIpy9y?GKK|rFoqbqxxX}_|+6+#FoOeo_ z+J8}@gY>DcwB^oSC~S?&RG-y^x`j=CPPp`yZ(duDu;)F| zjnDJ6H!Yma73WPm-(l=b1R7px2lo?y7M-c6CDHeQtiCdb@T|UYr+xBUuI_nR#$o31 z6~s-m`oVZjH7Kv~SOvJ$x>b5p>3scRBq|gO&j*q~3XnQm)zaZ>qgZ(BUYc zr&G+tu`Fg(Gff7Ag&*D3i6KTn+YJ3FJ3#pqphEEXeW>w&hZ};hH85T0QgQ3LfW6(Y zaYW4j&aC#znSRm89|*Xg_f`fT8_zn_S`_WyLip2BNq*QlYjm32s;u zSo3&Kbq#dnp9Ws_>~IlkS`w@segj2JT6TyoFoE%hfFTiEZ`689>4~`U`{-pC{n34> z9h{=GvYB5x!){$^sOmzmYSLHC?{*08Al6y~q9SWKY$U$ZGl{+}h-3WUr3ad(@YBs+ zr~o!`#v0O9))sG=-sV zwTo*iHn^{$SkK57Coy52IvL+>0se^YGEW#rZM86mt-#GD{UoJZ6QOnrW^Ur3pT8WCW>gq8bQW_`FC{y|9wZ9PcGUBGrx@x!hLmy3luJ|4dbBz9QE zO@%6iv+4RXjz=Vi@ni?W#!#}O`qt?jg^+gGvbeN#xd+~&i!zvoqGc^eTqf{L7&YP0 z<5>Aot*2+9mk|IRhB4%?kuq8f z^{5I`X5tvh!C2IJ>%YiYP}Ki^=SYX^s2BQg6zsC&97@EAuj!HrtsOIF4 zz%tk=w|2K#E}A;uH{4F4-I+0a`}BFc-B`-iKfeGqf_z+_(!UM(0BT;k&{7e8!_?pI?mo ze*LX?ampYk+^;&dzqm2w!6PRaKk(0CKv(QfIK-7(| zVAxP(B<7{F!jaJobf(}8Q9=YqYc_n-d~SIk;%0kui?h&p_=CrtW8GtOt6~JJO({k6 zMr%5c(>H#L!B2a{?~_apIvHjC(M~+nw-?olB&3dSQKa{C|3bE0^HF+Mu&F!KGWOwX zcxe+!Yv$^C9PcEBsFb4?&-yZGLr=1E-p3zQN&Vur@(ZPH5yG$N3imZOS>OGBGehKs z{PP#v9^}nUNS?_#q(b=N#uz^ZLz@2Kl&@a>aR_&@;T_J6yK=b6Q|bmbQ%OCJo09f; zWVU>Bsd`WQ?3c=Cz>TVU3#K}6bnKy#5N2&}@AV)xBwQY1jZZX2pKHebs2`fylXoCK zp}3*+$nnPOBxMZuk_fwT`bRCt$MmV8z&72jbEmUsWYXZrlkW(dmp43J(8Jd<+h$8} ztH29ymS3_PAaTMN=Wz8&XEPuT9$mks!qKNC3OeIpMYcF-WTCM zy-t+DJ!#|D+#c!Z21VsU1eLk6{5-u&fkyZ6E)%^yZY;-7miohMOkQHOe*K_)O)lK9 za3q=1M1*C5QejQ%FbcW&nd{b2_X(EyVEYX$N5sGm&XQ2vEe{Rt)K36N%!Y1_kX3Mr z?t}!#anc8C{bH>KchCPz3h?+K@aS--{HSRig&o-|lm^=Z^so!kzvEo-i~P5fV0W>4 zZiE-5T)+M(O?jJ3`FNGo3bKm&Al*m~Sgp@95@Pp1Uo*CA%eTtcX+Jq{m8R*=)@p4? z$hbyB^!S6c0q3I%QIYyS$`=Bgke5!lfXe4pbBvdQ5xSiJdK{|J`dp6&0_Hj=nm%S5OPDQbtaY$0F>_gXW;=|3jlHTOeE*Ww zx%$9cX+#y)GyR=YQVfK znpGOE|MVz%z~yQI1Q1Hm`+~~{o?trJmpfkF&~E@PKB9yBOkLpMs4t)A2hU$ETlc1-RKhX75V^Ddo1xgj7pNU z-dSjJ^jJOp7Z0%;S+Bi(54}dQ55Brd%Zgv0rE(20O;rSTZd!l?&$7n;l5?cSd(*$2 zI&|6jx?I>uxR%w~-SLbTEnUjI(|p^mO!i{DnXlxODZVkg5q*x3pHZbfJn8z8(7NW-VGU1MU-lNjOS zAL4Y@?w1?qPP!Mjq%5^-2fCCBBkqs)Ab#0bi%*SljGP|YG|76eW%bw|oxfer(rJ{2 zQ3arU@WV zMbN2jA*opr>cP76U|Ww`;2u=Itg<4IF>J)EwDZwP%ZOkY+|>T<7iNt2EFQl~*@`w9RN<@Wc%4KL2F^#mrqR zfkzAyp@Y8&6j6;u_)H}3GF{NOqtr{asHw(NF=%1c#Ts;KqX1`3i#BoyO@v;7w1O5i z$=O-vdN*n!becY!5#MJ`P{XsBAND0dI(HP%D&NUI+!2`^ukg=GhDuuT;-8gFgxN4( z81RDgbCnpCxYwc7+i`+VymVVh!)-LC+i5ChHQEF1{kx;v(OP4y=Or6j9Y{T$oh=5D z!nuf6G9t3PJa3-^>E+94sS5N3zsj{d=AhTWlVuLGeFsjpY^yf0R=zsc+= z6XEhMUUt$O*TOZi@0M^PzYxlUMX4{B9K%LDq(z1S`}jlad=qcl!)gNS$ST=G>~V+0 z3{Yp%Qh*Ti@p1V_W(D~2zGQi~xpnwYIEFKrn%TN)eb%-7Gw`X- z1Tf!9fe@xr1KxH9W16c(^^R0t?Oa6;CF9~3{>2%F$dq&FeT z2ZFsYfA_#&GH5e#s2gNMKGGDcvrm`|VQ@(s@@wuUiET9}(7jo2T}HJ6am2>X%ehYy z7V7I;MY3;LHyp=XZGJW=m5#D#`@?Km59au&J^DQUViz$`1s&Q8za^fMKjmb}*&hPr{+w&O1#?j$+VyYtOyMwBZqB@DJ8<6o^EPN2=q^sZ zSc<&})*|zz9O7>IM!vHiAjibCwUP1-S88CvF*3$ap`sRBwG}70#CvOA(v3ycAUQ81 zbG;ErrF!#>JKUK^<3PgD&Oj2YxZ2zf<5P7^CRmEr&F(VuV=wykYm{aoG7^>4*xCRw z%~c`0gdPr6domN5z9~u^9_cla?}C889agyXKCKYa$3Ggj(Wb+?*cuv3OkJ>OY?CXs z!KiTn#W6s$gOKY!WyiiKd3cQ_iG9*Kb@Gu4950H1r{QU;mRnq>S?*dNMH}(e$)3+5 z0Q<)60yZt9;b+BX?;?|z!B&H-ViR-@HcU_4!RCSV#caxjKN164fu)XIuWYb$x@0e$%={-*x!s?#m9gnQw%6h2 zMFng^FMW}P7i6U3)Mi(~U8ZQJMUiPlt~`e!tpDNh1)JmXQ*3p=CKN8oCpq<(1oR@c zWJSS?vyW3jw{Sn48AdrF+BN_=E2n3Rg$kH$80UWj+L6gPfWPOE7%EM$4<-!QZX2>J`<7w?~+uY|AO|6lVLjjbS6rT%0Y zK39Jp8snJJEjH_9ty5-_PA{(EZ}+scue24SyYF?}M)e1`EBH0@+9JMXHbuM_>I;5( zWJEUU&1$|zEyC8W=L3r_wUd7r(KS7TL;PB{k#_IUB$g>i04JMWt-nK zdYFJN@Kkl)%?L$xM?VU3?<#x={;Yj~pxC>15zo<<)gtAV(^tV-w!;_VW+3bbPcFaD zpxLYV+jMW03B_ar?DRu*&IwbYx%2jD)eRk$xy~c4uqaAlF$hHs8$}flPrvAb$P25< z8@UskC}x@RQoa~fmP+zPpl?m)tW31rF8E%v)+AKoocrRf1)16MaKD0-tFEB-$g0cP zmNgl*v?+8-B5<@J8ANGwy+3}FUFg#L95dN|=SljR+9^e*+xp_y4Vc zK1=(&;Qb4?_WXPDxXCr%;T!4k2KZ*=8q4@L_$IT@f%I3KvyqPOTp|I&2jhQtaZ9KK zY2grGQj2x34*fG|KJvrzK`pc|Qq$Zx26?7UI7$al`-h zO3g#$^Fwp&No(<*D}eP0xo1N7ruv%kx4qg~*Z+CAQ%CLO{Cv0f|2k^x{l5SB;pUrY zcs%KmSB&5{6c!EoOmLMWddPUgV`Ww=OKz-eS21%hYJlqGm#j#BZtMNZ>;1fXY1vt2 z@c*9JS%C*spzm&F#hgG8xjcnH@#x>!%scv+Ql8@jJ=p#HeCQ)++0Z~(<#^W7-)&>E zP64U(^%`Ln(fGZ7l7Vi@8gmAO7TV`+kA~~FUYNC{&;0E1^qpw$&{5fY_xitV)!{g$ z=9mGZhF*@x%2!{yG|;u0x@tX#7XIS-MA8;LWK)0jU`DTTXy;=BzvuUVo}LH;gLSHq zkmF)OKL)J?&L!2*%alCN3bSr&OB}Qwt7$CFwi&6mX{n7%PFcp&TWjB6YUBLfBBIR_ zOWL@sYTuj`sd5fh%M;hk->jsHsubH}8Wk;>1Iw(S7s9W|b(CWY5~beovNiJFeOZ7DAo^;M{iHrEBau_W7QnG`jDK3?ch zjab+Cwp+y^UPRGI1Xxmp`R(cPI=}k9uz_5xdfzI(t>>ml^hPK&z4GrCPh7come|8? zbeaQNg*g?K(EFO@8Zua;j4Oc7G?!MDWXMF08bIRFiKVHQM4m54lL+k6Xbv<7fN7xK zT8FAD?eB7lLFvg@#$t?Skkp;78s_tll33Wh&NMC9!A}LI5yTY}K@uq)c9O-r^%M$c zT92*-h0>;#SPq1QHQDKuW{mAC6NwRb%Z!+`ZB1&am?gEuNYjaTegmc4BO=WAl(g{9 zhH<+0y20pnARUCA&8|Kg<{bQAP;ak9#E3T_p#Tk5_nJ!35q7Dev@0kC%R40pzN*Nu zTO$V^qr6_Jp&T8w2%xlZbGGI-Y9PDIxDQ_KxNl>kdHqUIaO=>WBtL<6Ou{qXS8=Wk zC)$9D2(;GdNUzb9PQ^;DdebqP2#-8^D9+|g83lc)y_9un@o@3)6OxC|e=F<7^N1tl zZ_45W2)@Tqi}Ju8_0te3nW4A^fGG(23Io67*AmnHod1TlhGHle-9yI9mr2--Of z@sQD)SD@+Df85|{T@V_3bg_mg=*0WM)Yp#0T}FJr4|;JvJTKLa2|Yvo-~aeQm=DwV zpt?2sCvZUff56~g4}ub-7EzpEu*Lrn!&fPZhT*5r*FS~- zRUr2wC@m_)jPn;`g5mo82PL{7Q+;os+}$zse!E40d%H0p)DaSXy#M0`oiaNfpGNLm zqCx$r7JGm~e|#~a(N*bv@$9?X8bhRXque*e6(ACPrV0aaAh4C?dnoN~F)L#dE9bq2 ztJ^71xW?`Mhu2efeij0wzy8x*!=+LP$H=K#mN{_xZOYVBH{kuOxJ6(NkiYMK+z<2? z_p*Jy>Px+|)zFwBwt?1+PGDDchNmQH-f%ayJiJ+U;{JO8SQn+D(wcLvk)Z_n^^_?* z-=iKznyFV4)w)tyFVmn_Rx{ee#4}3%WKmr1Vy;S0zdMw5a)&Y8u4Q7-A;UoPjNfICDn}^N<@a21jdz zhH#*OW!z`?_)9C1c2hT2ttPE+)?#?%Js%`Xdd03p3d_yQC7y)09=S9K$ENpYA;BD@ zpU?R*(zhU_LUlE<*sXTe05+d1G9Hzm6SwIe7%eb##nycoUF{5ER-6D{nm_Kf+c>5Y zGPO&coI>?ibXXY<8#}R=Ixp;-bTyX+{3!7Ph-qy;IbW~_bwqyLbue#qPEQ9SDV@s2 zl)K_XU>knn#C4|e#JZ_ATZpx7<`O+Hn;?h@VoFBnA$dDXlc*|wjq{f14h~R(=F=(O z6luXqMyNIWjxsW&Mp9-8M$MV~q#~%7)I1PPlCRYePs!s)KjFf4TW5GT*v2odmZ?Is{7b`n!|3NSmNq~n&2~a%2Fe3qc*{m%6W(#o z#ujw^IM%*^d-s?*7gbF^axi?dFqRJd!^uiPwWgy`>e2>9O0|uk5V8P+sg6nv6_zVQ zACD0FEh8jInqhqB=HF%x9WWr(#Ew`Kk1l;$kZ_SCJn?eJ|9&tio^AdHcts4tCW*AZ zc|s72TC(p6U?3yN?F=ksv$BoPnzl8V{1!z%#N8_XR|Ed;~sYo}AhTaMUXLkV0;y7IdN+-KEjI>mgCH`I#4#Jv2@#W={aB%JhM)=z#D-Ur^OD ztqEY!76BjcSIRU$+KM|AV9}=!x0&210}!aEmM4hjS^mO=$EDv+u83( zIl+f&>Hw806Ey3UQY>O#&or%nsS+UsCROC{c(>gmP={!8DuYuwMg?=fs82zlq!9|4 znA#idOsceSNVgA3G=JR!N+pG<35Ww7Cbz*awJ^%7AwXxT#?PZUz&l`JjkxoOSKC)F zMd|N5lFciA{&n?*OI9kYrk%#MtQyle_yv;gU;8&>w4=J6Ee(}igTxjlJ|0#ZSm=*m zyg~azA+I~qQUHFk84qWh?%W#ut!jWIH?Wm6gsMwDRwOioLp3#+3YyS**7RvGtB(9 z(YCI$#gD?x_{)WIkDZ6QjbwaY%h~wK-0bX8Dse7`r3z$Z-PEXxpomZG?+_4@@%fJK zfOSe$doT6NJ3w z`^O8Fzf(sVc@`L>bDHgJNk0a)PM>1_j6#C{>pu+hil$&_LVDe&Xox~E`rCCj$PQqF zkBi<-P5=Cl&4ko?Wl*`0?CPnW;?VLzX?|s1%JqxFA^0Eud9mpIj$P^Tzgh<6yR$M% z&D;NhqH#{L?c;P*b~6u7`1T&o zpnD6t`%F=;8{gQ2t?SA2lwIeY*w#Ml{@Akwac=Gt< ziRw~lpx?$T8tu9vc=r3d1|l(8&*hm3DyIr&ERdO|sIg)g+^t}xfXvbxwgJ7>u+>Ru znOJ}Vq8?G7Z?w=t@JS=P_0Jy|1qccj(;7U{2=vEkX(X*uAWhQWgRz;r-ZTOue(CIJcGS5w_Vuz-;gCICS*#ep}vEm zW^MMA1RVi5E>~8r@I0F)R5B1X$#5}NN@bNYkZnHczdpRZ%zE#*O{MMSnbdZGLP11I z#QRVuCYm#;dDNfvlpBD2F6ukE!D!GU41s45+Rc!z#oMPn$!VhTG)WdWgl)GLaa5Q! z9{O~_RYDaSmpimcTOQ>u3_#H4hEF}~B?215E-~g0gj?=_Av`;W_P^j7Jyy_=Mi1`D z+e~a0DT?QT@R^-ebI4fFi;Q_0M4jND`BrU|nnj;x6DTgWWNbX=zIvfpCJiV2_rCSK_+DYgSe&$rg0vvp$bEILSF4&x!2c z0!R3l;{+9S@NgSHXTn|#aGu$Y=4*3}YcN~W{Y#vBu%< zFS7LghMO1Lc5W|wmQ~iJI;cUM^OSB4|LU^{QP&hp({4fci7rW79a0B*mU$#N$NH1t z{pYIOkI31#_W7A-g3fkYiyNnxZv+m4p;<{t5La1Uk8thZ_#ZfYSA5OvZ0P@F`+eP9&HMx+Pig)re|x|Etl#`} u7x#YtNXtyqj>*tLNQsU(1+p$P#{?-d!ifE)#IgVavOhLc1OkBp`hNfxTK$Uv literal 0 HcmV?d00001 diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index cdf54bc2..288a4955 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -1,16 +1,38 @@ //! Offline integration tests for `mega-evme replay --block` / `--tx-file`. //! -//! These run against an RPC capture envelope large enough to replay whole -//! blocks, which is too big to commit; point `MEGA_EVME_TEST_ENVELOPE` at one -//! and run them explicitly: +//! Replaying whole blocks needs an RPC capture covering every transaction of +//! each block, which is far larger than a single-transaction capture. The +//! envelope is committed as a gzipped archive and extracted into a temporary +//! directory once per test binary, so these run in CI without setup. They are +//! the only tests that exercise the batch driver's multi-target paths: more +//! than one target in a block, whole-block mode, grouping targets across +//! blocks, sweeping targets on both sides of a mid-block abort, and +//! block-global log indexing against real logs. +//! +//! Set `MEGA_EVME_TEST_ENVELOPE` to replay against a different capture instead. +//! +//! To regenerate the archive, capture both blocks into one envelope (the second +//! run merges into the first) and repack it. `--verify-receipt` is what puts the +//! receipts in the capture, which the verification and fixture-dump tests need: //! //! ```bash -//! MEGA_EVME_TEST_ENVELOPE= cargo test -p mega-evme -- --ignored +//! for block in 22945844 22945853; do +//! mega-evme replay --rpc --rpc.capture-file replay_batch_blocks.cache.json \ +//! --block "$block" --verify-receipt --json +//! done +//! tar -czf replay_batch_blocks.tar.gz replay_batch_blocks.cache.json //! ``` //! -//! They are `#[ignore]`d so CI, which has no envelope, skips them. +//! The endpoint must serve state at those blocks; a pruning node fails every +//! target with "state at block #N is pruned". + +use std::{ + path::{Path, PathBuf}, + process::Command, + sync::OnceLock, +}; -use std::process::Command; +use tempfile::TempDir; mod common; @@ -37,14 +59,42 @@ const OTHER_BLOCK: u64 = 22_945_853; const OTHER_BLOCK_TX: &str = "0x18302160f2395069a44e1654d173fa9eed95ead8f922f12bfe07b6bdcc0a14f2"; const OTHER_BLOCK_TX_INDEX: u64 = 23; -/// Path of the offline envelope, or a skip message when it is not configured. +/// Path of the offline envelope. +/// +/// The committed archive is extracted once per test binary into a temporary +/// directory that lives for the whole run; `MEGA_EVME_TEST_ENVELOPE` overrides +/// it with an already-extracted capture. Extraction shells out to `tar` rather +/// than linking a decompressor, since every platform that runs these tests has +/// one and the archive is only read here. fn envelope() -> String { - std::env::var("MEGA_EVME_TEST_ENVELOPE").expect( - "set MEGA_EVME_TEST_ENVELOPE to an RPC capture covering the replayed blocks; \ - these tests are #[ignore]d precisely because that envelope is not committed", - ) + if let Ok(path) = std::env::var("MEGA_EVME_TEST_ENVELOPE") { + return path; + } + + static EXTRACTED: OnceLock<(TempDir, PathBuf)> = OnceLock::new(); + let (_dir, capture) = EXTRACTED.get_or_init(|| { + let archive = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/replay_batch_blocks.tar.gz"); + let dir = tempfile::tempdir().expect("failed to create a temp dir for the envelope"); + let status = Command::new("tar") + .arg("-xzf") + .arg(&archive) + .arg("-C") + .arg(dir.path()) + .status() + .expect("failed to run tar"); + assert!(status.success(), "failed to extract {}", archive.display()); + + let capture = dir.path().join(ENVELOPE_NAME); + assert!(capture.is_file(), "{} does not contain {ENVELOPE_NAME}", archive.display()); + (dir, capture) + }); + capture.display().to_string() } +/// Name of the capture inside the committed archive. +const ENVELOPE_NAME: &str = "replay_batch_blocks.cache.json"; + fn mega_evme() -> Command { Command::new(env!("CARGO_BIN_EXE_mega-evme")) } @@ -180,7 +230,6 @@ fn run_error(stdout: &str) -> serde_json::Value { /// `--block N --json` emits exactly one NDJSON line per transaction of the /// block, in transaction order, and exits 0. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_block_emits_one_ndjson_line_per_transaction() { let stdout = replay(&["--block", &BLOCK.to_string(), "--json"], true); let lines = ndjson(&stdout); @@ -209,7 +258,6 @@ fn test_replay_block_emits_one_ndjson_line_per_transaction() { /// A batch line and a single-transaction replay of the same transaction must /// agree on the execution outcome. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_batch_matches_single_transaction_replay() { let batch = ndjson(&replay(&["--block", &BLOCK.to_string(), "--json"], true)); @@ -234,7 +282,6 @@ fn test_replay_batch_matches_single_transaction_replay() { /// `--tx-file` replays transactions from several blocks in one process and /// reports them ordered by (block, transaction index). #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_tx_file_spans_blocks_in_order() { // Deliberately unordered, with a comment, a blank line, and a duplicate. let list = format!( @@ -272,7 +319,6 @@ fn test_replay_tx_file_spans_blocks_in_order() { /// targets still replay, and the process exits non-zero with the class of the /// failure — here an unanswered lookup against the offline envelope. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_tx_file_reports_unresolved_targets_and_exits_nonzero() { let unknown = "0x0000000000000000000000000000000000000000000000000000000000000001"; let path = @@ -300,27 +346,27 @@ fn test_replay_tx_file_reports_unresolved_targets_and_exits_nonzero() { /// exits non-zero. /// /// The development envelope is captured by replays, which do not fetch receipts, -/// so this pins the endpoint-cannot-serve-the-receipt path end to end. +/// `--block N --verify-receipt` verifies every transaction of the block against +/// its on-chain receipt and exits 0 when they all reproduce. +/// +/// This is the multi-target verification fan-out: one receipt fetch and one +/// verdict per target, each carried on that target's own result line. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] -fn test_replay_block_verify_receipt_without_receipts_reports_rpc_errors() { +fn test_replay_block_verify_receipt_reports_a_verdict_per_target() { let (stdout, code) = replay_with_code(&["--block", &BLOCK.to_string(), "--verify-receipt", "--json"]); let lines = ndjson(&stdout); - assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is reported exactly once"); for line in &lines { + assert!(line.get("error").is_none(), "a verified target is not an error entry: {line}"); assert_eq!( - line["error"]["kind"].as_str(), - Some("rpc"), - "a receipt the envelope cannot serve is an infrastructure error: {line}" + line["verification"]["match"].as_bool(), + Some(true), + "every target must reproduce its on-chain receipt: {line}" ); - assert!(line.get("verification").is_none(), "an unverified target carries no verdict"); } - - // Unverified targets are RPC-class failures, never mismatches. - assert_eq!(code, Some(3), "a run of unverified targets exits 3"); - assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); + assert_eq!(code, Some(0), "a fully matching run exits 0"); } /// An abort caused by one transaction of the block is not an answer about the @@ -328,7 +374,6 @@ fn test_replay_block_verify_receipt_without_receipts_reports_rpc_errors() { /// `not_found`, and every target swept up behind it is reported as unanswered /// (`rpc`) with a message naming the transaction that aborted the block. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_block_sweeps_targets_behind_an_abort_as_unanswered() { let (missing, missing_index) = BLOCK_TXS[1]; let path = envelope_without_transaction("abort_block", missing); @@ -379,7 +424,6 @@ fn test_replay_block_sweeps_targets_behind_an_abort_as_unanswered() { /// (intrinsic/call gas) and aborts the block — an execution-class error, not /// `TransactionNotFound`. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_block_sweeps_targets_behind_execution_abort_as_rpc() { let (aborting, aborting_index) = EXEC_ABORT_TX; let path = envelope_with_zero_gas_transaction("exec_abort_block", aborting); @@ -426,7 +470,6 @@ fn test_replay_block_sweeps_targets_behind_execution_abort_as_rpc() { /// Targets swept up by an abort are reported in block transaction-index order, /// whatever order `--tx-file` listed them in. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_tx_file_sweeps_targets_in_block_order() { let missing = BLOCK_TXS[0].0; let path = envelope_without_transaction("abort_order", missing); @@ -463,7 +506,6 @@ fn test_replay_tx_file_sweeps_targets_in_block_order() { /// Batch mode rejects the single-transaction-only flags before doing any work. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_batch_rejects_single_transaction_flags() { let envelope = envelope(); for (extra, expected) in [ @@ -491,7 +533,6 @@ fn test_replay_batch_rejects_single_transaction_flags() { /// A parent block whose hash does not match the child block's `parentHash` is an /// infrastructure failure for every target of that block (reorg / divergent views). #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_block_rejects_mismatched_parent_hash() { let mut envelope: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) @@ -557,7 +598,6 @@ fn test_replay_block_rejects_mismatched_parent_hash() { /// outer block/tx identity and a block-global `logIndex` that starts above zero /// (preceding receipts already emitted logs). #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] fn test_replay_receipt_inner_log_metadata_nonzero_preceding_offset() { // Last transaction of BLOCK: multi-log, with many preceding logs in-block. const LATE_TX: &str = "0xb6a0b7a302c741f64b8e46861a3dcb2d5c1047f6f2cb89a35b5c2183c96296b7"; @@ -589,15 +629,18 @@ fn test_replay_receipt_inner_log_metadata_nonzero_preceding_offset() { /// no receipts skips every target on the fidelity gate and still exits 0. /// /// Fixture skips are expected (not infrastructure failures); the development -/// envelope is captured without receipts, so this pins the skip path end to end. +/// `--block N --dump-fixture-dir` writes a fixture for every transaction it can +/// express and skips the ones it cannot, without failing the run. +/// +/// Every OP-stack block opens with a deposit, which the fixture format cannot +/// represent. Reporting that as an error rather than a skip would make a +/// whole-block sweep exit non-zero on every block, so this pins the +/// classification end to end: 22 files written, the deposit skipped with its +/// reason, nothing reported as an error, and exit 0. #[test] -#[ignore = "requires MEGA_EVME_TEST_ENVELOPE"] -fn test_replay_block_dump_fixture_dir_skips_without_receipts() { - let dir = std::env::temp_dir().join(format!( - "mega_evme_dump_dir_skip_{}_{}", - std::process::id(), - BLOCK - )); +fn test_replay_block_dump_fixture_dir_writes_all_but_the_deposit() { + let dir = std::env::temp_dir() + .join(format!("mega_evme_dump_dir_sweep_{}_{BLOCK}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); let stdout = replay( @@ -605,24 +648,33 @@ fn test_replay_block_dump_fixture_dir_skips_without_receipts() { true, ); let lines = ndjson(&stdout); + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is reported exactly once"); - assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + let mut written = 0; + let mut skipped = Vec::new(); for line in &lines { - assert!(line.get("error").is_none(), "skips must not turn into error entries: {line}"); - let skipped = line["fixture"]["skipped"] - .as_str() - .unwrap_or_else(|| panic!("every line must carry a fixture skip reason: {line}")); - assert!( - skipped.contains("fidelity-gate-unavailable"), - "expected fidelity-gate-unavailable skip, got: {skipped}" - ); - assert!(line["fixture"].get("path").is_none(), "a skip must not report a path: {line}"); + assert!(line.get("error").is_none(), "a sweep must not produce error entries: {line}"); + if line["fixture"]["path"].is_string() { + written += 1; + } else { + skipped.push( + line["fixture"]["skipped"] + .as_str() + .unwrap_or_else(|| panic!("a line reported neither path nor skip: {line}")) + .to_string(), + ); + } } - // No fixtures written: the directory may exist (create_dir_all) but be empty. - if dir.exists() { - let entries: Vec<_> = std::fs::read_dir(&dir).expect("read dump dir").collect(); - assert!(entries.is_empty(), "fidelity skips must write no fixture files"); - } + assert_eq!(skipped.len(), 1, "only the index-0 deposit is unsupported: {skipped:?}"); + assert!( + skipped[0].contains("does not support deposit"), + "the skip must name the reason: {}", + skipped[0] + ); + assert_eq!(written, BLOCK_TX_COUNT - 1, "every other transaction is dumped"); + + let on_disk = std::fs::read_dir(&dir).expect("read dump dir").count(); + assert_eq!(on_disk, written, "each reported path is a file on disk"); let _ = std::fs::remove_dir_all(&dir); } From 7059a928752076b02356bcfbe1a7d80448496dab Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 14:54:09 +0800 Subject: [PATCH 25/64] test(mega-evme): resolve every fixture through one helper Four test files each rebuilt the path to tests/fixtures/ by hand, and the whole-block capture added its own tar extraction on top, so the knowledge of where fixtures live and how they are stored was spread across five places and would have grown a sixth with the next compressed fixture. common::fixture(name) is now the only place that knows either. A fixture is either the file itself or a `.tar.gz` holding exactly that one file, so the archive is renamed to match the mechanical rule rather than needing a suffix-stripping convention. Archives extract once per test binary into a temporary directory, still by shelling out to tar. Its doc comment records when compression is worth reaching for, because the obvious reason is wrong: git already compresses blobs, so a raw JSON fixture costs about the same in the repository as a pre-compressed one, and a compressed blob cannot delta against its previous revision. What compression buys is keeping a fixture large enough to break a pull-request diff out of that diff. The remaining fixtures are left uncompressed on that basis, including the benchmark one that is include_str!'d at compile time. The inner-log metadata test no longer returns early asserting the capture is log-less: it reads the whole-block capture, whose late transactions emit logs behind other log-emitting receipts, so it exercises the path it documents. --- bin/mega-evme/tests/common/mod.rs | 58 ++++++++++++++++++ bin/mega-evme/tests/exit_codes.rs | 16 +++-- ... => replay_batch_blocks.cache.json.tar.gz} | Bin bin/mega-evme/tests/replay_batch.rs | 39 ++---------- bin/mega-evme/tests/replay_dump.rs | 45 +++++++++++--- bin/mega-evme/tests/replay_verify.rs | 35 ++++------- 6 files changed, 122 insertions(+), 71 deletions(-) rename bin/mega-evme/tests/fixtures/{replay_batch_blocks.tar.gz => replay_batch_blocks.cache.json.tar.gz} (100%) diff --git a/bin/mega-evme/tests/common/mod.rs b/bin/mega-evme/tests/common/mod.rs index 63385990..f21c9cdf 100644 --- a/bin/mega-evme/tests/common/mod.rs +++ b/bin/mega-evme/tests/common/mod.rs @@ -8,10 +8,68 @@ #![allow(dead_code)] // Each test binary uses a different subset of helpers. +use std::{ + collections::HashSet, + path::{Path, PathBuf}, + process::Command, + sync::{Mutex, OnceLock}, +}; + use clap::Parser; use mega_evme::common::RpcArgs; +use tempfile::TempDir; use wiremock::{matchers, Mock, MockServer, ResponseTemplate}; +/// Resolve a fixture in `tests/fixtures/` by name, extracting it if it is +/// stored compressed. +/// +/// A fixture is either the file itself, or a `.tar.gz` holding exactly +/// that one file. Compression is worth it only where the raw file would bloat a +/// pull-request diff — git already compresses blobs, so it buys little on its +/// own, and a compressed blob cannot delta against its previous revision. +/// +/// Archives are extracted once per test binary into a temporary directory that +/// lives for the whole run. Extraction shells out to `tar` rather than linking a +/// decompressor: every platform that runs these tests has one, and this is the +/// only place that reads an archive. +pub(crate) fn fixture(name: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"); + let plain = dir.join(name); + if plain.is_file() { + return plain; + } + + let archive = dir.join(format!("{name}.tar.gz")); + assert!( + archive.is_file(), + "no fixture named {name}: neither {} nor {} exists", + plain.display(), + archive.display(), + ); + + static ROOT: OnceLock = OnceLock::new(); + static EXTRACTED: OnceLock>> = OnceLock::new(); + let root = ROOT.get_or_init(|| { + tempfile::tempdir().expect("failed to create a temp dir for extracted fixtures") + }); + let mut extracted = + EXTRACTED.get_or_init(|| Mutex::new(HashSet::new())).lock().expect("fixture lock"); + + let path = root.path().join(name); + if extracted.insert(name.to_string()) { + let status = Command::new("tar") + .arg("-xzf") + .arg(&archive) + .arg("-C") + .arg(root.path()) + .status() + .expect("failed to run tar"); + assert!(status.success(), "failed to extract {}", archive.display()); + assert!(path.is_file(), "{} does not contain {name}", archive.display()); + } + path +} + /// A mock JSON-RPC server tuned for mega-evme integration tests. /// /// All mounted mocks match `POST` (the JSON-RPC verb) and use priorities so diff --git a/bin/mega-evme/tests/exit_codes.rs b/bin/mega-evme/tests/exit_codes.rs index 57f5b064..31bbc1df 100644 --- a/bin/mega-evme/tests/exit_codes.rs +++ b/bin/mega-evme/tests/exit_codes.rs @@ -11,8 +11,14 @@ use std::process::{Command, Output}; mod common; /// Offline RPC capture used as the replay file. -const CACHE: &str = - concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/replay_offline.cache.json"); +/// Name of the committed offline capture, resolved through the shared fixture +/// helper so its location lives in exactly one place. +const CACHE: &str = "replay_offline.cache.json"; + +/// Path of the committed offline capture. +fn cache() -> std::path::PathBuf { + common::fixture(CACHE) +} /// The transaction the committed capture can replay. const TX_OK: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; @@ -77,7 +83,9 @@ fn run(args: &[&str]) -> Run { /// Run `replay` against the committed offline capture. fn replay(args: &[&str]) -> Run { - let mut argv = vec!["replay", "--rpc.replay-file", CACHE]; + let cache = cache(); + let mut argv = + vec!["replay", "--rpc.replay-file", cache.to_str().expect("fixture path is utf-8")]; argv.extend_from_slice(args); run(&argv) } @@ -86,7 +94,7 @@ fn replay(args: &[&str]) -> Run { /// return its path. fn cache_without_entry(name: &str, key: &str) -> std::path::PathBuf { let mut envelope: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) .expect("parse offline cache"); let entries = envelope["cache"].as_array_mut().expect("cache entries"); let before = entries.len(); diff --git a/bin/mega-evme/tests/fixtures/replay_batch_blocks.tar.gz b/bin/mega-evme/tests/fixtures/replay_batch_blocks.cache.json.tar.gz similarity index 100% rename from bin/mega-evme/tests/fixtures/replay_batch_blocks.tar.gz rename to bin/mega-evme/tests/fixtures/replay_batch_blocks.cache.json.tar.gz diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 288a4955..63f5ae4d 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -20,19 +20,13 @@ //! mega-evme replay --rpc --rpc.capture-file replay_batch_blocks.cache.json \ //! --block "$block" --verify-receipt --json //! done -//! tar -czf replay_batch_blocks.tar.gz replay_batch_blocks.cache.json +//! tar -czf replay_batch_blocks.cache.json.tar.gz replay_batch_blocks.cache.json //! ``` //! //! The endpoint must serve state at those blocks; a pruning node fails every //! target with "state at block #N is pruned". -use std::{ - path::{Path, PathBuf}, - process::Command, - sync::OnceLock, -}; - -use tempfile::TempDir; +use std::process::Command; mod common; @@ -61,38 +55,15 @@ const OTHER_BLOCK_TX_INDEX: u64 = 23; /// Path of the offline envelope. /// -/// The committed archive is extracted once per test binary into a temporary -/// directory that lives for the whole run; `MEGA_EVME_TEST_ENVELOPE` overrides -/// it with an already-extracted capture. Extraction shells out to `tar` rather -/// than linking a decompressor, since every platform that runs these tests has -/// one and the archive is only read here. +/// `MEGA_EVME_TEST_ENVELOPE` overrides the committed capture with another one. fn envelope() -> String { if let Ok(path) = std::env::var("MEGA_EVME_TEST_ENVELOPE") { return path; } - - static EXTRACTED: OnceLock<(TempDir, PathBuf)> = OnceLock::new(); - let (_dir, capture) = EXTRACTED.get_or_init(|| { - let archive = - Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/replay_batch_blocks.tar.gz"); - let dir = tempfile::tempdir().expect("failed to create a temp dir for the envelope"); - let status = Command::new("tar") - .arg("-xzf") - .arg(&archive) - .arg("-C") - .arg(dir.path()) - .status() - .expect("failed to run tar"); - assert!(status.success(), "failed to extract {}", archive.display()); - - let capture = dir.path().join(ENVELOPE_NAME); - assert!(capture.is_file(), "{} does not contain {ENVELOPE_NAME}", archive.display()); - (dir, capture) - }); - capture.display().to_string() + common::fixture(ENVELOPE_NAME).display().to_string() } -/// Name of the capture inside the committed archive. +/// Name of the committed capture, stored compressed alongside the other fixtures. const ENVELOPE_NAME: &str = "replay_batch_blocks.cache.json"; fn mega_evme() -> Command { diff --git a/bin/mega-evme/tests/replay_dump.rs b/bin/mega-evme/tests/replay_dump.rs index 173bbac3..4379bb86 100644 --- a/bin/mega-evme/tests/replay_dump.rs +++ b/bin/mega-evme/tests/replay_dump.rs @@ -14,9 +14,17 @@ use std::{ time::Duration, }; -/// Offline RPC capture (includes the on-chain receipt needed by the fidelity gate). -const CACHE: &str = - concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/replay_offline.cache.json"); +mod common; + +/// Offline RPC capture (includes the on-chain receipt needed by the fidelity +/// gate). Name of the committed offline capture, resolved through the shared fixture +/// helper so its location lives in exactly one place. +const CACHE: &str = "replay_offline.cache.json"; + +/// Path of the committed offline capture. +fn cache() -> std::path::PathBuf { + common::fixture(CACHE) +} /// The transaction captured in `CACHE` (a 75,514-gas Rex5 mainnet call). const TX: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; @@ -33,11 +41,12 @@ fn test_replay_dump_rejects_transaction_overrides() { let out = std::env::temp_dir().join(format!("mega_evme_dump_ovr_{}.json", std::process::id())); let _ = std::fs::remove_file(&out); + let cache = cache(); let output = mega_evme() .args([ "replay", "--rpc.replay-file", - CACHE, + cache.to_str().unwrap(), "--dump-fixture", out.to_str().unwrap(), "--override.gas-limit", @@ -64,8 +73,16 @@ fn test_replay_dump_fixture_writes_validatable_file() { let out = std::env::temp_dir().join(format!("mega_evme_dump_{}.json", std::process::id())); let _ = std::fs::remove_file(&out); + let cache = cache(); let output = mega_evme() - .args(["replay", "--rpc.replay-file", CACHE, "--dump-fixture", out.to_str().unwrap(), TX]) + .args([ + "replay", + "--rpc.replay-file", + cache.to_str().unwrap(), + "--dump-fixture", + out.to_str().unwrap(), + TX, + ]) .output() .expect("failed to run mega-evme"); @@ -93,11 +110,12 @@ fn test_replay_dump_is_byte_reproducible() { let out = std::env::temp_dir() .join(format!("mega_evme_repro_{}_{suffix}.json", std::process::id())); let _ = std::fs::remove_file(&out); + let cache = cache(); let output = mega_evme() .args([ "replay", "--rpc.replay-file", - CACHE, + cache.to_str().unwrap(), "--dump-fixture", out.to_str().unwrap(), TX, @@ -133,8 +151,16 @@ fn test_replay_dump_overwrites_atomically_without_tmp_residue() { // Seed a pre-existing "committed" fixture that the dump overwrites in place. std::fs::write(&out, br#"{"pre-existing":"corpus fixture"}"#).expect("seed existing fixture"); + let cache = cache(); let output = mega_evme() - .args(["replay", "--rpc.replay-file", CACHE, "--dump-fixture", out.to_str().unwrap(), TX]) + .args([ + "replay", + "--rpc.replay-file", + cache.to_str().unwrap(), + "--dump-fixture", + out.to_str().unwrap(), + TX, + ]) .output() .expect("failed to run mega-evme"); @@ -180,7 +206,7 @@ fn test_replay_dump_rejects_receipt_from_different_block() { // Doctor the capture: flip the receipt's blockHash. Cache entries are keyed // by the request, not the response, so the doctored entry still resolves. let mut envelope: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) .expect("parse offline cache"); let mut doctored = false; for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { @@ -235,11 +261,12 @@ fn test_replay_dump_rejects_spec_override() { let out = std::env::temp_dir().join(format!("mega_evme_dump_spec_{}.json", std::process::id())); let _ = std::fs::remove_file(&out); + let cache = cache(); let output = mega_evme() .args([ "replay", "--rpc.replay-file", - CACHE, + cache.to_str().unwrap(), "--dump-fixture", out.to_str().unwrap(), "--override.spec", diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index 586caf3e..a17f8d32 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -15,8 +15,9 @@ use std::{ mod common; /// Offline RPC capture, including the transaction's on-chain receipt. -const CACHE: &str = - concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/replay_offline.cache.json"); +/// Name of the committed offline capture, resolved through the shared fixture +/// helper so its location lives in exactly one place. +const CACHE: &str = "replay_offline.cache.json"; /// The transaction captured in `CACHE` (a 75,514-gas Rex5 mainnet call). const TX: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; @@ -98,7 +99,7 @@ fn replay_with_env(cache: &Path, args: &[&str], envs: &[(&str, &str)]) -> Run { /// The committed capture, unmodified. fn cache() -> PathBuf { - PathBuf::from(CACHE) + common::fixture(CACHE) } /// A temp path unique to this process and this test. @@ -110,7 +111,7 @@ fn temp_path(name: &str) -> PathBuf { /// `doctor`, and return its path. fn doctored_cache(name: &str, doctor: impl Fn(&mut serde_json::Value)) -> PathBuf { let mut envelope: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) .expect("parse offline cache"); let mut doctored = false; for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { @@ -136,7 +137,7 @@ fn doctored_cache(name: &str, doctor: impl Fn(&mut serde_json::Value)) -> PathBu /// modelling an endpoint that has pruned it. fn cache_without_receipt(name: &str) -> PathBuf { let mut envelope: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) .expect("parse offline cache"); let entries = envelope["cache"].as_array_mut().expect("cache entries"); let before = entries.len(); @@ -710,7 +711,7 @@ fn test_batch_dump_does_not_write_or_clobber_when_block_aborts_before_finish() { "0x91bbb37d27a588e217e5be6aeab0fb377ffea0ad3a2714d1f54ceb69852124f2"; let mut envelope: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(CACHE).expect("read offline cache")) + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) .expect("parse offline cache"); // Clone the dump target's TX response shape and rewrite hash + gas so the // later index is fetchable but rejected at execution (intrinsic gas). @@ -803,28 +804,14 @@ fn test_batch_dump_does_not_write_or_clobber_when_block_aborts_before_finish() { /// identity and a block-global `logIndex` that starts above zero when earlier /// receipts in the block already emitted logs. /// -/// The committed offline capture has no logs, so this test uses the dev -/// envelope (block 22945844, last tx) when `MEGA_EVME_TEST_ENVELOPE` is set — -/// the same fixture the ignored batch suite uses. Without the envelope the -/// non-empty path is covered by -/// `outcome::tests::test_op_receipt_to_tx_receipt_stamps_inner_log_metadata`. +/// The single-transaction capture this file otherwise uses is log-less, so this +/// one reads the whole-block capture, whose late transactions emit logs and sit +/// behind other log-emitting receipts. `MEGA_EVME_TEST_ENVELOPE` overrides it. #[test] fn test_replay_receipt_inner_log_metadata_matches_outer_receipt() { let envelope = match std::env::var("MEGA_EVME_TEST_ENVELOPE") { Ok(path) if !path.is_empty() => PathBuf::from(path), - _ => { - // Unit test covers non-empty logs + non-zero first_log_index; keep - // this integration test green in CI without the large envelope. - let run = replay(&cache(), &["--json", TX]); - assert!(run.success, "replay must exit 0.\nstderr: {}", run.stderr); - let summary = run.json(); - let logs = summary["receipt"]["logs"].as_array().expect("logs array"); - assert!( - logs.is_empty(), - "committed capture is log-less; set MEGA_EVME_TEST_ENVELOPE for multi-log coverage" - ); - return; - } + _ => common::fixture("replay_batch_blocks.cache.json"), }; // Last transaction of block 22945844: multi-log, with many preceding logs. From 6f1f8b554b92202dd090509457b239943bd90cf5 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 15:37:24 +0800 Subject: [PATCH 26/64] fix(mega-evme): keep the merged provider cache within its configured cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --rpc.cache-max-entries is documented as bounding what a run persists, but merge-on-persist wrote the union of the on-disk file and this run's entries without reapplying it. Runs that share a cache directory touch disjoint RPC keys, so the file grew past what any single run was allowed to keep, and every later start paid to parse all of it before the in-memory LRU could evict anything. Truncate the union to this run's cap, keeping this run's entries: they are already LRU-bounded by the same cap and are the ones it just proved it needs. The unit test alone did not pin this — it calls the merge with an explicit cap and so never touches the call site. The added store-level test does: it fails with 24 entries against a cap of 4 if the call site stops passing the cap. --- bin/mega-evme/src/cache/merge.rs | 61 +++++++++++++++++++ bin/mega-evme/src/cache/mod.rs | 9 +-- .../src/common/provider/cache_store.rs | 48 +++++++++++++-- 3 files changed, 110 insertions(+), 8 deletions(-) diff --git a/bin/mega-evme/src/cache/merge.rs b/bin/mega-evme/src/cache/merge.rs index 7c842104..cb8f1369 100644 --- a/bin/mega-evme/src/cache/merge.rs +++ b/bin/mega-evme/src/cache/merge.rs @@ -118,6 +118,36 @@ pub(crate) fn merge_kv_entries(base: Vec, overlay: Vec) -> Vec map.into_iter().map(|(key, value)| CacheKv { key, value }).collect() } +/// Merge `ours` over `on_disk` for a provider cache, keeping at most `cap` +/// entries. +/// +/// `--rpc.cache-max-entries` bounds what a run persists, so the union of a +/// sibling's file and ours must be bounded too: runs that share a cache +/// directory but touch disjoint RPC keys would otherwise grow the file without +/// limit, and every later start would parse all of it before the in-memory LRU +/// could evict anything. +/// +/// This process's entries are kept first — they are already LRU-bounded by the +/// same cap, and they are the ones this run just proved it needs. On-disk +/// entries then fill whatever room is left. +pub(crate) fn merge_provider_entries_capped( + on_disk: Vec, + ours: Vec, + cap: usize, +) -> Vec { + let mut map: BTreeMap = BTreeMap::new(); + for e in ours.into_iter().take(cap) { + map.insert(e.key, e.value); + } + for e in on_disk { + if map.len() >= cap { + break; + } + map.entry(e.key).or_insert(e.value); + } + map.into_iter().map(|(key, value)| CacheKv { key, value }).collect() +} + /// Detect whether `value` is a provider-cache array or a capture envelope. pub(crate) fn detect_shape(value: &serde_json::Value, path: &Path) -> Result { if value.is_array() { @@ -605,6 +635,37 @@ mod tests { assert_eq!(merged.cache, vec![kv(1, "disk"), kv(2, "ours")]); } + /// The merged provider cache never exceeds the configured cap, and this + /// process's entries survive the truncation. + /// + /// Runs sharing a cache directory touch disjoint RPC keys, so an uncapped + /// union grows the file without limit no matter how small each run's LRU is. + #[test] + fn test_merge_provider_entries_capped_bounds_the_union() { + let on_disk: Vec = (0..10).map(|i| kv(i, "disk")).collect(); + let ours: Vec = (100..104).map(|i| kv(i, "ours")).collect(); + + let merged = merge_provider_entries_capped(on_disk, ours.clone(), 6); + assert_eq!(merged.len(), 6, "the union is capped, not the sum of both sides"); + for entry in &ours { + assert!( + merged.iter().any(|m| m.key == entry.key && m.value == entry.value), + "this run's entries survive truncation: {:?}", + entry.key + ); + } + } + + /// A cap larger than the union keeps everything, and ours win on collision. + #[test] + fn test_merge_provider_entries_capped_keeps_all_below_the_cap() { + let on_disk = vec![kv(1, "disk"), kv(2, "disk")]; + let ours = vec![kv(2, "ours"), kv(3, "ours")]; + + let merged = merge_provider_entries_capped(on_disk, ours, 16); + assert_eq!(merged, vec![kv(1, "disk"), kv(2, "ours"), kv(3, "ours")]); + } + /// No opinion: loaded A, ours A (carried forward), disk now B → B wins and /// our cache entries still merge. A run given no `--bucket-capacity` reaches /// persist with `ours == loaded`; treating that as a conflict would fail the diff --git a/bin/mega-evme/src/cache/mod.rs b/bin/mega-evme/src/cache/mod.rs index f2853d0e..219fb5f2 100644 --- a/bin/mega-evme/src/cache/mod.rs +++ b/bin/mega-evme/src/cache/mod.rs @@ -12,10 +12,11 @@ use clap::{Parser, Subcommand}; use crate::common::{EvmeError, Result}; pub(crate) use merge::{ - lock_sidecar_path, merge_envelope_for_persist, merge_kv_entries, merge_provider_lists, - parse_rpc_cache_filename_chain_id, read_provider_cache, reread_envelope_for_merge, - write_bytes_atomic, write_envelope_atomic, write_provider_cache_atomic, CacheKv, EnvelopeDoc, - EnvelopeReread, ExternalEnvDoc, ENVELOPE_VERSION, + lock_sidecar_path, merge_envelope_for_persist, merge_provider_entries_capped, + merge_provider_lists, parse_rpc_cache_filename_chain_id, read_provider_cache, + reread_envelope_for_merge, write_bytes_atomic, write_envelope_atomic, + write_provider_cache_atomic, CacheKv, EnvelopeDoc, EnvelopeReread, ExternalEnvDoc, + ENVELOPE_VERSION, }; use merge::{load_cache_file, merge_envelopes_cli, CacheShape, LoadedCache}; diff --git a/bin/mega-evme/src/common/provider/cache_store.rs b/bin/mega-evme/src/common/provider/cache_store.rs index 4dcc9e85..4c137be4 100644 --- a/bin/mega-evme/src/common/provider/cache_store.rs +++ b/bin/mega-evme/src/common/provider/cache_store.rs @@ -31,9 +31,9 @@ use tracing::{info, warn}; use super::transport::TransportCache; use crate::{ cache::{ - lock_sidecar_path, merge_envelope_for_persist, merge_kv_entries, read_provider_cache, - reread_envelope_for_merge, write_bytes_atomic, write_envelope_atomic, CacheKv, EnvelopeDoc, - EnvelopeReread, ExternalEnvDoc, ENVELOPE_VERSION, + lock_sidecar_path, merge_envelope_for_persist, merge_provider_entries_capped, + read_provider_cache, reread_envelope_for_merge, write_bytes_atomic, write_envelope_atomic, + CacheKv, EnvelopeDoc, EnvelopeReread, ExternalEnvDoc, ENVELOPE_VERSION, }, common::{EvmeError, Result}, }; @@ -320,7 +320,10 @@ fn save_cache_atomic(cache: &SharedCache, target: &Path) -> std::io::Result<()> } }; - let merged = merge_kv_entries(disk_entries, our_entries); + // The union must respect the configured cap: a sibling's file plus ours can + // otherwise exceed what either run was allowed to keep. + let merged = + merge_provider_entries_capped(disk_entries, our_entries, cache.max_items() as usize); let serialized = serde_json::to_vec(&merged).map_err(|e| { std::io::Error::other(format!( "failed to serialize merged cache for {}: {e}", @@ -558,6 +561,43 @@ mod tests { } /// Interleaving: A holds only key A in memory; B persists key B; A then + /// Persisting into a shared cache directory keeps the file within the + /// configured cap. + /// + /// Runs that share a directory touch disjoint RPC keys, so merging a + /// sibling's file in wholesale would grow it past what either run was + /// allowed to keep, and every later start would parse all of it. + #[test] + fn test_provider_cache_persist_respects_the_configured_cap() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-1.json"); + + // A sibling with a bigger budget fills the file first. + let sibling = CacheLayer::new(64).cache(); + for i in 0..20u8 { + sibling.put(B256::repeat_byte(i), format!(r#"{{"result":"{i}"}}"#)).expect("put"); + } + RpcCacheStore::new(sibling, path.clone()).persist().expect("persist sibling"); + + // Ours is capped at 4 and holds keys the sibling never saw. + let ours = CacheLayer::new(4).cache(); + let mine: Vec = (100..104u8).map(B256::repeat_byte).collect(); + for key in &mine { + ours.put(*key, r#"{"result":"mine"}"#.to_string()).expect("put"); + } + RpcCacheStore::new(ours, path.clone()).persist().expect("persist ours"); + + let entries = crate::cache::read_provider_cache(&path).expect("read merged cache"); + assert!( + entries.len() <= 4, + "the merged file must respect this run's cap, got {} entries", + entries.len() + ); + for key in &mine { + assert!(entries.iter().any(|e| e.key == *key), "this run's entries survive: {key}"); + } + } + /// persists — on-disk file must contain the union (B's entries survive). #[test] fn test_provider_cache_persist_merges_interleaved_disk_entries() { From 003c2e25fb3ad066c81f7cf91f44d2242e6d5d48 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 5 Aug 2026 15:37:36 +0800 Subject: [PATCH 27/64] fix(mega-evme): check tx-file targets against the block that is replayed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolution asks eth_getTransactionByHash which block a target belongs to, then the driver fetches that block by number and replays it. Those are two separate answers from the endpoint, so a reorg or a load-balanced backend can serve two views and the targets get replayed against a block they are not in — reported as ordinary results rather than as the infrastructure inconsistency they are. Carry the inclusion hash through resolution and reject the block when it does not match, the same way the parent-linkage guard already rejects a parent that is not the parent. Two targets of one block number reporting different inclusion hashes is the same failure seen earlier, so the disagreeing target is reported as unanswered rather than replayed against whichever view happened to be recorded first. Both guards check headers only. The state reads behind the fork are still addressed by number, which the comment now says: anchoring them would mean forking by hash, and alloy hashes the block id into its cache keys, so every committed offline capture would stop resolving. --- bin/mega-evme/src/replay/batch.rs | 71 +++++++++++++++++++++++++++-- bin/mega-evme/tests/replay_batch.rs | 59 ++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 5 deletions(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index cb77651d..448c8ec4 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -308,6 +308,10 @@ struct BlockJob { block: Option>, /// Hashes of the transactions whose results are reported. targets: Vec, + /// Block hash the targets reported as their inclusion, when a resolution + /// step observed one (`--tx-file`). `None` for `--block`, whose targets come + /// from the block body itself and so cannot disagree with it. + inclusion_hash: Option, } /// Fixture work for one target, held until `finish()` succeeds. @@ -419,7 +423,12 @@ where eprintln!("Block {number} contains no transactions; nothing to replay"); vec![] } else { - vec![BlockJob { number: *number, block: Some(block), targets }] + vec![BlockJob { + number: *number, + block: Some(block), + targets, + inclusion_hash: None, + }] } } BatchMode::TxList(hashes) => { @@ -488,7 +497,7 @@ async fn resolve_targets

(provider: &P, hashes: &[B256]) -> (Vec, Ve where P: Provider, { - let mut grouped: BTreeMap> = BTreeMap::new(); + let mut grouped: BTreeMap, Option)> = BTreeMap::new(); let mut failures = Vec::new(); for hash in hashes { @@ -504,7 +513,30 @@ where message: "Transaction not found".to_string(), }), Ok(Some(tx)) => match tx.block_number { - Some(number) => grouped.entry(number).or_default().push(*hash), + Some(number) => { + let (targets, inclusion) = grouped.entry(number).or_default(); + // Two targets resolving to the same number but different + // block hashes means the endpoint served two views. Neither + // can be trusted, so the disagreeing target is reported as + // unanswered rather than silently replayed against one view. + match (*inclusion, tx.block_hash) { + (Some(seen), Some(theirs)) if seen != theirs => { + failures.push(FailedTx { + tx_hash: *hash, + kind: BatchErrorKind::Rpc, + message: format!( + "inclusion block hash {theirs} for block {number} differs \ + from {seen} reported by an earlier target of the same \ + block: the endpoint served divergent views" + ), + }); + continue; + } + (None, Some(theirs)) => *inclusion = Some(theirs), + _ => {} + } + targets.push(*hash); + } None => failures.push(FailedTx { tx_hash: *hash, kind: BatchErrorKind::Pending, @@ -516,7 +548,12 @@ where let jobs = grouped .into_iter() - .map(|(number, targets)| BlockJob { number, block: None, targets }) + .map(|(number, (targets, inclusion_hash))| BlockJob { + number, + block: None, + targets, + inclusion_hash, + }) .collect(); (jobs, failures) } @@ -543,7 +580,7 @@ async fn replay_block

( where P: Provider + Clone + std::fmt::Debug, { - let BlockJob { number, block, targets } = job; + let BlockJob { number, block, targets, inclusion_hash } = job; let verify_receipt = report.verify_receipt; let dump_dir = report.dump_fixture_dir.as_deref(); let overwrite = report.overwrite; @@ -568,6 +605,14 @@ where } }; + // Both guards below check the *headers* the endpoint served. The state + // reads behind the fork are still addressed by block number, so an endpoint + // that serves headers and state from different backends can still hand back + // state for a different block at this height. Anchoring state reads to the + // validated hash would need the fork to take a block hash rather than a + // number, and would change every cached RPC key (alloy hashes the block id + // into the cache key), invalidating every committed offline capture. + // // Parent/block linkage guard: across a reorg or a load-balanced endpoint // serving divergent views, `eth_getBlockByNumber(N-1)` can return a block // that is not the parent of the block being replayed. Forking from that @@ -583,6 +628,22 @@ where return fail_all(&targets, BatchErrorKind::Rpc, &message); } + // Inclusion guard: `--tx-file` resolved each target through + // `eth_getTransactionByHash`, which reported the block it belongs to. If the + // block fetched by that number is a different one, the endpoint served two + // views and the targets do not belong to what is about to be replayed. + if let Some(expected) = inclusion_hash { + let fetched = block.hash(); + if fetched != expected { + let message = format!( + "block {number} has hash {fetched}, but its targets were resolved as included in \ + {expected}: the endpoint served divergent views of this block (reorg in \ + progress, or a load-balanced endpoint); retry once the chain settles" + ); + return fail_all(&targets, BatchErrorKind::Rpc, &message); + } + } + // Fetch the on-chain receipts before the block runs. Needed for // `--verify-receipt` (mismatch vs unverified) and for `--dump-fixture-dir` // (fidelity gate). A receipt that cannot be fetched, or that describes a diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 63f5ae4d..2c1358be 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -502,6 +502,65 @@ fn test_replay_batch_rejects_single_transaction_flags() { } /// A parent block whose hash does not match the child block's `parentHash` is an +/// A `--tx-file` target whose reported inclusion block is not the block fetched +/// by that number is unanswered, not replayed. +/// +/// The endpoint answers `eth_getTransactionByHash` and `eth_getBlockByNumber` +/// separately, so a reorg or a load-balanced backend can serve two views. The +/// resolution step records the inclusion hash so the mismatch is caught before +/// the block runs, instead of replaying targets against a block they are not in. +#[test] +fn test_replay_tx_file_rejects_a_block_that_does_not_match_the_resolved_inclusion() { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let wrong_hash = "0x2222222222222222222222222222222222222222222222222222222222222222"; + let (target, _) = BLOCK_TXS[1]; + + // Rewrite only the transaction's own response: it now claims to belong to a + // block whose hash differs from the one `eth_getBlockByNumber` returns. + let marker = format!("\"hash\":\"{target}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object"); + result["blockHash"] = serde_json::Value::String(wrong_hash.into()); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "exactly one response describes the target transaction"); + + let envelope_path = + std::env::temp_dir().join(format!("mega_evme_batch_inclusion_{}.json", std::process::id())); + std::fs::write(&envelope_path, envelope.to_string()).expect("write doctored envelope"); + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_inclusion_{}.txt", std::process::id())); + std::fs::write(&list, format!("{target}\n")).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&envelope_path, &["--tx-file", list.to_str().unwrap(), "--json"]); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), 1, "the single target is reported once: {stdout}"); + assert_eq!( + lines[0]["error"]["kind"].as_str(), + Some("rpc"), + "divergent views are unanswered, not a wrong answer: {}", + lines[0] + ); + let message = lines[0]["error"]["message"].as_str().unwrap_or_default(); + assert!(message.contains("divergent views"), "message names the cause: {message}"); + assert_eq!(code, Some(3), "an unanswered target exits 3"); + + let _ = std::fs::remove_file(&envelope_path); + let _ = std::fs::remove_file(&list); +} + /// infrastructure failure for every target of that block (reorg / divergent views). #[test] fn test_replay_block_rejects_mismatched_parent_hash() { From 36081e7e67b1b5d66496a86d0fe3b33e56bf0339 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 11 Aug 2026 15:12:55 +0800 Subject: [PATCH 28/64] fix(mega-evme): read all-zero RPC accounts as nonexistent JSON-RPC cannot express account non-existence, so the forked backend materialized never-created accounts as existing empty accounts. That flipped the EIP-7702 per-authorization refund: a brand-new authority was judged already-in-trie and each replayed authorization refunded 12,500 gas the chain did not, making replayed gasUsed under-report the receipt. Map the all-zero (balance, nonce, code) answer back to None. This is safe post-EIP-161: an existing-but-empty account cannot occur on any chain this tool replays. --- bin/mega-evme/src/common/provider/mod.rs | 5 +- bin/mega-evme/src/common/state.rs | 27 ++- bin/mega-evme/src/replay/cmd.rs | 60 ++++- bin/mega-evme/tests/account_existence.rs | 209 ++++++++++++++++++ bin/mega-evme/tests/batch_cache_default.rs | 199 +++++++++++++++++ bin/mega-evme/tests/common/mod.rs | 16 ++ docs/mega-evme/commands/replay.md | 5 +- .../configuration/state-management.md | 7 +- 8 files changed, 518 insertions(+), 10 deletions(-) create mode 100644 bin/mega-evme/tests/account_existence.rs create mode 100644 bin/mega-evme/tests/batch_cache_default.rs diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index e1116a17..ed16e4a0 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -111,11 +111,14 @@ pub struct RpcArgs { /// /// Defaults to the platform cache directory (`$XDG_CACHE_HOME/mega-evme/rpc` on /// Linux, `~/Library/Caches/mega-evme/rpc` on macOS). Pass `--rpc.no-cache-file` - /// to disable on-disk persistence entirely. + /// to disable on-disk persistence entirely. Batch replay (`--tx-file` / `--block`) + /// uses the on-disk cache only when this flag names a directory explicitly. #[arg(long = "rpc.cache-dir", value_parser = parse_non_empty_path)] pub cache_dir: Option, /// Disable on-disk cache persistence. The in-memory LRU cache still applies. + /// This is already the default for batch replay (`--tx-file` / `--block`) + /// unless `--rpc.cache-dir` is passed. #[arg(long = "rpc.no-cache-file")] pub no_cache_file: bool, diff --git a/bin/mega-evme/src/common/state.rs b/bin/mega-evme/src/common/state.rs index bbafccdc..6460e780 100644 --- a/bin/mega-evme/src/common/state.rs +++ b/bin/mega-evme/src/common/state.rs @@ -454,6 +454,25 @@ where Forked(Box>>>), } +/// Normalizes an account fetched over RPC, mapping the all-zero answer to "does not exist". +/// +/// JSON-RPC cannot express account non-existence: `eth_getBalance`, `eth_getTransactionCount`, +/// and `eth_getCode` all answer `0`/`0`/empty for an account that was never created, so the RPC +/// backend materializes it as an *existing* empty account. That flips every consumer of +/// existence, most visibly the EIP-7702 per-authorization refund: a brand-new authority is +/// judged "already in the trie" and the replay refunds 12,500 gas per authorization that the +/// chain did not. +/// +/// Mapping all-zero back to `None` is safe because an existing-but-empty account cannot occur +/// on `MegaETH`: EIP-161 (Spurious Dragon) removes empty accounts on touch and forbids creating +/// them, every chain this tool replays activated it from genesis, and the genesis allocs carry +/// balance or code. The one shape this cannot distinguish — an account with zero +/// balance/nonce/code that still holds storage — also cannot exist post-EIP-161, since storage +/// is only reachable through code and contracts have a non-empty code hash or nonce. +fn normalize_rpc_account(account: Option) -> Option { + account.filter(|info| !info.is_empty()) +} + /// State database that can be backed by either [`EmptyDB`] or [`AlloyDB`] (forked from RPC) #[derive(Debug)] pub struct EvmeState @@ -645,9 +664,9 @@ where Ok(account) } EvmeBackend::Forked(db) => { - let account = db.basic(address).map_err(|e| { + let account = normalize_rpc_account(db.basic(address).map_err(|e| { EvmeError::RpcError(format!("Failed to fetch account {}: {:?}", address, e)) - })?; + })?); trace!(address = %address, account = ?account, "Loaded account basic from forked state"); Ok(account) } @@ -766,9 +785,9 @@ where Ok(account) } EvmeBackend::Forked(db) => { - let account = db.basic_ref(address).map_err(|e| { + let account = normalize_rpc_account(db.basic_ref(address).map_err(|e| { EvmeError::RpcError(format!("Failed to fetch account {}: {:?}", address, e)) - })?; + })?); trace!(address = %address, account = ?account, "Loaded account basic from forked state"); Ok(account) } diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index fb1cc57d..8050f907 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -26,7 +26,7 @@ use crate::{ common::{ op_receipt_to_tx_receipt, parse_bucket_capacity, print_execution_summary, print_execution_trace, print_receipt, BuildProviderOutput, EvmeExternalEnvs, EvmeOutcome, - ExecutionSummary, ExternalEnvSnapshot, OpTxReceipt, RpcCacheStore, TracerType, + ExecutionSummary, ExternalEnvSnapshot, OpTxReceipt, RpcArgs, RpcCacheStore, TracerType, TxOverrideArgs, }, replay::get_hardfork_config, @@ -458,7 +458,11 @@ impl Cmd { self.rpc_args.build_replay_provider().await? } else if let Some(rpc) = &self.rpc_args.rpc_url { info!(rpc = %rpc, "Provider mode: online RPC"); - self.rpc_args.build_provider().await? + if self.is_batch() { + self.batch_rpc_args().build_provider().await? + } else { + self.rpc_args.build_provider().await? + } } else { return Err(ReplayError::Other( "'mega-evme replay' requires '--rpc ', '--rpc.capture-file ', \ @@ -471,6 +475,27 @@ impl Cmd { Ok(ProviderContext { provider, cache_store, external_env, chain_id }) } + /// The RPC args a batch run actually uses: on-disk cache persistence is opt-in for batch. + /// + /// A batch scan walks linear history — its request keys are block-scoped and essentially + /// never repeat across runs, so a shared cache file buys almost no hits, while its + /// clean-exit persist re-reads, merges, and rewrites the whole file under a cross-process + /// lock. That exit tail grows linearly with the file and serializes across concurrent + /// batch processes sharing the default cache directory, so batch mode keeps the disk + /// cache off unless `--rpc.cache-dir` names one explicitly. The in-memory LRU (and its + /// intra-run reuse across a block's transactions) is unaffected. + fn batch_rpc_args(&self) -> RpcArgs { + let mut args = self.rpc_args.clone(); + if args.cache_dir.is_none() && !args.no_cache_file { + info!( + "Batch replay leaves the on-disk RPC cache disabled; \ + pass --rpc.cache-dir to enable it" + ); + args.no_cache_file = true; + } + args + } + /// Fetch the transaction, its block, and preceding transaction hashes from the provider. async fn fetch_replay_context

( &self, @@ -1032,6 +1057,37 @@ mod tests { cmd.validate().expect_err("batch mode must reject the flag").to_string() } + /// Parse an online `replay` invocation (no offline RPC flags). + fn parse_online(extra: &[&str]) -> Cmd { + let mut argv = vec!["replay", "--rpc", "http://localhost:1"]; + argv.extend_from_slice(extra); + Cmd::try_parse_from(argv).expect("online flags should parse") + } + + #[test] + fn test_batch_rpc_args_disables_disk_cache_by_default() { + let cmd = parse_online(&["--block", "1"]); + assert!(!cmd.rpc_args.no_cache_file, "the flag itself must default off"); + assert!( + cmd.batch_rpc_args().no_cache_file, + "a batch run without --rpc.cache-dir must not touch the disk cache", + ); + } + + #[test] + fn test_batch_rpc_args_explicit_cache_dir_opts_back_in() { + let cmd = parse_online(&["--block", "1", "--rpc.cache-dir", "/tmp/evme-cache"]); + let args = cmd.batch_rpc_args(); + assert!(!args.no_cache_file, "an explicit --rpc.cache-dir must keep persistence on"); + assert_eq!(args.cache_dir.as_deref(), Some(std::path::Path::new("/tmp/evme-cache"))); + } + + #[test] + fn test_batch_rpc_args_keeps_explicit_no_cache_file() { + let cmd = parse_online(&["--block", "1", "--rpc.no-cache-file"]); + assert!(cmd.batch_rpc_args().no_cache_file, "--rpc.no-cache-file must stay honored"); + } + #[test] fn test_replay_target_group_accepts_each_form() { assert!(matches!( diff --git a/bin/mega-evme/tests/account_existence.rs b/bin/mega-evme/tests/account_existence.rs new file mode 100644 index 00000000..d0b64048 --- /dev/null +++ b/bin/mega-evme/tests/account_existence.rs @@ -0,0 +1,209 @@ +//! Integration tests for the forked backend's account-existence normalization. +//! +//! JSON-RPC cannot express "this account was never created": `eth_getBalance`, +//! `eth_getTransactionCount`, and `eth_getCode` all answer `0`/`0`/empty for +//! it, and the RPC backend would otherwise materialize that answer as an +//! *existing* empty account. `EvmeState` maps the all-zero answer back to +//! `None` (safe post-EIP-161, where existing-but-empty accounts cannot occur). +//! +//! The DB-level tests pin the normalization boundary: only the fully-zero +//! account maps to `None`; any single non-zero dimension keeps it existing. +//! The execution-level tests pin the consumer that made the bug observable: +//! EIP-7702 refunds 12,500 gas per authorization only when the authority +//! already exists in the trie, so a brand-new authority materialized as an +//! existing empty account made every replayed type-4 transaction with fresh +//! authorities under-report `gasUsed` by 12,500 per authorization. + +use alloy_eips::{ + eip2930::{AccessList, AccessListItem}, + eip7702::{Authorization, RecoveredAuthority, RecoveredAuthorization}, +}; +use alloy_primitives::{address, Address, Bytes, B256, U256}; +use clap::Parser; +use mega_evm::{ + revm::{ + context::{result::ExecutionResult, tx::TxEnvBuilder}, + state::EvmState, + DatabaseRef, ExecuteEvm, + }, + MegaContext, MegaEvm, MegaSpecId, MegaTransaction, MegaTransactionNew as _, +}; +use mega_evme::common::{EvmeExternalEnvs, EvmeState, OpProvider, PreStateArgs}; +use op_alloy_network::Optimism; +use rstest::rstest; + +mod common; +use common::{test_rpc_args, MockRpcServer}; + +/// The transaction sender; funded through a prestate override, so its account +/// never reaches the RPC mock. +const CALLER: Address = address!("00000000000000000000000000000000c0ffee01"); + +/// The call target. Has no code on the mock chain. +const CALLEE: Address = address!("00000000000000000000000000000000c0ffee02"); + +/// The EIP-7702 authority whose existence the scenarios vary. +const AUTHORITY: Address = address!("00000000000000000000000000000000c0ffee03"); + +/// The delegation designator target. Never loaded (only written). +const DELEGATE: Address = address!("00000000000000000000000000000000c0ffee04"); + +/// EIP-7702 `PER_EMPTY_ACCOUNT_COST - PER_AUTH_BASE_COST`: the per-authorization +/// refund granted when the authority already exists in the trie. +const EXISTING_AUTHORITY_REFUND: u64 = 12_500; + +/// Build a forked `EvmeState` against a mock whose `eth_getBalance` / +/// `eth_getTransactionCount` / `eth_getCode` answers are fixed for every +/// address. `eth_getStorageAt` answers the zero word so OP L1-fee loading +/// resolves without a live node. +async fn forked_state( + server: &MockRpcServer, + balance: &str, + nonce: &str, + code: &str, +) -> EvmeState { + server.respond_eth_chain_id(4326, 1).await; + server.respond_method_result("eth_getBalance", balance, 2).await; + server.respond_method_result("eth_getTransactionCount", nonce, 2).await; + server.respond_method_result("eth_getCode", code, 2).await; + server + .respond_method_result( + "eth_getStorageAt", + "0x0000000000000000000000000000000000000000000000000000000000000000", + 2, + ) + .await; + + let prestate_args = PreStateArgs::parse_from(["mega-evme", "--fork", "--fork.block", "1"]); + let rpc_args = test_rpc_args(&server.uri(), None); + let (state, _cache_store) = + prestate_args.create_initial_state(&CALLER, &rpc_args).await.expect("create_initial_state"); + state +} + +// ─── DB-level: the normalization boundary ──────────────────────────────────── + +/// An account whose balance, nonce, and code are all zero does not exist. +#[tokio::test(flavor = "multi_thread")] +async fn test_forked_all_zero_account_reads_as_nonexistent() { + let server = MockRpcServer::start().await; + let state = forked_state(&server, "0x0", "0x0", "0x").await; + + let account = state.basic_ref(AUTHORITY).expect("basic_ref"); + assert_eq!(account, None, "an all-zero RPC answer must read as a nonexistent account"); +} + +/// A balance alone keeps the account existing (e.g. a plain EOA that only +/// ever received funds). +#[tokio::test(flavor = "multi_thread")] +async fn test_forked_balance_only_account_exists() { + let server = MockRpcServer::start().await; + let state = forked_state(&server, "0x1", "0x0", "0x").await; + + let account = state.basic_ref(AUTHORITY).expect("basic_ref").expect("account must exist"); + assert_eq!(account.balance, U256::from(1)); + assert_eq!(account.nonce, 0); +} + +/// A nonce alone keeps the account existing (e.g. an EOA that spent its +/// entire balance on fees). +#[tokio::test(flavor = "multi_thread")] +async fn test_forked_nonce_only_account_exists() { + let server = MockRpcServer::start().await; + let state = forked_state(&server, "0x0", "0x1", "0x").await; + + let account = state.basic_ref(AUTHORITY).expect("basic_ref").expect("account must exist"); + assert_eq!(account.balance, U256::ZERO); + assert_eq!(account.nonce, 1); +} + +/// Code alone keeps the account existing (e.g. a contract with neither +/// balance nor nonce is still a contract). +#[tokio::test(flavor = "multi_thread")] +async fn test_forked_code_only_account_exists() { + let server = MockRpcServer::start().await; + let state = forked_state(&server, "0x0", "0x0", "0x6001").await; + + let account = state.basic_ref(AUTHORITY).expect("basic_ref").expect("account must exist"); + assert!(!account.is_empty_code_hash(), "the fetched code must be reflected in the code hash"); +} + +// ─── Execution-level: the EIP-7702 refund consumer ─────────────────────────── + +/// Replay a type-4 transaction whose single authorization names `AUTHORITY`, +/// returning the gas it used. The access list pads execution gas so the +/// EIP-3529 refund cap (`gas_used / 5`) stays above the full 12,500 refund — +/// otherwise the two scenarios' gas would differ by the cap, not the refund. +async fn type4_gas_used(server: &MockRpcServer, spec: MegaSpecId) -> u64 { + let mut state = forked_state(server, "0x0", "0x0", "0x").await; + state.set_account_balance(CALLER, U256::from(10).pow(U256::from(18))); + + let authorization = RecoveredAuthorization::new_unchecked( + // chain_id 0 = valid on any chain, so the context's chain id is irrelevant. + Authorization { chain_id: U256::ZERO, address: DELEGATE, nonce: 0 }, + RecoveredAuthority::Valid(AUTHORITY), + ); + let access_list = AccessList(vec![AccessListItem { + address: CALLEE, + storage_keys: (0u64..40).map(|i| B256::from(U256::from(i))).collect(), + }]); + let tx_env = TxEnvBuilder::default() + .caller(CALLER) + .call(CALLEE) + .gas_limit(1_000_000) + .access_list(access_list) + .authorization_list_recovered(vec![authorization]) + .build_fill(); + + let context = + MegaContext::new(&mut state, spec).with_external_envs(EvmeExternalEnvs::new().into()); + let mut evm = MegaEvm::new(context); + let mut tx = MegaTransaction::new(tx_env); + tx.enveloped_tx = Some(Bytes::new()); + let outcome = evm.transact(tx).expect("type-4 replay must execute"); + + assert!( + matches!(outcome.result, ExecutionResult::Success { .. }), + "type-4 replay must succeed, got {:?}", + outcome.result, + ); + assert_delegated(&outcome.state); + outcome.result.tx_gas_used() +} + +/// The authorization must have actually applied — a skipped authorization +/// would make the gas comparison vacuous. +fn assert_delegated(state: &EvmState) { + let authority = state.get(&AUTHORITY).expect("the authority must be in the post-state"); + assert_eq!(authority.info.nonce, 1, "the applied authorization must bump the authority nonce"); + assert!( + authority.info.code.as_ref().is_some_and(|code| code.is_eip7702()), + "the authority must carry the delegation designator", + ); +} + +/// A brand-new authority (all-zero on RPC) must not earn the existing-account +/// refund: the replayed `gasUsed` is exactly 12,500 above the run whose +/// authority exists. Before normalization both runs were refunded and a replay +/// under-reported `gasUsed` against the on-chain receipt. +#[rstest] +#[case::rex5(MegaSpecId::REX5)] +#[case::rex6(MegaSpecId::REX6)] +#[tokio::test(flavor = "multi_thread")] +async fn test_type4_fresh_authority_is_not_refunded(#[case] spec: MegaSpecId) { + let fresh_server = MockRpcServer::start().await; + let gas_fresh_authority = type4_gas_used(&fresh_server, spec).await; + + // Same transaction, but every RPC account (including the authority) holds + // 1 wei. The caller and its balance come from the prestate override in + // both runs, so the authority's existence is the only difference. + let existing_server = MockRpcServer::start().await; + existing_server.respond_method_result("eth_getBalance", "0x1", 1).await; + let gas_existing_authority = type4_gas_used(&existing_server, spec).await; + + assert_eq!( + gas_fresh_authority, + gas_existing_authority + EXISTING_AUTHORITY_REFUND, + "a fresh authority must not earn the 12,500 existing-account refund", + ); +} diff --git a/bin/mega-evme/tests/batch_cache_default.rs b/bin/mega-evme/tests/batch_cache_default.rs new file mode 100644 index 00000000..655e8960 --- /dev/null +++ b/bin/mega-evme/tests/batch_cache_default.rs @@ -0,0 +1,199 @@ +//! Integration tests for the batch-mode on-disk cache default. +//! +//! Batch replay (`--tx-file` / `--block`) engages the on-disk RPC cache only +//! when `--rpc.cache-dir` names a directory explicitly. The clean-exit persist +//! re-reads, merges, and atomically rewrites the whole per-chain cache file +//! under a cross-process lock, so its cost grows with the file and serializes +//! across concurrent processes — while a linear history scan gets almost no +//! cache hits in return. With the default in place a batch exit performs zero +//! disk-cache work, making its cost independent of any cache a machine has +//! accumulated. Single-transaction replay keeps the previous default. +//! +//! The tests point the child's platform cache directory into a temp dir via +//! `HOME` / `XDG_CACHE_HOME`, so the real user cache is never touched. + +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +use tempfile::TempDir; + +mod common; +use common::MockRpcServer; + +/// Any syntactically valid transaction hash; every lookup fails at the mock. +const TX: &str = "0x1111111111111111111111111111111111111111111111111111111111111111"; + +/// A fake home directory the child process resolves its platform cache dir in. +struct FakeHome { + dir: TempDir, +} + +impl FakeHome { + fn new() -> Self { + Self { dir: tempfile::tempdir().expect("tempdir") } + } + + fn path(&self) -> &Path { + self.dir.path() + } + + /// Where the child's default per-chain cache file lands for chain 4326. + /// + /// Mirrors `dirs::cache_dir()` under the overridden environment: macOS + /// resolves `$HOME/Library/Caches`, other unixes `$XDG_CACHE_HOME` (which + /// the tests always set). + fn default_cache_file(&self) -> PathBuf { + let base = if cfg!(target_os = "macos") { + self.path().join("Library/Caches") + } else { + self.path().join("xdg-cache") + }; + base.join("mega-evme/rpc/rpc-cache-4326.json") + } + + /// Every `rpc-cache-*.json` anywhere under the fake home. + fn cache_files(&self) -> Vec { + fn walk(dir: &Path, hits: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + walk(&path, hits); + } else if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + if name.starts_with("rpc-cache-") && name.ends_with(".json") { + hits.push(path); + } + } + } + } + let mut hits = Vec::new(); + walk(self.path(), &mut hits); + hits + } +} + +/// Run `mega-evme replay` with the platform cache dir redirected into `home`. +fn replay(home: &FakeHome, args: &[&str]) { + let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .arg("replay") + .args(args) + .env("HOME", home.path()) + .env("XDG_CACHE_HOME", home.path().join("xdg-cache")) + .output() + .expect("failed to run mega-evme"); + // Every scenario here replays a transaction the mock cannot answer, so the + // run itself fails; the assertions are about the cache file side effects. + assert!( + output.status.code().is_some(), + "mega-evme must exit, not die on a signal: {output:?}" + ); +} + +/// A mock whose chain id resolves (4326) and whose every other request fails +/// without triggering the retry layer. +async fn failing_mock() -> MockRpcServer { + let server = MockRpcServer::start().await; + server.respond_eth_chain_id(4326, 1).await; + server.respond_jsonrpc_error(-32601, "no such method", 2).await; + server +} + +/// Write a `--tx-file` under the fake home and return its path. +fn tx_file(home: &FakeHome) -> PathBuf { + let path = home.path().join("targets.txt"); + std::fs::write(&path, format!("{TX}\n")).expect("write tx file"); + path +} + +/// A default-flag batch run must neither read nor write any on-disk cache: +/// a pre-existing default cache file survives byte-identical (even though its +/// content is garbage a load would have rejected), and no new cache file +/// appears anywhere. The exit therefore does no work proportional to the +/// cache a machine has accumulated, no matter how many targets were replayed. +#[tokio::test(flavor = "multi_thread")] +async fn test_batch_default_leaves_disk_cache_untouched() { + let home = FakeHome::new(); + let seeded = home.default_cache_file(); + std::fs::create_dir_all(seeded.parent().expect("cache file has a parent")).expect("mkdir"); + std::fs::write(&seeded, b"not even json").expect("seed cache file"); + + let server = failing_mock().await; + let targets = tx_file(&home); + + replay( + &home, + &[ + "--tx-file", + targets.to_str().expect("utf-8"), + "--rpc", + &server.uri(), + "--rpc.max-retries", + "0", + "--rpc.backoff-ms", + "1", + "--json", + ], + ); + + let bytes = std::fs::read(&seeded).expect("seeded file must still exist"); + assert_eq!(bytes, b"not even json", "the seeded default cache file must stay byte-identical"); + assert_eq!( + home.cache_files(), + vec![seeded], + "no other cache file may appear anywhere under the fake home", + ); +} + +/// An explicit `--rpc.cache-dir` opts a batch run back into persistence: the +/// per-chain cache file is written on exit. +#[tokio::test(flavor = "multi_thread")] +async fn test_batch_explicit_cache_dir_still_persists() { + let home = FakeHome::new(); + let cache_dir = home.path().join("explicit-cache"); + + let server = failing_mock().await; + let targets = tx_file(&home); + + replay( + &home, + &[ + "--tx-file", + targets.to_str().expect("utf-8"), + "--rpc", + &server.uri(), + "--rpc.cache-dir", + cache_dir.to_str().expect("utf-8"), + "--rpc.max-retries", + "0", + "--rpc.backoff-ms", + "1", + "--json", + ], + ); + + assert!( + cache_dir.join("rpc-cache-4326.json").exists(), + "an explicit --rpc.cache-dir must persist the cache file on exit", + ); +} + +/// Single-transaction replay keeps the previous default: the per-chain cache +/// file is persisted into the platform cache directory. +#[tokio::test(flavor = "multi_thread")] +async fn test_single_replay_still_persists_by_default() { + let home = FakeHome::new(); + + let server = failing_mock().await; + + replay( + &home, + &[TX, "--rpc", &server.uri(), "--rpc.max-retries", "0", "--rpc.backoff-ms", "1", "--json"], + ); + + assert!( + home.default_cache_file().exists(), + "single-transaction replay must keep persisting the default cache file", + ); +} diff --git a/bin/mega-evme/tests/common/mod.rs b/bin/mega-evme/tests/common/mod.rs index f21c9cdf..18712500 100644 --- a/bin/mega-evme/tests/common/mod.rs +++ b/bin/mega-evme/tests/common/mod.rs @@ -142,6 +142,22 @@ impl MockRpcServer { .await; } + /// Mount an unbounded mock that answers every JSON-RPC request for + /// `method` with the given hex `result`, regardless of params. + pub(crate) async fn respond_method_result(&self, method: &str, hex_result: &str, priority: u8) { + let body = serde_json::json!({ + "jsonrpc": "2.0", + "id": 0, + "result": hex_result, + }); + Mock::given(matchers::method("POST")) + .and(matchers::body_partial_json(serde_json::json!({ "method": method }))) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .with_priority(priority) + .mount(&self.server) + .await; + } + /// Mount a mock that returns `eth_chainId` with the given chain id. pub(crate) async fn respond_eth_chain_id(&self, chain_id: u64, priority: u8) { let body = serde_json::json!({ diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 47745b04..d6e91ac4 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -49,7 +49,10 @@ Batch mode does all of it once. A batch run builds a single provider and a single RPC cache, groups the requested transactions by their containing block, and processes the blocks in ascending order. Each block is executed exactly once: state is forked at the parent block, pre-execution changes are applied, and every transaction of the block runs in order, with each requested transaction's result recorded before it is committed. -The RPC cache is persisted once, on exit, even if some transactions failed — the captured responses are the artifact you need to debug the failure offline. +A capture file (`--rpc.capture-file`) is persisted once, on exit, even if some transactions failed — the captured responses are the artifact you need to debug the failure offline. +The per-chain on-disk RPC cache is opt-in for batch runs: it is loaded and persisted only when `--rpc.cache-dir` names a directory explicitly. +A batch scan walks linear history whose request keys essentially never repeat across runs, so a shared cache file buys almost no hits, while its clean-exit re-read-merge-rewrite grows with the file and serializes concurrent processes on the persist lock. +The in-memory cache still serves every repeated request within the run. A plain batch replay issues the same RPC calls as single-transaction replay, so an offline envelope captured by single-transaction runs serves a batch run without a cache miss. `--verify-receipt` and `--dump-fixture-dir` are the exception: both fetch the receipt of every target in the block, including transactions a single-transaction capture never asked about, so an older envelope will miss them and the run exits `3`. diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index 07beee76..bc473503 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -217,6 +217,9 @@ The default cache directory is the platform cache directory: - **Linux**: `$XDG_CACHE_HOME/mega-evme/rpc` - **macOS**: `~/Library/Caches/mega-evme/rpc` +Batch replay (`--tx-file` / `--block`) is the exception: it engages the on-disk cache only when `--rpc.cache-dir` names a directory explicitly, and otherwise behaves as if `--rpc.no-cache-file` were set. +A batch scan walks linear history whose request keys essentially never repeat across runs, so the file buys almost no hits, while its clean-exit persist re-reads, merges, and rewrites the whole file under the cross-process lock — a cost that grows with the file and serializes concurrent batch processes. + ### Concurrent cache-dir sharing Multiple `mega-evme` processes may share the same `--rpc.cache-dir` safely. @@ -255,8 +258,8 @@ Provider-cache merge also rejects inputs (and `--output`) whose `rpc-cache-{chai | Flag | Type | Default | Description | | ----------------------------- | ----- | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--rpc.cache-max-entries ` | `u32` | `0` | Maximum number of items in the in-memory RPC LRU cache (and therefore what is persisted to the cache file). `0` = effectively unlimited (caps at 1,048,576 entries; the cache index is preallocated proportional to the cap). Default. | -| `--rpc.cache-dir ` | path | Platform cache dir | Directory for per-chain cache files. Each chain's cache is stored as `{cache_dir}/rpc-cache-{chain_id}.json`. | -| `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies. | +| `--rpc.cache-dir ` | path | Platform cache dir | Directory for per-chain cache files. Each chain's cache is stored as `{cache_dir}/rpc-cache-{chain_id}.json`. Batch replay uses the on-disk cache only when this flag is passed explicitly. | +| `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies. Already the default for batch replay unless `--rpc.cache-dir` is passed. | | `--rpc.clear-cache` | flag | `false` | Delete the current chain's cache file before loading it. Recovery path for a polluted or corrupt cache. | The in-memory cache layer is always installed on a forked or online run and cannot be turned off; `--rpc.no-cache-file` disables only on-disk persistence. From 409de9aeed6d487dfb4cff4c4d9c3197c27be259 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 11 Aug 2026 15:12:55 +0800 Subject: [PATCH 29/64] feat(mega-evme): make the on-disk RPC cache opt-in for batch replay A batch scan walks linear history whose request keys essentially never repeat across runs, so a shared cache file buys almost no hits, while the clean-exit persist re-reads, merges, and rewrites the whole file under a cross-process lock. That exit tail grows linearly with the file (minutes at multi-GB sizes) and serializes concurrent batch processes sharing the default cache directory. Batch replay (--tx-file / --block) now engages the on-disk cache only when --rpc.cache-dir names a directory explicitly; otherwise it behaves as --rpc.no-cache-file, so exit does zero disk-cache work regardless of what any prior run accumulated. Single-transaction replay, run/tx --fork, and capture mode keep their previous defaults; the in-memory LRU is unaffected. --- bin/mega-evme/tests/batch_cache_default.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bin/mega-evme/tests/batch_cache_default.rs b/bin/mega-evme/tests/batch_cache_default.rs index 655e8960..4b714353 100644 --- a/bin/mega-evme/tests/batch_cache_default.rs +++ b/bin/mega-evme/tests/batch_cache_default.rs @@ -85,10 +85,7 @@ fn replay(home: &FakeHome, args: &[&str]) { .expect("failed to run mega-evme"); // Every scenario here replays a transaction the mock cannot answer, so the // run itself fails; the assertions are about the cache file side effects. - assert!( - output.status.code().is_some(), - "mega-evme must exit, not die on a signal: {output:?}" - ); + assert!(output.status.code().is_some(), "mega-evme must exit, not die on a signal: {output:?}"); } /// A mock whose chain id resolves (4326) and whose every other request fails From a77ed86ebc66d9e0bf3f0f85aaf3299f1b834aad Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 11 Aug 2026 15:12:55 +0800 Subject: [PATCH 30/64] refactor(mega-evme): derive DecodedRawTx through FromTxWithEncoded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DecodedRawTx held the real envelope yet hand-mapped every variant into a TxEnv, a mapping the upstream FromTxWithEncoded impl already provides — including the deposit parts and the enveloped bytes for L1 fee calculation — and keeps extending as transaction types are added. Back the struct with the derived MegaTransaction and drop the manual mapping. The CLI-flag path (TxArgs) has no real envelope and stays as is. --- bin/mega-evme/src/common/tx.rs | 211 +++++++++++++++++++++------------ bin/mega-evme/src/tx/cmd.rs | 4 +- 2 files changed, 136 insertions(+), 79 deletions(-) diff --git a/bin/mega-evme/src/common/tx.rs b/bin/mega-evme/src/common/tx.rs index 596b62ea..f452d1d7 100644 --- a/bin/mega-evme/src/common/tx.rs +++ b/bin/mega-evme/src/common/tx.rs @@ -4,14 +4,14 @@ use alloy_primitives::{address, Address, Bytes, Signature, B256, U256}; use clap::Args; use mega_evm::{ alloy_consensus::{ - transaction::SignerRecoverable, Sealed, Signed, Transaction as _, TxEip1559, TxEip2930, - TxEip7702, TxLegacy, + transaction::SignerRecoverable, Sealed, Signed, TxEip1559, TxEip2930, TxEip7702, TxLegacy, }, alloy_eips::{ eip2930::{AccessList, AccessListItem}, eip7702::{Authorization, RecoveredAuthority, RecoveredAuthorization, SignedAuthorization}, - Decodable2718, Encodable2718, Typed2718 as _, + Decodable2718, Encodable2718, }, + alloy_evm::FromTxWithEncoded, op_alloy_consensus::{OpTxEnvelope, TxDeposit}, op_revm::transaction::deposit::DepositTransactionParts, revm::{context::tx::TxEnv, primitives::TxKind}, @@ -367,20 +367,18 @@ impl TxArgs { /// Result of decoding a raw EIP-2718 transaction. #[derive(Debug)] pub struct DecodedRawTx { - /// The decoded transaction environment. - pub tx_env: TxEnv, - /// The original raw EIP-2718 encoded bytes. - pub raw_bytes: Bytes, - /// Deposit-specific fields, if this is a deposit transaction. - /// `(source_hash, mint, is_system_transaction)` - pub deposit: Option<(B256, Option, bool)>, + /// The decoded transaction. `enveloped_tx` carries the original raw bytes (used in L1 fee + /// calculation), and the deposit fields are filled for type-126 transactions. + pub tx: MegaTransaction, } impl DecodedRawTx { - /// Decodes raw EIP-2718 encoded transaction bytes into a [`TxEnv`]. + /// Decodes raw EIP-2718 encoded transaction bytes into a [`MegaTransaction`]. /// - /// Recovers the signer from the signature (or uses the `from` field for deposits) - /// and extracts all transaction fields. No CLI overrides are applied. + /// Recovers the signer from the signature (or uses the `from` field for deposits). The + /// per-variant field mapping is derived through [`FromTxWithEncoded`], so new transaction + /// types are picked up from the upstream impl instead of a hand-written mapping here. + /// No CLI overrides are applied. pub fn from_raw(raw_bytes: impl Into) -> Result { let raw_bytes = raw_bytes.into(); let envelope = OpTxEnvelope::decode_2718(&mut &raw_bytes[..]).map_err(|e| { @@ -391,114 +389,73 @@ impl DecodedRawTx { .recover_signer() .map_err(|e| EvmeError::InvalidInput(format!("Failed to recover signer: {e}")))?; - let deposit = envelope.as_deposit().map(|d| { - let mint = if d.mint == 0 { None } else { Some(d.mint) }; - (d.source_hash, mint, d.is_system_transaction) - }); - - let decoded_chain_id = envelope.chain_id(); - let (gas_price, gas_priority_fee) = match &envelope { - OpTxEnvelope::Legacy(_) | OpTxEnvelope::Eip2930(_) => { - (envelope.gas_price().unwrap_or(0), None) - } - OpTxEnvelope::Eip1559(_) | OpTxEnvelope::Eip7702(_) => { - (envelope.max_fee_per_gas(), envelope.max_priority_fee_per_gas()) - } - OpTxEnvelope::Deposit(_) | OpTxEnvelope::PostExec(_) => (0, None), - }; - - let authorization_list = envelope - .authorization_list() - .map(|list| list.iter().map(|sa| Either::Right(sa.clone().into_recovered())).collect()) - .unwrap_or_default(); - - let tx_env = TxEnv { - caller, - gas_price, - gas_priority_fee, - blob_hashes: Vec::new(), - max_fee_per_blob_gas: 0, - tx_type: envelope.ty(), - gas_limit: envelope.gas_limit(), - data: envelope.input().clone(), - nonce: envelope.nonce(), - value: envelope.value(), - access_list: envelope.access_list().cloned().unwrap_or_default(), - authorization_list, - kind: envelope.kind(), - chain_id: decoded_chain_id, - }; - - Ok(Self { tx_env, raw_bytes, deposit }) + Ok(Self { tx: MegaTransaction::from_encoded_tx(&envelope, caller, raw_bytes) }) } - /// Applies explicitly-set [`TxArgs`] fields as overrides to the decoded [`TxEnv`]. + /// Applies explicitly-set [`TxArgs`] fields as overrides to the decoded transaction. /// /// Only fields that were explicitly provided via CLI flags are overridden; /// `None` / empty fields in `tx_args` leave the base value unchanged. pub fn override_tx_env(mut self, tx_args: &TxArgs) -> Result { + let was_deposit = self.tx.base.tx_type == MegaTxType::Deposit as u8; + if let Some(tx_type) = tx_args.tx_type { - self.tx_env.tx_type = tx_type; + self.tx.base.tx_type = tx_type; } if let Some(gas) = tx_args.gas { - self.tx_env.gas_limit = gas; + self.tx.base.gas_limit = gas; } if let Some(basefee) = tx_args.basefee { - self.tx_env.gas_price = basefee as u128; + self.tx.base.gas_price = basefee as u128; } if let Some(priority_fee) = tx_args.priority_fee { - self.tx_env.gas_priority_fee = Some(priority_fee as u128); + self.tx.base.gas_priority_fee = Some(priority_fee as u128); } if let Some(sender) = tx_args.sender { - self.tx_env.caller = sender; + self.tx.base.caller = sender; } if let Some(ref value) = tx_args.value { - self.tx_env.value = parse_ether_value(value)?; + self.tx.base.value = parse_ether_value(value)?; } if let Some(nonce) = tx_args.nonce { - self.tx_env.nonce = nonce; + self.tx.base.nonce = nonce; } if tx_args.input.is_some() || tx_args.inputfile.is_some() { - self.tx_env.data = + self.tx.base.data = load_hex(tx_args.input.clone(), tx_args.inputfile.clone())?.unwrap_or_default(); } if tx_args.create.unwrap_or(false) { - self.tx_env.kind = TxKind::Create; + self.tx.base.kind = TxKind::Create; } else if let Some(receiver) = tx_args.receiver { - self.tx_env.kind = TxKind::Call(receiver); + self.tx.base.kind = TxKind::Call(receiver); } if !tx_args.access.is_empty() { - self.tx_env.access_list = tx_args.parse_access_list()?; + self.tx.base.access_list = tx_args.parse_access_list()?; } if !tx_args.auth.is_empty() { - let chain_id = self.tx_env.chain_id.unwrap_or(0); - self.tx_env.authorization_list = tx_args + let chain_id = self.tx.base.chain_id.unwrap_or(0); + self.tx.base.authorization_list = tx_args .parse_authorization_list(chain_id)? .into_iter() .map(Either::Right) .collect(); } - if let Some((ref mut source_hash, ref mut mint, _)) = self.deposit { + // Deposit overrides apply only to a transaction decoded as a deposit — for any + // other type the deposit parts are defaults that must not be given meaning. + if was_deposit { if let Some(sh) = tx_args.source_hash { - *source_hash = sh; + self.tx.deposit.source_hash = sh; } if tx_args.mint.is_some() { - *mint = tx_args.mint; + self.tx.deposit.mint = tx_args.mint; } } Ok(self) } /// Converts the decoded raw transaction into a [`MegaTransaction`]. - /// - /// Uses the stored raw bytes for `enveloped_tx` (used in L1 fee calculation). pub fn into_tx(self) -> MegaTransaction { - let mut tx = MegaTransaction::new(self.tx_env); - tx.enveloped_tx = Some(self.raw_bytes); - if let Some((source_hash, mint, is_system_transaction)) = self.deposit { - tx.deposit = DepositTransactionParts { source_hash, mint, is_system_transaction }; - } - tx + self.tx } } @@ -608,3 +565,103 @@ fn create_fake_envelope(tx_env: &TxEnv) -> Result { MegaTxType::PostExec => Err(EvmeError::UnsupportedTxType(tx_env.tx_type)), } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::b256; + + /// The EIP-155 appendix example: a chain-1 legacy transaction with a known + /// signer, exercising signature recovery on a real signed payload. + const EIP155_RAW: &str = "0xf86c098504a817c800825208943535353535353535353535353535353535353535880de0b6b3a76400008025a028ef61340bd939bc2195fe537567866003e1a15d3c71ff63e1590620aa636276a067cbe9d8997f761aecb703304b3800ccf555c9f3dc64214b297fb1966a3b6d83"; + const EIP155_SIGNER: Address = address!("9d8A62f656a8d1615C1294fd71e9CFb3E4855A4F"); + + fn eip155_raw_bytes() -> Bytes { + load_hex(Some(EIP155_RAW.to_string()), None).expect("valid hex").expect("non-empty") + } + + /// A `TxArgs` with no flag set, the base for override tests. + fn empty_tx_args() -> TxArgs { + TxArgs { + tx_type: None, + gas: None, + basefee: None, + priority_fee: None, + sender: None, + receiver: None, + nonce: None, + create: None, + value: None, + input: None, + inputfile: None, + source_hash: None, + mint: None, + auth: Vec::new(), + access: Vec::new(), + } + } + + #[test] + fn test_from_raw_legacy_recovers_signer_and_maps_fields() { + let raw = eip155_raw_bytes(); + let decoded = DecodedRawTx::from_raw(raw.clone()).expect("decode"); + + let base = &decoded.tx.base; + assert_eq!(base.caller, EIP155_SIGNER); + assert_eq!(base.tx_type, 0); + assert_eq!(base.nonce, 9); + assert_eq!(base.gas_limit, 21_000); + assert_eq!(base.gas_price, 20_000_000_000); + assert_eq!(base.kind, TxKind::Call(address!("3535353535353535353535353535353535353535"))); + assert_eq!(base.value, U256::from(10u64).pow(U256::from(18u64))); + assert_eq!(base.chain_id, Some(1)); + assert_eq!( + decoded.tx.enveloped_tx.as_ref(), + Some(&raw), + "the original raw bytes must back the L1 fee calculation", + ); + } + + #[test] + fn test_from_raw_deposit_fills_deposit_parts() { + let deposit = TxDeposit { + source_hash: b256!("1111111111111111111111111111111111111111111111111111111111111111"), + from: address!("00000000000000000000000000000000000000aa"), + to: TxKind::Call(address!("00000000000000000000000000000000000000bb")), + mint: 5, + value: U256::from(7), + gas_limit: 100_000, + is_system_transaction: false, + input: Bytes::from_static(b"\x01\x02"), + }; + let envelope = MegaTxEnvelope::Deposit(Sealed::new_unchecked(deposit.clone(), B256::ZERO)); + let raw = Bytes::from(envelope.encoded_2718()); + + let decoded = DecodedRawTx::from_raw(raw).expect("decode"); + let tx = decoded.into_tx(); + + assert_eq!(tx.base.tx_type, MegaTxType::Deposit as u8); + assert_eq!(tx.base.caller, deposit.from); + assert_eq!(tx.base.value, deposit.value); + assert_eq!(tx.deposit.source_hash, deposit.source_hash); + assert_eq!(tx.deposit.mint, Some(5)); + assert!(!tx.deposit.is_system_transaction); + } + + #[test] + fn test_override_tx_env_applies_explicit_flags_only() { + let overrides = + TxArgs { gas: Some(300_000), value: Some("2ether".to_string()), ..empty_tx_args() }; + + let decoded = DecodedRawTx::from_raw(eip155_raw_bytes()) + .expect("decode") + .override_tx_env(&overrides) + .expect("override"); + + let base = &decoded.tx.base; + assert_eq!(base.gas_limit, 300_000, "explicit --gas must override"); + assert_eq!(base.value, U256::from(2) * U256::from(10u64).pow(U256::from(18u64))); + assert_eq!(base.caller, EIP155_SIGNER, "unset flags must keep decoded values"); + assert_eq!(base.nonce, 9, "unset flags must keep decoded values"); + } +} diff --git a/bin/mega-evme/src/tx/cmd.rs b/bin/mega-evme/src/tx/cmd.rs index d9d59494..e9321df0 100644 --- a/bin/mega-evme/src/tx/cmd.rs +++ b/bin/mega-evme/src/tx/cmd.rs @@ -66,10 +66,10 @@ impl Cmd { let tx = if let Some(ref raw) = self.raw { let raw_bytes = load_hex(Some(raw.clone()), None)?.unwrap_or_default(); let decoded = DecodedRawTx::from_raw(raw_bytes)?.override_tx_env(&self.tx_args)?; - if decoded.tx_env.chain_id != Some(chain_id) { + if decoded.tx.base.chain_id != Some(chain_id) { warn!( chain_id, - decoded_chain_id = decoded.tx_env.chain_id, + decoded_chain_id = decoded.tx.base.chain_id, "Raw transaction chain_id does not match the configured chain_id" ); } From 6a715c8c87aa0150fb3596ea835b4fb18057ca3c Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 11 Aug 2026 23:54:02 +0800 Subject: [PATCH 31/64] fix(mega-evme): reject mined batch targets without inclusion hash A --tx-file target whose eth_getTransactionByHash answer is mined but carries a null blockHash is an unanchored view: fail it as rpc instead of queuing with inclusion_hash unset. --- bin/mega-evme/src/replay/batch.rs | 23 +++++++-- bin/mega-evme/tests/replay_batch.rs | 78 +++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 448c8ec4..cb3e2766 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -514,13 +514,28 @@ where }), Ok(Some(tx)) => match tx.block_number { Some(number) => { + // A mined transaction without an inclusion hash is an + // unanchored view: the number alone cannot prove which block + // body to replay against, so the target is unanswered rather + // than queued with inclusion_hash left unset. + let Some(theirs) = tx.block_hash else { + failures.push(FailedTx { + tx_hash: *hash, + kind: BatchErrorKind::Rpc, + message: format!( + "endpoint reported a mined transaction in block {number} \ + without an inclusion hash: unanchored view" + ), + }); + continue; + }; let (targets, inclusion) = grouped.entry(number).or_default(); // Two targets resolving to the same number but different // block hashes means the endpoint served two views. Neither // can be trusted, so the disagreeing target is reported as // unanswered rather than silently replayed against one view. - match (*inclusion, tx.block_hash) { - (Some(seen), Some(theirs)) if seen != theirs => { + match *inclusion { + Some(seen) if seen != theirs => { failures.push(FailedTx { tx_hash: *hash, kind: BatchErrorKind::Rpc, @@ -532,8 +547,8 @@ where }); continue; } - (None, Some(theirs)) => *inclusion = Some(theirs), - _ => {} + None => *inclusion = Some(theirs), + Some(_) => {} } targets.push(*hash); } diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 2c1358be..5fd5d901 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -561,6 +561,84 @@ fn test_replay_tx_file_rejects_a_block_that_does_not_match_the_resolved_inclusio let _ = std::fs::remove_file(&list); } +/// A mined `--tx-file` target whose `eth_getTransactionByHash` answer carries a +/// block number but no inclusion hash is unanswered: the endpoint served an +/// unanchored view, so the target is not queued and other blocks still replay. +#[test] +fn test_replay_tx_file_rejects_mined_target_without_inclusion_hash() { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let (target, _) = BLOCK_TXS[1]; + + // Keep the block number so the response still looks mined, but drop the + // inclusion hash. Cache entries are keyed by the request, so the doctored + // answer still resolves. + let marker = format!("\"hash\":\"{target}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object"); + assert!( + result.get("blockNumber").is_some_and(|n| !n.is_null()), + "fixture transaction must report a block number" + ); + result["blockHash"] = serde_json::Value::Null; + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "exactly one response describes the target transaction"); + + let envelope_path = std::env::temp_dir() + .join(format!("mega_evme_batch_null_inclusion_{}.json", std::process::id())); + std::fs::write(&envelope_path, envelope.to_string()).expect("write doctored envelope"); + // Pair the unanchored target with one from another block so a clean job + // still runs when resolution fails for only one hash. + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_null_inclusion_{}.txt", std::process::id())); + std::fs::write(&list, format!("{target}\n{OTHER_BLOCK_TX}\n")).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&envelope_path, &["--tx-file", list.to_str().unwrap(), "--json"]); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), 2, "every target is reported once: {stdout}"); + + let failed = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(target)) + .expect("doctored target must be reported"); + assert_eq!( + failed["error"]["kind"].as_str(), + Some("rpc"), + "a mined target without an inclusion hash is unanswered: {failed}" + ); + let message = failed["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("inclusion hash") && message.contains("unanchored"), + "message names the unanchored view: {message}" + ); + + let ok = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(OTHER_BLOCK_TX)) + .expect("other-block target must be reported"); + assert!(ok.get("error").is_none(), "targets in other blocks still replay: {ok}"); + assert_eq!(ok["block_number"].as_u64(), Some(OTHER_BLOCK)); + assert_eq!(ok["success"].as_bool(), Some(true)); + + assert_eq!(code, Some(3), "an unanswered target exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); + + let _ = std::fs::remove_file(&envelope_path); + let _ = std::fs::remove_file(&list); +} + /// infrastructure failure for every target of that block (reorg / divergent views). #[test] fn test_replay_block_rejects_mismatched_parent_hash() { From e1b4a02c50d0e4d91b2a18de091dee5ae5c21dbd Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Tue, 11 Aug 2026 23:58:56 +0800 Subject: [PATCH 32/64] fix(mega-evme): validate parent linkage on the single-transaction replay path The block and its parent are fetched by number in separate calls, so a reorg or a load-balanced endpoint serving divergent views can answer them from different chains. Replaying anyway forked from a pre-state that does not precede the block, surfacing later as a false receipt mismatch or a silently wrong replay. Reject the mismatch as an RPC-consistency failure (exit 3), mirroring the batch path's guard. --- bin/mega-evme/src/replay/cmd.rs | 24 +++++++ bin/mega-evme/tests/exit_codes.rs | 102 ++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 8050f907..5923014f 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -541,6 +541,30 @@ impl Cmd { .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? .ok_or(ReplayError::BlockNotFound(block_number))?; + // Parent/block linkage guard: the two blocks above were fetched by + // number in separate calls, so across a reorg or a load-balanced + // endpoint serving divergent views `eth_getBlockByNumber(N-1)` can + // return a block that is not the parent of the block being replayed. + // Forking from that state would silently execute against the wrong + // pre-state, and the divergence would surface later as a receipt + // mismatch rather than as the infrastructure failure it is. + // + // A pending transaction has no such pair: its state base *is* the + // latest block, so both fetches address the same block and there is no + // linkage to check. + if !is_pending { + let parent_hash = parent_block.hash(); + let expected_parent = block.header.parent_hash(); + if parent_hash != expected_parent { + return Err(ReplayError::RpcError(format!( + "parent block hash {parent_hash} != block parent_hash {expected_parent}: the \ + parent block describes a different chain than the block being replayed (reorg \ + in progress, or a load-balanced endpoint serving divergent views); retry once \ + the chain settles" + ))); + } + } + let mut preceding_tx_hashes = vec![]; if !is_pending { for hash in block.transactions.hashes() { diff --git a/bin/mega-evme/tests/exit_codes.rs b/bin/mega-evme/tests/exit_codes.rs index 31bbc1df..7e612699 100644 --- a/bin/mega-evme/tests/exit_codes.rs +++ b/bin/mega-evme/tests/exit_codes.rs @@ -107,6 +107,58 @@ fn cache_without_entry(name: &str, key: &str) -> std::path::PathBuf { path } +/// Write a copy of the committed capture whose parent-block response reports a +/// hash that does not link to the replayed block, and return its path together +/// with the hash the untouched capture reported. +/// +/// The capture answers `eth_getBlockByNumber` for exactly two heights: the +/// replayed block and its parent. Rewriting the lower-numbered body's own `hash` +/// models an endpoint serving divergent views of the chain, since the two blocks +/// are fetched in separate calls. Only that field changes — state reads are +/// keyed by block number, so every other response still resolves. +fn cache_with_unlinked_parent(name: &str, wrong_hash: &str) -> (std::path::PathBuf, String) { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) + .expect("parse offline cache"); + let entries = envelope["cache"].as_array_mut().expect("cache entries"); + + let mut blocks: Vec<(usize, u64)> = vec![]; + for (index, entry) in entries.iter().enumerate() { + let value = entry["value"].as_str().expect("entry value is a string"); + let Ok(response) = serde_json::from_str::(value) else { + continue; + }; + let Some(result) = response.get("result") else { + continue; + }; + // A block body is the only response carrying a parent hash. + if !result.is_object() || result.get("parentHash").is_none() { + continue; + } + let number = result["number"].as_str().expect("block number is a string"); + let number = + u64::from_str_radix(number.trim_start_matches("0x"), 16).expect("block number is hex"); + blocks.push((index, number)); + } + assert_eq!(blocks.len(), 2, "the capture must hold the replayed block and its parent"); + blocks.sort_unstable_by_key(|&(_, number)| number); + let (parent_index, _) = blocks[0]; + + let entry = &mut entries[parent_index]; + let mut response: serde_json::Value = + serde_json::from_str(entry["value"].as_str().expect("entry value is a string")) + .expect("parse parent block response"); + let result = &mut response["result"]; + let original = result["hash"].as_str().expect("parent block hash").to_string(); + result["hash"] = serde_json::Value::String(wrong_hash.to_string()); + entry["value"] = serde_json::Value::String(response.to_string()); + + let path = + std::env::temp_dir().join(format!("mega_evme_exit_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored cache"); + (path, original) +} + /// Write a `--tx-file` holding `contents`, and return its path. fn tx_file(name: &str, contents: &str) -> std::path::PathBuf { let path = @@ -192,6 +244,56 @@ fn test_state_read_failure_during_execution_is_an_rpc_failure() { ); } +/// A parent block that does not link to the replayed block is an unanswered +/// question, not a wrong answer: the single-transaction run exits 3, with or +/// without `--verify-receipt`. +/// +/// The block and its parent are fetched by number in two separate calls, so a +/// reorg (or a load-balanced endpoint serving divergent views) can answer them +/// from different chains. Replaying anyway would fork from a pre-state that does +/// not precede the block, and the divergence would surface later as a receipt +/// mismatch (exit 2) or as a silently wrong replay. +#[test] +fn test_unlinked_parent_block_is_an_rpc_failure() { + const WRONG_PARENT: &str = "0x1111111111111111111111111111111111111111111111111111111111111111"; + + let (path, expected_parent) = cache_with_unlinked_parent("unlinked_parent", WRONG_PARENT); + let cache = path.to_str().unwrap(); + + for extra in [&[][..], &["--verify-receipt"][..]] { + let mut argv = vec!["replay", "--rpc.replay-file", cache, "--json"]; + argv.extend_from_slice(extra); + argv.push(TX_OK); + let outcome = run(&argv); + + assert_eq!( + outcome.code(), + 3, + "a broken parent linkage exits 3 for {extra:?}.\nstderr: {}", + outcome.stderr + ); + let error = outcome.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains(WRONG_PARENT) && message.contains(&expected_parent), + "the message must name both hashes (parent {expected_parent}, served \ + {WRONG_PARENT}): {error}" + ); + assert!(message.contains("divergent views"), "the message must name the cause: {error}"); + assert!( + !outcome.stdout.contains("MISMATCH") && + !outcome.stderr.contains("verification mismatch"), + "an unanswered question must not be reported as a mismatch:\n{}\n{}", + outcome.stdout, + outcome.stderr, + ); + } + + let _ = std::fs::remove_file(&path); +} + /// A batch run's error object follows the per-target lines, so a parser reading /// the stream sees every target before the run-level verdict. #[test] From 80359dc1a4a392dc8e5c8fe8772e5ebea9eaf8b4 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:00:17 +0800 Subject: [PATCH 33/64] fix(mega-evme): treat block-body null tx lookup as RPC inconsistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hash taken from the fetched block body that then resolves to null is not proof the transaction is unknown — it contradicts data the endpoint already served. Classify that as rpc-failure (exit 3) via a dedicated error variant that still names the vanished hash for the abort sweep. --- bin/mega-evme/src/common/error.rs | 8 +++++++ bin/mega-evme/src/common/exit.rs | 8 ++++++- bin/mega-evme/src/replay/batch.rs | 33 ++++++++++++++++++++++++++--- bin/mega-evme/tests/replay_batch.rs | 31 ++++++++++++++++++--------- 4 files changed, 66 insertions(+), 14 deletions(-) diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index d3ac1e6a..a4f83ecd 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -16,6 +16,14 @@ pub enum EvmeError { #[error("Transaction not found: {0}")] TransactionNotFound(TxHash), + /// The block body listed this hash, but `eth_getTransactionByHash` returned null. + /// + /// That answer contradicts data the endpoint already served (the block body), + /// so the endpoint is inconsistent — typically a reorg or load-balanced + /// divergent views — rather than a definitive "unknown transaction". + #[error("Block body lists transaction {0} but the endpoint resolves it to null")] + BlockBodyTransactionNull(TxHash), + /// Block not found #[error("Block not found: {0}")] BlockNotFound(BlockNumber), diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs index 61ca6f60..e32a138d 100644 --- a/bin/mega-evme/src/common/exit.rs +++ b/bin/mega-evme/src/common/exit.rs @@ -142,7 +142,12 @@ impl ExitCode { match err { // The endpoint never answered: unreachable, transport-level // failure, or an offline replay file without the response. - EvmeError::RpcTransportError(_) | EvmeError::RpcError(_) => Self::RpcFailure, + // `BlockBodyTransactionNull` is the same class: the block body + // already listed the hash, so a null lookup is an inconsistent + // endpoint rather than a definitive unknown transaction. + EvmeError::RpcTransportError(_) | + EvmeError::RpcError(_) | + EvmeError::BlockBodyTransactionNull(_) => Self::RpcFailure, // A block error the EVM raised because a state read failed is that // read's failure, not an execution result: classify it by its // cause, so an endpoint that died mid-execution still reports the @@ -291,6 +296,7 @@ mod tests { EvmeError::RpcTransportError( alloy_provider::transport::TransportErrorKind::custom_str("connection refused"), ), + EvmeError::BlockBodyTransactionNull(B256::ZERO), ] { assert_eq!( ExitCode::from_evme_error(&err), diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index cb3e2766..634ef420 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -766,11 +766,16 @@ where // preceding transactions). block_executor.clear_accessed_block_hashes(); + // Every hash here came from the block body this endpoint already + // served. `Ok(None)` therefore means the endpoint is inconsistent + // (reorg or load-balanced divergent views), not that the hash is + // unknown — that definitive answer only applies to a user-supplied + // target lookup on the single-transaction path. let tx = provider .get_transaction_by_hash(*tx_hash) .await .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? - .ok_or(ReplayError::TransactionNotFound(*tx_hash))?; + .ok_or(ReplayError::BlockBodyTransactionNull(*tx_hash))?; let is_target = target_set.contains(tx_hash); let start = Instant::now(); @@ -1216,7 +1221,9 @@ where fn classify(err: &ReplayError) -> BatchErrorKind { match err { ReplayError::TransactionNotFound(_) => BatchErrorKind::NotFound, - ReplayError::RpcError(_) | ReplayError::RpcTransportError(_) => BatchErrorKind::Rpc, + ReplayError::RpcError(_) | + ReplayError::RpcTransportError(_) | + ReplayError::BlockBodyTransactionNull(_) => BatchErrorKind::Rpc, // A block error the EVM raised because a state read failed is that // read's failure: the same classification the run-level exit code uses. ReplayError::BlockExecutionError(_) @@ -1248,7 +1255,9 @@ fn swept_kind(_err: &ReplayError) -> BatchErrorKind { /// The transaction an aborting error is about, when it names one. fn aborting_tx_hash(err: &ReplayError) -> Option { match err { - ReplayError::TransactionNotFound(hash) => Some(*hash), + ReplayError::TransactionNotFound(hash) | ReplayError::BlockBodyTransactionNull(hash) => { + Some(*hash) + } ReplayError::BlockExecutionError(err) => block_error_tx_hash(err), _ => None, } @@ -1487,6 +1496,11 @@ mod tests { assert_eq!(swept_kind(&ReplayError::TransactionNotFound(B256::ZERO)), BatchErrorKind::Rpc); // Transport/RPC failure. assert_eq!(swept_kind(&ReplayError::RpcError("endpoint down".into())), BatchErrorKind::Rpc); + // Block-body null is rpc-class for the aborting target and for sweeps. + assert_eq!( + swept_kind(&ReplayError::BlockBodyTransactionNull(B256::ZERO)), + BatchErrorKind::Rpc + ); // Execution-class aborts (other, setup, internal) must not blame swept targets. assert_eq!( swept_kind(&ReplayError::Other("executor setup failed".into())), @@ -1503,6 +1517,19 @@ mod tests { ); } + /// A block-body hash resolving to null is an RPC inconsistency that still + /// names the vanished transaction for the abort sweep. + #[test] + fn test_block_body_transaction_null_classifies_as_rpc_and_names_the_hash() { + let hash = B256::repeat_byte(0xab); + let err = ReplayError::BlockBodyTransactionNull(hash); + assert_eq!(classify(&err), BatchErrorKind::Rpc); + assert_eq!(aborting_tx_hash(&err), Some(hash)); + // Contrasts with the user-supplied unknown-hash definitive answer. + assert_eq!(classify(&ReplayError::TransactionNotFound(hash)), BatchErrorKind::NotFound); + assert_eq!(aborting_tx_hash(&ReplayError::TransactionNotFound(hash)), Some(hash)); + } + /// A run whose only finding is divergence fails as the mismatch it is. #[test] fn test_batch_tally_mismatch_only_reports_the_verification_error() { diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 5fd5d901..ce159af0 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -340,10 +340,12 @@ fn test_replay_block_verify_receipt_reports_a_verdict_per_target() { assert_eq!(code, Some(0), "a fully matching run exits 0"); } -/// An abort caused by one transaction of the block is not an answer about the -/// targets behind it: only the transaction the endpoint denied is reported as -/// `not_found`, and every target swept up behind it is reported as unanswered -/// (`rpc`) with a message naming the transaction that aborted the block. +/// An abort caused by a block-body transaction resolving to null is not a +/// definitive "unknown hash": the hash came from the block the endpoint already +/// served, so the null is an RPC inconsistency. The aborting target is reported +/// as `rpc` (not `not_found`), and every target swept up behind it is also +/// unanswered (`rpc`) with a message naming the transaction that aborted the +/// block. #[test] fn test_replay_block_sweeps_targets_behind_an_abort_as_unanswered() { let (missing, missing_index) = BLOCK_TXS[1]; @@ -364,8 +366,16 @@ fn test_replay_block_sweeps_targets_behind_an_abort_as_unanswered() { if index == missing_index { assert_eq!( line["error"]["kind"].as_str(), - Some("not_found"), - "only the denied transaction is unknown: {line}" + Some("rpc"), + "a block-body hash resolving to null is an RPC inconsistency: {line}" + ); + assert!( + line["error"]["message"].as_str().is_some_and(|m| { + m.contains(missing) && + m.contains("Block body") && + m.contains("resolves it to null") + }), + "the aborting error must name the hash and the inconsistency: {line}" ); continue; } @@ -380,10 +390,11 @@ fn test_replay_block_sweeps_targets_behind_an_abort_as_unanswered() { ); } - // The denied transaction is an execution-class failure, which outranks the - // unanswered ones. - assert_eq!(code, Some(1), "a definitive negative answer exits 1"); - assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("execution-error")); + // Every failure is RPC-class (inconsistency + unanswered sweeps), so the + // run exits 3 rather than the definitive-answer class of a user-supplied + // unknown hash. + assert_eq!(code, Some(3), "a block-body null lookup exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); } /// An executor/setup abort on a mid-block transaction is still an execution From c3342f33f0d0f559f57a7aec209b901a672e6f23 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:04:11 +0800 Subject: [PATCH 34/64] docs(mega-evme): remove stray doc-comment fragment in batch tests --- bin/mega-evme/tests/replay_batch.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index ce159af0..fc51874f 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -512,7 +512,6 @@ fn test_replay_batch_rejects_single_transaction_flags() { } } -/// A parent block whose hash does not match the child block's `parentHash` is an /// A `--tx-file` target whose reported inclusion block is not the block fetched /// by that number is unanswered, not replayed. /// From 10f371464fba15210c19bd2dafa590e31d0ead08 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:08:55 +0800 Subject: [PATCH 35/64] fix(mega-evme): make panic hook writes non-panicking on closed pipes Rust ignores SIGPIPE, so a closed stdout turns the next NDJSON println! into a Broken pipe panic. The panic hook then used println!/eprintln! on the same closed streams, double-panicked, and aborted (SIGABRT) instead of exit(1). Write via fallible writeln! and discard errors so the hook always reaches the documented execution-error exit class. --- bin/mega-evme/src/common/exit.rs | 13 ++++++- bin/mega-evme/src/lib.rs | 15 +++++++-- bin/mega-evme/tests/exit_codes.rs | 56 +++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs index e32a138d..f384ff18 100644 --- a/bin/mega-evme/src/common/exit.rs +++ b/bin/mega-evme/src/common/exit.rs @@ -251,10 +251,21 @@ pub fn report_command_result(result: Result<(), Error>, json: bool) -> ExitCode /// Also used by failures that never become a command result — an argument /// parsing error is reported by `clap` itself, but a machine-readable run must /// still end its stdout with the object the taxonomy promises. +/// +/// Writes are fallible: a closed stdout is ignored rather than panicking. +/// The panic hook relies on this so a broken-pipe panic during normal output +/// still reaches `exit(1)` instead of aborting inside the hook. With an open +/// stdout the bytes match a successful `println!` of the same envelope. pub fn print_json_error(code: ExitCode, message: &str) { + use std::io::Write; + let envelope = ErrorEnvelope { error: ErrorBody { code: code.code(), kind: code.kind(), message } }; - println!("{}", serde_json::to_string(&envelope).expect("failed to serialize the error")); + // Serialization of this envelope cannot fail for ordinary messages; still + // avoid `.expect` so a hook-path write never panics on its way to exit(1). + if let Ok(json) = serde_json::to_string(&envelope) { + let _ = writeln!(std::io::stdout(), "{json}"); + } } #[cfg(test)] diff --git a/bin/mega-evme/src/lib.rs b/bin/mega-evme/src/lib.rs index 8eb99601..2c4e5687 100644 --- a/bin/mega-evme/src/lib.rs +++ b/bin/mega-evme/src/lib.rs @@ -32,21 +32,32 @@ pub use common::*; /// When the raw process argv contains `--json`, the hook also prints the /// standard structured error object on stdout before exiting so a machine- /// readable run never ends with empty stdout on panic. +/// +/// Every write the hook performs is fallible: a closed stdout or stderr must +/// not re-panic inside the hook, or the runtime aborts (SIGABRT) before the +/// documented `exit(1)`. Consumers that close the pipe early +/// (`… --json | head`) therefore still see exit class 1 rather than an +/// undefined signal death. pub fn set_thread_panic_hook() { use std::{ backtrace::Backtrace, + io::{self, Write}, panic::{set_hook, take_hook}, process::exit, }; let orig_hook = take_hook(); set_hook(Box::new(move |panic_info| { // Raw stderr rather than `tracing`: the subscriber may not be - // installed yet when a panic fires during CLI startup. - eprintln!("Custom backtrace: {}", Backtrace::capture()); + // installed yet when a panic fires during CLI startup. Discard write + // errors so a closed stderr cannot abort the process from here. + let _ = writeln!(io::stderr(), "Custom backtrace: {}", Backtrace::capture()); orig_hook(panic_info); if raw_argv_wants_json() { // Keep the panic text on stderr (via `orig_hook`); the structured // object is the machine-readable final stdout line. + // `print_json_error` itself is non-panicking on a closed stdout — + // the broken pipe that often triggered this panic must not cause a + // second panic before `exit(1)`. let message = format!("panic: {panic_info}"); print_json_error(ExitCode::ExecutionError, &message); } diff --git a/bin/mega-evme/tests/exit_codes.rs b/bin/mega-evme/tests/exit_codes.rs index 31bbc1df..c0bab4eb 100644 --- a/bin/mega-evme/tests/exit_codes.rs +++ b/bin/mega-evme/tests/exit_codes.rs @@ -373,3 +373,59 @@ fn test_help_in_json_mode_prints_no_error_object() { help.stdout ); } + +/// Closing stdout mid-batch must not abort the process. +/// +/// Rust ignores SIGPIPE, so the next NDJSON `println!` panics with a broken +/// pipe. The panic hook still has to reach `exit(1)` even when it cannot write +/// the structured error object to the same closed stdout — otherwise the +/// runtime aborts (SIGABRT, shell status 134) and scripts that branch on the +/// documented 0/1/2/3 exit classes see an undefined status. +#[test] +fn test_closed_stdout_during_json_batch_exits_one() { + use std::{ + io::{BufRead, BufReader}, + process::{Command, Stdio}, + }; + + // Multi-target offline batch: many NDJSON lines, so dropping the pipe after + // the first line still leaves further writes that hit the broken pipe. + let envelope = common::fixture("replay_batch_blocks.cache.json"); + let mut child = Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args([ + "replay", + "--rpc.replay-file", + envelope.to_str().expect("fixture path is utf-8"), + "--block", + "22945844", + "--json", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to spawn mega-evme"); + + let stdout = child.stdout.take().expect("child stdout was piped"); + let mut first_line = String::new(); + BufReader::new(stdout) + .read_line(&mut first_line) + .expect("failed to read the first NDJSON line"); + assert!( + !first_line.trim().is_empty(), + "batch --json must print at least one NDJSON line before further writes" + ); + // Dropping the BufReader closes the read end. The child's next stdout write + // then fails with EPIPE and panics into the process-wide hook. + // (Binding ends here; no further use of the pipe.) + + let status = child.wait().expect("failed to wait for mega-evme"); + assert_eq!( + status.code(), + Some(1), + "closed stdout must exit 1 (execution-error), not signal death.\nstatus: {status:?}" + ); + assert!( + status.code().is_some(), + "process must not be signal-killed (e.g. SIGABRT from a double panic in the hook)" + ); +} From a6f8884ef0c319163b0cfb877256bb04d49bb747 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:12:47 +0800 Subject: [PATCH 36/64] test(mega-evme): pin build_pre_state absent-means-nonexistent shape Rewrite the stale else-branch comment: forked backends normalize all-zero RPC answers to None, so nonexistent touched accounts are omitted from pre. Add a unit test covering touched+None and touched+Some outcomes. --- bin/mega-evme/src/replay/fixture.rs | 93 +++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/bin/mega-evme/src/replay/fixture.rs b/bin/mega-evme/src/replay/fixture.rs index 831e80f0..89cc3af6 100644 --- a/bin/mega-evme/src/replay/fixture.rs +++ b/bin/mega-evme/src/replay/fixture.rs @@ -423,12 +423,13 @@ where .basic_ref(*address) .map_err(|e| ReplayError::Other(format!("pre-state read for {address}: {e}")))? else { - // The database reports no account. RPC-backed databases (AlloyDB) - // always materialize an account (possibly all-empty), so on a forked - // replay this branch never fires and accounts created by the target - // transaction enter `pre` as explicit empty accounts — equivalent - // under EIP-161 state clearing. A database that can signal - // nonexistence omits the account here. + // The database reports no account. On a forked replay the RPC + // backend normalizes an all-zero (balance, nonce, code) answer to + // `None` (see `normalize_rpc_account` in `common/state.rs`), so + // this branch fires for every pre-transaction nonexistent account + // — including accounts the target transaction itself creates. + // Omitting them is correct state-test semantics: absence in `pre` + // means the account did not exist. continue; }; @@ -612,6 +613,86 @@ mod tests { crate::common::EvmeError::InvalidInput("database unavailable".to_string()) } + /// A database whose `basic_ref` answers are supplied per address. + /// + /// Used to pin the `build_pre_state` shape for both existence outcomes: + /// a touched address that returns `None` is omitted from `pre`, and a + /// touched address that returns `Some` is recorded with its fields. + struct MapDb { + accounts: std::collections::HashMap>, + } + + impl DatabaseRef for MapDb { + type Error = crate::common::EvmeError; + + fn basic_ref( + &self, + address: Address, + ) -> std::result::Result, Self::Error> { + Ok(self.accounts.get(&address).cloned().unwrap_or(None)) + } + + fn code_by_hash_ref(&self, _: B256) -> std::result::Result { + Err(unavailable()) + } + + fn storage_ref( + &self, + _: Address, + _: StorageKey, + ) -> std::result::Result { + Err(unavailable()) + } + + fn block_hash_ref(&self, _: u64) -> std::result::Result { + Err(unavailable()) + } + } + + /// Touched addresses with no pre-transaction account are omitted from `pre`; + /// touched addresses that exist are recorded with their fields. + /// + /// Absence-means-nonexistence is the state-test fixture shape: a forked + /// backend returns `None` for all-zero RPC answers, so accounts created by + /// the target transaction must not appear as explicit empty entries. + #[test] + fn test_build_pre_state_omits_nonexistent_and_records_existing() { + let missing = Address::repeat_byte(0xaa); + let present = Address::repeat_byte(0xbb); + let balance = U256::from(42u64); + let nonce = 7u64; + + let mut accounts = std::collections::HashMap::new(); + accounts.insert(missing, None); + accounts.insert( + present, + Some(RevmAccountInfo { + balance, + nonce, + code_hash: KECCAK256_EMPTY, + code: Some(Bytecode::default()), + ..Default::default() + }), + ); + let db = MapDb { accounts }; + + let mut evm_state = EvmState::default(); + evm_state.insert(missing, Default::default()); + evm_state.insert(present, Default::default()); + + let pre = build_pre_state(&db, &evm_state).expect("pre-state construction succeeds"); + + assert!( + !pre.contains_key(&missing), + "touched + basic_ref=None must be absent from pre (nonexistence)" + ); + let recorded = pre.get(&present).expect("touched + basic_ref=Some must appear in pre"); + assert_eq!(recorded.balance, balance); + assert_eq!(recorded.nonce, nonce); + assert!(recorded.code.is_empty()); + assert!(recorded.storage.is_empty()); + } + fn deposit_transaction() -> Transaction { let envelope = OpTxEnvelope::Deposit(Sealed::new(TxDeposit::default())); let inner = alloy_rpc_types_eth::Transaction { From fcb0beaf6b424344d63806d50afcb90163f40148 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:18:57 +0800 Subject: [PATCH 37/64] test(mega-evme): cover typed raw-tx envelope decoding Add EIP-2930, EIP-1559, and EIP-7702 DecodedRawTx::from_raw tests that sign real envelopes with a fixed Hardhat key so signer recovery and FromTxWithEncoded field mapping (access list, fees, authorizations) are regression-guarded. --- bin/mega-evme/src/common/tx.rs | 166 +++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/bin/mega-evme/src/common/tx.rs b/bin/mega-evme/src/common/tx.rs index f452d1d7..d8b64704 100644 --- a/bin/mega-evme/src/common/tx.rs +++ b/bin/mega-evme/src/common/tx.rs @@ -570,16 +570,55 @@ fn create_fake_envelope(tx_env: &TxEnv) -> Result { mod tests { use super::*; use alloy_primitives::b256; + use mega_evm::alloy_consensus::{crypto::secp256k1, SignableTransaction}; /// The EIP-155 appendix example: a chain-1 legacy transaction with a known /// signer, exercising signature recovery on a real signed payload. const EIP155_RAW: &str = "0xf86c098504a817c800825208943535353535353535353535353535353535353535880de0b6b3a76400008025a028ef61340bd939bc2195fe537567866003e1a15d3c71ff63e1590620aa636276a067cbe9d8997f761aecb703304b3800ccf555c9f3dc64214b297fb1966a3b6d83"; const EIP155_SIGNER: Address = address!("9d8A62f656a8d1615C1294fd71e9CFb3E4855A4F"); + /// Hardhat account #0 private key; recovered address is [`DEFAULT_SENDER`]. + const TEST_SECRET: B256 = + b256!("ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"); + + /// Fixed chain and receiver used by the typed-envelope signing vectors. + const TYPED_CHAIN_ID: u64 = 1; + const TYPED_TO: Address = address!("3535353535353535353535353535353535353535"); + const ACCESS_ADDR: Address = address!("1111111111111111111111111111111111111111"); + const ACCESS_KEY: B256 = + b256!("2222222222222222222222222222222222222222222222222222222222222222"); + const AUTH_DELEGATION: Address = address!("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + fn eip155_raw_bytes() -> Bytes { load_hex(Some(EIP155_RAW.to_string()), None).expect("valid hex").expect("non-empty") } + /// Signs a signable transaction body with [`TEST_SECRET`] and returns the + /// EIP-2718 envelope bytes for the given `MegaTxEnvelope` constructor. + fn sign_and_encode_envelope( + envelope: impl FnOnce(Signature) -> MegaTxEnvelope, + signature_hash: B256, + ) -> Bytes { + let sig = secp256k1::sign_message(TEST_SECRET, signature_hash).expect("sign must succeed"); + Bytes::from(envelope(sig).encoded_2718()) + } + + fn sample_access_list() -> AccessList { + AccessList(vec![AccessListItem { address: ACCESS_ADDR, storage_keys: vec![ACCESS_KEY] }]) + } + + /// Builds a genuinely signed EIP-7702 authorization for the fixed fields. + fn sample_signed_authorization() -> SignedAuthorization { + let auth = Authorization { + chain_id: U256::from(TYPED_CHAIN_ID), + address: AUTH_DELEGATION, + nonce: 3, + }; + let sig = secp256k1::sign_message(TEST_SECRET, auth.signature_hash()) + .expect("auth sign must succeed"); + auth.into_signed(sig) + } + /// A `TxArgs` with no flag set, the base for override tests. fn empty_tx_args() -> TxArgs { TxArgs { @@ -622,6 +661,133 @@ mod tests { ); } + #[test] + fn test_from_raw_eip2930_recovers_signer_and_preserves_access_list() { + let access_list = sample_access_list(); + let tx = TxEip2930 { + chain_id: TYPED_CHAIN_ID, + nonce: 4, + gas_price: 30_000_000_000, + gas_limit: 50_000, + to: TxKind::Call(TYPED_TO), + value: U256::from(1), + access_list: access_list.clone(), + input: Bytes::from_static(b"\xca\xfe"), + }; + let signature_hash = tx.signature_hash(); + let raw = sign_and_encode_envelope( + |sig| MegaTxEnvelope::Eip2930(tx.into_signed(sig)), + signature_hash, + ); + + let decoded = DecodedRawTx::from_raw(raw.clone()).expect("decode"); + let base = &decoded.tx.base; + + assert_eq!(base.caller, DEFAULT_SENDER, "signer recovery must match the test key"); + assert_eq!(base.tx_type, MegaTxType::Eip2930 as u8); + assert_eq!(base.nonce, 4); + assert_eq!(base.gas_limit, 50_000); + assert_eq!(base.chain_id, Some(TYPED_CHAIN_ID)); + assert_eq!(base.access_list, access_list, "access list addresses and keys must survive"); + assert_eq!( + decoded.tx.enveloped_tx.as_ref(), + Some(&raw), + "the original raw bytes must back the L1 fee calculation", + ); + } + + #[test] + fn test_from_raw_eip1559_recovers_signer_and_maps_fee_fields() { + let max_fee_per_gas = 40_000_000_000u128; + let max_priority_fee_per_gas = 2_000_000_000u128; + let tx = TxEip1559 { + chain_id: TYPED_CHAIN_ID, + nonce: 7, + gas_limit: 80_000, + max_fee_per_gas, + max_priority_fee_per_gas, + to: TxKind::Call(TYPED_TO), + value: U256::from(2), + access_list: AccessList::default(), + input: Bytes::new(), + }; + let signature_hash = tx.signature_hash(); + let raw = sign_and_encode_envelope( + |sig| MegaTxEnvelope::Eip1559(tx.into_signed(sig)), + signature_hash, + ); + + let decoded = DecodedRawTx::from_raw(raw.clone()).expect("decode"); + let base = &decoded.tx.base; + + assert_eq!(base.caller, DEFAULT_SENDER, "signer recovery must match the test key"); + assert_eq!(base.tx_type, MegaTxType::Eip1559 as u8); + assert_eq!(base.nonce, 7); + assert_eq!(base.gas_limit, 80_000); + assert_eq!(base.chain_id, Some(TYPED_CHAIN_ID)); + assert_eq!(base.gas_price, max_fee_per_gas, "gas_price must map from max_fee_per_gas"); + assert_eq!( + base.gas_priority_fee, + Some(max_priority_fee_per_gas), + "gas_priority_fee must map from max_priority_fee_per_gas", + ); + assert_eq!( + decoded.tx.enveloped_tx.as_ref(), + Some(&raw), + "the original raw bytes must back the L1 fee calculation", + ); + } + + #[test] + fn test_from_raw_eip7702_recovers_signer_and_preserves_authorization_list() { + let signed_auth = sample_signed_authorization(); + let expected_inner = signed_auth.inner().clone(); + let tx = TxEip7702 { + chain_id: TYPED_CHAIN_ID, + nonce: 11, + gas_limit: 120_000, + max_fee_per_gas: 50_000_000_000, + max_priority_fee_per_gas: 1_000_000_000, + to: TYPED_TO, + value: U256::ZERO, + access_list: AccessList::default(), + authorization_list: vec![signed_auth], + input: Bytes::new(), + }; + let signature_hash = tx.signature_hash(); + let raw = sign_and_encode_envelope( + |sig| MegaTxEnvelope::Eip7702(tx.into_signed(sig)), + signature_hash, + ); + + let decoded = DecodedRawTx::from_raw(raw.clone()).expect("decode"); + let base = &decoded.tx.base; + + assert_eq!(base.caller, DEFAULT_SENDER, "signer recovery must match the test key"); + assert_eq!(base.tx_type, MegaTxType::Eip7702 as u8); + assert_eq!(base.nonce, 11); + assert_eq!(base.gas_limit, 120_000); + assert_eq!(base.chain_id, Some(TYPED_CHAIN_ID)); + assert_eq!(base.authorization_list.len(), 1, "authorization list length must survive"); + match &base.authorization_list[0] { + Either::Right(recovered) => { + assert_eq!(*recovered.chain_id(), expected_inner.chain_id); + assert_eq!(*recovered.address(), expected_inner.address); + assert_eq!(recovered.nonce(), expected_inner.nonce); + } + Either::Left(signed) => { + assert_eq!(signed.inner().chain_id, expected_inner.chain_id); + assert_eq!(signed.inner().address, expected_inner.address); + assert_eq!(signed.inner().nonce, expected_inner.nonce); + } + } + assert_eq!( + decoded.tx.enveloped_tx.as_ref(), + Some(&raw), + "the original raw bytes must back the L1 fee calculation", + ); + } + #[test] fn test_from_raw_deposit_fills_deposit_parts() { let deposit = TxDeposit { From f5ebec6dc039243961b984b40f51361c98f1747d Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:23:20 +0800 Subject: [PATCH 38/64] fix(mega-evme): classify contradictory null number with hash as RPC A batch resolve_targets answer with block_number=None and block_hash=Some is self-contradictory metadata, not pending. Fail it as rpc, keep the genuine pending arm for (None, None), and make the (number, hash) match total. --- bin/mega-evme/src/replay/batch.rs | 46 ++++---- bin/mega-evme/tests/replay_batch.rs | 156 ++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 18 deletions(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 634ef420..75e80760 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -512,23 +512,11 @@ where kind: BatchErrorKind::NotFound, message: "Transaction not found".to_string(), }), - Ok(Some(tx)) => match tx.block_number { - Some(number) => { - // A mined transaction without an inclusion hash is an - // unanchored view: the number alone cannot prove which block - // body to replay against, so the target is unanswered rather - // than queued with inclusion_hash left unset. - let Some(theirs) = tx.block_hash else { - failures.push(FailedTx { - tx_hash: *hash, - kind: BatchErrorKind::Rpc, - message: format!( - "endpoint reported a mined transaction in block {number} \ - without an inclusion hash: unanchored view" - ), - }); - continue; - }; + // Every (block_number, block_hash) shape the endpoint can return is + // handled explicitly so a contradictory row cannot fall through a + // wildcard into the pending arm. + Ok(Some(tx)) => match (tx.block_number, tx.block_hash) { + (Some(number), Some(theirs)) => { let (targets, inclusion) = grouped.entry(number).or_default(); // Two targets resolving to the same number but different // block hashes means the endpoint served two views. Neither @@ -552,7 +540,29 @@ where } targets.push(*hash); } - None => failures.push(FailedTx { + // A mined transaction without an inclusion hash is an unanchored + // view: the number alone cannot prove which block body to replay + // against, so the target is unanswered rather than queued with + // inclusion_hash left unset. + (Some(number), None) => failures.push(FailedTx { + tx_hash: *hash, + kind: BatchErrorKind::Rpc, + message: format!( + "endpoint reported a mined transaction in block {number} \ + without an inclusion hash: unanchored view" + ), + }), + // A hash proves inclusion; a null number denies it. That pair is + // self-contradictory metadata, not a pending transaction. + (None, Some(hash_value)) => failures.push(FailedTx { + tx_hash: *hash, + kind: BatchErrorKind::Rpc, + message: format!( + "endpoint reported inclusion hash {hash_value} without a block \ + number: contradictory metadata" + ), + }), + (None, None) => failures.push(FailedTx { tx_hash: *hash, kind: BatchErrorKind::Pending, message: "Transaction is pending (no block number)".to_string(), diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index fc51874f..892ddc4d 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -649,6 +649,162 @@ fn test_replay_tx_file_rejects_mined_target_without_inclusion_hash() { let _ = std::fs::remove_file(&list); } +/// A `--tx-file` target whose `eth_getTransactionByHash` answer carries an +/// inclusion hash but no block number is unanswered: the endpoint served +/// contradictory metadata, so the target is not treated as pending. +#[test] +fn test_replay_tx_file_rejects_inclusion_hash_without_block_number() { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let (target, _) = BLOCK_TXS[1]; + + // Keep the inclusion hash so the response still claims a mined block, but + // drop the block number. Cache entries are keyed by the request, so the + // doctored answer still resolves. + let marker = format!("\"hash\":\"{target}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object"); + assert!( + result.get("blockHash").is_some_and(|h| !h.is_null()), + "fixture transaction must report an inclusion hash" + ); + assert!( + result.get("blockNumber").is_some_and(|n| !n.is_null()), + "fixture transaction must report a block number before doctoring" + ); + result["blockNumber"] = serde_json::Value::Null; + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "exactly one response describes the target transaction"); + + let envelope_path = std::env::temp_dir() + .join(format!("mega_evme_batch_null_number_with_hash_{}.json", std::process::id())); + std::fs::write(&envelope_path, envelope.to_string()).expect("write doctored envelope"); + // Pair the contradictory target with one from another block so a clean job + // still runs when resolution fails for only one hash. + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_null_number_with_hash_{}.txt", std::process::id())); + std::fs::write(&list, format!("{target}\n{OTHER_BLOCK_TX}\n")).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&envelope_path, &["--tx-file", list.to_str().unwrap(), "--json"]); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), 2, "every target is reported once: {stdout}"); + + let failed = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(target)) + .expect("doctored target must be reported"); + assert_eq!( + failed["error"]["kind"].as_str(), + Some("rpc"), + "contradictory metadata is unanswered as rpc, not pending: {failed}" + ); + let message = failed["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("contradictory") && + (message.contains("without a block number") || message.contains("block number")), + "message names the contradiction: {message}" + ); + assert!( + !message.contains("pending"), + "contradictory metadata must not be classified as pending: {message}" + ); + + let ok = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(OTHER_BLOCK_TX)) + .expect("other-block target must be reported"); + assert!(ok.get("error").is_none(), "targets in other blocks still replay: {ok}"); + assert_eq!(ok["block_number"].as_u64(), Some(OTHER_BLOCK)); + assert_eq!(ok["success"].as_bool(), Some(true)); + + assert_eq!(code, Some(3), "an unanswered target exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); + + let _ = std::fs::remove_file(&envelope_path); + let _ = std::fs::remove_file(&list); +} + +/// A genuinely pending `--tx-file` target (`blockNumber` and `blockHash` both +/// null) keeps the pending classification and is not queued for replay. +#[test] +fn test_replay_tx_file_classifies_null_number_and_hash_as_pending() { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let (target, _) = BLOCK_TXS[1]; + + let marker = format!("\"hash\":\"{target}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object"); + result["blockNumber"] = serde_json::Value::Null; + result["blockHash"] = serde_json::Value::Null; + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "exactly one response describes the target transaction"); + + let envelope_path = std::env::temp_dir() + .join(format!("mega_evme_batch_pending_nulls_{}.json", std::process::id())); + std::fs::write(&envelope_path, envelope.to_string()).expect("write doctored envelope"); + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_pending_nulls_{}.txt", std::process::id())); + std::fs::write(&list, format!("{target}\n{OTHER_BLOCK_TX}\n")).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&envelope_path, &["--tx-file", list.to_str().unwrap(), "--json"]); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), 2, "every target is reported once: {stdout}"); + + let failed = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(target)) + .expect("doctored target must be reported"); + assert_eq!( + failed["error"]["kind"].as_str(), + Some("pending"), + "null number and null hash is pending: {failed}" + ); + let message = failed["error"]["message"].as_str().unwrap_or_default(); + assert_eq!( + message, "Transaction is pending (no block number)", + "pending message is unchanged: {message}" + ); + + let ok = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(OTHER_BLOCK_TX)) + .expect("other-block target must be reported"); + assert!(ok.get("error").is_none(), "targets in other blocks still replay: {ok}"); + assert_eq!(ok["block_number"].as_u64(), Some(OTHER_BLOCK)); + assert_eq!(ok["success"].as_bool(), Some(true)); + + // Pending counts as an execution-class failure (exit 1), not rpc (exit 3). + assert_eq!(code, Some(1), "a pending target exits 1"); + + let _ = std::fs::remove_file(&envelope_path); + let _ = std::fs::remove_file(&list); +} + /// infrastructure failure for every target of that block (reorg / divergent views). #[test] fn test_replay_block_rejects_mismatched_parent_hash() { From 000911bd030904bd6c142dacd8aaa179f517ee0c Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:28:27 +0800 Subject: [PATCH 39/64] feat(mega-evme): make replay --override.spec a coherent what-if MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The override only patched the EVM semantics: the executor still received the chain's real schedule, so the pre-block predeploys, the EIP-2935 / EIP-4788 gating and the block-level limits followed the block's historical position while the opcodes followed the forced spec — a world no chain ever ran. Forcing Rex5 on a pre-Rex5 mainnet block even failed hard, because the registry was resolved under Rex5 but never deployed. Hand the executor a schedule synthesized from the forced spec instead, so setup, block-level limits and semantics all derive from it. The synthesized schedule takes activation from the spec but delegates per-fork parameters to the chain configuration: those are chain data, not spec data, and without them every Rex5+ override would fail closed in the registry deploy. The per-transaction limit patch is dropped, subsumed by resolving the fork from the schedule. Without an override nothing changes — the chain's own config is passed through, including the "no fork active" failure for a block older than the chain's first hardfork. --- bin/mega-evme/src/common/hardfork.rs | 150 ++++++- bin/mega-evme/src/replay/cmd.rs | 51 ++- bin/mega-evme/src/replay/hardforks.rs | 248 ++++++++++- bin/mega-evme/tests/common/mod.rs | 44 ++ bin/mega-evme/tests/replay_override_spec.rs | 403 ++++++++++++++++++ docs/mega-evme/commands/replay.md | 11 + .../mega-evme/configuration/chain-and-spec.md | 6 + 7 files changed, 886 insertions(+), 27 deletions(-) create mode 100644 bin/mega-evme/tests/replay_override_spec.rs diff --git a/bin/mega-evme/src/common/hardfork.rs b/bin/mega-evme/src/common/hardfork.rs index 3ffb9488..73a75dc2 100644 --- a/bin/mega-evme/src/common/hardfork.rs +++ b/bin/mega-evme/src/common/hardfork.rs @@ -1,23 +1,44 @@ +use core::any::Any; + use mega_evm::{ alloy_hardforks::{EthereumHardfork, ForkCondition}, alloy_op_hardforks::{EthereumHardforks, OpHardfork, OpHardforks}, - MegaHardfork, MegaHardforks, MegaSpecId, + MegaHardfork, MegaHardforkConfig, MegaHardforks, MegaSpecId, }; /// Fixed hardfork configuration for replay +/// +/// Activation follows the fixed spec alone: every hardfork whose spec is included in `spec` is +/// active at timestamp 0, and every later hardfork never activates. This is how mega-evme +/// expresses "a chain running spec N" without a real activation schedule. +/// +/// Per-fork parameters are *not* synthesized. A fixed-spec world still needs the chain's +/// parameters (the Rex5+ `SequencerRegistry` seeds are chain-specific data, not spec data), so an +/// optional parameter source can be attached with [`with_params_from`](Self::with_params_from). +/// Without one, parameter lookups return `None` as before. #[derive(Debug, Clone, Copy)] -pub struct FixedHardfork { +pub struct FixedHardfork<'a> { spec: MegaSpecId, + params: Option<&'a MegaHardforkConfig>, } -impl FixedHardfork { +impl<'a> FixedHardfork<'a> { /// Create a new [`FixedHardfork`] with the given `spec` pub fn new(spec: MegaSpecId) -> Self { - Self { spec } + Self { spec, params: None } + } + + /// Delegates per-fork parameter lookups to `config` while activation stays fixed. + /// + /// The delegation is wholesale rather than per parameter type: a parameter query carries no + /// activation check, so forwarding the whole lookup keeps every parameter type reachable — + /// including ones added later, which a hand-listed forwarding would silently drop. + pub fn with_params_from(self, config: &'a MegaHardforkConfig) -> Self { + Self { params: Some(config), ..self } } } -impl EthereumHardforks for FixedHardfork { +impl EthereumHardforks for FixedHardfork<'_> { fn ethereum_fork_activation(&self, fork: EthereumHardfork) -> ForkCondition { if fork <= EthereumHardfork::Prague { ForkCondition::Timestamp(0) @@ -27,7 +48,7 @@ impl EthereumHardforks for FixedHardfork { } } -impl OpHardforks for FixedHardfork { +impl OpHardforks for FixedHardfork<'_> { fn op_fork_activation(&self, fork: OpHardfork) -> ForkCondition { if fork <= OpHardfork::Isthmus { ForkCondition::Timestamp(0) @@ -37,7 +58,7 @@ impl OpHardforks for FixedHardfork { } } -impl MegaHardforks for FixedHardfork { +impl MegaHardforks for FixedHardfork<'_> { fn mega_fork_activation(&self, fork: MegaHardfork) -> ForkCondition { let mapped_spec = fork.spec_id(); if mapped_spec <= self.spec { @@ -46,4 +67,119 @@ impl MegaHardforks for FixedHardfork { ForkCondition::Never } } + + fn fork_params_any(&self, fork: MegaHardfork) -> Option<&(dyn Any + Send + Sync)> { + self.params?.fork_params_any(fork) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mega_evm::{ + hardfork_schedule, SequencerRegistryConfig, SequencerRegistryRex6Config, MAINNET_CHAIN_ID, + }; + + /// A chain configuration carrying both parameter types, so the delegation can be checked for a + /// type the published schedules do not (yet) attach. + fn config_with_all_params() -> MegaHardforkConfig { + // The unknown-chain fallback attaches both the Rex5 and the Rex6 registry parameters. + hardfork_schedule(0xdead_beef) + } + + /// Activation is a function of the fixed spec alone, in both directions: everything up to the + /// spec is active at timestamp 0, everything above it never activates — no matter which + /// timestamp is asked about, and no matter what the attached parameter source schedules. + #[test] + fn test_activation_follows_the_fixed_spec_only() { + let chain = hardfork_schedule(MAINNET_CHAIN_ID); + let fixed = FixedHardfork::new(MegaSpecId::REX5).with_params_from(&chain); + + for fork in MegaHardfork::VARIANTS { + let expected = if fork.spec_id() <= MegaSpecId::REX5 { + ForkCondition::Timestamp(0) + } else { + ForkCondition::Never + }; + assert_eq!(fixed.mega_fork_activation(*fork), expected, "{fork:?}"); + } + + // The chain's own schedule puts Rex5 far in the future; the fixed world ignores it. + assert_eq!(fixed.spec_id(0), MegaSpecId::REX5); + assert_eq!(fixed.spec_id(u64::MAX), MegaSpecId::REX5); + } + + /// Every spec on the ladder resolves to itself, including the patch hardforks that map back to + /// an earlier spec. + #[test] + fn test_every_spec_resolves_to_itself() { + for spec in [ + MegaSpecId::EQUIVALENCE, + MegaSpecId::MINI_REX, + MegaSpecId::REX, + MegaSpecId::REX1, + MegaSpecId::REX2, + MegaSpecId::REX3, + MegaSpecId::REX4, + MegaSpecId::REX5, + MegaSpecId::REX6, + MegaSpecId::REX7, + ] { + assert_eq!(FixedHardfork::new(spec).spec_id(0), spec, "{spec:?}"); + } + } + + /// Without a parameter source, parameter lookups stay empty — the behavior the `run` / `tx` + /// commands rely on. + #[test] + fn test_bare_fixed_hardfork_has_no_params() { + let fixed = FixedHardfork::new(MegaSpecId::REX6); + assert!(fixed.fork_params::().is_none()); + assert!(fixed.fork_params::().is_none()); + } + + /// With a parameter source, lookups return the source's values verbatim — for every parameter + /// type it carries, not just the one the current deploy path happens to need. + #[test] + fn test_params_are_delegated_to_the_chain_config() { + let chain = config_with_all_params(); + let fixed = FixedHardfork::new(MegaSpecId::REX7).with_params_from(&chain); + + assert_eq!( + fixed.fork_params::(), + chain.fork_params::(), + ); + assert_eq!( + fixed.fork_params::(), + chain.fork_params::(), + ); + assert!(fixed.fork_params::().is_some()); + assert!(fixed.fork_params::().is_some()); + } + + /// The mainnet schedule's Rex5 parameters survive the swap: this is what keeps a Rex5+ + /// override from failing closed in the pre-block `SequencerRegistry` deploy. + #[test] + fn test_mainnet_rex5_params_survive_the_swap() { + let chain = hardfork_schedule(MAINNET_CHAIN_ID); + let fixed = FixedHardfork::new(MegaSpecId::REX5).with_params_from(&chain); + + assert_eq!( + fixed.fork_params::(), + chain.fork_params::(), + ); + assert!(fixed.fork_params::().is_some()); + } + + /// Parameter lookups do not consult activation: a parameter attached to a fork the fixed spec + /// never activates is still returned. Delegating wholesale therefore cannot depend on the + /// order in which parameters and specs were chosen. + #[test] + fn test_params_lookup_is_independent_of_activation() { + let chain = config_with_all_params(); + let fixed = FixedHardfork::new(MegaSpecId::EQUIVALENCE).with_params_from(&chain); + + assert_eq!(fixed.mega_fork_activation(MegaHardfork::Rex5), ForkCondition::Never); + assert!(fixed.fork_params::().is_some()); + } } diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 5923014f..390f1731 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -14,8 +14,8 @@ use mega_evm::{ primitives::eip4844, DatabaseRef, }, - BlockLimits, EvmTxRuntimeLimits, MegaBlockExecutionCtx, MegaBlockExecutorFactory, - MegaEvmFactory, MegaHardforks, MegaSpecId, + BlockLimits, MegaBlockExecutionCtx, MegaBlockExecutorFactory, MegaEvmFactory, MegaHardforks, + MegaSpecId, }; use tracing::{debug, error, info, trace, warn}; @@ -29,7 +29,7 @@ use crate::{ ExecutionSummary, ExternalEnvSnapshot, OpTxReceipt, RpcArgs, RpcCacheStore, TracerType, TxOverrideArgs, }, - replay::get_hardfork_config, + replay::{get_hardfork_config, ReplayHardforks}, run, ChainArgs, EvmeState, }; @@ -268,6 +268,17 @@ impl Cmd { Ok(()) } + /// The spec forced by `--override.spec`, parsed. + fn resolve_spec_override(&self) -> Result> { + self.spec_override + .as_deref() + .map(|spec| { + MegaSpecId::from_str(spec) + .map_err(|e| ReplayError::Other(format!("Invalid spec: {e:?}"))) + }) + .transpose() + } + /// Whether this invocation selects a batch of transactions. fn is_batch(&self) -> bool { self.tx_file.is_some() || self.block.is_some() @@ -651,7 +662,16 @@ impl Cmd { where P: Provider + Clone + std::fmt::Debug, { - let hardforks = get_hardfork_config(ctx.chain_id); + // `--override.spec` replaces the whole execution world, not just the EVM semantics: the + // synthesized schedule drives the pre-block predeploys and the block-level limits too, so + // the replay is a coherent what-if rather than a mix of the historical setup with forced + // semantics. + let chain_hardforks = get_hardfork_config(ctx.chain_id); + let spec_override = self.resolve_spec_override()?; + if let Some(spec_override) = spec_override { + info!(spec_override = %spec_override, "Overriding EVM spec"); + } + let hardforks = ReplayHardforks::resolve(&chain_hardforks, spec_override); let spec = hardforks.spec_id(ctx.block.header.timestamp()); let chain_args = ChainArgs { chain_id: ctx.chain_id, spec: spec.to_string() }; debug!(chain_id = ctx.chain_id, spec = %spec, "Chain configuration"); @@ -667,7 +687,7 @@ impl Cmd { let block_env = retrieve_block_env(&ctx.block)?; trace!(?block_env, "Block environment built"); - let mut evm_env = EvmEnv::new(chain_args.create_cfg_env()?, block_env); + let evm_env = EvmEnv::new(chain_args.create_cfg_env()?, block_env); // For `--dump-fixture`, snapshot the two inputs a fixture // needs before the external env is moved into the factory: the effective @@ -745,12 +765,13 @@ impl Cmd { }; let evm_factory = MegaEvmFactory::new().with_external_env_factory(external_envs); - let block_executor_factory = MegaBlockExecutorFactory::new( - &hardforks, - evm_factory, - OpAlloyReceiptBuilder::default(), - ); - let mut block_limits = BlockLimits::from_hardfork_and_block_gas_limit( + let block_executor_factory = + MegaBlockExecutorFactory::new(hardforks, evm_factory, OpAlloyReceiptBuilder::default()); + // Both the per-transaction and the block-level dimensions come from the fork resolved out + // of the schedule above, so a spec override moves all of them at once. The no-override + // path keeps the "no fork active" failure: a block older than the chain's first hardfork + // has no limits to execute under. A synthesized schedule always has one active. + let block_limits = BlockLimits::from_hardfork_and_block_gas_limit( hardforks.hardfork(ctx.block.header.timestamp()).ok_or(ReplayError::Other(format!( "No `MegaHardfork` active at block timestamp: {}", ctx.block.header.timestamp() @@ -758,14 +779,6 @@ impl Cmd { ctx.block.header.gas_limit(), ); - if let Some(spec_override) = &self.spec_override { - info!(spec_override = %spec_override, "Overriding EVM spec"); - let spec = MegaSpecId::from_str(spec_override) - .map_err(|e| ReplayError::Other(format!("Invalid spec: {e:?}")))?; - evm_env.cfg_env.set_spec_and_mainnet_gas_params(spec); - block_limits = block_limits.with_tx_runtime_limits(EvmTxRuntimeLimits::from_spec(spec)); - } - // The spec the target transaction will execute under (after any override), // captured before `evm_env` is moved into the executor. let executed_spec = evm_env.cfg_env.spec; diff --git a/bin/mega-evme/src/replay/hardforks.rs b/bin/mega-evme/src/replay/hardforks.rs index 011514f2..a2b207c9 100644 --- a/bin/mega-evme/src/replay/hardforks.rs +++ b/bin/mega-evme/src/replay/hardforks.rs @@ -1,4 +1,12 @@ -use mega_evm::MegaHardforkConfig; +use core::any::Any; + +use mega_evm::{ + alloy_hardforks::{EthereumHardfork, ForkCondition}, + alloy_op_hardforks::{EthereumHardforks, OpHardfork, OpHardforks}, + MegaHardfork, MegaHardforkConfig, MegaHardforks, MegaSpecId, +}; + +use crate::common::FixedHardfork; /// Returns the hardfork configuration for a given chain ID. /// @@ -8,3 +16,241 @@ use mega_evm::MegaHardforkConfig; pub fn get_hardfork_config(chain_id: u64) -> MegaHardforkConfig { mega_evm::hardfork_schedule(chain_id) } + +/// The hardfork schedule a replay executes under. +/// +/// Without a spec override this is the chain's real schedule, so the replay reproduces the block +/// as it happened. With `--override.spec` it is a schedule synthesized from the forced spec, which +/// makes the override a coherent what-if: the pre-block predeploys, the EIP-2935 / EIP-4788 +/// gating, the block-level resource limits and the EVM semantics all come from the same spec, +/// instead of mixing the historical setup with forced semantics into a world that never existed. +/// +/// The synthesized schedule takes activation from the forced spec but keeps the chain's per-fork +/// parameters, which are chain data rather than spec data (the Rex5+ `SequencerRegistry` seeds). +#[derive(Debug, Clone, Copy)] +pub enum ReplayHardforks<'a> { + /// The chain's published activation schedule. + Chain(&'a MegaHardforkConfig), + /// A schedule synthesized from a forced spec, with parameters from the chain. + Forced(FixedHardfork<'a>), +} + +impl<'a> ReplayHardforks<'a> { + /// Selects the schedule for a replay: the chain's own, or one synthesized from + /// `spec_override`. + pub fn resolve(chain: &'a MegaHardforkConfig, spec_override: Option) -> Self { + match spec_override { + Some(spec) => Self::Forced(FixedHardfork::new(spec).with_params_from(chain)), + None => Self::Chain(chain), + } + } +} + +impl EthereumHardforks for ReplayHardforks<'_> { + fn ethereum_fork_activation(&self, fork: EthereumHardfork) -> ForkCondition { + match self { + Self::Chain(chain) => chain.ethereum_fork_activation(fork), + Self::Forced(forced) => forced.ethereum_fork_activation(fork), + } + } +} + +impl OpHardforks for ReplayHardforks<'_> { + fn op_fork_activation(&self, fork: OpHardfork) -> ForkCondition { + match self { + Self::Chain(chain) => chain.op_fork_activation(fork), + Self::Forced(forced) => forced.op_fork_activation(fork), + } + } +} + +impl MegaHardforks for ReplayHardforks<'_> { + fn mega_fork_activation(&self, fork: MegaHardfork) -> ForkCondition { + match self { + Self::Chain(chain) => chain.mega_fork_activation(fork), + Self::Forced(forced) => forced.mega_fork_activation(fork), + } + } + + fn fork_params_any(&self, fork: MegaHardfork) -> Option<&(dyn Any + Send + Sync)> { + match self { + Self::Chain(chain) => chain.fork_params_any(fork), + Self::Forced(forced) => forced.fork_params_any(fork), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mega_evm::{ + flat_system_contract_specs, BlockLimits, EvmTxRuntimeLimits, SequencerRegistryConfig, + SequencerRegistryRex6Config, MAINNET_CHAIN_ID, TESTNET_CHAIN_ID, + }; + + /// A mainnet timestamp inside the Rex4 window: Rex4 is active, Rex5 is not. + const REX4_TIMESTAMP: u64 = 1_776_700_000; + + /// A mainnet timestamp inside the `MiniRex` window, before any Rex fork. + const MINI_REX_TIMESTAMP: u64 = 1_764_000_000; + + /// Without an override the replay world is the chain's schedule, unchanged. + #[test] + fn test_without_override_the_chain_schedule_is_used() { + let chain = get_hardfork_config(MAINNET_CHAIN_ID); + let world = ReplayHardforks::resolve(&chain, None); + + for timestamp in [0, MINI_REX_TIMESTAMP, REX4_TIMESTAMP, u64::MAX] { + assert_eq!(world.spec_id(timestamp), chain.spec_id(timestamp), "at {timestamp}"); + assert_eq!(world.hardfork(timestamp), chain.hardfork(timestamp), "at {timestamp}"); + } + for fork in MegaHardfork::VARIANTS { + assert_eq!( + world.mega_fork_activation(*fork), + chain.mega_fork_activation(*fork), + "{fork:?}", + ); + } + assert_eq!( + world.fork_params::(), + chain.fork_params::(), + ); + } + + /// A block before the first `MegaHardfork` has no active fork, and the replay must report that + /// rather than silently pick one. Testnet's `MiniRex` activates at timestamp 0, so this is + /// checked on a config whose first fork activates later. + #[test] + fn test_without_override_a_block_before_any_fork_has_no_hardfork() { + let chain = + MegaHardforkConfig::new().with(MegaHardfork::Rex, ForkCondition::Timestamp(100)); + let world = ReplayHardforks::resolve(&chain, None); + + assert_eq!(world.hardfork(99), None); + assert_eq!(world.hardfork(100), Some(MegaHardfork::Rex)); + } + + /// With an override, the whole schedule follows the forced spec: it resolves to that spec at + /// the block's timestamp (and at any other), so every consumer that reads the schedule — + /// predeploys, block limits, EVM semantics — sees the same world. + #[test] + fn test_override_makes_the_schedule_follow_the_forced_spec() { + let chain = get_hardfork_config(MAINNET_CHAIN_ID); + let world = ReplayHardforks::resolve(&chain, Some(MegaSpecId::REX5)); + + assert_eq!(world.spec_id(MINI_REX_TIMESTAMP), MegaSpecId::REX5); + assert_eq!(world.spec_id(REX4_TIMESTAMP), MegaSpecId::REX5); + assert_eq!(world.hardfork(MINI_REX_TIMESTAMP), Some(MegaHardfork::Rex5)); + assert!(world.is_rex_5_active_at_timestamp(MINI_REX_TIMESTAMP)); + assert!(!world.is_rex_6_active_at_timestamp(MINI_REX_TIMESTAMP)); + } + + /// The forced schedule keeps the chain's per-fork parameters. Without this the pre-block + /// `SequencerRegistry` deploy fails closed on every Rex5+ override. + #[test] + fn test_override_keeps_the_chain_fork_params() { + for chain_id in [MAINNET_CHAIN_ID, TESTNET_CHAIN_ID] { + let chain = get_hardfork_config(chain_id); + let world = ReplayHardforks::resolve(&chain, Some(MegaSpecId::REX5)); + + assert_eq!( + world.fork_params::(), + chain.fork_params::(), + "chain {chain_id}", + ); + assert!( + world.fork_params::().is_some(), + "chain {chain_id} must carry the Rex5 registry parameters", + ); + } + } + + /// Every parameter type the chain carries is delegated, not just the one the Rex5 deploy path + /// needs today. + #[test] + fn test_override_delegates_every_params_type() { + // The unknown-chain fallback carries both registry parameter types. + let chain = get_hardfork_config(0xdead_beef); + let world = ReplayHardforks::resolve(&chain, Some(MegaSpecId::REX6)); + + assert_eq!( + world.fork_params::(), + chain.fork_params::(), + ); + assert!(world.fork_params::().is_some()); + } + + /// The predeploy set follows the override, in both directions: forcing a newer spec on an old + /// block installs contracts that did not exist at that block, and forcing an older spec on a + /// recent block withholds contracts that did. + #[test] + fn test_override_switches_the_predeploy_set() { + let chain = get_hardfork_config(MAINNET_CHAIN_ID); + + let historical = ReplayHardforks::resolve(&chain, None); + let upgraded = ReplayHardforks::resolve(&chain, Some(MegaSpecId::REX5)); + let downgraded = ReplayHardforks::resolve(&chain, Some(MegaSpecId::MINI_REX)); + + let at = |world: &ReplayHardforks<'_>, timestamp| { + flat_system_contract_specs(world, timestamp) + .into_iter() + .map(|spec| spec.address) + .collect::>() + }; + + // MegaLimitControl arrives with Rex4, so a MiniRex-era block gains it under a Rex5 + // override and a Rex4-era block loses it under a MiniRex override. + let historical_mini_rex = at(&historical, MINI_REX_TIMESTAMP); + let forced_rex5 = at(&upgraded, MINI_REX_TIMESTAMP); + assert!(forced_rex5.len() > historical_mini_rex.len()); + assert!(forced_rex5.contains(&mega_evm::LIMIT_CONTROL_ADDRESS)); + assert!(!historical_mini_rex.contains(&mega_evm::LIMIT_CONTROL_ADDRESS)); + + let historical_rex4 = at(&historical, REX4_TIMESTAMP); + let forced_mini_rex = at(&downgraded, REX4_TIMESTAMP); + assert!(historical_rex4.contains(&mega_evm::LIMIT_CONTROL_ADDRESS)); + assert!(!forced_mini_rex.contains(&mega_evm::LIMIT_CONTROL_ADDRESS)); + + // The registry is deployed separately from the flat predeploys; its gate reads the same + // schedule, so a Rex5 override activates it on a MiniRex-era block. + assert!(upgraded.is_rex_5_active_at_timestamp(MINI_REX_TIMESTAMP)); + assert!(!historical.is_rex_5_active_at_timestamp(MINI_REX_TIMESTAMP)); + } + + /// Block-level limits follow the override too, not only the per-transaction ones. The + /// block-level dimensions are the ones a per-transaction patch cannot reach: they come from + /// the hardfork resolved out of the schedule. + #[test] + fn test_override_switches_block_level_limits() { + let chain = get_hardfork_config(MAINNET_CHAIN_ID); + let gas_limit = 10_000_000_000; + + let historical = ReplayHardforks::resolve(&chain, None); + let forced = ReplayHardforks::resolve(&chain, Some(MegaSpecId::REX5)); + + let historical_limits = BlockLimits::from_hardfork_and_block_gas_limit( + historical.hardfork(MINI_REX_TIMESTAMP).expect("MiniRex is active"), + gas_limit, + ); + let forced_limits = BlockLimits::from_hardfork_and_block_gas_limit( + forced.hardfork(MINI_REX_TIMESTAMP).expect("the forced spec is always active"), + gas_limit, + ); + + // State growth metering arrives with Rex: the block-level budget is unlimited under + // MiniRex and bounded under the forced Rex5 world. + assert_eq!(historical_limits.block_state_growth_limit, u64::MAX); + assert_ne!(forced_limits.block_state_growth_limit, u64::MAX); + assert_eq!( + forced_limits, + BlockLimits::from_hardfork_and_block_gas_limit(MegaHardfork::Rex5, gas_limit), + ); + + // The per-transaction dimensions follow as well, which is what makes the previous + // per-transaction patch redundant rather than merely subsumed. + assert_eq!( + forced_limits.to_evm_tx_runtime_limits(), + EvmTxRuntimeLimits::from_spec(MegaSpecId::REX5), + ); + } +} diff --git a/bin/mega-evme/tests/common/mod.rs b/bin/mega-evme/tests/common/mod.rs index 18712500..db751066 100644 --- a/bin/mega-evme/tests/common/mod.rs +++ b/bin/mega-evme/tests/common/mod.rs @@ -158,6 +158,50 @@ impl MockRpcServer { .await; } + /// Mount an unbounded mock that answers every JSON-RPC request for `method` + /// with the given JSON `result`, regardless of params. + /// + /// The result is any JSON value, so this serves structured answers (blocks, + /// transactions) that [`Self::respond_method_result`]'s hex string cannot. + pub(crate) async fn respond_method_json( + &self, + method: &str, + result: serde_json::Value, + priority: u8, + ) { + let body = serde_json::json!({ "jsonrpc": "2.0", "id": 0, "result": result }); + Mock::given(matchers::method("POST")) + .and(matchers::body_partial_json(serde_json::json!({ "method": method }))) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .with_priority(priority) + .mount(&self.server) + .await; + } + + /// Mount an unbounded mock that answers `method` calls whose params match + /// `params` with the given JSON `result`. + /// + /// Needed where one method is called with different arguments in the same + /// run and the answers must differ — `eth_getBlockByNumber` for a block and + /// its parent, for instance. + pub(crate) async fn respond_method_params_json( + &self, + method: &str, + params: serde_json::Value, + result: serde_json::Value, + priority: u8, + ) { + let body = serde_json::json!({ "jsonrpc": "2.0", "id": 0, "result": result }); + Mock::given(matchers::method("POST")) + .and(matchers::body_partial_json( + serde_json::json!({ "method": method, "params": params }), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .with_priority(priority) + .mount(&self.server) + .await; + } + /// Mount a mock that returns `eth_chainId` with the given chain id. pub(crate) async fn respond_eth_chain_id(&self, chain_id: u64, priority: u8) { let body = serde_json::json!({ diff --git a/bin/mega-evme/tests/replay_override_spec.rs b/bin/mega-evme/tests/replay_override_spec.rs new file mode 100644 index 00000000..59960cd0 --- /dev/null +++ b/bin/mega-evme/tests/replay_override_spec.rs @@ -0,0 +1,403 @@ +//! Integration tests for `mega-evme replay --override.spec`. +//! +//! The override is a coherent what-if: the whole execution world switches to the +//! forced spec, as if the block had run on a chain at that spec. These tests pin +//! the consequences that are visible from outside the process — the pre-block +//! predeploys (their presence and their version) and the block-level resource +//! limits follow the override rather than the block's position in the chain's +//! schedule, and a fork whose parameters the chain never published cannot be +//! forced at all. +//! +//! They run against a mock JSON-RPC endpoint rather than a recorded capture: a +//! higher-spec override reads state the historical replay never touched (the +//! `SequencerRegistry` account, for one), and an offline capture answers a miss +//! with a hard error rather than the state the forced world needs. + +use std::process::Command; + +use serde_json::{json, Value}; + +mod common; +use common::MockRpcServer; + +/// `MegaETH` mainnet: the chain whose published schedule carries the Rex5 +/// `SequencerRegistry` parameters that a Rex5+ override needs. +const CHAIN_ID: u64 = 4326; + +/// The replayed block. Its parent is `BLOCK_NUMBER - 1`. +const BLOCK_NUMBER: u64 = 18_172_461; + +/// A mainnet timestamp inside the `MiniRex` window, well before Rex4 (the first +/// fork that deploys `MegaLimitControl`) and Rex5 (the first that deploys the +/// `SequencerRegistry`). Forcing a newer spec on this block is what makes the +/// two worlds — historical and forced — visibly different. +const MINI_REX_TIMESTAMP: u64 = 1_764_000_000; + +/// Hash of the replayed transaction, and the only transaction in the block. +const TX_HASH: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; + +/// Hash of the replayed block. +const BLOCK_HASH: &str = "0x2801837c261826beb8047e46139dfc4eb93ab5b3196ce23f312d3c7658262a62"; + +/// Hash of the parent block, which the replay forks its state from. +const PARENT_HASH: &str = "0xd482d481e9d11dd116ef6c41bf95ca608f159206c8f07900b1b53936d196ccb3"; + +/// Hash of the grandparent, so the parent block is a well-formed header. +const GRANDPARENT_HASH: &str = "0x152b00e0c659a9ea0827f7d3b7666951c100bb6a6761a90e20ed7f79099a82e1"; + +/// Sender of the replayed transaction. Funded by the mock's blanket balance. +const SENDER: &str = "0x14112799a39f2905b901067d3cd4a1f63c1cebda"; + +/// `SequencerRegistry`, deployed pre-block from Rex5 on. +const SEQUENCER_REGISTRY: &str = "0x6342000000000000000000000000000000000006"; + +/// `MegaLimitControl`, deployed pre-block from Rex4 on. +const LIMIT_CONTROL: &str = "0x6342000000000000000000000000000000000005"; + +/// `version()` — declared by `ISemver`, implemented by the registry bytecode, +/// and answering with the deployed version string. +const VERSION_SELECTOR: &str = "0x54fd4d50"; + +/// ABI-encoded `"1.0.0"`: the version the pre-Rex6 `SequencerRegistry` reports. +const VERSION_1_0_0: &str = concat!( + "0x", + "0000000000000000000000000000000000000000000000000000000000000020", + "0000000000000000000000000000000000000000000000000000000000000005", + "312e302e30000000000000000000000000000000000000000000000000000000", +); + +/// ABI-encoded `"2.0.0"`: the version the Rex6 `SequencerRegistry` reports. +const VERSION_2_0_0: &str = concat!( + "0x", + "0000000000000000000000000000000000000000000000000000000000000020", + "0000000000000000000000000000000000000000000000000000000000000005", + "322e302e30000000000000000000000000000000000000000000000000000000", +); + +/// A chain id with no published schedule. `mega-evm` answers those with an +/// all-activated schedule that carries every registry parameter type, so it is +/// the counterpart to mainnet for the parameter-availability cases below. +const UNKNOWN_CHAIN_ID: u64 = 0xdead_beef; + +/// Outcome of one `mega-evme replay` invocation. +struct Run { + code: Option, + stdout: String, + stderr: String, +} + +impl Run { + /// The single `--json` summary the run printed. + fn summary(&self) -> Value { + let mut values = common::json_values(&self.stdout); + if values.last().is_some_and(common::is_run_error) { + values.pop(); + } + assert_eq!( + values.len(), + 1, + "expected one summary on stdout:\n{}\nstderr:\n{}", + self.stdout, + self.stderr, + ); + values.pop().expect("checked above") + } + + /// The hex return data of a successful run, or `None` when the call + /// returned nothing (an account with no code answers empty). + fn output(&self) -> Option { + let summary = self.summary(); + assert_eq!( + summary["success"], + json!(true), + "the replay must succeed:\n{}\nstderr:\n{}", + self.stdout, + self.stderr, + ); + summary["output"].as_str().map(str::to_string) + } +} + +/// A block header the RPC backend and the replay accept, carrying only the +/// fields either of them reads. +fn block_json(number: u64, hash: &str, parent_hash: &str, timestamp: u64, txs: Vec<&str>) -> Value { + json!({ + "hash": hash, + "parentHash": parent_hash, + "number": format!("0x{number:x}"), + "timestamp": format!("0x{timestamp:x}"), + "gasLimit": "0x2540be400", + "gasUsed": "0x0", + "baseFeePerGas": "0xf4240", + "blobGasUsed": "0x0", + "excessBlobGas": "0x0", + "difficulty": "0x0", + "extraData": "0x00000000fa00000001", + "logsBloom": format!("0x{}", "0".repeat(512)), + "miner": "0x4200000000000000000000000000000000000011", + "mixHash": "0x5cd8791a477b467456670744425e11d5bd91fd54575d6d3bf80d761ab39d957f", + "nonce": "0x0000000000000000", + "parentBeaconBlockRoot": + "0x67123956bf748ccfcfa68f03531dd12c1c647f9f31cc91935ce4271fa7399e24", + "receiptsRoot": "0x16fe124682128dd43a5da7f2cee0a3bf076deaf12682d19c656914bbea4615e3", + "requestsHash": "0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x43e7", + "stateRoot": "0xa342aba318978654abcf7f09f9494ed271e2136040b628edacb6d384e9074416", + "transactionsRoot": "0x2f3c5d0b0c4c8d34dd4e1c8bb4b4a4b6d6a2a3d3b8f6a9a2c1d0e9f8a7b6c5d4", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "uncles": [], + "withdrawals": [], + "transactions": txs, + }) +} + +/// The replayed transaction: an EIP-1559 call to `to` with `input` as calldata. +fn tx_json(chain_id: u64, to: &str, input: &str) -> Value { + json!({ + "type": "0x2", + "chainId": format!("0x{chain_id:x}"), + "nonce": "0x0", + "gas": "0x249f0", + "maxFeePerGas": "0x200b20", + "maxPriorityFeePerGas": "0x186a0", + "gasPrice": "0x10c8e0", + "to": to, + "value": "0x0", + "accessList": [], + "input": input, + "r": "0xa19f0f1f52e2951452711b4f4aa5d177442c9a56abeb609b803fe2412ed24946", + "s": "0x7af21777b2e7d91c745d0077ba2726ee1bb75ccf00039a6218d64fdced768491", + "yParity": "0x0", + "v": "0x0", + "hash": TX_HASH, + "from": SENDER, + "blockHash": BLOCK_HASH, + "blockNumber": format!("0x{BLOCK_NUMBER:x}"), + "transactionIndex": "0x0", + }) +} + +/// A mock endpoint serving a one-transaction mainnet block at `timestamp`, +/// whose transaction calls `to` with `input`. +/// +/// Account reads are answered blanket: every account holds 1 ETH, has nonce 0, +/// no code, and zero storage. That leaves the pre-block deploys as the only +/// source of code on the forked state, which is what makes "did this spec's +/// predeploys land" observable from the transaction's own return data. +async fn mock_chain(to: &str, input: &str, timestamp: u64) -> MockRpcServer { + mock_chain_with_id(CHAIN_ID, to, input, timestamp).await +} + +/// [`mock_chain`], on the chain id of the caller's choosing. +async fn mock_chain_with_id(chain_id: u64, to: &str, input: &str, timestamp: u64) -> MockRpcServer { + let server = MockRpcServer::start().await; + server.respond_eth_chain_id(chain_id, 1).await; + server + .respond_method_params_json( + "eth_getBlockByNumber", + json!([format!("0x{BLOCK_NUMBER:x}"), false]), + block_json(BLOCK_NUMBER, BLOCK_HASH, PARENT_HASH, timestamp, vec![TX_HASH]), + 2, + ) + .await; + server + .respond_method_params_json( + "eth_getBlockByNumber", + json!([format!("0x{:x}", BLOCK_NUMBER - 1), false]), + block_json(BLOCK_NUMBER - 1, PARENT_HASH, GRANDPARENT_HASH, timestamp - 1, vec![]), + 2, + ) + .await; + server.respond_method_json("eth_getTransactionByHash", tx_json(chain_id, to, input), 3).await; + server.respond_method_result("eth_getBalance", "0xde0b6b3a7640000", 4).await; + server.respond_method_result("eth_getTransactionCount", "0x0", 4).await; + server.respond_method_result("eth_getCode", "0x", 4).await; + server + .respond_method_result( + "eth_getStorageAt", + "0x0000000000000000000000000000000000000000000000000000000000000000", + 4, + ) + .await; + server +} + +/// Replay the mock's transaction, optionally with extra flags. +fn replay(server: &MockRpcServer, args: &[&str]) -> Run { + let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args(["replay", TX_HASH, "--rpc", &server.uri()]) + .args(["--rpc.no-cache-file", "--rpc.max-retries", "0", "--rpc.backoff-ms", "1", "--json"]) + .args(args) + .output() + .expect("failed to run mega-evme"); + Run { + code: output.status.code(), + stdout: String::from_utf8(output.stdout).expect("stdout is utf-8"), + stderr: String::from_utf8(output.stderr).expect("stderr is utf-8"), + } +} + +/// Without an override the block executes where it sits in the chain's +/// schedule: on a MiniRex-era block the Rex5 `SequencerRegistry` was never +/// deployed, so the call reaches an account with no code. +#[tokio::test(flavor = "multi_thread")] +async fn test_without_override_predeploys_follow_the_block_timestamp() { + let server = mock_chain(SEQUENCER_REGISTRY, VERSION_SELECTOR, MINI_REX_TIMESTAMP).await; + let run = replay(&server, &[]); + + assert_eq!(run.code, Some(0), "stdout:\n{}\nstderr:\n{}", run.stdout, run.stderr); + assert_eq!(run.output(), None, "a MiniRex-era block must not carry the Rex5 SequencerRegistry",); +} + +/// Forcing Rex5 on a MiniRex-era block installs the Rex5 predeploys, including +/// the `SequencerRegistry` — whose pre-block deploy needs the chain's Rex5 +/// parameters. A synthesized schedule that dropped them would fail the run +/// before any transaction executed. +#[tokio::test(flavor = "multi_thread")] +async fn test_override_installs_the_forced_spec_predeploys() { + let server = mock_chain(SEQUENCER_REGISTRY, VERSION_SELECTOR, MINI_REX_TIMESTAMP).await; + let run = replay(&server, &["--override.spec", "Rex5"]); + + assert!( + !run.stderr.contains("SequencerRegistryConfig not configured"), + "the forced world must keep the chain's Rex5 registry parameters:\n{}", + run.stderr, + ); + assert_eq!(run.code, Some(0), "stdout:\n{}\nstderr:\n{}", run.stdout, run.stderr); + assert_eq!( + run.output().as_deref(), + Some(VERSION_1_0_0), + "the forced spec's registry version must answer the call", + ); +} + +/// Forcing an older spec withholds predeploys the block did have: +/// `MegaLimitControl` arrives with Rex4, and neither its bytecode nor its +/// interception is present in a forced `MiniRex` world. +#[tokio::test(flavor = "multi_thread")] +async fn test_override_to_an_older_spec_withholds_later_predeploys() { + let selector = remaining_compute_gas_selector(); + let server = mock_chain(LIMIT_CONTROL, &selector, MINI_REX_TIMESTAMP).await; + let run = replay(&server, &["--override.spec", "MiniRex"]); + + assert_eq!(run.code, Some(0), "stdout:\n{}\nstderr:\n{}", run.stdout, run.stderr); + assert_eq!(run.output(), None, "MegaLimitControl must not answer in a forced MiniRex world"); +} + +/// The block-level resource limits follow the override too. `MegaLimitControl` +/// reports the compute gas left in the current call, which is the forced spec's +/// per-transaction compute budget minus what the transaction has spent — the +/// `MiniRex` budget the block's own schedule carries is five times larger, so the +/// two are never confusable. +#[tokio::test(flavor = "multi_thread")] +async fn test_override_switches_the_block_limits() { + let selector = remaining_compute_gas_selector(); + let server = mock_chain(LIMIT_CONTROL, &selector, MINI_REX_TIMESTAMP).await; + let run = replay(&server, &["--override.spec", "Rex5"]); + + assert_eq!(run.code, Some(0), "stdout:\n{}\nstderr:\n{}", run.stdout, run.stderr); + let output = run.output().expect("MegaLimitControl must answer under the forced Rex5 world"); + let remaining = decode_remaining_compute_gas(&output); + + let forced_budget = compute_gas_budget(mega_evm::MegaSpecId::REX5); + let historical_budget = compute_gas_budget(mega_evm::MegaSpecId::MINI_REX); + assert!( + historical_budget > forced_budget, + "the scenario needs the two budgets to differ to tell the worlds apart", + ); + assert!( + remaining <= forced_budget && remaining > forced_budget - 1_000_000, + "remaining compute gas {remaining} must come from the forced Rex5 budget \ + {forced_budget}, not the block's MiniRex budget {historical_budget}", + ); +} + +/// An override below the chain's own spec moves the predeploys back to that +/// spec's versions, not just their presence: on a chain running Rex6 the +/// registry is v2.0.0, and forcing Rex5 deploys v1.0.0 instead. +#[tokio::test(flavor = "multi_thread")] +async fn test_override_downgrade_switches_the_predeploy_version() { + let server = mock_chain_with_id( + UNKNOWN_CHAIN_ID, + SEQUENCER_REGISTRY, + VERSION_SELECTOR, + MINI_REX_TIMESTAMP, + ) + .await; + + let historical = replay(&server, &[]); + assert_eq!( + historical.output().as_deref(), + Some(VERSION_2_0_0), + "a chain with no published schedule runs the latest spec", + ); + + let forced = replay(&server, &["--override.spec", "Rex5"]); + assert_eq!(forced.code, Some(0), "stdout:\n{}\nstderr:\n{}", forced.stdout, forced.stderr); + assert_eq!( + forced.output().as_deref(), + Some(VERSION_1_0_0), + "the forced Rex5 world must deploy the Rex5 registry version", + ); +} + +/// A forced spec needs the chain's parameters for every fork it activates. Where +/// the chain's schedule carries them the what-if runs; where it does not, the +/// pre-block deploy fails closed with the missing-parameter error rather than +/// inventing a value the chain never published. Mainnet carries the Rex5 +/// parameters but not the Rex6 ones, so which arm applies is read from the +/// chain configuration instead of being assumed here. +#[tokio::test(flavor = "multi_thread")] +async fn test_override_needs_the_chain_params_of_every_fork_it_activates() { + use mega_evm::{MegaHardforks, SequencerRegistryRex6Config}; + + let configured = mega_evm::hardfork_schedule(CHAIN_ID) + .fork_params::() + .is_some(); + + let server = mock_chain(SEQUENCER_REGISTRY, VERSION_SELECTOR, MINI_REX_TIMESTAMP).await; + let run = replay(&server, &["--override.spec", "Rex6"]); + + if configured { + assert_eq!(run.code, Some(0), "stdout:\n{}\nstderr:\n{}", run.stdout, run.stderr); + assert_eq!(run.output().as_deref(), Some(VERSION_2_0_0)); + } else { + assert_eq!( + run.code, + Some(1), + "a fork whose parameters the chain does not carry must not be forced silently:\n{}", + run.stdout, + ); + assert!( + run.stderr.contains("SequencerRegistryRex6Config not configured"), + "the refusal must name the missing parameters:\n{}", + run.stderr, + ); + } +} + +/// A spec's per-transaction compute gas budget, which the block limits carry +/// into execution. +fn compute_gas_budget(spec: mega_evm::MegaSpecId) -> u64 { + mega_evm::EvmTxRuntimeLimits::from_spec(spec).tx_compute_gas_limit +} + +/// Calldata for `IMegaLimitControl.remainingComputeGas()`. +fn remaining_compute_gas_selector() -> String { + use mega_evm::{alloy_sol_types::SolCall, IMegaLimitControl}; + + format!( + "0x{}", + alloy_primitives::hex::encode(IMegaLimitControl::remainingComputeGasCall {}.abi_encode()) + ) +} + +/// Decode the `uint64` `MegaLimitControl` returns. +fn decode_remaining_compute_gas(output: &str) -> u64 { + use mega_evm::{alloy_sol_types::SolCall, IMegaLimitControl}; + + let bytes = alloy_primitives::hex::decode(output).expect("output is hex"); + IMegaLimitControl::remainingComputeGasCall::abi_decode_returns(&bytes) + .expect("output decodes as uint64") +} diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index d6e91ac4..69f24604 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -448,6 +448,17 @@ Useful when you want to test how the transaction would behave under a different mega-evme replay --override.spec Rex2 ``` +The override replaces the entire execution world, not just the EVM semantics. +The block is executed as if it had run on a chain whose schedule activates the forced spec at genesis: the pre-block system contract deploys, the EIP-2935 and EIP-4788 pre-block calls, the block-level resource limits, and the EVM semantics all come from the forced spec. +This keeps a forced replay coherent — mixing the historical setup with forced semantics would execute a world that never existed on any chain. + +A consequence worth stating explicitly: replaying an old block under a newer spec installs predeploys that did not exist at that block (for example, forcing `Rex5` on a pre-`Rex5` block deploys the `SequencerRegistry`), and forcing an older spec withholds predeploys the block did have, or installs an earlier version of them. +That is intentional — it is what "how would this transaction behave under spec X" means. +The replayed state therefore diverges from the chain's historical state by construction, so `--verify-receipt` will normally report a mismatch and `--dump-fixture` is rejected outright. + +The forced spec does not synthesize chain configuration. +Per-fork parameters (currently the `SequencerRegistry` seeds a chain publishes for `Rex5` and `Rex6`) are taken from the chain's own configuration, so a fork the chain has not configured cannot be forced: the run fails before executing, naming the missing parameters, rather than proceeding with an invented value. + ## Transaction Overrides Override flags let you modify the transaction before re-executing it. diff --git a/docs/mega-evme/configuration/chain-and-spec.md b/docs/mega-evme/configuration/chain-and-spec.md index 6befb4d3..f180fd28 100644 --- a/docs/mega-evme/configuration/chain-and-spec.md +++ b/docs/mega-evme/configuration/chain-and-spec.md @@ -8,6 +8,12 @@ These options control which MegaETH spec and chain ID the EVM uses during execut They are available in the `run` and `tx` commands. The `replay` command auto-detects the spec from the chain ID and block timestamp (see [replay](../commands/replay.md#spec-auto-detection)). +In all three commands a chosen spec defines the whole execution world, not only the opcode and gas rules: the system contracts predeployed before execution, the block-level resource limits, and the EVM semantics all come from that one spec. +For `run` and `tx` there is nothing else it could mean — there is no historical block to contradict it. +For `replay`, [`--override.spec`](../commands/replay.md#--overridespec-spec) makes the same choice explicitly: the block is replayed as if it had run on a chain at the forced spec, so replaying an old block under a newer spec installs predeploys that never existed at that block, and forcing an older spec withholds or downgrades the ones that did. +That is intentional, and it is what makes the answer a coherent what-if rather than a mixture of two worlds. +Chain-specific configuration is not synthesized along with it: a fork whose parameters the chain has not published (the `SequencerRegistry` seeds, today) cannot be forced, and the run fails up front naming what is missing. + ## Options | Flag | Default | Aliases | Description | From 7ed74d451ddfa089704d5522ec76130616c22614 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:37:13 +0800 Subject: [PATCH 40/64] docs(mega-evme): scope the batch jq selectors to per-transaction lines The documented batch selectors matched the run-level error object a failed --json run appends to stdout, so one failed target counted as two. Require .tx_hash, which only per-target lines carry, and say why. --- docs/mega-evme/commands/replay.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 69f24604..a8618c83 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -159,9 +159,11 @@ Where `corpus.txt` looks like: Count the transactions that did not succeed: ```bash -jq -c 'select(.error != null or .success == false)' results.ndjson | wc -l +jq -c 'select(.tx_hash and (.error != null or .success == false))' results.ndjson | wc -l ``` +A failed run ends its stdout with a run-level `{"error": …}` object (see [Exit codes](../overview.md#exit-codes)), which carries no `tx_hash`, so selecting on `.tx_hash` keeps the count to per-transaction lines. + ## Receipt Verification Replaying a transaction only proves that the local EVM produced _some_ result; equivalence verification needs that result checked against what the chain recorded. @@ -262,10 +264,12 @@ Verify a whole corpus in one process and collect the divergences: mega-evme replay --rpc https://mainnet.megaeth.com/rpc \ --tx-file ./corpus.txt --verify-receipt --json > results.ndjson -jq -c 'select(.verification.match == false)' results.ndjson # mismatched -jq -c 'select(.error != null)' results.ndjson # unverified +jq -c 'select(.tx_hash and .verification.match == false)' results.ndjson # mismatched +jq -c 'select(.tx_hash and .error != null)' results.ndjson # unverified ``` +Both selectors require `.tx_hash` so that the run-level `{"error": …}` object a failed run appends to stdout is not counted as an unverified transaction. + Capture once online, then re-verify the same corpus offline: ```bash From b3a22a56e100d636d8b79f628ed5b28d7e576c1e Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:45:24 +0800 Subject: [PATCH 41/64] test(mega-evme): harden lane test pins against review findings Drain closed-stdout panic stderr to avoid RUST_BACKTRACE hangs, pin 7702 authority recovery and non-default typed fields, record present- but-empty accounts in build_pre_state, and e2e-pin the panic-hook JSON envelope via a test-utils inject on open stdout. --- bin/mega-evme/src/common/tx.rs | 33 +++++++++++++--- bin/mega-evme/src/main.rs | 13 +++++++ bin/mega-evme/src/replay/fixture.rs | 18 ++++++++- bin/mega-evme/tests/exit_codes.rs | 59 ++++++++++++++++++++++++++++- 4 files changed, 116 insertions(+), 7 deletions(-) diff --git a/bin/mega-evme/src/common/tx.rs b/bin/mega-evme/src/common/tx.rs index d8b64704..4f12e9b0 100644 --- a/bin/mega-evme/src/common/tx.rs +++ b/bin/mega-evme/src/common/tx.rs @@ -687,6 +687,10 @@ mod tests { assert_eq!(base.tx_type, MegaTxType::Eip2930 as u8); assert_eq!(base.nonce, 4); assert_eq!(base.gas_limit, 50_000); + assert_eq!(base.gas_price, 30_000_000_000, "gas_price must survive typed decode"); + assert_eq!(base.kind, TxKind::Call(TYPED_TO), "to must survive typed decode"); + assert_eq!(base.value, U256::from(1), "value must survive typed decode"); + assert_eq!(base.data, Bytes::from_static(b"\xca\xfe"), "input must survive typed decode"); assert_eq!(base.chain_id, Some(TYPED_CHAIN_ID)); assert_eq!(base.access_list, access_list, "access list addresses and keys must survive"); assert_eq!( @@ -725,6 +729,8 @@ mod tests { assert_eq!(base.nonce, 7); assert_eq!(base.gas_limit, 80_000); assert_eq!(base.chain_id, Some(TYPED_CHAIN_ID)); + assert_eq!(base.kind, TxKind::Call(TYPED_TO), "to must survive typed decode"); + assert_eq!(base.value, U256::from(2), "value must survive typed decode"); assert_eq!(base.gas_price, max_fee_per_gas, "gas_price must map from max_fee_per_gas"); assert_eq!( base.gas_priority_fee, @@ -742,12 +748,14 @@ mod tests { fn test_from_raw_eip7702_recovers_signer_and_preserves_authorization_list() { let signed_auth = sample_signed_authorization(); let expected_inner = signed_auth.inner().clone(); + let max_fee_per_gas = 50_000_000_000u128; + let max_priority_fee_per_gas = 1_000_000_000u128; let tx = TxEip7702 { chain_id: TYPED_CHAIN_ID, nonce: 11, gas_limit: 120_000, - max_fee_per_gas: 50_000_000_000, - max_priority_fee_per_gas: 1_000_000_000, + max_fee_per_gas, + max_priority_fee_per_gas, to: TYPED_TO, value: U256::ZERO, access_list: AccessList::default(), @@ -768,17 +776,32 @@ mod tests { assert_eq!(base.nonce, 11); assert_eq!(base.gas_limit, 120_000); assert_eq!(base.chain_id, Some(TYPED_CHAIN_ID)); + assert_eq!(base.gas_price, max_fee_per_gas, "gas_price must map from max_fee_per_gas"); + assert_eq!( + base.gas_priority_fee, + Some(max_priority_fee_per_gas), + "gas_priority_fee must map from max_priority_fee_per_gas", + ); + assert_eq!(base.kind, TxKind::Call(TYPED_TO), "to must survive typed decode"); assert_eq!(base.authorization_list.len(), 1, "authorization list length must survive"); match &base.authorization_list[0] { Either::Right(recovered) => { assert_eq!(*recovered.chain_id(), expected_inner.chain_id); assert_eq!(*recovered.address(), expected_inner.address); assert_eq!(recovered.nonce(), expected_inner.nonce); + // Authority recovery is independent of field survival: a + // recovery regression that yields RecoveredAuthority::Invalid + // must not pass this test. + assert_eq!( + recovered.authority(), + Some(DEFAULT_SENDER), + "authorization authority must recover to the test signer", + ); } Either::Left(signed) => { - assert_eq!(signed.inner().chain_id, expected_inner.chain_id); - assert_eq!(signed.inner().address, expected_inner.address); - assert_eq!(signed.inner().nonce, expected_inner.nonce); + panic!( + "from_raw must recover the authorization authority, got unrecovered signed auth: {signed:?}" + ); } } assert_eq!( diff --git a/bin/mega-evme/src/main.rs b/bin/mega-evme/src/main.rs index b4ebb4c6..8defa229 100644 --- a/bin/mega-evme/src/main.rs +++ b/bin/mega-evme/src/main.rs @@ -16,6 +16,19 @@ use mega_evme::{cmd::MainCmd, print_json_error, report_command_result, set_threa async fn main() -> ExitCode { set_thread_panic_hook(); + // Test-only injection: force a panic after the process-wide hook is + // installed so integration tests can pin the structured JSON envelope on + // an open stdout. Same gate as the fixture pre-state inject + // (`test-utils`, enabled for the test-profile binary via the self + // dev-dependency). Production builds never carry this branch. + // `manual_assert` is allowed: this must be a plain `panic!` payload so the + // hook message stays `panic: …`, not an assertion-failure rewrite. + #[cfg(feature = "test-utils")] + #[allow(clippy::manual_assert)] + if std::env::var_os("MEGA_EVME_INJECT_PANIC").is_some() { + panic!("injected panic for panic-hook JSON envelope test"); + } + let cmd = match MainCmd::try_parse() { Ok(cmd) => cmd, Err(err) => { diff --git a/bin/mega-evme/src/replay/fixture.rs b/bin/mega-evme/src/replay/fixture.rs index 89cc3af6..b66e3448 100644 --- a/bin/mega-evme/src/replay/fixture.rs +++ b/bin/mega-evme/src/replay/fixture.rs @@ -650,15 +650,19 @@ mod tests { } /// Touched addresses with no pre-transaction account are omitted from `pre`; - /// touched addresses that exist are recorded with their fields. + /// touched addresses that exist are recorded with their fields — including + /// an explicitly present-but-empty account (`Some(AccountInfo::default())`). /// /// Absence-means-nonexistence is the state-test fixture shape: a forked /// backend returns `None` for all-zero RPC answers, so accounts created by /// the target transaction must not appear as explicit empty entries. + /// Presence of an empty account is a different DB answer and must still be + /// recorded; `build_pre_state` does not filter empties with `is_empty()`. #[test] fn test_build_pre_state_omits_nonexistent_and_records_existing() { let missing = Address::repeat_byte(0xaa); let present = Address::repeat_byte(0xbb); + let empty_present = Address::repeat_byte(0xcc); let balance = U256::from(42u64); let nonce = 7u64; @@ -674,11 +678,15 @@ mod tests { ..Default::default() }), ); + // Present-but-empty: the DB returns Some with zero fields. This must + // stay in `pre` so a future `is_empty()` filter cannot creep in. + accounts.insert(empty_present, Some(RevmAccountInfo::default())); let db = MapDb { accounts }; let mut evm_state = EvmState::default(); evm_state.insert(missing, Default::default()); evm_state.insert(present, Default::default()); + evm_state.insert(empty_present, Default::default()); let pre = build_pre_state(&db, &evm_state).expect("pre-state construction succeeds"); @@ -691,6 +699,14 @@ mod tests { assert_eq!(recorded.nonce, nonce); assert!(recorded.code.is_empty()); assert!(recorded.storage.is_empty()); + + let empty_recorded = pre + .get(&empty_present) + .expect("touched + basic_ref=Some(default) must appear in pre (empty is not absent)"); + assert_eq!(empty_recorded.balance, U256::ZERO); + assert_eq!(empty_recorded.nonce, 0); + assert!(empty_recorded.code.is_empty()); + assert!(empty_recorded.storage.is_empty()); } fn deposit_transaction() -> Transaction { diff --git a/bin/mega-evme/tests/exit_codes.rs b/bin/mega-evme/tests/exit_codes.rs index c0bab4eb..e26c9630 100644 --- a/bin/mega-evme/tests/exit_codes.rs +++ b/bin/mega-evme/tests/exit_codes.rs @@ -384,8 +384,9 @@ fn test_help_in_json_mode_prints_no_error_object() { #[test] fn test_closed_stdout_during_json_batch_exits_one() { use std::{ - io::{BufRead, BufReader}, + io::{BufRead, BufReader, Read}, process::{Command, Stdio}, + thread, }; // Multi-target offline batch: many NDJSON lines, so dropping the pipe after @@ -400,11 +401,25 @@ fn test_closed_stdout_during_json_batch_exits_one() { "22945844", "--json", ]) + // Bound panic-hook stderr volume regardless of the ambient env: a full + // backtrace can fill the pipe buffer and deadlock child-vs-`wait()` if + // stderr is never drained. We still drain (below) so a caller that + // exports `RUST_BACKTRACE=full` cannot hang this test either. + .env("RUST_BACKTRACE", "0") .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .expect("failed to spawn mega-evme"); + // Drain stderr on a reader thread before any wait, so a noisy panic hook + // cannot fill the pipe and stall the child forever. + let stderr = child.stderr.take().expect("child stderr was piped"); + let stderr_drain = thread::spawn(move || { + let mut sink = Vec::new(); + let _ = BufReader::new(stderr).read_to_end(&mut sink); + sink + }); + let stdout = child.stdout.take().expect("child stdout was piped"); let mut first_line = String::new(); BufReader::new(stdout) @@ -419,6 +434,7 @@ fn test_closed_stdout_during_json_batch_exits_one() { // (Binding ends here; no further use of the pipe.) let status = child.wait().expect("failed to wait for mega-evme"); + let _stderr_bytes = stderr_drain.join().expect("stderr drain thread panicked"); assert_eq!( status.code(), Some(1), @@ -429,3 +445,44 @@ fn test_closed_stdout_during_json_batch_exits_one() { "process must not be signal-killed (e.g. SIGABRT from a double panic in the hook)" ); } + +/// A panic under `--json` with an open stdout ends the stream with the standard +/// error envelope (`code: 1`, `kind: "execution-error"`). +/// +/// The closed-stdout case only proves `exit(1)` when the hook cannot write. +/// This test pins the machine-readable object the hook prints when stdout is +/// still open. Triggered via the test-only `MEGA_EVME_INJECT_PANIC` hook (same +/// `test-utils` gate as the fixture pre-state inject), not via invalid input. +#[test] +fn test_panic_under_json_prints_execution_error_envelope() { + let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args(["--json"]) + .env("MEGA_EVME_INJECT_PANIC", "1") + .env("RUST_BACKTRACE", "0") + .output() + .expect("failed to run mega-evme"); + + assert_eq!( + output.status.code(), + Some(1), + "injected panic must exit 1.\nstderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let stdout = String::from_utf8(output.stdout).expect("stdout is utf-8"); + let values = common::json_values(&stdout); + let last = values + .last() + .unwrap_or_else(|| panic!("panic under --json must not leave stdout empty:\n{stdout}")); + assert!( + common::is_run_error(last), + "the final stdout line must be the run-level error object, got: {last}" + ); + assert_eq!(last["error"]["code"].as_u64(), Some(1)); + assert_eq!(last["error"]["kind"].as_str(), Some("execution-error")); + let message = last["error"]["message"].as_str().expect("error.message must be a string"); + assert!( + message.starts_with("panic: "), + "panic-hook message must keep the `panic: …` prefix: {message}" + ); +} From 153d4487cac443fed6f5124a715393158a7bf3a8 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:51:36 +0800 Subject: [PATCH 42/64] fix(mega-evme): skip caching JSON-RPC null results in capture transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A transient `"result": null` (e.g. eth_getTransactionByHash for a briefly- invisible transaction) must not be baked into --rpc.capture-file fixtures. Cmd::run persists captures even on failed runs, and offline replay would otherwise serve the null forever with no in-tool recovery (--rpc.clear-cache conflicts with capture mode). Trade-off: offline replay of a run that failed on a genuine null now fails with a cache-miss error naming the request instead of replaying the null answer — more honest, and re-capturing against a healthy endpoint fills the entry. --- .../src/common/provider/transport.rs | 204 +++++++++++++++++- bin/mega-evme/tests/common/mod.rs | 17 ++ bin/mega-evme/tests/provider.rs | 64 ++++++ 3 files changed, 277 insertions(+), 8 deletions(-) diff --git a/bin/mega-evme/src/common/provider/transport.rs b/bin/mega-evme/src/common/provider/transport.rs index 07739555..810a04b7 100644 --- a/bin/mega-evme/src/common/provider/transport.rs +++ b/bin/mega-evme/src/common/provider/transport.rs @@ -128,6 +128,15 @@ fn transport_cache_key(method: &str, params: Option<&serde_json::value::RawValue keccak256(format!("{method}\x00{params_str}")) } +/// Whether a success response carries a JSON `null` result. +/// +/// A null is never load-bearing for offline replay — every consumer fails its +/// target or run on it — so capture must not bake it into the fixture. The +/// correct offline representation of "no answer" is a cache miss. +fn is_null_success_result(resp: &alloy_json_rpc::Response) -> bool { + resp.payload.as_success().is_some_and(|raw| raw.get().trim() == "null") +} + /// Transport wrapper that records all JSON-RPC responses into a /// [`TransportCache`]. Used in capture mode: cache hits are served locally, /// misses are forwarded to the inner transport and the response is cached. @@ -174,11 +183,16 @@ where return fut; } - // Cache miss: forward to inner, cache successful responses only. - // JSON-RPC error bodies (e.g. transient rate-limit errors that the - // endpoint surfaces via `error` instead of an HTTP status) must not - // be baked into the fixture — otherwise replay would replay the - // error forever. + // Cache miss: forward to inner, cache non-null successful responses + // only. + // + // - JSON-RPC error bodies (e.g. transient rate-limit errors that the endpoint surfaces + // via `error` instead of an HTTP status) must not be baked into the fixture — + // otherwise replay would replay the error forever. + // - `"result": null` is likewise skipped: a transient null (e.g. + // eth_getTransactionByHash for a briefly-invisible tx) would otherwise freeze "not + // found" into the fixture with no in-tool recovery (`--rpc.clear-cache` conflicts + // with capture mode). Offline, a cache miss names the exact request instead. let cache = self.cache.clone(); let method = r.method().to_string(); let fut = self.inner.call(req); @@ -190,6 +204,11 @@ where method = %method, "Skipping cache for JSON-RPC error response", ); + } else if is_null_success_result(resp) { + tracing::warn!( + method = %method, + "Skipping cache for JSON-RPC null result", + ); } else if let Ok(serialized) = serde_json::to_string(resp) { cache.put(key, serialized); } @@ -260,18 +279,77 @@ impl tower::Service for ReplayTransport { #[cfg(test)] mod tests { - use std::path::PathBuf; + use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + task::{Context, Poll}, + }; + + use alloy_json_rpc::{Id, Response, ResponsePayload}; + use serde_json::value::to_raw_value; + use tower::Service; use super::*; + /// Inner transport that returns a fixed single response for every call. + #[derive(Clone)] + struct FixedResponseTransport { + response: Arc, + calls: Arc, + } + + impl FixedResponseTransport { + fn new(response: Response) -> Self { + Self { response: Arc::new(response), calls: Arc::new(AtomicUsize::new(0)) } + } + } + + impl Service for FixedResponseTransport { + type Response = ResponsePacket; + type Error = TransportError; + type Future = TransportFut<'static>; + + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _req: RequestPacket) -> Self::Future { + self.calls.fetch_add(1, Ordering::SeqCst); + let response = ResponsePacket::Single((*self.response).clone()); + Box::pin(async move { Ok(response) }) + } + } + + fn success_response(result_json: &str) -> Response { + Response { + id: Id::Number(1), + payload: ResponsePayload::Success( + to_raw_value(&serde_json::from_str::(result_json).unwrap()) + .unwrap(), + ), + } + } + + fn single_request(method: &'static str) -> RequestPacket { + RequestPacket::Single( + alloy_json_rpc::Request::new(method, Id::Number(1), ()) + .serialize() + .expect("serialize request"), + ) + } + /// The replay transport is always ready. /// Single-request cache misses return a descriptive Custom error (safe because /// the retry layer is never installed on the replay path). /// Batch misses return `BackendGone`. #[tokio::test] async fn test_replay_transport_cache_miss() { - use tower::Service; - let mut transport = ReplayTransport::new(PathBuf::from("/tmp/test.cache.json"), TransportCache::new()); @@ -292,4 +370,114 @@ mod tests { assert!(msg.contains("eth_blockNumber"), "error should include method: {msg}"); assert!(msg.contains("test.cache.json"), "error should include fixture path: {msg}"); } + + /// A success with a non-null result is cached and served on the next call. + #[tokio::test] + async fn test_caching_transport_caches_non_null_success() { + let inner = FixedResponseTransport::new(success_response(r#""0x42""#)); + let calls = Arc::clone(&inner.calls); + let cache = TransportCache::new(); + let mut transport = CachingTransport::new(inner, cache.clone()); + + let first = transport.call(single_request("eth_blockNumber")).await.expect("first call"); + assert!(matches!(first, ResponsePacket::Single(_))); + assert_eq!(cache.len(), 1, "non-null success must be cached"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + // Second call must hit the cache and not touch the inner transport. + let second = transport.call(single_request("eth_blockNumber")).await.expect("cache hit"); + assert!(matches!(second, ResponsePacket::Single(_))); + assert_eq!(calls.load(Ordering::SeqCst), 1, "cache hit must not call inner"); + } + + /// A success with `"result": null` is returned to the caller but never put + /// in the cache — absent entry stays absent. + #[tokio::test] + async fn test_caching_transport_skips_null_result() { + let inner = FixedResponseTransport::new(success_response("null")); + let calls = Arc::clone(&inner.calls); + let cache = TransportCache::new(); + let mut transport = CachingTransport::new(inner, cache.clone()); + + let response = transport.call(single_request("eth_getTransactionByHash")).await; + let response = response.expect("null result is still a transport success"); + match response { + ResponsePacket::Single(resp) => { + assert!(is_null_success_result(&resp), "caller must receive the null unchanged"); + } + other => panic!("expected single response, got {other:?}"), + } + assert_eq!(cache.len(), 0, "null result must not be cached"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + // A second call is still a miss and re-forwards — never a cached null. + let _ = transport.call(single_request("eth_getTransactionByHash")).await; + assert_eq!(cache.len(), 0, "repeated nulls must still leave the cache empty"); + assert_eq!(calls.load(Ordering::SeqCst), 2, "null must not create a cache hit"); + } + + /// A null response must never reach `put`, so an existing non-null entry + /// for the same key is preserved even if the live endpoint later returns null. + /// + /// The production path usually serves the existing entry as a hit before the + /// network is consulted; this test forces the miss→null path after a prior + /// put by clearing nothing and re-issuing through a transport whose inner + /// now returns null, after first caching a non-null under the same key via + /// a separate `CachingTransport` sharing the cache. + #[tokio::test] + async fn test_caching_transport_null_does_not_overwrite_existing_entry() { + let cache = TransportCache::new(); + + // Seed a non-null entry under eth_blockNumber. + let seed_inner = FixedResponseTransport::new(success_response(r#""0x42""#)); + let mut seeder = CachingTransport::new(seed_inner, cache.clone()); + seeder.call(single_request("eth_blockNumber")).await.expect("seed"); + assert_eq!(cache.len(), 1); + + // A second CachingTransport sharing the same cache: on a hit the null + // never reaches put. Serve the hit and assert the entry is unchanged. + let null_inner = FixedResponseTransport::new(success_response("null")); + let null_calls = Arc::clone(&null_inner.calls); + let mut transport = CachingTransport::new(null_inner, cache.clone()); + let served = transport.call(single_request("eth_blockNumber")).await.expect("hit"); + match served { + ResponsePacket::Single(resp) => { + assert!( + !is_null_success_result(&resp), + "existing non-null entry must be served, not overwritten by a live null", + ); + } + other => panic!("expected single response, got {other:?}"), + } + assert_eq!(null_calls.load(Ordering::SeqCst), 0, "cache hit must not call inner"); + assert_eq!(cache.len(), 1, "null path must never drop the existing entry"); + + // Inspect the cached payload is still the non-null success. + let key = transport_cache_key("eth_blockNumber", None); + let cached = cache.get(&key).expect("entry present"); + let cached_resp: Response = serde_json::from_str(&cached).expect("cached JSON"); + assert!(!is_null_success_result(&cached_resp)); + } + + /// JSON-RPC error responses are still skipped (regression pin). + #[tokio::test] + async fn test_caching_transport_skips_error_response() { + let error = Response { id: Id::Number(1), payload: ResponsePayload::internal_error() }; + let inner = FixedResponseTransport::new(error); + let cache = TransportCache::new(); + let mut transport = CachingTransport::new(inner, cache.clone()); + + let _ = transport.call(single_request("eth_blockNumber")).await; + assert_eq!(cache.len(), 0, "error responses must not be cached"); + } + + #[test] + fn test_is_null_success_result_detects_null_only() { + assert!(is_null_success_result(&success_response("null"))); + assert!(!is_null_success_result(&success_response(r#""0x1""#))); + assert!(!is_null_success_result(&success_response("0"))); + assert!(!is_null_success_result(&success_response("{}"))); + let error = Response { id: Id::Number(1), payload: ResponsePayload::internal_error() }; + assert!(!is_null_success_result(&error)); + } } diff --git a/bin/mega-evme/tests/common/mod.rs b/bin/mega-evme/tests/common/mod.rs index 18712500..46fe29ef 100644 --- a/bin/mega-evme/tests/common/mod.rs +++ b/bin/mega-evme/tests/common/mod.rs @@ -142,6 +142,23 @@ impl MockRpcServer { .await; } + /// Mount an unbounded mock that always returns a successful JSON-RPC body + /// with `"result": null` (HTTP 200). Models a transient not-found answer + /// (e.g. `eth_getTransactionByHash` for a briefly-invisible transaction) + /// that capture must not bake into the fixture. + pub(crate) async fn respond_jsonrpc_null_result(&self, priority: u8) { + let body = serde_json::json!({ + "jsonrpc": "2.0", + "id": 0, + "result": null, + }); + Mock::given(matchers::method("POST")) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .with_priority(priority) + .mount(&self.server) + .await; + } + /// Mount an unbounded mock that answers every JSON-RPC request for /// `method` with the given hex `result`, regardless of params. pub(crate) async fn respond_method_result(&self, method: &str, hex_result: &str, priority: u8) { diff --git a/bin/mega-evme/tests/provider.rs b/bin/mega-evme/tests/provider.rs index 3b3c8b85..ea712f94 100644 --- a/bin/mega-evme/tests/provider.rs +++ b/bin/mega-evme/tests/provider.rs @@ -929,6 +929,70 @@ async fn test_capture_does_not_cache_jsonrpc_error_response() { assert!(cached.get("error").is_none(), "cached entry must not be an error response"); } +/// A success with `"result": null` must be served to the caller but must not +/// be baked into the capture fixture. Offline replay of that fixture then +/// fails with a cache-miss error naming the request, not a silent not-found +/// from a frozen null. +#[tokio::test(flavor = "multi_thread")] +async fn test_capture_does_not_cache_null_result_and_offline_misses() { + let server = MockRpcServer::start().await; + // eth_chainId must succeed so build_capture_provider can complete. + server.respond_eth_chain_id(4326, 1).await; + // Any other call resolves to a null result at HTTP 200. + server.respond_jsonrpc_null_result(2).await; + + let dir = tempdir().expect("tempdir"); + let cache_file = dir.path().join("null.cache.json"); + + let capture_args = RpcArgs::parse_from([ + "mega-evme", + "--rpc", + &server.uri(), + "--rpc.capture-file", + cache_file.to_str().unwrap(), + ]); + let output = capture_args.build_capture_provider().await.expect("capture build"); + + // Null is a transport-level success; the provider fails deserializing it + // into a block number. Capture must still observe the response and skip it. + let _ = output.provider.get_block_number().await; + output.cache_store.persist().expect("persist should still succeed"); + + // The envelope must hold only eth_chainId — no entry for the null call. + let raw = std::fs::read_to_string(&cache_file).expect("read envelope"); + let envelope: serde_json::Value = serde_json::from_str(&raw).expect("parse envelope"); + let entries = envelope["cache"].as_array().expect("cache is a JSON array"); + assert_eq!( + entries.len(), + 1, + "only eth_chainId should be cached; null results must be skipped. entries = {entries:#?}", + ); + let cached: serde_json::Value = + serde_json::from_str(entries[0]["value"].as_str().expect("value is a JSON string")) + .expect("cached response is valid JSON"); + assert!( + cached.get("result").is_some_and(|r| !r.is_null()), + "sole cached entry must be a non-null success, got {cached}", + ); + + // Offline replay: the null was never captured, so the same request is a + // cache miss that names the method — not a silent null / not_found. + let replay_args = + RpcArgs::parse_from(["mega-evme", "--rpc.replay-file", cache_file.to_str().unwrap()]); + let replay = replay_args.build_replay_provider().await.expect("replay build"); + let err = replay + .provider + .get_block_number() + .await + .expect_err("missing null entry must surface as cache miss"); + let msg = format!("{err}"); + assert!(msg.contains("cache miss"), "offline error must say 'cache miss', got: {msg}",); + assert!( + msg.contains("eth_blockNumber"), + "offline error must name the missing method, got: {msg}", + ); +} + /// Cross-chain contamination guard: an existing envelope claiming chain X /// combined with an endpoint returning chain Y must hard-error, not silently /// mix responses from two chains. From d07f08f980a209eba40e10da8959f53304296044 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 00:56:56 +0800 Subject: [PATCH 43/64] fix(mega-evme): make every cache-file write take the sidecar lock `cache merge` wrote its output with a bare atomic rename: no lock, no re-read. Run against a path a live replay is persisting to, whichever rename landed last silently dropped the other side's entries. It now takes the same exclusive sidecar lock clean-exit persist takes, folds whatever the output holds at that moment into the union as one more input (the named inputs winning on collision), writes, and releases. Lock acquisition failure no longer degrades to an unlocked write, which was the very lost-update race the lock exists to prevent. Provider persist skips instead (best-effort artifact, warn names the file and says the entries were not saved); capture persist and `cache merge` hard-error. The flock helper moves to `cache/lock.rs` so both writers share one definition, and an existing output that parses but cannot be folded (the other shape, another chain id or version) is now refused by both shapes rather than overwritten. Mutual exclusion is covered by a two-process test: it holds the sidecar lock, spawns the real binary, shows the merge makes no progress while held, writes a concurrent writer's entries under that lock, and asserts both sides survive after release. --- bin/mega-evme/AGENTS.md | 3 +- bin/mega-evme/src/cache/lock.rs | 90 +++++ bin/mega-evme/src/cache/merge.rs | 218 +++++++++++- bin/mega-evme/src/cache/mod.rs | 324 +++++++++++++++++- .../src/common/provider/cache_store.rs | 174 +++++++--- bin/mega-evme/tests/cache_merge_lock.rs | 207 +++++++++++ docs/mega-evme/commands/cache.md | 35 +- .../configuration/state-management.md | 9 +- 8 files changed, 973 insertions(+), 87 deletions(-) create mode 100644 bin/mega-evme/src/cache/lock.rs create mode 100644 bin/mega-evme/tests/cache_merge_lock.rs diff --git a/bin/mega-evme/AGENTS.md b/bin/mega-evme/AGENTS.md index f7485d60..6f010ef0 100644 --- a/bin/mega-evme/AGENTS.md +++ b/bin/mega-evme/AGENTS.md @@ -10,7 +10,7 @@ CLI toolbox for direct MegaEVM execution (`run`, `tx`, `replay`, `cache`) with o - `src/run/`: bytecode execution command. - `src/tx/`: full transaction execution command with raw-tx override support. - `src/replay/`: RPC-backed historical transaction replay through block executor, plus the batch driver. -- `src/cache/`: cache-file merge utilities (provider-cache and capture-envelope JSON shapes) backing the `cache merge` subcommand and the lock-protected merge-on-persist. +- `src/cache/`: cache-file merge utilities (provider-cache and capture-envelope JSON shapes) backing the `cache merge` subcommand and the lock-protected merge-on-persist, plus the sidecar advisory lock every cache-file writer takes. ## KEY PATTERNS - Shared argument groups are flattened from `run` argument structs into sibling commands. @@ -33,4 +33,5 @@ CLI toolbox for direct MegaEVM execution (`run`, `tx`, `replay`, `cache`) with o - Change replay hardfork/spec selection: `src/replay/{cmd.rs,hardforks.rs}`. - Change receipt/summary formatting: `src/common/outcome.rs` and printer helpers. - Change cache merge behavior (CLI or merge-on-persist): `src/cache/{mod.rs,merge.rs}`. +- Change how cache files are locked against concurrent writers: `src/cache/lock.rs` — the one place a cache-file write may acquire its lock, and every caller must fail closed when it cannot. - Change process exit classification: `src/common/exit.rs` — the single exit site for command results. diff --git a/bin/mega-evme/src/cache/lock.rs b/bin/mega-evme/src/cache/lock.rs new file mode 100644 index 00000000..1a4310d6 --- /dev/null +++ b/bin/mega-evme/src/cache/lock.rs @@ -0,0 +1,90 @@ +//! Advisory locking for cache files shared by concurrent processes. +//! +//! Every writer of a cache file — clean-exit persist and the offline +//! `cache merge` subcommand alike — takes the exclusive lock on that file's +//! sidecar before it re-reads the file, merges, and renames the result into +//! place. A writer that skips the lock can only be correct by luck: two +//! read-modify-write cycles that interleave lose whichever side renamed first. +//! +//! The lock lives on a sidecar (`.lock`) rather than the cache file +//! itself because the target is replaced by rename on every write, and a lock +//! held on the replaced inode protects nothing. + +use std::{ + fs, + fs::OpenOptions, + path::{Path, PathBuf}, +}; + +/// Path of the advisory lock sidecar for `target` (`.lock`). +pub(crate) fn lock_sidecar_path(target: &Path) -> PathBuf { + let mut os = target.as_os_str().to_owned(); + os.push(".lock"); + PathBuf::from(os) +} + +/// RAII exclusive lock on the sidecar file for a cache target. +/// +/// The lock is released when this guard is dropped (file handle closed). +/// The sidecar file itself is left on disk. +#[derive(Debug)] +pub(crate) struct ExclusiveFileLock { + _file: fs::File, +} + +/// Acquire an exclusive advisory lock on `.lock`, blocking until held. +/// +/// The sidecar is created if missing and left in place after unlock. +/// +/// Callers must fail closed on `Err`: an unlocked write is exactly the +/// lost-update race the lock exists to prevent, so a failed acquisition means +/// "do not write", never "write anyway". +pub(crate) fn acquire_exclusive_lock(target: &Path) -> std::io::Result { + let lock_path = lock_sidecar_path(target); + if let Some(parent) = lock_path.parent() { + fs::create_dir_all(parent)?; + } + // truncate(false): the sidecar is only a flock target; keep any existing bytes. + let file = + OpenOptions::new().create(true).read(true).write(true).truncate(false).open(&lock_path)?; + // Blocking exclusive advisory lock. + file.lock()?; + Ok(ExclusiveFileLock { _file: file }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_lock_sidecar_path_suffix() { + let p = Path::new("/tmp/rpc-cache-1.json"); + assert_eq!(lock_sidecar_path(p), PathBuf::from("/tmp/rpc-cache-1.json.lock")); + } + + /// The sidecar is created on acquisition and left in place after unlock. + #[test] + fn test_acquire_exclusive_lock_creates_and_keeps_sidecar() { + let dir = tempfile::tempdir().expect("tempdir"); + let target = dir.path().join("rpc-cache-1.json"); + let sidecar = lock_sidecar_path(&target); + assert!(!sidecar.exists()); + + let guard = acquire_exclusive_lock(&target).expect("acquire"); + assert!(sidecar.exists(), "sidecar created while held"); + drop(guard); + assert!(sidecar.exists(), "sidecar left in place after unlock"); + } + + /// An un-openable sidecar path surfaces as an error rather than a silent + /// unlocked write. + #[test] + fn test_acquire_exclusive_lock_reports_unopenable_sidecar() { + let dir = tempfile::tempdir().expect("tempdir"); + let target = dir.path().join("rpc-cache-1.json"); + // A directory in the sidecar's place cannot be opened as a file. + fs::create_dir(lock_sidecar_path(&target)).expect("occupy sidecar path"); + + acquire_exclusive_lock(&target).expect_err("un-openable sidecar must not silently succeed"); + } +} diff --git a/bin/mega-evme/src/cache/merge.rs b/bin/mega-evme/src/cache/merge.rs index cb8f1369..2efe4749 100644 --- a/bin/mega-evme/src/cache/merge.rs +++ b/bin/mega-evme/src/cache/merge.rs @@ -79,13 +79,6 @@ pub(crate) fn canonicalize_bucket_capacities(caps: &[(u32, u64)]) -> Vec<(u32, u map.into_iter().collect() } -/// Path of the advisory lock sidecar for `target` (`.lock`). -pub(crate) fn lock_sidecar_path(target: &Path) -> PathBuf { - let mut os = target.as_os_str().to_owned(); - os.push(".lock"); - PathBuf::from(os) -} - /// Parse `rpc-cache-{chain_id}.json` from a path's file name. /// /// Returns `None` when the name does not match the per-chain provider-cache @@ -204,6 +197,63 @@ pub(crate) fn read_provider_cache(path: &Path) -> Result> { } } +/// Classification of the file already at a provider-shape merge's output. +/// +/// The counterpart of [`EnvelopeReread`] for the other shape, and typed for the +/// same reason: content that cannot be parsed at all is safe to replace, while +/// a file that parses into something this merge cannot fold is a file the merge +/// must not silently destroy. +#[derive(Debug)] +pub(crate) enum ProviderReread { + /// The output holds provider-cache entries (or does not exist yet). + Ok(Vec), + /// Corrupt, unreadable, or undecodable content — safe to warn and replace. + Degradable(String), + /// Readable, but not a provider cache: refusing beats overwriting. + Hard(EvmeError), +} + +/// Re-read the existing merge output for a provider-shape merge. +pub(crate) fn reread_provider_cache_for_merge(path: &Path) -> ProviderReread { + if !path.exists() { + return ProviderReread::Ok(Vec::new()); + } + let content = match fs::read_to_string(path) { + Ok(c) => c, + Err(e) => { + return ProviderReread::Degradable(format!( + "Failed to read cache file {}: {e}", + path.display() + )); + } + }; + let value: serde_json::Value = match serde_json::from_str(&content) { + Ok(v) => v, + Err(e) => { + return ProviderReread::Degradable(format!( + "Failed to parse cache file {}: {e}", + path.display() + )); + } + }; + match detect_shape(&value, path) { + Ok(CacheShape::Provider) => match serde_json::from_value(value) { + Ok(entries) => ProviderReread::Ok(entries), + Err(e) => ProviderReread::Degradable(format!( + "Failed to decode provider-cache entries in {}: {e}", + path.display() + )), + }, + Ok(CacheShape::Envelope) => ProviderReread::Hard(EvmeError::InvalidInput(format!( + "Expected provider-cache array in '{}', found capture envelope", + path.display() + ))), + // An unrecognized shape is still structured JSON somebody wrote: it is + // not this merge's output to overwrite. + Err(e) => ProviderReread::Hard(e), + } +} + /// Classification of an envelope re-read during concurrent persist merge. /// /// Typed so hard identity failures (version / `chain_id` / wrong shape) are not @@ -448,6 +498,67 @@ pub(crate) fn merge_envelopes_cli(docs: &[(PathBuf, EnvelopeDoc)]) -> Result Result { + if on_disk.version != merged_inputs.version { + return Err(EvmeError::InvalidInput(format!( + "Envelope version mismatch with existing output '{}': output has version {}, \ + inputs have version {}", + output.display(), + on_disk.version, + merged_inputs.version, + ))); + } + if on_disk.chain_id != merged_inputs.chain_id { + return Err(EvmeError::InvalidInput(format!( + "Envelope chain_id mismatch with existing output '{}': output has chain_id {}, \ + inputs have chain_id {}", + output.display(), + on_disk.chain_id, + merged_inputs.chain_id, + ))); + } + + let external_env = match (&on_disk.external_env, &merged_inputs.external_env) { + (Some(disk), Some(ours)) => { + let (disk_c, ours_c) = (disk.canonicalized(), ours.canonicalized()); + if disk_c != ours_c { + return Err(EvmeError::InvalidInput(format!( + "Conflicting external_env snapshots with existing output '{}': \ + output {disk_c:?}, inputs {ours_c:?}", + output.display(), + ))); + } + Some(ours_c) + } + (Some(disk), None) => Some(disk.canonicalized()), + (None, Some(ours)) => Some(ours.canonicalized()), + (None, None) => None, + }; + + Ok(EnvelopeDoc { + version: merged_inputs.version, + chain_id: merged_inputs.chain_id, + cache: merge_kv_entries(on_disk.cache, merged_inputs.cache), + external_env, + }) +} + /// Atomically write `entries` as a provider-cache JSON array to `path`. pub(crate) fn write_provider_cache_atomic(path: &Path, entries: &[CacheKv]) -> Result<()> { let dir = path.parent().unwrap_or_else(|| Path::new(".")); @@ -885,10 +996,97 @@ mod tests { ); } + /// Folding the existing output unions its entries in, with the named inputs + /// winning on key collision. + #[test] + fn test_fold_output_envelope_unions_with_inputs_winning() { + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "output"), kv(9, "concurrent")], + external_env: None, + }; + let inputs = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![kv(1, "inputs"), kv(2, "inputs")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(2, 20), (1, 10)] }), + }; + let folded = + fold_output_envelope(Path::new("out.json"), on_disk, inputs).expect("fold output"); + assert_eq!(folded.cache, vec![kv(1, "inputs"), kv(2, "inputs"), kv(9, "concurrent")]); + // The written snapshot is canonical. + assert_eq!( + folded.external_env, + Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10), (2, 20)] }) + ); + } + + /// A snapshot on the existing output that disagrees with the inputs is a + /// conflict, exactly as it is between two inputs. #[test] - fn test_lock_sidecar_path_suffix() { - let p = Path::new("/tmp/rpc-cache-1.json"); - assert_eq!(lock_sidecar_path(p), PathBuf::from("/tmp/rpc-cache-1.json.lock")); + fn test_fold_output_envelope_rejects_conflicting_external_env() { + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 42)] }), + }; + let inputs = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 99)] }), + }; + let err = + fold_output_envelope(Path::new("out.json"), on_disk, inputs).expect_err("conflict"); + let msg = err.to_string(); + assert!(msg.contains("external_env"), "msg={msg}"); + assert!(msg.contains("out.json"), "msg={msg}"); + assert!(msg.contains("42") && msg.contains("99"), "msg={msg}"); + } + + /// Same effective capacities in different order are not a conflict. + #[test] + fn test_fold_output_envelope_order_insensitive_external_env() { + let on_disk = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(2, 20), (1, 10)] }), + }; + let inputs = EnvelopeDoc { + version: 1, + chain_id: 7, + cache: vec![], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10), (2, 20)] }), + }; + let folded = fold_output_envelope(Path::new("out.json"), on_disk, inputs) + .expect("order-only difference must not conflict"); + assert_eq!( + folded.external_env, + Some(ExternalEnvDoc { bucket_capacities: vec![(1, 10), (2, 20)] }) + ); + } + + /// Identity failures against the existing output name that file. + #[test] + fn test_fold_output_envelope_rejects_identity_mismatch() { + let inputs = EnvelopeDoc { version: 1, chain_id: 7, cache: vec![], external_env: None }; + + let other_chain = + EnvelopeDoc { version: 1, chain_id: 8, cache: vec![], external_env: None }; + let err = fold_output_envelope(Path::new("out.json"), other_chain, inputs.clone()) + .expect_err("chain_id mismatch"); + let msg = err.to_string(); + assert!(msg.contains("chain_id") && msg.contains("out.json"), "msg={msg}"); + + let other_version = + EnvelopeDoc { version: 2, chain_id: 7, cache: vec![], external_env: None }; + let err = fold_output_envelope(Path::new("out.json"), other_version, inputs) + .expect_err("version mismatch"); + let msg = err.to_string(); + assert!(msg.contains("version") && msg.contains("out.json"), "msg={msg}"); } /// Filename-derived chain id for the standard provider-cache naming scheme. diff --git a/bin/mega-evme/src/cache/mod.rs b/bin/mega-evme/src/cache/mod.rs index 219fb5f2..ff90eec0 100644 --- a/bin/mega-evme/src/cache/mod.rs +++ b/bin/mega-evme/src/cache/mod.rs @@ -3,6 +3,7 @@ //! Currently ships `cache merge` for consolidating per-worker provider-cache //! files or capture envelopes after historical sharded campaigns. +mod lock; mod merge; use std::path::PathBuf; @@ -11,15 +12,18 @@ use clap::{Parser, Subcommand}; use crate::common::{EvmeError, Result}; +pub(crate) use lock::{acquire_exclusive_lock, lock_sidecar_path}; pub(crate) use merge::{ - lock_sidecar_path, merge_envelope_for_persist, merge_provider_entries_capped, - merge_provider_lists, parse_rpc_cache_filename_chain_id, read_provider_cache, - reread_envelope_for_merge, write_bytes_atomic, write_envelope_atomic, - write_provider_cache_atomic, CacheKv, EnvelopeDoc, EnvelopeReread, ExternalEnvDoc, - ENVELOPE_VERSION, + merge_envelope_for_persist, merge_provider_entries_capped, merge_provider_lists, + parse_rpc_cache_filename_chain_id, read_provider_cache, reread_envelope_for_merge, + write_bytes_atomic, write_envelope_atomic, write_provider_cache_atomic, CacheKv, EnvelopeDoc, + EnvelopeReread, ExternalEnvDoc, ENVELOPE_VERSION, }; -use merge::{load_cache_file, merge_envelopes_cli, CacheShape, LoadedCache}; +use merge::{ + fold_output_envelope, load_cache_file, merge_envelopes_cli, reread_provider_cache_for_merge, + CacheShape, LoadedCache, ProviderReread, +}; use tracing::warn; /// `mega-evme cache` — offline cache-file utilities. @@ -126,24 +130,66 @@ impl MergeArgs { let total_in: usize = loaded.iter().map(|(_, _, d)| d.entry_count()).sum(); let input_count = loaded.len(); + if first_shape == CacheShape::Provider { + // Provider-cache files carry chain identity only in the + // `rpc-cache-{id}.json` filename. Reject merges that would + // union different chains; warn when a path cannot be checked. + // Checked before the output is locked so a doomed merge leaves no + // sidecar behind. + check_provider_cache_chain_identity( + loaded + .iter() + .map(|(p, _, _)| p.as_path()) + .chain(std::iter::once(self.output.as_path())), + )?; + } + + // The output is a shared file: a live run may be persisting to the same + // path under the same sidecar lock. Take that lock and hold it across + // read-merge-rename, so neither side's entries are lost to whichever + // rename lands last. + let _output_lock = acquire_exclusive_lock(&self.output).map_err(|e| { + EvmeError::InvalidInput(format!( + "Failed to acquire the cache lock {} for output '{}': {e}. \ + Refusing to merge without it: an unlocked write would silently drop \ + entries written by a concurrent process.", + lock_sidecar_path(&self.output).display(), + self.output.display(), + )) + })?; + + // Entries the output file already held when the lock was granted. + let mut folded_in = 0usize; + let unique_out = match first_shape { CacheShape::Provider => { - // Provider-cache files carry chain identity only in the - // `rpc-cache-{id}.json` filename. Reject merges that would - // union different chains; warn when a path cannot be checked. - check_provider_cache_chain_identity( - loaded - .iter() - .map(|(p, _, _)| p.as_path()) - .chain(std::iter::once(self.output.as_path())), - )?; - let mut acc = Vec::new(); for (_, _, data) in loaded { let LoadedCache::Provider(entries) = data else { unreachable!() }; // Later inputs win on collision. acc = merge_provider_lists(acc, entries); } + + // Whatever is at the output now joins the union as one more + // input: a concurrent writer may have landed entries there + // while this merge waited for the lock. + let on_disk = match reread_provider_cache_for_merge(&self.output) { + ProviderReread::Ok(entries) => entries, + ProviderReread::Hard(err) => return Err(err), + ProviderReread::Degradable(msg) => { + warn!( + path = %self.output.display(), + error = %msg, + "Failed to read the existing merge output; \ + it will be replaced by the merged inputs", + ); + Vec::new() + } + }; + folded_in = on_disk.len(); + // The named inputs win over the output's prior entries. + let acc = merge_provider_lists(on_disk, acc); + let unique = acc.len(); write_provider_cache_atomic(&self.output, &acc)?; unique @@ -157,14 +203,46 @@ impl MergeArgs { }) .collect(); let merged = merge_envelopes_cli(&docs)?; + + let merged = if self.output.exists() { + // Typed classification: identity/schema failures must not be + // papered over by overwriting the file we cannot read. + match reread_envelope_for_merge(&self.output) { + EnvelopeReread::Ok(on_disk) => { + folded_in = on_disk.cache.len(); + fold_output_envelope(&self.output, on_disk, merged)? + } + EnvelopeReread::Hard(err) => return Err(err), + EnvelopeReread::Degradable(msg) => { + warn!( + path = %self.output.display(), + error = %msg, + "Failed to read the existing merge output; \ + it will be replaced by the merged inputs", + ); + merged + } + } + } else { + merged + }; + let unique = merged.cache.len(); write_envelope_atomic(&self.output, &merged)?; unique } }; + // Name the folded-in entries so the arithmetic still adds up when the + // output already held some. + let folded = if folded_in > 0 { + format!(" + {folded_in} already in the output") + } else { + String::new() + }; println!( - "Merged {input_count} inputs ({total_in} entries in) → {unique_out} unique entries out" + "Merged {input_count} inputs ({total_in} entries in{folded}) \ + → {unique_out} unique entries out" ); Ok(()) } @@ -349,6 +427,218 @@ mod tests { assert!(msg.contains("chain 1") && msg.contains("chain 4326"), "msg={msg}"); } + /// A provider-shaped output already on disk joins the union as one more + /// input: entries a concurrent writer left there survive the merge, and the + /// named inputs win where the keys collide. + #[test] + fn test_cache_merge_folds_the_existing_provider_output() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let b = dir.path().join("b.json"); + let out = dir.path().join("out.json"); + + write(&a, &serde_json::to_string(&vec![kv(1, "from-a")]).unwrap()); + write(&b, &serde_json::to_string(&vec![kv(2, "from-b")]).unwrap()); + // The output already holds a sibling's entry plus a stale copy of key 2. + write( + &out, + &serde_json::to_string(&vec![kv(2, "from-output"), kv(9, "concurrent")]).unwrap(), + ); + + MergeArgs { inputs: vec![a, b], output: out.clone() }.run().expect("merge"); + + let merged: Vec = + serde_json::from_str(&fs::read_to_string(&out).unwrap()).unwrap(); + assert_eq!(merged, vec![kv(1, "from-a"), kv(2, "from-b"), kv(9, "concurrent")]); + } + + /// The envelope shape folds its existing output the same way. + #[test] + fn test_cache_merge_folds_the_existing_envelope_output() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + let env_a = EnvelopeDoc { + version: 1, + chain_id: 4326, + cache: vec![kv(1, "from-a"), kv(2, "from-a")], + external_env: None, + }; + let existing = EnvelopeDoc { + version: 1, + chain_id: 4326, + cache: vec![kv(2, "from-output"), kv(9, "concurrent")], + external_env: Some(ExternalEnvDoc { bucket_capacities: vec![(1, 100)] }), + }; + write(&a, &serde_json::to_string_pretty(&env_a).unwrap()); + write(&out, &serde_json::to_string_pretty(&existing).unwrap()); + + MergeArgs { inputs: vec![a], output: out.clone() }.run().expect("merge"); + + let merged: EnvelopeDoc = serde_json::from_str(&fs::read_to_string(&out).unwrap()).unwrap(); + assert_eq!(merged.cache, vec![kv(1, "from-a"), kv(2, "from-a"), kv(9, "concurrent")]); + // The output's snapshot is preserved when the inputs carry none. + assert_eq!(merged.external_env, Some(ExternalEnvDoc { bucket_capacities: vec![(1, 100)] })); + } + + /// An unreadable provider output degrades to the merged inputs (warned), + /// matching the persist path's handling of a corrupt on-disk file. + #[test] + fn test_cache_merge_replaces_corrupt_provider_output() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + write(&a, &serde_json::to_string(&vec![kv(1, "from-a")]).unwrap()); + write(&out, "not-json{{{"); + + MergeArgs { inputs: vec![a], output: out.clone() }.run().expect("merge"); + + let merged: Vec = + serde_json::from_str(&fs::read_to_string(&out).unwrap()).unwrap(); + assert_eq!(merged, vec![kv(1, "from-a")]); + } + + /// An existing envelope output on another chain is an identity failure, not + /// something to overwrite. + #[test] + fn test_cache_merge_rejects_existing_envelope_output_on_another_chain() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + write( + &a, + &serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(1, "a")], + external_env: None, + }) + .unwrap(), + ); + let existing = serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 2, + cache: vec![kv(9, "concurrent")], + external_env: None, + }) + .unwrap(); + write(&out, &existing); + + let err = MergeArgs { inputs: vec![a], output: out.clone() }.run().unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("chain_id"), "msg={msg}"); + assert!(msg.contains("out.json"), "the output must be named: msg={msg}"); + assert_eq!(fs::read_to_string(&out).unwrap(), existing, "output left untouched"); + } + + /// A capture envelope sitting at a provider merge's output is a hard error + /// for the same reason as the mirrored case below: a mistyped `--output` + /// must not destroy a file the merge cannot fold. + #[test] + fn test_cache_merge_rejects_wrong_shaped_existing_provider_output() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + write(&a, &serde_json::to_string(&vec![kv(1, "from-a")]).unwrap()); + let existing = serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(9, "concurrent")], + external_env: None, + }) + .unwrap(); + write(&out, &existing); + + let err = MergeArgs { inputs: vec![a], output: out.clone() }.run().unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("envelope"), "msg={msg}"); + assert_eq!(fs::read_to_string(&out).unwrap(), existing, "output left untouched"); + } + + /// A provider-shaped file sitting at an envelope merge's output is a hard + /// error: it cannot be folded, and overwriting it would destroy it. + #[test] + fn test_cache_merge_rejects_wrong_shaped_existing_output() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + write( + &a, + &serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(1, "a")], + external_env: None, + }) + .unwrap(), + ); + let existing = serde_json::to_string(&vec![kv(9, "concurrent")]).unwrap(); + write(&out, &existing); + + let err = MergeArgs { inputs: vec![a], output: out.clone() }.run().unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("envelope"), "msg={msg}"); + assert_eq!(fs::read_to_string(&out).unwrap(), existing, "output left untouched"); + } + + /// Provider merge fails closed when the output lock cannot be acquired: the + /// existing output is left exactly as it was. + #[test] + fn test_cache_merge_provider_fails_closed_when_the_output_lock_is_unavailable() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + write(&a, &serde_json::to_string(&vec![kv(1, "from-a")]).unwrap()); + let existing = serde_json::to_string(&vec![kv(9, "concurrent")]).unwrap(); + write(&out, &existing); + // A directory in the sidecar's place makes the lock un-acquirable. + fs::create_dir(lock_sidecar_path(&out)).expect("occupy sidecar path"); + + let err = MergeArgs { inputs: vec![a], output: out.clone() }.run().unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("lock"), "msg={msg}"); + assert!(msg.contains("out.json.lock"), "the lock path must be named: msg={msg}"); + assert_eq!(fs::read_to_string(&out).unwrap(), existing, "no unlocked write happened"); + } + + /// Envelope merge fails closed on the same condition. + #[test] + fn test_cache_merge_envelope_fails_closed_when_the_output_lock_is_unavailable() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + write( + &a, + &serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(1, "a")], + external_env: None, + }) + .unwrap(), + ); + let existing = serde_json::to_string_pretty(&EnvelopeDoc { + version: 1, + chain_id: 1, + cache: vec![kv(9, "concurrent")], + external_env: None, + }) + .unwrap(); + write(&out, &existing); + fs::create_dir(lock_sidecar_path(&out)).expect("occupy sidecar path"); + + let err = MergeArgs { inputs: vec![a], output: out.clone() }.run().unwrap_err(); + assert!(err.to_string().contains("lock"), "msg={err}"); + assert_eq!(fs::read_to_string(&out).unwrap(), existing, "no unlocked write happened"); + } + /// Unit-testable predicate: non-matching filename cannot supply chain identity. #[test] fn test_provider_cache_chain_identity_non_matching_filename_is_none() { diff --git a/bin/mega-evme/src/common/provider/cache_store.rs b/bin/mega-evme/src/common/provider/cache_store.rs index 4c137be4..3235be1b 100644 --- a/bin/mega-evme/src/common/provider/cache_store.rs +++ b/bin/mega-evme/src/common/provider/cache_store.rs @@ -17,10 +17,14 @@ //! therefore share one `--rpc.cache-dir` without losing each other's entries. //! The lock sidecar is left in place after the process exits (the flock is released //! when the lock file handle is closed). +//! +//! Persist fails closed on the lock: if the lock cannot be acquired, nothing is +//! written. Writing unlocked would reintroduce exactly the lost-update race the +//! lock exists to prevent, and it would do so silently — a sibling process's +//! entries would vanish under our rename. use std::{ fmt, fs, - fs::OpenOptions, path::{Path, PathBuf}, }; @@ -31,9 +35,10 @@ use tracing::{info, warn}; use super::transport::TransportCache; use crate::{ cache::{ - lock_sidecar_path, merge_envelope_for_persist, merge_provider_entries_capped, - read_provider_cache, reread_envelope_for_merge, write_bytes_atomic, write_envelope_atomic, - CacheKv, EnvelopeDoc, EnvelopeReread, ExternalEnvDoc, ENVELOPE_VERSION, + acquire_exclusive_lock, lock_sidecar_path, merge_envelope_for_persist, + merge_provider_entries_capped, read_provider_cache, reread_envelope_for_merge, + write_bytes_atomic, write_envelope_atomic, CacheKv, EnvelopeDoc, EnvelopeReread, + ExternalEnvDoc, ENVELOPE_VERSION, }, common::{EvmeError, Result}, }; @@ -234,50 +239,26 @@ impl fmt::Debug for RpcCacheStore { } } -/// RAII exclusive lock on the sidecar file for `target`. -/// -/// The lock is released when this guard is dropped (file handle closed). -/// The sidecar file itself is left on disk. -struct ExclusiveFileLock { - _file: fs::File, -} - -/// Acquire an exclusive advisory lock on `.lock`, blocking until held. -/// -/// The sidecar is created if missing and left in place after unlock. -fn acquire_exclusive_lock(target: &Path) -> std::io::Result { - let lock_path = lock_sidecar_path(target); - if let Some(parent) = lock_path.parent() { - fs::create_dir_all(parent)?; - } - // truncate(false): the sidecar is only a flock target; keep any existing bytes. - let file = - OpenOptions::new().create(true).read(true).write(true).truncate(false).open(&lock_path)?; - // Blocking exclusive advisory lock. - file.lock()?; - Ok(ExclusiveFileLock { _file: file }) -} - /// Atomically persist `cache` to `target` via lock + re-read-merge + temp rename. /// /// All error paths include `target` in the returned [`std::io::Error`] so the /// warn-log in [`RpcCacheStore::persist`] identifies which file failed. /// -/// Lock acquisition failure degrades to an unlocked write with a `warn!`. +/// Lock acquisition failure aborts the persist: the provider cache is a +/// best-effort artifact, so skipping it costs a re-fetch, while an unlocked +/// write can silently delete a sibling process's entries. /// A missing or corrupt on-disk file during re-read degrades to persisting /// our entries only (with a `warn!` for corrupt). fn save_cache_atomic(cache: &SharedCache, target: &Path) -> std::io::Result<()> { - let _guard = match acquire_exclusive_lock(target) { - Ok(g) => Some(g), - Err(err) => { - warn!( - path = %target.display(), - error = %err, - "Failed to acquire RPC cache lock; persisting without lock", - ); - None - } - }; + let _guard = acquire_exclusive_lock(target).map_err(|e| { + std::io::Error::other(format!( + "failed to acquire the cache lock {} for {}: {e}; \ + cache entries were not saved (an unlocked write could drop a \ + concurrent process's entries)", + lock_sidecar_path(target).display(), + target.display(), + )) + })?; let dir = target.parent().unwrap_or_else(|| Path::new(".")); fs::create_dir_all(dir).map_err(|e| { @@ -402,24 +383,22 @@ impl CacheFileEnvelope { /// corrupt JSON degrades to ours-only with a warning. /// /// Lock contention blocks until the lock is free. Failure to create/acquire - /// the lock degrades to an unlocked write with a `warn!`. Write failures - /// remain hard errors. + /// the lock is a hard error — the envelope is the primary output of capture + /// mode, so an unlocked write that silently drops a concurrent writer's + /// entries is worse than a failed run. Write failures remain hard errors. pub(super) fn save( &self, path: &Path, loaded_external_env: Option<&ExternalEnvSnapshot>, ) -> Result<()> { - let _guard = match acquire_exclusive_lock(path) { - Ok(g) => Some(g), - Err(err) => { - warn!( - path = %path.display(), - error = %err, - "Failed to acquire envelope lock; persisting without lock", - ); - None - } - }; + let _guard = acquire_exclusive_lock(path).map_err(|e| { + EvmeError::FixtureError(format!( + "Failed to acquire the cache lock {} for envelope '{}': {e}. \ + Refusing to write it unlocked: a concurrent writer's entries would be lost.", + lock_sidecar_path(path).display(), + path.display(), + )) + })?; let ours = self.to_merge_doc()?; let loaded_doc = loaded_external_env @@ -948,6 +927,95 @@ mod tests { } } + /// Provider persist fails closed when the lock cannot be acquired: nothing + /// is written, and the file a sibling process left behind is intact. + /// + /// The store swallows the failure (the provider cache is best-effort), so + /// the observable contract is the untouched file, not the return value. + #[test] + fn test_provider_cache_persist_skips_when_the_lock_is_unavailable() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-1.json"); + + // A sibling's file is already on disk. + let sibling = CacheLayer::new(16).cache(); + sibling.put(B256::repeat_byte(0xbb), "from-sibling".into()).expect("put"); + RpcCacheStore::new(sibling, path.clone()).persist().expect("persist sibling"); + let before = fs::read_to_string(&path).expect("read sibling file"); + + // A directory in the sidecar's place makes the lock un-acquirable. + fs::remove_file(lock_sidecar_path(&path)).expect("remove sidecar"); + fs::create_dir(lock_sidecar_path(&path)).expect("occupy sidecar path"); + + let ours = CacheLayer::new(16).cache(); + ours.put(B256::repeat_byte(0xaa), "ours".into()).expect("put"); + RpcCacheStore::new(ours, path.clone()) + .persist() + .expect("provider persist stays best-effort"); + + assert_eq!(fs::read_to_string(&path).unwrap(), before, "no unlocked write happened"); + } + + /// The skipped persist reports why, naming the lock and stating that the + /// entries were not saved. + #[test] + fn test_save_cache_atomic_reports_the_lock_failure() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-1.json"); + fs::create_dir(lock_sidecar_path(&path)).expect("occupy sidecar path"); + + let cache = CacheLayer::new(16).cache(); + cache.put(B256::repeat_byte(0xaa), "ours".into()).expect("put"); + let err = save_cache_atomic(&cache, &path).expect_err("lock failure must not write"); + let msg = err.to_string(); + assert!(msg.contains("rpc-cache-1.json.lock"), "msg={msg}"); + assert!(msg.contains("were not saved"), "msg={msg}"); + assert!(!path.exists(), "nothing was written"); + } + + /// Envelope persist hard-errors when the lock cannot be acquired: the + /// capture is the primary output, so a silently unlocked write is worse + /// than a failed run. + #[test] + fn test_envelope_persist_errors_when_the_lock_is_unavailable() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + + CacheFileEnvelope::new(&TransportCache::new(), 7, None).save(&path, None).expect("seed"); + let before = fs::read_to_string(&path).expect("read seeded envelope"); + + fs::remove_file(lock_sidecar_path(&path)).expect("remove sidecar"); + fs::create_dir(lock_sidecar_path(&path)).expect("occupy sidecar path"); + + let cache = TransportCache::new(); + cache + .merge(&serde_json::json!([{ + "key": keccak256("a"), + "value": r#"{"result":"a"}"#, + }])) + .expect("seed ours"); + let err = CacheFileEnvelope::new(&cache, 7, None) + .save(&path, None) + .expect_err("lock failure must abort the capture persist"); + let msg = err.to_string(); + assert!(msg.contains("capture.json.lock"), "msg={msg}"); + assert!(msg.contains("unlocked"), "msg={msg}"); + assert_eq!(fs::read_to_string(&path).unwrap(), before, "no unlocked write happened"); + } + + /// The store surfaces the envelope lock failure to its caller. + #[test] + fn test_store_envelope_persist_errors_when_the_lock_is_unavailable() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("capture.json"); + fs::create_dir(lock_sidecar_path(&path)).expect("occupy sidecar path"); + + let store = RpcCacheStore::new_envelope(TransportCache::new(), path.clone(), 7, None); + let err = store.persist().expect_err("capture persist must fail closed"); + assert!(err.to_string().contains("lock"), "msg={err}"); + assert!(!path.exists(), "nothing was written"); + } + /// Corrupt on-disk provider cache during re-read does not abort; ours are written. #[test] fn test_provider_cache_persist_degrades_on_corrupt_disk() { diff --git a/bin/mega-evme/tests/cache_merge_lock.rs b/bin/mega-evme/tests/cache_merge_lock.rs new file mode 100644 index 00000000..2c75f61f --- /dev/null +++ b/bin/mega-evme/tests/cache_merge_lock.rs @@ -0,0 +1,207 @@ +//! Two-process serialization tests for the `cache merge` output lock. +//! +//! `cache merge` writes a file a live `mega-evme` run may be persisting to at +//! the same time. Both writers take the exclusive advisory lock on the output's +//! sidecar and re-read the file while holding it, so neither side's entries are +//! lost to whichever rename lands last. +//! +//! Mutual exclusion cannot be demonstrated inside one process: a single-process +//! test that "takes the lock" and then calls the merge in-process either +//! deadlocks or proves nothing about a second process. These tests hold the +//! sidecar lock in the test process, spawn the real binary, show it makes no +//! progress while the lock is held, write a concurrent writer's entries under +//! that same lock, and only then release — so the merge's output can contain +//! those entries only by re-reading the file after it acquired the lock. + +use std::{ + fs::{self, File, OpenOptions}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +use alloy_primitives::B256; +use serde_json::{json, Value}; + +/// How long the lock is held while the spawned merge must make no progress. +/// +/// Merging two one-entry files takes milliseconds, so staying alive for this +/// long is only explainable by the lock. +const HOLD: Duration = Duration::from_secs(2); + +/// Upper bound on how long the merge may take after the lock is released. +/// Generous on purpose: this bound exists to fail a hung merge with a message +/// instead of hanging the suite, not to measure anything. +const COMPLETION_DEADLINE: Duration = Duration::from_secs(60); + +/// One `{key, value}` cache entry, keyed by a repeated byte. +fn kv(byte: u8, value: &str) -> Value { + json!({ "key": B256::repeat_byte(byte), "value": value }) +} + +/// The advisory lock sidecar the binary locks for `output`. +fn sidecar(output: &Path) -> PathBuf { + let mut os = output.as_os_str().to_owned(); + os.push(".lock"); + PathBuf::from(os) +} + +/// Take the exclusive lock on the output's sidecar and keep it until the +/// returned handle is dropped. +fn hold_output_lock(output: &Path) -> File { + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(sidecar(output)) + .expect("open the output sidecar"); + file.lock().expect("hold the output lock"); + file +} + +/// Spawn the real `mega-evme cache merge` process. +fn spawn_merge(inputs: &[&Path], output: &Path) -> Child { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_mega-evme")); + cmd.args(["cache", "merge"]); + for input in inputs { + cmd.arg(input); + } + cmd.arg("--output").arg(output).stdout(Stdio::piped()).stderr(Stdio::piped()); + cmd.spawn().expect("spawn mega-evme cache merge") +} + +/// Assert the spawned merge makes no progress for the whole hold window. +fn assert_blocked_while_held(child: &mut Child) { + let deadline = Instant::now() + HOLD; + while Instant::now() < deadline { + if let Some(status) = child.try_wait().expect("poll the merge") { + panic!("the merge completed while the output lock was held (status {status})"); + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + child.try_wait().expect("poll the merge").is_none(), + "the merge must still be waiting for the output lock", + ); +} + +/// Wait for the released merge to finish and return its stdout. +fn finish(mut child: Child) -> String { + let deadline = Instant::now() + COMPLETION_DEADLINE; + loop { + if child.try_wait().expect("poll the merge").is_some() { + break; + } + assert!( + Instant::now() < deadline, + "the merge did not finish within {COMPLETION_DEADLINE:?} of the lock being released", + ); + std::thread::sleep(Duration::from_millis(20)); + } + let out = child.wait_with_output().expect("collect the merge output"); + assert!( + out.status.success(), + "the merge must succeed once the lock is free.\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr), + ); + String::from_utf8_lossy(&out.stdout).into_owned() +} + +/// Value stored for the entry keyed by a repeated byte, if present. +fn value_of(entries: &[Value], byte: u8) -> Option { + let key = json!(B256::repeat_byte(byte)); + entries + .iter() + .find(|e| e.get("key") == Some(&key)) + .and_then(|e| e.get("value")) + .and_then(Value::as_str) + .map(str::to_owned) +} + +/// Provider shape: the merge waits for the lock, then folds in what a +/// concurrent writer left in the output while it waited. +#[test] +fn test_cache_merge_serializes_with_a_concurrent_provider_writer() { + let dir = tempfile::tempdir().expect("tempdir"); + let a = dir.path().join("a.json"); + let b = dir.path().join("b.json"); + let out = dir.path().join("out.json"); + + fs::write(&a, serde_json::to_string(&vec![kv(1, "from-a")]).unwrap()).expect("write a"); + fs::write(&b, serde_json::to_string(&vec![kv(2, "from-b")]).unwrap()).expect("write b"); + + let lock = hold_output_lock(&out); + let mut child = spawn_merge(&[&a, &b], &out); + assert_blocked_while_held(&mut child); + + // A concurrent writer lands its entries the way a clean-exit persist does: + // while it holds the same lock the merge is waiting for. + fs::write(&out, serde_json::to_string(&vec![kv(9, "from-concurrent-writer")]).unwrap()) + .expect("concurrent write"); + drop(lock); + + let stdout = finish(child); + + let merged: Vec = + serde_json::from_str(&fs::read_to_string(&out).expect("read merged output")) + .expect("merged output is a provider-cache array"); + assert_eq!( + value_of(&merged, 9).as_deref(), + Some("from-concurrent-writer"), + "the entry written while the merge was blocked must survive: {merged:?}", + ); + assert_eq!(value_of(&merged, 1).as_deref(), Some("from-a"), "input entry lost: {merged:?}"); + assert_eq!(value_of(&merged, 2).as_deref(), Some("from-b"), "input entry lost: {merged:?}"); + assert_eq!(merged.len(), 3, "the union is exactly both sides: {merged:?}"); + assert!( + stdout.contains("already in the output"), + "the summary must report the folded-in entries: {stdout}", + ); +} + +/// Envelope shape: same protocol, same guarantee. +#[test] +fn test_cache_merge_serializes_with_a_concurrent_envelope_writer() { + let dir = tempfile::tempdir().expect("tempdir"); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + let envelope = |entries: Vec| { + json!({ + "version": 1, + "chain_id": 4326, + "cache": entries, + "external_env": null, + }) + }; + fs::write(&a, serde_json::to_string_pretty(&envelope(vec![kv(1, "from-a")])).unwrap()) + .expect("write a"); + + let lock = hold_output_lock(&out); + let mut child = spawn_merge(&[&a], &out); + assert_blocked_while_held(&mut child); + + fs::write( + &out, + serde_json::to_string_pretty(&envelope(vec![kv(9, "from-concurrent-writer")])).unwrap(), + ) + .expect("concurrent write"); + drop(lock); + + finish(child); + + let merged: Value = + serde_json::from_str(&fs::read_to_string(&out).expect("read merged output")) + .expect("merged output is an envelope"); + let entries = merged["cache"].as_array().expect("cache array").clone(); + assert_eq!( + value_of(&entries, 9).as_deref(), + Some("from-concurrent-writer"), + "the entry written while the merge was blocked must survive: {entries:?}", + ); + assert_eq!(value_of(&entries, 1).as_deref(), Some("from-a"), "input entry lost: {entries:?}"); + assert_eq!(entries.len(), 2, "the union is exactly both sides: {entries:?}"); + assert_eq!(merged["chain_id"], json!(4326)); +} diff --git a/docs/mega-evme/commands/cache.md b/docs/mega-evme/commands/cache.md index 2328171a..8c3c41ec 100644 --- a/docs/mega-evme/commands/cache.md +++ b/docs/mega-evme/commands/cache.md @@ -28,11 +28,30 @@ Inputs are auto-detected by JSON shape: All inputs in one invocation must share the same shape. Mixing a provider-cache file with a capture envelope is a hard error that names the offending path. +### Output locking + +`--output` may be a file a live `mega-evme` run is persisting to. +The merge therefore uses the same protocol that clean-exit persist uses (see [State Management](../configuration/state-management.md#concurrent-cache-dir-sharing)): + +1. Take the exclusive advisory lock on the output's sidecar (`.lock`), blocking until it is free. +2. Under that lock, read whatever the output file holds now and fold it into the union as one more input. +3. Write via temp file + atomic rename, then release the lock. + +Folding the current output in is what makes the lock worth taking: entries a concurrent process wrote while the merge waited are carried into the merged result instead of being overwritten. +The merge's own inputs win where their keys collide with the output's prior entries. + +If the lock cannot be acquired at all (for example the sidecar path is not writable), the merge fails with an error and writes nothing. +An unlocked write would silently drop a concurrent process's entries, which is the failure the merge exists to prevent. + +An existing output that cannot be parsed at all (corrupt JSON) is replaced by the merged inputs, with a warning. +An existing output that parses but cannot be folded — the other cache shape, an unrecognized JSON shape, a different `chain_id`, a different envelope `version` — is a hard error that names the output path and leaves the file untouched. +Both shapes classify it the same way: a mistyped `--output` should not destroy a file the merge cannot read as its own. + ### Provider-cache merge - Union entries by `key`. -- Later inputs win on collision. -- Output is a provider-cache-shaped JSON array, written atomically (temp file + rename). +- Later inputs win on collision; the inputs win over entries already in `--output`. +- Output is a provider-cache-shaped JSON array, written atomically (temp file + rename) under the output lock. - Chain identity is taken only from the standard filename `rpc-cache-{chain_id}.json` (provider-cache bodies have no chain field). Every input path and `--output` that matches that pattern must name the same chain id; a mismatch is a hard error that names the conflicting files. Paths that do not match the pattern emit a warning that chain identity cannot be validated for them, and the merge proceeds for those paths without a filename-based check. @@ -40,9 +59,11 @@ Mixing a provider-cache file with a capture envelope is a hard error that names ### Envelope merge - Every input must use the current envelope `version` and the same `chain_id` (else hard error naming the mismatch). -- Union the `cache` arrays by key; later inputs win on collision. + An envelope already at `--output` must agree with them too. +- Union the `cache` arrays by key; later inputs win on collision, and the inputs win over entries already in `--output`. - `external_env`: if two inputs carry non-identical snapshots, hard error; otherwise propagate the non-null snapshot. -- Output is a pretty-printed envelope, written atomically. + A snapshot already at `--output` is held to the same rule. +- Output is a pretty-printed envelope, written atomically under the output lock. ### Summary @@ -52,6 +73,12 @@ On success, `cache merge` prints one line and exits 0: Merged 3 inputs (120 entries in) → 95 unique entries out ``` +When the output file already held entries, they are counted separately, so the arithmetic still adds up: + +``` +Merged 3 inputs (120 entries in + 12 already in the output) → 101 unique entries out +``` + ### Examples Merge sharded worker provider caches after a multi-process campaign: diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index bc473503..92c3a1f5 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -232,10 +232,14 @@ On clean-exit persist, each process: The lock sidecar is left in place after the process exits; only the flock is released when the handle closes. Lock contention blocks for a short critical section rather than failing the finished run. -If the lock cannot be acquired at all (for example the directory is not writable), persist logs a warning and falls back to an unlocked write. +If the lock cannot be acquired at all (for example the directory is not writable), persist fails closed and writes nothing. +For the provider cache that means the file is left untouched and a warning names it — the cache is a best-effort artifact, so the cost is a re-fetch on the next run. +An unlocked write is not offered as a fallback: it is exactly the lost-update race the lock exists to prevent, and it would delete a sibling process's entries silently. A missing or corrupt on-disk file during the re-read degrades to writing this process's entries only (also warned). -Capture envelopes (`--rpc.capture-file`) use the same lock + re-read-merge path, with additional hard-error checks before writing: +Capture envelopes (`--rpc.capture-file`) use the same lock + re-read-merge path, with additional hard-error checks before writing. +A capture that cannot take the lock fails the run rather than writing unlocked: the envelope is the primary output of capture mode, so losing a concurrent writer's entries is worse than reporting the failure. +The checks are: - The on-disk envelope `version` and `chain_id` must match this process's capture. - `external_env` uses optimistic concurrency against the snapshot observed when this process opened the capture file: @@ -252,6 +256,7 @@ A corrupt or unreadable on-disk envelope during re-read degrades to writing this To consolidate historical per-worker cache directories offline, use [`cache merge`](../commands/cache.md). Provider-cache merge also rejects inputs (and `--output`) whose `rpc-cache-{chain_id}.json` filenames disagree on chain id. +`cache merge` follows the same lock protocol for its `--output`: it takes the output's sidecar lock, folds whatever the file holds at that moment into the union, writes, and releases — so merging into a file a live run is still persisting to loses neither side's entries. ### Cache Flags From 6c7348f71650ad38f6b7e80d21c015b2c6dec7dd Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 01:05:24 +0800 Subject: [PATCH 44/64] fix(mega-evme): print cache merge safeguard warnings on stderr `cache merge` warns when a filename cannot supply chain identity and when an unreadable output is about to be replaced. Both only existed as `warn!`, and the CLI leaves the tracing filter at `off` unless `-v` or `RUST_LOG` raises it, so a default run showed nothing: the cross-chain safeguard degraded silently to no check, and discarded output entries went unannounced. Route both through a `warn_user` helper that writes the line to stderr unconditionally and keeps the structured event for log sinks. Merge behavior is unchanged; stdout still carries the summary alone. --- bin/mega-evme/src/cache/mod.rs | 56 +++-- .../tests/cache_merge_diagnostics.rs | 208 ++++++++++++++++++ docs/mega-evme/commands/cache.md | 4 + 3 files changed, 250 insertions(+), 18 deletions(-) create mode 100644 bin/mega-evme/tests/cache_merge_diagnostics.rs diff --git a/bin/mega-evme/src/cache/mod.rs b/bin/mega-evme/src/cache/mod.rs index ff90eec0..97006c2e 100644 --- a/bin/mega-evme/src/cache/mod.rs +++ b/bin/mega-evme/src/cache/mod.rs @@ -6,7 +6,7 @@ mod lock; mod merge; -use std::path::PathBuf; +use std::{fmt, path::PathBuf}; use clap::{Parser, Subcommand}; @@ -62,10 +62,29 @@ impl Cmd { } } +/// Emit a diagnostic that protects the user from a silently wrong merge, on +/// stderr unconditionally and through the structured log sinks. +/// +/// The CLI leaves the tracing filter at `off` unless `-v` flags or `RUST_LOG` +/// raise it, so a `warn!`-only diagnostic reaches nobody on a default command +/// line: a safeguard reporting that it could not run, or a write about to +/// discard data already on disk, would be announced into a disabled subscriber. +/// stderr therefore carries the human line regardless of verbosity, and the +/// tracing event still carries it to a `--log.file` sink. At raised verbosity +/// without `--log.file` both channels land on stderr and the line appears +/// twice, which is preferable to dropping either one. +/// +/// Reserved for warnings a user must act on; ordinary progress reporting stays +/// on `tracing` alone. +fn warn_user(message: fmt::Arguments<'_>) { + eprintln!("warning: {message}"); + warn!("{message}"); +} + /// Validate that provider-cache paths agreeing with `rpc-cache-{id}.json` all /// name the same chain id. /// -/// Paths that do not match the pattern emit a `warn!` (chain identity cannot be +/// Paths that do not match the pattern warn the user (chain identity cannot be /// validated for them) and are otherwise ignored. Two or more matching paths /// with different ids are a hard error naming the conflicting files. pub(crate) fn check_provider_cache_chain_identity<'a>( @@ -75,11 +94,11 @@ pub(crate) fn check_provider_cache_chain_identity<'a>( for path in paths { match parse_rpc_cache_filename_chain_id(path) { None => { - warn!( - path = %path.display(), - "Provider-cache path does not match rpc-cache-{{id}}.json; \ + warn_user(format_args!( + "Provider-cache path '{}' does not match rpc-cache-{{id}}.json; \ chain identity cannot be validated for this file", - ); + path.display(), + )); } Some(id) => match &seen { None => seen = Some((id, path.to_path_buf())), @@ -177,12 +196,13 @@ impl MergeArgs { ProviderReread::Ok(entries) => entries, ProviderReread::Hard(err) => return Err(err), ProviderReread::Degradable(msg) => { - warn!( - path = %self.output.display(), - error = %msg, - "Failed to read the existing merge output; \ - it will be replaced by the merged inputs", - ); + // Replacing the output drops whatever it held: the user + // must hear about it whatever the verbosity is. + warn_user(format_args!( + "{msg}. Replacing the existing merge output '{}' with the \ + merged inputs; any entries it held are discarded", + self.output.display(), + )); Vec::new() } }; @@ -214,12 +234,12 @@ impl MergeArgs { } EnvelopeReread::Hard(err) => return Err(err), EnvelopeReread::Degradable(msg) => { - warn!( - path = %self.output.display(), - error = %msg, - "Failed to read the existing merge output; \ - it will be replaced by the merged inputs", - ); + // Same data loss as the provider shape above. + warn_user(format_args!( + "{msg}. Replacing the existing merge output '{}' with the \ + merged inputs; any entries it held are discarded", + self.output.display(), + )); merged } } diff --git a/bin/mega-evme/tests/cache_merge_diagnostics.rs b/bin/mega-evme/tests/cache_merge_diagnostics.rs new file mode 100644 index 00000000..5f5ac772 --- /dev/null +++ b/bin/mega-evme/tests/cache_merge_diagnostics.rs @@ -0,0 +1,208 @@ +//! Binary-level tests for the diagnostics `cache merge` owes the user. +//! +//! `cache merge` has safeguards that can only warn: chain identity that cannot +//! be derived from a filename, and an unreadable output file that the merge is +//! about to replace. Both report a result that is silently wrong or lossy, and +//! both are worthless if the user never sees them. +//! +//! The CLI initializes tracing with the filter at `off` unless `-v` flags or +//! `RUST_LOG` raise it, so these cannot be asserted through a tracing capture +//! in-process: doing that would prove the event is emitted while the default +//! command line still shows nothing. These tests therefore run the real binary +//! with no verbosity flags and `RUST_LOG` removed from its environment, and +//! read what an operator would actually see on stderr. + +use std::{ + fs, + path::Path, + process::{Command, Output}, +}; + +use alloy_primitives::B256; +use serde_json::{json, Value}; + +/// One `{key, value}` provider-cache entry, keyed by a repeated byte. +fn kv(byte: u8, value: &str) -> Value { + json!({ "key": B256::repeat_byte(byte), "value": value }) +} + +/// A capture envelope holding `entries`. +fn envelope(entries: Vec) -> Value { + json!({ "version": 1, "chain_id": 4326, "cache": entries, "external_env": null }) +} + +/// Run `mega-evme cache merge` exactly as a default command line would: no `-v` +/// flags, and no inherited `RUST_LOG` that could raise the filter for us. +fn run_merge(inputs: &[&Path], output: &Path) -> Output { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_mega-evme")); + cmd.args(["cache", "merge"]); + for input in inputs { + cmd.arg(input); + } + cmd.arg("--output").arg(output); + cmd.env_remove("RUST_LOG"); + cmd.output().expect("run mega-evme cache merge") +} + +/// Assert the merge succeeded, and return `(stdout, stderr)`. +fn succeeds(out: &Output) -> (String, String) { + let stdout = String::from_utf8_lossy(&out.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&out.stderr).into_owned(); + assert_eq!( + out.status.code(), + Some(0), + "the merge must still succeed.\nstdout: {stdout}\nstderr: {stderr}", + ); + (stdout, stderr) +} + +/// Value stored for the entry keyed by a repeated byte, if present. +fn value_of(entries: &[Value], byte: u8) -> Option { + let key = json!(B256::repeat_byte(byte)); + entries + .iter() + .find(|e| e.get("key") == Some(&key)) + .and_then(|e| e.get("value")) + .and_then(Value::as_str) + .map(str::to_owned) +} + +/// Read the merged provider-cache array at `path`. +fn read_provider(path: &Path) -> Vec { + serde_json::from_str(&fs::read_to_string(path).expect("read merged output")) + .expect("merged output is a provider-cache array") +} + +/// Filenames that carry no chain id leave the cross-chain safeguard unable to +/// run. The merge proceeds — renamed shards are legitimate — but the user is +/// told, on stderr, without asking for verbosity. +#[test] +fn test_cache_merge_warns_on_stderr_when_chain_identity_cannot_be_validated() { + let dir = tempfile::tempdir().expect("tempdir"); + let a = dir.path().join("worker-a.json"); + let b = dir.path().join("worker-b.json"); + let out = dir.path().join("merged.json"); + + fs::write(&a, serde_json::to_string(&vec![kv(1, "from-a")]).unwrap()).expect("write a"); + fs::write(&b, serde_json::to_string(&vec![kv(2, "from-b")]).unwrap()).expect("write b"); + + let output = run_merge(&[&a, &b], &out); + let (stdout, stderr) = succeeds(&output); + + assert!( + stderr.contains("chain identity cannot be validated"), + "the safeguard must announce that it could not run: stderr={stderr}", + ); + assert!( + stderr.contains("worker-a.json") && stderr.contains("worker-b.json"), + "each unvalidatable file must be named: stderr={stderr}", + ); + + // Behavior is otherwise unchanged: the merge still produced the union. + let merged = read_provider(&out); + assert_eq!(value_of(&merged, 1).as_deref(), Some("from-a"), "{merged:?}"); + assert_eq!(value_of(&merged, 2).as_deref(), Some("from-b"), "{merged:?}"); + assert_eq!(merged.len(), 2, "{merged:?}"); + assert!(stdout.contains("Merged"), "the summary still goes to stdout: {stdout}"); +} + +/// The warning is specific to unvalidatable names: filenames that agree on a +/// chain id let the safeguard run, and a quiet merge stays quiet. +#[test] +fn test_cache_merge_is_silent_when_filenames_agree_on_the_chain_id() { + let dir = tempfile::tempdir().expect("tempdir"); + let worker0 = dir.path().join("worker0"); + let worker1 = dir.path().join("worker1"); + let merged_dir = dir.path().join("merged"); + for d in [&worker0, &worker1, &merged_dir] { + fs::create_dir(d).expect("create dir"); + } + let a = worker0.join("rpc-cache-4326.json"); + let b = worker1.join("rpc-cache-4326.json"); + let out = merged_dir.join("rpc-cache-4326.json"); + + fs::write(&a, serde_json::to_string(&vec![kv(1, "from-a")]).unwrap()).expect("write a"); + fs::write(&b, serde_json::to_string(&vec![kv(2, "from-b")]).unwrap()).expect("write b"); + + let output = run_merge(&[&a, &b], &out); + let (_, stderr) = succeeds(&output); + + assert!( + !stderr.contains("chain identity"), + "a validated same-chain merge must not warn: stderr={stderr}", + ); + assert!(stderr.is_empty(), "a clean merge writes nothing to stderr: stderr={stderr}"); + + let merged = read_provider(&out); + assert_eq!(merged.len(), 2, "{merged:?}"); +} + +/// An unreadable provider output is replaced by the merged inputs, dropping +/// whatever it held. That data loss reaches stderr at default verbosity too. +#[test] +fn test_cache_merge_warns_on_stderr_when_replacing_an_unreadable_provider_output() { + let dir = tempfile::tempdir().expect("tempdir"); + let inputs = dir.path().join("inputs"); + let merged_dir = dir.path().join("merged"); + for d in [&inputs, &merged_dir] { + fs::create_dir(d).expect("create dir"); + } + // Convention-following names on both sides, so the only warning that can + // fire here is the one under test. + let a = inputs.join("rpc-cache-4326.json"); + let out = merged_dir.join("rpc-cache-4326.json"); + + fs::write(&a, serde_json::to_string(&vec![kv(1, "from-a")]).unwrap()).expect("write a"); + fs::write(&out, "not-json{{{").expect("write a corrupt output"); + + let output = run_merge(&[&a], &out); + let (_, stderr) = succeeds(&output); + + assert!( + stderr.contains("Replacing the existing merge output"), + "the replacement must be announced: stderr={stderr}", + ); + assert!( + stderr.contains("discarded"), + "the user must be told entries are lost: stderr={stderr}", + ); + assert!( + !stderr.contains("chain identity"), + "no chain-identity warning is due here: stderr={stderr}", + ); + + let merged = read_provider(&out); + assert_eq!(value_of(&merged, 1).as_deref(), Some("from-a"), "{merged:?}"); + assert_eq!(merged.len(), 1, "{merged:?}"); +} + +/// The envelope shape replaces an unreadable output the same way, and warns the +/// same way. +#[test] +fn test_cache_merge_warns_on_stderr_when_replacing_an_unreadable_envelope_output() { + let dir = tempfile::tempdir().expect("tempdir"); + let a = dir.path().join("a.json"); + let out = dir.path().join("out.json"); + + fs::write(&a, serde_json::to_string_pretty(&envelope(vec![kv(1, "from-a")])).unwrap()) + .expect("write a"); + fs::write(&out, "not-json{{{").expect("write a corrupt output"); + + let output = run_merge(&[&a], &out); + let (_, stderr) = succeeds(&output); + + assert!( + stderr.contains("Replacing the existing merge output"), + "the replacement must be announced: stderr={stderr}", + ); + assert!( + stderr.contains("discarded"), + "the user must be told entries are lost: stderr={stderr}", + ); + + let merged: Value = serde_json::from_str(&fs::read_to_string(&out).expect("read output")) + .expect("merged output is an envelope"); + let entries = merged["cache"].as_array().expect("cache array").clone(); + assert_eq!(value_of(&entries, 1).as_deref(), Some("from-a"), "{entries:?}"); + assert_eq!(entries.len(), 1, "{entries:?}"); +} diff --git a/docs/mega-evme/commands/cache.md b/docs/mega-evme/commands/cache.md index 8c3c41ec..320bc547 100644 --- a/docs/mega-evme/commands/cache.md +++ b/docs/mega-evme/commands/cache.md @@ -47,6 +47,10 @@ An existing output that cannot be parsed at all (corrupt JSON) is replaced by th An existing output that parses but cannot be folded — the other cache shape, an unrecognized JSON shape, a different `chain_id`, a different envelope `version` — is a hard error that names the output path and leaves the file untouched. Both shapes classify it the same way: a mistyped `--output` should not destroy a file the merge cannot read as its own. +Warnings about a merge that may be silently wrong or lossy — an output being replaced, or chain identity that cannot be validated — are printed on stderr regardless of verbosity. +They do not depend on `-v` flags or `RUST_LOG`, which only add the structured log event alongside them. +Stdout carries the summary line only, so it stays parseable. + ### Provider-cache merge - Union entries by `key`. From 45ea3148edde9d18a22b35a1b2f2dbf3f67839af Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 08:48:49 +0800 Subject: [PATCH 45/64] fix(mega-evme): classify single-path block-body nulls as rpc The single-transaction replay path derives its preceding-transaction hashes from the block body the endpoint already served, so a lookup resolving to null contradicts an answer that endpoint gave itself. Report it as BlockBodyTransactionNull (rpc class, exit 3) instead of TransactionNotFound (execution class, exit 1), matching the batch driver. The initial user-supplied target lookup keeps TransactionNotFound: nothing the endpoint served claimed that hash exists, so the null is a definitive answer about the caller's own question. --- bin/mega-evme/src/replay/cmd.rs | 13 ++- bin/mega-evme/tests/replay_batch.rs | 123 ++++++++++++++++++++++++++++ docs/mega-evme/overview.md | 2 + 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 390f1731..a7e1dfd9 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -518,6 +518,10 @@ impl Cmd { P: Provider, { info!(tx_hash = %tx_hash, "Fetching transaction"); + // The user supplied this hash and nothing the endpoint served so far + // claims it exists, so `Ok(None)` is a definitive "unknown transaction" + // rather than an inconsistency — unlike the block-body-derived lookups + // further down, which the endpoint has already vouched for. let target_tx = provider .get_transaction_by_hash(tx_hash) .await @@ -807,11 +811,18 @@ impl Cmd { info!(preceding_count = ctx.preceding_tx_hashes.len(), "Executing preceding transactions",); for tx_hash in &ctx.preceding_tx_hashes { debug!(tx_hash = %tx_hash, "Executing preceding transaction"); + // These hashes were read out of the block body this endpoint already + // served, so `Ok(None)` contradicts an answer it gave itself: the + // endpoint is inconsistent (reorg, or a load-balanced backend serving + // divergent views), not definitively denying the hash. Only the + // user-supplied target lookup keeps `TransactionNotFound`, because + // there the null is a definitive answer about a hash the caller asked + // about. let tx = provider .get_transaction_by_hash(*tx_hash) .await .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? - .ok_or(ReplayError::TransactionNotFound(*tx_hash))?; + .ok_or(ReplayError::BlockBodyTransactionNull(*tx_hash))?; let outcome = block_executor .run_transaction(tx.as_recovered()) .map_err(ReplayError::BlockExecutionError)?; diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 892ddc4d..52443c12 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -129,6 +129,34 @@ fn envelope_without_transaction(name: &str, tx_hash: &str) -> std::path::PathBuf path } +/// Write a copy of the envelope with the `eth_getTransactionByHash` entry for +/// `tx_hash` removed entirely, and return its path. +/// +/// Distinct from [`envelope_without_transaction`]: there the endpoint answers +/// "no such transaction", here it does not answer at all — an offline cache +/// miss, which is how a transport failure reaches the same call site. +fn envelope_dropping_transaction(name: &str, tx_hash: &str) -> std::path::PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let marker = format!("\"hash\":\"{tx_hash}\""); + let entries = envelope["cache"].as_array_mut().expect("cache entries"); + let before = entries.len(); + entries.retain(|entry| { + !entry["value"].as_str().expect("entry value is a string").contains(&marker) + }); + assert_eq!( + before - entries.len(), + 1, + "the envelope must hold exactly one response for {tx_hash}" + ); + + let path = + std::env::temp_dir().join(format!("mega_evme_batch_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored envelope"); + path +} + /// Write a copy of the envelope whose `eth_getTransactionByHash` response for /// `tx_hash` still returns the transaction object, but with `gas` set to `0x0` /// so execution/setup fails (intrinsic gas / validation) rather than a missing @@ -198,6 +226,20 @@ fn run_error(stdout: &str) -> serde_json::Value { value } +/// The structured error object a failing single-transaction `--json` run ends +/// with. +/// +/// Single-transaction output is pretty-printed rather than NDJSON, so the +/// object is recovered by streaming every JSON value on stdout instead of +/// reading the last line. +fn single_run_error(stdout: &str) -> serde_json::Value { + let values = common::json_values(stdout); + let last = + values.last().unwrap_or_else(|| panic!("a failing --json run must not leave stdout empty")); + assert!(common::is_run_error(last), "the last stdout value must be the error object: {last}"); + last.clone() +} + /// `--block N --json` emits exactly one NDJSON line per transaction of the /// block, in transaction order, and exits 0. #[test] @@ -397,6 +439,87 @@ fn test_replay_block_sweeps_targets_behind_an_abort_as_unanswered() { assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); } +/// A preceding transaction of a single-transaction replay resolving to null is +/// an endpoint inconsistency (exit 3), not a definitive unknown transaction. +/// +/// The single-transaction path derives its preceding hashes from the block body +/// the endpoint already served, so the batch driver's reasoning applies +/// unchanged: a null lookup contradicts an answer the endpoint gave itself. +/// Doctors the index-0 transaction's response and replays a mid-block target, +/// which executes that transaction before its own. +#[test] +fn test_replay_single_transaction_preceding_null_is_an_rpc_failure() { + let missing = BLOCK_TXS[0].0; + let (target, target_index) = BLOCK_TXS[1]; + assert!(target_index > 0, "the target must have preceding transactions to execute"); + let path = envelope_without_transaction("single_preceding_null", missing); + + let (stdout, code) = replay_envelope_with_code(&path, &["--json", target]); + let _ = std::fs::remove_file(&path); + + assert_eq!(code, Some(3), "a preceding block-body null exits 3: {stdout}"); + let error = single_run_error(&stdout); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + assert!( + error["error"]["message"].as_str().is_some_and(|m| { + m.contains(missing) && m.contains("Block body") && m.contains("resolves it to null") + }), + "the failure must name the hash and the inconsistency: {error}" + ); +} + +/// The user-supplied target of a single-transaction replay resolving to null +/// keeps the definitive-answer class (exit 1). +/// +/// Nothing the endpoint served claims that hash exists, so the null is an +/// answer about the caller's own question rather than a contradiction — the one +/// lookup on this path that stays `TransactionNotFound`. +#[test] +fn test_replay_single_transaction_target_null_is_not_found() { + let target = BLOCK_TXS[1].0; + let path = envelope_without_transaction("single_target_null", target); + + let (stdout, code) = replay_envelope_with_code(&path, &["--json", target]); + let _ = std::fs::remove_file(&path); + + assert_eq!(code, Some(1), "an unknown user-supplied hash exits 1: {stdout}"); + let error = single_run_error(&stdout); + assert_eq!(error["error"]["code"].as_u64(), Some(1)); + assert_eq!(error["error"]["kind"].as_str(), Some("execution-error")); + assert!( + error["error"]["message"] + .as_str() + .is_some_and(|m| m.contains("Transaction not found") && m.contains(target)), + "the failure must stay a definitive not-found naming the hash: {error}" + ); +} + +/// A preceding transaction the endpoint never answers is an unanswered +/// question, not a null answer: same exit class (3), different message. +/// +/// Pins that reclassifying the null answer did not swallow the transport +/// failure reaching the same call site. +#[test] +fn test_replay_single_transaction_preceding_transport_error_is_an_rpc_failure() { + let missing = BLOCK_TXS[0].0; + let target = BLOCK_TXS[1].0; + let path = envelope_dropping_transaction("single_preceding_miss", missing); + + let (stdout, code) = replay_envelope_with_code(&path, &["--json", target]); + let _ = std::fs::remove_file(&path); + + assert_eq!(code, Some(3), "an unanswered preceding lookup exits 3: {stdout}"); + let error = single_run_error(&stdout); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!(message.contains("cache miss"), "the failure must name the missed request: {error}"); + assert!( + !message.contains("resolves it to null"), + "an unanswered lookup is not a null answer: {error}" + ); +} + /// An executor/setup abort on a mid-block transaction is still an execution /// failure for that transaction only: every target behind it is unanswered /// (`rpc`), not blamed as execution. diff --git a/docs/mega-evme/overview.md b/docs/mega-evme/overview.md index fc909249..a2410002 100644 --- a/docs/mega-evme/overview.md +++ b/docs/mega-evme/overview.md @@ -80,6 +80,8 @@ Every command reports its outcome through the same set of exit codes, so a pipel Codes `1` and `3` separate the two ways a question can go wrong: `1` means the tool answered, and the answer is negative; `3` means the question went unanswered, so retrying against a healthy endpoint may still produce a result. A state read that fails while the EVM is executing — an offline replay file without the response, or an endpoint that dies mid-transaction — belongs to `3` as well, even though it surfaces as a block execution error. +A hash the endpoint itself listed in a block body but then resolves to null belongs to `3` too: the null contradicts an answer the endpoint already gave, so it describes an inconsistent endpoint (a reorg, or a load-balanced backend serving divergent views) rather than an unknown transaction. +Only a hash the caller supplied directly stays in `1` when it resolves to null, since nothing the endpoint served claimed it existed. Two paths cannot be classified that way: a read that fails inside the pre-block system calls (EIP-4788 beacon root, EIP-2935 block hashes) or inside the sandboxed execution of the keyless-deploy system contract has its cause rendered into a message by the layer that raises it, so `mega-evme` cannot tell it from an execution failure and reports `1`. A batch run (`--tx-file` / `--block`) reports every target on its own line and then exits once for the run as a whole, ranking the failure classes it saw: any execution or internal failure exits `1`, otherwise any RPC failure exits `3`, otherwise any verification mismatch exits `2`. From 14aa29e8653bed0842afc2687c3b50504ef9b547 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 08:50:50 +0800 Subject: [PATCH 46/64] fix(mega-evme): lock clear-cache and type provider persist reread Serialize --rpc.clear-cache under the same sidecar exclusive lock as persist/merge and fail closed when the lock cannot be acquired. Classify on-disk reread in save_cache_atomic so corrupt degrades to ours-only while foreign shapes skip the write with a visible warning. --- bin/mega-evme/src/cache/merge.rs | 5 + bin/mega-evme/src/cache/mod.rs | 20 +-- .../src/common/provider/cache_store.rs | 101 ++++++++++-- bin/mega-evme/src/common/provider/mod.rs | 17 +++ bin/mega-evme/tests/cache_clear_lock.rs | 144 ++++++++++++++++++ bin/mega-evme/tests/provider.rs | 52 +++++++ 6 files changed, 315 insertions(+), 24 deletions(-) create mode 100644 bin/mega-evme/tests/cache_clear_lock.rs diff --git a/bin/mega-evme/src/cache/merge.rs b/bin/mega-evme/src/cache/merge.rs index 2efe4749..19386d34 100644 --- a/bin/mega-evme/src/cache/merge.rs +++ b/bin/mega-evme/src/cache/merge.rs @@ -173,6 +173,11 @@ pub(crate) fn detect_shape(value: &serde_json::Value, path: &Path) -> Result Result> { if !path.exists() { return Ok(Vec::new()); diff --git a/bin/mega-evme/src/cache/mod.rs b/bin/mega-evme/src/cache/mod.rs index 97006c2e..53862a08 100644 --- a/bin/mega-evme/src/cache/mod.rs +++ b/bin/mega-evme/src/cache/mod.rs @@ -15,15 +15,16 @@ use crate::common::{EvmeError, Result}; pub(crate) use lock::{acquire_exclusive_lock, lock_sidecar_path}; pub(crate) use merge::{ merge_envelope_for_persist, merge_provider_entries_capped, merge_provider_lists, - parse_rpc_cache_filename_chain_id, read_provider_cache, reread_envelope_for_merge, + parse_rpc_cache_filename_chain_id, reread_envelope_for_merge, reread_provider_cache_for_merge, write_bytes_atomic, write_envelope_atomic, write_provider_cache_atomic, CacheKv, EnvelopeDoc, - EnvelopeReread, ExternalEnvDoc, ENVELOPE_VERSION, + EnvelopeReread, ExternalEnvDoc, ProviderReread, ENVELOPE_VERSION, }; -use merge::{ - fold_output_envelope, load_cache_file, merge_envelopes_cli, reread_provider_cache_for_merge, - CacheShape, LoadedCache, ProviderReread, -}; +// Used by unit tests that assert the provider-array on-disk shape after a merge. +#[cfg(test)] +pub(crate) use merge::read_provider_cache; + +use merge::{fold_output_envelope, load_cache_file, merge_envelopes_cli, CacheShape, LoadedCache}; use tracing::warn; /// `mega-evme cache` — offline cache-file utilities. @@ -62,8 +63,9 @@ impl Cmd { } } -/// Emit a diagnostic that protects the user from a silently wrong merge, on -/// stderr unconditionally and through the structured log sinks. +/// Emit a diagnostic that protects the user from a silently wrong merge or +/// persist decision, on stderr unconditionally and through the structured log +/// sinks. /// /// The CLI leaves the tracing filter at `off` unless `-v` flags or `RUST_LOG` /// raise it, so a `warn!`-only diagnostic reaches nobody on a default command @@ -76,7 +78,7 @@ impl Cmd { /// /// Reserved for warnings a user must act on; ordinary progress reporting stays /// on `tracing` alone. -fn warn_user(message: fmt::Arguments<'_>) { +pub(crate) fn warn_user(message: fmt::Arguments<'_>) { eprintln!("warning: {message}"); warn!("{message}"); } diff --git a/bin/mega-evme/src/common/provider/cache_store.rs b/bin/mega-evme/src/common/provider/cache_store.rs index 3235be1b..25133fe9 100644 --- a/bin/mega-evme/src/common/provider/cache_store.rs +++ b/bin/mega-evme/src/common/provider/cache_store.rs @@ -36,9 +36,9 @@ use super::transport::TransportCache; use crate::{ cache::{ acquire_exclusive_lock, lock_sidecar_path, merge_envelope_for_persist, - merge_provider_entries_capped, read_provider_cache, reread_envelope_for_merge, - write_bytes_atomic, write_envelope_atomic, CacheKv, EnvelopeDoc, EnvelopeReread, - ExternalEnvDoc, ENVELOPE_VERSION, + merge_provider_entries_capped, reread_envelope_for_merge, reread_provider_cache_for_merge, + warn_user, write_bytes_atomic, write_envelope_atomic, CacheKv, EnvelopeDoc, EnvelopeReread, + ExternalEnvDoc, ProviderReread, ENVELOPE_VERSION, }, common::{EvmeError, Result}, }; @@ -196,7 +196,9 @@ impl RpcCacheStore { match inner { RpcCacheStoreInner::ProviderCache { cache, path } => { match save_cache_atomic(&cache, &path) { - Ok(()) => info!(path = %path.display(), "Persisted RPC cache"), + Ok(true) => info!(path = %path.display(), "Persisted RPC cache"), + // Intentional skip (e.g. foreign on-disk shape) already warned inside. + Ok(false) => {} Err(err) => warn!( path = %path.display(), error = %err, @@ -241,15 +243,22 @@ impl fmt::Debug for RpcCacheStore { /// Atomically persist `cache` to `target` via lock + re-read-merge + temp rename. /// -/// All error paths include `target` in the returned [`std::io::Error`] so the -/// warn-log in [`RpcCacheStore::persist`] identifies which file failed. +/// Returns `Ok(true)` when the file was written, `Ok(false)` when the write was +/// intentionally skipped (a recognizable foreign on-disk shape), and `Err` on +/// lock/IO failure. All error paths include `target` in the returned +/// [`std::io::Error`] so the warn-log in [`RpcCacheStore::persist`] identifies +/// which file failed. /// /// Lock acquisition failure aborts the persist: the provider cache is a /// best-effort artifact, so skipping it costs a re-fetch, while an unlocked /// write can silently delete a sibling process's entries. -/// A missing or corrupt on-disk file during re-read degrades to persisting -/// our entries only (with a `warn!` for corrupt). -fn save_cache_atomic(cache: &SharedCache, target: &Path) -> std::io::Result<()> { +/// +/// On-disk re-read is typed (same classification as `cache merge`): +/// - missing / provider array → merge and write; +/// - corrupt / unreadable → degrade to ours-only (with a `warn!`); +/// - recognizable foreign shape (capture envelope, …) → skip the write with a visible warning so a +/// mispointed cache-dir cannot destroy the foreign file. +fn save_cache_atomic(cache: &SharedCache, target: &Path) -> std::io::Result { let _guard = acquire_exclusive_lock(target).map_err(|e| { std::io::Error::other(format!( "failed to acquire the cache lock {} for {}: {e}; \ @@ -289,16 +298,30 @@ fn save_cache_atomic(cache: &SharedCache, target: &Path) -> std::io::Result<()> // Drop the NamedTempFile so it is unlinked; we only needed the dump bytes. drop(our_tmp); - let disk_entries = match read_provider_cache(target) { - Ok(entries) => entries, - Err(err) => { + let disk_entries = match reread_provider_cache_for_merge(target) { + ProviderReread::Ok(entries) => entries, + ProviderReread::Degradable(msg) => { warn!( path = %target.display(), - error = %err, + error = %msg, "Failed to re-read on-disk RPC cache during merge; persisting our entries only", ); Vec::new() } + ProviderReread::Hard(err) => { + // Best-effort provider persist must not destroy a foreign file that + // a shared-dir misconfiguration pointed it at (e.g. a capture + // envelope). Skip the write so the foreign content survives. + // stderr via `warn_user`: default CLI tracing is off, so a + // `warn!`-only line would never reach the operator who needs to + // fix the shared-dir misconfiguration. + warn_user(format_args!( + "Skipping RPC cache persist to '{}': {err}. On-disk file is not a \ + provider cache; leaving it intact", + target.display(), + )); + return Ok(false); + } }; // The union must respect the configured cap: a sibling's file plus ours can @@ -312,7 +335,7 @@ fn save_cache_atomic(cache: &SharedCache, target: &Path) -> std::io::Result<()> )) })?; write_bytes_atomic(target, &serialized)?; - Ok(()) + Ok(true) } /// On-disk envelope format shared by `--rpc.capture-file` (write) and @@ -973,6 +996,53 @@ mod tests { assert!(!path.exists(), "nothing was written"); } + /// Persisting a provider cache onto a capture envelope must leave the + /// envelope intact: a shared-dir misconfiguration must not destroy a + /// foreign file the provider shape cannot fold. + #[test] + fn test_provider_cache_persist_skips_foreign_envelope_target() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-1.json"); + + let envelope = serde_json::json!({ + "version": 1, + "chain_id": 7, + "cache": [], + "external_env": null, + }); + let before = serde_json::to_string_pretty(&envelope).unwrap(); + fs::write(&path, &before).expect("seed envelope"); + + let cache = CacheLayer::new(16).cache(); + cache.put(B256::repeat_byte(0xaa), "ours".into()).expect("put"); + let wrote = save_cache_atomic(&cache, &path).expect("skip is Ok(false), not Err"); + assert!(!wrote, "foreign shape must not be overwritten"); + assert_eq!(fs::read_to_string(&path).unwrap(), before, "envelope left intact"); + + // Store path is best-effort: same skip, same intact file, no hard error. + let store_cache = CacheLayer::new(16).cache(); + store_cache.put(B256::repeat_byte(0xbb), "store".into()).expect("put"); + RpcCacheStore::new(store_cache, path.clone()) + .persist() + .expect("provider persist stays best-effort on foreign skip"); + assert_eq!(fs::read_to_string(&path).unwrap(), before, "store path also leaves envelope"); + } + + /// An unrecognized structured JSON shape is also foreign: skip, do not replace. + #[test] + fn test_provider_cache_persist_skips_unrecognized_foreign_shape() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rpc-cache-1.json"); + let before = r#"{"not":"a-provider-cache","nor":"an-envelope"}"#; + fs::write(&path, before).expect("seed foreign"); + + let cache = CacheLayer::new(16).cache(); + cache.put(B256::repeat_byte(0xaa), "ours".into()).expect("put"); + let wrote = save_cache_atomic(&cache, &path).expect("skip is Ok"); + assert!(!wrote); + assert_eq!(fs::read_to_string(&path).unwrap(), before); + } + /// Envelope persist hard-errors when the lock cannot be acquired: the /// capture is the primary output, so a silently unlocked write is worse /// than a failed run. @@ -1026,7 +1096,8 @@ mod tests { let key = B256::repeat_byte(0xcc); let cache = CacheLayer::new(16).cache(); cache.put(key, "ok".into()).expect("put"); - RpcCacheStore::new(cache, path.clone()).persist().expect("persist"); + let wrote = save_cache_atomic(&cache, &path).expect("corrupt degrades to write"); + assert!(wrote, "corrupt target is replaced with ours"); let loaded = CacheLayer::new(16).cache(); loaded.load_cache(path).expect("load"); diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index ed16e4a0..24f13c0c 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -39,6 +39,7 @@ use self::{ transport::{CachingTransport, ReplayTransport, TransportCache}, }; use super::{EvmeError, Result}; +use crate::cache::{acquire_exclusive_lock, lock_sidecar_path}; /// OP-stack provider type used throughout mega-evme. pub type OpProvider = DynProvider; @@ -192,6 +193,21 @@ impl RpcArgs { let cache_store = match cache_path { Some(path) => { if self.clear_cache { + // Same sidecar lock as persist / `cache merge`: without it a + // writer mid re-read-merge-rename can land after the unlink + // (undoing the clear), or the clear can delete a file the + // locked writer just re-read. Fail closed if the lock cannot + // be acquired — the user asked for a deletion that is not + // safe to do unlocked. + let _clear_lock = acquire_exclusive_lock(&path).map_err(|e| { + EvmeError::RpcError(format!( + "Failed to acquire the cache lock {} for clear-cache of {}: {e}. \ + Refusing to clear without it: a concurrent writer could race \ + the unlink and silently recreate or rely on the file.", + lock_sidecar_path(&path).display(), + path.display(), + )) + })?; if let Err(e) = fs::remove_file(&path) { if e.kind() != std::io::ErrorKind::NotFound { return Err(EvmeError::RpcError(format!( @@ -202,6 +218,7 @@ impl RpcArgs { } else { info!(path = %path.display(), "Cleared existing RPC cache"); } + // Lock released on drop before load / provider build continue. } if let Some(parent) = path.parent() { if let Err(e) = fs::create_dir_all(parent) { diff --git a/bin/mega-evme/tests/cache_clear_lock.rs b/bin/mega-evme/tests/cache_clear_lock.rs new file mode 100644 index 00000000..42592349 --- /dev/null +++ b/bin/mega-evme/tests/cache_clear_lock.rs @@ -0,0 +1,144 @@ +//! Cross-process serialization for `--rpc.clear-cache` against the sidecar lock. +//! +//! Clear and persist share the exclusive advisory lock on `.lock`. A +//! clear that unlinks without that lock can race a writer mid re-read-merge- +//! rename: the writer's rename lands after the clear (undoing it), or the clear +//! deletes the file the locked writer just re-read. These tests hold the lock +//! in the test process, spawn a real `mega-evme` that takes the clear-cache +//! path, show the clear does not complete while the lock is held, then release +//! so the clear finishes. + +use std::{ + fs::{self, File, OpenOptions}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +mod common; +use common::MockRpcServer; + +/// How long the lock is held while the spawned clear must make no progress. +const HOLD: Duration = Duration::from_secs(2); + +/// Upper bound on how long clear may take after the lock is released. +const COMPLETION_DEADLINE: Duration = Duration::from_secs(60); + +/// The advisory lock sidecar the binary locks for `target`. +fn sidecar(target: &Path) -> PathBuf { + let mut os = target.as_os_str().to_owned(); + os.push(".lock"); + PathBuf::from(os) +} + +/// Take the exclusive lock on the cache file's sidecar and keep it until the +/// returned handle is dropped. +fn hold_cache_lock(target: &Path) -> File { + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(sidecar(target)) + .expect("open the cache sidecar"); + file.lock().expect("hold the cache lock"); + file +} + +/// Spawn a single-tx online `replay` that hits `build_provider` with +/// `--rpc.clear-cache`. After the clear the lookup fails; we only care that +/// clear ran under the lock. +fn spawn_clear_cache(rpc: &str, cache_dir: &Path) -> Child { + Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args([ + "replay", + "--rpc", + rpc, + "--rpc.cache-dir", + cache_dir.to_str().expect("utf-8 cache dir"), + "--rpc.clear-cache", + "--rpc.max-retries", + "0", + "--rpc.backoff-ms", + "1", + // Dummy hash: chain-id is enough for clear; the later tx fetch fails. + "0x0000000000000000000000000000000000000000000000000000000000000001", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn mega-evme replay --rpc.clear-cache") +} + +/// Assert the spawned process makes no progress for the whole hold window. +fn assert_blocked_while_held(child: &mut Child) { + let deadline = Instant::now() + HOLD; + while Instant::now() < deadline { + if let Some(status) = child.try_wait().expect("poll the clear-cache process") { + panic!("clear-cache completed while the cache lock was held (status {status})",); + } + std::thread::sleep(Duration::from_millis(50)); + } + assert!( + child.try_wait().expect("poll the clear-cache process").is_none(), + "clear-cache must still be waiting for the cache lock", + ); +} + +/// Wait until the child exits (success or failure) after the lock is free. +fn finish(mut child: Child) -> std::process::Output { + let deadline = Instant::now() + COMPLETION_DEADLINE; + loop { + if child.try_wait().expect("poll the clear-cache process").is_some() { + break; + } + assert!( + Instant::now() < deadline, + "clear-cache did not finish within {COMPLETION_DEADLINE:?} of the lock being released", + ); + std::thread::sleep(Duration::from_millis(20)); + } + child.wait_with_output().expect("collect clear-cache output") +} + +/// Clear waits on a held sidecar lock, does not unlink while blocked, and +/// wipes the seeded content only after the lock is released. +/// +/// The file may reappear empty if a later clean-exit-adjacent persist runs +/// after the clear (online replay still installs a disk store); the proof is +/// that the pre-clear seed is gone, not that the path stays unlinked. +#[tokio::test(flavor = "multi_thread")] +async fn test_clear_cache_serializes_with_a_held_sidecar_lock() { + let server = MockRpcServer::start().await; + let chain_id: u64 = 55; + server.respond_eth_chain_id(chain_id, 1).await; + // After clear, eth_getTransactionByHash (and friends) may be called; a + // null result fails the run cleanly without hanging. + server.respond_jsonrpc_null_result(10).await; + + let dir = tempfile::tempdir().expect("tempdir"); + let cache_file = dir.path().join(format!("rpc-cache-{chain_id}.json")); + let seed = r#"[{"key":"0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","value":"seed-before-clear"}]"#; + fs::write(&cache_file, seed).expect("seed cache file"); + assert!(cache_file.exists()); + + let lock = hold_cache_lock(&cache_file); + let mut child = spawn_clear_cache(&server.uri(), dir.path()); + assert_blocked_while_held(&mut child); + assert_eq!( + fs::read_to_string(&cache_file).expect("read while held"), + seed, + "clear must not unlink while the sidecar lock is held", + ); + + drop(lock); + let out = finish(child); + let after = fs::read_to_string(&cache_file).unwrap_or_default(); + assert!( + !after.contains("seed-before-clear"), + "clear must wipe the seeded content once the lock is free.\n\ + after={after}\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr), + ); +} diff --git a/bin/mega-evme/tests/provider.rs b/bin/mega-evme/tests/provider.rs index ea712f94..adb66e3d 100644 --- a/bin/mega-evme/tests/provider.rs +++ b/bin/mega-evme/tests/provider.rs @@ -388,6 +388,54 @@ async fn test_build_provider_clear_cache_deletes_file_before_load() { ); } +/// `--rpc.clear-cache` fails closed when the sidecar lock cannot be acquired: +/// the user asked for a deletion that is not safe to do unlocked, so the file +/// must remain and the build must hard-error rather than unlink without the lock. +#[tokio::test(flavor = "multi_thread")] +async fn test_build_provider_clear_cache_fails_closed_when_lock_unacquirable() { + let server = MockRpcServer::start().await; + server.respond_eth_chain_id(77, 1).await; + + let dir = tempdir().expect("tempdir"); + let cache_file = dir.path().join("rpc-cache-77.json"); + let seed = r#"[{"key":"0x0000000000000000000000000000000000000000000000000000000000000001","value":"seed"}]"#; + std::fs::write(&cache_file, seed).expect("seed cache"); + // A directory in the sidecar's place makes the lock un-acquirable. + std::fs::create_dir(format!("{}.lock", cache_file.display())).expect("occupy sidecar"); + + let args = RpcArgs::parse_from([ + "mega-evme", + "--rpc", + &server.uri(), + "--rpc.cache-max-entries", + "256", + "--rpc.cache-dir", + dir.path().to_str().unwrap(), + "--rpc.clear-cache", + ]); + + let err = args.build_provider().await.expect_err("clear-cache must fail closed on lock"); + match err { + EvmeError::RpcError(msg) => { + assert!(msg.contains("lock"), "error must name the lock failure, got: {msg}"); + assert!( + msg.contains("rpc-cache-77.json.lock") || msg.contains(".lock"), + "error must name the sidecar, got: {msg}", + ); + assert!( + msg.contains("Refusing to clear") || msg.contains("clear"), + "error must state the clear was refused, got: {msg}", + ); + } + other => panic!("expected EvmeError::RpcError, got {other:?}"), + } + assert_eq!( + std::fs::read_to_string(&cache_file).expect("file still readable"), + seed, + "no unlocked unlink happened", + ); +} + /// `--rpc.clear-cache` must hard-error when the file exists but cannot be /// unlinked, rather than warn-and-continue. Silent fallback would reload /// exactly the content the user asked to wipe, defeating the recovery path. @@ -409,6 +457,10 @@ async fn test_build_provider_clear_cache_hard_errors_on_unlink_failure() { // Seed a "polluted" cache file the user would want to wipe. let cache_file = dir.path().join(format!("rpc-cache-{chain_id}.json")); std::fs::write(&cache_file, r#"{"polluted":"content"}"#).expect("write seed"); + // Pre-create the lock sidecar while the directory is still writable. + // Clear acquires that sidecar before unlinking; without it, a read-only + // parent would fail lock creation first and never exercise the unlink path. + std::fs::write(format!("{}.lock", cache_file.display()), b"").expect("seed sidecar"); // Revoke write permission on the parent dir so `remove_file` fails. // Read/execute stays on so the file is still visible to `path.exists()` From b5b6cfe7b76e8156089a7ead7050abc5257f7b5f Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 09:06:00 +0800 Subject: [PATCH 47/64] fix(mega-evme): anchor the single-path replay to its fetched block The parent/block linkage guard passes even when both numbered fetches coherently answer from a replacement block the target is not part of: the lookup reported inclusion in one block, the endpoint served another. The target then executed after every transaction of that block, which counted as preceding, and the run reported a plausible wrong result with exit 0. Anchor a mined target to the block that is about to be replayed: the inclusion hash the lookup reported must equal the fetched block's hash, a mined lookup without an inclusion hash is rejected as an unanchored view, and the fetched body must list the target, since its position there is what defines the preceding transactions. Each violation is an RPC consistency failure (exit 3), matching the batch driver's classes. A pending target fetched the latest height twice, once as its state base and once as the block it is replayed in, and skipped every coherence check: two divergent answers produced a mixed-view execution reporting exit 0. Fetch that block once and fill both roles from it, so the two cannot disagree at all. --- bin/mega-evme/src/replay/cmd.rs | 74 +++++++- bin/mega-evme/tests/common/mod.rs | 47 +++++ bin/mega-evme/tests/exit_codes.rs | 165 +++++++++++++++++ bin/mega-evme/tests/replay_pending.rs | 244 ++++++++++++++++++++++++++ docs/mega-evme/commands/replay.md | 8 + 5 files changed, 531 insertions(+), 7 deletions(-) create mode 100644 bin/mega-evme/tests/replay_pending.rs diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index a7e1dfd9..8d3b5304 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -545,16 +545,31 @@ impl Cmd { "Block numbers determined", ); - let parent_block = provider - .get_block_by_number(state_base_block.into()) - .await - .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? - .ok_or(ReplayError::BlockNotFound(state_base_block))?; + // A mined target forks from its parent block, which is a different block + // than the one it is replayed in. A pending target forks from the latest + // block, which *is* the block it is replayed in: fetching that one height + // twice lets a reorg or a load-balanced endpoint answer the two roles + // from different blocks, and the replay would then run a pre-state from + // one view under a block environment from another. The pending path + // therefore fetches once and uses the same block for both roles, so the + // two roles cannot disagree at all. + let parent_block = if is_pending { + None + } else { + Some( + provider + .get_block_by_number(state_base_block.into()) + .await + .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? + .ok_or(ReplayError::BlockNotFound(state_base_block))?, + ) + }; let block = provider .get_block_by_number(block_number.into()) .await .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? .ok_or(ReplayError::BlockNotFound(block_number))?; + let parent_block = parent_block.unwrap_or_else(|| block.clone()); // Parent/block linkage guard: the two blocks above were fetched by // number in separate calls, so across a reorg or a load-balanced @@ -565,8 +580,8 @@ impl Cmd { // mismatch rather than as the infrastructure failure it is. // // A pending transaction has no such pair: its state base *is* the - // latest block, so both fetches address the same block and there is no - // linkage to check. + // latest block, so one fetch fills both roles and there is no linkage, + // no inclusion, and no membership to check. if !is_pending { let parent_hash = parent_block.hash(); let expected_parent = block.header.parent_hash(); @@ -578,16 +593,61 @@ impl Cmd { the chain settles" ))); } + + // Inclusion guard: the linkage above only proves the two fetched + // blocks belong to one chain, not that the target belongs to them. + // The lookup that resolved the target reported which block includes + // it, in a separate call, so a reorg or a load-balanced endpoint can + // answer both numbered fetches from a replacement block the target is + // not part of. Replaying that block anyway executes the target + // against a body it never ran in. + let fetched = block.hash(); + match target_tx.block_hash { + Some(reported) if reported != fetched => { + return Err(ReplayError::RpcError(format!( + "block {block_number} has hash {fetched}, but the target transaction was \ + resolved as included in {reported}: the endpoint served divergent views \ + of this block (reorg in progress, or a load-balanced endpoint); retry \ + once the chain settles" + ))) + } + Some(_) => {} + // A mined transaction without an inclusion hash is an unanchored + // view: the block number alone cannot prove which block body the + // target belongs to, so there is nothing to anchor the replay to. + None => { + return Err(ReplayError::RpcError(format!( + "endpoint reported a mined transaction in block {block_number} without an \ + inclusion hash: unanchored view" + ))) + } + } } + // The preceding transactions are the block-body entries ahead of the + // target, so the target's own position in that body defines the set and + // the body must contain it. A body that does not list the target would + // silently make every transaction of the block count as preceding and + // execute the target after the whole block. let mut preceding_tx_hashes = vec![]; if !is_pending { + let mut found = false; for hash in block.transactions.hashes() { if hash == tx_hash { + found = true; break; } preceding_tx_hashes.push(hash); } + if !found { + return Err(ReplayError::RpcError(format!( + "block {block_number} ({}) does not list target transaction {tx_hash}, which \ + the endpoint resolved as included in it: the endpoint served divergent views \ + of this block (reorg in progress, or a load-balanced endpoint); retry once \ + the chain settles", + block.hash(), + ))); + } } debug!(chain_id, preceding_count = preceding_tx_hashes.len(), "Replay context ready"); diff --git a/bin/mega-evme/tests/common/mod.rs b/bin/mega-evme/tests/common/mod.rs index d9f4cbc0..6d188a91 100644 --- a/bin/mega-evme/tests/common/mod.rs +++ b/bin/mega-evme/tests/common/mod.rs @@ -219,6 +219,53 @@ impl MockRpcServer { .await; } + /// Mount a mock that answers the first `n` calls of `method` with matching + /// `params` and then stops matching, so a lower-priority mock for the same + /// request serves every later call. + /// + /// Models an endpoint whose answer to one repeated request changes + /// mid-run — a reorg landing between two calls, or a load balancer moving + /// the run to another backend. + pub(crate) async fn respond_method_params_json_n_times( + &self, + method: &str, + params: serde_json::Value, + result: serde_json::Value, + n: u64, + priority: u8, + ) { + let body = serde_json::json!({ "jsonrpc": "2.0", "id": 0, "result": result }); + Mock::given(matchers::method("POST")) + .and(matchers::body_partial_json( + serde_json::json!({ "method": method, "params": params }), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + .up_to_n_times(n) + .with_priority(priority) + .mount(&self.server) + .await; + } + + /// How many single JSON-RPC requests for `method` the server has received. + /// + /// Batched requests (a JSON array body) are not counted: every call these + /// tests make is a single request. + pub(crate) async fn received_method_count(&self, method: &str) -> usize { + let requests = self.server.received_requests().await.expect("received_requests"); + requests + .iter() + .filter(|request| { + serde_json::from_slice::(&request.body) + .ok() + .and_then(|body| { + body.get("method").and_then(serde_json::Value::as_str).map(str::to_string) + }) + .as_deref() == + Some(method) + }) + .count() + } + /// Mount a mock that returns `eth_chainId` with the given chain id. pub(crate) async fn respond_eth_chain_id(&self, chain_id: u64, priority: u8) { let body = serde_json::json!({ diff --git a/bin/mega-evme/tests/exit_codes.rs b/bin/mega-evme/tests/exit_codes.rs index 849d870d..2aa535a3 100644 --- a/bin/mega-evme/tests/exit_codes.rs +++ b/bin/mega-evme/tests/exit_codes.rs @@ -23,6 +23,9 @@ fn cache() -> std::path::PathBuf { /// The transaction the committed capture can replay. const TX_OK: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; +/// Number of the block `TX_OK` was mined in, as the capture reports it. +const BLOCK_NUMBER: u64 = 18_172_461; + /// A hash the capture holds no response for: the question goes unanswered. const UNANSWERABLE_TX: &str = "0x0000000000000000000000000000000000000000000000000000000000000001"; @@ -159,6 +162,91 @@ fn cache_with_unlinked_parent(name: &str, wrong_hash: &str) -> (std::path::PathB (path, original) } +/// Write a copy of the committed capture whose `eth_getTransactionByHash` +/// response for `TX_OK` carries `block_hash` as its inclusion hash, and return +/// its path together with the hash the untouched capture reported. +/// +/// Entries are keyed by the request, so the doctored answer still resolves. The +/// transaction lookup and the block fetch are separate calls, so rewriting only +/// the lookup models an endpoint that describes the target's inclusion +/// differently than the block it serves for that number. Passing +/// [`serde_json::Value::Null`] models a lookup that reports a mined transaction +/// without anchoring it to any block at all. +fn cache_with_inclusion_hash( + name: &str, + block_hash: serde_json::Value, +) -> (std::path::PathBuf, String) { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) + .expect("parse offline cache"); + // Only the transaction's own response carries it as the `hash` field; the + // block body lists bare hashes and a receipt names it `transactionHash`. + let marker = format!("\"hash\":\"{TX_OK}\""); + let mut original = None; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object for {TX_OK}"); + assert!( + result.get("blockNumber").is_some_and(|n| !n.is_null()), + "the captured transaction must report a block number" + ); + original = Some(result["blockHash"].as_str().expect("inclusion hash").to_string()); + result["blockHash"] = block_hash.clone(); + entry["value"] = serde_json::Value::String(response.to_string()); + } + let original = original.expect("the capture must hold exactly one response for the target"); + + let path = + std::env::temp_dir().join(format!("mega_evme_exit_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored cache"); + (path, original) +} + +/// Write a copy of the committed capture whose replayed block no longer lists +/// `TX_OK` in its body, and return its path together with that block's hash. +/// +/// The block keeps its own hash, so the lookup's inclusion hash still matches +/// what the endpoint serves for that number: only the membership the preceding +/// transaction set is derived from is gone. +fn cache_without_target_in_block_body(name: &str) -> (std::path::PathBuf, String) { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) + .expect("parse offline cache"); + let mut block_hash = None; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + let Ok(mut response) = serde_json::from_str::(value) else { + continue; + }; + // A block body is the only response carrying a transaction list. + let Some(txs) = response + .get_mut("result") + .and_then(|result| result.get_mut("transactions")) + .and_then(|txs| txs.as_array_mut()) + else { + continue; + }; + if !txs.iter().any(|hash| hash.as_str() == Some(TX_OK)) { + continue; + } + txs.retain(|hash| hash.as_str() != Some(TX_OK)); + block_hash = Some(response["result"]["hash"].as_str().expect("block hash").to_string()); + entry["value"] = serde_json::Value::String(response.to_string()); + } + let block_hash = block_hash.expect("exactly one captured block body lists the target"); + + let path = + std::env::temp_dir().join(format!("mega_evme_exit_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored cache"); + (path, block_hash) +} + /// Write a `--tx-file` holding `contents`, and return its path. fn tx_file(name: &str, contents: &str) -> std::path::PathBuf { let path = @@ -294,6 +382,83 @@ fn test_unlinked_parent_block_is_an_rpc_failure() { let _ = std::fs::remove_file(&path); } +/// A block whose hash is not the one the target was resolved as included in is +/// an unanswered question: the single-transaction run exits 3 rather than +/// replaying the target against a block it never ran in. +/// +/// The parent linkage can hold while both numbered fetches answer from a +/// replacement block, so the linkage guard alone does not anchor the target. +#[test] +fn test_block_that_does_not_match_the_reported_inclusion_is_an_rpc_failure() { + const WRONG_INCLUSION: &str = + "0x2222222222222222222222222222222222222222222222222222222222222222"; + + let (path, served) = + cache_with_inclusion_hash("wrong_inclusion", serde_json::json!(WRONG_INCLUSION)); + let run = run(&["replay", "--rpc.replay-file", path.to_str().unwrap(), "--json", TX_OK]); + let _ = std::fs::remove_file(&path); + + assert_eq!(run.code(), 3, "a divergent inclusion exits 3.\nstderr: {}", run.stderr); + let error = run.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains(WRONG_INCLUSION) && message.contains(&served), + "the message must name both hashes (served {served}, reported {WRONG_INCLUSION}): {error}" + ); + assert!(message.contains("divergent views"), "the message must name the cause: {error}"); +} + +/// A block body that does not list the target is an unanswered question too: the +/// run exits 3 instead of treating every transaction of the block as preceding +/// and executing the target after the whole block. +#[test] +fn test_target_absent_from_the_block_body_is_an_rpc_failure() { + let (path, block_hash) = cache_without_target_in_block_body("absent_target"); + let run = run(&["replay", "--rpc.replay-file", path.to_str().unwrap(), "--json", TX_OK]); + let _ = std::fs::remove_file(&path); + + assert_eq!(run.code(), 3, "a target absent from the body exits 3.\nstderr: {}", run.stderr); + let error = run.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains(TX_OK) && message.contains(&block_hash), + "the message must name the target and the block it is missing from: {error}" + ); + assert!( + !run.stdout.contains("\"success\""), + "the run must not produce an execution summary:\n{}", + run.stdout + ); +} + +/// A mined lookup carrying no inclusion hash is an unanchored view: the block +/// number alone cannot prove which body the target belongs to, so the run exits +/// 3 — the same class the batch driver rejects it with. +#[test] +fn test_mined_target_without_an_inclusion_hash_is_an_rpc_failure() { + let (path, _) = cache_with_inclusion_hash("unanchored", serde_json::Value::Null); + let run = run(&["replay", "--rpc.replay-file", path.to_str().unwrap(), "--json", TX_OK]); + let _ = std::fs::remove_file(&path); + + assert_eq!(run.code(), 3, "an unanchored view exits 3.\nstderr: {}", run.stderr); + let error = run.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("inclusion hash") && message.contains("unanchored"), + "the message must name the unanchored view: {error}" + ); + assert!( + message.contains(&BLOCK_NUMBER.to_string()), + "the message must name the block number the lookup reported: {error}" + ); +} + /// A batch run's error object follows the per-target lines, so a parser reading /// the stream sees every target before the run-level verdict. #[test] diff --git a/bin/mega-evme/tests/replay_pending.rs b/bin/mega-evme/tests/replay_pending.rs new file mode 100644 index 00000000..bf43d8d4 --- /dev/null +++ b/bin/mega-evme/tests/replay_pending.rs @@ -0,0 +1,244 @@ +//! Integration tests for the pending-transaction single-transaction replay path. +//! +//! A pending target has no parent/block pair: its state base *is* the latest +//! block, which is also the block it is replayed in. The two roles must +//! therefore be filled by one and the same block, and these tests pin that from +//! outside the process — an endpoint that changes its answer between two calls +//! at the same height must not be able to produce a mixed-view replay. +//! +//! They run against a mock JSON-RPC endpoint rather than a recorded capture. An +//! offline capture cannot represent this case at all: identical requests are +//! served from the same keyed entry, so one fetch and two fetches are +//! indistinguishable offline, and a capture recorded for a mined replay answers +//! its state reads at the parent height while a pending replay reads them at the +//! latest one. + +use std::process::Command; + +use serde_json::{json, Value}; + +mod common; +use common::MockRpcServer; + +/// `MegaETH` mainnet, whose published schedule the replayed block runs under. +const CHAIN_ID: u64 = 4326; + +/// Height the endpoint reports as `latest`, and the only block it serves. +const LATEST: u64 = 18_172_461; + +/// A mainnet timestamp inside the `MiniRex` window. +const TIMESTAMP: u64 = 1_764_000_000; + +/// Hash of the block the endpoint serves first for `LATEST`. +const LATEST_HASH: &str = "0x2801837c261826beb8047e46139dfc4eb93ab5b3196ce23f312d3c7658262a62"; + +/// Hash of the replacement block the endpoint serves for `LATEST` from the +/// second call on — the divergent view a second fetch would pick up. +const REPLACEMENT_HASH: &str = "0x3333333333333333333333333333333333333333333333333333333333333333"; + +/// Parent of `LATEST_HASH`, so its header is well formed. +const PARENT_HASH: &str = "0xd482d481e9d11dd116ef6c41bf95ca608f159206c8f07900b1b53936d196ccb3"; + +/// Parent of the replacement block: a different chain, as a reorg would leave it. +const REPLACEMENT_PARENT_HASH: &str = + "0x4444444444444444444444444444444444444444444444444444444444444444"; + +/// Hash of the pending transaction being replayed. +const TX_HASH: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; + +/// Sender of the pending transaction, funded by the mock's blanket balance. +const SENDER: &str = "0x14112799a39f2905b901067d3cd4a1f63c1cebda"; + +/// Recipient of the pending transaction: an account with no code, so the call +/// succeeds without depending on any contract the mock does not serve. +const RECIPIENT: &str = "0x681e908b8ab57c49c74d770f369754ccc3e1ae09"; + +/// A block header the RPC backend and the replay accept, carrying only the +/// fields either of them reads. +fn block_json(hash: &str, parent_hash: &str) -> Value { + json!({ + "hash": hash, + "parentHash": parent_hash, + "number": format!("0x{LATEST:x}"), + "timestamp": format!("0x{TIMESTAMP:x}"), + "gasLimit": "0x2540be400", + "gasUsed": "0x0", + "baseFeePerGas": "0xf4240", + "blobGasUsed": "0x0", + "excessBlobGas": "0x0", + "difficulty": "0x0", + "extraData": "0x00000000fa00000001", + "logsBloom": format!("0x{}", "0".repeat(512)), + "miner": "0x4200000000000000000000000000000000000011", + "mixHash": "0x5cd8791a477b467456670744425e11d5bd91fd54575d6d3bf80d761ab39d957f", + "nonce": "0x0000000000000000", + "parentBeaconBlockRoot": + "0x67123956bf748ccfcfa68f03531dd12c1c647f9f31cc91935ce4271fa7399e24", + "receiptsRoot": "0x16fe124682128dd43a5da7f2cee0a3bf076deaf12682d19c656914bbea4615e3", + "requestsHash": "0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x43e7", + "stateRoot": "0xa342aba318978654abcf7f09f9494ed271e2136040b628edacb6d384e9074416", + "transactionsRoot": "0x2f3c5d0b0c4c8d34dd4e1c8bb4b4a4b6d6a2a3d3b8f6a9a2c1d0e9f8a7b6c5d4", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "uncles": [], + "withdrawals": [], + "transactions": [], + }) +} + +/// The replayed transaction, reported as pending: no block number and no +/// inclusion hash. +fn pending_tx_json() -> Value { + json!({ + "type": "0x2", + "chainId": format!("0x{CHAIN_ID:x}"), + "nonce": "0x0", + "gas": "0x249f0", + "maxFeePerGas": "0x200b20", + "maxPriorityFeePerGas": "0x186a0", + "gasPrice": "0x10c8e0", + "to": RECIPIENT, + "value": "0x0", + "accessList": [], + "input": "0x", + "r": "0xa19f0f1f52e2951452711b4f4aa5d177442c9a56abeb609b803fe2412ed24946", + "s": "0x7af21777b2e7d91c745d0077ba2726ee1bb75ccf00039a6218d64fdced768491", + "yParity": "0x0", + "v": "0x0", + "hash": TX_HASH, + "from": SENDER, + "blockHash": Value::Null, + "blockNumber": Value::Null, + "transactionIndex": Value::Null, + }) +} + +/// A mock endpoint holding one pending transaction, whose `latest` height is +/// answered with [`LATEST_HASH`] once and with [`REPLACEMENT_HASH`] from the +/// second call on. +/// +/// Account reads are answered blanket: every account holds 1 ETH, has nonce 0, +/// no code, and zero storage. +async fn mock_chain() -> MockRpcServer { + let server = MockRpcServer::start().await; + server.respond_eth_chain_id(CHAIN_ID, 1).await; + server.respond_method_result("eth_blockNumber", &format!("0x{LATEST:x}"), 2).await; + server + .respond_method_params_json_n_times( + "eth_getBlockByNumber", + json!([format!("0x{LATEST:x}"), false]), + block_json(LATEST_HASH, PARENT_HASH), + 1, + 2, + ) + .await; + server + .respond_method_json( + "eth_getBlockByNumber", + block_json(REPLACEMENT_HASH, REPLACEMENT_PARENT_HASH), + 3, + ) + .await; + server.respond_method_json("eth_getTransactionByHash", pending_tx_json(), 3).await; + server.respond_method_result("eth_getBalance", "0xde0b6b3a7640000", 4).await; + server.respond_method_result("eth_getTransactionCount", "0x0", 4).await; + server.respond_method_result("eth_getCode", "0x", 4).await; + server + .respond_method_result( + "eth_getStorageAt", + "0x0000000000000000000000000000000000000000000000000000000000000000", + 4, + ) + .await; + server +} + +/// Outcome of one `mega-evme replay` invocation. +struct Run { + code: Option, + stdout: String, + stderr: String, +} + +impl Run { + /// The single `--json` summary the run printed. + fn summary(&self) -> Value { + let mut values = common::json_values(&self.stdout); + if values.last().is_some_and(common::is_run_error) { + values.pop(); + } + assert_eq!( + values.len(), + 1, + "expected one summary on stdout:\n{}\nstderr:\n{}", + self.stdout, + self.stderr, + ); + values.pop().expect("checked above") + } +} + +/// Replay the mock's pending transaction. +fn replay(server: &MockRpcServer) -> Run { + let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args(["replay", TX_HASH, "--rpc", &server.uri()]) + .args(["--rpc.no-cache-file", "--rpc.max-retries", "0", "--rpc.backoff-ms", "1", "--json"]) + .output() + .expect("failed to run mega-evme"); + Run { + code: output.status.code(), + stdout: String::from_utf8(output.stdout).expect("stdout is utf-8"), + stderr: String::from_utf8(output.stderr).expect("stderr is utf-8"), + } +} + +/// A pending replay fetches the latest block exactly once and fills both the +/// state-base and the replayed-block role from that one answer. +/// +/// The endpoint changes its answer for the same height after the first call, so +/// a second fetch would hand the run a replacement block: the pre-state would +/// come from one view and the block environment from the other, and the run +/// would still exit 0 while reporting a receipt anchored to a block it never +/// forked from. One fetch removes that possibility structurally rather than +/// detecting it afterwards. +#[tokio::test(flavor = "multi_thread")] +async fn test_pending_replay_fetches_the_latest_block_once() { + let server = mock_chain().await; + + let run = replay(&server); + + assert_eq!(run.code, Some(0), "stdout:\n{}\nstderr:\n{}", run.stdout, run.stderr); + assert_eq!( + server.received_method_count("eth_getBlockByNumber").await, + 1, + "the two roles must be filled by a single fetch:\n{}", + run.stdout, + ); + let receipt = &run.summary()["receipt"]; + assert_eq!( + receipt["blockHash"].as_str(), + Some(LATEST_HASH), + "the replay must report the block it forked from, not the replacement: {receipt}", + ); + assert_eq!(receipt["blockNumber"].as_str(), Some(format!("0x{LATEST:x}").as_str())); +} + +/// A pending target still replays against a coherent endpoint: the reused block +/// fills both roles, so the transaction executes on top of the latest block and +/// reports its result there. +#[tokio::test(flavor = "multi_thread")] +async fn test_pending_replay_executes_against_the_latest_block() { + let server = mock_chain().await; + + let run = replay(&server); + + assert_eq!(run.code, Some(0), "stdout:\n{}\nstderr:\n{}", run.stdout, run.stderr); + let summary = run.summary(); + assert_eq!(summary["success"], json!(true), "the pending transaction must execute: {summary}"); + assert_eq!( + summary["receipt"]["transactionHash"].as_str(), + Some(TX_HASH), + "the receipt must describe the replayed transaction: {summary}", + ); +} diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index a8618c83..210548c6 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -30,6 +30,14 @@ The transaction hash to replay (32-byte hex). `mega-evme` re-executes the transaction locally using state and block context sourced from either an RPC endpoint or a local fixture file. This gives you a fully reproducible execution without needing a local archive node. +Resolving a mined transaction takes three separate calls — the transaction lookup, the block it reports, and that block's parent — which a reorg in progress or a load-balanced endpoint can answer from different views of the chain. +Replaying a mixed view yields a plausible but wrong result, so the answers are checked against each other and a disagreement is reported as an RPC failure (exit `3`) instead of being replayed: +the parent block must be the replayed block's parent, the fetched block must be the one the transaction was resolved as included in, and that block's body must list the transaction — its position there is what defines the preceding transactions replayed ahead of it. +A mined transaction the endpoint reports without an inclusion hash is rejected the same way: the block number alone cannot anchor the replay to a block body. + +A pending transaction has no such pair, since its state base is the latest block, which is also the block it is replayed in. +That block is fetched once and fills both roles, so the two cannot disagree. + ### `--rpc ` Aliases: `--rpc-url` From a48ee02353b683c0da0b9294e84bb7d2c255204e Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 10:21:52 +0800 Subject: [PATCH 48/64] fix(mega-evme): hold clear-cache lock through load Keep the sidecar exclusive lock from before unlink until after the exists-check and load_cache so a concurrent persist or cache merge cannot recreate the file in the released window and have the clearing invocation load the entries the user asked to remove. Add a cross-process test that injects a cache file while clear is queued on the lock and asserts the injected entries do not survive. --- bin/mega-evme/src/common/provider/mod.rs | 35 ++++++++++++----- bin/mega-evme/tests/cache_clear_lock.rs | 50 ++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 24f13c0c..51913d62 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -192,14 +192,22 @@ impl RpcArgs { let cache = cache_layer.cache(); let cache_store = match cache_path { Some(path) => { - if self.clear_cache { - // Same sidecar lock as persist / `cache merge`: without it a - // writer mid re-read-merge-rename can land after the unlink - // (undoing the clear), or the clear can delete a file the - // locked writer just re-read. Fail closed if the lock cannot - // be acquired — the user asked for a deletion that is not - // safe to do unlocked. - let _clear_lock = acquire_exclusive_lock(&path).map_err(|e| { + // Same sidecar lock as persist / `cache merge`. Held for the + // whole clear critical section: acquire → unlink → exists-check + // → load (or the decision that nothing is on disk to load). + // Releasing after unlink but before load leaves a window where a + // concurrent locked writer can recreate the file with the + // entries the user asked to remove, and this invocation then + // loads them. Fail closed if the lock cannot be acquired — the + // user asked for a deletion that is not safe to do unlocked. + // + // Lock ordering with same-process persist: this guard lives only + // for provider build and is dropped before `BuildProviderOutput` + // returns; clean-exit `RpcCacheStore::persist` acquires later. + // The two critical sections never overlap in one process, so + // clear cannot deadlock against its own later persist. + let clear_lock = if self.clear_cache { + Some(acquire_exclusive_lock(&path).map_err(|e| { EvmeError::RpcError(format!( "Failed to acquire the cache lock {} for clear-cache of {}: {e}. \ Refusing to clear without it: a concurrent writer could race \ @@ -207,7 +215,11 @@ impl RpcArgs { lock_sidecar_path(&path).display(), path.display(), )) - })?; + })?) + } else { + None + }; + if self.clear_cache { if let Err(e) = fs::remove_file(&path) { if e.kind() != std::io::ErrorKind::NotFound { return Err(EvmeError::RpcError(format!( @@ -218,7 +230,6 @@ impl RpcArgs { } else { info!(path = %path.display(), "Cleared existing RPC cache"); } - // Lock released on drop before load / provider build continue. } if let Some(parent) = path.parent() { if let Err(e) = fs::create_dir_all(parent) { @@ -242,6 +253,10 @@ impl RpcArgs { ); } } + // Release after unlink + exists + load; later provider construction + // and exit-time persist run without this guard (they never overlap + // it in the same process — see lock-ordering note above). + drop(clear_lock); RpcCacheStore::new(cache, path) } None => RpcCacheStore::noop(), diff --git a/bin/mega-evme/tests/cache_clear_lock.rs b/bin/mega-evme/tests/cache_clear_lock.rs index 42592349..ee5cdd87 100644 --- a/bin/mega-evme/tests/cache_clear_lock.rs +++ b/bin/mega-evme/tests/cache_clear_lock.rs @@ -142,3 +142,53 @@ async fn test_clear_cache_serializes_with_a_held_sidecar_lock() { String::from_utf8_lossy(&out.stderr), ); } + +/// A concurrent writer that lands under the lock while clear is queued must +/// still be wiped: clear's critical section is unlink + exists-check + load, +/// so the file written just before clear acquires cannot be reloaded into the +/// clearing session. +/// +/// Inverse of `test_clear_cache_serializes_with_a_held_sidecar_lock`: that test +/// seeds before the hold and proves clear does not act while blocked; this one +/// writes only while clear is blocked (as a clean-exit persist would, under +/// the same lock) and proves the injected entries do not survive the clear. +#[tokio::test(flavor = "multi_thread")] +async fn test_clear_cache_wipes_file_written_while_queued_on_lock() { + let server = MockRpcServer::start().await; + let chain_id: u64 = 56; + server.respond_eth_chain_id(chain_id, 1).await; + server.respond_jsonrpc_null_result(10).await; + + let dir = tempfile::tempdir().expect("tempdir"); + let cache_file = dir.path().join(format!("rpc-cache-{chain_id}.json")); + // No seed before the hold: the only pollution is what the concurrent + // writer leaves under the lock while clear waits. + assert!(!cache_file.exists(), "precondition: cache file must be absent"); + + let lock = hold_cache_lock(&cache_file); + let mut child = spawn_clear_cache(&server.uri(), dir.path()); + assert_blocked_while_held(&mut child); + + // Concurrent persist finishes under the lock clear is waiting for: its + // rename lands, then it releases — clear acquires next and must treat this + // file as the one to wipe, not as content to load after an early unlock. + let injected = r#"[{"key":"0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","value":"injected-while-clear-queued"}]"#; + fs::write(&cache_file, injected).expect("write while clear is queued"); + assert_eq!( + fs::read_to_string(&cache_file).expect("read injected"), + injected, + "injected file must still be present while clear is blocked", + ); + + drop(lock); + let out = finish(child); + let after = fs::read_to_string(&cache_file).unwrap_or_default(); + assert!( + !after.contains("injected-while-clear-queued"), + "clear must delete the concurrent write and start empty; injected \ + entries must not reappear via load-then-persist.\n\ + after={after}\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr), + ); +} From a57bbc428ad37d0f3fb4eef132b17bf9f4b7de5c Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 10:25:32 +0800 Subject: [PATCH 49/64] fix(mega-evme): classify single-path target metadata before any fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single-transaction replay path decided "pending" from `block_number` alone, so an inclusion hash paired with a null number was replayed against latest with exit 0 — every inclusion and body guard skipped — while a block number paired with a null hash was only rejected after the block and parent fetches, where a missing block or a broken parent linkage could mask it. Match the `(block_number, block_hash)` pair exhaustively right after the target lookup: both present is mined, neither is pending, and the two mixed shapes fail as RPC failures from the metadata alone, before any block is fetched. The mined arm carries its inclusion hash forward, so the later inclusion guard no longer needs a null case at all. --- bin/mega-evme/src/replay/cmd.rs | 68 ++++++++----- bin/mega-evme/tests/replay_pending.rs | 134 ++++++++++++++++++++++++-- docs/mega-evme/commands/replay.md | 8 +- 3 files changed, 176 insertions(+), 34 deletions(-) diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 8d3b5304..0004c893 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -529,14 +529,44 @@ impl Cmd { .ok_or_else(|| ReplayError::TransactionNotFound(tx_hash))?; debug!(block_number = ?target_tx.block_number, "Transaction found"); - let (state_base_block, block_number, is_pending) = if let Some(n) = target_tx.block_number { - (n - 1, n, false) + // Classify the target from its `(block_number, block_hash)` pair before + // anything else is fetched. Every shape the endpoint can return is + // handled explicitly, so a contradictory row cannot fall through into the + // pending arm, and a shape that can never be replayed is answered from the + // metadata alone — no block fetch precedes the verdict, and no fetch + // failure can mask it. A mined target keeps its inclusion hash here, which + // is what the block fetched below is anchored against. + let mined: Option<(u64, B256)> = match (target_tx.block_number, target_tx.block_hash) { + (Some(number), Some(inclusion)) => Some((number, inclusion)), + // A mined transaction without an inclusion hash is an unanchored + // view: the block number alone cannot prove which block body the + // target belongs to, so there is nothing to anchor the replay to. + (Some(number), None) => { + return Err(ReplayError::RpcError(format!( + "endpoint reported a mined transaction in block {number} without an \ + inclusion hash: unanchored view" + ))) + } + // A hash proves inclusion; a null number denies it. That pair is + // self-contradictory metadata, not a pending transaction. + (None, Some(inclusion)) => { + return Err(ReplayError::RpcError(format!( + "endpoint reported inclusion hash {inclusion} without a block \ + number: contradictory metadata" + ))) + } + (None, None) => None, + }; + let is_pending = mined.is_none(); + + let (state_base_block, block_number) = if let Some((n, _)) = mined { + (n - 1, n) } else { let latest = provider .get_block_number() .await .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))?; - (latest, latest, true) + (latest, latest) }; debug!( state_base_block = state_base_block, @@ -582,7 +612,7 @@ impl Cmd { // A pending transaction has no such pair: its state base *is* the // latest block, so one fetch fills both roles and there is no linkage, // no inclusion, and no membership to check. - if !is_pending { + if let Some((_, reported)) = mined { let parent_hash = parent_block.hash(); let expected_parent = block.header.parent_hash(); if parent_hash != expected_parent { @@ -600,27 +630,17 @@ impl Cmd { // it, in a separate call, so a reorg or a load-balanced endpoint can // answer both numbered fetches from a replacement block the target is // not part of. Replaying that block anyway executes the target - // against a body it never ran in. + // against a body it never ran in. The reported hash is the one the + // metadata classification kept, so a mined target always has one to + // anchor against. let fetched = block.hash(); - match target_tx.block_hash { - Some(reported) if reported != fetched => { - return Err(ReplayError::RpcError(format!( - "block {block_number} has hash {fetched}, but the target transaction was \ - resolved as included in {reported}: the endpoint served divergent views \ - of this block (reorg in progress, or a load-balanced endpoint); retry \ - once the chain settles" - ))) - } - Some(_) => {} - // A mined transaction without an inclusion hash is an unanchored - // view: the block number alone cannot prove which block body the - // target belongs to, so there is nothing to anchor the replay to. - None => { - return Err(ReplayError::RpcError(format!( - "endpoint reported a mined transaction in block {block_number} without an \ - inclusion hash: unanchored view" - ))) - } + if reported != fetched { + return Err(ReplayError::RpcError(format!( + "block {block_number} has hash {fetched}, but the target transaction was \ + resolved as included in {reported}: the endpoint served divergent views of \ + this block (reorg in progress, or a load-balanced endpoint); retry once the \ + chain settles" + ))); } } diff --git a/bin/mega-evme/tests/replay_pending.rs b/bin/mega-evme/tests/replay_pending.rs index bf43d8d4..4328f2e8 100644 --- a/bin/mega-evme/tests/replay_pending.rs +++ b/bin/mega-evme/tests/replay_pending.rs @@ -1,4 +1,5 @@ -//! Integration tests for the pending-transaction single-transaction replay path. +//! Integration tests for the pending-transaction single-transaction replay path +//! and for the target-metadata classification that decides who enters it. //! //! A pending target has no parent/block pair: its state base *is* the latest //! block, which is also the block it is replayed in. The two roles must @@ -6,6 +7,11 @@ //! outside the process — an endpoint that changes its answer between two calls //! at the same height must not be able to produce a mixed-view replay. //! +//! Only a target reporting neither a block number nor an inclusion hash is +//! pending. The other `(block_number, block_hash)` shapes are classified from the +//! metadata alone, before any block is fetched, and these tests pin that too by +//! counting the requests the endpoint receives. +//! //! They run against a mock JSON-RPC endpoint rather than a recorded capture. An //! offline capture cannot represent this case at all: identical requests are //! served from the same keyed entry, so one fetch and two fetches are @@ -87,9 +93,10 @@ fn block_json(hash: &str, parent_hash: &str) -> Value { }) } -/// The replayed transaction, reported as pending: no block number and no -/// inclusion hash. -fn pending_tx_json() -> Value { +/// The replayed transaction, carrying the `(blockNumber, blockHash)` pair the +/// endpoint reports for it. Everything else is the same transaction, so a test +/// varies only the metadata the classification reads. +fn tx_json(block_number: Value, block_hash: Value) -> Value { json!({ "type": "0x2", "chainId": format!("0x{CHAIN_ID:x}"), @@ -108,19 +115,32 @@ fn pending_tx_json() -> Value { "v": "0x0", "hash": TX_HASH, "from": SENDER, - "blockHash": Value::Null, - "blockNumber": Value::Null, + "blockHash": block_hash, + "blockNumber": block_number, "transactionIndex": Value::Null, }) } +/// The replayed transaction, reported as pending: no block number and no +/// inclusion hash. +fn pending_tx_json() -> Value { + tx_json(Value::Null, Value::Null) +} + /// A mock endpoint holding one pending transaction, whose `latest` height is /// answered with [`LATEST_HASH`] once and with [`REPLACEMENT_HASH`] from the /// second call on. +async fn mock_chain() -> MockRpcServer { + mock_chain_serving(pending_tx_json()).await +} + +/// A mock endpoint that resolves the target to `tx`, and otherwise behaves like +/// [`mock_chain`]: the `latest` height is answered with [`LATEST_HASH`] once and +/// with [`REPLACEMENT_HASH`] from the second call on. /// /// Account reads are answered blanket: every account holds 1 ETH, has nonce 0, /// no code, and zero storage. -async fn mock_chain() -> MockRpcServer { +async fn mock_chain_serving(tx: Value) -> MockRpcServer { let server = MockRpcServer::start().await; server.respond_eth_chain_id(CHAIN_ID, 1).await; server.respond_method_result("eth_blockNumber", &format!("0x{LATEST:x}"), 2).await; @@ -140,7 +160,7 @@ async fn mock_chain() -> MockRpcServer { 3, ) .await; - server.respond_method_json("eth_getTransactionByHash", pending_tx_json(), 3).await; + server.respond_method_json("eth_getTransactionByHash", tx, 3).await; server.respond_method_result("eth_getBalance", "0xde0b6b3a7640000", 4).await; server.respond_method_result("eth_getTransactionCount", "0x0", 4).await; server.respond_method_result("eth_getCode", "0x", 4).await; @@ -177,6 +197,19 @@ impl Run { ); values.pop().expect("checked above") } + + /// The structured error object a failing `--json` run ends with. + fn error_object(&self) -> Value { + let values = common::json_values(&self.stdout); + let last = values.last().unwrap_or_else(|| { + panic!("a failing --json run must not leave stdout empty:\nstderr:\n{}", self.stderr) + }); + assert!( + common::is_run_error(last), + "the last stdout value must be the error object, got: {last}" + ); + last.clone() + } } /// Replay the mock's pending transaction. @@ -242,3 +275,88 @@ async fn test_pending_replay_executes_against_the_latest_block() { "the receipt must describe the replayed transaction: {summary}", ); } + +/// An inclusion hash paired with a null block number is contradictory metadata, +/// not a pending transaction: the hash proves inclusion while the null number +/// denies it. The run answers that from the metadata alone — exit 3 without a +/// single block fetch — rather than reading the null number as "pending", +/// skipping every inclusion and body guard, and replaying the target against +/// latest with exit 0. +#[tokio::test(flavor = "multi_thread")] +async fn test_inclusion_hash_without_a_block_number_is_rejected_before_any_fetch() { + const INCLUSION: &str = "0x5555555555555555555555555555555555555555555555555555555555555555"; + + let server = mock_chain_serving(tx_json(Value::Null, json!(INCLUSION))).await; + + let run = replay(&server); + + assert_eq!( + run.code, + Some(3), + "contradictory metadata exits 3.\nstdout:\n{}\nstderr:\n{}", + run.stdout, + run.stderr, + ); + assert_eq!( + server.received_method_count("eth_getBlockByNumber").await, + 0, + "the verdict must precede every block fetch:\n{}", + run.stdout, + ); + assert_eq!( + server.received_method_count("eth_blockNumber").await, + 0, + "the verdict must precede the latest-height lookup too:\n{}", + run.stdout, + ); + let error = run.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains(INCLUSION) && message.contains("contradictory metadata"), + "the message must name the hash and the contradiction: {error}" + ); + assert!( + !run.stdout.contains("\"success\""), + "the run must not produce an execution summary:\n{}", + run.stdout, + ); +} + +/// A block number paired with a null inclusion hash is an unanchored view: the +/// number alone cannot prove which block body the target belongs to. The run +/// answers that from the metadata alone — exit 3 without a single block fetch — +/// so neither a missing block nor a broken parent linkage can mask it. +#[tokio::test(flavor = "multi_thread")] +async fn test_mined_target_without_an_inclusion_hash_is_rejected_before_any_fetch() { + let server = mock_chain_serving(tx_json(json!(format!("0x{LATEST:x}")), Value::Null)).await; + + let run = replay(&server); + + assert_eq!( + run.code, + Some(3), + "an unanchored view exits 3.\nstdout:\n{}\nstderr:\n{}", + run.stdout, + run.stderr, + ); + assert_eq!( + server.received_method_count("eth_getBlockByNumber").await, + 0, + "the verdict must precede every block fetch:\n{}", + run.stdout, + ); + let error = run.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("inclusion hash") && message.contains("unanchored"), + "the message must name the unanchored view: {error}" + ); + assert!( + message.contains(&LATEST.to_string()), + "the message must name the block number the lookup reported: {error}" + ); +} diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 210548c6..9ba18e6f 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -30,10 +30,14 @@ The transaction hash to replay (32-byte hex). `mega-evme` re-executes the transaction locally using state and block context sourced from either an RPC endpoint or a local fixture file. This gives you a fully reproducible execution without needing a local archive node. -Resolving a mined transaction takes three separate calls — the transaction lookup, the block it reports, and that block's parent — which a reorg in progress or a load-balanced endpoint can answer from different views of the chain. +The transaction lookup's own block number and inclusion hash are classified first, before any block is fetched. +Both present is a mined target; neither present is a pending one. +The two mixed shapes cannot be replayed at all and are reported as RPC failures (exit `3`) from the metadata alone, so no fetch precedes the verdict and no later failure can mask it: +a block number without an inclusion hash is an unanchored view, since the number alone cannot anchor the replay to a block body, and an inclusion hash without a block number is contradictory metadata, since the hash proves inclusion while the missing number denies it. + +Resolving a mined transaction then takes two more calls — the block the lookup reports and that block's parent — which a reorg in progress or a load-balanced endpoint can answer from different views of the chain. Replaying a mixed view yields a plausible but wrong result, so the answers are checked against each other and a disagreement is reported as an RPC failure (exit `3`) instead of being replayed: the parent block must be the replayed block's parent, the fetched block must be the one the transaction was resolved as included in, and that block's body must list the transaction — its position there is what defines the preceding transactions replayed ahead of it. -A mined transaction the endpoint reports without an inclusion hash is rejected the same way: the block number alone cannot anchor the replay to a block body. A pending transaction has no such pair, since its state base is the latest block, which is also the block it is replayed in. That block is fetched once and fills both roles, so the two cannot disagree. From 5039318f468749154e381bb35babbdf2cdc26d05 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 11:08:05 +0800 Subject: [PATCH 50/64] fix(mega-evme): classify batch inclusion per target, not first-seen Validate each --tx-file target's reported inclusion hash against the fetched block instead of a job-level first-seen anchor, so same-height stale/canonical pairs get order-independent outcomes. Classify anchored-but-absent targets as rpc (endpoint self-contradiction) rather than not_found; genuine null resolution stays not_found. --- bin/mega-evme/src/replay/batch.rs | 211 ++++++++++++++++--------- bin/mega-evme/tests/replay_batch.rs | 234 ++++++++++++++++++++++++++++ 2 files changed, 373 insertions(+), 72 deletions(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 75e80760..8413fae5 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -300,18 +300,26 @@ impl BatchTally { } } +/// One target of a [`BlockJob`], carrying the inclusion hash it resolved with. +/// +/// Per-target inclusion (rather than a job-level first-seen anchor) keeps +/// outcomes order-independent: two same-height targets that report different +/// hashes each validate against the fetched block on their own. +struct JobTarget { + hash: B256, + /// Inclusion block hash from `eth_getTransactionByHash` (`--tx-file`). + /// `None` for `--block` targets, which come from the body itself. + inclusion_hash: Option, +} + /// One block's worth of work. struct BlockJob { /// Number of the block holding the targets. number: u64, /// Block body, present when planning already fetched it (`--block`). block: Option>, - /// Hashes of the transactions whose results are reported. - targets: Vec, - /// Block hash the targets reported as their inclusion, when a resolution - /// step observed one (`--tx-file`). `None` for `--block`, whose targets come - /// from the block body itself and so cannot disagree with it. - inclusion_hash: Option, + /// Targets whose results are reported for this block. + targets: Vec, } /// Fixture work for one target, held until `finish()` succeeds. @@ -426,8 +434,12 @@ where vec![BlockJob { number: *number, block: Some(block), - targets, - inclusion_hash: None, + // Whole-block mode takes its targets from the body, so there + // is no separate inclusion claim to reconcile later. + targets: targets + .into_iter() + .map(|hash| JobTarget { hash, inclusion_hash: None }) + .collect(), }] } } @@ -493,11 +505,16 @@ where /// /// Returns the per-block jobs in ascending block order, plus the failures for /// hashes that could not be resolved (in the order they were requested). +/// +/// Grouping is by block number only. Each target keeps the inclusion hash its +/// own lookup reported; agreement with the fetched block is checked later in +/// [`replay_block`], so two same-height targets that disagree with each other +/// still get independent outcomes instead of a first-seen race. async fn resolve_targets

(provider: &P, hashes: &[B256]) -> (Vec, Vec) where P: Provider, { - let mut grouped: BTreeMap, Option)> = BTreeMap::new(); + let mut grouped: BTreeMap> = BTreeMap::new(); let mut failures = Vec::new(); for hash in hashes { @@ -517,33 +534,15 @@ where // wildcard into the pending arm. Ok(Some(tx)) => match (tx.block_number, tx.block_hash) { (Some(number), Some(theirs)) => { - let (targets, inclusion) = grouped.entry(number).or_default(); - // Two targets resolving to the same number but different - // block hashes means the endpoint served two views. Neither - // can be trusted, so the disagreeing target is reported as - // unanswered rather than silently replayed against one view. - match *inclusion { - Some(seen) if seen != theirs => { - failures.push(FailedTx { - tx_hash: *hash, - kind: BatchErrorKind::Rpc, - message: format!( - "inclusion block hash {theirs} for block {number} differs \ - from {seen} reported by an earlier target of the same \ - block: the endpoint served divergent views" - ), - }); - continue; - } - None => *inclusion = Some(theirs), - Some(_) => {} - } - targets.push(*hash); + grouped + .entry(number) + .or_default() + .push(JobTarget { hash: *hash, inclusion_hash: Some(theirs) }); } // A mined transaction without an inclusion hash is an unanchored // view: the number alone cannot prove which block body to replay - // against, so the target is unanswered rather than queued with - // inclusion_hash left unset. + // against, so the target is unanswered rather than queued without + // an inclusion claim. (Some(number), None) => failures.push(FailedTx { tx_hash: *hash, kind: BatchErrorKind::Rpc, @@ -573,12 +572,7 @@ where let jobs = grouped .into_iter() - .map(|(number, (targets, inclusion_hash))| BlockJob { - number, - block: None, - targets, - inclusion_hash, - }) + .map(|(number, targets)| BlockJob { number, block: None, targets }) .collect(); (jobs, failures) } @@ -605,13 +599,18 @@ async fn replay_block

( where P: Provider + Clone + std::fmt::Debug, { - let BlockJob { number, block, targets, inclusion_hash } = job; + let BlockJob { number, block, targets } = job; let verify_receipt = report.verify_receipt; let dump_dir = report.dump_fixture_dir.as_deref(); let overwrite = report.overwrite; + let target_hashes = || targets.iter().map(|t| t.hash); if number == 0 { - return fail_all(&targets, BatchErrorKind::Rpc, "Block 0 has no parent block to fork from"); + return fail_all( + target_hashes(), + BatchErrorKind::Rpc, + "Block 0 has no parent block to fork from", + ); } let block = match block { @@ -619,14 +618,14 @@ where None => match fetch_block(provider, number).await { Ok(block) => block, Err(e) => { - return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + return fail_all(target_hashes(), BatchErrorKind::Rpc, &e.to_string()); } }, }; let parent_block = match fetch_block(provider, number - 1).await { Ok(block) => block, Err(e) => { - return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + return fail_all(target_hashes(), BatchErrorKind::Rpc, &e.to_string()); } }; @@ -650,24 +649,58 @@ where block describes a different chain than the block being replayed (reorg in progress, \ or a load-balanced endpoint serving divergent views); retry once the chain settles" ); - return fail_all(&targets, BatchErrorKind::Rpc, &message); + return fail_all(target_hashes(), BatchErrorKind::Rpc, &message); } - // Inclusion guard: `--tx-file` resolved each target through - // `eth_getTransactionByHash`, which reported the block it belongs to. If the - // block fetched by that number is a different one, the endpoint served two - // views and the targets do not belong to what is about to be replayed. - if let Some(expected) = inclusion_hash { - let fetched = block.hash(); - if fetched != expected { - let message = format!( - "block {number} has hash {fetched}, but its targets were resolved as included in \ - {expected}: the endpoint served divergent views of this block (reorg in \ - progress, or a load-balanced endpoint); retry once the chain settles" - ); - return fail_all(&targets, BatchErrorKind::Rpc, &message); + // Per-target inclusion and membership guards. `--tx-file` resolved each + // target through `eth_getTransactionByHash`, which reported the block it + // belongs to. Agreement is checked against the fetched body, not against + // a first-seen peer, so two same-height targets that report different + // hashes get independent outcomes. A target whose reported hash matches + // the body but is missing from it is an endpoint self-contradiction (`rpc`), + // not a definitive "unknown hash". + let fetched = block.hash(); + let body_txs: HashSet = block.transactions.hashes().collect(); + let mut entries = Vec::with_capacity(targets.len()); + let mut active: Vec = Vec::new(); + for target in &targets { + if let Some(reported) = target.inclusion_hash { + if reported != fetched { + entries.push(failure( + target.hash, + BatchErrorKind::Rpc, + format!( + "block {number} has hash {fetched}, but the target transaction was \ + resolved as included in {reported}: the endpoint served divergent \ + views of this block (reorg in progress, or a load-balanced \ + endpoint); retry once the chain settles" + ), + )); + continue; + } + if !body_txs.contains(&target.hash) { + entries.push(failure( + target.hash, + BatchErrorKind::Rpc, + format!( + "block {number} ({fetched}) does not list target transaction {}, which \ + the endpoint resolved as included in it: the endpoint served \ + divergent views of this block (reorg in progress, or a \ + load-balanced endpoint); retry once the chain settles", + target.hash, + ), + )); + continue; + } } + active.push(target.hash); } + // Every target either failed an inclusion/membership check or was a + // `--block` target already taken from the body. Nothing left to execute. + if active.is_empty() { + return entries; + } + let targets = active; // Fetch the on-chain receipts before the block runs. Needed for // `--verify-receipt` (mismatch vs unverified) and for `--dump-fixture-dir` @@ -700,13 +733,13 @@ where let cfg_env = match chain_args.create_cfg_env() { Ok(cfg) => cfg, Err(e) => { - return fail_all(&targets, BatchErrorKind::Execution, &e.to_string()); + return fail_remaining(&targets, entries, BatchErrorKind::Execution, &e.to_string()); } }; let block_env = match retrieve_block_env(&block) { Ok(env) => env, Err(e) => { - return fail_all(&targets, BatchErrorKind::Execution, &e.to_string()); + return fail_remaining(&targets, entries, BatchErrorKind::Execution, &e.to_string()); } }; let executed_spec = cfg_env.spec; @@ -714,7 +747,7 @@ where let Some(hardfork) = hardforks.hardfork(timestamp) else { let message = format!("No `MegaHardfork` active at block timestamp: {timestamp}"); - return fail_all(&targets, BatchErrorKind::Execution, &message); + return fail_remaining(&targets, entries, BatchErrorKind::Execution, &message); }; let block_limits = BlockLimits::from_hardfork_and_block_gas_limit(hardfork, block.header.gas_limit()); @@ -736,7 +769,7 @@ where { Ok(database) => database, Err(e) => { - return fail_all(&targets, BatchErrorKind::Rpc, &e.to_string()); + return fail_remaining(&targets, entries, BatchErrorKind::Rpc, &e.to_string()); } }; @@ -748,7 +781,7 @@ where if let Err(e) = block_executor.apply_pre_execution_changes() { let error = ReplayError::BlockExecutionError(e); - return fail_all(&targets, classify(&error), &error.to_string()); + return fail_remaining(&targets, entries, classify(&error), &error.to_string()); } let target_set: HashSet = targets.iter().copied().collect(); @@ -870,8 +903,8 @@ where .await; // Finish the block even when it aborted midway: targets that already ran - // still have a receipt worth reporting. - let mut entries = Vec::with_capacity(targets.len()); + // still have a receipt worth reporting. `entries` already holds any + // inclusion/membership failures recorded before the block started. match block_executor.finish() { Ok((evm, block_result)) => { let (db, _) = evm.finish(); @@ -986,10 +1019,11 @@ where } } - // Any target that produced no entry either sat behind the abort or is not - // part of this block at all. They are appended in block transaction-index - // order, keeping the run's ascending (block, index) order; a target the - // block does not contain has no index and keeps its input position. + // Any active target that produced no entry sat behind an abort (or is a + // residual not-in-body case for `--block`, which has no inclusion claim). + // They are appended in block transaction-index order, keeping the run's + // ascending (block, index) order; a target the block does not contain has + // no index and keeps its input position among the active set. let reported: HashSet = entries.iter().map(BatchEntry::tx_hash).collect(); let block_txs: HashSet = tx_hashes.iter().copied().collect(); let unreported = tx_hashes @@ -1000,9 +1034,21 @@ where match &loop_result { Ok(()) => { - let message = format!("Transaction is not part of block {number}"); + // Active targets are already filtered for inclusion agreement; a + // remaining absence from the body is still an endpoint + // inconsistency (the target was queued against this block), not a + // definitive not-found. for tx_hash in unreported { - entries.push(failure(*tx_hash, BatchErrorKind::NotFound, message.clone())); + entries.push(failure( + *tx_hash, + BatchErrorKind::Rpc, + format!( + "block {number} ({fetched}) does not list target transaction {tx_hash}, \ + which the endpoint resolved as included in it: the endpoint served \ + divergent views of this block (reorg in progress, or a load-balanced \ + endpoint); retry once the chain settles" + ), + )); } } Err(e) => { @@ -1291,8 +1337,29 @@ fn failure(tx_hash: B256, kind: BatchErrorKind, message: String) -> BatchEntry { } /// Report the same failure for every target of a block that never started. -fn fail_all(targets: &[B256], kind: BatchErrorKind, message: &str) -> Vec { - targets.iter().map(|hash| failure(*hash, kind, message.to_string())).collect() +fn fail_all( + targets: impl IntoIterator, + kind: BatchErrorKind, + message: &str, +) -> Vec { + targets.into_iter().map(|hash| failure(hash, kind, message.to_string())).collect() +} + +/// Append the same failure for every remaining target, keeping any entries +/// already recorded (for example inclusion mismatches decided earlier). +fn fail_remaining( + targets: &[B256], + mut entries: Vec, + kind: BatchErrorKind, + message: &str, +) -> Vec { + let reported: HashSet = entries.iter().map(BatchEntry::tx_hash).collect(); + for hash in targets { + if !reported.contains(hash) { + entries.push(failure(*hash, kind, message.to_string())); + } + } + entries } /// Write one entry to stdout: a compact NDJSON line, or the human-readable diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 52443c12..8f661cd8 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -694,6 +694,240 @@ fn test_replay_tx_file_rejects_a_block_that_does_not_match_the_resolved_inclusio let _ = std::fs::remove_file(&list); } +/// Two same-height targets that report different inclusion hashes get +/// independent outcomes: the one that matches the fetched block replays, the +/// one that does not fails as `rpc`. Outcomes must not depend on file order. +/// +/// The resolution step used to keep a first-seen job-level anchor and reject +/// later peers that disagreed, so a stale target listed first could poison the +/// canonical peer. Both orders are exercised against the same doctored capture. +#[test] +fn test_replay_tx_file_inclusion_mismatch_is_order_independent() { + let (stale_target, _) = BLOCK_TXS[1]; + let (canonical_target, canonical_index) = BLOCK_TXS[2]; + let wrong_hash = "0x2222222222222222222222222222222222222222222222222222222222222222"; + + // Doctor only the stale target's inclusion hash; the canonical target and + // the block body keep their original agreement. + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let marker = format!("\"hash\":\"{stale_target}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object"); + result["blockHash"] = serde_json::Value::String(wrong_hash.into()); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "exactly one response describes the stale target"); + + let envelope_path = std::env::temp_dir() + .join(format!("mega_evme_batch_inclusion_order_{}.json", std::process::id())); + std::fs::write(&envelope_path, envelope.to_string()).expect("write doctored envelope"); + + for (label, first, second) in [ + ("stale_first", stale_target, canonical_target), + ("canonical_first", canonical_target, stale_target), + ] { + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_inclusion_order_{label}_{}.txt", std::process::id())); + std::fs::write(&list, format!("{first}\n{second}\n")).expect("write tx list"); + + let (stdout, code) = replay_envelope_with_code( + &envelope_path, + &["--tx-file", list.to_str().unwrap(), "--json"], + ); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), 2, "{label}: every target is reported once: {stdout}"); + + let stale = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(stale_target)) + .unwrap_or_else(|| panic!("{label}: stale target must be reported")); + assert_eq!( + stale["error"]["kind"].as_str(), + Some("rpc"), + "{label}: mismatched inclusion is unanswered: {stale}" + ); + let message = stale["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("divergent views") && + message.contains(wrong_hash) && + message.contains("resolved as included"), + "{label}: message names both views: {message}" + ); + + let ok = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(canonical_target)) + .unwrap_or_else(|| panic!("{label}: canonical target must be reported")); + assert!(ok.get("error").is_none(), "{label}: matching inclusion still replays: {ok}"); + assert_eq!(ok["block_number"].as_u64(), Some(BLOCK)); + assert_eq!(ok["tx_index"].as_u64(), Some(canonical_index)); + assert_eq!(ok["success"].as_bool(), Some(true)); + + assert_eq!(code, Some(3), "{label}: an unanswered target exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); + + let _ = std::fs::remove_file(&list); + } + + let _ = std::fs::remove_file(&envelope_path); +} + +/// A target whose reported inclusion hash matches the fetched block, but which +/// the block body does not list, is an endpoint self-contradiction (`rpc`), not +/// a definitive `not_found`. +/// +/// The lookup said "in block B"; B's body lacks it. That is the same class as +/// the single-transaction membership guard, not an answer that the hash is +/// unknown. +#[test] +fn test_replay_tx_file_anchored_but_absent_target_is_rpc() { + let (target, _) = BLOCK_TXS[1]; + + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + + // Keep the transaction lookup intact (correct number + hash) but drop the + // hash from the block body it claims to belong to. + let mut body_doctored = 0; + let mut block_hash = None; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + let Ok(mut response) = serde_json::from_str::(value) else { + continue; + }; + let Some(result) = response.get_mut("result") else { + continue; + }; + if !result.is_object() { + continue; + } + let number = result.get("number").and_then(|n| { + n.as_str().and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + }); + if number != Some(BLOCK) { + continue; + } + let Some(txs) = result.get_mut("transactions").and_then(|t| t.as_array_mut()) else { + continue; + }; + let before = txs.len(); + txs.retain(|tx| tx.as_str() != Some(target)); + if txs.len() != before { + block_hash = result.get("hash").and_then(|h| h.as_str()).map(str::to_string); + entry["value"] = serde_json::Value::String(response.to_string()); + body_doctored += 1; + } + } + assert_eq!(body_doctored, 1, "exactly one block body for {BLOCK} must list the target"); + let block_hash = block_hash.expect("block hash"); + + // Pair with another-block target so a clean job still runs alongside the + // inconsistency: only the absent target fails. + let envelope_path = std::env::temp_dir() + .join(format!("mega_evme_batch_anchored_absent_{}.json", std::process::id())); + std::fs::write(&envelope_path, envelope.to_string()).expect("write doctored envelope"); + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_anchored_absent_{}.txt", std::process::id())); + std::fs::write(&list, format!("{target}\n{OTHER_BLOCK_TX}\n")).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&envelope_path, &["--tx-file", list.to_str().unwrap(), "--json"]); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), 2, "every target is reported once: {stdout}"); + + let failed = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(target)) + .expect("absent target must be reported"); + assert_eq!( + failed["error"]["kind"].as_str(), + Some("rpc"), + "anchored-but-absent is an RPC inconsistency, not not_found: {failed}" + ); + let message = failed["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains(target) && + message.contains(&block_hash) && + message.contains("does not list") && + message.contains("divergent views"), + "message names the target, the block, and the cause: {message}" + ); + + let ok = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(OTHER_BLOCK_TX)) + .expect("other-block target must be reported"); + assert!(ok.get("error").is_none(), "targets in other blocks still replay: {ok}"); + assert_eq!(ok["block_number"].as_u64(), Some(OTHER_BLOCK)); + assert_eq!(ok["success"].as_bool(), Some(true)); + + assert_eq!(code, Some(3), "an unanswered target exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); + + let _ = std::fs::remove_file(&envelope_path); + let _ = std::fs::remove_file(&list); +} + +/// A hash whose resolution answers `null` keeps the definitive `not_found` +/// class: the endpoint denied the hash, rather than claiming inclusion and then +/// contradicting itself. +#[test] +fn test_replay_tx_file_null_resolution_stays_not_found() { + let (target, _) = BLOCK_TXS[1]; + let path = envelope_without_transaction("tx_file_null_resolution", target); + + // Pair with a clean other-block target so the run still produces a success + // line next to the definitive not-found. + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_null_resolution_{}.txt", std::process::id())); + std::fs::write(&list, format!("{target}\n{OTHER_BLOCK_TX}\n")).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--tx-file", list.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&path); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), 2, "every target is reported once: {stdout}"); + + let failed = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(target)) + .expect("null-resolution target must be reported"); + assert_eq!( + failed["error"]["kind"].as_str(), + Some("not_found"), + "a null lookup is a definitive not_found: {failed}" + ); + assert_eq!( + failed["error"]["message"].as_str(), + Some("Transaction not found"), + "not_found message is unchanged: {failed}" + ); + + let ok = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(OTHER_BLOCK_TX)) + .expect("other-block target must be reported"); + assert!(ok.get("error").is_none(), "targets in other blocks still replay: {ok}"); + assert_eq!(ok["success"].as_bool(), Some(true)); + + // Definitive not_found is an execution-class failure (exit 1), not rpc. + assert_eq!(code, Some(1), "a definitive not_found exits 1"); + + let _ = std::fs::remove_file(&list); +} + /// A mined `--tx-file` target whose `eth_getTransactionByHash` answer carries a /// block number but no inclusion hash is unanswered: the endpoint served an /// unanchored view, so the target is not queued and other blocks still replay. From 10eaaaeffae098c6b4c9b1bded6fd80368c82cc0 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 11:08:43 +0800 Subject: [PATCH 51/64] fix(mega-evme): treat --rpc.clear-cache as a batch disk-cache opt-in Batch replay defaults the on-disk RPC cache off and forced `no_cache_file` whenever `--rpc.cache-dir` was absent, which silently swallowed `--rpc.clear-cache`: the documented recovery flag parsed, did nothing, and left the polluted default-path cache for the next non-batch run. Deleting the cache file only means something while the disk cache is engaged, so `--rpc.clear-cache` now opts a batch run back in exactly as `--rpc.cache-dir` does. The file is cleared under the sidecar lock, the run starts empty, and the cache persists on exit. An explicit `--rpc.no-cache-file` still wins over both flags and keeps its existing meaning: with no cache file in play there is nothing to delete, load, or persist. Covered by unit rows over the flag combinations and binary-level tests that seed a default-path cache under a fake HOME. --- bin/mega-evme/src/common/provider/mod.rs | 11 ++- bin/mega-evme/src/replay/cmd.rs | 51 +++++++++- bin/mega-evme/tests/batch_cache_default.rs | 97 ++++++++++++++++++- docs/mega-evme/commands/replay.md | 6 +- .../configuration/state-management.md | 13 ++- 5 files changed, 167 insertions(+), 11 deletions(-) diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 51913d62..2fcf0e5d 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -113,19 +113,24 @@ pub struct RpcArgs { /// Defaults to the platform cache directory (`$XDG_CACHE_HOME/mega-evme/rpc` on /// Linux, `~/Library/Caches/mega-evme/rpc` on macOS). Pass `--rpc.no-cache-file` /// to disable on-disk persistence entirely. Batch replay (`--tx-file` / `--block`) - /// uses the on-disk cache only when this flag names a directory explicitly. + /// uses the on-disk cache only when this flag or `--rpc.clear-cache` is passed + /// explicitly. #[arg(long = "rpc.cache-dir", value_parser = parse_non_empty_path)] pub cache_dir: Option, /// Disable on-disk cache persistence. The in-memory LRU cache still applies. - /// This is already the default for batch replay (`--tx-file` / `--block`) - /// unless `--rpc.cache-dir` is passed. + /// Takes precedence over `--rpc.clear-cache`: with no cache file in play there is + /// nothing to delete, load, or persist. This is already the default for batch replay + /// (`--tx-file` / `--block`) unless `--rpc.cache-dir` or `--rpc.clear-cache` is passed. #[arg(long = "rpc.no-cache-file")] pub no_cache_file: bool, /// Delete the current chain's cache file before loading it. Recovery path for a /// polluted or corrupt cache file. If the unlink itself fails (e.g. insufficient /// permissions), `mega-evme` aborts rather than silently reloading the stale file. + /// Passing this flag engages the on-disk cache, including in batch replay + /// (`--tx-file` / `--block`), where it is otherwise off by default. Has no effect + /// alongside `--rpc.no-cache-file`. #[arg(long = "rpc.clear-cache")] pub clear_cache: bool, diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 0004c893..d456685f 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -493,11 +493,18 @@ impl Cmd { /// clean-exit persist re-reads, merges, and rewrites the whole file under a cross-process /// lock. That exit tail grows linearly with the file and serializes across concurrent /// batch processes sharing the default cache directory, so batch mode keeps the disk - /// cache off unless `--rpc.cache-dir` names one explicitly. The in-memory LRU (and its + /// cache off unless the invocation says otherwise. The in-memory LRU (and its /// intra-run reuse across a block's transactions) is unaffected. + /// + /// Two flags say otherwise. `--rpc.cache-dir` names the file to use, and `--rpc.clear-cache` + /// asks for that file to be deleted — a request that only means something while the disk + /// cache is engaged, so forcing the cache off would silently swallow it and leave the + /// polluted file in place for the next run. Either flag therefore opts the batch run back + /// into the disk cache. An explicit `--rpc.no-cache-file` still wins over both, keeping the + /// same meaning it has outside batch mode. fn batch_rpc_args(&self) -> RpcArgs { let mut args = self.rpc_args.clone(); - if args.cache_dir.is_none() && !args.no_cache_file { + if args.cache_dir.is_none() && !args.clear_cache && !args.no_cache_file { info!( "Batch replay leaves the on-disk RPC cache disabled; \ pass --rpc.cache-dir to enable it" @@ -1210,12 +1217,52 @@ mod tests { assert_eq!(args.cache_dir.as_deref(), Some(std::path::Path::new("/tmp/evme-cache"))); } + #[test] + fn test_batch_rpc_args_explicit_clear_cache_opts_back_in() { + let cmd = parse_online(&["--block", "1", "--rpc.clear-cache"]); + let args = cmd.batch_rpc_args(); + assert!( + !args.no_cache_file, + "--rpc.clear-cache asks for the disk cache file to be deleted, which only \ + happens while the disk cache is engaged", + ); + assert!(args.clear_cache, "the clear request itself must survive"); + assert_eq!(args.cache_dir, None, "the default cache path is the one being cleared"); + } + #[test] fn test_batch_rpc_args_keeps_explicit_no_cache_file() { let cmd = parse_online(&["--block", "1", "--rpc.no-cache-file"]); assert!(cmd.batch_rpc_args().no_cache_file, "--rpc.no-cache-file must stay honored"); } + /// `--rpc.no-cache-file` and `--rpc.clear-cache` are not mutually exclusive at the + /// parser level. Batch mode must not reinterpret the pair: it passes both through so + /// the combination behaves exactly as it does in single-transaction mode. + #[test] + fn test_batch_rpc_args_passes_no_cache_file_with_clear_cache_through() { + let cmd = parse_online(&["--block", "1", "--rpc.no-cache-file", "--rpc.clear-cache"]); + let args = cmd.batch_rpc_args(); + assert!(args.no_cache_file, "--rpc.no-cache-file must stay honored alongside clear"); + assert!(args.clear_cache, "the clear flag must be passed through unmodified"); + } + + /// `--rpc.cache-dir` and `--rpc.clear-cache` together name the file to clear. + #[test] + fn test_batch_rpc_args_cache_dir_with_clear_cache_opts_back_in() { + let cmd = parse_online(&[ + "--block", + "1", + "--rpc.cache-dir", + "/tmp/evme-cache", + "--rpc.clear-cache", + ]); + let args = cmd.batch_rpc_args(); + assert!(!args.no_cache_file, "both flags opt the batch run into the disk cache"); + assert!(args.clear_cache); + assert_eq!(args.cache_dir.as_deref(), Some(std::path::Path::new("/tmp/evme-cache"))); + } + #[test] fn test_replay_target_group_accepts_each_form() { assert!(matches!( diff --git a/bin/mega-evme/tests/batch_cache_default.rs b/bin/mega-evme/tests/batch_cache_default.rs index 4b714353..14b4df9f 100644 --- a/bin/mega-evme/tests/batch_cache_default.rs +++ b/bin/mega-evme/tests/batch_cache_default.rs @@ -1,7 +1,7 @@ //! Integration tests for the batch-mode on-disk cache default. //! //! Batch replay (`--tx-file` / `--block`) engages the on-disk RPC cache only -//! when `--rpc.cache-dir` names a directory explicitly. The clean-exit persist +//! when the invocation asks for it explicitly. The clean-exit persist //! re-reads, merges, and atomically rewrites the whole per-chain cache file //! under a cross-process lock, so its cost grows with the file and serializes //! across concurrent processes — while a linear history scan gets almost no @@ -9,6 +9,11 @@ //! disk-cache work, making its cost independent of any cache a machine has //! accumulated. Single-transaction replay keeps the previous default. //! +//! `--rpc.clear-cache` opts a batch run back in the same way `--rpc.cache-dir` +//! does: deleting the cache file is a request that only means something while +//! the disk cache is engaged, so forcing it off would make the documented +//! recovery flag a no-op and leave the polluted file for the next run. +//! //! The tests point the child's platform cache directory into a temp dir via //! `HOME` / `XDG_CACHE_HOME`, so the real user cache is never touched. @@ -143,6 +148,96 @@ async fn test_batch_default_leaves_disk_cache_untouched() { ); } +/// An explicit `--rpc.clear-cache` opts a batch run back into the disk cache at +/// the default path: the seeded file is deleted before the run and a fresh cache +/// file is persisted on exit. Forcing the cache off instead would make the flag +/// parse and do nothing, leaving the polluted file for the next non-batch run. +#[tokio::test(flavor = "multi_thread")] +async fn test_batch_clear_cache_clears_and_repersists_default_path() { + let home = FakeHome::new(); + let seeded = home.default_cache_file(); + std::fs::create_dir_all(seeded.parent().expect("cache file has a parent")).expect("mkdir"); + std::fs::write(&seeded, b"not even json").expect("seed cache file"); + + let server = failing_mock().await; + let targets = tx_file(&home); + + replay( + &home, + &[ + "--tx-file", + targets.to_str().expect("utf-8"), + "--rpc", + &server.uri(), + "--rpc.clear-cache", + "--rpc.max-retries", + "0", + "--rpc.backoff-ms", + "1", + "--json", + ], + ); + + let bytes = std::fs::read(&seeded).expect("a fresh cache file must exist after the run"); + assert_ne!( + bytes, b"not even json", + "the seeded cache file must have been cleared, not carried forward", + ); + // The clear only happened because the disk cache was engaged, so the exit + // persist must have written a well-formed provider cache in its place. + serde_json::from_slice::(&bytes) + .expect("the persisted cache file must be valid JSON"); + assert_eq!( + home.cache_files(), + vec![seeded], + "the run must not create a cache file anywhere else under the fake home", + ); +} + +/// `--rpc.no-cache-file` wins over `--rpc.clear-cache`: with no cache file in +/// play there is nothing to delete, load, or persist, so a seeded file survives +/// byte-identical. Batch mode passes the pair through unchanged, so both target +/// forms behave the same way. +#[tokio::test(flavor = "multi_thread")] +async fn test_no_cache_file_wins_over_clear_cache_in_both_modes() { + for batch in [false, true] { + let home = FakeHome::new(); + let seeded = home.default_cache_file(); + std::fs::create_dir_all(seeded.parent().expect("cache file has a parent")).expect("mkdir"); + std::fs::write(&seeded, b"not even json").expect("seed cache file"); + + let server = failing_mock().await; + let uri = server.uri(); + let targets = tx_file(&home); + + let mut args = + if batch { vec!["--tx-file", targets.to_str().expect("utf-8")] } else { vec![TX] }; + args.extend_from_slice(&[ + "--rpc", + &uri, + "--rpc.no-cache-file", + "--rpc.clear-cache", + "--rpc.max-retries", + "0", + "--rpc.backoff-ms", + "1", + "--json", + ]); + replay(&home, &args); + + let bytes = std::fs::read(&seeded).expect("the seeded file must still exist"); + assert_eq!( + bytes, b"not even json", + "--rpc.no-cache-file must keep the disk cache out of play (batch = {batch})", + ); + assert_eq!( + home.cache_files(), + vec![seeded], + "no cache file may be written anywhere (batch = {batch})", + ); + } +} + /// An explicit `--rpc.cache-dir` opts a batch run back into persistence: the /// per-chain cache file is written on exit. #[tokio::test(flavor = "multi_thread")] diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 9ba18e6f..950da49f 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -62,10 +62,14 @@ Batch mode does all of it once. A batch run builds a single provider and a single RPC cache, groups the requested transactions by their containing block, and processes the blocks in ascending order. Each block is executed exactly once: state is forked at the parent block, pre-execution changes are applied, and every transaction of the block runs in order, with each requested transaction's result recorded before it is committed. A capture file (`--rpc.capture-file`) is persisted once, on exit, even if some transactions failed — the captured responses are the artifact you need to debug the failure offline. -The per-chain on-disk RPC cache is opt-in for batch runs: it is loaded and persisted only when `--rpc.cache-dir` names a directory explicitly. +The per-chain on-disk RPC cache is opt-in for batch runs: it is loaded and persisted only when `--rpc.cache-dir` names a directory explicitly, or when `--rpc.clear-cache` asks for the cache file to be deleted. A batch scan walks linear history whose request keys essentially never repeat across runs, so a shared cache file buys almost no hits, while its clean-exit re-read-merge-rewrite grows with the file and serializes concurrent processes on the persist lock. The in-memory cache still serves every repeated request within the run. +`--rpc.clear-cache` counts as an explicit opt-in because deleting the cache file only means something while the disk cache is engaged: a batch run that forced the cache off would parse the flag, do nothing, and leave the polluted file in place for the next run. +With it, the cache file (at the default path, or under `--rpc.cache-dir`) is deleted under the sidecar lock, the run starts from an empty cache, and the cache is persisted on exit. +An explicit `--rpc.no-cache-file` still wins over both flags and keeps the disk cache off, exactly as in single-transaction mode. + A plain batch replay issues the same RPC calls as single-transaction replay, so an offline envelope captured by single-transaction runs serves a batch run without a cache miss. `--verify-receipt` and `--dump-fixture-dir` are the exception: both fetch the receipt of every target in the block, including transactions a single-transaction capture never asked about, so an older envelope will miss them and the run exits `3`. diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index 92c3a1f5..194d8ef7 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -217,9 +217,14 @@ The default cache directory is the platform cache directory: - **Linux**: `$XDG_CACHE_HOME/mega-evme/rpc` - **macOS**: `~/Library/Caches/mega-evme/rpc` -Batch replay (`--tx-file` / `--block`) is the exception: it engages the on-disk cache only when `--rpc.cache-dir` names a directory explicitly, and otherwise behaves as if `--rpc.no-cache-file` were set. +Batch replay (`--tx-file` / `--block`) is the exception: it engages the on-disk cache only when the invocation asks for it explicitly, and otherwise behaves as if `--rpc.no-cache-file` were set. A batch scan walks linear history whose request keys essentially never repeat across runs, so the file buys almost no hits, while its clean-exit persist re-reads, merges, and rewrites the whole file under the cross-process lock — a cost that grows with the file and serializes concurrent batch processes. +Two flags ask for it: `--rpc.cache-dir`, which names the file to use, and `--rpc.clear-cache`, which asks for that file to be deleted. +Clearing only means something while the disk cache is engaged, so a batch run that forced the cache off would parse the recovery flag, do nothing, and leave the polluted file in place for the next run. +With `--rpc.clear-cache`, a batch run deletes the cache file under the sidecar lock, starts from an empty cache, and persists on exit — the same sequence as single-transaction mode. +An explicit `--rpc.no-cache-file` still wins over both flags. + ### Concurrent cache-dir sharing Multiple `mega-evme` processes may share the same `--rpc.cache-dir` safely. @@ -263,9 +268,9 @@ Provider-cache merge also rejects inputs (and `--output`) whose `rpc-cache-{chai | Flag | Type | Default | Description | | ----------------------------- | ----- | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--rpc.cache-max-entries ` | `u32` | `0` | Maximum number of items in the in-memory RPC LRU cache (and therefore what is persisted to the cache file). `0` = effectively unlimited (caps at 1,048,576 entries; the cache index is preallocated proportional to the cap). Default. | -| `--rpc.cache-dir ` | path | Platform cache dir | Directory for per-chain cache files. Each chain's cache is stored as `{cache_dir}/rpc-cache-{chain_id}.json`. Batch replay uses the on-disk cache only when this flag is passed explicitly. | -| `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies. Already the default for batch replay unless `--rpc.cache-dir` is passed. | -| `--rpc.clear-cache` | flag | `false` | Delete the current chain's cache file before loading it. Recovery path for a polluted or corrupt cache. | +| `--rpc.cache-dir ` | path | Platform cache dir | Directory for per-chain cache files. Each chain's cache is stored as `{cache_dir}/rpc-cache-{chain_id}.json`. Batch replay uses the on-disk cache only when this flag or `--rpc.clear-cache` is passed explicitly. | +| `--rpc.no-cache-file` | flag | `false` | Disable on-disk cache persistence. The in-memory LRU cache still applies. Wins over `--rpc.clear-cache`. Already the default for batch replay unless `--rpc.cache-dir` or `--rpc.clear-cache` is passed. | +| `--rpc.clear-cache` | flag | `false` | Delete the current chain's cache file before loading it. Recovery path for a polluted or corrupt cache. Engages the on-disk cache, including in batch replay. No effect alongside `--rpc.no-cache-file`. | The in-memory cache layer is always installed on a forked or online run and cannot be turned off; `--rpc.no-cache-file` disables only on-disk persistence. At the default cap the cache index is preallocated to tens of MiB regardless of how many entries a run actually stores, which is the trade for never re-fetching during a long verification sweep. From 6b97fc3a20f19bc68483bc6ccc2c7c07d323a03b Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 11:17:39 +0800 Subject: [PATCH 52/64] fix(mega-evme): tally batch abort root cause and name failing fetches When a non-target aborts a block, count classify(err) into the batch tally so the run exit reflects the root cause while swept targets stay rpc/unanswered. Deferred fixture discards inherit the abort class, and body-tx transport failures carry the hash via BlockBodyTransactionFetch. --- bin/mega-evme/src/common/error.rs | 14 ++ bin/mega-evme/src/common/exit.rs | 14 +- bin/mega-evme/src/replay/batch.rs | 296 ++++++++++++++++++++++++---- bin/mega-evme/tests/replay_batch.rs | 163 +++++++++++++++ docs/mega-evme/commands/replay.md | 6 +- 5 files changed, 444 insertions(+), 49 deletions(-) diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index a4f83ecd..2d504e7c 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -24,6 +24,20 @@ pub enum EvmeError { #[error("Block body lists transaction {0} but the endpoint resolves it to null")] BlockBodyTransactionNull(TxHash), + /// The block body listed this hash, but fetching the transaction failed. + /// + /// A transport error or offline cache miss on a body-listed hash is the same + /// class as a null answer: the endpoint failed to deliver a transaction it + /// claimed to include, rather than answering "unknown hash" about a user + /// query. The hash is carried so abort output can name the failing fetch. + #[error("Block body lists transaction {tx_hash} but fetching it failed: {message}")] + BlockBodyTransactionFetch { + /// Hash the block body listed and the lookup failed for. + tx_hash: TxHash, + /// Transport or cache-miss detail from the failed lookup. + message: String, + }, + /// Block not found #[error("Block not found: {0}")] BlockNotFound(BlockNumber), diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs index f384ff18..497c3423 100644 --- a/bin/mega-evme/src/common/exit.rs +++ b/bin/mega-evme/src/common/exit.rs @@ -142,12 +142,14 @@ impl ExitCode { match err { // The endpoint never answered: unreachable, transport-level // failure, or an offline replay file without the response. - // `BlockBodyTransactionNull` is the same class: the block body - // already listed the hash, so a null lookup is an inconsistent - // endpoint rather than a definitive unknown transaction. + // `BlockBodyTransactionNull` / `BlockBodyTransactionFetch` are the + // same class: the block body already listed the hash, so a null or + // failed lookup is an inconsistent endpoint rather than a + // definitive unknown transaction. EvmeError::RpcTransportError(_) | EvmeError::RpcError(_) | - EvmeError::BlockBodyTransactionNull(_) => Self::RpcFailure, + EvmeError::BlockBodyTransactionNull(_) | + EvmeError::BlockBodyTransactionFetch { .. } => Self::RpcFailure, // A block error the EVM raised because a state read failed is that // read's failure, not an execution result: classify it by its // cause, so an endpoint that died mid-execution still reports the @@ -308,6 +310,10 @@ mod tests { alloy_provider::transport::TransportErrorKind::custom_str("connection refused"), ), EvmeError::BlockBodyTransactionNull(B256::ZERO), + EvmeError::BlockBodyTransactionFetch { + tx_hash: B256::ZERO, + message: "cache miss in offline replay file".to_string(), + }, ] { assert_eq!( ExitCode::from_evme_error(&err), diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 8413fae5..8c68fd60 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -93,22 +93,52 @@ struct FixtureReport { /// Why the fixture was not written for this target. #[serde(skip_serializing_if = "Option::is_none")] skipped: Option, - /// Why writing the fixture failed. Counts as an execution-class failure. + /// Why writing the fixture failed. #[serde(skip_serializing_if = "Option::is_none")] error: Option, + /// Batch tally class for [`Self::error`]. Construction and write failures + /// are execution-class; a draft discarded because the block aborted inherits + /// the abort's class so a transient RPC abort does not become exit 1. + /// Not serialized: the wire shape stays `path` / `skipped` / `error`. + #[serde(skip)] + error_kind: BatchErrorKind, } impl FixtureReport { fn written(path: &Path) -> Self { - Self { path: Some(path.display().to_string()), skipped: None, error: None } + Self { + path: Some(path.display().to_string()), + skipped: None, + error: None, + error_kind: BatchErrorKind::Execution, + } } fn skipped(reason: impl Into) -> Self { - Self { path: None, skipped: Some(reason.into()), error: None } + Self { + path: None, + skipped: Some(reason.into()), + error: None, + error_kind: BatchErrorKind::Execution, + } } + /// Construction or write failure of this target's fixture (execution-class). fn error(message: impl Into) -> Self { - Self { path: None, skipped: None, error: Some(message.into()) } + Self { + path: None, + skipped: None, + error: Some(message.into()), + error_kind: BatchErrorKind::Execution, + } + } + + /// Fixture discarded because the block aborted after the draft was built. + /// + /// The target keeps its execution result; only the fixture field fails, and + /// the failure class matches the abort so the run exit reflects the cause. + fn abort_error(message: impl Into, kind: BatchErrorKind) -> Self { + Self { path: None, skipped: None, error: Some(message.into()), error_kind: kind } } /// Whether the fixture the run was asked to write could not be written. @@ -265,9 +295,33 @@ impl BatchTally { } } // A fixture the run was asked to write and could not is a failure of - // that target, even though its replay produced a result. - if fixture.is_some_and(FixtureReport::is_error) { - self.counts.execution += 1; + // that target, even though its replay produced a result. Construction + // and write failures are execution-class; an abort-inherited discard + // uses the abort's class (see [`FixtureReport::abort_error`]). + if let Some(fixture) = fixture.filter(|f| f.is_error()) { + match fixture.error_kind { + BatchErrorKind::Rpc => self.counts.rpc += 1, + BatchErrorKind::NotFound | BatchErrorKind::Pending | BatchErrorKind::Execution => { + self.counts.execution += 1 + } + } + } + } + + /// Count a mid-block abort whose root-cause class is not already carried by + /// a per-target failure entry. + /// + /// Swept targets always stay `rpc` ("unanswered"). When the aborting + /// transaction is not itself a reported target, that class would otherwise + /// be lost and a deterministic executor abort would exit 3. The abort is + /// tallied once without emitting an extra NDJSON line or incrementing + /// `reported`. + fn record_uncounted_abort(&mut self, kind: BatchErrorKind) { + match kind { + BatchErrorKind::Rpc => self.counts.rpc += 1, + BatchErrorKind::NotFound | BatchErrorKind::Pending | BatchErrorKind::Execution => { + self.counts.execution += 1 + } } } @@ -461,8 +515,8 @@ where }; for job in jobs { - let entries = replay_block(provider, chain_id, job, external_envs.clone(), &report).await; - for entry in entries { + let outcome = replay_block(provider, chain_id, job, external_envs.clone(), &report).await; + for entry in outcome.entries { if let BatchEntry::Executed(tx) = &entry { match &tx.fixture { Some(fixture) if fixture.path.is_some() => fixtures_written += 1, @@ -474,6 +528,10 @@ where tally.record(&entry); emit(&entry, report.json); } + // Root-cause class of a non-target abort is not on any per-target line. + if let Some(kind) = outcome.uncounted_abort { + tally.record_uncounted_abort(kind); + } } info!( @@ -577,6 +635,23 @@ where (jobs, failures) } +/// Outcome of replaying one block's targets. +struct BlockReplayOutcome { + /// One entry per target of the job (executed or failed). + entries: Vec, + /// Root-cause class of a mid-block abort that no reported entry carries. + /// + /// Present when the aborting transaction is not a target: swept targets stay + /// `rpc`, and this class is tallied so the run exit reflects the abort. + uncounted_abort: Option, +} + +impl BlockReplayOutcome { + fn entries_only(entries: Vec) -> Self { + Self { entries, uncounted_abort: None } + } +} + /// Replay one block, reporting an entry for every target it was asked about. /// /// The block is executed exactly once: every transaction runs in order, and each @@ -595,7 +670,7 @@ async fn replay_block

( job: BlockJob, external_envs: EvmeExternalEnvs, report: &ReportArgs, -) -> Vec +) -> BlockReplayOutcome where P: Provider + Clone + std::fmt::Debug, { @@ -606,11 +681,11 @@ where let target_hashes = || targets.iter().map(|t| t.hash); if number == 0 { - return fail_all( + return BlockReplayOutcome::entries_only(fail_all( target_hashes(), BatchErrorKind::Rpc, "Block 0 has no parent block to fork from", - ); + )); } let block = match block { @@ -618,14 +693,22 @@ where None => match fetch_block(provider, number).await { Ok(block) => block, Err(e) => { - return fail_all(target_hashes(), BatchErrorKind::Rpc, &e.to_string()); + return BlockReplayOutcome::entries_only(fail_all( + target_hashes(), + BatchErrorKind::Rpc, + &e.to_string(), + )); } }, }; let parent_block = match fetch_block(provider, number - 1).await { Ok(block) => block, Err(e) => { - return fail_all(target_hashes(), BatchErrorKind::Rpc, &e.to_string()); + return BlockReplayOutcome::entries_only(fail_all( + target_hashes(), + BatchErrorKind::Rpc, + &e.to_string(), + )); } }; @@ -649,7 +732,11 @@ where block describes a different chain than the block being replayed (reorg in progress, \ or a load-balanced endpoint serving divergent views); retry once the chain settles" ); - return fail_all(target_hashes(), BatchErrorKind::Rpc, &message); + return BlockReplayOutcome::entries_only(fail_all( + target_hashes(), + BatchErrorKind::Rpc, + &message, + )); } // Per-target inclusion and membership guards. `--tx-file` resolved each @@ -698,7 +785,7 @@ where // Every target either failed an inclusion/membership check or was a // `--block` target already taken from the body. Nothing left to execute. if active.is_empty() { - return entries; + return BlockReplayOutcome::entries_only(entries); } let targets = active; @@ -733,13 +820,23 @@ where let cfg_env = match chain_args.create_cfg_env() { Ok(cfg) => cfg, Err(e) => { - return fail_remaining(&targets, entries, BatchErrorKind::Execution, &e.to_string()); + return BlockReplayOutcome::entries_only(fail_remaining( + &targets, + entries, + BatchErrorKind::Execution, + &e.to_string(), + )); } }; let block_env = match retrieve_block_env(&block) { Ok(env) => env, Err(e) => { - return fail_remaining(&targets, entries, BatchErrorKind::Execution, &e.to_string()); + return BlockReplayOutcome::entries_only(fail_remaining( + &targets, + entries, + BatchErrorKind::Execution, + &e.to_string(), + )); } }; let executed_spec = cfg_env.spec; @@ -747,7 +844,12 @@ where let Some(hardfork) = hardforks.hardfork(timestamp) else { let message = format!("No `MegaHardfork` active at block timestamp: {timestamp}"); - return fail_remaining(&targets, entries, BatchErrorKind::Execution, &message); + return BlockReplayOutcome::entries_only(fail_remaining( + &targets, + entries, + BatchErrorKind::Execution, + &message, + )); }; let block_limits = BlockLimits::from_hardfork_and_block_gas_limit(hardfork, block.header.gas_limit()); @@ -769,7 +871,12 @@ where { Ok(database) => database, Err(e) => { - return fail_remaining(&targets, entries, BatchErrorKind::Rpc, &e.to_string()); + return BlockReplayOutcome::entries_only(fail_remaining( + &targets, + entries, + BatchErrorKind::Rpc, + &e.to_string(), + )); } }; @@ -781,7 +888,12 @@ where if let Err(e) = block_executor.apply_pre_execution_changes() { let error = ReplayError::BlockExecutionError(e); - return fail_remaining(&targets, entries, classify(&error), &error.to_string()); + return BlockReplayOutcome::entries_only(fail_remaining( + &targets, + entries, + classify(&error), + &error.to_string(), + )); } let target_set: HashSet = targets.iter().copied().collect(); @@ -817,7 +929,10 @@ where let tx = provider .get_transaction_by_hash(*tx_hash) .await - .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? + .map_err(|e| ReplayError::BlockBodyTransactionFetch { + tx_hash: *tx_hash, + message: e.to_string(), + })? .ok_or(ReplayError::BlockBodyTransactionNull(*tx_hash))?; let is_target = target_set.contains(tx_hash); @@ -978,21 +1093,21 @@ where // Materialize only when the block loop completed cleanly: a // mid-block abort after this target built a Ready draft must // not publish (or clobber) a fixture for a block that failed. - let fixture = target.fixture.map(|deferred| { - if loop_result.is_ok() { - materialize_deferred_fixture(deferred) - } else { - // Drop the ready draft without writing; the target still - // reports its receipt below when finish succeeded. - match deferred { - DeferredFixture::Report(report) => report, - DeferredFixture::Ready { path, .. } => FixtureReport::error(format!( + // Keep the execution result; only the fixture field fails, and + // it inherits the abort's class so a transient RPC abort exits 3. + let fixture = target.fixture.map(|deferred| match &loop_result { + Ok(()) => materialize_deferred_fixture(deferred), + Err(abort) => match deferred { + DeferredFixture::Report(report) => report, + DeferredFixture::Ready { path, .. } => FixtureReport::abort_error( + format!( "fixture not written: block aborted before a clean finish \ - (draft for {} was discarded)", + (draft for {} was discarded): {abort}", path.display() - )), - } - } + ), + classify(abort), + ), + }, }); entries.push(BatchEntry::Executed(Box::new(ExecutedTx { tx_hash: target.tx_hash, @@ -1032,6 +1147,7 @@ where .chain(targets.iter().filter(|hash| !block_txs.contains(*hash))) .filter(|hash| !reported.contains(*hash)); + let mut uncounted_abort = None; match &loop_result { Ok(()) => { // Active targets are already filtered for inclusion agreement; a @@ -1054,10 +1170,13 @@ where Err(e) => { warn!(block = number, error = %e, "Aborted block replay; skipping its remaining targets"); let aborting = aborting_tx_hash(e); + let root_kind = classify(e); + let mut root_on_target = false; for tx_hash in unreported { if aborting == Some(*tx_hash) { // The abort is this target's own answer. - entries.push(failure(*tx_hash, classify(e), e.to_string())); + root_on_target = true; + entries.push(failure(*tx_hash, root_kind, e.to_string())); } else { // The abort belongs to another transaction of the block, so // nothing was established about this target: it went @@ -1069,10 +1188,37 @@ where )); } } + // When the aborter is not a reported target, no failure entry carries + // the abort's own class. Tallied separately so the run exit reflects + // the root cause (e.g. exit 1 for a deterministic non-target abort). + // Fixture abort-errors on executed targets may also carry the class; + // double-counting the same class still yields the correct exit. + if !root_on_target { + // If finish failed for pending targets that already include the + // aborter as a Failed entry, the class is already counted. + let already_counted = aborting.is_some_and(|hash| { + entries.iter().any(|entry| match entry { + BatchEntry::Failed(tx) => tx.tx_hash == hash && tx.kind == root_kind, + BatchEntry::Executed(_) => false, + }) + }); + // Abort-inherited fixture failures on executed targets already + // contribute the abort class to the tally. + let fixture_carries_class = entries.iter().any(|entry| match entry { + BatchEntry::Executed(tx) => tx + .fixture + .as_ref() + .is_some_and(|f| f.is_error() && f.error_kind == root_kind), + BatchEntry::Failed(_) => false, + }); + if !already_counted && !fixture_carries_class { + uncounted_abort = Some(root_kind); + } + } } } - entries + BlockReplayOutcome { entries, uncounted_abort } } /// Inputs for [`prepare_target_fixture`], grouped so the dump path stays a single @@ -1279,7 +1425,8 @@ fn classify(err: &ReplayError) -> BatchErrorKind { ReplayError::TransactionNotFound(_) => BatchErrorKind::NotFound, ReplayError::RpcError(_) | ReplayError::RpcTransportError(_) | - ReplayError::BlockBodyTransactionNull(_) => BatchErrorKind::Rpc, + ReplayError::BlockBodyTransactionNull(_) | + ReplayError::BlockBodyTransactionFetch { .. } => BatchErrorKind::Rpc, // A block error the EVM raised because a state read failed is that // read's failure: the same classification the run-level exit code uses. ReplayError::BlockExecutionError(_) @@ -1296,14 +1443,13 @@ fn classify(err: &ReplayError) -> BatchErrorKind { /// The abort says nothing about this target: whatever class caused the block to /// stop (unknown hash, RPC failure, executor/setup error on another /// transaction), a non-aborting swept target is unanswered (`rpc`). Only the -/// transaction that caused the abort keeps its own classified kind. +/// transaction that caused the abort keeps its own classified kind (when it is +/// a reported target); otherwise the run tallies the abort class separately so +/// the exit code still reflects the root cause. /// /// The error is taken and ignored on purpose: the signature keeps the decision /// visible at the call site, so a future change that wants to classify by cause -/// has to argue against this rule rather than silently add a parameter. Note the -/// consequence — a deterministic execution failure in a *non-target* transaction -/// sweeps every target as `rpc` (exit `3`, "retrying may help") even though -/// retrying will not help. +/// has to argue against this rule rather than silently add a parameter. fn swept_kind(_err: &ReplayError) -> BatchErrorKind { BatchErrorKind::Rpc } @@ -1314,6 +1460,7 @@ fn aborting_tx_hash(err: &ReplayError) -> Option { ReplayError::TransactionNotFound(hash) | ReplayError::BlockBodyTransactionNull(hash) => { Some(*hash) } + ReplayError::BlockBodyTransactionFetch { tx_hash, .. } => Some(*tx_hash), ReplayError::BlockExecutionError(err) => block_error_tx_hash(err), _ => None, } @@ -1567,6 +1714,9 @@ mod tests { /// Every non-aborting swept target is unanswered (`rpc`), even when the /// abort itself is an execution-class failure of another transaction. + /// + /// The abort's own class is tallied separately when the aborter is not a + /// target (`record_uncounted_abort`); swept entries stay `rpc`. #[test] fn test_swept_kind_always_rpc_regardless_of_abort_class() { // Unknown hash: already unanswered for the cause, and for swept peers. @@ -1607,6 +1757,66 @@ mod tests { assert_eq!(aborting_tx_hash(&ReplayError::TransactionNotFound(hash)), Some(hash)); } + /// A block-body fetch failure (transport / cache miss) is rpc-class and + /// names the hash, matching the null-answer pattern. + #[test] + fn test_block_body_transaction_fetch_classifies_as_rpc_and_names_the_hash() { + let hash = B256::repeat_byte(0xcd); + let err = ReplayError::BlockBodyTransactionFetch { + tx_hash: hash, + message: "cache miss in offline replay file".into(), + }; + assert_eq!(classify(&err), BatchErrorKind::Rpc); + assert_eq!(aborting_tx_hash(&err), Some(hash)); + let message = err.to_string(); + assert!(message.contains(&hash.to_string()) || message.contains(&format!("{hash:#x}"))); + assert!(message.contains("fetching it failed"), "unexpected message: {message}"); + assert!(message.contains("cache miss"), "unexpected message: {message}"); + } + + /// An uncounted non-target abort contributes its class to the exit without + /// a synthetic reported entry. + #[test] + fn test_batch_tally_uncounted_abort_drives_exit_class() { + let mut tally = BatchTally::default(); + // Two targets swept as unanswered behind a non-target execution abort. + tally.record(&failure(B256::repeat_byte(0x01), BatchErrorKind::Rpc, "swept".into())); + tally.record(&failure(B256::repeat_byte(0x02), BatchErrorKind::Rpc, "swept".into())); + tally.record_uncounted_abort(BatchErrorKind::Execution); + + assert_eq!(tally.reported, 2, "uncounted abort is not a reported target"); + assert_eq!(tally.counts.rpc, 2); + assert_eq!(tally.counts.execution, 1); + let err = tally.into_error().expect("run failed"); + let ReplayError::BatchFailed(counts) = err else { + panic!("expected batch failure: {err:?}"); + }; + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::ExecutionError); + } + + /// A fixture discarded after a transport abort counts as rpc, not execution, + /// so the run exit matches the abort class. + #[test] + fn test_batch_tally_abort_inherited_fixture_error_is_rpc_class() { + let mut tally = BatchTally::default(); + tally.record_executed( + None, + Some(&FixtureReport::abort_error( + "fixture not written: block aborted: transport", + BatchErrorKind::Rpc, + )), + ); + + assert_eq!(tally.replayed, 1); + assert_eq!(tally.counts.rpc, 1); + assert_eq!(tally.counts.execution, 0); + let err = tally.into_error().expect("run failed"); + let ReplayError::BatchFailed(counts) = err else { + panic!("expected batch failure: {err:?}"); + }; + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::RpcFailure); + } + /// A run whose only finding is divergence fails as the mismatch it is. #[test] fn test_batch_tally_mismatch_only_reports_the_verification_error() { diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 8f661cd8..88fd42a2 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -48,6 +48,12 @@ const BLOCK_TXS: [(&str, u64); 3] = [ const EXEC_ABORT_TX: (&str, u64) = ("0xa637d68cda9423d67826e008b1c90295193f30f19cd74a6f4acf54022d56cae2", 2); +/// Non-target body transaction between `BLOCK_TXS[1]` (index 3) and +/// `BLOCK_TXS[2]` (index 22). Used to abort after an early dumpable target +/// without putting the aborter on the reported target list. +const MID_BLOCK_NON_TARGET: (&str, u64) = + ("0x63e032fdff2676824fd6a71df09d88f62146d07effc7e4ed7e246034df2e9b22", 4); + /// Last transaction of the envelope's second block. const OTHER_BLOCK: u64 = 22_945_853; const OTHER_BLOCK_TX: &str = "0x18302160f2395069a44e1654d173fa9eed95ead8f922f12bfe07b6bdcc0a14f2"; @@ -572,6 +578,163 @@ fn test_replay_block_sweeps_targets_behind_execution_abort_as_rpc() { assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("execution-error")); } +/// A non-target deterministic executor abort still exits 1: swept targets stay +/// `rpc` ("unanswered"), but the run tallies the abort's own class so a +/// retryable exit is not reported for a permanent failure. +/// +/// `EXEC_ABORT_TX` is doctored and kept out of the `--tx-file` target list; only +/// later targets of the same block are requested. +#[test] +fn test_replay_tx_file_non_target_execution_abort_exits_execution() { + let (aborting, aborting_index) = EXEC_ABORT_TX; + let (target_a, target_a_index) = BLOCK_TXS[1]; + let (target_b, _) = BLOCK_TXS[2]; + assert!(target_a_index > aborting_index, "targets must sit behind the non-target aborter"); + let path = envelope_with_zero_gas_transaction("non_target_exec_abort", aborting); + let list = format!("{target_a}\n{target_b}\n"); + let list_path = std::env::temp_dir() + .join(format!("mega_evme_tx_list_non_target_exec_{}.txt", std::process::id())); + std::fs::write(&list_path, list).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--tx-file", list_path.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list_path); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), 2, "only requested targets are reported: {stdout}"); + for line in &lines { + assert_eq!( + line["error"]["kind"].as_str(), + Some("rpc"), + "swept target stays unanswered: {line}" + ); + assert!( + line["error"]["message"].as_str().is_some_and(|m| { + m.contains(aborting) || m.contains("aborted") || m.contains("Block replay") + }), + "the message must name the non-target abort: {line}" + ); + } + + assert_eq!(code, Some(1), "a non-target execution abort exits 1, not 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("execution-error")); +} + +/// A non-target transport abort (cache miss) exits 3 and names the failing +/// fetch so swept entries are distinguishable from the cause. +#[test] +fn test_replay_tx_file_non_target_transport_abort_names_hash_and_exits_rpc() { + let (aborting, aborting_index) = EXEC_ABORT_TX; + let (target_a, target_a_index) = BLOCK_TXS[1]; + let (target_b, _) = BLOCK_TXS[2]; + assert!(target_a_index > aborting_index, "targets must sit behind the non-target aborter"); + let path = envelope_dropping_transaction("non_target_transport_abort", aborting); + let list = format!("{target_a}\n{target_b}\n"); + let list_path = std::env::temp_dir() + .join(format!("mega_evme_tx_list_non_target_rpc_{}.txt", std::process::id())); + std::fs::write(&list_path, list).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--tx-file", list_path.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list_path); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), 2, "only requested targets are reported: {stdout}"); + for line in &lines { + assert_eq!( + line["error"]["kind"].as_str(), + Some("rpc"), + "swept target stays unanswered: {line}" + ); + assert!( + line["error"]["message"].as_str().is_some_and(|m| m.contains(aborting)), + "the abort message must name the failing fetch: {line}" + ); + } + + assert_eq!(code, Some(3), "a transport abort exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); +} + +/// With `--dump-fixture-dir`, a target that executed and drafted a fixture +/// before a later transport abort keeps its result line; only the fixture field +/// fails, and that failure inherits the abort's `rpc` class so the run exits 3 +/// rather than converting the discard into an execution-class error. +#[test] +fn test_replay_tx_file_dump_fixture_inherits_transport_abort_class() { + let (aborting, aborting_index) = MID_BLOCK_NON_TARGET; + // Early non-deposit target builds a Ready draft; a later target forces the + // job past the non-target aborter between them. + let (early, early_index) = BLOCK_TXS[1]; + let (late, late_index) = BLOCK_TXS[2]; + assert!(early_index < aborting_index && aborting_index < late_index); + + let path = envelope_dropping_transaction("fixture_inherits_rpc_abort", aborting); + let list = format!("{early}\n{late}\n"); + let list_path = std::env::temp_dir() + .join(format!("mega_evme_tx_list_fixture_abort_{}.txt", std::process::id())); + std::fs::write(&list_path, list).expect("write tx list"); + let dir = + std::env::temp_dir().join(format!("mega_evme_fixture_abort_dir_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create dump dir"); + + let (stdout, code) = replay_envelope_with_code( + &path, + &[ + "--tx-file", + list_path.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list_path); + let _ = std::fs::remove_dir_all(&dir); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), 2, "both targets are reported: {stdout}"); + + let early_line = lines.iter().find(|l| l["tx_hash"] == early).expect("early target line"); + assert!( + early_line.get("error").is_none(), + "the early target keeps its execution result line: {early_line}" + ); + assert!( + early_line["success"].is_boolean(), + "result fields are present on the early target: {early_line}" + ); + let fixture = &early_line["fixture"]; + assert!( + fixture["error"].is_string(), + "the drafted fixture must report the abort (not a skip): {early_line}" + ); + assert!( + fixture["error"].as_str().is_some_and(|m| m.contains("aborted") || m.contains("discarded")), + "fixture error names the abort: {early_line}" + ); + assert!(fixture.get("path").is_none(), "no fixture file is written after abort"); + assert!(fixture.get("skipped").is_none(), "abort discard is an error, not a skip"); + + let late_line = lines.iter().find(|l| l["tx_hash"] == late).expect("late target line"); + assert_eq!( + late_line["error"]["kind"].as_str(), + Some("rpc"), + "the late target is swept as unanswered: {late_line}" + ); + assert!( + late_line["error"]["message"].as_str().is_some_and(|m| m.contains(aborting)), + "swept message names the failing fetch: {late_line}" + ); + + // Abort is transport-class; fixture discard inherits it — not exit 1. + assert_eq!(code, Some(3), "transport abort with fixture discard exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); +} + /// Targets swept up by an abort are reported in block transaction-index order, /// whatever order `--tx-file` listed them in. #[test] diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 9ba18e6f..8c02d957 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -140,10 +140,12 @@ With [`--dump-fixture-dir`](#--dump-fixture-dir-dir), each result line additiona ### Exit Status A batch run exits `0` when every requested transaction produced an execution result and nothing the run was asked to do failed, and non-zero otherwise — see [Exit codes](../overview.md#exit-codes) for how the failure classes are ranked. -Fixture skips (fidelity gate, BLOCKHASH readers, unsupported shapes) are not failures and do not fail the run; a fixture the run was asked to write and could not is an execution-class failure of its target. +Fixture skips (fidelity gate, BLOCKHASH readers, unsupported shapes) are not failures and do not fail the run; a fixture construction or write failure is an execution-class failure of its target. +When a mid-block abort discards a drafted fixture, that fixture error inherits the abort's class (so a transport abort still exits `3`). The NDJSON stream is written to stdout in both cases; diagnostics go to stderr. -The exit code can understate a failure in one case: when the transaction that aborts a block is not itself a target, no target can claim the abort's own class, so every target is reported as `rpc` and the run exits `3` — "the question went unanswered" — even if the underlying cause was a deterministic execution failure that retrying will not fix. +Swept targets behind an abort always report as `rpc` ("unanswered"). +When the aborting transaction is not itself a target, the run still tallies the abort's own class so the process exit reflects the root cause — a non-target executor abort exits `1`, a transport abort exits `3`. `--block 0` is rejected as invalid input; a block that genuinely holds no transactions produces no stdout lines, exits `0`, and says so on stderr. ### Examples From 6792c52d521f3fa4a59275e54aff4ec81fd2535a Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 11:23:32 +0800 Subject: [PATCH 53/64] fix(mega-evme): classify dump-fixture receipt anomalies as RPC failures The single-transaction --dump-fixture fidelity gate fetched the target's on-chain receipt with its own error mapping: a null receipt became TransactionNotFound (exit 1) and a divergent-inclusion receipt became Other (exit 1), while --verify-receipt classified both identical endpoint conditions as RpcError (exit 3). Route the dump path's fetch through verify::fetch_receipt and map the inclusion check to RpcError, so both modes report the same exit code, the same failure class, and the same message for an unanswered or divergent receipt. --- bin/mega-evme/src/replay/cmd.rs | 25 ++- bin/mega-evme/tests/replay_dump.rs | 314 +++++++++++++++++++++++++---- docs/mega-evme/commands/replay.md | 1 + 3 files changed, 287 insertions(+), 53 deletions(-) diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index d456685f..0ca23774 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -793,8 +793,9 @@ impl Cmd { // self-validation alone cannot catch. let fixture_inputs = if self.dump_fixture.is_some() { // A pending transaction has no receipt yet, so the fidelity gate cannot - // run; fail clearly instead of surfacing the receipt lookup's confusing - // `TransactionNotFound`. + // run; fail clearly here, where the missing receipt is a definitive + // property of the target, instead of surfacing the receipt lookup's + // infrastructure-class failure, which would invite a pointless retry. if ctx.target_tx.block_number.is_none() { return Err(ReplayError::Other( "--dump-fixture does not support pending transactions: the fidelity \ @@ -811,18 +812,22 @@ impl Cmd { let mut oracle_storage = external_envs.oracle_storage(); oracle_storage.sort_unstable(); let mega_env = state_test::types::MegaEnv { bucket_capacities, oracle_storage }; - let receipt = provider - .get_transaction_receipt(ctx.tx_hash) - .await - .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? - .ok_or(ReplayError::TransactionNotFound(ctx.tx_hash))?; + // Fetched through the same helper `--verify-receipt` uses, so both + // modes classify an unserved receipt identically. A null answer for a + // target this run has already resolved as mined is the endpoint + // failing to answer — a pruned receipt, or a backend serving a + // divergent view — not a definitive statement that the transaction + // does not exist, so it is a retryable infrastructure failure rather + // than an execution verdict. + let receipt = verify::fetch_receipt(provider, ctx.tx_hash).await?; // Anchor the receipt to the replayed block: across a reorg or a // load-balanced endpoint serving divergent views, the receipt can // describe a different inclusion than the block fetched earlier - // (including a receipt with `blockHash: null`). Same check as - // `--verify-receipt` so dump and verify agree on unanchored receipts. + // (including a receipt with `blockHash: null`). Same check and same + // failure class as `--verify-receipt`, so dump and verify agree on + // unanchored receipts. verify::check_inclusion(receipt.block_hash(), ctx.block.hash()) - .map_err(ReplayError::Other)?; + .map_err(ReplayError::RpcError)?; // RLP-hash the receipt's logs with the same helper the state-test // runner uses for `logsRoot`, so the dump can check the replay's logs // against the chain (the rich RPC logs' `inner` is the consensus log). diff --git a/bin/mega-evme/tests/replay_dump.rs b/bin/mega-evme/tests/replay_dump.rs index 4379bb86..0519200a 100644 --- a/bin/mega-evme/tests/replay_dump.rs +++ b/bin/mega-evme/tests/replay_dump.rs @@ -9,7 +9,8 @@ //! `state-test --bench`; see `bench/replay/`.) use std::{ - process::Command, + path::{Path, PathBuf}, + process::{Command, Output}, sync::{Arc, Mutex}, time::Duration, }; @@ -22,7 +23,7 @@ mod common; const CACHE: &str = "replay_offline.cache.json"; /// Path of the committed offline capture. -fn cache() -> std::path::PathBuf { +fn cache() -> PathBuf { common::fixture(CACHE) } @@ -33,6 +34,122 @@ fn mega_evme() -> Command { Command::new(env!("CARGO_BIN_EXE_mega-evme")) } +/// A temp path unique to this process and this test. +fn temp_path(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("mega_evme_dump_{name}_{}.json", std::process::id())) +} + +/// Run `replay --dump-fixture` offline against `cache`, writing to `out`. +fn dump(cache: &Path, out: &Path) -> Output { + mega_evme() + .args([ + "replay", + "--rpc.replay-file", + cache.to_str().expect("cache path is utf-8"), + "--dump-fixture", + out.to_str().expect("fixture path is utf-8"), + "--json", + TX, + ]) + .output() + .expect("failed to run mega-evme") +} + +/// Run `replay --verify-receipt` offline against `cache`, the mode whose +/// classification the dump path reuses. +fn verify(cache: &Path) -> Output { + mega_evme() + .args([ + "replay", + "--rpc.replay-file", + cache.to_str().expect("cache path is utf-8"), + "--verify-receipt", + "--json", + TX, + ]) + .output() + .expect("failed to run mega-evme") +} + +/// The structured error object a failing `--json` run ends its stdout with. +fn error_object(stdout: &str) -> serde_json::Value { + let values = common::json_values(stdout); + let last = values + .last() + .unwrap_or_else(|| panic!("a failing --json run must not leave stdout empty:\n{stdout}")); + assert!(common::is_run_error(last), "the last stdout value must be the error object: {last}"); + last["error"].clone() +} + +/// Write a copy of the committed capture in which the `result` of every cached +/// response `selects` accepts is rewritten by `doctor`, and return its path. +/// +/// Cache entries are keyed by the request, not the response, so a doctored entry +/// still resolves and the run meets the doctored answer where it would meet the +/// real one. +fn rewrite_cache( + name: &str, + selects: impl Fn(&serde_json::Value) -> bool, + doctor: impl Fn(&mut serde_json::Value), +) -> PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) + .expect("parse offline cache"); + let mut doctored = false; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse cached response"); + if !selects(&response["result"]) { + continue; + } + doctor(&mut response["result"]); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored = true; + } + assert!(doctored, "offline capture should contain the entry being doctored"); + + let path = temp_path(name); + std::fs::write(&path, envelope.to_string()).expect("write doctored cache"); + path +} + +/// A copy of the committed capture whose on-chain receipt response is rewritten +/// by `doctor`. The receipt is the only cached response carrying +/// `cumulativeGasUsed`. +fn cache_with_doctored_receipt(name: &str, doctor: impl Fn(&mut serde_json::Value)) -> PathBuf { + rewrite_cache(name, |result| result.get("cumulativeGasUsed").is_some(), doctor) +} + +/// A copy of the committed capture with the on-chain receipt dropped entirely. +/// Offline, the absent entry surfaces as a cache miss — the transport failing to +/// answer the receipt request at all. +fn cache_without_receipt(name: &str) -> PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) + .expect("parse offline cache"); + let entries = envelope["cache"].as_array_mut().expect("cache entries"); + let before = entries.len(); + entries.retain(|entry| { + !entry["value"].as_str().expect("entry value is a string").contains("cumulativeGasUsed") + }); + assert!(entries.len() < before, "offline capture should contain the receipt entry"); + + let path = temp_path(name); + std::fs::write(&path, envelope.to_string()).expect("write pruned cache"); + path +} + +/// A copy of the committed capture whose target-transaction lookup answers null, +/// modelling an endpoint that does not know the hash the caller asked about. +fn cache_with_null_target_transaction(name: &str) -> PathBuf { + rewrite_cache( + name, + |result| result.get("hash").and_then(serde_json::Value::as_str) == Some(TX), + |result| *result = serde_json::Value::Null, + ) +} + /// `--dump-fixture` is incompatible with transaction overrides (the isolated /// execution would not represent the on-chain transaction), and must be /// rejected before any execution, writing nothing. @@ -199,61 +316,172 @@ fn test_replay_dump_overwrites_atomically_without_tmp_residue() { /// The fidelity gate must reject a receipt that describes a different inclusion /// than the replayed block (a reorg in progress, or a load-balanced endpoint -/// serving divergent views). Doctor the captured receipt's `blockHash` and -/// expect a clear error with no fixture written. +/// serving divergent views). The endpoint served a view the run cannot use, so +/// this is a retryable infrastructure failure (exit 3) — the same class +/// `--verify-receipt` gives the identical condition — and no fixture is written. #[test] fn test_replay_dump_rejects_receipt_from_different_block() { - // Doctor the capture: flip the receipt's blockHash. Cache entries are keyed - // by the request, not the response, so the doctored entry still resolves. - let mut envelope: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(cache()).expect("read offline cache")) - .expect("parse offline cache"); - let mut doctored = false; - for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { - let value = entry["value"].as_str().expect("entry value is a string"); - // The receipt is the only cached response carrying cumulativeGasUsed. - if !value.contains("cumulativeGasUsed") { - continue; - } - let mut response: serde_json::Value = - serde_json::from_str(value).expect("parse receipt response"); - response["result"]["blockHash"] = serde_json::Value::String( - "0x1111111111111111111111111111111111111111111111111111111111111111".into(), - ); - entry["value"] = serde_json::Value::String(response.to_string()); - doctored = true; - } - assert!(doctored, "offline cache should contain the receipt entry"); - - let doctored_cache = - std::env::temp_dir().join(format!("mega_evme_reorg_cache_{}.json", std::process::id())); - std::fs::write(&doctored_cache, envelope.to_string()).expect("write doctored cache"); - let out = - std::env::temp_dir().join(format!("mega_evme_dump_reorg_{}.json", std::process::id())); + let doctored_cache = cache_with_doctored_receipt("reorg_cache", |receipt| { + receipt["blockHash"] = + "0x1111111111111111111111111111111111111111111111111111111111111111".into(); + }); + let out = temp_path("reorg"); let _ = std::fs::remove_file(&out); - let output = mega_evme() - .args([ - "replay", - "--rpc.replay-file", - doctored_cache.to_str().unwrap(), - "--dump-fixture", - out.to_str().unwrap(), - TX, - ]) - .output() - .expect("failed to run mega-evme"); + let output = dump(&doctored_cache, &out); let _ = std::fs::remove_file(&doctored_cache); - assert!(!output.status.success(), "a receipt from a different block must abort the dump"); let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(3), + "a receipt from a different block is an infrastructure failure.\nstderr: {stderr}", + ); assert!( stderr.contains("different inclusion"), "expected reorg/divergent-endpoint hint, got stderr:\n{stderr}" ); + let error = error_object(&String::from_utf8_lossy(&output.stdout)); + assert_eq!(error["kind"].as_str(), Some("rpc-failure"), "got: {error}"); assert!(!out.exists(), "must not write a fixture when the receipt anchor mismatches"); } +/// A receipt the endpoint answers with `null` leaves the fidelity gate's +/// question unanswered: the run resolved this very transaction as mined moments +/// earlier, so the null is a pruned receipt or a divergent backend, not a +/// definitive "unknown transaction". It must exit 3 as an infrastructure +/// failure, name the receipt, and write no fixture. +#[test] +fn test_replay_dump_null_receipt_is_an_rpc_failure() { + let doctored_cache = cache_with_doctored_receipt("null_receipt_cache", |receipt| { + *receipt = serde_json::Value::Null; + }); + let out = temp_path("null_receipt"); + let _ = std::fs::remove_file(&out); + + let output = dump(&doctored_cache, &out); + let _ = std::fs::remove_file(&doctored_cache); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(3), + "an unanswered receipt is an infrastructure failure.\nstderr: {stderr}", + ); + let error = error_object(&String::from_utf8_lossy(&output.stdout)); + assert_eq!(error["kind"].as_str(), Some("rpc-failure"), "got: {error}"); + let message = error["message"].as_str().expect("the error object carries a message"); + assert!( + message.contains("No on-chain receipt for transaction") && message.contains(TX), + "the message must name the unanswered receipt and its transaction: {message}" + ); + assert!( + !message.contains("Transaction not found"), + "a replayed transaction must not be reported as unknown: {message}" + ); + assert!(!out.exists(), "must not write a fixture when the receipt is unanswered"); +} + +/// A receipt request the transport never answers (offline: the capture holds no +/// receipt entry) is the third receipt-fetch anomaly, and stays an +/// infrastructure failure with no fixture written. +#[test] +fn test_replay_dump_unanswered_receipt_request_is_an_rpc_failure() { + let pruned_cache = cache_without_receipt("pruned_receipt_cache"); + let out = temp_path("pruned_receipt"); + let _ = std::fs::remove_file(&out); + + let output = dump(&pruned_cache, &out); + let _ = std::fs::remove_file(&pruned_cache); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(3), + "an unanswered receipt request is an infrastructure failure.\nstderr: {stderr}", + ); + let error = error_object(&String::from_utf8_lossy(&output.stdout)); + assert_eq!(error["kind"].as_str(), Some("rpc-failure"), "got: {error}"); + assert!( + error["message"].as_str().is_some_and(|m| m.contains("eth_getTransactionReceipt")), + "the message must name the unanswered request: {error}" + ); + assert!(!out.exists(), "must not write a fixture when the receipt request goes unanswered"); +} + +/// Negative control for the classification above: a null answer for the *target +/// transaction* is the endpoint answering the caller's own question with a +/// definitive "unknown transaction", and keeps its execution class (exit 1). +/// Only the fidelity gate's receipt question moved to the infrastructure class, +/// not every null the run can meet. +#[test] +fn test_replay_dump_null_target_transaction_stays_an_execution_error() { + let doctored_cache = cache_with_null_target_transaction("null_target_cache"); + let out = temp_path("null_target"); + let _ = std::fs::remove_file(&out); + + let output = dump(&doctored_cache, &out); + let _ = std::fs::remove_file(&doctored_cache); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(1), + "an unknown target transaction is a definitive negative answer.\nstderr: {stderr}", + ); + let error = error_object(&String::from_utf8_lossy(&output.stdout)); + assert_eq!(error["kind"].as_str(), Some("execution-error"), "got: {error}"); + assert!( + error["message"].as_str().is_some_and(|m| m.contains("Transaction not found")), + "got: {error}" + ); + assert!(!out.exists(), "must not write a fixture when the target is unknown"); +} + +/// The dump path and the verify path fetch the same receipt for the same +/// transaction, so an endpoint anomaly must produce the identical failure: same +/// exit code, same class, same message. A pipeline can then branch on the exit +/// code without knowing which mode produced it. +#[test] +fn test_dump_and_verify_classify_receipt_anomalies_identically() { + // A receipt the endpoint does not serve at all. + assert_dump_and_verify_agree("agree_null_receipt", |receipt| { + *receipt = serde_json::Value::Null; + }); + // A receipt describing a different inclusion than the replayed block. + assert_dump_and_verify_agree("agree_reorg", |receipt| { + receipt["blockHash"] = + "0x1111111111111111111111111111111111111111111111111111111111111111".into(); + }); +} + +/// Run both modes against a capture whose receipt response is rewritten by +/// `doctor`, and assert the two runs fail with the same exit code and the same +/// error object. +fn assert_dump_and_verify_agree(name: &str, doctor: impl Fn(&mut serde_json::Value)) { + let doctored_cache = cache_with_doctored_receipt(&format!("{name}_cache"), doctor); + let out = temp_path(name); + let _ = std::fs::remove_file(&out); + + let dumped = dump(&doctored_cache, &out); + let verified = verify(&doctored_cache); + let _ = std::fs::remove_file(&doctored_cache); + let _ = std::fs::remove_file(&out); + + assert_eq!( + dumped.status.code(), + verified.status.code(), + "{name}: dump and verify must exit with the same code.\ndump stderr: {}\nverify stderr: {}", + String::from_utf8_lossy(&dumped.stderr), + String::from_utf8_lossy(&verified.stderr), + ); + assert_eq!( + error_object(&String::from_utf8_lossy(&dumped.stdout)), + error_object(&String::from_utf8_lossy(&verified.stdout)), + "{name}: dump and verify must report the same failure object", + ); +} + /// `--dump-fixture` must reject `--override.spec` (a forced spec would make the /// fixture a what-if, not the on-chain transaction) and write nothing. #[test] diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 950da49f..6df15689 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -380,6 +380,7 @@ The fixture still self-validates and reproduces gas exactly; only such balance-d A target transaction that reads a block hash via `BLOCKHASH` is also rejected: fixtures carry no historical block hashes, so the isolated re-execution could not reproduce the values the replay observed. Block hash reads by preceding transactions in the same block do not matter — only the target transaction's reads are checked. Because the fidelity gate reads the receipt, an offline dump (`--rpc.replay-file`) requires the receipt to be present in the capture — so capture and dump together in the online run, then re-dump offline reproducibly. +A receipt the endpoint does not serve — no receipt at all, or one describing a different inclusion than the replayed block — is classified exactly as under [`--verify-receipt`](#--verify-receipt): an RPC failure (exit `3`), because the question went unanswered rather than answered no. When combined with `--rpc.capture-file`, the capture file is written even if execution or the fidelity gate fails, so the captured RPC responses remain available for debugging the failure offline. ```bash From 73c5582ae7aa8c8e24513f0c90dcddf7ab6a9d03 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 11:32:09 +0800 Subject: [PATCH 54/64] fix(mega-evme): batch receipt failures keep results and classify as rpc Route dump-dir and verify-receipt unanswered receipt questions as rpc-class findings on kept result lines (exit 3), early-return when no job targets sit in the fetched body, and document the --block 0 vs resolved-into-0 asymmetry. --- bin/mega-evme/src/replay/batch.rs | 245 +++++++++++++++++++-------- bin/mega-evme/src/replay/verify.rs | 84 ++++++++- bin/mega-evme/tests/replay_batch.rs | 135 ++++++++++++++- bin/mega-evme/tests/replay_verify.rs | 196 +++++++++++++++++++-- docs/mega-evme/commands/replay.md | 25 ++- 5 files changed, 577 insertions(+), 108 deletions(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 8c68fd60..b517ecf2 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -81,10 +81,11 @@ pub(super) struct ReportArgs { /// Per-target fixture dump outcome reported on the NDJSON / human result line. /// /// Exactly one field is set: the fixture was written, expectedly skipped -/// (fidelity gate, BLOCKHASH, unsupported shape), or could not be written. A -/// write failure is reported here rather than replacing the target's result, -/// so a target that did replay keeps its result — including its receipt -/// verification verdict — and still fails the run. +/// (fidelity mismatch, BLOCKHASH, unsupported shape), or could not be written. +/// A write failure — and an unanswered receipt question for the fidelity gate — +/// is reported here rather than replacing the target's result, so a target that +/// did replay keeps its result — including its receipt verification verdict — +/// and still fails the run. #[derive(Debug, Clone, Serialize)] struct FixtureReport { /// Absolute or as-written path of a successfully written fixture. @@ -97,8 +98,10 @@ struct FixtureReport { #[serde(skip_serializing_if = "Option::is_none")] error: Option, /// Batch tally class for [`Self::error`]. Construction and write failures - /// are execution-class; a draft discarded because the block aborted inherits - /// the abort's class so a transient RPC abort does not become exit 1. + /// are execution-class; an unanswered on-chain receipt (transport, pruned, + /// divergent inclusion, missing from the offline envelope) is rpc-class; a + /// draft discarded because the block aborted inherits the abort's class so + /// a transient RPC abort does not become exit 1. /// Not serialized: the wire shape stays `path` / `skipped` / `error`. #[serde(skip)] error_kind: BatchErrorKind, @@ -133,6 +136,21 @@ impl FixtureReport { } } + /// Fidelity gate could not run because the on-chain receipt question went + /// unanswered (transport, null, reorg, or offline envelope missing it). + /// + /// Distinct from a genuine skip (BLOCKHASH, unsupported shape, fidelity + /// mismatch): the dump was requested and the receipt call failed, so the + /// run exits non-zero as rpc-class. + fn rpc_error(message: impl Into) -> Self { + Self { + path: None, + skipped: None, + error: Some(message.into()), + error_kind: BatchErrorKind::Rpc, + } + } + /// Fixture discarded because the block aborted after the draft was built. /// /// The target keeps its execution result; only the fixture field fails, and @@ -280,7 +298,9 @@ impl BatchTally { /// /// A verdict and a fixture failure are independent findings about the same /// target: a replay that diverged from its receipt is counted as a mismatch - /// whether or not its fixture could be written. + /// whether or not its fixture could be written. An unanswered receipt + /// question (verification unavailable, or dump-dir fidelity gate starved of + /// a receipt) is rpc-class and does not count as verified. fn record_executed( &mut self, verification: Option<&VerificationOutcome>, @@ -289,15 +309,21 @@ impl BatchTally { self.reported += 1; self.replayed += 1; if let Some(verification) = verification { - self.verified += 1; - if !verification.matched { - self.counts.mismatched += 1; + if verification.is_unavailable() { + // Compared path never ran: the receipt question went unanswered. + self.counts.rpc += 1; + } else { + self.verified += 1; + if !verification.matched { + self.counts.mismatched += 1; + } } } // A fixture the run was asked to write and could not is a failure of // that target, even though its replay produced a result. Construction - // and write failures are execution-class; an abort-inherited discard - // uses the abort's class (see [`FixtureReport::abort_error`]). + // and write failures are execution-class; an unanswered receipt for the + // fidelity gate is rpc-class; an abort-inherited discard uses the + // abort's class (see [`FixtureReport::abort_error`]). if let Some(fixture) = fixture.filter(|f| f.is_error()) { match fixture.error_kind { BatchErrorKind::Rpc => self.counts.rpc += 1, @@ -681,10 +707,14 @@ where let target_hashes = || targets.iter().map(|t| t.hash); if number == 0 { + // Distinct from `--block 0` (invalid request, exit 1): an endpoint that + // resolves a hash into block 0 is contradictory endpoint data — the + // same unanswered class as unanchored / contradictory metadata. return BlockReplayOutcome::entries_only(fail_all( target_hashes(), BatchErrorKind::Rpc, - "Block 0 has no parent block to fork from", + "endpoint resolved the target into block 0, which has no parent block \ + to fork from: contradictory endpoint data", )); } @@ -701,43 +731,6 @@ where } }, }; - let parent_block = match fetch_block(provider, number - 1).await { - Ok(block) => block, - Err(e) => { - return BlockReplayOutcome::entries_only(fail_all( - target_hashes(), - BatchErrorKind::Rpc, - &e.to_string(), - )); - } - }; - - // Both guards below check the *headers* the endpoint served. The state - // reads behind the fork are still addressed by block number, so an endpoint - // that serves headers and state from different backends can still hand back - // state for a different block at this height. Anchoring state reads to the - // validated hash would need the fork to take a block hash rather than a - // number, and would change every cached RPC key (alloy hashes the block id - // into the cache key), invalidating every committed offline capture. - // - // Parent/block linkage guard: across a reorg or a load-balanced endpoint - // serving divergent views, `eth_getBlockByNumber(N-1)` can return a block - // that is not the parent of the block being replayed. Forking from that - // state would silently execute against the wrong pre-state. - let parent_hash = parent_block.hash(); - let expected_parent = block.header.parent_hash(); - if parent_hash != expected_parent { - let message = format!( - "parent block hash {parent_hash} != block parent_hash {expected_parent}: the parent \ - block describes a different chain than the block being replayed (reorg in progress, \ - or a load-balanced endpoint serving divergent views); retry once the chain settles" - ); - return BlockReplayOutcome::entries_only(fail_all( - target_hashes(), - BatchErrorKind::Rpc, - &message, - )); - } // Per-target inclusion and membership guards. `--tx-file` resolved each // target through `eth_getTransactionByHash`, which reported the block it @@ -746,6 +739,11 @@ where // hashes get independent outcomes. A target whose reported hash matches // the body but is missing from it is an endpoint self-contradiction (`rpc`), // not a definitive "unknown hash". + // + // When none of the job's targets appear in the body, every target already + // has its definitive answer here — skip parent fetch, state forking, and + // the execute loop entirely. Otherwise `last_target_index` would be `None` + // and the foreign block would be walked for nothing. let fetched = block.hash(); let body_txs: HashSet = block.transactions.hashes().collect(); let mut entries = Vec::with_capacity(targets.len()); @@ -779,6 +777,22 @@ where )); continue; } + } else if !body_txs.contains(&target.hash) { + // `--block` targets come from the body, so this arm is defensive. + // A residual not-in-body without an inclusion claim is still an + // unanswered view of this height, not a definitive not-found. + entries.push(failure( + target.hash, + BatchErrorKind::Rpc, + format!( + "block {number} ({fetched}) does not list target transaction {}, which \ + was queued against it: the endpoint served divergent views of this \ + block (reorg in progress, or a load-balanced endpoint); retry once \ + the chain settles", + target.hash, + ), + )); + continue; } active.push(target.hash); } @@ -789,6 +803,45 @@ where } let targets = active; + // Both guards below check the *headers* the endpoint served. The state + // reads behind the fork are still addressed by block number, so an endpoint + // that serves headers and state from different backends can still hand back + // state for a different block at this height. Anchoring state reads to the + // validated hash would need the fork to take a block hash rather than a + // number, and would change every cached RPC key (alloy hashes the block id + // into the cache key), invalidating every committed offline capture. + // + // Parent/block linkage guard: across a reorg or a load-balanced endpoint + // serving divergent views, `eth_getBlockByNumber(N-1)` can return a block + // that is not the parent of the block being replayed. Forking from that + // state would silently execute against the wrong pre-state. + let parent_block = match fetch_block(provider, number - 1).await { + Ok(block) => block, + Err(e) => { + return BlockReplayOutcome::entries_only(fail_remaining( + &targets, + entries, + BatchErrorKind::Rpc, + &e.to_string(), + )); + } + }; + let parent_hash = parent_block.hash(); + let expected_parent = block.header.parent_hash(); + if parent_hash != expected_parent { + let message = format!( + "parent block hash {parent_hash} != block parent_hash {expected_parent}: the parent \ + block describes a different chain than the block being replayed (reorg in progress, \ + or a load-balanced endpoint serving divergent views); retry once the chain settles" + ); + return BlockReplayOutcome::entries_only(fail_remaining( + &targets, + entries, + BatchErrorKind::Rpc, + &message, + )); + } + // Fetch the on-chain receipts before the block runs. Needed for // `--verify-receipt` (mismatch vs unverified) and for `--dump-fixture-dir` // (fidelity gate). A receipt that cannot be fetched, or that describes a @@ -1070,22 +1123,21 @@ where target.tx_index, first_log_index, ); + // Keep the execution result even when the receipt question went + // unanswered: the target did replay, so its summary, local + // receipt, and timing stay on the result line. The verification + // field carries the failure; the tally counts it as rpc. let verification = if verify_receipt { match onchain_receipts.get(&target.tx_hash) { Some(Ok(onchain)) => { Some(verify::compare(onchain, &ReceiptFacts::from_receipt(&receipt))) } - // Without an on-chain receipt there is nothing to compare - // against: report the target as unverified. - unverified => { - let message = match unverified { - Some(Err(message)) => message.clone(), - _ => "No on-chain receipt was fetched for this transaction" - .to_string(), - }; - entries.push(failure(target.tx_hash, BatchErrorKind::Rpc, message)); - continue; + Some(Err(message)) => { + Some(VerificationOutcome::unavailable(message.clone())) } + None => Some(VerificationOutcome::unavailable( + "No on-chain receipt was fetched for this transaction", + )), } } else { None @@ -1239,8 +1291,11 @@ struct DumpFixtureArgs<'a> { /// Prepare a fixture for one successfully executed target against pre-commit state. /// -/// Expected skips (missing receipt, fidelity mismatch, BLOCKHASH, unsupported -/// transaction shapes) become a final [`FixtureReport`] and never fail the run. +/// Genuine skips (fidelity mismatch, BLOCKHASH, unsupported transaction shapes) +/// become a final [`FixtureReport::skipped`] and never fail the run. +/// An unanswered on-chain receipt (transport, pruned/null, divergent inclusion, +/// or offline envelope lacking it) becomes a rpc-class fixture error so the run +/// exits 3 while the target keeps its execution result line. /// Database and other construction failures become a fixture error (execution-class). /// A successfully built draft is carried as [`DeferredFixture::Ready`] and only /// written by [`materialize_deferred_fixture`] after `finish()` succeeds. @@ -1266,19 +1321,18 @@ where overwrite, } = args; - // Fidelity gate needs the on-chain receipt; without it the dump is skipped, - // not failed — the envelope may simply lack receipts (expected in sweeps - // over captures that never fetched them). + // Fidelity gate needs the on-chain receipt. When the receipt question went + // unanswered the dump fails as rpc (not a fidelity-gate skip): the run was + // asked to write a fixture and could not obtain the receipt it needs. Genuine + // gate skips (BLOCKHASH, unsupported shape, fidelity mismatch) stay skips. let facts = match onchain { Some(Ok(facts)) => facts, Some(Err(message)) => { - return DeferredFixture::Report(FixtureReport::skipped(format!( - "fidelity-gate-unavailable: {message}" - ))); + return DeferredFixture::Report(FixtureReport::rpc_error(message.clone())); } None => { - return DeferredFixture::Report(FixtureReport::skipped( - "fidelity-gate-unavailable: no on-chain receipt was fetched for this transaction", + return DeferredFixture::Report(FixtureReport::rpc_error( + "no on-chain receipt was fetched for this transaction", )); } }; @@ -1618,7 +1672,7 @@ mod tests { /// A verification verdict as a run would have reported it. fn verdict(matched: bool) -> VerificationOutcome { - VerificationOutcome { matched, diff: None } + VerificationOutcome::compared(matched, None) } /// Build a tally from the outcomes a run would have reported: `failures` @@ -1817,6 +1871,44 @@ mod tests { assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::RpcFailure); } + /// An unanswered receipt for `--verify-receipt` keeps the target as + /// replayed, counts as rpc (not mismatched), and is not "verified". + #[test] + fn test_batch_tally_verification_unavailable_is_rpc_and_still_replayed() { + let mut tally = BatchTally::default(); + tally.record_executed(Some(&VerificationOutcome::unavailable("receipt pruned")), None); + + assert_eq!(tally.replayed, 1, "the target still replayed"); + assert_eq!(tally.verified, 0, "no comparison ran"); + assert_eq!(tally.counts.mismatched, 0, "unverified is not a mismatch"); + assert_eq!(tally.counts.rpc, 1); + let err = tally.into_error().expect("run failed"); + let ReplayError::BatchFailed(counts) = err else { + panic!("expected batch failure: {err:?}"); + }; + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::RpcFailure); + } + + /// An unanswered receipt for `--dump-fixture-dir` is a rpc-class fixture + /// error, not a skip that exits 0. + #[test] + fn test_batch_tally_fixture_receipt_unavailable_is_rpc_class() { + let mut tally = BatchTally::default(); + tally.record_executed( + None, + Some(&FixtureReport::rpc_error("no on-chain receipt was fetched for this transaction")), + ); + + assert_eq!(tally.replayed, 1); + assert_eq!(tally.counts.rpc, 1); + assert_eq!(tally.counts.execution, 0); + let err = tally.into_error().expect("run failed"); + let ReplayError::BatchFailed(counts) = err else { + panic!("expected batch failure: {err:?}"); + }; + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::RpcFailure); + } + /// A run whose only finding is divergence fails as the mismatch it is. #[test] fn test_batch_tally_mismatch_only_reports_the_verification_error() { @@ -1925,9 +2017,9 @@ mod tests { #[test] fn test_materialize_deferred_fixture_passes_reports_through() { let skipped = materialize_deferred_fixture(DeferredFixture::Report( - FixtureReport::skipped("fidelity-gate-unavailable: no receipt"), + FixtureReport::skipped("fidelity gate failed: gas_used"), )); - assert_eq!(skipped.skipped.as_deref(), Some("fidelity-gate-unavailable: no receipt")); + assert_eq!(skipped.skipped.as_deref(), Some("fidelity gate failed: gas_used")); assert!(skipped.path.is_none()); assert!(skipped.error.is_none()); @@ -1936,5 +2028,12 @@ mod tests { ))); assert!(err.error.as_ref().is_some_and(|m| m.contains("construction failed"))); assert!(err.path.is_none()); + + let rpc = materialize_deferred_fixture(DeferredFixture::Report(FixtureReport::rpc_error( + "no on-chain receipt was fetched for this transaction", + ))); + assert!(rpc.is_error()); + assert_eq!(rpc.error_kind, BatchErrorKind::Rpc); + assert!(rpc.skipped.is_none()); } } diff --git a/bin/mega-evme/src/replay/verify.rs b/bin/mega-evme/src/replay/verify.rs index 36e65156..b462a825 100644 --- a/bin/mega-evme/src/replay/verify.rs +++ b/bin/mega-evme/src/replay/verify.rs @@ -53,26 +53,79 @@ impl ReceiptFacts { } /// The verdict for one verified transaction. -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +/// +/// Three shapes on the wire: +/// - compared and equal: `{"match": true}` +/// - compared and diverged: `{"match": false, "diff": …}` +/// - receipt question unanswered: `{"error": "…"}` — the target still replayed; only the comparison +/// could not run (transport, pruned, reorg). +/// +/// Serialize is hand-written so an unavailable outcome never emits a false +/// `match` that a consumer would read as a divergence. +#[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct VerificationOutcome { /// Whether the local replay reproduced the on-chain receipt. - #[serde(rename = "match")] + /// + /// Meaningless when [`Self::error`] is set (kept for a simple bool check + /// on the compared path); the wire shape omits `match` in that case. pub matched: bool, - /// The mismatched dimensions; absent when the replay matched. - #[serde(skip_serializing_if = "Option::is_none")] + /// The mismatched dimensions; absent when the replay matched or when the + /// comparison never ran. pub diff: Option, + /// Why the on-chain receipt could not be compared, when the target still + /// produced a local result. Mutually exclusive with a real match/diff. + pub error: Option, } impl VerificationOutcome { + /// A completed comparison against an on-chain receipt. + pub(super) fn compared(matched: bool, diff: Option) -> Self { + Self { matched, diff, error: None } + } + + /// The target replayed, but the on-chain receipt question went unanswered. + pub(super) fn unavailable(message: impl Into) -> Self { + Self { matched: false, diff: None, error: Some(message.into()) } + } + + /// Whether this outcome is an unanswered receipt fetch, not a comparison. + pub(super) const fn is_unavailable(&self) -> bool { + self.error.is_some() + } + /// The one-line human verdict printed for a verified transaction. pub(super) fn verdict_line(&self) -> String { - match &self.diff { - None => "verification: MATCH".to_string(), - Some(diff) => format!("verification: MISMATCH ({})", diff.describe()), + if let Some(error) = &self.error { + format!("verification: FAILED ({error})") + } else if let Some(diff) = &self.diff { + format!("verification: MISMATCH ({})", diff.describe()) + } else { + "verification: MATCH".to_string() } } } +impl Serialize for VerificationOutcome { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeMap; + if let Some(error) = &self.error { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("error", error)?; + return map.end(); + } + let fields = 1 + usize::from(self.diff.is_some()); + let mut map = serializer.serialize_map(Some(fields))?; + map.serialize_entry("match", &self.matched)?; + if let Some(diff) = &self.diff { + map.serialize_entry("diff", diff)?; + } + map.end() + } +} + /// The mismatched dimensions of a verification. Dimensions that agree are /// absent, so a diff never has to be scanned for "everything equal" entries. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] @@ -235,9 +288,9 @@ pub(super) fn compare(onchain: &ReceiptFacts, replay: &ReceiptFacts) -> Verifica } if diff.is_empty() { - VerificationOutcome { matched: true, diff: None } + VerificationOutcome::compared(true, None) } else { - VerificationOutcome { matched: false, diff: Some(diff) } + VerificationOutcome::compared(false, Some(diff)) } } @@ -394,6 +447,19 @@ mod tests { assert_eq!(outcome.verdict_line(), "verification: MATCH"); } + /// An unanswered receipt serializes as `{"error": …}` with no `match` field, + /// so consumers never read it as a false mismatch. + #[test] + fn test_unavailable_outcome_serializes_as_error_only() { + let outcome = VerificationOutcome::unavailable("receipt pruned below retention"); + assert!(outcome.is_unavailable()); + assert_eq!( + json(&outcome), + serde_json::json!({ "error": "receipt pruned below retention" }) + ); + assert_eq!(outcome.verdict_line(), "verification: FAILED (receipt pruned below retention)"); + } + #[test] fn test_compare_empty_logs_on_both_sides_match() { let outcome = compare(&facts(vec![]), &facts(vec![])); diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 88fd42a2..f8be9176 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -1043,6 +1043,129 @@ fn test_replay_tx_file_anchored_but_absent_target_is_rpc() { let _ = std::fs::remove_file(&list); } +/// When every target of a job is absent from the fetched block body, each gets +/// its definitive `rpc` answer pre-loop and the block is never executed. +/// +/// After doctoring the body, the envelope is stripped of every response that is +/// not a target transaction lookup or the doctored block body (parent header, +/// state, receipts, body-tx objects). A clean early return needs only those two +/// shapes; forking state or walking the body would miss and fail differently. +#[test] +fn test_replay_tx_file_all_targets_absent_from_body_skips_block_execution() { + let targets: Vec<&str> = BLOCK_TXS.iter().map(|(h, _)| *h).collect(); + + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + + let mut body_doctored = 0; + let mut block_hash = None; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + let Ok(mut response) = serde_json::from_str::(value) else { + continue; + }; + let Some(result) = response.get_mut("result") else { + continue; + }; + if !result.is_object() { + continue; + } + let number = result.get("number").and_then(|n| { + n.as_str().and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + }); + if number != Some(BLOCK) { + continue; + } + let Some(txs) = result.get_mut("transactions").and_then(|t| t.as_array_mut()) else { + continue; + }; + let before = txs.len(); + txs.retain(|tx| { + let hash = tx.as_str().unwrap_or(""); + !targets.contains(&hash) + }); + if txs.len() != before { + block_hash = result.get("hash").and_then(|h| h.as_str()).map(str::to_string); + entry["value"] = serde_json::Value::String(response.to_string()); + body_doctored += 1; + } + } + assert_eq!(body_doctored, 1, "exactly one block body for {BLOCK} must list the targets"); + let block_hash = block_hash.expect("block hash"); + + // Cache keys are request hashes, so filter by response shape: keep only the + // target transaction lookups and the doctored block-at-height body. + let entries = envelope["cache"].as_array_mut().expect("cache entries"); + entries.retain(|entry| { + let value = entry["value"].as_str().unwrap_or(""); + let Ok(response) = serde_json::from_str::(value) else { + return false; + }; + let Some(result) = response.get("result") else { + return false; + }; + if !result.is_object() { + return false; + } + // Transaction lookup for one of our targets. + if let Some(hash) = result.get("hash").and_then(|h| h.as_str()) { + if targets.contains(&hash) && result.get("blockNumber").is_some() { + return true; + } + } + // Doctored block body at the job height. + let number = result.get("number").and_then(|n| { + n.as_str().and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()) + }); + number == Some(BLOCK) && result.get("transactions").is_some() + }); + + let envelope_path = std::env::temp_dir() + .join(format!("mega_evme_batch_all_absent_{}.json", std::process::id())); + std::fs::write(&envelope_path, envelope.to_string()).expect("write doctored envelope"); + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_all_absent_{}.txt", std::process::id())); + std::fs::write(&list, format!("{}\n", targets.join("\n"))).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&envelope_path, &["--tx-file", list.to_str().unwrap(), "--json"]); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), targets.len(), "every target is reported once: {stdout}"); + + for target in &targets { + let failed = lines + .iter() + .find(|line| line["tx_hash"].as_str() == Some(target)) + .unwrap_or_else(|| panic!("target {target} must be reported")); + assert_eq!(failed["error"]["kind"].as_str(), Some("rpc"), "all-absent is rpc: {failed}"); + let message = failed["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("does not list") && message.contains(&block_hash), + "message names absence and block: {message}" + ); + assert!( + failed.get("success").is_none() && failed.get("receipt").is_none(), + "no execution result for an unexecuted target: {failed}" + ); + } + + assert_eq!(code, Some(3), "all-absent exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); + // If the driver had forked state or walked body transactions, the stripped + // envelope would have produced a cache-miss error naming parent/state — not + // a clean per-target membership rpc answer for every hash. + assert!( + !stdout.contains("cache miss") && + !stdout.contains("not found in the offline") && + !stdout.contains("not present in the offline"), + "early return must not touch parent/state/body-tx paths:\n{stdout}" + ); + + let _ = std::fs::remove_file(&envelope_path); + let _ = std::fs::remove_file(&list); +} + /// A hash whose resolution answers `null` keeps the definitive `not_found` /// class: the endpoint denied the hash, rather than claiming inclusion and then /// contradicting itself. @@ -1419,18 +1542,18 @@ fn test_replay_receipt_inner_log_metadata_nonzero_preceding_offset() { } } -/// Sweeping a block with `--dump-fixture-dir` against an envelope that carries -/// no receipts skips every target on the fidelity gate and still exits 0. -/// -/// Fixture skips are expected (not infrastructure failures); the development -/// `--block N --dump-fixture-dir` writes a fixture for every transaction it can -/// express and skips the ones it cannot, without failing the run. +/// Sweeping a block with `--dump-fixture-dir` writes a fixture for every +/// transaction it can express and skips genuine unsupported shapes (deposit) +/// without failing the run. /// /// Every OP-stack block opens with a deposit, which the fixture format cannot /// represent. Reporting that as an error rather than a skip would make a /// whole-block sweep exit non-zero on every block, so this pins the /// classification end to end: 22 files written, the deposit skipped with its /// reason, nothing reported as an error, and exit 0. +/// +/// (An unanswered on-chain receipt is a separate rpc-class fixture error and is +/// covered by the doctored dump-dir tests in `replay_verify`.) #[test] fn test_replay_block_dump_fixture_dir_writes_all_but_the_deposit() { let dir = std::env::temp_dir() diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index a17f8d32..f3aa1788 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -374,10 +374,11 @@ fn test_batch_verify_receipt_reports_a_mismatch_and_exits_nonzero() { ); } -/// In batch mode an unavailable receipt turns the target into an `rpc` error -/// entry — reported as unverified, never as a mismatch. +/// In batch mode an unavailable receipt keeps the replayed result line and +/// reports the failure on `verification.error` — never as a mismatch, and never +/// by discarding the execution summary. #[test] -fn test_batch_verify_receipt_missing_receipt_is_an_rpc_error_entry() { +fn test_batch_verify_receipt_missing_receipt_keeps_result_and_is_rpc() { let path = cache_without_receipt("batch_pruned"); let list = tx_file("batch_pruned"); @@ -388,8 +389,24 @@ fn test_batch_verify_receipt_missing_receipt_is_an_rpc_error_entry() { assert_eq!(run.code(), 3, "an unverified target exits 3.\nstderr: {}", run.stderr); let lines = run.ndjson(); assert_eq!(lines.len(), 1, "one line per requested transaction"); - assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); - assert!(lines[0].get("verification").is_none(), "an unverified target carries no verdict"); + assert!( + lines[0].get("error").is_none(), + "a replayed target keeps its result line, not a bare error entry: {}", + lines[0] + ); + assert!(lines[0]["receipt"].is_object(), "local receipt is kept: {}", lines[0]); + assert_eq!(lines[0]["success"].as_bool(), Some(true)); + assert_eq!(lines[0]["gas_used"].as_u64(), Some(GAS_USED)); + assert!( + lines[0]["verification"]["error"].is_string(), + "verification carries the unanswered receipt: {}", + lines[0] + ); + assert!( + lines[0]["verification"].get("match").is_none(), + "unavailable is not a match/mismatch verdict: {}", + lines[0] + ); assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("rpc-failure")); assert!( !run.stderr.contains("verification mismatch"), @@ -398,9 +415,10 @@ fn test_batch_verify_receipt_missing_receipt_is_an_rpc_error_entry() { ); } -/// The reorg guard applies in batch mode too, as an `rpc` error entry. +/// The reorg guard applies in batch mode too: the result is kept and the +/// divergent-inclusion failure is reported on `verification.error`. #[test] -fn test_batch_verify_receipt_reorg_is_an_rpc_error_entry() { +fn test_batch_verify_receipt_reorg_keeps_result_and_is_rpc() { let path = doctored_cache("batch_reorg", |receipt| { receipt["blockHash"] = "0x1111111111111111111111111111111111111111111111111111111111111111".into(); @@ -413,9 +431,9 @@ fn test_batch_verify_receipt_reorg_is_an_rpc_error_entry() { assert_eq!(run.code(), 3, "an unverified target exits 3.\nstderr: {}", run.stderr); let lines = run.ndjson(); - assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); + assert!(lines[0].get("error").is_none(), "result line is kept: {}", lines[0]); assert!( - lines[0]["error"]["message"] + lines[0]["verification"]["error"] .as_str() .is_some_and(|message| message.contains("different inclusion")), "expected the reorg/divergent-endpoint hint: {}", @@ -449,9 +467,10 @@ fn test_verify_receipt_null_block_hash_is_an_infrastructure_error() { ); } -/// Batch mode reports a null `blockHash` as an `rpc` error entry. +/// Batch mode reports a null `blockHash` on the kept result line as +/// `verification.error` (rpc), not as a bare error entry. #[test] -fn test_batch_verify_receipt_null_block_hash_is_an_rpc_error_entry() { +fn test_batch_verify_receipt_null_block_hash_keeps_result_and_is_rpc() { let path = doctored_cache("batch_null_block_hash", |receipt| { receipt["blockHash"] = serde_json::Value::Null; }); @@ -463,15 +482,164 @@ fn test_batch_verify_receipt_null_block_hash_is_an_rpc_error_entry() { assert_eq!(run.code(), 3, "an unverified target exits 3.\nstderr: {}", run.stderr); let lines = run.ndjson(); - assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); + assert!(lines[0].get("error").is_none(), "result line is kept: {}", lines[0]); assert!( - lines[0]["error"]["message"] + lines[0]["verification"]["error"] .as_str() .is_some_and(|message| message.contains("no block hash")), "expected a missing-inclusion-hash hint: {}", lines[0] ); - assert!(lines[0].get("verification").is_none(), "an unverified target carries no verdict"); + assert!( + lines[0]["verification"].get("match").is_none(), + "unavailable is not a match/mismatch verdict: {}", + lines[0] + ); +} + +/// Dump-dir with a nulled receipt response fails the fidelity gate as rpc on +/// the kept result line (not a silent `fidelity-gate-unavailable` skip). +#[test] +fn test_batch_dump_fixture_dir_null_receipt_is_rpc_fixture_error() { + let path = doctored_cache("batch_dump_null_receipt", |receipt| { + // Doctor the whole result to null by replacing the entry value below. + let _ = receipt; + }); + // Null the receipt response entirely (result: null), modelling a pruned + // or unanswered eth_getTransactionReceipt. + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).expect("read doctored cache")) + .expect("parse"); + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains("cumulativeGasUsed") { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse receipt response"); + response["result"] = serde_json::Value::Null; + entry["value"] = serde_json::Value::String(response.to_string()); + } + std::fs::write(&path, envelope.to_string()).expect("rewrite null-receipt cache"); + + let list = tx_file("batch_dump_null_receipt"); + let dir = std::env::temp_dir() + .join(format!("mega_evme_batch_dump_null_receipt_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let run = replay( + &path, + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(run.code(), 3, "unanswered receipt for dump exits 3.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1); + assert!(lines[0].get("error").is_none(), "result line is kept: {}", lines[0]); + assert!(lines[0]["receipt"].is_object(), "local receipt is kept: {}", lines[0]); + assert_eq!(lines[0]["success"].as_bool(), Some(true)); + assert!( + lines[0]["fixture"]["error"].is_string(), + "fixture reports the unanswered receipt: {}", + lines[0] + ); + assert!( + lines[0]["fixture"].get("skipped").is_none(), + "receipt fetch failure is not a fidelity-gate skip: {}", + lines[0] + ); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("rpc-failure")); +} + +/// Dump-dir against an envelope that never captured the receipt is the same +/// unanswered class: rpc fixture error, result kept, exit 3. +#[test] +fn test_batch_dump_fixture_dir_missing_receipt_is_rpc_fixture_error() { + let path = cache_without_receipt("batch_dump_no_receipt"); + let list = tx_file("batch_dump_no_receipt"); + let dir = std::env::temp_dir() + .join(format!("mega_evme_batch_dump_no_receipt_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let run = replay( + &path, + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(run.code(), 3, "missing receipt for dump exits 3.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1); + assert!(lines[0].get("error").is_none(), "result line is kept: {}", lines[0]); + assert!(lines[0]["receipt"].is_object(), "local receipt is kept: {}", lines[0]); + assert!( + lines[0]["fixture"]["error"].as_str().is_some_and(|m| m.contains("receipt") || + m.contains("not found") || + m.contains("cache")), + "fixture.error names the unanswered receipt: {}", + lines[0] + ); + assert!( + lines[0]["fixture"].get("skipped").is_none(), + "missing receipt is not a silent skip: {}", + lines[0] + ); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("rpc-failure")); +} + +/// A genuine fidelity-gate skip (local gas disagrees with on-chain receipt) +/// still exits 0: the receipt question was answered, the dump was correctly +/// refused, and skips never fail the run. +#[test] +fn test_batch_dump_fixture_dir_fidelity_mismatch_stays_skip() { + let path = doctored_cache("batch_dump_fidelity_skip", |receipt| { + receipt["gasUsed"] = "0x1".into(); + }); + let list = tx_file("batch_dump_fidelity_skip"); + let dir = std::env::temp_dir() + .join(format!("mega_evme_batch_dump_fidelity_skip_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let run = replay( + &path, + &[ + "--tx-file", + list.to_str().unwrap(), + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(run.code(), 0, "a fidelity skip exits 0.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1); + assert!( + lines[0]["fixture"]["skipped"].as_str().is_some_and(|m| m.contains("fidelity gate failed")), + "expected fidelity-gate skip: {}", + lines[0] + ); + assert!(lines[0]["fixture"].get("error").is_none()); } /// Batch `--dump-fixture-dir` writes a self-validating fixture for a target diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 8c02d957..a40aec31 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -140,13 +140,15 @@ With [`--dump-fixture-dir`](#--dump-fixture-dir-dir), each result line additiona ### Exit Status A batch run exits `0` when every requested transaction produced an execution result and nothing the run was asked to do failed, and non-zero otherwise — see [Exit codes](../overview.md#exit-codes) for how the failure classes are ranked. -Fixture skips (fidelity gate, BLOCKHASH readers, unsupported shapes) are not failures and do not fail the run; a fixture construction or write failure is an execution-class failure of its target. +Fixture skips (fidelity mismatch, BLOCKHASH readers, unsupported shapes) are not failures and do not fail the run; a fixture construction or write failure is an execution-class failure of its target; an unanswered on-chain receipt for the fidelity gate is an rpc-class failure of its target. When a mid-block abort discards a drafted fixture, that fixture error inherits the abort's class (so a transport abort still exits `3`). The NDJSON stream is written to stdout in both cases; diagnostics go to stderr. Swept targets behind an abort always report as `rpc` ("unanswered"). When the aborting transaction is not itself a target, the run still tallies the abort's own class so the process exit reflects the root cause — a non-target executor abort exits `1`, a transport abort exits `3`. -`--block 0` is rejected as invalid input; a block that genuinely holds no transactions produces no stdout lines, exits `0`, and says so on stderr. +`--block 0` is rejected as invalid input (exit `1`): the user asked for a genesis block that cannot be replayed. +An endpoint that resolves a transaction hash into block 0 is contradictory endpoint data instead — each such target is reported as `rpc` and the run exits `3`, in the same family as unanchored views and contradictory metadata. +A block that genuinely holds no transactions produces no stdout lines, exits `0`, and says so on stderr. ### Examples @@ -208,7 +210,9 @@ Anything that prevents the comparison from running is an infrastructure failure - The receipt describes a different inclusion than the replayed block (its `blockHash` differs from the replayed block, or is null — a reorg in progress, or a load-balanced endpoint serving divergent views): reported as an `rpc` failure, because comparing against it would compare the replay to the wrong on-chain execution, and a receipt with no inclusion hash cannot be anchored at all. - The target is a pending transaction, which has no receipt yet: rejected up front in single-transaction mode, and reported as a `pending` error entry in batch mode. -In batch mode each of these becomes an error entry for that transaction, exactly like any other infrastructure failure. +In batch mode, when a target already produced an execution result and only the receipt fetch failed, the target keeps its full result line (execution summary, local receipt, timing) and reports the failure on that line as `"verification": {"error": "…"}`. +The target still counts as `replayed`; the unanswered receipt is tallied as `rpc` and the run exits `3`. +A target that never reached execution (pending, not-found, block setup failure) remains a bare error entry, exactly like any other infrastructure failure before replay. Transaction overrides and `--override.spec` are still accepted with `--verify-receipt`, but they make the replay a what-if that the chain never executed, so the comparison will normally report a mismatch. @@ -223,6 +227,12 @@ A match carries nothing else: { "match": true } ``` +An unanswered receipt (fetch failed, pruned, reorg / divergent inclusion) carries only the error — no `match` field, so it is never confused with a divergence: + +```json +{ "error": "No on-chain receipt was fetched for this transaction" } +``` + A mismatch carries a `diff` holding only the dimensions that disagreed, each as `{"onchain": …, "replay": …}`: ```json @@ -252,9 +262,11 @@ Without `--json`, each transaction gets one verdict line after its usual output: ``` verification: MATCH verification: MISMATCH (gas_used: onchain 75514 vs replay 75500) +verification: FAILED (No on-chain receipt was fetched for this transaction) ``` The mismatch line names every dimension that disagreed, comma-separated. +The failed line is used when the comparison never ran. ### Exit Status @@ -405,7 +417,7 @@ The fixture draft is built against the pre-commit state (same moment as the sing | Gate | Outcome | | -------------------------------------------------------------------------------- | ------------------------------------------------------------- | -| On-chain receipt unavailable (not in capture, pruned, reorg/divergent inclusion) | `fixture.skipped` with `fidelity-gate-unavailable: …` | +| On-chain receipt unavailable (not in capture, pruned, reorg/divergent inclusion) | `fixture.error` with the reason; rpc-class failure | | Fidelity mismatch (gas / status / logs root) | `fixture.skipped` with `fidelity gate failed: …` | | Target reads `BLOCKHASH` | `fixture.skipped` (fixtures carry no historical block hashes) | | Unsupported shape (deposit, EIP-7702, unknown spec mapping) | `fixture.skipped` | @@ -420,13 +432,14 @@ Human mode prints one fixture line per target. An end-of-run `INFO` summary reports written / skipped / failed counts. A failed dump is reported on the target's own result line rather than replacing it: the transaction did replay, so its receipt — and, with [`--verify-receipt`](#receipt-verification), its verdict — is still what the run was asked for, and a divergence found on such a target is still counted as a mismatch. -Fixture skips do not fail the run; a failed dump does, as an execution-class failure of that target. +Fixture skips do not fail the run; a failed dump does — as an execution-class failure for construction/write failures, or as an rpc-class failure when the on-chain receipt question went unanswered. Registration into `bench/replay/manifest.json` is not performed — corpus curation stays manual. `--dump-fixture-dir` cannot be combined with `--dump-fixture`, and is rejected in single-transaction mode. ```bash -# Sweep a whole block offline into per-tx fixtures (skips targets without receipts): +# Sweep a whole block offline into per-tx fixtures (targets whose capture lacks +# a receipt fail as rpc on the result line and exit 3): mega-evme replay --rpc.replay-file ./fixtures/blocks.json \ --block 22945844 --dump-fixture-dir ./fixtures/out --json From 0901653db72767d6a0854e59e709690417be98ca Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 11:33:53 +0800 Subject: [PATCH 55/64] fix(mega-evme): reject a receipt served for another transaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eth_getTransactionReceipt is queried by transaction hash, but nothing checked that the answer describes that transaction: the guard compared only block hashes, and ReceiptFacts discards transactionHash. An inconsistent endpoint or a tampered capture could therefore have verify report a verdict about a different transaction — including a spurious match when the consensus facts coincide — and have the dump path anchor a fixture to it. Validate the identity in verify::fetch_receipt, the single seam that single verify, single dump and batch all fetch through, and classify a mismatch as an RPC failure whose message names both the served and the requested hash. --- bin/mega-evme/src/replay/verify.rs | 66 +++++++++++++++++++++++++-- bin/mega-evme/tests/replay_dump.rs | 39 ++++++++++++++++ bin/mega-evme/tests/replay_verify.rs | 67 ++++++++++++++++++++++++++++ docs/mega-evme/commands/replay.md | 21 ++++----- 4 files changed, 179 insertions(+), 14 deletions(-) diff --git a/bin/mega-evme/src/replay/verify.rs b/bin/mega-evme/src/replay/verify.rs index 36e65156..0f43e1b6 100644 --- a/bin/mega-evme/src/replay/verify.rs +++ b/bin/mega-evme/src/replay/verify.rs @@ -7,7 +7,8 @@ //! and testable without a provider. //! //! Anything that prevents the comparison from running at all (a receipt the -//! endpoint cannot serve, or a receipt describing a different inclusion than the +//! endpoint cannot serve, a receipt describing a different transaction than the +//! one requested, or a receipt describing a different inclusion than the //! replayed block) is an infrastructure failure, never a mismatch: a target that //! could not be verified must not be reported as a divergence. @@ -293,12 +294,15 @@ fn compare_log(index: usize, onchain: &Log, replay: &Log) -> Option(provider: &P, tx_hash: B256) -> Result where P: Provider, { - provider + let receipt = provider .get_transaction_receipt(tx_hash) .await .map_err(|e| ReplayError::RpcError(format!("Failed to fetch receipt: {e}")))? @@ -307,7 +311,34 @@ where "No on-chain receipt for transaction {tx_hash}: the transaction is unknown to \ the endpoint, or the endpoint has pruned its receipt" )) - }) + })?; + check_transaction_identity(receipt.inner.transaction_hash, tx_hash) + .map_err(ReplayError::RpcError)?; + Ok(receipt) +} + +/// Check that a fetched receipt describes the transaction it was requested for. +/// +/// `eth_getTransactionReceipt` is asked by transaction hash, but nothing in the +/// answer forces the endpoint to honour it: an inconsistent backend, or a +/// tampered offline capture, can serve another transaction's receipt. Comparing +/// against it would report a verdict about the wrong transaction — a mismatch +/// blamed on the replay, or a spurious match when the two transactions happen to +/// share their consensus facts — and the dump path would anchor a fixture to it. +/// Returns the explanatory message so each mode can wrap it in the error shape it +/// reports. +pub(super) fn check_transaction_identity( + receipt_tx_hash: B256, + requested_tx_hash: B256, +) -> std::result::Result<(), String> { + if receipt_tx_hash == requested_tx_hash { + return Ok(()); + } + Err(format!( + "receipt is for transaction {receipt_tx_hash}, but transaction {requested_tx_hash} was \ + requested: the endpoint served the receipt of a different transaction (an inconsistent \ + backend, or a tampered capture); the transaction is unverified" + )) } /// Check that a fetched receipt describes the block the replay executed. @@ -616,6 +647,33 @@ mod tests { ); } + #[test] + fn test_check_transaction_identity_accepts_the_requested_transaction() { + let hash = b256!("0x3333333333333333333333333333333333333333333333333333333333333333"); + + assert!(check_transaction_identity(hash, hash).is_ok()); + } + + /// A receipt for another transaction is rejected, and the message names both + /// hashes so the served/requested confusion is diagnosable from the error + /// alone. + #[test] + fn test_check_transaction_identity_rejects_another_transactions_receipt() { + let served = b256!("0x3333333333333333333333333333333333333333333333333333333333333333"); + let requested = b256!("0x4444444444444444444444444444444444444444444444444444444444444444"); + + let message = check_transaction_identity(served, requested) + .expect_err("a receipt for another transaction must be rejected"); + + assert!( + message.contains(&format!("{served}")) && + message.contains(&format!("{requested}")) && + message.contains("different transaction") && + message.contains("unverified"), + "message must name both hashes and explain the target is unverified: {message}" + ); + } + #[test] fn test_check_inclusion_rejects_a_missing_block_hash() { let replayed = b256!("0x2222222222222222222222222222222222222222222222222222222222222222"); diff --git a/bin/mega-evme/tests/replay_dump.rs b/bin/mega-evme/tests/replay_dump.rs index 0519200a..a3a24f83 100644 --- a/bin/mega-evme/tests/replay_dump.rs +++ b/bin/mega-evme/tests/replay_dump.rs @@ -346,6 +346,40 @@ fn test_replay_dump_rejects_receipt_from_different_block() { assert!(!out.exists(), "must not write a fixture when the receipt anchor mismatches"); } +/// The fidelity gate must reject a receipt that describes a different +/// transaction than the one requested: anchoring a fixture to another +/// transaction's gas, status and logs would bake a wrong expectation into the +/// artifact. The endpoint answered a question that was never asked, so this is an +/// infrastructure failure (exit 3) and no fixture is written. +#[test] +fn test_replay_dump_rejects_receipt_for_another_transaction() { + const OTHER_TX: &str = "0x00000000000000000000000000000000000000000000000000000000feed0001"; + + let doctored_cache = cache_with_doctored_receipt("wrong_tx_cache", |receipt| { + receipt["transactionHash"] = OTHER_TX.into(); + }); + let out = temp_path("wrong_tx"); + let _ = std::fs::remove_file(&out); + + let output = dump(&doctored_cache, &out); + let _ = std::fs::remove_file(&doctored_cache); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + output.status.code(), + Some(3), + "a receipt for another transaction is an infrastructure failure.\nstderr: {stderr}", + ); + let error = error_object(&String::from_utf8_lossy(&output.stdout)); + assert_eq!(error["kind"].as_str(), Some("rpc-failure"), "got: {error}"); + let message = error["message"].as_str().expect("the error object carries a message"); + assert!( + message.contains(OTHER_TX) && message.contains(TX), + "the message must name both the served and the requested transaction: {message}" + ); + assert!(!out.exists(), "must not write a fixture when the receipt is for another transaction"); +} + /// A receipt the endpoint answers with `null` leaves the fidelity gate's /// question unanswered: the run resolved this very transaction as mined moments /// earlier, so the null is a pruned receipt or a divergent backend, not a @@ -453,6 +487,11 @@ fn test_dump_and_verify_classify_receipt_anomalies_identically() { receipt["blockHash"] = "0x1111111111111111111111111111111111111111111111111111111111111111".into(); }); + // A receipt describing a different transaction than the one requested. + assert_dump_and_verify_agree("agree_wrong_tx", |receipt| { + receipt["transactionHash"] = + "0x00000000000000000000000000000000000000000000000000000000feed0001".into(); + }); } /// Run both modes against a capture whose receipt response is rewritten by diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index a17f8d32..701e51d0 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -25,6 +25,10 @@ const TX: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616 /// Gas the transaction used on-chain, which a faithful replay reproduces. const GAS_USED: u64 = 75_514; +/// A transaction hash that is not the replayed target, used to model an endpoint +/// answering a receipt request with another transaction's receipt. +const OTHER_TX: &str = "0x00000000000000000000000000000000000000000000000000000000feed0001"; + /// Outcome of one `mega-evme replay` invocation. struct Run { success: bool, @@ -305,6 +309,37 @@ fn test_verify_receipt_reorg_is_an_infrastructure_error() { ); } +/// A receipt describing a different transaction than the one requested is an +/// infrastructure failure: comparing against it would report a verdict about the +/// wrong transaction, so the target is unverified and the message names both the +/// requested and the served hash. +#[test] +fn test_verify_receipt_for_another_transaction_is_an_infrastructure_error() { + let path = doctored_cache("wrong_tx", |receipt| { + receipt["transactionHash"] = OTHER_TX.into(); + }); + + let run = replay(&path, &["--verify-receipt", "--json", TX]); + let _ = std::fs::remove_file(&path); + + assert_eq!(run.code(), 3, "a receipt for another transaction exits 3.\nstderr: {}", run.stderr); + assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("rpc-failure")); + let message = run.error_object()["error"]["message"] + .as_str() + .expect("the error object carries a message") + .to_string(); + assert!( + message.contains(OTHER_TX) && message.contains(TX), + "the message must name both the served and the requested transaction: {message}" + ); + assert!( + !run.stderr.contains("verification mismatch") && !run.stdout.contains("MISMATCH"), + "a receipt for another transaction must not be reported as a mismatch:\n{}\n{}", + run.stdout, + run.stderr, + ); +} + /// A receipt the endpoint cannot serve (e.g. pruned below its retention height) /// is an infrastructure failure, not a mismatch. #[test] @@ -423,6 +458,38 @@ fn test_batch_verify_receipt_reorg_is_an_rpc_error_entry() { ); } +/// The identity guard applies in batch mode too, as an `rpc` error entry naming +/// both hashes, and the target carries no verdict. +#[test] +fn test_batch_verify_receipt_for_another_transaction_is_an_rpc_error_entry() { + let path = doctored_cache("batch_wrong_tx", |receipt| { + receipt["transactionHash"] = OTHER_TX.into(); + }); + let list = tx_file("batch_wrong_tx"); + + let run = replay(&path, &["--tx-file", list.to_str().unwrap(), "--verify-receipt", "--json"]); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + + assert_eq!(run.code(), 3, "an unverified target exits 3.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "one line per requested transaction"); + assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); + assert!( + lines[0]["error"]["message"] + .as_str() + .is_some_and(|message| message.contains(OTHER_TX) && message.contains(TX)), + "expected both the served and the requested transaction: {}", + lines[0] + ); + assert!(lines[0].get("verification").is_none(), "an unverified target carries no verdict"); + assert!( + !run.stderr.contains("verification mismatch"), + "an unverifiable target must not fail as a mismatch:\n{}", + run.stderr + ); +} + /// A receipt with a null `blockHash` cannot be anchored to the replayed block: /// infrastructure failure, never a match/mismatch verdict. #[test] diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index 6df15689..8ebfdd2e 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -208,6 +208,7 @@ Anything that prevents the comparison from running is an infrastructure failure - The endpoint fails the receipt call, or has pruned the receipt below its retention height (common on non-archive endpoints): reported as an `rpc` failure. - The receipt describes a different inclusion than the replayed block (its `blockHash` differs from the replayed block, or is null — a reorg in progress, or a load-balanced endpoint serving divergent views): reported as an `rpc` failure, because comparing against it would compare the replay to the wrong on-chain execution, and a receipt with no inclusion hash cannot be anchored at all. +- The receipt describes a different transaction than the one requested (its `transactionHash` is not the hash the receipt was asked for — an inconsistent endpoint, or a tampered capture): reported as an `rpc` failure, because the verdict would describe the wrong transaction, and two transactions sharing their consensus facts would even yield a spurious match. - The target is a pending transaction, which has no receipt yet: rejected up front in single-transaction mode, and reported as a `pending` error entry in batch mode. In batch mode each of these becomes an error entry for that transaction, exactly like any other infrastructure failure. @@ -380,7 +381,7 @@ The fixture still self-validates and reproduces gas exactly; only such balance-d A target transaction that reads a block hash via `BLOCKHASH` is also rejected: fixtures carry no historical block hashes, so the isolated re-execution could not reproduce the values the replay observed. Block hash reads by preceding transactions in the same block do not matter — only the target transaction's reads are checked. Because the fidelity gate reads the receipt, an offline dump (`--rpc.replay-file`) requires the receipt to be present in the capture — so capture and dump together in the online run, then re-dump offline reproducibly. -A receipt the endpoint does not serve — no receipt at all, or one describing a different inclusion than the replayed block — is classified exactly as under [`--verify-receipt`](#--verify-receipt): an RPC failure (exit `3`), because the question went unanswered rather than answered no. +A receipt the endpoint does not serve — no receipt at all, one describing a different inclusion than the replayed block, or one describing a different transaction than the one requested — is classified exactly as under [`--verify-receipt`](#--verify-receipt): an RPC failure (exit `3`), because the question went unanswered rather than answered no. When combined with `--rpc.capture-file`, the capture file is written even if execution or the fidelity gate fails, so the captured RPC responses remain available for debugging the failure offline. ```bash @@ -406,15 +407,15 @@ Existing files are refused unless `--overwrite` is also set — a refused overwr Per-target gating mirrors the single-transaction rules, but records a skip instead of failing the run. The fixture draft is built against the pre-commit state (same moment as the single-transaction dump) and only written after the block finishes successfully — a commit-time rejection or finish failure never creates or replaces a fixture file. -| Gate | Outcome | -| -------------------------------------------------------------------------------- | ------------------------------------------------------------- | -| On-chain receipt unavailable (not in capture, pruned, reorg/divergent inclusion) | `fixture.skipped` with `fidelity-gate-unavailable: …` | -| Fidelity mismatch (gas / status / logs root) | `fixture.skipped` with `fidelity gate failed: …` | -| Target reads `BLOCKHASH` | `fixture.skipped` (fixtures carry no historical block hashes) | -| Unsupported shape (deposit, EIP-7702, unknown spec mapping) | `fixture.skipped` | -| Fixture construction failure (database / pre-state reads) | `fixture.error` with the reason; execution-class failure | -| Finalize / write / self-validation failure, refused overwrite | `fixture.error` with the reason; execution-class failure | -| Pending / unresolvable target | already an error entry; no fixture report | +| Gate | Outcome | +| ----------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | +| On-chain receipt unavailable (not in capture, pruned, reorg/divergent inclusion, receipt for another transaction) | `fixture.skipped` with `fidelity-gate-unavailable: …` | +| Fidelity mismatch (gas / status / logs root) | `fixture.skipped` with `fidelity gate failed: …` | +| Target reads `BLOCKHASH` | `fixture.skipped` (fixtures carry no historical block hashes) | +| Unsupported shape (deposit, EIP-7702, unknown spec mapping) | `fixture.skipped` | +| Fixture construction failure (database / pre-state reads) | `fixture.error` with the reason; execution-class failure | +| Finalize / write / self-validation failure, refused overwrite | `fixture.error` with the reason; execution-class failure | +| Pending / unresolvable target | already an error entry; no fixture report | `BLOCKHASH` access is isolated per transaction: the access record is cleared before each transaction of the block, so preceding readers do not poison a later target's dump. From 1434eda87c44ab436b7df0d1a79db050383e1d8b Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 11:54:06 +0800 Subject: [PATCH 56/64] fix(mega-evme): keep batch tallies truthful and NDJSON ordered Non-target aborts floor the run exit without inflating per-target failure counts. Shared receipt failures under --verify-receipt and --dump-fixture-dir count once while both result fields stay present. Same-block entries emit in documented (block, tx_index) order with absent-last placement. Docs match anchored-absence rpc and the dual unverified jq selector. --- bin/mega-evme/src/common/error.rs | 26 ++ bin/mega-evme/src/common/exit.rs | 72 +++++- bin/mega-evme/src/replay/batch.rs | 344 ++++++++++++++++++++------- bin/mega-evme/tests/replay_batch.rs | 114 ++++++++- bin/mega-evme/tests/replay_verify.rs | 54 +++++ docs/mega-evme/commands/replay.md | 7 +- 6 files changed, 521 insertions(+), 96 deletions(-) diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index 2d504e7c..26e34b20 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -116,12 +116,33 @@ pub enum EvmeError { Other(String), } +/// Exit-code floor contributed by a non-target mid-block abort. +/// +/// Per-target failure counters stay strictly about reported targets. When the +/// aborting transaction is not itself a target, this floor still ranks the run +/// exit by the abort's root cause without inflating the "N of M target(s) +/// failed" totals. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum BatchExitFloor { + /// No non-target abort contributed a floor. + #[default] + None, + /// A non-target abort was execution-class (setup, executor rejection, …). + Execution, + /// A non-target abort was rpc-class (transport, cache miss, …). + Rpc, +} + /// How many targets of a batch replay failed, by failure class. /// /// A batch run reports every target on its own output line and then fails once /// with this summary, so the exit-code mapping can apply the batch precedence /// (execution before RPC before mismatch) without re-reading the per-target /// lines or parsing an error message. +/// +/// [`Self::execution`], [`Self::rpc`], [`Self::mismatched`], and [`Self::total`] +/// count only reported targets. [`Self::exit_floor`] is consulted solely for +/// exit ranking when a non-target abort's class is not carried by any target. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct BatchFailureCounts { /// Targets that failed for an execution, setup, or definitive-answer reason @@ -134,10 +155,15 @@ pub struct BatchFailureCounts { pub mismatched: usize, /// Targets the run reported on. pub total: usize, + /// Non-target abort class that floors the run exit without being a target + /// failure count. Display ignores this for "N of M"; exit mapping uses it. + pub exit_floor: BatchExitFloor, } impl core::fmt::Display for BatchFailureCounts { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + // Totals stay per-target: a non-target abort floor must not print + // "3 of 2 target transaction(s) failed". write!( f, "{} of {} target transaction(s) failed ({} execution, {} rpc)", diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs index 497c3423..854cb9d4 100644 --- a/bin/mega-evme/src/common/exit.rs +++ b/bin/mega-evme/src/common/exit.rs @@ -182,14 +182,21 @@ impl ExitCode { /// verified, so reporting such a run as a mismatch would overstate what it /// found. /// + /// [`BatchFailureCounts::exit_floor`] ranks with the same precedence when a + /// non-target abort's class is not carried by any reported target: an + /// execution floor outranks target-only rpc failures, and an rpc floor still + /// yields rpc when every target was clean of infrastructure failures. + /// /// Counts that record no failure at all reach this mapping only through a /// batch aggregation bug, since the run reports a failure precisely when it /// counted one. That is an internal error, not a success: a failure can /// never produce exit `0`. pub const fn from_batch_failures(counts: &BatchFailureCounts) -> Self { - if counts.execution > 0 { + use crate::common::BatchExitFloor; + + if counts.execution > 0 || matches!(counts.exit_floor, BatchExitFloor::Execution) { Self::ExecutionError - } else if counts.rpc > 0 { + } else if counts.rpc > 0 || matches!(counts.exit_floor, BatchExitFloor::Rpc) { Self::RpcFailure } else if counts.mismatched > 0 { Self::VerificationMismatch @@ -397,16 +404,70 @@ mod tests { /// Batch precedence: execution beats rpc beats mismatch. #[test] fn test_batch_failure_precedence() { - let mixed = BatchFailureCounts { execution: 1, rpc: 2, mismatched: 3, total: 6 }; + let mixed = BatchFailureCounts { + execution: 1, + rpc: 2, + mismatched: 3, + total: 6, + ..Default::default() + }; assert_eq!(ExitCode::from_batch_failures(&mixed), ExitCode::ExecutionError); - let rpc_only = BatchFailureCounts { execution: 0, rpc: 2, mismatched: 3, total: 6 }; + let rpc_only = BatchFailureCounts { + execution: 0, + rpc: 2, + mismatched: 3, + total: 6, + ..Default::default() + }; assert_eq!(ExitCode::from_batch_failures(&rpc_only), ExitCode::RpcFailure); - let mismatch_only = BatchFailureCounts { execution: 0, rpc: 0, mismatched: 3, total: 6 }; + let mismatch_only = BatchFailureCounts { + execution: 0, + rpc: 0, + mismatched: 3, + total: 6, + ..Default::default() + }; assert_eq!(ExitCode::from_batch_failures(&mismatch_only), ExitCode::VerificationMismatch); } + /// A non-target execution abort floors exit 1 even when every reported + /// target failure is rpc (swept unanswered). + #[test] + fn test_batch_exit_floor_execution_outranks_target_rpc() { + use crate::common::BatchExitFloor; + + let counts = BatchFailureCounts { + execution: 0, + rpc: 2, + mismatched: 0, + total: 2, + exit_floor: BatchExitFloor::Execution, + }; + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::ExecutionError); + assert_eq!( + counts.to_string(), + "2 of 2 target transaction(s) failed (0 execution, 2 rpc)", + "floor must not inflate the target totals" + ); + } + + /// A non-target rpc abort floors exit 3 when targets alone would not. + #[test] + fn test_batch_exit_floor_rpc_when_targets_clean_of_infra() { + use crate::common::BatchExitFloor; + + let counts = BatchFailureCounts { + execution: 0, + rpc: 0, + mismatched: 1, + total: 1, + exit_floor: BatchExitFloor::Rpc, + }; + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::RpcFailure); + } + /// A batch failure that counted nothing is an internal error, never a /// success: an error variant must not be able to produce exit 0. #[test] @@ -434,6 +495,7 @@ mod tests { rpc: 1, mismatched: 1, total: 3, + ..Default::default() }); assert_eq!(ExitCode::from_evme_error(&with_execution), ExitCode::ExecutionError); } diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index b517ecf2..8fec63a4 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -18,7 +18,7 @@ //! about. Offline, those come back as `rpc` entries and the run exits `3`. use std::{ - collections::{BTreeMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, path::{Path, PathBuf}, str::FromStr, time::{Duration, Instant}, @@ -50,8 +50,8 @@ use tracing::{debug, info, warn}; use crate::{ common::{ - op_receipt_to_tx_receipt, print_execution_summary, print_receipt, BatchFailureCounts, - EvmeExternalEnvs, ExecutionSummary, ExitCode, OpTxReceipt, + op_receipt_to_tx_receipt, print_execution_summary, print_receipt, BatchExitFloor, + BatchFailureCounts, EvmeExternalEnvs, ExecutionSummary, ExitCode, OpTxReceipt, }, replay::get_hardfork_config, ChainArgs, EvmeState, @@ -260,6 +260,11 @@ struct FailedTx { /// A batch reports each target as it goes and fails once at the end, so the /// outcome classes are counted here rather than recovered from the emitted /// lines. +/// +/// Per-target counters ([`Self::counts`], [`Self::reported`]) stay strictly +/// about emitted target entries. A non-target abort's class is carried only as +/// [`Self::exit_floor`] so the human "N of M" totals stay truthful while the +/// run exit still reflects the root cause. #[derive(Debug, Default)] struct BatchTally { /// Targets the run reported on, one per emitted entry. @@ -268,8 +273,10 @@ struct BatchTally { replayed: usize, /// Targets compared against an on-chain receipt. verified: usize, - /// Failed and mismatched targets, by class. + /// Failed and mismatched targets, by class (reported targets only). counts: BatchFailureCounts, + /// Run-level exit floor from a non-target abort not carried by any target. + exit_floor: BatchExitFloor, } impl BatchTally { @@ -301,6 +308,10 @@ impl BatchTally { /// whether or not its fixture could be written. An unanswered receipt /// question (verification unavailable, or dump-dir fidelity gate starved of /// a receipt) is rpc-class and does not count as verified. + /// + /// When both `--verify-receipt` and `--dump-fixture-dir` fail on the same + /// unanswered receipt, both result fields stay on the line but the shared + /// rpc failure is counted once. fn record_executed( &mut self, verification: Option<&VerificationOutcome>, @@ -308,10 +319,12 @@ impl BatchTally { ) { self.reported += 1; self.replayed += 1; + let mut receipt_rpc_counted = false; if let Some(verification) = verification { if verification.is_unavailable() { // Compared path never ran: the receipt question went unanswered. self.counts.rpc += 1; + receipt_rpc_counted = true; } else { self.verified += 1; if !verification.matched { @@ -326,7 +339,14 @@ impl BatchTally { // abort's class (see [`FixtureReport::abort_error`]). if let Some(fixture) = fixture.filter(|f| f.is_error()) { match fixture.error_kind { - BatchErrorKind::Rpc => self.counts.rpc += 1, + BatchErrorKind::Rpc => { + // Same missing receipt as verification.error: one target, + // one rpc count. Independent fixture rpc failures (none + // today share the gate without verification) still count. + if !receipt_rpc_counted { + self.counts.rpc += 1; + } + } BatchErrorKind::NotFound | BatchErrorKind::Pending | BatchErrorKind::Execution => { self.counts.execution += 1 } @@ -334,21 +354,29 @@ impl BatchTally { } } - /// Count a mid-block abort whose root-cause class is not already carried by + /// Record a mid-block abort whose root-cause class is not already carried by /// a per-target failure entry. /// /// Swept targets always stay `rpc` ("unanswered"). When the aborting /// transaction is not itself a reported target, that class would otherwise /// be lost and a deterministic executor abort would exit 3. The abort is - /// tallied once without emitting an extra NDJSON line or incrementing - /// `reported`. + /// recorded as an exit floor only: it does not emit an NDJSON line, does + /// not increment `reported`, and does not inflate the per-target counters. fn record_uncounted_abort(&mut self, kind: BatchErrorKind) { - match kind { - BatchErrorKind::Rpc => self.counts.rpc += 1, + let floor = match kind { + BatchErrorKind::Rpc => BatchExitFloor::Rpc, BatchErrorKind::NotFound | BatchErrorKind::Pending | BatchErrorKind::Execution => { - self.counts.execution += 1 + BatchExitFloor::Execution } - } + }; + // Multiple blocks can each contribute a floor; keep the more severe. + self.exit_floor = match (self.exit_floor, floor) { + (BatchExitFloor::Execution, _) | (_, BatchExitFloor::Execution) => { + BatchExitFloor::Execution + } + (BatchExitFloor::Rpc, _) | (_, BatchExitFloor::Rpc) => BatchExitFloor::Rpc, + (BatchExitFloor::None, BatchExitFloor::None) => BatchExitFloor::None, + }; } /// Targets that failed, by any class other than a receipt mismatch. @@ -363,10 +391,14 @@ impl BatchTally { /// run whose only finding is divergence fails as the mismatch it is. /// Fixture skips never count as failures; a fixture that could not be /// written does, as an execution-class failure of its target. + /// + /// A non-target abort floor alone also fails the run (with empty target + /// failure counters) so the exit still reflects the root cause. fn into_error(self) -> Option { - if self.failed() > 0 { + if self.failed() > 0 || self.exit_floor != BatchExitFloor::None { return Some(ReplayError::BatchFailed(BatchFailureCounts { total: self.reported, + exit_floor: self.exit_floor, ..self.counts })); } @@ -673,11 +705,57 @@ struct BlockReplayOutcome { } impl BlockReplayOutcome { - fn entries_only(entries: Vec) -> Self { - Self { entries, uncounted_abort: None } + /// Order entries into documented stream order before returning. + /// + /// Pre-execution inclusion/membership failures are collected before the + /// execute loop, while canonical results are appended after `finish()`. + /// Without a final reorder, a later same-block target's inclusion failure + /// would precede an earlier target's execution result. + fn ordered( + entries: Vec, + job_targets: &[JobTarget], + block_tx_order: Option<&[B256]>, + uncounted_abort: Option, + ) -> Self { + Self { entries: order_block_entries(entries, job_targets, block_tx_order), uncounted_abort } } } +/// Order a block's entries: targets present in the body by ascending transaction +/// index, then targets the block cannot place (inclusion/membership failures) +/// last, in job input order. +fn order_block_entries( + entries: Vec, + job_targets: &[JobTarget], + block_tx_order: Option<&[B256]>, +) -> Vec { + if entries.len() <= 1 { + return entries; + } + let mut by_hash: HashMap = HashMap::with_capacity(entries.len()); + for entry in entries { + by_hash.insert(entry.tx_hash(), entry); + } + let mut ordered = Vec::with_capacity(by_hash.len()); + if let Some(block_txs) = block_tx_order { + for hash in block_txs { + if let Some(entry) = by_hash.remove(hash) { + ordered.push(entry); + } + } + } + // Residual targets (absent from the body, or no body order available) keep + // the job's input order — the documented absent-last placement. + for target in job_targets { + if let Some(entry) = by_hash.remove(&target.hash) { + ordered.push(entry); + } + } + // Defensive: anything not listed on the job (should not happen). + ordered.extend(by_hash.into_values()); + ordered +} + /// Replay one block, reporting an entry for every target it was asked about. /// /// The block is executed exactly once: every transaction runs in order, and each @@ -700,22 +778,27 @@ async fn replay_block

( where P: Provider + Clone + std::fmt::Debug, { - let BlockJob { number, block, targets } = job; + let BlockJob { number, block, targets: job_targets } = job; let verify_receipt = report.verify_receipt; let dump_dir = report.dump_fixture_dir.as_deref(); let overwrite = report.overwrite; - let target_hashes = || targets.iter().map(|t| t.hash); + let target_hashes = || job_targets.iter().map(|t| t.hash); if number == 0 { // Distinct from `--block 0` (invalid request, exit 1): an endpoint that // resolves a hash into block 0 is contradictory endpoint data — the // same unanswered class as unanchored / contradictory metadata. - return BlockReplayOutcome::entries_only(fail_all( - target_hashes(), - BatchErrorKind::Rpc, - "endpoint resolved the target into block 0, which has no parent block \ - to fork from: contradictory endpoint data", - )); + return BlockReplayOutcome::ordered( + fail_all( + target_hashes(), + BatchErrorKind::Rpc, + "endpoint resolved the target into block 0, which has no parent block \ + to fork from: contradictory endpoint data", + ), + &job_targets, + None, + None, + ); } let block = match block { @@ -723,14 +806,17 @@ where None => match fetch_block(provider, number).await { Ok(block) => block, Err(e) => { - return BlockReplayOutcome::entries_only(fail_all( - target_hashes(), - BatchErrorKind::Rpc, - &e.to_string(), - )); + return BlockReplayOutcome::ordered( + fail_all(target_hashes(), BatchErrorKind::Rpc, &e.to_string()), + &job_targets, + None, + None, + ); } }, }; + // Body order for the documented ascending `(block, tx_index)` stream. + let block_tx_order: Vec = block.transactions.hashes().collect(); // Per-target inclusion and membership guards. `--tx-file` resolved each // target through `eth_getTransactionByHash`, which reported the block it @@ -744,11 +830,15 @@ where // has its definitive answer here — skip parent fetch, state forking, and // the execute loop entirely. Otherwise `last_target_index` would be `None` // and the foreign block would be walked for nothing. + // + // Pre-execution failures are buffered into `entries` and reordered with + // executed results at return time so a later-index inclusion failure cannot + // precede an earlier target's result line. let fetched = block.hash(); - let body_txs: HashSet = block.transactions.hashes().collect(); - let mut entries = Vec::with_capacity(targets.len()); + let body_txs: HashSet = block_tx_order.iter().copied().collect(); + let mut entries = Vec::with_capacity(job_targets.len()); let mut active: Vec = Vec::new(); - for target in &targets { + for target in &job_targets { if let Some(reported) = target.inclusion_hash { if reported != fetched { entries.push(failure( @@ -799,7 +889,7 @@ where // Every target either failed an inclusion/membership check or was a // `--block` target already taken from the body. Nothing left to execute. if active.is_empty() { - return BlockReplayOutcome::entries_only(entries); + return BlockReplayOutcome::ordered(entries, &job_targets, Some(&block_tx_order), None); } let targets = active; @@ -818,12 +908,12 @@ where let parent_block = match fetch_block(provider, number - 1).await { Ok(block) => block, Err(e) => { - return BlockReplayOutcome::entries_only(fail_remaining( - &targets, - entries, - BatchErrorKind::Rpc, - &e.to_string(), - )); + return BlockReplayOutcome::ordered( + fail_remaining(&targets, entries, BatchErrorKind::Rpc, &e.to_string()), + &job_targets, + Some(&block_tx_order), + None, + ); } }; let parent_hash = parent_block.hash(); @@ -834,12 +924,12 @@ where block describes a different chain than the block being replayed (reorg in progress, \ or a load-balanced endpoint serving divergent views); retry once the chain settles" ); - return BlockReplayOutcome::entries_only(fail_remaining( - &targets, - entries, - BatchErrorKind::Rpc, - &message, - )); + return BlockReplayOutcome::ordered( + fail_remaining(&targets, entries, BatchErrorKind::Rpc, &message), + &job_targets, + Some(&block_tx_order), + None, + ); } // Fetch the on-chain receipts before the block runs. Needed for @@ -873,23 +963,23 @@ where let cfg_env = match chain_args.create_cfg_env() { Ok(cfg) => cfg, Err(e) => { - return BlockReplayOutcome::entries_only(fail_remaining( - &targets, - entries, - BatchErrorKind::Execution, - &e.to_string(), - )); + return BlockReplayOutcome::ordered( + fail_remaining(&targets, entries, BatchErrorKind::Execution, &e.to_string()), + &job_targets, + Some(&block_tx_order), + None, + ); } }; let block_env = match retrieve_block_env(&block) { Ok(env) => env, Err(e) => { - return BlockReplayOutcome::entries_only(fail_remaining( - &targets, - entries, - BatchErrorKind::Execution, - &e.to_string(), - )); + return BlockReplayOutcome::ordered( + fail_remaining(&targets, entries, BatchErrorKind::Execution, &e.to_string()), + &job_targets, + Some(&block_tx_order), + None, + ); } }; let executed_spec = cfg_env.spec; @@ -897,12 +987,12 @@ where let Some(hardfork) = hardforks.hardfork(timestamp) else { let message = format!("No `MegaHardfork` active at block timestamp: {timestamp}"); - return BlockReplayOutcome::entries_only(fail_remaining( - &targets, - entries, - BatchErrorKind::Execution, - &message, - )); + return BlockReplayOutcome::ordered( + fail_remaining(&targets, entries, BatchErrorKind::Execution, &message), + &job_targets, + Some(&block_tx_order), + None, + ); }; let block_limits = BlockLimits::from_hardfork_and_block_gas_limit(hardfork, block.header.gas_limit()); @@ -924,12 +1014,12 @@ where { Ok(database) => database, Err(e) => { - return BlockReplayOutcome::entries_only(fail_remaining( - &targets, - entries, - BatchErrorKind::Rpc, - &e.to_string(), - )); + return BlockReplayOutcome::ordered( + fail_remaining(&targets, entries, BatchErrorKind::Rpc, &e.to_string()), + &job_targets, + Some(&block_tx_order), + None, + ); } }; @@ -941,16 +1031,18 @@ where if let Err(e) = block_executor.apply_pre_execution_changes() { let error = ReplayError::BlockExecutionError(e); - return BlockReplayOutcome::entries_only(fail_remaining( - &targets, - entries, - classify(&error), - &error.to_string(), - )); + return BlockReplayOutcome::ordered( + fail_remaining(&targets, entries, classify(&error), &error.to_string()), + &job_targets, + Some(&block_tx_order), + None, + ); } let target_set: HashSet = targets.iter().copied().collect(); - let tx_hashes: Vec = block.transactions.hashes().collect(); + // Prefer the already-collected body order so stream ordering and the loop + // walk the same sequence. + let tx_hashes = block_tx_order.clone(); // Highest block index among this job's targets: once that transaction has // committed we can stop — later non-targets are not needed for receipts or // fixtures, and requiring them would force incomplete offline captures to @@ -1270,7 +1362,7 @@ where } } - BlockReplayOutcome { entries, uncounted_abort } + BlockReplayOutcome::ordered(entries, &job_targets, Some(&block_tx_order), uncounted_abort) } /// Inputs for [`prepare_target_fixture`], grouped so the dump path stays a single @@ -1727,7 +1819,16 @@ mod tests { let ReplayError::BatchFailed(counts) = err else { panic!("infrastructure failures must aggregate: {err:?}"); }; - assert_eq!(counts, BatchFailureCounts { execution: 2, rpc: 1, mismatched: 1, total: 5 }); + assert_eq!( + counts, + BatchFailureCounts { + execution: 2, + rpc: 1, + mismatched: 1, + total: 5, + ..Default::default() + } + ); assert!( counts.to_string().contains("3 of 5 target transaction(s) failed"), "unexpected message: {counts}" @@ -1751,7 +1852,16 @@ mod tests { let ReplayError::BatchFailed(counts) = err else { panic!("a failed fixture must fail the run: {err:?}"); }; - assert_eq!(counts, BatchFailureCounts { execution: 1, rpc: 0, mismatched: 1, total: 1 }); + assert_eq!( + counts, + BatchFailureCounts { + execution: 1, + rpc: 0, + mismatched: 1, + total: 1, + ..Default::default() + } + ); assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::ExecutionError); } @@ -1828,8 +1938,8 @@ mod tests { assert!(message.contains("cache miss"), "unexpected message: {message}"); } - /// An uncounted non-target abort contributes its class to the exit without - /// a synthetic reported entry. + /// An uncounted non-target abort floors the exit class without a synthetic + /// reported entry and without inflating per-target failure totals. #[test] fn test_batch_tally_uncounted_abort_drives_exit_class() { let mut tally = BatchTally::default(); @@ -1839,15 +1949,85 @@ mod tests { tally.record_uncounted_abort(BatchErrorKind::Execution); assert_eq!(tally.reported, 2, "uncounted abort is not a reported target"); - assert_eq!(tally.counts.rpc, 2); - assert_eq!(tally.counts.execution, 1); + assert_eq!(tally.counts.rpc, 2, "target counters stay per-target"); + assert_eq!(tally.counts.execution, 0, "abort must not inflate execution count"); + assert_eq!(tally.exit_floor, BatchExitFloor::Execution); let err = tally.into_error().expect("run failed"); let ReplayError::BatchFailed(counts) = err else { panic!("expected batch failure: {err:?}"); }; + assert_eq!( + counts.to_string(), + "2 of 2 target transaction(s) failed (0 execution, 2 rpc)", + "aggregate message must stay truthful about targets" + ); assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::ExecutionError); } + /// One unanswered receipt with both `--verify-receipt` and + /// `--dump-fixture-dir` counts as a single rpc failure, while both result + /// fields remain present on the executed entry. + #[test] + fn test_batch_tally_shared_receipt_failure_counted_once() { + let mut tally = BatchTally::default(); + tally.record_executed( + Some(&VerificationOutcome::unavailable("receipt pruned")), + Some(&FixtureReport::rpc_error("no on-chain receipt was fetched for this transaction")), + ); + + assert_eq!(tally.reported, 1); + assert_eq!(tally.replayed, 1); + assert_eq!(tally.verified, 0); + assert_eq!(tally.counts.rpc, 1, "shared receipt failure is one rpc count"); + assert_eq!(tally.counts.execution, 0); + let err = tally.into_error().expect("run failed"); + let ReplayError::BatchFailed(counts) = err else { + panic!("expected batch failure: {err:?}"); + }; + assert_eq!(counts.to_string(), "1 of 1 target transaction(s) failed (0 execution, 1 rpc)"); + assert_eq!(ExitCode::from_batch_failures(&counts), ExitCode::RpcFailure); + } + + /// Independent findings on the same target still both count: a receipt + /// mismatch plus a fixture write failure is not a shared root cause. + #[test] + fn test_batch_tally_mismatch_and_fixture_error_are_independent() { + let mut tally = BatchTally::default(); + tally.record_executed(Some(&verdict(false)), Some(&FixtureReport::error("disk full"))); + + assert_eq!(tally.counts.mismatched, 1); + assert_eq!(tally.counts.execution, 1); + assert_eq!(tally.counts.rpc, 0); + } + + /// Documented stream order: body-index first, then absent targets last in + /// job input order — independent of the order entries were collected. + #[test] + fn test_order_block_entries_body_index_before_absent_last() { + let early = B256::repeat_byte(0x11); + let mid = B256::repeat_byte(0x22); + let late_absent = B256::repeat_byte(0x33); + let job_targets = vec![ + JobTarget { hash: late_absent, inclusion_hash: Some(B256::ZERO) }, + JobTarget { hash: early, inclusion_hash: Some(B256::ZERO) }, + JobTarget { hash: mid, inclusion_hash: Some(B256::ZERO) }, + ]; + // Collected in the buggy pre-execution-first order: absent then results. + let entries = vec![ + failure(late_absent, BatchErrorKind::Rpc, "inclusion".into()), + failure(mid, BatchErrorKind::Rpc, "swept".into()), + failure(early, BatchErrorKind::Rpc, "swept".into()), + ]; + let body = [early, mid, B256::repeat_byte(0x99)]; + let ordered = order_block_entries(entries, &job_targets, Some(&body)); + let hashes: Vec = ordered.iter().map(BatchEntry::tx_hash).collect(); + assert_eq!( + hashes, + vec![early, mid, late_absent], + "body order first, absent last: {hashes:?}" + ); + } + /// A fixture discarded after a transport abort counts as rpc, not execution, /// so the run exit matches the abort class. #[test] diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index f8be9176..ea22fceb 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -199,11 +199,24 @@ fn replay_envelope_with_code( envelope_path: &std::path::Path, args: &[&str], ) -> (String, Option) { + let (stdout, _stderr, code) = replay_envelope_full(envelope_path, args); + (stdout, code) +} + +/// Run `replay` against `envelope_path` and return stdout, stderr, and exit code. +fn replay_envelope_full( + envelope_path: &std::path::Path, + args: &[&str], +) -> (String, String, Option) { let mut cmd = mega_evme(); cmd.args(["replay", "--rpc.replay-file", envelope_path.to_str().expect("path is utf-8")]); cmd.args(args); let output = cmd.output().expect("failed to run mega-evme"); - (String::from_utf8(output.stdout).expect("stdout is utf-8"), output.status.code()) + ( + String::from_utf8(output.stdout).expect("stdout is utf-8"), + String::from_utf8(output.stderr).expect("stderr is utf-8"), + output.status.code(), + ) } /// Parse NDJSON stdout into one JSON value per line, dropping the structured @@ -579,8 +592,11 @@ fn test_replay_block_sweeps_targets_behind_execution_abort_as_rpc() { } /// A non-target deterministic executor abort still exits 1: swept targets stay -/// `rpc` ("unanswered"), but the run tallies the abort's own class so a -/// retryable exit is not reported for a permanent failure. +/// `rpc` ("unanswered"), but the run floors the exit on the abort's own class so +/// a retryable exit is not reported for a permanent failure. +/// +/// Per-target totals stay truthful ("2 of 2"): the abort is not a synthetic +/// third target failure. /// /// `EXEC_ABORT_TX` is doctored and kept out of the `--tx-file` target list; only /// later targets of the same block are requested. @@ -596,8 +612,8 @@ fn test_replay_tx_file_non_target_execution_abort_exits_execution() { .join(format!("mega_evme_tx_list_non_target_exec_{}.txt", std::process::id())); std::fs::write(&list_path, list).expect("write tx list"); - let (stdout, code) = - replay_envelope_with_code(&path, &["--tx-file", list_path.to_str().unwrap(), "--json"]); + let (stdout, stderr, code) = + replay_envelope_full(&path, &["--tx-file", list_path.to_str().unwrap(), "--json"]); let _ = std::fs::remove_file(&path); let _ = std::fs::remove_file(&list_path); let lines = ndjson(&stdout); @@ -618,7 +634,19 @@ fn test_replay_tx_file_non_target_execution_abort_exits_execution() { } assert_eq!(code, Some(1), "a non-target execution abort exits 1, not 3"); - assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("execution-error")); + let err = run_error(&stdout); + assert_eq!(err["error"]["kind"].as_str(), Some("execution-error")); + let message = err["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("2 of 2 target transaction(s) failed"), + "aggregate must stay truthful about targets (not 3 of 2): {message}" + ); + assert!( + stderr.contains("2 of 2 target transaction(s) failed") || + message.contains("2 of 2 target transaction(s) failed"), + "stderr/stdout aggregate must not count the non-target abort as a target: \ + stderr={stderr}\nmessage={message}" + ); } /// A non-target transport abort (cache miss) exits 3 and names the failing @@ -857,6 +885,80 @@ fn test_replay_tx_file_rejects_a_block_that_does_not_match_the_resolved_inclusio let _ = std::fs::remove_file(&list); } +/// Same-block mix of a successful early target and a later inclusion failure: +/// NDJSON line order follows ascending `(block, tx_index)`, so the earlier +/// result precedes the later inclusion failure even when the failure was +/// decided before the execute loop. +/// +/// Looks up by hash alone cannot catch this; the test asserts line positions. +#[test] +fn test_replay_tx_file_same_block_mixed_inclusion_preserves_line_order() { + let (early_target, early_index) = BLOCK_TXS[1]; + let (late_target, late_index) = BLOCK_TXS[2]; + assert!(early_index < late_index); + let wrong_hash = "0x2222222222222222222222222222222222222222222222222222222222222222"; + + // Doctor only the later target's inclusion hash so it fails the membership + // guard while the earlier target still replays. + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let marker = format!("\"hash\":\"{late_target}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object"); + result["blockHash"] = serde_json::Value::String(wrong_hash.into()); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "exactly one response describes the late target"); + + let envelope_path = std::env::temp_dir() + .join(format!("mega_evme_batch_mixed_order_{}.json", std::process::id())); + std::fs::write(&envelope_path, envelope.to_string()).expect("write doctored envelope"); + + // List the later (failing) target first so a pre-execution-first emit would + // put its failure line before the earlier result. + let list = std::env::temp_dir() + .join(format!("mega_evme_tx_list_mixed_order_{}.txt", std::process::id())); + std::fs::write(&list, format!("{late_target}\n{early_target}\n")).expect("write tx list"); + + let (stdout, code) = + replay_envelope_with_code(&envelope_path, &["--tx-file", list.to_str().unwrap(), "--json"]); + let _ = std::fs::remove_file(&envelope_path); + let _ = std::fs::remove_file(&list); + let lines = ndjson(&stdout); + assert_eq!(lines.len(), 2, "every target is reported once: {stdout}"); + + // End-to-end line order: earlier body index first, absent/divergent last. + assert_eq!( + lines[0]["tx_hash"].as_str(), + Some(early_target), + "line 0 must be the earlier target's result, got: {}", + lines[0] + ); + assert!(lines[0].get("error").is_none(), "earlier target still replays: {}", lines[0]); + assert_eq!(lines[0]["tx_index"].as_u64(), Some(early_index)); + assert_eq!(lines[0]["success"].as_bool(), Some(true)); + + assert_eq!( + lines[1]["tx_hash"].as_str(), + Some(late_target), + "line 1 must be the later target's inclusion failure, got: {}", + lines[1] + ); + assert_eq!(lines[1]["error"]["kind"].as_str(), Some("rpc"), "inclusion failure: {}", lines[1]); + + assert_eq!(code, Some(3), "an unanswered target exits 3"); +} + /// Two same-height targets that report different inclusion hashes get /// independent outcomes: the one that matches the fetched block replays, the /// one that does not fails as `rpc`. Outcomes must not depend on file order. diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index f3aa1788..ead16d87 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -604,6 +604,60 @@ fn test_batch_dump_fixture_dir_missing_receipt_is_rpc_fixture_error() { assert_eq!(run.error_object()["error"]["kind"].as_str(), Some("rpc-failure")); } +/// Combined `--verify-receipt --dump-fixture-dir` with a missing receipt: both +/// result fields report the failure, the shared receipt failure is counted once +/// ("1 of 1"), and the run exits 3. +#[test] +fn test_batch_verify_and_dump_missing_receipt_counted_once() { + let path = cache_without_receipt("batch_verify_dump_no_receipt"); + let list = tx_file("batch_verify_dump_no_receipt"); + let dir = std::env::temp_dir() + .join(format!("mega_evme_batch_verify_dump_no_receipt_{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + let run = replay( + &path, + &[ + "--tx-file", + list.to_str().unwrap(), + "--verify-receipt", + "--dump-fixture-dir", + dir.to_str().unwrap(), + "--json", + ], + ); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_dir_all(&dir); + + assert_eq!(run.code(), 3, "shared missing receipt exits 3.\nstderr: {}", run.stderr); + let lines = run.ndjson(); + assert_eq!(lines.len(), 1, "one target: {}", run.stdout); + assert!(lines[0].get("error").is_none(), "result line is kept: {}", lines[0]); + assert!(lines[0]["receipt"].is_object(), "local receipt is kept: {}", lines[0]); + assert!( + lines[0]["verification"]["error"].is_string(), + "verification.error carries the unanswered receipt: {}", + lines[0] + ); + assert!( + lines[0]["fixture"]["error"].is_string(), + "fixture.error also carries the unanswered receipt: {}", + lines[0] + ); + let err = run.error_object(); + let message = err["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("1 of 1 target transaction(s) failed"), + "shared receipt failure must not double-count: {message}" + ); + assert!( + message.contains("1 rpc") || message.contains("(0 execution, 1 rpc)"), + "exactly one rpc failure in the aggregate: {message}" + ); + assert_eq!(err["error"]["kind"].as_str(), Some("rpc-failure")); +} + /// A genuine fidelity-gate skip (local gas disagrees with on-chain receipt) /// still exits 0: the receipt question was answered, the dump was correctly /// refused, and skips never fail the run. diff --git a/docs/mega-evme/commands/replay.md b/docs/mega-evme/commands/replay.md index a40aec31..84bb9401 100644 --- a/docs/mega-evme/commands/replay.md +++ b/docs/mega-evme/commands/replay.md @@ -128,8 +128,8 @@ Execution outcomes are not errors: a reverted or halted transaction is a normal A failure while running the block aborts it, because the executor state no longer matches the chain. The transaction the failure is about — the hash the endpoint denied, or the one the executor rejected — is reported with that failure's own kind. Every target behind it is reported as `rpc` with a message naming the aborting cause: nothing was established about those transactions, so they went unanswered rather than being unknown. -Targets that never ran are still emitted in the block's transaction-index order, keeping the whole stream in ascending `(block, tx_index)` order; a hash the block does not contain is reported last within its block, in input order, as `not_found`. -Hashes that could not be resolved to a block at all (unknown, pending, or an endpoint failure during resolution) are emitted before every block result, since the run cannot place them in the stream's order. +Targets that never ran are still emitted in the block's transaction-index order, keeping the whole stream in ascending `(block, tx_index)` order; a hash the endpoint claimed for this block but that the body does not list is reported last within its block, in input order, as `rpc` (an unanswered, divergent view — not a definitive unknown hash). +Hashes that could not be resolved to a block at all (unknown as `not_found`, pending, or an endpoint failure during resolution) are emitted before every block result, since the run cannot place them in the stream's order. Without `--json`, each transaction is printed with a header naming its hash, block, and index, followed by the same summary and receipt the single-transaction mode prints. A final one-line summary (transactions replayed, transactions failed, elapsed time) is logged at `INFO` level, so pass `-vvv` to see it. @@ -291,9 +291,10 @@ mega-evme replay --rpc https://mainnet.megaeth.com/rpc \ --tx-file ./corpus.txt --verify-receipt --json > results.ndjson jq -c 'select(.tx_hash and .verification.match == false)' results.ndjson # mismatched -jq -c 'select(.tx_hash and .error != null)' results.ndjson # unverified +jq -c 'select(.tx_hash and (.error != null or .verification.error != null))' results.ndjson # unverified ``` +Receipt-fetch failures on a target that still replayed live under `.verification.error` on the result line (the line keeps `receipt` / `success`); infrastructure failures that prevented execution live under `.error`. Both selectors require `.tx_hash` so that the run-level `{"error": …}` object a failed run appends to stdout is not counted as an unverified transaction. Capture once online, then re-verify the same corpus offline: From 1c4bc11f494ce23316bb13eb43d994c5e5af88d1 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 12:07:38 +0800 Subject: [PATCH 57/64] test(mega-evme): pin the batch wrong-transaction receipt to the keep-result shape --- bin/mega-evme/tests/replay_verify.rs | 35 +++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/bin/mega-evme/tests/replay_verify.rs b/bin/mega-evme/tests/replay_verify.rs index 276d6563..9c912e09 100644 --- a/bin/mega-evme/tests/replay_verify.rs +++ b/bin/mega-evme/tests/replay_verify.rs @@ -476,10 +476,11 @@ fn test_batch_verify_receipt_reorg_keeps_result_and_is_rpc() { ); } -/// The identity guard applies in batch mode too, as an `rpc` error entry naming -/// both hashes, and the target carries no verdict. +/// The identity guard applies in batch mode too: the target replayed, so it +/// keeps its result line, and the failure to answer its receipt question is +/// reported on `verification.error` naming both hashes — tallied rpc once. #[test] -fn test_batch_verify_receipt_for_another_transaction_is_an_rpc_error_entry() { +fn test_batch_verify_receipt_for_another_transaction_keeps_result_and_is_rpc() { let path = doctored_cache("batch_wrong_tx", |receipt| { receipt["transactionHash"] = OTHER_TX.into(); }); @@ -492,15 +493,37 @@ fn test_batch_verify_receipt_for_another_transaction_is_an_rpc_error_entry() { assert_eq!(run.code(), 3, "an unverified target exits 3.\nstderr: {}", run.stderr); let lines = run.ndjson(); assert_eq!(lines.len(), 1, "one line per requested transaction"); - assert_eq!(lines[0]["error"]["kind"].as_str(), Some("rpc")); assert!( - lines[0]["error"]["message"] + lines[0].get("error").is_none(), + "a replayed target keeps its result line, not a bare error entry: {}", + lines[0] + ); + assert!(lines[0]["receipt"].is_object(), "local receipt is kept: {}", lines[0]); + assert_eq!(lines[0]["success"].as_bool(), Some(true)); + assert_eq!(lines[0]["gas_used"].as_u64(), Some(GAS_USED)); + assert!( + lines[0]["verification"]["error"] .as_str() .is_some_and(|message| message.contains(OTHER_TX) && message.contains(TX)), "expected both the served and the requested transaction: {}", lines[0] ); - assert!(lines[0].get("verification").is_none(), "an unverified target carries no verdict"); + assert!( + lines[0]["verification"].get("match").is_none(), + "a receipt served for another transaction is not a match/mismatch verdict: {}", + lines[0] + ); + let err = run.error_object(); + assert_eq!(err["error"]["kind"].as_str(), Some("rpc-failure")); + let message = err["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("1 of 1 target transaction(s) failed"), + "the one unverified target is the one failure: {message}" + ); + assert!( + message.contains("(0 execution, 1 rpc)"), + "an unanswered receipt question is tallied rpc, not execution: {message}" + ); assert!( !run.stderr.contains("verification mismatch"), "an unverifiable target must not fail as a mismatch:\n{}", From e799f80caf03937a68c605db8e27308ae150a818 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 12:17:27 +0800 Subject: [PATCH 58/64] fix(mega-evme): classify stringified pre-block RPC failures as exit 3 Pre-block EIP-2935/EIP-4788 system-call database failures are rendered into BlockValidationError message fields by mega-evm, so the typed EvmeError chain is gone before exit classification. Recover the RPC class from stable Display prefixes this crate owns (RPC error: / RPC transport error:), so offline cache misses and transport failures during pre-block no longer look like permanent execution errors to retry scripts. --- bin/mega-evme/src/common/error.rs | 18 +++ bin/mega-evme/src/common/exit.rs | 190 ++++++++++++++++++++++++++++-- bin/mega-evme/tests/exit_codes.rs | 87 ++++++++++++++ 3 files changed, 282 insertions(+), 13 deletions(-) diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index 26e34b20..0be04a33 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -5,6 +5,24 @@ use mega_evm::{ revm::{bytecode::BytecodeDecodeError, database_interface::bal::EvmDatabaseError}, }; +/// Stable `Display` prefix of [`EvmeError::RpcTransportError`]. +/// +/// Pre-block system calls (EIP-2935 / EIP-4788) and similar mega-evm wrappers +/// render a database failure into a message string with `to_string()`, so the +/// typed variant is gone by the time exit classification runs. The classifier +/// recovers the RPC class by looking for this exact prefix inside that message. +/// The `#[error(...)]` text on the variant must keep the same prefix; the +/// round-trip unit test in `exit` enforces that. +pub const RPC_TRANSPORT_ERROR_PREFIX: &str = "RPC transport error: "; + +/// Stable `Display` prefix of [`EvmeError::RpcError`]. +/// +/// Same recovery contract as [`RPC_TRANSPORT_ERROR_PREFIX`]: fork-state reads +/// map transport/cache failures into this variant, and stringified block errors +/// still carry this prefix so exit classification can treat them as unanswered +/// questions rather than execution failures. +pub const RPC_ERROR_PREFIX: &str = "RPC error: "; + /// Error types for the replay command #[derive(Debug, thiserror::Error)] pub enum EvmeError { diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs index 854cb9d4..c58be4e8 100644 --- a/bin/mega-evme/src/common/exit.rs +++ b/bin/mega-evme/src/common/exit.rs @@ -27,7 +27,7 @@ //! assigned a class. use mega_evm::{ - alloy_evm::block::BlockExecutionError, + alloy_evm::block::{BlockExecutionError, BlockValidationError}, alloy_op_evm::OpTxError, revm::{context::result::EVMError, database_interface::bal::EvmDatabaseError}, }; @@ -36,7 +36,7 @@ use tracing::error; use crate::{ cmd::Error, - common::{BatchFailureCounts, EvmeError}, + common::{BatchFailureCounts, EvmeError, RPC_ERROR_PREFIX, RPC_TRANSPORT_ERROR_PREFIX}, }; /// The concrete EVM error a `mega-evme` block executor produces. @@ -58,14 +58,23 @@ type BlockEvmError = EVMError, OpTxError>; /// step is typed: the rendered message is never inspected. /// /// Not every block error carries its cause this way. A read that fails inside -/// the pre-block system calls (EIP-4788 beacon root, EIP-2935 block hashes) or -/// inside the sandboxed execution used by the keyless-deploy system contract -/// has its cause rendered into a message string by the layer that raised it, so -/// the type is gone before the error arrives here and the failure stays -/// execution-class. +/// the pre-block system calls (EIP-4788 beacon root, EIP-2935 block hashes) +/// is stringified into a [`BlockValidationError`] message field by mega-evm +/// before the error reaches here. For that path, see +/// [`stringified_rpc_failure`]. The keyless-deploy sandbox erases the cause +/// entirely (selector-only `InternalError`); that path cannot be recovered +/// bin-side. fn database_cause(err: &BlockExecutionError) -> Option<&EvmeError> { - let internal = err.as_internal()?; - let boxed = internal.as_evm().map(|(_, error)| error).or_else(|| internal.as_other())?; + let boxed = match err { + BlockExecutionError::Internal(internal) => { + internal.as_evm().map(|(_, error)| error).or_else(|| internal.as_other())? + } + // Validation::EVM / Other box a non-tx failure the same way Internal does. + BlockExecutionError::Validation( + BlockValidationError::EVM { error, .. } | BlockValidationError::Other(error), + ) => error.as_ref(), + BlockExecutionError::Validation(_) => return None, + }; if let Some(evm_error) = boxed.downcast_ref::() { return match evm_error { @@ -88,6 +97,41 @@ const fn external_cause(err: &EvmDatabaseError) -> Option<&EvmeError> } } +/// Whether a rendered message still carries an [`EvmeError`] RPC-class prefix. +/// +/// Used only after typed recovery fails. The prefixes are the stable +/// [`Display`] texts of [`EvmeError::RpcError`] / [`EvmeError::RpcTransportError`], +/// which this crate owns; mega-evm and alloy-evm wrappers that call +/// `to_string()` leave them embedded in the outer message. +fn message_carries_rpc_class(message: &str) -> bool { + message.contains(RPC_ERROR_PREFIX) || message.contains(RPC_TRANSPORT_ERROR_PREFIX) +} + +/// Recover an RPC-class failure that mega-evm stringified into a validation +/// message, losing the typed [`EvmeError`] chain. +/// +/// Pre-block EIP-2935 / EIP-4788 helpers map a system-call database error to +/// [`BlockValidationError::BlockHashContractCall`] / +/// [`BlockValidationError::BeaconRootContractCall`] with `message: e.to_string()`. +/// The type is gone, but the message still contains this crate's RPC +/// [`Display`] prefixes. Other validation variants either keep a typed box +/// (handled by [`database_cause`]) or are genuine consensus/execution failures. +fn stringified_rpc_failure(err: &BlockExecutionError) -> bool { + let BlockExecutionError::Validation(validation) = err else { + return false; + }; + let message = match validation { + BlockValidationError::BlockHashContractCall { message } | + BlockValidationError::BeaconRootContractCall { message, .. } | + BlockValidationError::WithdrawalRequestsContractCall { message } | + BlockValidationError::ConsolidationRequestsContractCall { message } => message.as_str(), + // Typed boxes are recovered by `database_cause`; consensus-only variants + // never embed an RPC `Display` prefix. + _ => return false, + }; + message_carries_rpc_class(message) +} + /// Process exit status of a `mega-evme` run. /// /// The discriminants are the wire contract with calling scripts; see the module @@ -152,10 +196,18 @@ impl ExitCode { EvmeError::BlockBodyTransactionFetch { .. } => Self::RpcFailure, // A block error the EVM raised because a state read failed is that // read's failure, not an execution result: classify it by its - // cause, so an endpoint that died mid-execution still reports the - // question as unanswered. - EvmeError::BlockExecutionError(err) => database_cause(err) - .map_or(Self::ExecutionError, Self::from_evme_error), + // cause when the type survives, or by the stable RPC Display + // prefixes when mega-evm stringified the failure into a validation + // message (pre-block EIP-2935 / EIP-4788 system calls). + EvmeError::BlockExecutionError(err) => { + if let Some(cause) = database_cause(err) { + Self::from_evme_error(cause) + } else if stringified_rpc_failure(err) { + Self::RpcFailure + } else { + Self::ExecutionError + } + } // Answered, definitively negative. EvmeError::TransactionNotFound(_) | EvmeError::BlockNotFound(_) | @@ -281,6 +333,7 @@ pub fn print_json_error(code: ExitCode, message: &str) { mod tests { use super::*; use alloy_primitives::B256; + use mega_evm::alloy_evm::block::BlockValidationError; /// Every failure class the taxonomy defines keeps its documented code. #[test] @@ -291,6 +344,36 @@ mod tests { assert_eq!(ExitCode::RpcFailure.code(), 3); } + /// The classifier recovers stringified RPC failures by the same prefixes + /// [`EvmeError`]'s `Display` emits — a drift between the two would silently + /// reclassify pre-block cache misses as execution errors. + #[test] + fn test_rpc_display_prefixes_round_trip_with_classifier() { + let rpc = EvmeError::RpcError("cache miss in offline replay file".to_string()); + let rpc_text = rpc.to_string(); + assert!( + rpc_text.starts_with(RPC_ERROR_PREFIX), + "RpcError Display must start with RPC_ERROR_PREFIX, got: {rpc_text}" + ); + assert!(message_carries_rpc_class(&rpc_text)); + + let transport = EvmeError::RpcTransportError( + alloy_provider::transport::TransportErrorKind::custom_str("connection refused"), + ); + let transport_text = transport.to_string(); + assert!( + transport_text.starts_with(RPC_TRANSPORT_ERROR_PREFIX), + "RpcTransportError Display must start with RPC_TRANSPORT_ERROR_PREFIX, got: \ + {transport_text}" + ); + assert!(message_carries_rpc_class(&transport_text)); + + // Nested the way mega-evm stringifies a system-call DB failure. + let nested = format!("Database error: {rpc_text}"); + assert!(message_carries_rpc_class(&nested)); + assert!(!message_carries_rpc_class("failed to apply blockhash contract call: halt")); + } + /// The `kind` namespace is kebab-case and one name per class. #[test] fn test_exit_code_kinds_are_kebab_case() { @@ -381,6 +464,87 @@ mod tests { ); } + /// Pre-block EIP-2935 stringifies the system-call error into + /// `BlockHashContractCall { message }`. The typed cause is gone, but the + /// message still embeds this crate's RPC Display prefix — classify as + /// unanswered, not as an execution rejection. + #[test] + fn test_stringified_blockhash_contract_call_rpc_failure_maps_to_three() { + let message = format!( + "Database error: {}", + EvmeError::RpcError("Failed to fetch storage for history slot: cache miss".into()) + ); + let err = EvmeError::BlockExecutionError(BlockExecutionError::Validation( + BlockValidationError::BlockHashContractCall { message }, + )); + + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::RpcFailure, + "unexpected class: {err}" + ); + } + + /// Pre-block EIP-4788 uses the same stringification path with a different + /// validation variant; the classifier must treat it the same way. + #[test] + fn test_stringified_beacon_root_contract_call_rpc_failure_maps_to_three() { + let message = format!( + "Database error: {}", + EvmeError::RpcError("Failed to fetch storage for beacon root: cache miss".into()) + ); + let err = EvmeError::BlockExecutionError(BlockExecutionError::Validation( + BlockValidationError::BeaconRootContractCall { + parent_beacon_block_root: Box::new(B256::ZERO), + message, + }, + )); + + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::RpcFailure, + "unexpected class: {err}" + ); + } + + /// A pre-block system-call failure that is not an unanswered RPC question + /// (for example a genuine EVM halt during the call) stays execution-class. + #[test] + fn test_stringified_blockhash_contract_call_without_rpc_prefix_maps_to_one() { + let err = EvmeError::BlockExecutionError(BlockExecutionError::Validation( + BlockValidationError::BlockHashContractCall { message: "OutOfGas".to_string() }, + )); + + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::ExecutionError, + "unexpected class: {err}" + ); + } + + /// Keyless-deploy sandbox DB failures are erased to a selector-only + /// `InternalError` inside mega-evm before any message reaches bin-side + /// classification. A synthetic stringified sandbox path that still carried + /// the RPC Display prefix (as `SandboxDbError` does before that final + /// erasure) is classified as RPC when it appears in a message-bearing + /// wrapper — pinning the prefix rule used for pre-block recovery. + #[test] + fn test_stringified_sandbox_style_rpc_prefix_maps_to_three() { + // SandboxDbError(e.to_string()) where e is EvmeError::RpcError(...). + let sandbox_db_message = + EvmeError::RpcError("Failed to fetch account during sandbox read".into()).to_string(); + assert!( + sandbox_db_message.starts_with(RPC_ERROR_PREFIX), + "sandbox stringification preserves the RPC prefix: {sandbox_db_message}" + ); + // If that message were embedded in a validation wrapper the same way + // pre-block helpers do, classification would recover it. + let err = EvmeError::BlockExecutionError(BlockExecutionError::Validation( + BlockValidationError::BlockHashContractCall { message: sandbox_db_message }, + )); + assert_eq!(ExitCode::from_evme_error(&err), ExitCode::RpcFailure); + } + /// A completed run whose replay diverged from the chain exits 2. #[test] fn test_verification_mismatch_maps_to_two() { diff --git a/bin/mega-evme/tests/exit_codes.rs b/bin/mega-evme/tests/exit_codes.rs index 2aa535a3..4dfecee9 100644 --- a/bin/mega-evme/tests/exit_codes.rs +++ b/bin/mega-evme/tests/exit_codes.rs @@ -37,6 +37,23 @@ const UNANSWERABLE_TX: &str = "0x00000000000000000000000000000000000000000000000 const IN_EXECUTION_STATE_READ: &str = "0x0d9aee1b171e0c4a2be0107def891d838cc94d71e4046cb95b00a1c2a61cffed"; +/// Request fingerprint of the EIP-2935 history-storage slot the pre-block +/// system call writes when replaying `TX_OK`'s block. +/// +/// Dropping this entry makes `apply_pre_execution_changes` fail inside the +/// blockhash contract call. mega-evm stringifies that database failure into +/// `BlockHashContractCall { message }`, so classification must recover the RPC +/// class from the stable `RPC error:` Display prefix rather than from a typed +/// cause chain. +const PRE_BLOCK_HISTORY_STORAGE_READ: &str = + "0x3abed4482ce079cf23c80a8e43bd75f8ac32b8b108e925b39f0a5c93de4aff48"; + +/// Request fingerprint of an EIP-4788 beacon-roots storage slot the pre-block +/// system call touches for the same block — a second stringified validation +/// path (`BeaconRootContractCall`) with the same recovery rule. +const PRE_BLOCK_BEACON_ROOT_STORAGE_READ: &str = + "0x49e3c5174c528b49897a0556c762d4fb88e1ad5e6aa8f8795ddbc37aa6c278f0"; + /// Outcome of one `mega-evme` invocation. struct Run { code: Option, @@ -332,6 +349,76 @@ fn test_state_read_failure_during_execution_is_an_rpc_failure() { ); } +/// A cache miss during the pre-block EIP-2935 blockhash system call is an +/// unanswered RPC question even though mega-evm stringifies it into +/// `BlockHashContractCall { message }` before the exit classifier sees it. +/// +/// Without prefix recovery this lands as exit 1 / `execution-error`; with it, +/// single-run and batch both report the RPC class (exit 3). +#[test] +fn test_pre_block_blockhash_system_call_cache_miss_is_an_rpc_failure() { + let path = cache_without_entry("pre_block_2935", PRE_BLOCK_HISTORY_STORAGE_READ); + let cache = path.to_str().unwrap(); + + let single = run(&["replay", "--rpc.replay-file", cache, "--json", TX_OK]); + assert_eq!( + single.code(), + 3, + "a pre-block history-storage miss exits 3 (was exit 1 before stringified-RPC recovery).\n\ + stderr: {}", + single.stderr + ); + let error = single.error_object(); + assert_eq!(error["error"]["code"].as_u64(), Some(3)); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("blockhash contract call") && + message.contains("RPC error:") && + message.contains("cache miss"), + "the message must name the stringified pre-block path and the miss: {error}" + ); + + let list = tx_file("pre_block_2935", &format!("{TX_OK}\n")); + let batch = run(&["replay", "--rpc.replay-file", cache, "--tx-file", list.to_str().unwrap()]); + let _ = std::fs::remove_file(&list); + let _ = std::fs::remove_file(&path); + + assert_eq!(batch.code(), 3, "the batch run exits 3 too.\nstderr: {}", batch.stderr); + assert!( + batch.stdout.contains("Error (rpc):"), + "the target is reported as unanswered:\n{}", + batch.stdout + ); +} + +/// Same recovery for the EIP-4788 beacon-root pre-block system call, which uses +/// `BeaconRootContractCall { message }` rather than the blockhash variant. +#[test] +fn test_pre_block_beacon_root_system_call_cache_miss_is_an_rpc_failure() { + let path = cache_without_entry("pre_block_4788", PRE_BLOCK_BEACON_ROOT_STORAGE_READ); + let cache = path.to_str().unwrap(); + + let single = run(&["replay", "--rpc.replay-file", cache, "--json", TX_OK]); + let _ = std::fs::remove_file(&path); + + assert_eq!( + single.code(), + 3, + "a pre-block beacon-root miss exits 3.\nstderr: {}", + single.stderr + ); + let error = single.error_object(); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("beacon root contract call") && + message.contains("RPC error:") && + message.contains("cache miss"), + "the message must name the stringified beacon-root path and the miss: {error}" + ); +} + /// A parent block that does not link to the replayed block is an unanswered /// question, not a wrong answer: the single-transaction run exits 3, with or /// without `--verify-receipt`. From 8a8cad003b41b79eb2f96331f29a4aa1de027238 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 12:35:12 +0800 Subject: [PATCH 59/64] fix(mega-evme): anchor stringified RPC classification to cause boundary Strip only the documented pre-block Display wrappers before matching RPC prefixes with starts_with, so an execution-class message that merely embeds "RPC error: " no longer misclassifies as exit 3. --- bin/mega-evme/src/common/error.rs | 11 ++- bin/mega-evme/src/common/exit.rs | 149 ++++++++++++++++++++++++++---- 2 files changed, 136 insertions(+), 24 deletions(-) diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index 0be04a33..fce8b98e 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -10,17 +10,18 @@ use mega_evm::{ /// Pre-block system calls (EIP-2935 / EIP-4788) and similar mega-evm wrappers /// render a database failure into a message string with `to_string()`, so the /// typed variant is gone by the time exit classification runs. The classifier -/// recovers the RPC class by looking for this exact prefix inside that message. -/// The `#[error(...)]` text on the variant must keep the same prefix; the -/// round-trip unit test in `exit` enforces that. +/// recovers the RPC class by stripping recognized outer wrappers and requiring +/// the remainder to start with this exact prefix. The `#[error(...)]` text on +/// the variant must keep the same prefix; the round-trip unit test in `exit` +/// enforces that. pub const RPC_TRANSPORT_ERROR_PREFIX: &str = "RPC transport error: "; /// Stable `Display` prefix of [`EvmeError::RpcError`]. /// /// Same recovery contract as [`RPC_TRANSPORT_ERROR_PREFIX`]: fork-state reads /// map transport/cache failures into this variant, and stringified block errors -/// still carry this prefix so exit classification can treat them as unanswered -/// questions rather than execution failures. +/// still start with this prefix (after wrapper strip) so exit classification +/// can treat them as unanswered questions rather than execution failures. pub const RPC_ERROR_PREFIX: &str = "RPC error: "; /// Error types for the replay command diff --git a/bin/mega-evme/src/common/exit.rs b/bin/mega-evme/src/common/exit.rs index c58be4e8..22a81421 100644 --- a/bin/mega-evme/src/common/exit.rs +++ b/bin/mega-evme/src/common/exit.rs @@ -97,14 +97,68 @@ const fn external_cause(err: &EvmDatabaseError) -> Option<&EvmeError> } } -/// Whether a rendered message still carries an [`EvmeError`] RPC-class prefix. +/// Outer `Display` wrapper of alloy-evm [`BlockValidationError::BlockHashContractCall`]. /// -/// Used only after typed recovery fails. The prefixes are the stable -/// [`Display`] texts of [`EvmeError::RpcError`] / [`EvmeError::RpcTransportError`], -/// which this crate owns; mega-evm and alloy-evm wrappers that call -/// `to_string()` leave them embedded in the outer message. +/// Producer: alloy-evm's `#[error("failed to apply blockhash contract call: {message}")]` +/// on that variant; mega-evm's EIP-2935 pre-block path +/// (`transact_blockhashes_contract_call`) fills `message` with `e.to_string()`. +/// The classifier usually sees only the inner `message` field, but this constant +/// is also stripped so a full rendered validation string classifies the same way. +const BLOCKHASH_CONTRACT_CALL_WRAPPER: &str = "failed to apply blockhash contract call: "; + +/// Outer `Display` stem of alloy-evm [`BlockValidationError::BeaconRootContractCall`]. +/// +/// Producer: alloy-evm's beacon-root validation error / mega-evm's EIP-4788 +/// pre-block path (`transact_beacon_root_contract_call`). The full `Display` +/// inserts `at {parent_beacon_block_root}:` between this stem and the message; +/// the variant's `message` field itself does not carry this wrapper. +const BEACON_ROOT_CONTRACT_CALL_WRAPPER: &str = "failed to apply beacon root contract call: "; + +/// revm [`EvmDatabaseError::Database`] `Display` layer around the external DB error. +/// +/// Producer: revm-database-interface `EvmDatabaseError` formats +/// `Database error: {error}`; mega-evm pre-block helpers stringify the system-call +/// failure with `e.to_string()`, so this layer sits immediately outside the +/// crate-owned RPC `Display` prefix in the validation `message` field. +const DATABASE_ERROR_WRAPPER: &str = "Database error: "; + +/// Strip every recognized outer wrapper produced on the pre-block stringification +/// path, leaving the cause boundary for prefix matching. +/// +/// Only the wrappers documented above are removed, and only from the front of +/// the string (repeatedly). Anything else — including an RPC-looking substring +/// that appears mid-message — is left alone so incidental embeds stay +/// execution-class. +fn strip_recognized_wrappers(message: &str) -> &str { + let mut rest = message; + loop { + if let Some(stripped) = rest.strip_prefix(BLOCKHASH_CONTRACT_CALL_WRAPPER) { + rest = stripped; + continue; + } + if let Some(stripped) = rest.strip_prefix(BEACON_ROOT_CONTRACT_CALL_WRAPPER) { + rest = stripped; + continue; + } + if let Some(stripped) = rest.strip_prefix(DATABASE_ERROR_WRAPPER) { + rest = stripped; + continue; + } + break; + } + rest +} + +/// Whether a rendered message is an RPC-class failure at the cause boundary. +/// +/// Used only after typed recovery fails. Recognized mega-evm / revm / alloy-evm +/// outer wrappers are stripped first; the remainder must then +/// [`str::starts_with`] a stable [`EvmeError`] RPC [`Display`] prefix this crate +/// owns. A mere `contains` would misclassify an execution failure whose message +/// incidentally embeds `"RPC error: "` (revert data, user input, etc.). fn message_carries_rpc_class(message: &str) -> bool { - message.contains(RPC_ERROR_PREFIX) || message.contains(RPC_TRANSPORT_ERROR_PREFIX) + let cause = strip_recognized_wrappers(message); + cause.starts_with(RPC_ERROR_PREFIX) || cause.starts_with(RPC_TRANSPORT_ERROR_PREFIX) } /// Recover an RPC-class failure that mega-evm stringified into a validation @@ -113,9 +167,10 @@ fn message_carries_rpc_class(message: &str) -> bool { /// Pre-block EIP-2935 / EIP-4788 helpers map a system-call database error to /// [`BlockValidationError::BlockHashContractCall`] / /// [`BlockValidationError::BeaconRootContractCall`] with `message: e.to_string()`. -/// The type is gone, but the message still contains this crate's RPC -/// [`Display`] prefixes. Other validation variants either keep a typed box -/// (handled by [`database_cause`]) or are genuine consensus/execution failures. +/// The type is gone, but after stripping the recognized outer wrappers the +/// remainder still starts with this crate's RPC [`Display`] prefixes. Other +/// validation variants either keep a typed box (handled by [`database_cause`]) +/// or are genuine consensus/execution failures. fn stringified_rpc_failure(err: &BlockExecutionError) -> bool { let BlockExecutionError::Validation(validation) = err else { return false; @@ -126,7 +181,7 @@ fn stringified_rpc_failure(err: &BlockExecutionError) -> bool { BlockValidationError::WithdrawalRequestsContractCall { message } | BlockValidationError::ConsolidationRequestsContractCall { message } => message.as_str(), // Typed boxes are recovered by `database_cause`; consensus-only variants - // never embed an RPC `Display` prefix. + // never start with an RPC `Display` prefix after wrapper strip. _ => return false, }; message_carries_rpc_class(message) @@ -346,7 +401,8 @@ mod tests { /// The classifier recovers stringified RPC failures by the same prefixes /// [`EvmeError`]'s `Display` emits — a drift between the two would silently - /// reclassify pre-block cache misses as execution errors. + /// reclassify pre-block cache misses as execution errors. Outer wrappers + /// from the pre-block path are stripped before the `starts_with` check. #[test] fn test_rpc_display_prefixes_round_trip_with_classifier() { let rpc = EvmeError::RpcError("cache miss in offline replay file".to_string()); @@ -368,10 +424,64 @@ mod tests { ); assert!(message_carries_rpc_class(&transport_text)); - // Nested the way mega-evm stringifies a system-call DB failure. - let nested = format!("Database error: {rpc_text}"); + // Nested the way mega-evm stringifies a system-call DB failure into the + // validation message field (`Database error: RPC error: …`). + let nested = format!("{DATABASE_ERROR_WRAPPER}{rpc_text}"); assert!(message_carries_rpc_class(&nested)); - assert!(!message_carries_rpc_class("failed to apply blockhash contract call: halt")); + assert_eq!(strip_recognized_wrappers(&nested), rpc_text.as_str()); + + // Full alloy-evm Display of BlockHashContractCall around that message. + let with_blockhash = format!("{BLOCKHASH_CONTRACT_CALL_WRAPPER}{nested}"); + assert!(message_carries_rpc_class(&with_blockhash)); + assert_eq!(strip_recognized_wrappers(&with_blockhash), rpc_text.as_str()); + + // Beacon-root stem (message field path has no stem; full Display may). + let with_beacon = format!("{BEACON_ROOT_CONTRACT_CALL_WRAPPER}{nested}"); + assert!(message_carries_rpc_class(&with_beacon)); + + // Execution-class pre-block halt: recognized wrapper, no RPC cause. + assert!(!message_carries_rpc_class(&format!("{BLOCKHASH_CONTRACT_CALL_WRAPPER}halt"))); + } + + /// An execution-class message that merely *contains* an RPC prefix substring + /// (e.g. revert data or user-echoed text) must not classify as exit 3. + /// Only a cause that *starts with* the prefix after wrapper strip is RPC. + #[test] + fn test_embedded_rpc_prefix_in_execution_message_maps_to_one() { + // Bare embed: contains the prefix, does not start with it. + let embedded = "system contract reverted: payload embeds RPC error: spoofed"; + assert!( + embedded.contains(RPC_ERROR_PREFIX) && !embedded.starts_with(RPC_ERROR_PREFIX), + "fixture must contain but not start with the RPC prefix" + ); + assert!(!message_carries_rpc_class(embedded)); + + let err = EvmeError::BlockExecutionError(BlockExecutionError::Validation( + BlockValidationError::BlockHashContractCall { message: embedded.to_string() }, + )); + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::ExecutionError, + "embedded RPC substring must stay execution-class: {err}" + ); + + // Same embed behind the Database-error wrapper: after strip the cause + // still does not start with the RPC prefix. + let wrapped_embed = + format!("{DATABASE_ERROR_WRAPPER}OutOfGas while echoing {RPC_ERROR_PREFIX}spoofed"); + assert!( + wrapped_embed.contains(RPC_ERROR_PREFIX) && + !strip_recognized_wrappers(&wrapped_embed).starts_with(RPC_ERROR_PREFIX), + "post-strip cause must not start with the RPC prefix" + ); + let err = EvmeError::BlockExecutionError(BlockExecutionError::Validation( + BlockValidationError::BlockHashContractCall { message: wrapped_embed }, + )); + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::ExecutionError, + "wrapped embed must stay execution-class: {err}" + ); } /// The `kind` namespace is kebab-case and one name per class. @@ -465,13 +575,14 @@ mod tests { } /// Pre-block EIP-2935 stringifies the system-call error into - /// `BlockHashContractCall { message }`. The typed cause is gone, but the - /// message still embeds this crate's RPC Display prefix — classify as - /// unanswered, not as an execution rejection. + /// `BlockHashContractCall { message }`. The typed cause is gone, but after + /// stripping the Database-error wrapper the remainder starts with this + /// crate's RPC Display prefix — classify as unanswered, not as an + /// execution rejection. #[test] fn test_stringified_blockhash_contract_call_rpc_failure_maps_to_three() { let message = format!( - "Database error: {}", + "{DATABASE_ERROR_WRAPPER}{}", EvmeError::RpcError("Failed to fetch storage for history slot: cache miss".into()) ); let err = EvmeError::BlockExecutionError(BlockExecutionError::Validation( @@ -490,7 +601,7 @@ mod tests { #[test] fn test_stringified_beacon_root_contract_call_rpc_failure_maps_to_three() { let message = format!( - "Database error: {}", + "{DATABASE_ERROR_WRAPPER}{}", EvmeError::RpcError("Failed to fetch storage for beacon root: cache miss".into()) ); let err = EvmeError::BlockExecutionError(BlockExecutionError::Validation( From b465ed567c1df7f5b3bf8c29b623ac630b1e2cda Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 16:25:07 +0800 Subject: [PATCH 60/64] fix(mega-evme): authenticate fetched transactions against the requested hash A load-balanced endpoint or a tampered offline capture can answer eth_getTransactionByHash(H) with another transaction. All three fetch sites (batch loop, single-path preceding loop, single-path target) now recompute the hash from the served consensus encoding and re-derive the signer from the signature before executing anything, refusing mismatches as an inconsistent-endpoint failure (exit 3). The response's own hash field and from field are never trusted: alloy caches the served hash into trie_hash()/tx_hash(), and as_recovered() trusts the served from. Mock harnesses now serve internally-authentic transactions (hash and from computed from the signed body); the execution-abort tests switch from body tampering (now refused at the fetch) to draining the sender's parent-block balance. --- bin/mega-evme/src/common/error.rs | 9 +- bin/mega-evme/src/replay/batch.rs | 8 + bin/mega-evme/src/replay/cmd.rs | 10 + bin/mega-evme/src/replay/verify.rs | 54 ++++- bin/mega-evme/tests/replay_batch.rs | 210 +++++++++++++++++++- bin/mega-evme/tests/replay_override_spec.rs | 77 +++++-- bin/mega-evme/tests/replay_pending.rs | 61 +++++- 7 files changed, 386 insertions(+), 43 deletions(-) diff --git a/bin/mega-evme/src/common/error.rs b/bin/mega-evme/src/common/error.rs index fce8b98e..f32932a1 100644 --- a/bin/mega-evme/src/common/error.rs +++ b/bin/mega-evme/src/common/error.rs @@ -45,10 +45,11 @@ pub enum EvmeError { /// The block body listed this hash, but fetching the transaction failed. /// - /// A transport error or offline cache miss on a body-listed hash is the same - /// class as a null answer: the endpoint failed to deliver a transaction it - /// claimed to include, rather than answering "unknown hash" about a user - /// query. The hash is carried so abort output can name the failing fetch. + /// A transport error, an offline cache miss, or a served transaction that + /// fails authentication against the requested hash is the same class as a + /// null answer: the endpoint failed to deliver a transaction it claimed to + /// include, rather than answering "unknown hash" about a user query. The + /// hash is carried so abort output can name the failing fetch. #[error("Block body lists transaction {tx_hash} but fetching it failed: {message}")] BlockBodyTransactionFetch { /// Hash the block body listed and the lookup failed for. diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index 8fec63a4..df100613 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -1079,6 +1079,14 @@ where message: e.to_string(), })? .ok_or(ReplayError::BlockBodyTransactionNull(*tx_hash))?; + // A served object that fails authentication is the same class as a + // null answer on a body-listed hash: the endpoint failed to deliver + // a transaction it claimed to include. Executing it instead would + // advance the block state on the wrong transaction, or report + // another transaction's outcome under a target hash. + verify::authenticate_transaction(&tx, *tx_hash).map_err(|message| { + ReplayError::BlockBodyTransactionFetch { tx_hash: *tx_hash, message } + })?; let is_target = target_set.contains(tx_hash); let start = Instant::now(); diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 0ca23774..14abae1e 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -534,6 +534,10 @@ impl Cmd { .await .map_err(|e| ReplayError::RpcError(format!("Failed to fetch transaction: {e}")))? .ok_or_else(|| ReplayError::TransactionNotFound(tx_hash))?; + // Authenticate before trusting anything in the answer: the inclusion + // metadata read next, and the execution below, must describe the + // requested transaction rather than whatever the endpoint served. + verify::authenticate_transaction(&target_tx, tx_hash).map_err(ReplayError::RpcError)?; debug!(block_number = ?target_tx.block_number, "Transaction found"); // Classify the target from its `(block_number, block_hash)` pair before @@ -915,6 +919,12 @@ impl Cmd { .await .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? .ok_or(ReplayError::BlockBodyTransactionNull(*tx_hash))?; + // A served object that fails authentication is the same class as a + // null answer on a body-listed hash: the endpoint failed to deliver + // a transaction it claimed to include. + verify::authenticate_transaction(&tx, *tx_hash).map_err(|message| { + ReplayError::BlockBodyTransactionFetch { tx_hash: *tx_hash, message } + })?; let outcome = block_executor .run_transaction(tx.as_recovered()) .map_err(ReplayError::BlockExecutionError)?; diff --git a/bin/mega-evme/src/replay/verify.rs b/bin/mega-evme/src/replay/verify.rs index 2d7e432a..569e29b3 100644 --- a/bin/mega-evme/src/replay/verify.rs +++ b/bin/mega-evme/src/replay/verify.rs @@ -15,10 +15,11 @@ use core::fmt; use alloy_consensus::TxReceipt; -use alloy_primitives::{Address, Bytes, Log, B256}; +use alloy_primitives::{keccak256, Address, Bytes, Log, B256}; use alloy_provider::Provider; use alloy_rpc_types_eth::{Log as RpcLog, TransactionReceipt}; -use op_alloy_rpc_types::OpTransactionReceipt; +use mega_evm::{alloy_consensus::transaction::SignerRecoverable, alloy_eips::Encodable2718}; +use op_alloy_rpc_types::{OpTransactionReceipt, Transaction}; use serde::Serialize; use super::{ReplayError, Result}; @@ -394,6 +395,55 @@ pub(super) fn check_transaction_identity( )) } +/// Check that a fetched transaction is the one it was requested for. +/// +/// `eth_getTransactionByHash` is asked by transaction hash, but nothing in the +/// answer forces the endpoint to honour it: an inconsistent backend, or a +/// tampered offline capture, can serve another transaction under the requested +/// hash — and the replay would execute it, advancing the block state on the +/// wrong transaction or reporting another transaction's outcome under the +/// target's name. The served envelope is authenticated against the request +/// rather than trusted: the transaction hash is recomputed from the served +/// consensus encoding (the response's own `hash` field is as unauthenticated as +/// the rest of it), and the sender is re-derived from the signature, since the +/// served `from` field is not covered by the hash of a signed transaction (a +/// deposit's `from` is part of its encoding, so the hash already covers it). +/// Returns the explanatory message so each call site can wrap it in the error +/// shape it reports. +pub(super) fn authenticate_transaction( + tx: &Transaction, + requested_tx_hash: B256, +) -> std::result::Result<(), String> { + let envelope = tx.inner.inner.inner(); + // Hash the consensus encoding directly: `trie_hash()`/`tx_hash()` return + // the envelope's *cached* hash, which an RPC deserialization seeds from the + // response's own `hash` field — the very value being authenticated. + let computed = keccak256(envelope.encoded_2718()); + if computed != requested_tx_hash { + return Err(format!( + "the served transaction hashes to {computed}, but transaction {requested_tx_hash} \ + was requested: the endpoint served a different transaction (an inconsistent \ + backend, or a tampered capture)" + )); + } + let recovered = envelope.recover_signer().map_err(|e| { + format!( + "transaction {requested_tx_hash}: the served transaction's signature does not \ + recover a signer ({e}): the endpoint served a corrupted transaction (an \ + inconsistent backend, or a tampered capture)" + ) + })?; + let served = tx.inner.inner.signer(); + if recovered != served { + return Err(format!( + "transaction {requested_tx_hash}: the served `from` address {served} does not match \ + the signer {recovered} recovered from the signature: the endpoint served an \ + inconsistent transaction (a corrupted backend, or a tampered capture)" + )); + } + Ok(()) +} + /// Check that a fetched receipt describes the block the replay executed. /// /// Across a reorg, or against a load-balanced endpoint serving divergent views, diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index ea22fceb..20e0b1d0 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -163,10 +163,73 @@ fn envelope_dropping_transaction(name: &str, tx_hash: &str) -> std::path::PathBu path } +/// Write a copy of the envelope whose `eth_getBalance` answer for `tx_hash`'s +/// sender at its parent block is zero, and return its path. +/// +/// The served transaction stays byte-identical — it still authenticates against +/// the requested hash — but the block executor rejects it (the sender cannot +/// fund its gas) and aborts the block: an execution-class abort raised through +/// served *state*, which carries no proof and cannot be authenticated the way a +/// consensus object can. +fn envelope_with_drained_sender(name: &str, tx_hash: &str) -> std::path::PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + // The transaction's own response names its sender and inclusion block. + let marker = format!("\"hash\":\"{tx_hash}\""); + let mut sender_block: Option<(String, u64)> = None; + for entry in envelope["cache"].as_array().expect("cache entries") { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = &response["result"]; + let from = result["from"].as_str().expect("transaction `from`").to_string(); + let number = u64::from_str_radix( + result["blockNumber"].as_str().expect("blockNumber").trim_start_matches("0x"), + 16, + ) + .expect("hex block number"); + assert!( + sender_block.replace((from, number)).is_none(), + "the envelope must hold exactly one response for {tx_hash}" + ); + } + let (from, number) = sender_block.expect("the envelope must hold the transaction"); + // The state fork reads the sender at the parent block. The entry is keyed + // by the same `method\x00params` digest the capturing transport writes, so + // the key is recomputed rather than searched for by value. + let params = format!("[\"{from}\",\"0x{:x}\"]", number - 1); + let key = format!("{}", alloy_primitives::keccak256(format!("eth_getBalance\x00{params}"))); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + if entry["key"].as_str() != Some(key.as_str()) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(entry["value"].as_str().expect("entry value is a string")) + .expect("parse balance response"); + response["result"] = serde_json::Value::String("0x0".into()); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "the envelope must hold the sender's parent-block balance"); + + let path = + std::env::temp_dir().join(format!("mega_evme_batch_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored envelope"); + path +} + /// Write a copy of the envelope whose `eth_getTransactionByHash` response for -/// `tx_hash` still returns the transaction object, but with `gas` set to `0x0` -/// so execution/setup fails (intrinsic gas / validation) rather than a missing -/// lookup. Models an executor abort mid-block. +/// `tx_hash` still returns the transaction object, but with `gas` set to `0x0`. +/// +/// The tampered body no longer hashes to the requested hash, so the replay must +/// refuse to execute it: authentication fails before the transaction reaches +/// the block executor. Models a tampered capture (or a corrupted backend) +/// serving a body that does not match the hash it was asked for. fn envelope_with_zero_gas_transaction(name: &str, tx_hash: &str) -> std::path::PathBuf { let mut envelope: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) @@ -194,6 +257,43 @@ fn envelope_with_zero_gas_transaction(name: &str, tx_hash: &str) -> std::path::P path } +/// Write a copy of the envelope whose `eth_getTransactionByHash` response for +/// `tx_hash` keeps the signed body byte-identical but reports a different +/// `from` address, and return its path. +/// +/// A signed transaction's `from` is not part of its encoding — it is derived +/// from the signature — so the tampered answer still hashes to the requested +/// hash. Executing it would run the transaction under the wrong sender; +/// authentication must instead re-derive the signer and reject the served +/// `from`. +fn envelope_with_reassigned_sender(name: &str, tx_hash: &str) -> std::path::PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let marker = format!("\"hash\":\"{tx_hash}\""); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains(&marker) { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse transaction response"); + let result = response.get_mut("result").expect("transaction result"); + assert!(result.is_object(), "expected a transaction object for {tx_hash}"); + result["from"] = + serde_json::Value::String("0x000000000000000000000000000000000000dead".into()); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "the envelope must hold exactly one response for {tx_hash}"); + + let path = + std::env::temp_dir().join(format!("mega_evme_batch_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored envelope"); + path +} + /// Run `replay` against `envelope_path` and return its stdout plus its exit code. fn replay_envelope_with_code( envelope_path: &std::path::Path, @@ -543,14 +643,14 @@ fn test_replay_single_transaction_preceding_transport_error_is_an_rpc_failure() /// failure for that transaction only: every target behind it is unanswered /// (`rpc`), not blamed as execution. /// -/// Doctors the envelope so a mid-block type-0x2 call has gas `0x0` — the lookup -/// succeeds, but the block executor rejects it as an invalid transaction -/// (intrinsic/call gas) and aborts the block — an execution-class error, not -/// `TransactionNotFound`. +/// Doctors the sender's parent-block balance to zero — the lookup succeeds and +/// the transaction authenticates, but the block executor rejects it as an +/// invalid transaction (the sender cannot fund its gas) and aborts the block — +/// an execution-class error, not `TransactionNotFound`. #[test] fn test_replay_block_sweeps_targets_behind_execution_abort_as_rpc() { let (aborting, aborting_index) = EXEC_ABORT_TX; - let path = envelope_with_zero_gas_transaction("exec_abort_block", aborting); + let path = envelope_with_drained_sender("exec_abort_block", aborting); let (stdout, code) = replay_envelope_with_code(&path, &["--block", &BLOCK.to_string(), "--json"]); @@ -598,15 +698,15 @@ fn test_replay_block_sweeps_targets_behind_execution_abort_as_rpc() { /// Per-target totals stay truthful ("2 of 2"): the abort is not a synthetic /// third target failure. /// -/// `EXEC_ABORT_TX` is doctored and kept out of the `--tx-file` target list; only -/// later targets of the same block are requested. +/// `EXEC_ABORT_TX`'s sender is drained and the transaction kept out of the +/// `--tx-file` target list; only later targets of the same block are requested. #[test] fn test_replay_tx_file_non_target_execution_abort_exits_execution() { let (aborting, aborting_index) = EXEC_ABORT_TX; let (target_a, target_a_index) = BLOCK_TXS[1]; let (target_b, _) = BLOCK_TXS[2]; assert!(target_a_index > aborting_index, "targets must sit behind the non-target aborter"); - let path = envelope_with_zero_gas_transaction("non_target_exec_abort", aborting); + let path = envelope_with_drained_sender("non_target_exec_abort", aborting); let list = format!("{target_a}\n{target_b}\n"); let list_path = std::env::temp_dir() .join(format!("mega_evme_tx_list_non_target_exec_{}.txt", std::process::id())); @@ -649,6 +749,94 @@ fn test_replay_tx_file_non_target_execution_abort_exits_execution() { ); } +/// A served transaction whose body does not hash to the requested hash must +/// not execute: the batch refuses it as a failed body-listed fetch (`rpc`) and +/// sweeps the targets behind it, instead of advancing the block state on the +/// wrong transaction. +/// +/// Doctors a mid-block transaction's `gas` — any body change breaks the hash. +#[test] +fn test_replay_block_tampered_transaction_body_fails_authentication_as_rpc() { + let (tampered, tampered_index) = EXEC_ABORT_TX; + let path = envelope_with_zero_gas_transaction("tampered_body_block", tampered); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--block", &BLOCK.to_string(), "--json"]); + let _ = std::fs::remove_file(&path); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + for (index, line) in lines.iter().enumerate() { + let index = index as u64; + if index < tampered_index { + assert!( + line.get("error").is_none(), + "targets before the tampered fetch replay: {line}" + ); + continue; + } + assert_eq!( + line["error"]["kind"].as_str(), + Some("rpc"), + "a tampered or swept target went unanswered — never an execution verdict: {line}" + ); + } + let message = lines[tampered_index as usize]["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("the endpoint served a different transaction"), + "the tampered target must name the authentication failure: {message}" + ); + + assert_eq!(code, Some(3), "an unauthenticated body-listed fetch exits 3"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("rpc-failure")); +} + +/// A served transaction whose signed body authenticates but whose `from` field +/// does not match the signature's signer must be refused: executing it would +/// run the transaction under the wrong sender. +#[test] +fn test_replay_single_transaction_reassigned_sender_fails_authentication() { + let (target, _) = BLOCK_TXS[1]; + let path = envelope_with_reassigned_sender("single_reassigned_from", target); + + let (stdout, code) = replay_envelope_with_code(&path, &["--json", target]); + let _ = std::fs::remove_file(&path); + + assert_eq!(code, Some(3), "an unauthenticated target fetch exits 3: {stdout}"); + let error = single_run_error(&stdout); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("does not match the signer") && message.contains(target), + "the failure must name the sender mismatch and the transaction: {message}" + ); +} + +/// A tampered preceding transaction is refused before execution on the single +/// path too: the target's pre-state depends on it, so the run fails as a failed +/// body-listed fetch (`rpc`) naming the tampered hash. +#[test] +fn test_replay_single_transaction_tampered_preceding_fails_authentication() { + let (tampered, tampered_index) = EXEC_ABORT_TX; + let (target, target_index) = BLOCK_TXS[1]; + assert!(target_index > tampered_index, "the tampered transaction must precede the target"); + let path = envelope_with_zero_gas_transaction("single_tampered_preceding", tampered); + + let (stdout, code) = replay_envelope_with_code(&path, &["--json", target]); + let _ = std::fs::remove_file(&path); + + assert_eq!(code, Some(3), "an unauthenticated preceding fetch exits 3: {stdout}"); + let error = single_run_error(&stdout); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("Block body lists transaction") && + message.contains(tampered) && + message.contains("served a different transaction"), + "the failure must name the tampered body-listed fetch: {message}" + ); +} + /// A non-target transport abort (cache miss) exits 3 and names the failing /// fetch so swept entries are distinguishable from the cause. #[test] diff --git a/bin/mega-evme/tests/replay_override_spec.rs b/bin/mega-evme/tests/replay_override_spec.rs index 59960cd0..61d29a01 100644 --- a/bin/mega-evme/tests/replay_override_spec.rs +++ b/bin/mega-evme/tests/replay_override_spec.rs @@ -33,8 +33,16 @@ const BLOCK_NUMBER: u64 = 18_172_461; /// two worlds — historical and forced — visibly different. const MINI_REX_TIMESTAMP: u64 = 1_764_000_000; -/// Hash of the replayed transaction, and the only transaction in the block. -const TX_HASH: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; +/// Signature of the replayed transaction: a fixed, well-formed secp256k1 pair. +/// +/// The replay authenticates every served transaction — its hash is recomputed +/// from the encoding and its sender re-derived from the signature — so the mock +/// cannot serve invented `hash`/`from` constants. The authentic identity is +/// computed by [`tx_identity`] from the transaction being built; the sender is +/// whatever address this signature recovers to for it, funded like every other +/// account by the mock's blanket balance. +const SIG_R: &str = "0xa19f0f1f52e2951452711b4f4aa5d177442c9a56abeb609b803fe2412ed24946"; +const SIG_S: &str = "0x7af21777b2e7d91c745d0077ba2726ee1bb75ccf00039a6218d64fdced768491"; /// Hash of the replayed block. const BLOCK_HASH: &str = "0x2801837c261826beb8047e46139dfc4eb93ab5b3196ce23f312d3c7658262a62"; @@ -45,9 +53,6 @@ const PARENT_HASH: &str = "0xd482d481e9d11dd116ef6c41bf95ca608f159206c8f07900b1b /// Hash of the grandparent, so the parent block is a well-formed header. const GRANDPARENT_HASH: &str = "0x152b00e0c659a9ea0827f7d3b7666951c100bb6a6761a90e20ed7f79099a82e1"; -/// Sender of the replayed transaction. Funded by the mock's blanket balance. -const SENDER: &str = "0x14112799a39f2905b901067d3cd4a1f63c1cebda"; - /// `SequencerRegistry`, deployed pre-block from Rex5 on. const SEQUENCER_REGISTRY: &str = "0x6342000000000000000000000000000000000006"; @@ -152,8 +157,42 @@ fn block_json(number: u64, hash: &str, parent_hash: &str, timestamp: u64, txs: V }) } +/// The authentic identity of the replayed transaction: `(hash, from)`. +/// +/// Builds the same consensus object the replay will deserialize from +/// [`tx_json`], hashes its encoding, and recovers its signer — the two values +/// the replay authenticates the served answer against. +fn tx_identity(chain_id: u64, to: &str, input: &str) -> (String, String) { + use mega_evm::{ + alloy_consensus::{transaction::SignerRecoverable, SignableTransaction, TxEip1559}, + op_alloy_consensus::OpTxEnvelope, + }; + + let tx = TxEip1559 { + chain_id, + nonce: 0, + gas_limit: 0x249f0, + max_fee_per_gas: 0x200b20, + max_priority_fee_per_gas: 0x186a0, + to: alloy_primitives::TxKind::Call(to.parse().expect("`to` is an address")), + value: alloy_primitives::U256::ZERO, + access_list: Default::default(), + input: input.parse::().expect("calldata is hex"), + }; + let signature = alloy_primitives::Signature::new( + SIG_R.parse().expect("r is a hex word"), + SIG_S.parse().expect("s is a hex word"), + false, + ); + let signed = tx.into_signed(signature); + let hash = format!("{:#x}", signed.hash()); + let from = OpTxEnvelope::Eip1559(signed).recover_signer().expect("signature recovers"); + (hash, format!("{from:#x}")) +} + /// The replayed transaction: an EIP-1559 call to `to` with `input` as calldata. fn tx_json(chain_id: u64, to: &str, input: &str) -> Value { + let (hash, from) = tx_identity(chain_id, to, input); json!({ "type": "0x2", "chainId": format!("0x{chain_id:x}"), @@ -166,18 +205,25 @@ fn tx_json(chain_id: u64, to: &str, input: &str) -> Value { "value": "0x0", "accessList": [], "input": input, - "r": "0xa19f0f1f52e2951452711b4f4aa5d177442c9a56abeb609b803fe2412ed24946", - "s": "0x7af21777b2e7d91c745d0077ba2726ee1bb75ccf00039a6218d64fdced768491", + "r": SIG_R, + "s": SIG_S, "yParity": "0x0", "v": "0x0", - "hash": TX_HASH, - "from": SENDER, + "hash": hash, + "from": from, "blockHash": BLOCK_HASH, "blockNumber": format!("0x{BLOCK_NUMBER:x}"), "transactionIndex": "0x0", }) } +/// A mock chain and the hash of the one transaction it serves, which is what +/// the replay is pointed at. +struct MockChain { + server: MockRpcServer, + tx_hash: String, +} + /// A mock endpoint serving a one-transaction mainnet block at `timestamp`, /// whose transaction calls `to` with `input`. /// @@ -185,19 +231,20 @@ fn tx_json(chain_id: u64, to: &str, input: &str) -> Value { /// no code, and zero storage. That leaves the pre-block deploys as the only /// source of code on the forked state, which is what makes "did this spec's /// predeploys land" observable from the transaction's own return data. -async fn mock_chain(to: &str, input: &str, timestamp: u64) -> MockRpcServer { +async fn mock_chain(to: &str, input: &str, timestamp: u64) -> MockChain { mock_chain_with_id(CHAIN_ID, to, input, timestamp).await } /// [`mock_chain`], on the chain id of the caller's choosing. -async fn mock_chain_with_id(chain_id: u64, to: &str, input: &str, timestamp: u64) -> MockRpcServer { +async fn mock_chain_with_id(chain_id: u64, to: &str, input: &str, timestamp: u64) -> MockChain { + let (tx_hash, _) = tx_identity(chain_id, to, input); let server = MockRpcServer::start().await; server.respond_eth_chain_id(chain_id, 1).await; server .respond_method_params_json( "eth_getBlockByNumber", json!([format!("0x{BLOCK_NUMBER:x}"), false]), - block_json(BLOCK_NUMBER, BLOCK_HASH, PARENT_HASH, timestamp, vec![TX_HASH]), + block_json(BLOCK_NUMBER, BLOCK_HASH, PARENT_HASH, timestamp, vec![&tx_hash]), 2, ) .await; @@ -220,13 +267,13 @@ async fn mock_chain_with_id(chain_id: u64, to: &str, input: &str, timestamp: u64 4, ) .await; - server + MockChain { server, tx_hash } } /// Replay the mock's transaction, optionally with extra flags. -fn replay(server: &MockRpcServer, args: &[&str]) -> Run { +fn replay(chain: &MockChain, args: &[&str]) -> Run { let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) - .args(["replay", TX_HASH, "--rpc", &server.uri()]) + .args(["replay", &chain.tx_hash, "--rpc", &chain.server.uri()]) .args(["--rpc.no-cache-file", "--rpc.max-retries", "0", "--rpc.backoff-ms", "1", "--json"]) .args(args) .output() diff --git a/bin/mega-evme/tests/replay_pending.rs b/bin/mega-evme/tests/replay_pending.rs index 4328f2e8..7474a857 100644 --- a/bin/mega-evme/tests/replay_pending.rs +++ b/bin/mega-evme/tests/replay_pending.rs @@ -49,16 +49,53 @@ const PARENT_HASH: &str = "0xd482d481e9d11dd116ef6c41bf95ca608f159206c8f07900b1b const REPLACEMENT_PARENT_HASH: &str = "0x4444444444444444444444444444444444444444444444444444444444444444"; -/// Hash of the pending transaction being replayed. -const TX_HASH: &str = "0x41d34e7e13dfe0f85da9d407e2b2c381955d8c7eed428b17dc82327b2616b000"; - -/// Sender of the pending transaction, funded by the mock's blanket balance. -const SENDER: &str = "0x14112799a39f2905b901067d3cd4a1f63c1cebda"; +/// Signature of the pending transaction: a fixed, well-formed secp256k1 pair. +/// +/// The replay authenticates every served transaction — its hash is recomputed +/// from the encoding and its sender re-derived from the signature — so the mock +/// cannot serve invented `hash`/`from` constants; [`tx_identity`] computes the +/// authentic pair. The sender is whatever address this signature recovers to, +/// funded like every other account by the mock's blanket balance. +const SIG_R: &str = "0xa19f0f1f52e2951452711b4f4aa5d177442c9a56abeb609b803fe2412ed24946"; +const SIG_S: &str = "0x7af21777b2e7d91c745d0077ba2726ee1bb75ccf00039a6218d64fdced768491"; /// Recipient of the pending transaction: an account with no code, so the call /// succeeds without depending on any contract the mock does not serve. const RECIPIENT: &str = "0x681e908b8ab57c49c74d770f369754ccc3e1ae09"; +/// The authentic identity of the pending transaction: `(hash, from)`. +/// +/// Builds the same consensus object the replay will deserialize from +/// [`tx_json`], hashes its encoding, and recovers its signer — the two values +/// the replay authenticates the served answer against. +fn tx_identity() -> (String, String) { + use mega_evm::{ + alloy_consensus::{transaction::SignerRecoverable, SignableTransaction, TxEip1559}, + op_alloy_consensus::OpTxEnvelope, + }; + + let tx = TxEip1559 { + chain_id: CHAIN_ID, + nonce: 0, + gas_limit: 0x249f0, + max_fee_per_gas: 0x200b20, + max_priority_fee_per_gas: 0x186a0, + to: alloy_primitives::TxKind::Call(RECIPIENT.parse().expect("`to` is an address")), + value: alloy_primitives::U256::ZERO, + access_list: Default::default(), + input: alloy_primitives::Bytes::new(), + }; + let signature = alloy_primitives::Signature::new( + SIG_R.parse().expect("r is a hex word"), + SIG_S.parse().expect("s is a hex word"), + false, + ); + let signed = tx.into_signed(signature); + let hash = format!("{:#x}", signed.hash()); + let from = OpTxEnvelope::Eip1559(signed).recover_signer().expect("signature recovers"); + (hash, format!("{from:#x}")) +} + /// A block header the RPC backend and the replay accept, carrying only the /// fields either of them reads. fn block_json(hash: &str, parent_hash: &str) -> Value { @@ -97,6 +134,7 @@ fn block_json(hash: &str, parent_hash: &str) -> Value { /// endpoint reports for it. Everything else is the same transaction, so a test /// varies only the metadata the classification reads. fn tx_json(block_number: Value, block_hash: Value) -> Value { + let (hash, from) = tx_identity(); json!({ "type": "0x2", "chainId": format!("0x{CHAIN_ID:x}"), @@ -109,12 +147,12 @@ fn tx_json(block_number: Value, block_hash: Value) -> Value { "value": "0x0", "accessList": [], "input": "0x", - "r": "0xa19f0f1f52e2951452711b4f4aa5d177442c9a56abeb609b803fe2412ed24946", - "s": "0x7af21777b2e7d91c745d0077ba2726ee1bb75ccf00039a6218d64fdced768491", + "r": SIG_R, + "s": SIG_S, "yParity": "0x0", "v": "0x0", - "hash": TX_HASH, - "from": SENDER, + "hash": hash, + "from": from, "blockHash": block_hash, "blockNumber": block_number, "transactionIndex": Value::Null, @@ -214,8 +252,9 @@ impl Run { /// Replay the mock's pending transaction. fn replay(server: &MockRpcServer) -> Run { + let (tx_hash, _) = tx_identity(); let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) - .args(["replay", TX_HASH, "--rpc", &server.uri()]) + .args(["replay", &tx_hash, "--rpc", &server.uri()]) .args(["--rpc.no-cache-file", "--rpc.max-retries", "0", "--rpc.backoff-ms", "1", "--json"]) .output() .expect("failed to run mega-evme"); @@ -271,7 +310,7 @@ async fn test_pending_replay_executes_against_the_latest_block() { assert_eq!(summary["success"], json!(true), "the pending transaction must execute: {summary}"); assert_eq!( summary["receipt"]["transactionHash"].as_str(), - Some(TX_HASH), + Some(tx_identity().0.as_str()), "the receipt must describe the replayed transaction: {summary}", ); } From b9f9bcf65389d29d73630e79a1398b04d5230965 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 16:29:36 +0800 Subject: [PATCH 61/64] fix(mega-evme): attribute hashless block aborts to the in-flight transaction A rejection raised about a transaction does not always embed its hash in the error (the block-gas admission check's TransactionGasLimitMoreThanAvailableBlockGas, for one). Attribution from error introspection alone then swept the aborter itself as an unanswered peer: its NDJSON line said rpc with a 'aborted before this transaction' message while the run exit was already floored to execution. The batch loop now records which transaction's iteration raised the abort and attributes the error there first; introspection remains the fallback for errors raised outside the loop (a failed finish can still name a transaction). --- bin/mega-evme/src/replay/batch.rs | 13 ++++- bin/mega-evme/tests/replay_batch.rs | 78 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/bin/mega-evme/src/replay/batch.rs b/bin/mega-evme/src/replay/batch.rs index df100613..74f8f066 100644 --- a/bin/mega-evme/src/replay/batch.rs +++ b/bin/mega-evme/src/replay/batch.rs @@ -1059,8 +1059,16 @@ where // Run the block's transactions in order. Any failure aborts the block: the // executor state no longer matches the chain, so the remaining targets // cannot be replayed faithfully. + // + // `in_flight` names the transaction whose iteration raised the abort. It is + // the attribution ground truth: some rejections raised *about* a + // transaction do not embed its hash in the error (the block-gas admission + // check, for one), and attributing from error introspection alone would + // sweep the aborter itself as an unanswered peer. + let mut in_flight: Option = None; let loop_result: Result<()> = async { for (tx_index, tx_hash) in tx_hashes.iter().enumerate() { + in_flight = Some(*tx_hash); // Isolate BLOCKHASH reads per transaction so a fixture dump sees only // the target's own accesses (mirrors the single-tx clear after // preceding transactions). @@ -1321,7 +1329,10 @@ where } Err(e) => { warn!(block = number, error = %e, "Aborted block replay; skipping its remaining targets"); - let aborting = aborting_tx_hash(e); + // The iteration that raised the abort is authoritative; error + // introspection only covers errors raised outside the loop (a + // failed `finish`, for one), which can still name a transaction. + let aborting = in_flight.or_else(|| aborting_tx_hash(e)); let root_kind = classify(e); let mut root_on_target = false; for tx_hash in unreported { diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 20e0b1d0..43a7eb1f 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -294,6 +294,42 @@ fn envelope_with_reassigned_sender(name: &str, tx_hash: &str) -> std::path::Path path } +/// Write a copy of the envelope whose header for block `number` advertises +/// `gas_limit`, and return its path. +/// +/// Shrinking the advertised limit makes the block-gas admission check reject +/// the first transaction whose own gas limit exceeds what remains — a +/// deterministic execution-class rejection whose error names no transaction +/// hash, raised while that transaction is in flight. +fn envelope_with_block_gas_limit(name: &str, number: u64, gas_limit: u64) -> std::path::PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let number_hex = format!("0x{number:x}"); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains("\"transactions\"") { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse block response"); + let result = &mut response["result"]; + if result["number"].as_str() != Some(number_hex.as_str()) { + continue; + } + result["gasLimit"] = serde_json::Value::String(format!("0x{gas_limit:x}")); + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "the envelope must hold exactly one header for block {number}"); + + let path = + std::env::temp_dir().join(format!("mega_evme_batch_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored envelope"); + path +} + /// Run `replay` against `envelope_path` and return its stdout plus its exit code. fn replay_envelope_with_code( envelope_path: &std::path::Path, @@ -837,6 +873,48 @@ fn test_replay_single_transaction_tampered_preceding_fails_authentication() { ); } +/// A deterministic rejection that does not name its transaction still lands on +/// the transaction it was raised about: the in-flight target keeps the +/// execution-class abort as its own answer, and only the targets behind it are +/// swept as unanswered. +/// +/// Shrinks the block's advertised gas limit so admission rejects the index-1 +/// transaction (`TransactionGasLimitMoreThanAvailableBlockGas` names no hash); +/// the index-0 deposit still fits the shrunken limit. +#[test] +fn test_replay_block_hashless_abort_lands_on_the_in_flight_target() { + let path = envelope_with_block_gas_limit("hashless_abort", BLOCK, 200_000_000); + + let (stdout, code) = + replay_envelope_with_code(&path, &["--block", &BLOCK.to_string(), "--json"]); + let _ = std::fs::remove_file(&path); + let lines = ndjson(&stdout); + + assert_eq!(lines.len(), BLOCK_TX_COUNT, "every target is still reported exactly once"); + assert!(lines[0].get("error").is_none(), "the deposit fits the shrunken limit: {}", lines[0]); + let aborter = &lines[1]; + assert_eq!( + aborter["error"]["kind"].as_str(), + Some("execution"), + "the in-flight target keeps the hashless rejection as its own answer: {aborter}" + ); + let message = aborter["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("gas") && !message.contains("aborted before this transaction"), + "the aborter's line carries the rejection itself, not a swept notice: {message}" + ); + for line in &lines[2..] { + assert_eq!( + line["error"]["kind"].as_str(), + Some("rpc"), + "targets behind the abort went unanswered: {line}" + ); + } + + assert_eq!(code, Some(1), "a deterministic rejection exits 1"); + assert_eq!(run_error(&stdout)["error"]["kind"].as_str(), Some("execution-error")); +} + /// A non-target transport abort (cache miss) exits 3 and names the failing /// fetch so swept entries are distinguishable from the cause. #[test] From 4310b415eca2d9e5f81fd6fb20614ed50e100956 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 16:31:46 +0800 Subject: [PATCH 62/64] fix(mega-evme): classify a null endpoint-resolved block as an RPC failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the single-transaction path, the replayed block and its parent are fetched at heights the endpoint itself resolved — the target's inclusion metadata, or the reported latest height for a pending target. A null answer there is the endpoint contradicting itself (reorg in progress, or a load-balanced endpoint serving divergent views), which the batch path already reports as a retryable rpc failure; the single path reported a definitive exit-1 'Block not found' for the same context. Both fetches now fail with the divergent-views rpc classification (exit 3); BlockNotFound stays reserved for user-supplied heights. --- bin/mega-evme/src/replay/cmd.rs | 20 +++++++- bin/mega-evme/tests/replay_batch.rs | 77 +++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/bin/mega-evme/src/replay/cmd.rs b/bin/mega-evme/src/replay/cmd.rs index 14abae1e..95c7b572 100644 --- a/bin/mega-evme/src/replay/cmd.rs +++ b/bin/mega-evme/src/replay/cmd.rs @@ -594,6 +594,22 @@ impl Cmd { // one view under a block environment from another. The pending path // therefore fetches once and uses the same block for both roles, so the // two roles cannot disagree at all. + // Both heights below come from the endpoint's own answers — the target's + // inclusion metadata for a mined transaction, the reported latest height + // for a pending one — so a null block is the endpoint contradicting + // itself (a reorg in progress, or a load-balanced endpoint serving + // divergent views), not a definitive "unknown block". `BlockNotFound` + // (exit 1) stays reserved for user-supplied heights, where the null is + // the answer; here the same context is an infrastructure failure + // (exit 3), matching the batch path's classification. + let missing_resolved_block = |number: u64| { + ReplayError::RpcError(format!( + "endpoint did not serve block {number}, which it itself resolved (the target's \ + inclusion metadata, or its reported latest height): the endpoint served \ + divergent views (reorg in progress, or a load-balanced endpoint); retry once \ + the chain settles" + )) + }; let parent_block = if is_pending { None } else { @@ -602,14 +618,14 @@ impl Cmd { .get_block_by_number(state_base_block.into()) .await .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? - .ok_or(ReplayError::BlockNotFound(state_base_block))?, + .ok_or_else(|| missing_resolved_block(state_base_block))?, ) }; let block = provider .get_block_by_number(block_number.into()) .await .map_err(|e| ReplayError::RpcError(format!("RPC transport error: {e}")))? - .ok_or(ReplayError::BlockNotFound(block_number))?; + .ok_or_else(|| missing_resolved_block(block_number))?; let parent_block = parent_block.unwrap_or_else(|| block.clone()); // Parent/block linkage guard: the two blocks above were fetched by diff --git a/bin/mega-evme/tests/replay_batch.rs b/bin/mega-evme/tests/replay_batch.rs index 43a7eb1f..2d335e52 100644 --- a/bin/mega-evme/tests/replay_batch.rs +++ b/bin/mega-evme/tests/replay_batch.rs @@ -294,6 +294,41 @@ fn envelope_with_reassigned_sender(name: &str, tx_hash: &str) -> std::path::Path path } +/// Write a copy of the envelope whose `eth_getBlockByNumber` answer for block +/// `number` is null, and return its path. +/// +/// The height was resolved by the endpoint's own answers (the target's +/// inclusion metadata names the block, whose parent must then exist), so the +/// null models an endpoint contradicting itself across a reorg or divergent +/// load-balanced views — not a user asking about an unknown height. +fn envelope_without_block(name: &str, number: u64) -> std::path::PathBuf { + let mut envelope: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(envelope()).expect("read envelope")) + .expect("parse envelope"); + let number_hex = format!("0x{number:x}"); + let mut doctored = 0; + for entry in envelope["cache"].as_array_mut().expect("cache entries").iter_mut() { + let value = entry["value"].as_str().expect("entry value is a string"); + if !value.contains("\"transactions\"") { + continue; + } + let mut response: serde_json::Value = + serde_json::from_str(value).expect("parse block response"); + if response["result"]["number"].as_str() != Some(number_hex.as_str()) { + continue; + } + response["result"] = serde_json::Value::Null; + entry["value"] = serde_json::Value::String(response.to_string()); + doctored += 1; + } + assert_eq!(doctored, 1, "the envelope must hold exactly one header for block {number}"); + + let path = + std::env::temp_dir().join(format!("mega_evme_batch_{name}_{}.json", std::process::id())); + std::fs::write(&path, envelope.to_string()).expect("write doctored envelope"); + path +} + /// Write a copy of the envelope whose header for block `number` advertises /// `gas_limit`, and return its path. /// @@ -873,6 +908,48 @@ fn test_replay_single_transaction_tampered_preceding_fails_authentication() { ); } +/// A parent block the endpoint itself resolved — the target claims inclusion +/// in its child — answering null is an endpoint self-contradiction: a +/// retryable infrastructure failure (exit 3, matching the batch path), not a +/// definitive exit-1 "block not found". +#[test] +fn test_replay_single_transaction_null_resolved_parent_is_an_rpc_failure() { + let (target, _) = BLOCK_TXS[1]; + let path = envelope_without_block("single_null_parent", BLOCK - 1); + + let (stdout, code) = replay_envelope_with_code(&path, &["--json", target]); + let _ = std::fs::remove_file(&path); + + assert_eq!(code, Some(3), "a null resolved parent exits 3: {stdout}"); + let error = single_run_error(&stdout); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("divergent views") && message.contains(&(BLOCK - 1).to_string()), + "the failure must name the divergent view and the block: {message}" + ); +} + +/// The replayed block itself answering null after the target's metadata named +/// it is the same self-contradiction as a null parent: exit 3, not exit 1. +#[test] +fn test_replay_single_transaction_null_resolved_block_is_an_rpc_failure() { + let (target, _) = BLOCK_TXS[1]; + let path = envelope_without_block("single_null_block", BLOCK); + + let (stdout, code) = replay_envelope_with_code(&path, &["--json", target]); + let _ = std::fs::remove_file(&path); + + assert_eq!(code, Some(3), "a null resolved block exits 3: {stdout}"); + let error = single_run_error(&stdout); + assert_eq!(error["error"]["kind"].as_str(), Some("rpc-failure")); + let message = error["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("divergent views") && message.contains(&BLOCK.to_string()), + "the failure must name the divergent view and the block: {message}" + ); +} + /// A deterministic rejection that does not name its transaction still lands on /// the transaction it was raised about: the in-flight target keeps the /// execution-class abort as its own answer, and only the targets behind it are From bd5bedc28f8907e8951a42189a484e3aa41a6f09 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 16:34:48 +0800 Subject: [PATCH 63/64] fix(mega-evme): classify clear-cache filesystem failures as execution errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A --rpc.clear-cache that fails locally — the sidecar lock cannot be acquired, or the cache file cannot be unlinked — was wrapped as an RPC error, telling automation the endpoint failed to answer (exit 3) even though retrying or switching the RPC cannot fix the local filesystem. Both sites now classify as InvalidInput (exit 1), the same class as cache merge's lock failure. Docs cover the new classification, plus the transaction-authentication and resolved-block-null conventions from the two preceding fixes. --- bin/mega-evme/src/common/provider/mod.rs | 10 ++++++++-- bin/mega-evme/tests/provider.rs | 20 +++++++++++++++---- .../configuration/state-management.md | 1 + docs/mega-evme/overview.md | 4 +++- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/bin/mega-evme/src/common/provider/mod.rs b/bin/mega-evme/src/common/provider/mod.rs index 2fcf0e5d..70846621 100644 --- a/bin/mega-evme/src/common/provider/mod.rs +++ b/bin/mega-evme/src/common/provider/mod.rs @@ -211,9 +211,15 @@ impl RpcArgs { // returns; clean-exit `RpcCacheStore::persist` acquires later. // The two critical sections never overlap in one process, so // clear cannot deadlock against its own later persist. + // Local filesystem failures below are the operator's + // environment refusing the requested operation — retrying or + // switching the endpoint cannot fix them, so they classify as + // execution-class input failures (exit 1), not as the endpoint + // failing to answer (exit 3). Same class as `cache merge`'s + // lock failure. let clear_lock = if self.clear_cache { Some(acquire_exclusive_lock(&path).map_err(|e| { - EvmeError::RpcError(format!( + EvmeError::InvalidInput(format!( "Failed to acquire the cache lock {} for clear-cache of {}: {e}. \ Refusing to clear without it: a concurrent writer could race \ the unlink and silently recreate or rely on the file.", @@ -227,7 +233,7 @@ impl RpcArgs { if self.clear_cache { if let Err(e) = fs::remove_file(&path) { if e.kind() != std::io::ErrorKind::NotFound { - return Err(EvmeError::RpcError(format!( + return Err(EvmeError::InvalidInput(format!( "Failed to clear RPC cache at {}: {e}", path.display(), ))); diff --git a/bin/mega-evme/tests/provider.rs b/bin/mega-evme/tests/provider.rs index adb66e3d..80e01bb0 100644 --- a/bin/mega-evme/tests/provider.rs +++ b/bin/mega-evme/tests/provider.rs @@ -415,8 +415,14 @@ async fn test_build_provider_clear_cache_fails_closed_when_lock_unacquirable() { ]); let err = args.build_provider().await.expect_err("clear-cache must fail closed on lock"); + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::ExecutionError, + "a local lock failure is not the endpoint's fault: retrying or switching \ + the RPC cannot fix it, so it must not classify as an rpc failure", + ); match err { - EvmeError::RpcError(msg) => { + EvmeError::InvalidInput(msg) => { assert!(msg.contains("lock"), "error must name the lock failure, got: {msg}"); assert!( msg.contains("rpc-cache-77.json.lock") || msg.contains(".lock"), @@ -427,7 +433,7 @@ async fn test_build_provider_clear_cache_fails_closed_when_lock_unacquirable() { "error must state the clear was refused, got: {msg}", ); } - other => panic!("expected EvmeError::RpcError, got {other:?}"), + other => panic!("expected EvmeError::InvalidInput, got {other:?}"), } assert_eq!( std::fs::read_to_string(&cache_file).expect("file still readable"), @@ -488,14 +494,20 @@ async fn test_build_provider_clear_cache_hard_errors_on_unlink_failure() { std::fs::set_permissions(dir.path(), orig_perms).expect("chmod restore"); let err = result.expect_err("clear-cache must hard-error on unlink failure"); + assert_eq!( + ExitCode::from_evme_error(&err), + ExitCode::ExecutionError, + "a local unlink failure is not the endpoint's fault: retrying or switching \ + the RPC cannot fix it, so it must not classify as an rpc failure", + ); match err { - EvmeError::RpcError(msg) => { + EvmeError::InvalidInput(msg) => { assert!( msg.contains("Failed to clear RPC cache"), "error must name the failed operation, got: {msg}", ); } - other => panic!("expected EvmeError::RpcError, got {other:?}"), + other => panic!("expected EvmeError::InvalidInput, got {other:?}"), } assert!(cache_file.exists(), "the cache file should still be on disk — unlink failed"); } diff --git a/docs/mega-evme/configuration/state-management.md b/docs/mega-evme/configuration/state-management.md index 194d8ef7..a542c257 100644 --- a/docs/mega-evme/configuration/state-management.md +++ b/docs/mega-evme/configuration/state-management.md @@ -223,6 +223,7 @@ A batch scan walks linear history whose request keys essentially never repeat ac Two flags ask for it: `--rpc.cache-dir`, which names the file to use, and `--rpc.clear-cache`, which asks for that file to be deleted. Clearing only means something while the disk cache is engaged, so a batch run that forced the cache off would parse the recovery flag, do nothing, and leave the polluted file in place for the next run. With `--rpc.clear-cache`, a batch run deletes the cache file under the sidecar lock, starts from an empty cache, and persists on exit — the same sequence as single-transaction mode. +A clear that fails locally — the sidecar lock cannot be acquired, or the file cannot be unlinked — is an execution-class failure (exit `1`), not an RPC failure: retrying or switching the endpoint cannot fix the local filesystem. An explicit `--rpc.no-cache-file` still wins over both flags. ### Concurrent cache-dir sharing diff --git a/docs/mega-evme/overview.md b/docs/mega-evme/overview.md index a2410002..f37f0460 100644 --- a/docs/mega-evme/overview.md +++ b/docs/mega-evme/overview.md @@ -81,7 +81,9 @@ Every command reports its outcome through the same set of exit codes, so a pipel Codes `1` and `3` separate the two ways a question can go wrong: `1` means the tool answered, and the answer is negative; `3` means the question went unanswered, so retrying against a healthy endpoint may still produce a result. A state read that fails while the EVM is executing — an offline replay file without the response, or an endpoint that dies mid-transaction — belongs to `3` as well, even though it surfaces as a block execution error. A hash the endpoint itself listed in a block body but then resolves to null belongs to `3` too: the null contradicts an answer the endpoint already gave, so it describes an inconsistent endpoint (a reorg, or a load-balanced backend serving divergent views) rather than an unknown transaction. -Only a hash the caller supplied directly stays in `1` when it resolves to null, since nothing the endpoint served claimed it existed. +A served transaction that fails authentication belongs to `3` for the same reason: replay recomputes each fetched transaction's hash from its encoding and re-derives its sender from its signature, and an answer whose body does not match the requested hash — or whose `from` does not match its own signature — is the endpoint serving an inconsistent object, not an answer about the requested transaction. +A block the endpoint itself resolved — the inclusion block of a mined target, its parent, or the reported latest height for a pending one — that then comes back null is the same self-contradiction and belongs to `3`. +Only a hash or block height the caller supplied directly stays in `1` when it resolves to null, since nothing the endpoint served claimed it existed. Two paths cannot be classified that way: a read that fails inside the pre-block system calls (EIP-4788 beacon root, EIP-2935 block hashes) or inside the sandboxed execution of the keyless-deploy system contract has its cause rendered into a message by the layer that raises it, so `mega-evme` cannot tell it from an execution failure and reports `1`. A batch run (`--tx-file` / `--block`) reports every target on its own line and then exits once for the run as a whole, ranking the failure classes it saw: any execution or internal failure exits `1`, otherwise any RPC failure exits `3`, otherwise any verification mismatch exits `2`. From 7da1ae12c9cf7203f191b1f8aee10ccbaec7e612 Mon Sep 17 00:00:00 2001 From: RealiCZ Date: Wed, 12 Aug 2026 19:02:31 +0800 Subject: [PATCH 64/64] test(mega-evme): pin the halt-logs receipt regression against mainnet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three mainnet CREATEs on the Rex spec halted after their constructor's checkpoint had been committed, so the constructor's log reached a receipt the chain records as empty — a receipts-root divergence caught by the full-history replay gate. Capture them together with their on-chain receipts and replay them offline under --verify-receipt, so the expectation is the chain's own receipt rather than a hand-written value. Neutering the result-seam log strip turns both tests red. The capture is committed compressed; the shared fixture helper extracts it. --- .../halt_logs_repro.cache.json.tar.gz | Bin 0 -> 125996 bytes bin/mega-evme/tests/replay_halt_logs.rs | 94 ++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 bin/mega-evme/tests/fixtures/halt_logs_repro.cache.json.tar.gz create mode 100644 bin/mega-evme/tests/replay_halt_logs.rs diff --git a/bin/mega-evme/tests/fixtures/halt_logs_repro.cache.json.tar.gz b/bin/mega-evme/tests/fixtures/halt_logs_repro.cache.json.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..6edb488dbaf059c456215c9e0b54e4a8642a764a GIT binary patch literal 125996 zcmV)EK)}BriwFRcPkd?s1MIy^tY$~DF63@5aPS?#m4LBTg2RArJKv8!YTa9YRN20+ z>UOo;k2*eT<!Uf&mi-445%t08=E87=Q$VFEVqj zfBmn$*Iy6&98Udq?Y&q2YvsyEWW*Pd5s`2H;p=ek{15jYes%VT$K}!Ehi|5kzAkTm zbN2ATYcJ@-7&A(}LBFlh?KeKtZ^or%Z*VQ}OIaht8;l!n&1-M47uA4^K2Ofh!(%ih z{c!Pp-+f)my$inI?$jGEvWYMC^Dq9J|MF`;f94d+p!Y+z(Rz zwmb+AQlanv8~iX^-~aJRcpM&_-+6%V|LgzqKm0GQ|LecI`lo;Y^Pm04@qc>r-};Mx z^1uGSfBN6P{^PZ8|God`fBfJ7qxg^i@4xsb|H1$K-aq?y{;PlbpZ#<5&;I+@fB!E% z-tTvObnnjDIb;9H&wl=gfAq7Te|>)b=;e&cA*Wt$6G1 z!@CdPD);K02e~|cD?NPhprrGMkKg*kTmLxS&%dsB?v=j;Wqwu8f1Rz3DrIMql&ZSq ztEsXk#dsFkdi{rw*sW`~?q2)!lWQM;8$Y}DDBjoi@;m-K-v5++^!csF@q7M#x_;w4 zU+3R_^vTt4!}V)t`5k^1MgHNV&p*7E-v991kM7_6I=z33r~AKu5+2nd& zx6Wu`JZFEIAAa}X-ouc8&HwW3PX4tt%9v2%FVVL8OQjz-p_vi+W+;x$p7@e`_KP}n}2+Z3CI7+ z1e;X#;;;V7y0`5+?_SlvylSqAU#e?Y?Juuh^ZJ)Yz0Iy!cJ&=D)H*``d+_;fc=~<` zH2N=34qU?jg|&8v|68lT|DV2y4qo>8d(QtQd*lB3o%^M)bjht`ZyG1Hvz)2dIp=SD z^ugO#Z@u&W2cN$8=F@O~{`k#9JoMLK%Id}^XX1-b@;5iHf1z)Db7pV;@bx=4+0FMp z{^a-PAAfTD`i(n`-Magp{OI@BuKxyQH@~^{&W-oJkT>4D_U?^OK47mq|Hkj|rjNe3 z=uOYC+F#f9qHb(N#jme^{K3_Kw|xA=AKeqoqIz8&ZPDVP1v7eGD|ykYj!Us$fAQe$ zv+(&%cH`Z5?p*!&!}{LG@4XY=zs1t~H_S)h`#;9diGlmaUNCbjPDyKe_$cO%~+`-+unRd+Q6q@Ba2Pe1HA-_w)5@-`^IW zvfqC8vAiihU|)!j)y?}iez@_&t^2>Zena2>=Jx#?;`TQ;-@W?u_PZb6z4^mk{yF>Z zyWiiv{kyd$RZ+Rf`9eD~33?|paU{>SGx-@T!3zWdb=w?BK2 z-Tvl1_4(7QPoHCKf4KeN)^{I$clCp-SI^#l@8hfEgK3{XH?Hn~u3dXwad(WME+CCp zz~V;(7O!)8@d5@!{q@xw*Ui1hpT2+X+Re9%y8Xq~+mG(Z@73LR-h21_E&1(t;r(mZ z^TYSO`1bneUwreHd*{7(K2q$ns~_6)kIo(^@zo9YPe1+m_V)*9q(`rnW7`P1ufym95*^7!n|!v|M>^#=b1U8k?Zod7`E9uOgx+-JPhVakLVWxveR<_qUtSS!vM;av;>#;{ zGA_AR`15f&dvfm_m$>!(`$whShqko)S}#7xql~j+TJeL2kl9-|eihDs`}od&;kcGr zThR;8AC593m0*{DczddKXshx&4<0=^pGtTNC2_f;x(hz)`5tHPIiF|zc`Nt{-cOQ7 z8dj_mxwMISJSIg8AmOULb&jW6n2-_D$|{aOc;#BUBAf`xxN0OQE2xls;Z?b$OVYVj zHW82xDyiTl1emo(u^KpP)LE!f99JwUZ4Fh$oL1h2kZa*Q6c$2sXkRCyQJLVSwt}z2 z+1FE94bB>yMe-R>%NnfwqQk~Tv59CZz?zIg9py8Hg3 zQXY6vHY5>aP$KKVEs$3V#za)CrEp4TRx55PDm!3^$Q<`R)s!HrH(w~KN4%YJl4?M8?=~TXe7y!<1M`h6PlsHH{jRLSUSD^o)g+G|z@f%voWlaw#L1 zT&nRQwDPh>a+bZu+k$Yxdr(>d0dc2dR6&AWhMKA~Ua+e06O-edgc2PSW3iZ%8!7i1 zskQK2xLT9bq53sV7RD7WvM4E8?~;?CNfBK1Uh0C$(m|sOQCLQKZxw~4zuSG(4)WUet3GVl;4)e*XM7+^P`x*h0u8T_|EzFyLWMC!j8Of z-eTqg3svbTV_@D!5S7v#qz%j?v;Y%H`G~v#5tg&FkDvsg_^wz$kBkPehHnPYabs%TVJ zQE&$MDdpcTNXW%5CZzd@kj04!v_M5*43-=VQUkGtsEh?81sXY|-~oTQ!q%Wgnuibo zssV(YstH)%X&FSoR0HS-XE*>?kih>?Z8a#^)QlBeYKY*dCj+rPut4BMxQaEfU-PeS^yrSS^?Yyz%B5m^R-GQO~yNXQUIU=@Yn}o zGy^$vtPp?%iMypJ0NXl4Ly1A2*u`z!_GPyl?0yK8r~;ajSTT$1kbNBD5kdocm~h#x z>LQdhQ@Ad+(js6=7Uop4q;;X6KE>!P=LxGtN+=TwV`zc5wei)S+C|RxrtyM476pB8 zah;1xT?9C&9D)HPCbpuFG6W5(WN5XtyhP|V)m!ehwGx;H+$4K$e2gp?Xfo)dh}D4; z_^4$%U9aJLeO%MXBEj`66GcOP;UHmNG=tV$y(uY5Fb^nX7<(qIz(RsFa594wHKv1~ z+DOHosf|@-2@VppCA&D*1~9t!HAl1#?EtH>SbSD1VhI{8tgOfYO$J(HI+a+YSV1cX zKgKJQO15(WHmFF11+KpodNTmW#vxr8Um`-I< zVo#hBt2Ze?@v;(4Six5z?!iGV$$f2NCcp$LVF1Q~ORck1Q98{O8DUdStN>|cK>lKI zRsG3)^;tlQv7mqHy`oK1pmz@F7o+7B$_T`^vfM(mwBBm)IcP#5Yo;Zdr?FVfsUXGs zmEuhrmTdyTEw9mxVN(NW_aOwxtQB3f=v#yU!?MhV6p?CTsD)H#tug8W0-hC(RidDN zVUZXvB35FE)k4Oe4BFiZeFQyQx43p#B^&GOu%@i&ql+02M~+nwa<8=QW5FWOZ?2() zX${f^e-la-tQ4Q6!{7n03!s?f^ND>F`(0eoMsF%17Ql2Zv6iI+*k#bAvQy9`gfGxf zn0G+yHuDI)2yW^%*kJ$}3U~4fHoJ{9WTJ$;)xtpR5+KL_Oc!l&&7?MDmx1bUq)wQN zz$52$hXR=0Dpgbp5Zg)}xR3N$Jgv04UW(s5DAh|jM=5=4GIaY$-%C(p!XoQWS>2yf zl^0YuN`WC4Pf0WW?q!kc73~}JA?}t*1x8DI9xw#9y8h3fq{-V)rV+0{X1}n%F$i?{^#~>)Bhum7WD24qNWtt8JEpvX> z%DVF7@fHpdPV*{AeQE948U;>}_24LQE)R@?ifsFNap3vfbvoPI%&3g~O)3_$^U#E7 z^#v?3$SRwhaLS7ez?@T%MKEhpk`N`q3yd+wgh3)W5o6NIx3}XbTR#c_H$)+VwfSx% zSee5se0TZjf#aUje7_e9jy`_B+;2>)2en04q|9LQI^T1*HKGR^!C%l6&^33t1s9sDwuaubw42pA-Ne8x1bmcil7}P`3 zht^c%?tmK8I;HP1jFo%CWR;l_Hv=7aS%v2W{p_`iM%fr`s-aHv)ZT!3q~G+f&8~Xc zoIv^uI01C5#s_TU02=595Ck9uaLm-k3&1M3IL49_9O535ep^5R)Fc}p0l&kyX`f^4 zass02F1y-TlcXWlZK8WzZRp-hsB=tD-?GNnNdx@5vkH_AyQ?5*DtU3TsnwYmqt<)& zcY!ZriCn@L4NHr;*NXogntK4rV{`w)`taOT1kQ-&or?3E#B!&8B%89;;4-CvSwI!c zC59*k7Bcuh^mIj=PqEhESxSJkko!IdQFIa+9d3M>_O*6aIath#9Hr;2$;oMy+-c(k zxkeI1lO|!B%xJt2@=AjEAt7wF#VD06^nOvfO^g?X8HL)bs&+1ed`hfAP$W-Qv`Z<; z)6EwNbjHISew!RNB-eojc#|*? zUfcIcO)Ful1#ew`Ye#>fFthm{vo${L4%F@y-oELdamZ!s| zF#0sLW?NFDEnqGk*AsR6hFK;GswHp(*0IqzB zSnW}k#EY~dgXv)D*UA#c59~1(+{wr^D}4dqv{9Yh z5F|wu>sbW*aVn>X2~}44M2-@H`|5oJqn8EupS=4d_Dqnk%tEisLa)q1&#Mf+G7J43 zQGZ^Ug#EVv1J1|ZdGBKnwblBF z7qFeZ`j2_}uW9$;&Onk9K|I?>B^#`-F+&n%E-94T03-OoYbDMMb*K8QNw%oO!NUsI zAYo0oUumVQh`i(rG#B6v$a)?iBcnGK3=Cow%2Y@dOI}jTA+Lft_WYi!uCJn)6e46I z7)&k-c*mo8$%EF&yD$qKqEcYSHBf+;s-Q;v7j(h6Z#gNmEiYqsASiD;RRcTd=!<2> zLQd5tUBLdLb*h3NC9ajQz6cX|(g7bRo#f?fc;G?-gXLI4O6&qh-{A#(#L>4unaiVx zXLrtTl~nFLI{&n1e%Xp5Jm{Xk^x|)?(res*Ln)C}!LtA!0*ft`L^?D?D1@vgaG(lm z^g2V?WF~-ogi)DPOR#$}9 z5F^6(Crk@4V)2%Vp`tK}OW|XpaL}TlF0cy49-_^sO z9k&n!&!$4)d#4I3P~~c_sJ1#((_#Ti);>ZO#iAsx8HG0hMnN`at4tOv0muRlCEg!sXMjZ2$e;DhNdXdi2icrdP%%f^MTj!^1T5jBq=*z2k_#0uPlA3p z3xLdyBA>B{+{v5ter;^&#epe{G%qU@$ZV_(&^Q6v62>snc7yk1(*l`r5Q_l8+5!e? zW2oGkl#O({)Bt7L*^^_SF3=!f#g?A@`Fk5%N|~&{iag(9*;NIKkpn~aAwqSAmJt=N zN(7dw2*z3{dqHJwC_K(*Z*@!=Y9|XZD!0wE$tq)`69ViKOU;(-oQO(r<0YBt6?C_l z1muD&&SXk=U=d&Cv52yuAtU6Beh_XT!I)mp#b66=8jOMZq6wX$GC_5UYaSLnP&J@biYiS z`=>kbh(cN0%*8j7YTCYoc2RKoW~cWOgCPJFIWX zxo}v2Mn&Zl;2q>RISbgfqv#?Ovnp!v=2SXTdpU48w<4?vibabeFJ6J@qm5@L-pilT#n#?JYhiN-I^EG4D&v!*@lhG?S zKA_G=Pk9bG9tOy#v<1qOSD?(kcpFQRkP0mkY68(yYE*3jVKF7tgp*x1pCv^Q7htLo zQQD>GYM`V=U?Um4QhMtxB?{C=SYvB}Sc3HM0NPpwuV`=d8S)tIqw5uolul?{Dvh9X z;C_|N$~AMyOopif6G4j#*|H_C3`;SK0FDfeHjp|0)JDDla4gIrbPVaBb`dgIS|tRz z!N?Du+`D%v@}oWOkaK zrj7Epz0dnUz_^GS2x~FgAS6LLy@$k_=6m<@elI9~zE`6qPy=`FHymTf!Z*lm%*a>#c0= z8r2+K|HzCfmbo#UDqJmoFMly*K%vne<-DmJlcY8R~&45ag}A?&IT!9$+wN|uzXf! zv$-35Q7@iFw-#5l>$kb8GHtGevT||NsBgNG!sNvj#&vP!f>>NpVw2WZSyj7>P#9E7 z2ss-q@AJ_y8YhZV4Xvqx``AQ-y3GialdG_<%_%ue*KNz6ljERZee4Ul26Y!Fo*b2f z8suU{?W`uE<~F0CJ&%HBF_st(MYDvl^w)k}H5vFSVlv z(e|v;xmF#oNTrWYc6lultFz&BbH*2aOyI$9=u08}cyUMR^UWP}y4dZT)& zjH8tNz8Gumenn@s4VqvPm><&?r|^jOI{ zl|t%3Aofx=?scfktK9o0!+4vcl_925Jz56MOsA-Ok1>m*v_Mm^q#78~y~bR49iS=j zHj^;;sH+)zdJlOrW(sanL~YafcuV{1m(%Zd)g&{;=uIIFp02CtQnB&VvW=Ns!%LkbV9AdVq%BlM$J;e7+N>s`ySPDm zHe>{3=J7r1@R%gt6n@{J7db+bY@dRLmi=%?`%%QE2Thua5~-w1_Fht{t&BYHIwDvn zXNB03apVU_5>1m76HHO;glcZ~(651a36JXLdT&exSY^ZczTkedbl69Sg(aSZ(b{&s z_j^5W?wIJcG+X^{E$ui^fQ7h$inhn>=-f;4d6{P&eQb6sa!YNY8ZMDjsLY}VOn|9m zJ?KOFL^zuyBl*U*pf5RQ+ilPG4%I-XnC)oo9`FEid}LFZb*2-qP1U1M`4UR^mxs*1V*w=jU7-;~W56we2UM3sz2cGT7|3!_?{2Ybm zQO0!WAY=eRlVGt!WgRn=?_??DUKg z1Q7#Qkd?f|p+%8{t!v}kv=fXcADSBl6WbO}Ap1!6K$IHov$)%AeHwL>L3~NQeDs|{ z6vad-G`GF)J!Gl(z3GK*%|Ul{s^pM;Dy9aZAgdijY)XJoSj{DRlbcVw^;vnSg^8E} zAZ73h%g!_o2{0sr7`(3qnkh;p4UXle0Lw9|O=nT5c0M*aHB);e?O+HA+3P0kDL12-Y7zUqs${@!Ytmax8A!}he_-72MT_K-he`U zJYlvT>ID8<)JzG7^@o)$$?Btlec?FB2bkF8-B z`}9B$va;>TT1VVrTT(C3FQp2d&@b8Q4p==YeKFRBB2L`G?$xlAdTBT4!%9q|&2l-1 zggFSmc(V7h&FFbHp%RP%ci#Dh)byfDP0IglTgwF5m{?oalCb+OLeLblvqS`h&?j8r zN%~Pw5=d`}8}Npp=erI_$=xx99ARwp#jr7!xx zH#}`0X_A$^&-)Tru0f>Z9?%s9FzJCKy_isH3$_|H4$jVr!}Y6O-givug1?TM_tx!J zl8i&#^r%D)B5sXQkadTL$jde8O?yl$>E9U2)G_T=L|SEg?ZW%$Pz|D7!Fvcdm05pC zvtxlyeBRdQsFVhZbzY#_1^}c^uk8YTNe#}#kLXKw=t~nuXb9hl);`*hv>R*vam`}p zm|k!}`A=Bk{rwHE4C-F;%hn6sc2yR%-*~I>tnYk~EbiYCp9(Xlb>p95NsTMw$NO4J zgd`YR3LQea$7R)JRA1|cn;z8A@D6Y}ahX0+$n_+lwcu_{fkb>&S$OcrO1=UCusQ0; zvP>X+t9cTN2nYWq&&jpqkUm1`rnmJGN+%Cpw`m7ah`zr#hla-3dEf!P=+u z9c)L3l8)!iis#J*&#%Ci$?ZPBhb%N4oG!pMb4Xd9AC+* zuUp}VU4&XjYuTCS*PHe^&vKm)+YZfjmVW5ZSquz7`89?$L2OwQHkOnFtM|a3W43I@ zAz8}{aY#0u_he(Iwh*7r6K~JF>FG}F;F7G&>Xk<4#C~Jp-@>iiaHxOH5!BcqN z5pGwEKEUmoO;4~HJ)LSBC`9l3b5Aor?E_sf^TNrknHSxjdSvb=R8T1aQCnvv+pfb* zJ>TxZ-rA|OYc$Pg+-=O(5-=?g+TeQR*#XWqd|5PIvAV%x zK1vk|liN~aMV&nB9V_{cwHvGKC`Rfv>c0QWQ>(SvugDWO9z$V-S@!JgV!uqc^f0uA z+*{b?04IcK$p=Q(7i)*!UR~s64Hdp_&@G+jw=bcV`x8*ekL*U^6Q^DbEZt?tcw2%J zmjloH)`!kfXDKXaDJ&+-#0DXo3lTDgOe_$x-nO~(c04?0$XsZ1A6$Q97ce5aWG$tB zp+i<@;^E901=~-7C#$;M^y7!$;>ET*Zm1#{A&nxzrw-#?3v(tE2 zwgx%jnZDcM^+pe=F>jB2ktdCOeI|R&uzGg?D66IpIul~- zz4vmf_rmOJWhnJ|OUkvk2ES`bIA(9$v4-fwJCe}a5AKSBGdE3UaDFLxM`m0UPdeyg zr&2T?F4^$QBz_AKN*)|)*a7KVMR7%Vs$|#uN;t0lx44Uw1S39Um$JNj5u(nKTaz=J{&&wuKr}I^T6svPV{7|4-#**7 zxJtp(KzBckIkhjBS{k>|(E&^2g5#5|VrgmI=%8xGOJ;$WW^1Ed zSR1>S^h#SJ*Um5X;g{=+fbo*YeGzCAXV8d__Eei6Cl{(!9C8aCzc4P@s1DV%pL(fX zZR=AP*$9tvgB=YWgk+W)dW)}^^|FD2jV_bLs7bkP&y->5AWX%^KK+c5`M@EI)n`Uz z!(l(19sPay>kX=0>JeQZCrU7ZP5OA}U}&MXh7>=$jHVbDQ)*FC-eH7K5}w zt&Z)td04gkjzmvnC^l&%+w6?*T6_lHS5tpm-*IU3`^X&f{Jq$NnY*R`ubF*W$#}F{w!{~|3yi@nM>)`l zMtIq_(_@y?HS$Zbt}7gnWZuU^w&n?eH!0L~c9K%fPQGKeIl)u8&oS`I!8x`euirCN z%yg=4NPV?LZcng zJus?lo;-&z?=fxS*yvIc$V1~wy6yqnGL^?&@9zuz0pzp$1Mo=xgMDw62mJZ`wpM+S z&oA1Ohkv+UXh0N>e&qg+sJ4BCUxa^jj;YWIH4Fb}Z!YkU=9JB3V^@8_zUCbIrjz^j z*q7pKeqe{xby`q68c4?iqRAwR=@D}yUsqAfQrmde)d zH3glYGD1SG9HFBT5M%-#hs*HAo(28aj;bj~Mq!=56)#)(kFFDq2^$`XZdhKs&lGuH z^_>Vh`e4>>@KlWh)tC*caox54A6ACD29o#AU@sYEk=riw~A%93VluF6+LsZDa!dM_H7+)lBI^na(_w&`z?Z zBYH?%W8N?AP~hGM@5QR|$~U_d={bh{t(q+fb4dH$lP}eduXRJBM9*}LiQGjJt!JFV zXfU`3)`eiX>$1_yi(2lTx1a#MwSFXdvys)Jzolb4!K&DQ6G;(#6P9~%cw=OX_2PT{ zGcD{5hMs((8NesUxgIS^@n;i^uL_n_o9p;jwq> zMh9Guofxz<^gZ`^y$?jQe2(t3>@r{B<=CfPstH}bcaJu~*%Ue5N0g+@-guK8h-RYS z=?~mNHAV6hHd}K#9}FV|<;kGCQjdR}#|~U{uQqm|tlI*GAb*!>{?V@e`&hfHW&82P zn+xeM`ar2!mu+q}cMq|h{33lGmU(Dg=#mb+Y4mNjmZJ>@l;ZWg+MC_gH2tO3LU*Jt zl_;6pk!a1*y+wRq&#WJ_CzzUcSI8DF)M7_NX4369{%nn_xqUAdA$m@uwn~IwVZSGT zI&b@7h`biD#3e%Xs8V-;HZGdYS`SNDA&Q`#wB^ju2uyp7lXn*88Yn zwhuHry_1J9pR*ggH61wWU9AN|fYE7>Ky=U=4W0F^jZ=wcSqVs9FKba!L4SnuOlBGA z7HVXw>wcH>I%&~7o(|%hcC6ZdTSH8uw?@joGiLfe+Q6SvL}%Y;6uRU?eQdoigOeGG z-5iI&iO$s~+hU);p82xn!qM7ySuy-QRcjN)F{qhK%({>;r1R~0(#+b{&i%9Ri@@j{ z&a-jM{rw?vFx_r9}B`!$ry(Q*s*Q~U8YHM}2shD6s!Z^z!=-$}pa&=ovy z9=jK9m`^I7oorw0vLB-UvKxar6;va(oq6Hd5lA|IfIJpcEmlIA>}MHoQNCwQTf!;_FmVmMxIT3 z6kB74M$BX8)Qb4?$x|F+c6(l&5*NM4$IW}(#qnj++&Hh6e!}Cun(6KLT5||TAIr_jj>(JYWhHHuQC%lCvC`eU*9H8 zJ;OTiZR-w+j} zc77^dfSlx@i^%#e1^D|ijp(RNEV1N$?2mx9&81_5rrj9!9-+qL#ObZkESXhbe90@ zil@~5rD6%@VnI$KIc}LshW5M2w(^{~v*Bq!XmXOUa=~Mx-q=i{Tp|B1#s#fLGmyX> zYuaqZ!EQxsh$!3zygx1^f9(F{m&=$==k$}C?RVMf$mp>a$uNHa!D2mjN!ZqFYAd|& z=MT=~bmvl&2G_VXWqCE-YKDoHtz>B-xO`7buiMkYUoAnaWpqY@RnxAYsaCeVJg5g- zxf`2PVvadc#Ci@F?Cz125FODSl3<7A7uFN=wvT|ZC-t}4QJ*)5*@*qx^j1!BU|pIY z>Z@y(wN^T0V`LlINWb|>nWCP5HBI~2V)SCMmYPR0?de(ksdvgp0^B@tg@{*i55p%%fUDgz7ywhFiv7a_1ke0Byy)XiQY15 z!J$Y8RMV}=ytP~Gw3}_y@iI1KF`~Td*c9>6V??I&F?!3k`X%Om$sV(^=?B?>{pd68 zl+eCBqs+hUKGHcS`K0xQg@g(Mjflknf(SfQB62#5ihjo!Eq!lL z|KYr=ouf#egSNEKdx5n{?9x@&oQe1Pg&m?%MgOM7k#2uOkQLAy_A&qX@6Hmid?pzI_zDuH)9Kr^L;Eg#dsXgF7o2CjtL}Cw|NlFwgpY z8pb9ILBsjvL(evQ#;zT9K1YDG_|wk8JLf=seg8o=I-tbW(|T9I&-J{xyyw&C969ge zxZXxBin;xCt)-&bur}vPv;)=JU%FG|uwKanw6E%%tqn|{T~7P$nZ6q!V~;&;5hGIR ztsN7o`l(!_g?!VR9^b%ET^}dw!}vba=#uqfoppfPA`^lmK13$W$04bMNULxHYY zU!}60qtVb$$Iwf!ggwtfid&qx;CB88snezTO@r9XOHL=wOFY4@MBeNPe;v(4erR60 z6|-d(Jf&TyoP@A!*VcNxs9no_lkTI`W~|a&ZZJ@bB-+c5H|06he1Qi|i$*`Ck1O+0 zSvt|yTe81$W9gI~vdvD>7d;WN(MR^ zE|7uFu?wW0v-6&fbb5vVE!}6*0jX!pmM4V-fL%(x(Ut@8y}Ld>o%8d)8x{H+6K49w zN*&*iN7rjato5*z@v}rPX{xX&V%M z*d|38Kv>HkK-`h?2QbAIw@q)HbzNwbQ#P9A85q1r&3Mtg-kK-X9yN5?!DUQo&QLNk zJ8i|_j6Ae^&x`KAo!k_Gu|X1D?i(4LPMG`I4mosaF~!$Uo;jT>9Et^%QMhUM+p%$0 zN4F`t`>g36J+o2_xjc?z=>$1N;&}pCan_Nv~J5RX9mF&l}&|Wn4W%hM{ z!7GlUPqt%t#jKyIJzYJ-;pggxmsBg0UQK6rToebhiGjh)i(ze<-%L9;8>*S2V933E zqKh^XC&UoQ$XW~_e%pDJ_eWTmtr!o&)`zT2-9KR~HbBle3$-)0(cr`2=b`iSnC5dv zWK_)(aY$>Zxm8*pE3OB=A$%RJ>WPWh4<`K0LK`6#^ZC+#@h z<1S>Xyd6nKDFyw~XQ*XX>e4J_vGIO6oUdZCu(3T(vSnXOQ)Z{bylmSC)pZY{O_m3H z^c1}Ay*qoI&hZH2-4|$7-3N*2HBGtgHvUv7YcX;}%XDRPr-<%}Kh`AWF`6RKT84nb z+SrtFuGiDJ{kt@0TVJU4_mtSsbwK^7ki+dbvfD2fu{&$ zxyewpkHp$~&SuKeRQ;@j_g$Wu?nZVI2~Ot&wX=Dl?PYzNyjrmaH&*2ARja>FXkH)n zF_L+c?@i{o%D)uP9Plrdl#^hIS96%preI55G?!EDGvV3v?Uo6E5Sd;|(4%&MB7Q9U z#6j!PaV^g5XmR~Kg|@CIpSyjHzxWgN=hd1O*;-qlfvKNSUdRx=nfV)C+ZF_SlSJrTdbG)%i5kc zU-}|%;=nG9nr?gZZYxr&TO+&~y5xb?qUfL?@3v$9XDkCUXmM;1yxZ~Nw2Jpvre5sl zUer>mk&A;d=8bc3OiUq9M#Vz&ecH5bKbHVh>oRMyt9L$uf+=8u5%mu%kfH+>=Dk%#ULBVr7s^yOd@*X> z+Fykj8=lN9wy>&^6X5`Tb^no4w_;aC!Gg2)qUh(X{E(fSEYIj5YvRO3n)OTfXtHh zd3ol{JCc0R4%m3iu5VqJ+K`u%^`cvd(#S+ZnS=?2iGd_qlv$62cT-c(=7Tf~?8}=U3ij70k`OS;?U!`R`3Fs0$K}ccbWl&m= z^GB9QY~zL4{ykS7rbbbw>*MPr zrpmOnEb*Xhq7+lwa)V(5c*Kn5R5JlXUM9cVUFz)kSkSy6J}ElInI(NvjIt1sSrnC`y6 zsT4D=R4OhThXJ!-Ccw5)Wl#j*U93UI$eE2+X9_KbR+%IvuF#mK%7WQcdh^NsSf-Jc zDkK}*m*v0)!Gm0$jyt$+-`2+u?~f8dr3jo~OtHz*PA!NS{4!wbN=5@x zYUiV2T>kv$E3dzS&!2zMe*LNaeB+HPcgy!ze)R@b4yX&!j244addfKJi}u+Ro~oon zkaGzpWcTd=*juG$A^-@7@x&D1U6(8 z7yxk>^9;`~Il}Wj&Y+ZO-B{%FR&?@*9RFm6lu$(-<#;daLk4kPDf0mOUX@k)@@HQ3 zC%Fv72SfrW^kctU5i1}ihhg@sN5+&fGDVJKmJF^RK26Dm*OWNe(-dHgGp>*6(BwUC=e#lUO~$s+jw%t0_8--EpVv} zDp78FXxEm!)KcYG!!bH1l{Q;gg@*!|3cQpkQXeoB)j-Vpl#-4-WB?LNaLz_xCjn?F zEg|PbF{utHt5x>>Zv#o2smd+Pcc_BNq30qjob&j zD{Kv_Q5488S;6v*1ZOCCOIeD5tGYt*w;9+K^C^J_HIstpYTKc*EPz_LQ!u>Y6UnQVKrhkQCMr{w|WGkU)G`q*!Ar3rishH7UxM0LCTvPy=BV7w5PIVk52)0Ks^xc^j)FZY^g zSU)@hYOxx}9FS7qaO-qH6d8NS&(XBR+x20oK#L1npJ*8F6D(mjzLx-7em$tg}=!4d*rks2) z^>vh{O0EH;G`pkkrSGAz_otL%K^HOJAjW&~&J4XijMB7Q7xNg;U`A_A>`%?5?G0hn z#Z8tG?5CHeK_8T=+t3+Zt+eF5H*lR@iQ{``r#Cq7!PR@I?+Ksh=UQ24KXnV(r@M6Q zW<<z$hWi|Z2@N1ecUD_aW;xB>f4ORaBCd>63U@pU@p7--qYyH zBmI=?`qz?cv7VT#AMMC=KX?TQyhuQ(XT|=TUe0Z5Z|)nMjO=sA+aP-zo>%$Xw4tq1 zu|;QmLucSLIDxKBK|n z;CwaOfv!+OzU&|C=}7h|eHur#H?nprfv8*kj>h|xn}c_3g@Be8pyOFT2M+_y1C-c~L@|dbmx;Zb_db`6mjWh2$5ZrgR;RY*6kL)(Ew)#`wtIJs@pc58cce}fvy`gB|6x~=TtY7D;Le#$wx zcvGg_bWI6C=W7;PYLw7|>gbGKDqT;-bd*P%v#50AF(Uwur!kxDO|HL* zn#lOFq`Az9kK>Q7jwRi$0Fca&o(uo$pS84TPflp~}(V5A+<< zSJ~7W+CH9wrz%;YcD4x_t);L+{iCedNr}P=S*tCBwktDuP*zby@^^PS2TnFT@WKoo zF;n_nXs9LIB>0KypU`_+_?kAR!f2zfTF$Ggh62=_s=`b3I0+SaVHwJ_SII}7WK}BF zh}%-AHs@2-XujXb{STQ2q=Blgs&$KNSLRgfSfVb+0=B=Rk5JT9pk%>Vb`fJ=h`oXm zjTMR62;mwEXMr+NSn^bfPTArUon0oaJJ~4Zo@E}4;@R@d3!N^(GZd0OV70_EV=p?AZoa&4cI7DhW1LjV`2zhN5J4UaJzpnGLgE zMJ*$w0HtS_(1RIPvsScA?5wB%Y zXFAjkD~CW6+8RJ_F_EuLlqiomL8Ii=9>qs?>NM;%u|($y>VdyoTBu4! z^3npAsz%j-Rb7D;6qBlY%E<0LnO&h%07kpKW)@r>Z~T(g$eeYlALd}}yjgxTq z_#+^e4T6lkm9n1G2@S0^|Y+?+F%TH74sygl?#6wU(UeO7N+R?@YtphFr0 zS@g(PRXYdCqA~11ajfbXV!{%VrBIL?)Od1f(6+GZP9?mxT_nk36m5-eu62=m1Brko zB*fvGE?TT;@J|O;z)}FXV+5azi2^)LR!~GtF%rs$2aVw)$@|_W!^x=`00%l{fNX3pc@!R(2j>fC z4omu=RN1OvU4*WfxD7!-GI+ob&;>MQNLHblBuDVbpsFR40v*`_Jl0(@&b|)v_Ycy& zGJCo-@8sfg0qtj07#&&IYy{f{KjUOg4Bxa&PbgCxq3m)qC&ZS7&eE zdpP^ctQo`qzkha(;4j0|O&q%HFZ%J_3KyS76!}iAcM>?v_Zx&kKGD0rSYdXX`smL6 zI}=?Bt)TCzQ5U}a{aKm(tD$64+S9&V?%Xd}hxZ>%ZW~^iEQSwTd5YEaZa5D+bXlkE z96lJp(*>u}bU+ViT`}-I3We|`R>1;*G{pqdc0M?C$wnmdUB@Euo{TQZM5v0k?s%C0AORbn`Ue|P8n>-;!;7w&EGfCMZF(JehxV+eTzsSor4D4%SmIM7@pmryaY z=0!P0wm3*hTRp93mM4d}cztU)*%&Kw`hL5AyY)@yn|GMHJibf{`drwc(#}XC{rvcB|MlB zPh9y>!Ald=DpR8ukVK&C^f1ogbz-ur`-g_GA(t&98mQ zY^khu+E+TdA^=5D)J_H%q{u2?l6So3oXFuOM5V}&F;p^bS6{)1YtWdB&S}}xMA1k* z#Y%Q+tWAz*Vxf~rA+IR)p(z?#QgSXqQ>IuP=n5R@p(I5JZd7BS<|quKPYUwWYsh6` znQ{(e3;aVF%g7OksT@5G6*MD=Z*YA;Kn7?ss>!Jx{#Si~`ozK5gTZJ)d8Hs}sBb=N zs1r>-)7fnn*jLx!EWMm?)Tl;tGy}>mr(}ZA_Bk&U&q|xPKn;TDoIM>Cn+S?Qi0cHI zs0;>50034ssESGqcugDX4C0TS>r3RAhUln203pq@_&>)mBCxI=9Lp#ka?uBJ`_I?5CQ>7b@Z z1p@cZLUT`~pz{K}(VQLt^E^3UWrt@4sJV)Y#g*zb#H3`=o1iLVe4_+(aTx8YC_>L^ zEDLZ0FbIZ)NExxEC5KuMWsYozWm*h$f&>|BIN389lW0EW)1h@ftH|UaatTe&QR<@rV5}7 zh-+F5&@d^lMGXYkz;yui3zRiLN30TuT9O4klp1r60WZP18VNk2QGByBDI6Z-j={si zW#kD7)@jhAn&^Z(pin0ZcnpUKkx>4M$R=56P*}m}7oR$XMU&_>bm%g4v`veCmskhn z)mO0A6o#1|t5(yQY8Vj8K59UjfT7$^R16dz1cnmyT>#BFvq76ERr4#!9F%9VOjICO z1h1j41<-2@=pWFgVbg^BK$JtycXS#hwVFg5)N&B6g;tapT9_0?;B&B7l`!KK%tHXy z)(}QDF*=BFl9~=|L?zIfBoD}9!FI6B!7?krIH%25u($vg05+Bb#n>rpL4JR;EtDtu z0oCD76N&FUeDYux(`HgD`ggyn^{mWzTRQ7=9{S9kuO5W+Cyys76+LZ_fPNA*77)1t z{S{ZS$b`?(!36nOL4O9$peRpB0-zB=@rO!)!5EF97U0g{9D-Ms$sgHhutN>`1>w^U z-T@H=76VN?9o!3&1HD=UYEyi;#XsF@a%@EP;@G^ zGjM2D0Zhnn1#m(>yHLm+THv8>o@%FiCTpo6EhZ<3U}(+2PAXyCGUS6I7a8Xyq&%oCzzQ zQOZAJvB47OWd_8gqQbW^C~%K-Vi(n(DRG6l1G=LE%N8<1P5|;0FwhEqZ&;=CQ(P4( zM{9xR1^KWDiWE(x==)kAq)wd%F818)u~Nu{Srynk;KiCIqp3z)urjn3ig5%WfxN-k z>7*f-0bdmOOz=!nk{lDkzJW#vktWMd^+ z@Mj>@1R7T8mzRa>cDftop3~s|A_DLDu&Rp$fyku&odqB;8Dt!Unu=tU&l0E(sEX2z8i%p8z=MT+ zk_zQLz;ss*sE^6kUbuL&&jwpVscZWP1a9++J}ORFqyjf{$R|o4NS1qOJTXEYvtZ-U z12d2UL=Chis0ipPurMW@$okYi?i+;HG|_pPTE=Gw0K|cKF&_qKoLNhU6VvJ6n$aeY zr9Ca^4z?vHEG#8?@?zrqqDHQ1B>Ay9Nl=&4rBnqb25Bc8`B4?h(vU1PdP-bj(OwE( zkla)5-W<@Om$jvHxwFA)%ev6yq67g4%*qILkT@r;RyHF-pq*Vx!iR660Z{%sQwgM1 zC=BfM-NU^eu4*C~jAMq{3d9F|A}Ci(1bWK>2p5!+3;uwbtD&uF86_k*aX_HH;!h6! zQ!iq{PR;|x#Ij;z4PB(u*MQFf;F%SS3k6+LR%kF}UXe@?r1y@_2PqKw2>=ZGgt4+z z?@#TbU%z(>Bjg0AS53<{c}pm1lQAZfC~ z!iOdk43Az$s}=e6M0+x(Ij|GArjg0;Y)G+HloZ8lYbE8QBHtY>Z1ACxpj3i2NLo9A zTuc>t{6bRx7*1Nk6z=Gxc9We44sX+X z%jmU^B{R7%uW9q7-Qk#I&|-is;)*U(0Jf}}G9CtKXDZRI6-!X3A-oBCk%c>e>(W_{ z5wgtk4DC-qrf{3$kD@ABa3o7c2JIQ?a-_Ebh6(FLRSej@L<3b-*+c=Qm142Mv6bUS z2CYw(4SwI}@c1-b;KIZx!atKiZ%r4$Kx2-S85N)i_?Gf)Nk9n5X~2aF-9-O??7dr$ zBuSPY_O4&CnCFFH_X{6a0we;2L=cdkRu6XNwm91}nw}w-BKY6+eP-q!?hziD5mnXO zJKD_ZuB`O*aJS2`bG~ygoo&`NC@cXDW@$~b$@Yvx`!-aPKSuts95z&K`AYN*%fVNu zw<#taQ;;fk54@a#tfK)WVfvp0;LuD4rEMrN%HIL~<_vm!8xDR;WC_d1Ih=O^6;zzJ0aHj3dh5Fm{>Oym7X;puua^;P45b(FDrGc? zGXfEU&1EL|GJpXh9>UzyXcLQ94AQsNF#8u!B_;c|barwK{5e??&x^OO&!A{DyP}@2v{D0YgR#z%!G* zAn@?5+DdQ)VWU9i_so-v*#Yj>~F6M zXWa^W*>0O$uHIwutS!fSvJU$s1J>Nyphe@@y!9S1F=QFUs zfQqS`>@t;(&DWW|-$PGNi-+A8reO_9ZAz$kvB?!OBr!r3?8roxEzlx_lCu|}zT!ZD zu4!okbg>NRDbuM)k2qD?5f^20#Hmb0KT8=s#C%iD3#(*KjrMj6l{V9(VBghM!Yio^ zKYWS>QZfa@$OASuOsvf-*UVj^*K9?7- zV6%~Uf=yR6g~_0jYbo?F;&fPL*wceSGerRkjZisL?y^8s;Vx6(RRbRBG&Y+n{_mpp z(cH@AC@r&r>23pU;dKe($rY&}RwT241QameQ?C6LD_W_=vpFHqGD!kY0($UxMfa|^ zSW|`cGO^{6(4_=@3)s9q^6J&Q`s&LmH=o-wUw7?QYI8!10!7K#X1lL+eM2#4@s;js zWLFzu2CM>x5*1Qe8^?$jFsFCT1`h!H&8kSE#K)>gUSV)GnN|O2RrcA+iq5Mx-(4M* zEySvC19%~wvNMld$yHGZycc3d=T%qdp+?XC-dW#fRrZNO`&QDbdQ(;vAQl6UMd(#Q zN8CTl0w+sZw)#hEt0=Utj!%hISH~tsv8JPBq? z&Wb`!K$D#Hpu}t994m&z1`)u z+8o;EQ*9A-U8iSM1yFnWcsp|3XHopwNMhu;bIfsG-I_4XS|Fn3^H^-Y8cR>>Uu#}~ zdiV1ZLd)jXvsu=AjiBp6)ra}#tquh{yWVOI zjc3ogKGN);S5lADc>6qS#I4Vdk|J9jsCB2(7jKH@ydsLlOdduZ+rBfiQQfF{r+bGE zq|}IVbD;u%WCy`zV>FFnnA$TIGGg~?YRPE;*U^KuWF%k$Op`Sj z#zHWAHnL#%Dwn$I^aeq79hE*tJ7Y_3H-8>&zO)@Jm1g71tJ#e9Q{xF&^>qSm^Q?}e z$uDQq35@tfI$D)=&L_lUuW#v348R-Wvd&pEFvSiqQ-jnB-Pfi77TQODL=9e-HgUW9%?|qt|-R5_kt(iZmt%ci= z1ZF?^qk$&I)B~HkVuWqfo|{chN6X7KQns}KzWHX0(rKjbG*Y{b^lofoJ5fJ^E8Uue z5B>oiiIr|Hc*m@3wzrR)P4+ep66ArpHD&B~oWjxIWmTej=M)~WO&k@7&W*@#-26vS z@jW+(-1+YPxbbtgC0qZG#zq2geYWQLRd+@VymYlUn$_1v%Sp4*>1#VfzvpTfbl`EU z9X0k>?DdXD#tV&vPRCgNZ&WA6`tjaoBMlJCv0+6&j8T*owzW>lLj;7PZ*W7 z#L{K9%2>QUpD-$@%2hdV$}0R$;~Z-*X4SFG-5i>$7>&bjwYRV<^Zk*Tk!zz17sJ2POvx^Z?lf81*lMS zH@1ppcC=~RnvZcmgFft{oK(s#WlKYGvt`{!8YWwg9otw1FWuxgqp_Eo?H)z&mE+cy z$xWh&DLZFNb?WbO)VdR4uSpr)b~-1|qsGR)hTaGHJ9Jk)zdsw@E8OU+j^h0-4>#Pu z&5{rPid7oz#w|yE#KC3u@2j2KT*t^&W6Qy&($)@ar}ZHAX7BfMa3M&=eJS6Z=XzC_ z)0~@flYs4^1}9AkEWgjN<36fuvL>e)O`>Vx;djZ53;bc-yF*=0d#JX_uA`%Uwt>cL z6vV>T(%T-N$2<52gH@=vx}3g4IhpOZITFi;)~dZT^U+{#%{V<`vyFGeJ~~}(HVHPm zp3-iektFhgw_I>~-ZPZ?Y+}W+<)!VoU6$1|l7S`R>3Cy) zUuB)V^@LW|6Xfj)E}1H`>cyqN#-M@OyDaGvN>s+;slg4_%qL_~*Avw33C3dxnMYGa zolT}Cz65q%#+Xyal(Y~P^g5m}8k&v(|JxH@?X@wVVETH(W?x%7?6hXWW&La?KiXlf zkGw%W?3y3P_l1WkudjIN)fZ09UZ;P_KzXdyd}_vPrX<{CB_tXpEnDicGUk-Sxf#~k zn!D69T_ltJzDx|+kp+f}K5hB=TL&5uK( z_qq{qg`s9M+o~!tt3QthtOHlSuCGjNHflX{{B^W`KeTWDajWx4z2htmUFk-_qw+TA zX|DJ2b)9x-!`r-=z;!pTszxn4;+>KWP8;}+m9=(OW=OHJ5@qdDD$+?bkhhF8ix_qf z>_&Uo^_sT62Xg6t`B*1-*!i0t`klVRA>-e6*DAW*dRfR{k$TcjJsXx+2OEU>ZX3jC z1^k|U2sA+}gc^K!Y3`H953bz$L_&{a@&~@A4t%Y(@)}=T_^I2`apanJu<6#ePrO%- zhQIuNuNl*VGMYaNHg><36EAS7zPzaG&Z&J)EE8#aTQ#niE&=eg8vU{oyxM39r;)W? zdow+;M^zRhpCD7sFGY4BZO@|9AS4#5(lHt@6=kafQT66?F3eEa8oKkDF~&omf+Hr` z&$W8gXoigHne9x8k-4DTDF{E>ZXGs$s76KdrLXe+-AsB2v!(gOt9I2$zqkHBQkva$ zYp`tK{NqdfIeu zc60t#(_~D%^4Kvdxii^0nyWI8&fa^z=0P63l!Tt1UiXt3qq4efaE}W3?a6qCH(7T7w1< zRNW8Ti=N}378J_M@hLlgr*6L+{V(*xSz;dxGv&0@YFTrlSf6F1`SIByT}VZF*p$@G zqP-CcZ|wvW9=Mt=-@`;1TL&rn)v7Oft5 zk+xz!Wj@1qgT?jPe{+eyfpjT-2Je_BY17k>okETyEa(1y>ayDk5$W2Arcd`v@{S>e z?a=ulspZ(8e(v=rS+r{N#z=@qB0GM+d1L18$epu%-&e!a^c;^Hyk+aZq>AJlCG%3iZxeYDZS0Zo%T2CafcpW8aweJvjD(z4dze8zwU^mM@v-W z8M7NqV2_zAxuQ1z6#Yr+79lb9y11;cQEZ%?SB`yOq$%tpNDQL4h!j%$_%1heD@_*b zGzxa2xgW=8eR!nlr}J^uhi7V>kTDzeS#erFUco+$r`73$BG*~6>J!SeTHH~lTa@We zlxdCmfHJK;)3={|z>(=@uCCbcbf?JB4X2XN265wOlPit?f9M&P>iUci?nnXW=Xd_? z^W0hC;t%FD@`BT>xTt++2BPuv`xsOEv18CrR^ow!L#Mgx^3;?b>$Rxu`ct;t@o;&d z?TZsJ&cr~yV)>RA#n77M=l9z>44HM3Jm1UtIxVl1y)IAwhVRGs%@ZGQXOk=7M%2;x zexjIR5ART1opI6AcZK5uD8{~Vri9*9DGK^|(=o9ucVT&Eouy}M$UKt&v$8r24^9Di zZ#2SY+1o=;C$KA_U&%mdbbH}r0VEW10*9rQOpStdvZ+*R)F6BT70^J#^g!fT0GFWm z*7A1gf%r~Kj1Tfi#5?)q&4WrjYaCjWg0nFTorh~XWs_rTEX0Vvptgs~Vjm%6_k2;6 z!s_kv@$#J{-w;`5`|z`3eZw`w*4Zysy;29OlP+v;O|XlV$gx`Ha)2hA935*^0`--Y z9e|>%Awz46nH9-QGH>^%k@_3@l&fxAs9rUIcf}$LmGv(>2(3s1VP0j`G|;eNd|{a< z!8(TQj!|UO{oSlA@bu@267G-=VcTOmge{JZ-)CqZyYczc)e-2QSNMPEj2QJ|h5v`G z@iP_U)%Y2D-(cFp)dkM91|1fV`23V#=@O>PhR3a|(H=jg-TL>uHiEQg^V${IeY$o_ z1qt%Za`!s{PWGn3uGTG(Tx}ucB8Bm{zTMTgKyelCLejHct!BN?N0s50yl@*)Nsl9{ zBOFy8LWy2?&|Ng$#duDuKn3`E6=>joT!jVTZl6~dpC4mp9o(D2kV)HJvtyx2l7;kG ztCgI-P!2{_Gux{q%|HcLSccZ7s8~ZbS(eogDnrmL$0-Hl>#Hyfd`tn5ziy*cWlsSa z43y|T_&A!RMpq%&3W%e+(e`u%POQ)j*KntqffA!-Dgf0nAOoL^J~2?^+eUleDD^7v zcqWyXiKB${O>zC)O|5)u_~hR_qYj zbYT0C(4&0n%v%P|_mJz7icn3T8$R#NNf09a&IrZkg5>2cn~%+ZHx>N5Gjcmt2+X8` z)xq{F>-s`i7YPaKd>(zXqV}YtcWdUnY|dqQ6-+6l!_NuuV5`#V44v6EyhI zb5Cecje&*6v<|YBB?nteXU&z7B4$GO9lQxbMLXChYwsbv>5Hd;)q?ig`YS-SFVQ|UwP%`a2wm#Os2 zRQk@>m#Os2RQhEq-QMwKD*ZB*{#BfUFH`A9tMg?l{W6t`FH`B4sr1WK`eiD8#Z;UY~5d{ZkU&8~X7H`|L- zwPrRwX8rn3qepfjD4#m}cMF}h$hug~%r*~I%o_c6yHROQ0fFaU46`+K`G@q=&;NGo z{Vqjw5B1%_23xvoXdtuqt2zBYrhmHcT(4LIcmM6b|I<%@U{U=0yKw%JICo$O&@8o1 zw5&lGVB4b4mGz5d@YN}Z2-RFF^-05pRM@os-~IFpXFEFb9-A#}#e=^AS5Mn$6h^UZ z&v$)&bA_7i3&Oy(*IHHl$gf-dbha)t-yH7!9bYT%7W$p~_Kp|d@dcW%iZunTHpYJU zM!x-JyM<0!=KBY8{AYhZ-5thS=z#y+FE`+uE>||PHl`bIeghJijgPSU-F$PR$rp5Jf)Zp4xiGpqG_4^1i}fC;tT-QZH*wGmP(Myj!4p)j!Z<~x1dOb% z4kepJVB5>0@GyuEz;M?MK=ue8uf^FOg4NjrA{1!~GY>+ScxBJR*Nk6YbYPLE4lMG% zA54_&QDNEav)e8J2s~RJw=(M|v1Sl3R?h*?XFoyF3(FEkr9_01R4+TacvS@kjMcIY zfP^`mJ*Y;vkn*u9mssu=Uz(mkUz|)5E23M?4HE3oCg!fAODf5I)X^o}w?TTf{L)pQ z7uLFI;)Sj*8XJ}kbfc^qyL^qei!+vZ7+1t!W5&&($%JWG|;L*$Q|(H7*M0klEDwDahJQ)w>0g?jFL|u7&eHqyd#BweZ+;Is|4+CC?aGa6R(IC` z7l#+c4Sj8^waEY5ydzEz9BYJNw@Vu)1}Md>qBuW`eV!w;w0+8!4%R&B?#8k`b@)WI z@&QoStS4^I3-0hdW#mUs5X;Gj>*n@^ZPW_NY%Tz~&B}abzcuP)znzvp=d><{)^DI=>AcEy>W>j_@@cV-pesTHs8 z64Fom|Ik$ko5B0E%b$lSgsr-QcoPM?OuKM_cipF?W9=G6rRD`E)Ha-OYss8pAD)J24mK_&mcNtL259X_YHH7`?iyg|@@nAin-_BUg=>(vwNmGtc`_~kH>J&!xUQ=(LMfMDmgPwJs^(@4)#_KZWjxJ_a3K7z; zN)>yp#9=-DUJI>T`P84$Q&gm=X)9xGom)MnM+@(;u=eRYmPg<5q#vV5KU*Ewo#I~g zxvy9Y)?M*8U;E7iKkCouc@L{)k2*Ds)tHvY^C_{Ti{t*=*Ce8>M~P@C*C-V+9IuC@ zd-P(`H(Bp}lM5^E5^?XF08}_vZF*?S)=vljTGU|Mv_9WgJRyCR<(+B=DStn2^(**D zE~MIuHW}WgS65fL)9PXn22`x0#*&TI-F(6mPfQFEy}ngOW^@SA!no(Qy7stFj-WYT zd4T8IhaVz;XMMywt~Y8=-Jh=2Z(*xzH7cFmkihqP(%k66H06+;mUzo*3Znx2USV|g z`g*!gVqc++Q*U)lA9sD6D(CYX`(9t%_m8881|{I$+rAf>YZaqxl7u3S1*@{eU<2Dc z$MhK${D8H$F(0Ezf?eJam(|Iaql4Rj)6bD)c29aDZ!>GGLJ&8j<(+GQ0X zY>xZLqvof6<=L$fiiSS&%~^kV;)z*RkEmj+2E>kV91YPFMfCEGiPcL%+@2RUOsPhF zx-hZF^ndNXv3RbbG%NXwT~pEihFbXZ@U)3D2Dz_ZHT-t^=C|vPr#_5s5FEQWYA?P{ zs@MD40Do9-R~LlYli-M6$OEW(0_k#8em!MXY@C&;&UH!DR;g`#KI|BbIvf)T!d^-4 z@rlzzIQL@XmAjX3e@kx{61hNk^x!Z9Rx#%2G4|*vw~sMHS2ar{4mJRmGu`WGW1>+) z*WyhtKyJ+l>Wmco0L#*`d8=a(cxfWBX?TC|F~-+wmS~;5mlaIOWale3vq1u4kMJU` zTU9_At+OthATTMw3Elv+0yhXY5_#7U^w>7_g)tfv8|zfLR75nbG@NT&AtWpGVXZWn zaI1kJyhq<{ZURiwHH)S7S(kUyDei~(LTgqH@FYtD99Aj?Ada3xNC^@nMq7L?S%Kf{ zf+_gYyamR|g^<0PW?hy4WErc(>*l)eGdhWMg+AEY|bx#iRR!2+_3#{1GwHMhV z8;%DGW2XbSdxo54c_N6@1Rcdkhf|l{oT7T1;tOM>#m|6HOX>w;u?O}!4Tk@*+&x4F zi$c0;GR|daDtg;G&7jcs-lt~!FveF%#etjnYFlC%IEXkIc-+{DM^tGGj~fJH5-|b< zYr6g+!z-|&Y|-?0*D=1*PbpsanfcL_h{F44D;*z?BS+Xm46V z|1iMaNtq4_L@x^MA+oUlK&nvrdxPc#(cz&d~AD5?kIV+82Eu*C$s%KT% zBD*_E;ow#CZMN2sBWotL5Gu*jmIiFldS+q9w{v|S=lH@PLxP~vDL{nAmQ55Ul?Ex6 zBG?v-edi1{Stcl-)hLFg#HPWpjOZ@*cTb0Zbsu=(ePP$i!_V0Ka@>d4+IVYhfw8rK zdl*`0sxVBPYqXx{0uYM@NKLoWE3~U-aQK@}MFk*^>e{=c22uJ0qJpIEHpZ{8Ach5b zJxhdt$JbX_&^Lc=VL|`e!*YJ(u#n$6Ea$flOZrX2(qiquby(6VEU2qdwv>(mL35RE z5^&k3*r4(?4;4OD-MoO5qdhC9+)NC>y%uVrckUDx1fT^K5$Me@D5!N7VTEvql&>bH znxIL=Q6K|8CL1wFuIuP@?)#3?RF+{Xo3qcre1xLRiMPLpErjMW_|78-|< zc(Z^iguhCyK{m-0|E@YVq4f4ID=w(q#|6FGRPFoXf-+FQGT!0?CZ((tbY9SfY^L>= zRm0!52`Eh>LQ9s}70b4OOnHt)hPN^QLtu~uOGvNa;G%=_p|(+6Khlh-GNAr!n=Np% za&(n6M{BI@*+LlwrU*d)P&D7h!P@-@{AH9{Q0rec{vz&X^q&9x_K&{4AAkLZ{J%p6 zg@4WXhkDj;gnw8+q({hr-wJ=Q1rHc_{0;F}%q9P3_@j$fha>Yh(?6(rHSwG2Ulf+I zmJ|L`IE*#->YYhKRb!kDRe8v;qEfEFWUNP2D!7NM#b*YjSfec+m|*etgulG4HfSBd zWL)Z5dX_Iwsu@VZ?kuFd!zC!f;PXWmUqTEe1g`+js-X$=UHs(=#w2p{0nkAt8Dq{3 zxJrefd(nv<&s+60}aVb{H1`WlELp4@TO7KCh4Qj08b8c#)2uF z;VL>G(M%16J>YLMd`9|@1U4u3t$W(z@Efg`My~(ZR zV1aN8G?3Dq@qpPjc0YpoBTH_9+C?nl{c1%I-Y(79S?(XJ3Sn1Y7&9=h*3NsMMj&MK zE-<2NG?%qds3lFU&?vgex>=A4Z&W%oT~!NjmvmE)#lT(}BpVyX0@PkHT%mbd@P4p* z=mso_)-wZ2DjVt9iWAvHQZa-UT+RC3iuq@I{TUI9R}rzeIxhDJ858m94&J`lw^=@c zw?{B5WC5I&-w8v$7sGuw3i&O>==Z*V%khBzZTd+VV#7lyiQ#pS84;Arpp4eRU0>_64id9bN9dC{gnJO+6ke4t`at`>Is;RjKZsudhmVUzO^Is;N~JoZVr5AbE-Vg8~>b5!yd)0Ka-W-rzCtUe_94h>S28jlz(;!AWhtmI5vrdoi_| z1R>E=@Tq!lsVjzLFf%Zt*1G`<%JOX+i}>bDjmNK7J01WfmX3W@N>qr`(k(+8l7{Tg zH5(&~GBGQovTC9mY1rGm#L!rc$e4FC49r7-@hd}wXl$(}T5d$F7><+}9G(897ZPm< zH0j~CRA`0@yrY)J@(beK-Wu|U?E>>>uyil&LX)NNNCAb{*kl&U#=?S2L4%9Oh$C4_ z3PrROlVdn*SZlja%3cY>a%Vm6(m_rS(OfhK*s8m$& zUiQ*Ofq-Nxfh}Olihanl5n1~XY!$j#>}0YD6qigj?;okJJ||z+<1g#+m-YC|di-TQ z{<0o_S&zT0$6watFYED__4vzr{AE457Q0Wn z=a`#uxmlcVi(?dRYXA%^7pK5`-?cxz(`fgDiYOydLqiI@GR0tHcO7TI$8Ehl~Iwy)xAo-bh^2hY2_Vb?)A*YEzjEB>T5^0DO&1y5+UB_r4 zfRkVzLNY9WtML|TRLRg!SYgcP85jG@KP3Hs{9JzD_T0o;im5GB_8qPmW)3VCjT3>* zH!@XC(})C-V9A)6#V}CSs}a>iZT!rA{ORXCg!Q$A9XaDzfBAFzbNdf}_`^me9*`qK z?&IJ=yny;#WX7@dB-7}-30Z&<6o|cS3{&(RZNlCcc5@dGk+Z)Xo;^}Qc)~+ep!}2? z^LUuGQz3-~9#Zk>Tw}onmx}ImOsJ72V11k!$bV|3{q)B_AD_zv>uQSi5CH}!Cl@BV zK*YGD>6Bs)olK=MC$o~lOQH~BgbWcJW*?y6|KUIW@^`=g!yeW9${POv{p&tt{nrSg zS64sI7ya|6eIC9BQPiJ$|EYk(oO8T>e#lj~TS~^i`{^Hl+UQbQ90)MEJzwMh_@!;x z9?mAGSBBM3|JZO0|M8F8uOJSp@-wc!{Syw;|D67u?#Qxlh6cIL|34iV6*Gi{@L&<9 zLs;_cWOg?90O>feP#omwl%3;pW?i&}3{~>7{XGL5WHalSK!afRil%Y=&pI?JF#hkJo~(!PJhdN zCw@u3ecqO?#cP|co~8mPR}_adwk#3+@ib~3Tx0TBPB9NlQE!?ArFqBeCCXUINI3XD zY)Ch#LUXA5w*Gp0$QT{_t}eO@$@z|E_!f`j8)&+!h_!AuCRAqPl+qnIy`)`uG0H z3d8j5_~V<7VFbbak1seW?{@yX-DH-HXZtZl+C8%2p19_)u@)w%aXf%cWY5U*S^$87 z1*0W8K_CJbt4g7YlgxWO!Cg0|x{h_`+4w?Mr%g`!7l7aoQ_W9hWc~Z08V3jnCNTi4 z_ZM>lzbeR>p4-^L)&2hE&&!tggfy` zRPYlXFqv;phi1%e;~#p69t6vV6b8-AROrB}yez?Iee6PrF$O$G*axrMW=i2>+Qn83x*8I;8&&^w3ju#@~v zW0~}ga;Lc9D3By$i2OWOmVbU}I`(MfTC98dxvL5dO^vo$vA>HsvsU!1f$#(KFw~|e zr%xr)iN#8YBGUr51kea*05Nh=FJhW*XH9F<8WtA^SacIe95ieNDHJ**7T5?lS z+2gN^c!wqR(llJ`apDCL$FjW8m-0nobq)TeYCGGJkQcf>)$>AsL1Wu#XuAx=_R|&c z`&F*!ycg@ijcAE&Ny6~VZGMvq1hLK7nHEF^VjR)VFwkIq7ZGv5(X$eHG1UqSfvFXI_VC>O-aR$vX!zm#!3U@Q#P_*N$9H|VXm!@} zvH89APIrk8hwO>NpdeKDF`V<0PQ5FU?%?0^UnfZh^kSm$V%*$iHZTEON;O2@SK5`* z6bg&!IgvCbw+#bFFHzabtOeT&=N=p3aA?*phew<4b|A(f(PDP9thim$qr!O!7EK|z zww@)@LHe%^asWB$ie&P4!B4$f8I0{UxD*GSVvr8UZK50NGN>9FHv6JL=_gdw$r`C{G@dwmXp%c;olsw4>Gu+Ja>kB?}f=s>@cdtO+&`lA-2JcWI`^r34}bLEEAQuL1E(%9^^f-c@AUt^ML@}s zT-qR@y?CdnRw<)o$`(qTLDo^HQ7RX6o%6L6w)`FBBV$A!KrA-R?Jv0-cdB-+&D~#% z=qZLKZ?S|yQ-g<=qu_gUD{4kTj!T35q=WFQIN%W#EJZBnou}x%x1E$$t~`!ZeG3jn z9&IL^zCNW7=x)Rn^qc#v0A6h~foevU5H`{RZD|I;D4g~APOa}l(D0=ddo($WeY0~lSl|k$e2eeZ*?S81WgcGS;3Bd=VX;%dC8m+#ADBr{y5G^j!IJ< zlTbYazhdR`+4e=&`)(ewB8FkGR#THQTtol|u5#ftxoo28Hj2sW>hA4f^VWlhps!bO zMnoi2$|W&Rl9)IE6hIgTS|?EnsrOEKFmr-J9ZEm~2Cr9s+puh`S;TP``QExc z5Q5g3$2Gc2ip?;?z7Xg)#Wc+VS6W>s)dqCU+v!@eKJVr8E$kY96#nZ;c*u7PB_d!c z;6WpbS~eoiOK=`1typBG$R&8o&#T3H8nk`XO>KE$DFT0BCkZeJkt(nVLQk90HZRA0W`*J50}^w2wO01MDQ_S^6S34{|G=SN)NGGh7E0#g_qbkdgbO6i^+0vbR;>pG(tUv5gS=R^|i4nPOytv zTZRwv&_?@MXG;VI76n6KBXLnq8wQ}wJo*-_m7!0E+yVwhwbU9` zZGJuRO#-AhLbs;v+idBM@}1OFqR+7$Qm&5jfi&yi-`7m$KLOwibXzbx9P|gMwt?7z zckMwlC^oGtr4}8a@hK4>WRmL1Tp&rvqY+nU=$d%;eg|wI)t}1Ww6* zf*iHV%yKQ@u~yf8^EFT*|1T{Q;{YJSS>ihj9LsC}>MBl2Gj&67PyRJUit8K{-NIwd zY-IA6(PL|9`qm)fp9E@T;J0jI zvu!ZPI%n#zGDx^y$>FU_?3&e{D@R9Vx02oRRnw-H=j5s*Bykmx>C}I-Ph6$ZZVjD} zVA{wq&o1-0PG9ks%0}(hVXHNZbDyU8`zuL!v{36z>hL_NkxOk+Q{Yr4Pr^Bz4tTE4 z&%Eez^zfm&6PtTKWRk%Hf#7@0?4PuyKeS3K0PcdjFmSXC3t2Dt;9vMJUvMcK$xe_s z#B3#9W~)dW&4BIONlz^q<3@CE{b&dbn1e8{J}pQ6u}Y?~0443XRgOVakMl(E4byV2 zvT&5JE+M#IpVjZ;d>9=xuHEr9u*^`#YkrMuiFsR(AmU(0yS;A(Q>})wetbz4aOGHf zK@VY3@`Q*&zps%uuq$tw@Z-tzieKh*_z6qMLuj`xCDCllR?zd}LZl)$SL<6TS!u50 zT&I?xW(`t3EHB$V?8?c$qjsUkbO)(6pp`ZZ(7AWqHpUzyOM`xHruS_htOy`*v*x06 z{atJ0HLOxTU?5Q3(_3AfCACmt){a7Hb2Z6;r8uYnQ_FYeFarVyj&GA|b8Z2od&MRg zFAs#O4i(-Fs^d=ijjKkCFqUT*kW4>xH@cgqHD;tegAwjt1##c2DUnf1U|7JJ@}GG> z9^l6OkVGR%zjOjd=0Z`R@)39&3ml46xl|0r(>ymT)&p7yLH~GR*?}IN_==XOUwoS5 zOTFG7<1!u*V8QD9^ZWpR0WHM7qS+?Fn7^cR&1Ga?Q%|*fjI&Q{0M%=|%G((bCR}4r zQakLLA-}c4zVI-=f#F5^Sn@`Rj@IlN=r@y_-?f{@aE3@V>{UNq#~L%tJ@4S#;aZ&` z{(yV3WH62orAOg>0`MGCQt9F8^fPOhedhIk6NajO*|P9{*`3_x9MstL-4X$>by=*9 z(;Jkf#~Velvl~VK0W8DC&B%6BIEd`PrvoO`&{}-N<^#MAHR&=-Ey78Gw{!I;53R=_^MFqz;qy4PWr>zxTFh@YJZrJ`>ScHCC>&c-tP;4 zdislvq>cinI zBS-Pz*D?lZwRx~e7dJ#Q0g~czQC9L>IlCQIy&csVu7thRL%bF1R#}MlTv%%X%FZ8i zbC>L~Qx4Fn$Mxvl!T@X5nnQN4TwDI-RBk7;#ol$u=ibMuHKU^G^A^sID1~`<=;;0m zxOflkK`UJLALyj9c4MVxLrdkSo&@X$mBXH5n3D$r!slvNkW96b<5>NyQq}}2o3)`?i3u?vw+1G|j zJ}XO9MN?I)n+B;?dWGSAdQeJ!c$cxI<90O!lVi7?a-|1oHkx#n!@OB1&CS2k%W`dE z(3v{$wcCzz^sJy{D2MSDPHI30mX!kZ+6e%{FQBG^zfFG@-q~i7h_nsqzN7ioiRJd= zur(w9lw}JwX-GU>w$$70v>w#|+ALV|m?cJXgvI zGIEuo-Nq}46Js5Rr=g;Uj&T`Y_Kqrk&T%}}n)~+`GqEF`jjO20kJbtbG)O)xlL=G^ zopc;vQSZM9nQ}=5lNK+}ge(x9@}hpLj<| z*;55&>#)Jh$~5<5i5K@0$7@f2%jK%EJV91 zLR(h{d1^BU=voJSD{AB!Nf6cjXcPM>zKCy?r0-KhkXjiaam78nkD(jT>+wwlYbx? z1S)8L!~UO@^zX=+Md8w*pb4N@#=ilE#}FZ47z*MvTWkixQitLTM3%Mk7Rn)01`;`* zmiMBz+->$b%>QgQJgHcHS=nO71b#@6lW4dua>H}YY&3yz4=7Eq(QM_++ok7s&Tai3 zm#3XIXY_|GpMG|dGuXCBx^byiA{4>o4e0jbAr=VrS3M10o}4pPqlNZ|?E=H&g2P zc@n^BaSq6MErS2pGMIn%+$KBQCcc@_(pPfzORANm-1LGpsQcVRUN^s==s#l~A1!X8 zhip2}xDxbX6D0Zz8pXVoqwqhz)^8q6WLH+GUc#sNy2`YG;fUa~m#toy*>hd@^POTc zaJs&PmgRxC2{LdYYvVXgx()ec51(a6v*__zBm2Ce|AYgHgzPQ`ehIM^Kg4u>VzEZ> z=+q*UAjx-{)43wOzhqz4PCv}fjXe=qsCV4SDB#c3z`BWcH!Nza@eso(2wD)6;UU!< zt4kC-u|4dvu+FM_Z3r2_@1=-&)U;ITpkh<@VgVQ}xWy(*ix*%w;qVZu(pX6urW#3| zyG5arx1NZJ2n4rn1qvVbrJ%*GLw_ic|DM}v3lBci>uw5LI^`M0h@>p2P5!>`iPu3m zMdaOd?kxlsI|dskT8!kv3YD$qnF#<1Wd=_GycS7zGEDz6D@w~u6wUIa2p}ZZaW(JM zBQP?|ZPgaijhq^vb4d&toky6jB`(%&fv3<`xyzR(WWsirXL4tz+$w_2loBV)w%=?X z2gku1$}nU<46>)EJWYnbQjpSkez`Id8S_U-n^KB5d`KaZ+LxQ!6@``)z( zhYFeOM0D@cV^q_##Q3A$DyMHLe@jT|=mmoUDVXqHf_jZoPyNX`k@P;5#1_GT;wxq7 z%A7djdR41m;V@QGU~m#(=Gf-D8Oy&S?#8JH^jFyDe)ei=qt4$8sB`;>@+LXUU@v{k zpw@d6@p&&OF#hSCQ|K|81WjSdk@vc0vvx1b+r{0nKV$L=M2q&&Pj9XEA3d$jC|7$+ zDN3^du{74dFNcBJ)^t81-QAH5W1l)yz#pC6Iwj_ihno@N8j+Cx-osl=xdl`&=m5cG3(!1nhG3@@A^{If=^UhjzK%oIBwJaj_k^m zXY{2n`!yh|)b72*WB2|lV7l9vMzZDm1GH(QvM$GP+=Po|HlLgW%x&0Yi7WN+Ou4Gr zg`02GI9n4fMLUpqUmgDH{jped%7OcMys5vlnv?dkwLAzI1FO68*V#Ym7Ig($nd_kG zTKC%hSo6ma_QJMNkMjF!>f!H^*<4b|;j_;<3>^ zLymcNohMgetl2ExB`!`*wb%KHJNd47T)gi=y#V)=C-YMkX0l#&&)AfDyGVRf>J{E+ z^y?GTOs?!)Svmz)eC?j#{r989^`%MnYxPSLQKDX5hMw;&b9n}^7c8HcOqvvooAOvy zo408;SAWU2_;Sat7Qpj0+Y6FH5Sv~DHh;D>XEUsiWN{KS3_fD~U>_R|)Tw=tKZ6Gi zmPlFhrWN*rOvcQwD-T?jtTXFMFwA&oWxi{u_sjadgCYV@bo1#Ak|DX;-8+m$jUly1 z90C>L#fXHbF)%iLDskUD+ZtRjDLuRMH1!rCg*)xq8mCUv*g$mMG6wtR6rC%Z(a7ys zM&C@mQgNJ8`rK3c%p?6LjE%v- z6A6m4{3Y^XGb;oc;h#uqwIL=b;yD%7(tiMRPosnPiUhymfh2h8WRZcq6?3t}`t|gp z-cDcnYw6rX1vHu>O03^1r|WS*U%H9O`0=RWm9ZqxKJR*}@5?PdFZ5%Tz!v?`l16^wCwQNTN? z(<)j8t7^z2-g>))dQJ0d`Xb6@3DM&+E^x{F6b;-2FMVR~Lt`{-;*SNJnf>LaVi^R& zvcaHH{I*~qqklg&7lDsOKHk!`N`EGm-NuVUi=={o-1JKUb&h(SWs0S1wk5v0%Q zv2!WDbwIkt@^D<6B+O52B<9-HAgirGi<9OTuW^fwIK%L_#i+rk@H5_duEma9R``}D*uK4xvDA?H@c>?EB-?Y@`{ zm}uX*tyrmpTk{;u%H@J=9ItvGmp|@`^&9iGG;NouY)x3K#Kh#CYrAW`xsW1^1`*#7 zE?XUEKXf+rWM9ywdnl8&NtPyx%+XQbjN8_8jWRlUCRXkX(bAV9xBsuy?9y(tx>egM-mXoins-;BNjJL!B1Kp54pZh*fKNcjNS$ zO2IH`3d4Z^`tmkLP>=RS^f zbiY?h+sh}+XP^erY`GU_WO>BJj&FP#t##6WY!U?WwQ4Rj>+fVEP~d}AV8A~foI5%4 zii_E!2wF5L`F47Lz%)eW>6qiyS14Ylsf|9zW>GGtdtro%yli?vuqkJO7-OiTa(Nn; z$}w`549x3n@@aiwvg6e`(JUR)zf!}`_qamFWa^F`wz3D=uXf4s6P24(|9fP1E^wST zLL*8UuJ531xG9vp-ZouNc)+oLmfR0GP1|oZnCRJ<^yM0AGd4OHwA?3i`DKmW4R_TN zuP707i!i3KuWI;Vj_p;xToB`o{kgO3J>)leJoqa97qxzs|Mz1RIU{CYy?9QTsOWb{kU!9x044oRg@^*c=E@7XOG&(QzB@UWuWf5^V@{|Tlfc~ z!5Q@Lcmsmj|lafR%=R2&GdD1<64Y?H$!&YUjL;4sC^Z8 zyAM&hL*z|NXiRl0GkOa5fT-xRfM>%$n&%AdN66uH$&(BLzBNOiB(-}O1hdozy({L} zFz|&~@rjd+g)|RWO;B-?PD&DQDaR{tBhQ2*M%A}gQ91Ny#P1$X-UFu)*K!+x-OXz4 zw#WT7&o)zVEHgtr&7XxRE}pBt5Z8wf*{j;qficT(clCLjwQKIer8LRA0WO3gMC*oU zQ;!}POKJ|?7)LcAygg(Q$KrPU>u1H8`W7&C!z{aBb#2bkk}pn|V87k*y>>FTRO@|4 z@wCTzpA2z8t20@8)91L*-Dq@wFptLq)v8*MB8IfTY3>U8^0AM$Be5c^ znz!HO4Ov=l(LerpwoKi&EhO6+JS$s7PDfrzOc2Otey)<}W!rZC4(Rl(PN(gPh ztE@Hccm}zXerT~0C*3AQxmglqIYv?^j(}r#z&9?V<&b9dbQ7%PvaW>5d7y|;Jr6?p zZ{#3*lA7u&pwTi$dL~bFnYR!ruJWVqEEq8K8-WR#P|?s!CE?#; z&Rq_iFwMR^QI-sJ>MmA>2Ue1EBu&(OF%vo7eobDi3&Q0UYXF8ld2R&OsL`Udx!s4| z0pxj;dAIyB8k351rPLn&6HTfLlcACXk>=cXq0qITuukM(7uLa^{GW-uYe7zGjGVMmL4P8nB!Sjvax20I%7a&h`$ zDpEgd&all`n}w{+(-o!w)-5ioi?fwFxZ5$1_B|2c)jy+ok(ipu%+R-Bbl9S~JlQ9i zlN4Vm!Oa&kn-m79ZUFg;LZZ9gTiFK4?AE)i2|y~)$-;`H7f*ZfUBk@c@D0CryilvW zv^*VVo6jX)Gv!N**BZ%8qZK8(soAXgwV>Qjx!OVIBw~_X$au6ZJ1T`cF}kXzy~ez+ z`9<(@OKWLmH1QWe}ZDredgE5%DP-(_26ZX^RxyF?J{aI;9>!B#BmA^1S3HQ6cAG{PK?{cvrA!D&B_GS82_1 z{_FmVaqsiy{;%_>3e{oI%V2wDLL6N4dB6q9U>iojZ~Cj(xqGf9uY}P`iNUqHPhLDb zHSv$X#mF3Oz|NTs=x;X*8wQ2ys(%(qYnYUW(gP*A^&$Opw=BrYdpyRZpVyC#mp??R z($Tv6`gB?t%lB4lJxGV}{aF_Yy>aD?-L#Ix;VMh%seFtYyCC1loZTwuczHUEo2Ty; zL9a8?MJ`&+-a_qUy=8xQETGH|8D=K4;)du%^q1E=&o`Ljmlnlow0Ijf^lk!DY&0hh zN^57GC#7D`tm6P$(ek&W((ia|4|2*NH=&ywytUX;m8QPhn|0VHxD-Qr)=i&?t;nh2 ztIqBznlc;XG$w$h3-uMI}dk{tO}m6{`V<85>@hlx$C$Be}T^9(DyEBwOnn0oZ+M!T_av}?1W)YM@; zXWH6x!7=$`kI=%^#;W%B=at_=l5w-a9vJ^c%_x($lSt?GsTE9~PpHXpt(hmx8b zJT6!!6h}e&T2rdYWfh^v^Bgp?t!*xLTqWauyfnl-GLi##ZL-s}pDdM!Cq*G|X8Z0L z>az&-f(95BE+#fP*9R`0jzQ$216QJCGYaiQaMF%w-{lNX0+SQ~}W=oE98Loomqpio$<>Agf4DqT6cxs^196QrmRk>gD(Jur{ z4O>Pca<8O(Z5tw-TiC^mWXntT_O{;4*+>{K@WR@s8ThIs?ivEY5hmW zqpJ1&r?(k>ehfNqrkIteYd!BV((d3!v~y$toZbtQo9HPSw(pi>=GNhg$yUZ7aM)qX zQz6TehHFC?cM3~$I+oDdf+5&OSD#k<#f5iY0(M(5mV?SK*I~x^%mmE1Z2tqk98E$U$ko~T_`T_+_=A`t&vsmU4MM3 z4Dltpk?t6~Ue|n0F}-OP+@`$)*;OpciW}hTX~@wW6VKOUM&mh-@8qiw+HNNgb(6T@ zR~WE}dv=mt_8Ozio6~9COtO{HB33wS+jBM5e+aKDJ#w{pG+;UJ4O6%%Im={C1ymY( zE^yB5>mA8UOD&*G!hH~tZKk2h2P_3CDePugvwZYp52&lyFrIT+{A z>309TlswSszfx3P>~!?5q1GmLcRl8A7(ur#S=$lsAH?8snlvppO z@-~-PyEgsuX)5ub?y{EUae~2M$$2;@)_VKC%J7U_-KK%`@QRq4qYoylEoE;tkV2*kNio>VP!L~)<#oapRuCiRW zQdIKC6}_Rn@m|5F&vdjkWB4MXVVYlBWcN*Ux4RtzA1V^DL>EtA8=X4zPz{wr>Y0h+ zPv@T_(Vnqm^qAs}%1+(ReeFhaTNfvCWRE^4EOtK4#y>x)GprSQ;<~n5TtsSkeC8&2 zygoW#51y^BZpSxHMKgLX%pVp?(6=TlpEny70Um*)o_G9n(q}f|jH>Y&Oh~g_^NoKcvl61IFcBG|aC}nG zNaJ+aw!>6*w~M8sle&wOJt~bwosK20T}_qrS$1{tk<=D8f^(<<+C3X_%)L(>JAw`s)2FYuYY6SdZn{-Za*$xf3o+4z%wk=R8$I*= zHyx486(7=o$oj{|0wyb8=ThbS!-?{2dDDGKb%6z*^97>aSLk6APR#M*UvI6&StGZO zjMay8`~olb<)-zhYZf4avQ3z>Mh^~NrHV31YbgAE!29ZwK!;n?;;!X9QBHy?QLPeK z(!)5@WlrX)$>#7C$vMtxDf&Y+>1)1m-7-#cL4lCoGsMZK*&4``1a9-KgG#1;AC8qu zqz&y|8j0TaV<+}d6?AXUYh>PUwj2g#==j1;yU4{5y|uVjkNNLhNL+yJqw|(@_`8F* z*5KFjVRj`bT(=&PMB6{he#|+R$aD(Ck%qIO#QNvyy}{sdH0P zk~C8DWTe3$s@&@`^e96XxZfgkd|P0L`$@_qVs@eSlSQDP8TsR08q>VU(QxMrxj7=@ zV-iibRB)GGf8Wq8alC32(zHtRZm=Zmt}v*y?5$4w>})oI`y${UrM zzKjBMfKkDAyPTKnz(?dvZm+pCq_KL8WAX~K)j((Mz6QgHNdA>!q&MTyREo#@;v>&U zfAZi53oGDB@aVto#mc$zd(Xx#hw8V&bhF~0AP8R`iB6W}h>xvzjGWQe!v~R~*CGF` zuUZ8%c#D-RJjAs@q8b3J_9(H&83_IV2Qu)Ps=C#(xM$O_p>+1XzxG<9sZ68{+sgKdj=5Sk9C>RdV&D zEEx{=3i!+Z~nO0!;VRhFwL^e2H8t-gUzG!j9$Qv2Ii6mqSFr%1e*h zwH;ov<9bc2lQho$)4)*#UOd^8B=!xrl_<6?V3_le)8;WA+)&{n+*4_GN`|X>2oYg}_{n86fOVqiNNy$&FSWg~Kt&=yT_!Fc3MQYZC7i*#z4lPq?Fjy-qnb zc??_KorS0+zCipCx{P>*QO{{5cj!!}Dczx3BJleZf$^)hB-VXN(bSXk*|739W62^; zkSUUgvMfz0+`KFpo%{Ysld3}lr8*cyioCL`$&p84T;K@LGgn@UgX67-2~>TX5|xLj z)-JnZy`1^6<$Py;yncSOoHoDr)L@TY6m?B#v6-eE&E&@TZ}n{C@iW}ydI^I_+tyj< zX}WGW-Jl8Wo+OI}GS02Jw1*9}-NIiBk7eG3F6nK&B%sid3weLuMo{L*X``H$20WhS zy^Sn%ws(q1ht9zb9`g^Ht}?#vO7+<#&gzWGHT&7tGkaG1BiiB7cMVCZ_lfO?3L_*8 zq%?OGx`C|D%=&jH6R;a?=WopygTY3f19I~vamQzla7|L2;UX1lgUo^eZAlw*zCw046eD??4=<*2a8Zy!x-9`nSymjL>@TAsl`Oo5U% z9J`V_Bql;U7wXj(by=JI18!`xsH5}v0XL?VonFgE#$nsPtlV=Pyz{S%yeL24e0^xU zVc&ijfiqgB5*V|dL#o}!j=xSs+x#=nS-%a)X#|HYTe*zuGE8#80nFaad`JFi%UNZQ z!MwZrv42eA44(9M6E==+w;A2xe_){A6x-TYzGn9$L?u@IqNJ17gOQLg& z4D%}K1wKSQt?6z5J6~iS94FG5-GU7%BTJBfijO7bd zXTK^l7yr`sZzx-wqmknfTl2$F7iZs`@LwRBE0&olTA8-KoN(rfT!=3urBRH^iU-7G z7xnJq$yN16Uo_-}(Cy!kyyiV(p!J0@Hf)r)$Z9*U;J_004bVOzx61`0IO^r5WY0S-^HqU82`l6`}zTkf}VO#MRwUi_JL@fgNV~!{B zXV%>37xDo^=DPr);+YJLe`|;nzXDFux;t~|?LFF6spgv7xIGSWCovjeuw&X|sxlO? z@9gV!q+sOLlT6jbQ^}=h{xH-{7?XOg9L0-x<4~Q3u$Q_oICKf~#xNb9iSHqtUG2N* zuNoV$zXtbtLTsUx46>hL3S|vUxaM97NmAx?EX`fATR5&AQYsq%eI&Y@yQi_9K6Hxt zS>s$e%WSoRAV!4%Kl(Kp@}X<^cmnqDE1N(;%?UHIvEq9KDQq(zark>%ImApZG2`vU zBHpgzBy!br$BU8mk|#1U>Xf95O+Iw?cHsHK8jk3UzR!yqsrMLg(s?LC29_%A5K9Cf zV2Y+yFD!^>8BR}# zDLDJVH4<<0_m?|CjYS*I)qb%r`Fyu`f4*;jo?iPr+kAbw?Bxh50RiK^=>UUCNJ$L0 zu^eijy^z*IlISh-vza|G$y48AVOK571?;=7(4FhAe9nQM6jx-T7`0XG!1NxVt3$>< z;g3SG`P;D)_C28w4JQq7s+n`l| z_A~wKy_FD8NN}GB084kQXf0A0xXk-r*ZpE>wlZoICuzMM=liDk{E(#d`8q}5`(Dqn zcrS4h#qYlGdF$t^hT)gMr0;G$iYO^SXNGl7130**H5WxbnZ9iJd|!9J<9|Wuet+5w zH-EE!ePwjNZu*#iKe~J$+1!xVZKFmQueu;~B$-$J+R)sIRJkECt7T&!t13zE3+0)d zTOW{IUHoeA-o$&&nZN#?dztxu`Y%xK^V&b{^Ja+Iul&8A`6`RXCnB0|-u-d-?6cK$ zb_)YJfe8qKQ0Bz-uHJ#O?~WkUZ8O~E%`q5W(z7oodlHc$q;~+3q9`A-wIWgKjWe%= zD6yjYwNW$x)JK(S;_eW>Q#j1Ax5W3o%-3YqwBL1ya9Wb{ZKb`ua^|4*^@_lkav$~0 z@OgLr{V=~j80AP1w9xT#%vWi#LntfzwQG@m#gI4O@ALK=j|Jr=6--Zm2B zaZcswJwB{^cdzNaSYzY%P2`4Pl~;m7yu^Aog)A8Ki;YDDN4iEZXo%9+wRge`adL?I zU(nQ$Z`K7iGy6>|kpc&UK}H=p`$*R!bSDh|e1nb#3q{+EA;?sjULG(d4jxoqLIYir zN)<%)`aLeO^=CGcqzJjwh)F4zMp@Nxu%%7!1AwiGhxC}!W* zh@=SYtc8?WUK`Np;q6bki4`p$)7|+>{5bM=0Fug*D*@e7CBZ*Jx^}gWr*x$E4}eW3v{6h@iHYq%KL{Me3$38+t^@?>@l zOkOZRA?ifn^{lZ;1BrhHKfO~V(;!7YK*+#Lqic_$iR22RFhd1drhj~uRFkDXUomM6 zjC-9nBp`OA<}?KlZ&kI>-$CH9>WwoTXS*gs8rUxUFf$~iCmxqc6d)>s@hT}6;A2a< zjOnHx6?l}{Xn5<$9)vP9a-_JW(5~TdG`>brt`6a6GDGzM%ILa(8_asCUM8sTeBRA!dP;4%W^M94l z(c*|_{PWs{D9}wYn?ysFHp6x~_woh0#!>`#V1Dfd8;9w3p@ea_jYf2RYtDUtNWL1b zzl7q1(>VX5T5-g4#KCyItfwl%N~45@HWWPtJ|TIS`eUY4196Vpcw4Z@=kV5bl>({4}V5^rBpKV^OLi4j1z$aS9X?Mx?9IpCP z!&j`u_qWq{dEl1 zWBuspsmHsI=6lK0u}_QX;%1K+M$T7`P4}nfr^6fHY0P_-xui2dUHy;t-@&?8U|Fjw z#=u}aFfmT_IhsX@6;x8leHzptyx!pS(u}vb?_Fco!Ssr`E(erji;#inq;KHjgPQIS zzp?RZ$$~ejpd$$0b)jZhNlJil-or~SL3KgM3OX1zc+9yRrg=O%t30UTtp(io#@Dkh zAavU$VuqkK!cWg{@J|FY(vWvZ9)7|3jw!%HpG7}*3h`|{QFx|`TpvrQA7)AB>ErU- z+9EeSyW@*4o9uxZY`I9yq?t=bQg3ya~Di5#ykWdhNwllQYZzqeppr>xRjpe zo_KqrkypJmYC5Q^Pp0aq)YqUqBcOl8MDPn_3D{A&j+Rv8pUeFMxHf_agY6{KI$DO4Br{A$5AYUOj zQc!!k5z~~q7D@kJ&y-MSGWhurqfS4J1UM?WPsDL1-ySs$f2@P z;53R@9$m$HekL+2h>!;EVa}iTxGjCRmodhyNCw>9k#;o$UWNeYU;4k|G5{tPMI3&^ zfOkY^-BR{oH0Wi?Vk8@(eWC)mBNw5kjh29?|qgDARrM{hW0Y1S-rMINS=BDL4Us4rsP z^gq1Foiy(JKEzt8TCz6SnS*c{=z|6N-4$p(VLvjj84ndEQ*AbcbaBDFg7~Sp+|Z+{ z+~a<0zh6q>&lLQ!^DmkPdtt%+!j(yFjKxf zz6qfl7XHqdRe(vMVViaTb;`U&ieNOy;sWzNnqFR5_(B^+1AXZS@G&uoDri1R7>jJh zH&TJg3QP_A*(8WB@-tr=*%O=xC{U6jQossSb~po4^(uq0f2~Keq7+*gCD{_up&+M?J3N;G}Od&`C3>2-wkbe@rjmuJ4s~PQx=wF;6 zghHUaUJvOU_sZBxtfmbmo^MjqgYNmCTeV-mP=uEB5aB2pq1Ksz846%_9?Rh65L8H{ z1s<%da}Qn(X5xpvZ(1qKt&;FyE{>r%tLbm+Xa-3tU^o^0hwGFrdkzkpO0)uZVZ;4^ zZP4}u^8hNnPE*Qk*|jj}N(s1*k*gr{etKq28p*^+`WD5xMhFhzcZ>>N%PSLcTH*9_ z52dE##;`Pm&@HB_Hi|KOpr$=-v!UtR`0Q|OnEe6kDu$^ zU-91W<2h`%$fq(=xi)dykVt=S%6H=TTDejpdgHc8Rq}i{;rBa*;fldI_5x+#h2gmB zGT;NS59Vvpa!rH;F)XD}@@ZmmxD`itq#OzSQB*2_f#H*5dy3P}@7Azs52-|7&Zt}Z zYXj9PIuH_BRcy|bOzo(PStr={;e#*_c07!QlVo4dZqR20KHd({f(d1T$$3cg=dAS=n#jMGN z^T_T2#ISmE)Ok3bl<0R_e(M<7M@`1mu3Re>!+6kGnao9%79R6uXccAFfl~ah_3-)F z7g!<+SK6V4l3~FbI#Y?`oPh%*2G{ zZDD$g$`x7!$o3XidRoqvf;bp;a?J$^CTWRIx>DxPZw3n2EhhIbsQTfZ`U*ONe9z%q zC?^fGtcOK*oQyIuvF=re;fvXluxRLTQUN`--2Hc5(wl8txQ^|a!PQXbVl&4HdcnpS zX-LFO3v!ZwC|9r-RYlP>f`lDn%^1$4V-Vt;atkw<70kVE3HgN@(B5dD^umh-75%|` zL!R}C z$o_H%tTn+?;f^{J6<}_=%1bs9KLr75x9J7?TA+a2R%cf9NQbRMlU1F)&W5Q|4Ap|o z?Lp+?{cE?VqHrjuGg6$RPZlc`KV^uJ=v%9zQ@U+byFRiz^=j*)uq4KFI<@ruk}mIx zkW0rfSRcml+jIS6v4@*}>tOgX)T0t)ZZVU|ExQqO9OsNX~Jc2@N ziZv!Y75N--Py=p6gSB+Y{B(9+m;%f8e+TQ;D@@b#2JTTP-EW*gQaoeh?&LmHjTP<; zJM3pt@AzhN^eJwEUch;w)zYSCJRP;ypqwO^dE7G4N!a|zg!rX^Xl-T{`I<$oYdC72 z{+?dDDhAWEVY;Phi;AD9h$c!dxsYWfR{t+^tEJqf1;9=%~(? zw~#crtmzE4MRd&t{|Dc#m6`E3Pd_VoOrtS!rrs+jpLa~Gm_D0cvz2NHE@T?X4Ti`q zQb88#=lw7@bkW7I<$&)CqJ|o}>&6hbYC20}3||1zQ{OFfgt0YuO(P6~8Y-lsC?vpZ z4oyf*oQK#?L<)-77)>jFUv!6xkUnch?l**RnT9wM^fN^T>E+~&B_YGQWuDDBb2h!i z+k}sy)8zCK7-v`CFDPeW-W^CtK*5BgT@V2;C}U*q<5#fZ1xGK_Rd4QW6M^r;4zX;H zPqqVy!egQ~Q|9k@k-v6%9!wyD%x3-umiR-Kz0p)~@+$*RH+xo@p#VZdUFo=3WC&v4CE^;f6SYih z#H!ntofX}{IzaTD)@79%TSjxZ@*^JU)Q zlTz1|ZpLu-H{2Iwu0Iy^%=)UI|7`#FHE~QnVeMwExQ;D_Px5uy_>^S0OOTVY~{yWynwZEDJ>fD#}Ru%FP95(~fpa#R@=(TMAt z{C)+GMnY8D02i3xc2N-_b9s6@HRgJoqrZ4y9~7@gzW8|oa`yB1DcMno%+PF__Tz32 zi*eF!6s;W;8A-Wah%kj6d1e-&IJr=;U@N!yV001n%LNh3stK#yK(Ejks6S`Lp(0=r z9$(MMIDtDGP-(rTwCra2ZDSma?e+!jw5jO;PYej30USeTd`2}(=7u4Y2}@Ae6#j|1 zHw*|YL!uXgY+#i7aeM-pHKtlg6ohuE9#;`VOP&0vYFoi*p?RLCnv~Z=c}g7}#WkMo zx!N&+#L)VYrw2p0UkKJtps~NvDBX=*WLBAFCM%@s^4tvqdTb;C3!GjUxoxaz2!Y<1 zGXn0MTyKTpkHR>770^~@>w=;iJYAf)WA-W)E8qB+Zv~D&U1J&>v;ySD+^Bd*m*(6S z`JW#yOLCDFE6azSyNt@4ZJj-2JN?{=#Ls5Y+{0OUF4gySlg+wbcDRKd_-c~c?nTSw z0MH2XSZZ6`Pg$w)>%BvaZ#AceJvNr^Jq)|L`fZ@_t6#mjS@(EZtECR8th5K*R?iw4z-bbHo?X0>lqosA4Fy(it`MZ z&=qv#cN~cR8ZU)D;p36}(eZw}%|ZF7+2h7Maw2WW8axZj0i>9&@GLOd*TMN)TI(Dp zf6f&Sz*|^kH@P3w+R*GVa}xx&IPsO?wnB&y`tg=fUH(~Jr7A=}M;KcUrJEhn`|&k4 zDu5gWU@9cnPCi}qY%Hu4ypE9XDV;7Sc4Yq`*mrgJ;gZ@2zoO6DEv!bL2Ajuob}KKP`JEJccbZl8GXz@n-j~G&ULK;f z-g8~EJG!%ozRnZ;3m0v}hdk3RVaW1D%1^t9H51`zti8W}KgDX%h-}Z1My=ddEk3%f z4v8Vj@Y>hc{iBQaKsqArabI`5oJkS3YYS{jjo)y;DvoHMqS1|WUhxY@x1?`@TPt4t zhi%F?VK%S=TVAQI&ZSB~3nOr)yqH4IvnvMKM%1h|b0BELQn#B_ulUC?9W0;a^)LQ2 zdn4G-LtCKtNrhDUF!$pfTZYJOFfP?6K;pfz(`H48_SOP*0{)8yqysJjm~erE%j__D z;tK`bS}mk&N*%U$hcxK_e!uWmim24!W23k55TMvC*3O-K^txlvHycEPe{hN34vi5M zgT|Hu_%m8H%-TfwbFDH;>zj7Uy}c~njo11GE#F3}Y9A^Ne|mP5MRsTEx(orC z)Mhh9;D6+2@QOcs&2avUIv2Q;PsT&jxKJ^hHRlBpund0QXe zgr@9h(|#j!uEq4goa8Vbs$yUnSQ^7H_KJoVG`O{$`EyTY1gofiAsW7u6O9q;=(l7q zhJTZA^3O{aH`SzGe%t(eU#DFm-*NZVg}Sxk6M5Y>a}dXUl{2jPv2JfH6(!n~!${}# z@<{D;Z?yX6!14+=9sb+9El$0fJCk0wPS5Mw=+%xjlb+BG{Ra`#xq#JeRXP7lq;lu5 za`nb)HIbT1g@Up;vw3aC4W*+kvkh-49(R`ECDAgex8vGaw%+$=1!|_Ws8{t`)9P}9 zky>O{!kIs6L@ut56z*w#$<1@{f0$m^qU)0{Urg;>GS%=#&IVeo>U$Gl79J6cwP2;# zv@~CYsJhH_yiR)*YYSRE{X5PLS`IZ(X>{m@_}jR3*R*vnYd=rj`uI!RsnIhx6Nw1S zD%22zyb;5;r*C{wx7k`GB4d%DrZaEKLT;8B61 zLaI`TyO(?sJDvd2G_Bbl_qp1bZLBT9aIo#0T3mL7^5rl1cS|~|Mj!;b?+J#MtNCL5 zQ1SAg-;HIKfd*Q}y3BWj&_~B2pW0&L73k4ggiDufmx zhAXE3pw`4ncH!pQvM(nw+G2viXLfx+nDT;P)O&E<4hpQrFO%>yYMvFsz{R70`rj1c zUM42WjQOVvRAz3E2~7DoO6bm$pZ~ap;PRjI@HC_!QZ2Q9dcsh&{gH}@7C}kJLkhwW zDee^+*f~8=+JxWoTDP)wS6zq7|h1?j6&MGS?c**Z%N7JFz5-}pMps3aUW8-w|M0!bOYI4MPC za4JPw*OCGtCAIPkIn;C2)RfAWM6bF~ER@jKxL4{#f7u}p{)Zt+$&+r#tc}!-?Yzp@ zw^V?D10E`>8C@zyt49(s4~|!K`Q923=0Ln);1h7h^UZPrEZMerJT5d+oQ@Ws*Nk*= z3k8Z@xYClSO#%to3-$VDA>t=WZRvx25jK&<#f~MwKrbncV_C1TqP{dI)XX=%IlmRB zB?Alroi_Wb5_HT zg9TountEa~_*?>0V7Xi@B7PMq8$hpBDh-a<5^qo(urlNYtjh2V@`5R%O}?DhGH~Kl z4AFHqa~2E1!^@9&h(_L^om6V+75Tc@F)TP+U|Gjvar)CGpcb57MpM5~Qe%v+?gXLI7HrOSSm}g4SA<4-fE;fZb>Ul#vaOB~znB%H$ zz|lc9sv8{TOPIlrualx?H0|`DOm$E560}DM+*b8<&jpdjDy7dhaw;3c(XB@q18inE z?kloRwN*j_X+x-f<#Fe_-G`p|KfMpRdX5X=yVojAYg72+FtAf3{(ulkzvKnOm5Ui% zO*%hnjvH{T8G|GOhf-*|ma`&t!PdcO&!WU2BE{8BOZkpVm2$U1Z1BxWoN*O9t4^<& zI4v7YKW_D>5+Q`>adrojO;6fci;ajHNNgyK?3}DaA)dEojo&zyB(PNmi9yQ1fw!58 zGjaNFALe9!P2-^!L^?xo7(6Ai6Q(xRq_~>kiEMG&D}FkgAvFu#a{A?n@gkIY zTx?g$hTH~!6t|;#aPh&Bh;Vj;bKH=7G$IutIwZG&AS@C;eAZ(~l0G@h&_;#>FFdN4 z^Q~-D4&wB+hGj*S671)}^ft4c0e(DBP5OQW7hoFwn7m#mddRIhI9wNF9KqtZLEJGaOS8~D9*$I4%f-&QKxi!ci`tg>TJw+y7`jPDPVxTXM@Pz?hp?+ROq z6DmopjXJxBnZ|9KRHPh;D*Bp1-Va7uXn;e;Zz79AfJi>0%{2)IX5Ny=%;ArpX66_BHB{v zbL}5$x3MB|UB!nqq&(yuLvDvlv& zZ=fFxKxkmgSkiS?V6j1O4LoF3QfuJxLi8HKITA6&c@%QjKQ2~F{HbRP{*{H?f>i96 zbc^> z=}hgp=67TwxCe$M&zMUXKq+&~<$uLi|G(u8u0CM(pZcS9??aV8x2}B*Rs9WBqyM+z zUqKF7Wx{#R*8t_7x{q5F)h>GPw}gC7qSvo!6rr7j=rT2AirX6S0Lg|nD|~(3>;Bpk z`%v&q_unD&MrMgZ&2Ey3yl;@#rqxtrimC)jP-?s|XzVy4aIj5^1#fD>N~SkuH0198 zWxVj;vucdTIQd6y?I^^uuU4dm;WhRRQ>~CJtFly`CQu+5$h|!v3G>J=%Bq;>>?msa%gIN;brS}=LloZjp>MgXF#xIwUn}6gA0v{vTx{m z;`gvG_5Jg<|9VpdQT9ix8k-~hyN?Vk->B18hK_mWGChLWwKmMp8EmmZA`xo%4bRN+ zGyjhl{riCg?sJduzWD z*XP@_c31JV+(4oZ;z2#%cS=nk!kQXUyp_Cx?@6nZ!PD(0Ca&L8zlBjkr~yVXljfo> zyIPQ$QdWxpK5br~JooOJ8@LyP(BJBQ{j#gyN+N8&b(vXcrpvID9&H7=KBP9nw3?q$ zr&?87WfBY+irxzZNf`mNn`!C0+T*XI?Poel-g` zZL8N=&Gh~@#o~A%D|8}B-r52RT<1laU8SJ_nMuPSiJBNk`+eVkkj!tNQ8ZYdc8ta% zP(svbWW||_QXFu(kQMV-iHo?D32VHSVp0^VY-WKKL!7@?d$<2}*9JW{-3j#H92uXG z+Go?&!w(L-mWp0TQQ*XNA}djiagr6em|L|%6P?^dNah&viA4tA3K@Vs(l`Ola$%S3D+IQgZC}M30QQ+5$)<&Q@o_M}~{J66%WQf}u8#_YCk{a&4l+djriH zni;NH@R}Q*)(EK)Q$ zE`F880&wMDNX?3FlFqF3HEM#5IaN2Pg&!!aEB59Rj#r*VhgZf-VUImYcgN<2ZRY4P zo1{DIBYjJk0*fDUv_+?$OxuI2m0tzMC6>F^<=`oP=p^=X!8;u#kS`wgx;HoD6nr z*)s~3*^59%1Ok$^N`%5VZH!@=BY0TFyP6S*1)w{B?I|Aw%|+wL70s8Bb)1aZ<*C~j`{t=>VxSbUJ?{>Za?^Cj}Wi!ys z{Fl_-{x%egv@!B9cK73rzQ0WW=jnWQ3YnU4b7jWguv6AuSwuaVn}M_z&R%F5Kq?|! zOvSTK5IA3e^aN2FiSnz3uow$mqVjmby(&gbQUcUkVPFNu2H)9&Dm7LNcZSwRn}Mba zkp@rYL3{t6OJ^pNq|abBCj(Vd!vo1K_YngoV&k>#6fj#8pEn?Rz;E=dxgp63?kb zu_8|w@iV~>8O$Jbm?2zYIS@wi278;MtEYLJ46r!+r<#qd&)#1F+;S-uLb289A6){26>+MRn?GKYwz z1_c8@Od{#3nMWF)Kqysd9b4G&ob1+v9jHsV!x96K8ckvn3l~XZbN0mqD}j#)KyygF zmuF;*oN#);K0{}bazb--n77s?8h&QzNduQx5R*26@>f(LeW4+8SFKRJ^n^BIFbxax za&CMKnpQcV%N7-Z4YvKG#c3_I8KHX=JTjQ%;$a+$2IgO zsfU{VfAByCjES=aRC1~+k-wu~ID!-nHi#{T8BOQB-+z-L8;9+Nh@4j7&%eVznD2VR zn!XvB5?mNN<{s8oLDlq2S9xc6?E*jOM7wRkwxB3d8232MNICP)OQC-t$lh+6!j9YC zE*K+`-tBv97pmF9E&g?zA;5a<2p)clnQAxPS_lf=Ew!4- z{dNhg+U-uAk-Kh!#Bb~Gxst*4@`tqfu+ zuHODQHsC0EJmxx=F6OXt)l2*=D5<)}0WOzNXD2;)f6~xau)Mne%|z>V`Q0-BzO|Kx z@?Jl#;H(62=4)6gTk6vOw26=SusSF)iLLDG5GqPnGhS598QeTsX+9U0fZ|f~##cea zdBV)!e(ikl|GI%A9n0A6!J~bD)OAcWJd^NdX$C1*7~W^*4)XU~s(0ePrZ;j-W%Z)< zaKYqfw=NXUu*Vd9LqF%rT57x3hXy^)A5>wmFFih2Z$z z3~!WPhRvO%&H+ljLxR@czSl>fhHveAnK27&*LCRXzuES#gclprFpu;0%Ih*OBW}#n ztex&hvix=FT~pSut?YYIygxt0Ewz8LATnFCSGRa4_8mL$r8>#aQB1a%HOGpO#5TBL zDByLn>M9;0s;6AaFonPf_%c3QcCD9eWEXHUb^FckXR>S~q2<``#*oeQ3sI++Ka}Oq z%rb7_1fS%6g5s0I3Xn3$S2VH#N`vkO?iBu|ZSv0=|93ka@_aBBO0s7iy7QQuK}8{y z34;a%GAfm=8Nks)yIH&x7}=A!QqJu*-Ji3uOlsvCg9dL!YjP}#7VB3D6ycv_uF2SA zcCielu+JH^IGr%h_Ko{?KfGszGyJn_QTJ$EUatoXroRmY=*@kDJ(^TBdy7I9>jRv( zjdgGMNvb!b)aRNSdHM{y`#Ke{dyKzkYKPh!15auROBL7o*lwMR9H?$@A@~C5r0=>S z5o;VAYgYN)vTjmtatk?tUrTJpcTCfV(j|hGXE)Nm4bR=uzCSO`B`^OfQTQq5F6x1O zFq;Q!mX$}kFnO^Ojstq~x~Cds z7ZLW=R>7lBRbK@%O&UnCI1R6Wc$gg?w}mF40p8e=-`||?L*)8m^Znr~cB7*lQ&g}( zO(%$)IC92~^>l{882mlI;7bRn=rNajWsN~WR0G|DH#;a&vSx%nNcE46q`V2}%6jzJ zQw4unpmMzt@Ptg=Gx!obBY3*$;IXqoZO@W;48JbkO;M@=9%G3)mL7K3S>e=`>ip`7vU0Y+F{MP%!fWs#CTWxM0QC zk^1agK8=}nK&`bSgDi(JE=SxNkxY$_;iXCouIe% z62Xy1tS-$mf>2V-XtvA=VFss|Dd{j8-brP=z`(96iw{G)5EpWXCakznxODoLrHW0d zc&&<)uNm162n~bF!1oBO5~Ym+(n+2dd(BgAQFIB&_AR=$`8@Jv%oQn;bRTW1DQpaJ z6Nz!uxI!`Uz2Hsk*s^B#-^ju^tjKwDrDP#u#)h0tY*Tv;{zdt{=pv#mwbfVsa+Cm61L>oCiDVIfE7B zy{R(|jc4@IXf_igOh;fI^6YaH1UG$Ym>{Fp384}Kb z&gc>7+No|{6@8OssL3fwZ@?0Z&U4gClv}41ohAQY7 zGs1(fcEGk-`Oe8d1VAN5^k9`j?l$<@IWdhPcU=~gHv3uoLf$#{I6;1z!7ZI6M&l6i z7fdq!07D?5lSBpbEd_F4JL;vF7}81DcQ$1Zaf$>E%{Xx#6qivl4j?qG#exySkHw-H zWHdm|Y460tK}fx4xDaoep3@)y5mBvkb+C@vSuTeZ(B!OPOL0ksx2|YsB6;N>z1LLv zQJ<3im~83$_l4P?$u@tDwl}m8>W@zA7J_&%N<&CZ4N#?PR#rWpX(~V<$s_^}T)L~s z`>`SW5`v!Ji_X(Jfk6JTT^n!SzTN2OplGL-66%>k6!rb^bUzQCUaBmSzWQgn<_y&1 z%NFiL!lPX~?p^rri@oao%po0JjmISE!Ww#MngBrZHUBPV7rc#$+%@)(l86nHG{dx! z)u8XD-BV4g>ioD;T$gfh`_|tUDr>Kr)-j9FGN};?*l(P*pN7#FCE_hTmI3Qdp;?E_*X#+^+IF zWBzeV{Xk1Fq>ku7D++=y%+5Y#O2+y*E0DFu$fA;!rdZbGg0{ZEPGT{s@JNN69`cXf z^Bq_nrAY(qphuZ>zw9^dfL9Gt2iPpAR(X-xxrMe~0Bwp&U_dqsA#eRcbjK=doN*KFSQPCH(u!hy^*WvC&nf1?R`H6KMNtn>L~=lE zsc{Nklt*Q)vq;iKsoP9LDr?ZzMKkG$wYx4ta{qyro`>&XD?yNTCu}x`10#p#wDNsyeW6YuqF#P+{C4E2@a-oz*I>4GNr|ygV`#>7COk(aqn| z11+Bf5moGtR0b&o3HBjQxkwpw5j4>NE20vu#wZ6U1Nz8|=E}`8lF=`a1LNMh7i!(9 zn5uwuQpC+Ul7NoE_F{BKQ?Y@PO9kGI!9mc3N$_q~vHY&d77!J7PHZO&>7`Us8Ix*# zV`M0E&m#6Tq~j*$EIiA)#ON0q`4A<0AiT8-wH(bz39~Q`#8|z#n14zbtl8*20ibiy z*}-EOZZQ@24q+NO2@+eZXg9?fG-xwHB_W|pVyF|0wu&C>9g_?s%QZk+SjG<|ZB?HV zp0KRRI_1er!v!y~tzec>Eo3i2a&FNLebBJr0;SECd7(swF{#BcXR0DolJY<}LuiH9 zWb;4@qH4FQJ!UAhlo{@NQfJ%4xjDV$UHZbvbO@rtGbFnCm|k{VC~=Q)j;6XUI}c%s z9T}QIrVnl|U8u>?Nh1vJ#Dka7qE!w_B?Onxv*KUZ+m^ygL#m8VrI{7vQPOdQd@^vl zhozc;P|o$x&R2}K200qe7Z6o$L~#v8ZLyQQ*x_4G4=Ger z=ch@w_|^uL-$HT|gPG}_fE$KB0$UuW=l695Xwbz;E0&uLIX#Pv?3PzYrubO9cTqv3<)Duk|d@gY)mQNz?>S3VM%2ddwoR$)ySO{o`DAWxxhIS4F_E|; z!tGE8@UP%M@f&;w$w>gBGkotQU=ag1gF~Lh*;F>!!O+7Fu93IXJAm_tT0PPxY4+I{ z2Q0`rW%z5@VXZ+ogqecq#fv4v8ppkDVDCDUwb>7OK+p!=WJ_};|9i0mK2y0e%#6!3 zPI5$@cLbZmMvUdSAc;{wG>g=RG@#}?DW*&erRG45@Na`lXwfrvyWzv)i*|htw7d9N z$AkV+Xk8dtB$Gk6A}v+k$j1`)U#CQ{rW72a{a%i z)SLzk9pLX-KVGNN@(6uq;0wG28^4Z}I(@;yg?9R?Le`=_9!q1ER zxu{3K_Ie)!ISzdS4k}tzgOT`A2}qW-k9~yi;sK4;E{K|IcI8p&nLjQfMwcj7N|u$K zP@tAo1Hy1J;l>CmAbhZf+JqqyG}M7|1FV`+i15Dnk_C*U5juv*f$c*ivb9jf!buI0 zUcm|;y!darfwMIzQMaQ+RKxVXB5O{ZYB5t>vwuxqPV%@3&y z42n6e{c(Y=fDBl6GCpZWUX75@^MCp0gOeiLu3?rj3@jv+c_8r=fCCU14U33DgFWAc zvDb_U=LOBdPjjcV#DMXLcZ_T>r&i%PgnfA@djea`#%&(HKj2mI3g-P;L+8prq)V}MQZu#xA2%yCA}U18WNRbmp7lOA^)!B#LH5tPba&7j#x!*&7ey9)XY3*er~BTTP#j0E%iNe#`U zz>;>JX?Ss+`#=)d3%9&M}tewM;U+)hqBJY%5;{<5OqSY$=s>x z_VuF$b*ieOp|YYhzdi3f40Wnub+s2FTzc#z>B)4YTy_DR4)w)aEj z;?XQa!O^4#kN^$A;=_e*a`<6HGt0n>vA8F?EPnXnSuNqgOTPN8#`Y%{TglR%)9KBkdpJ18TbN zqWqlfR`AX$*2xV$j=ISc;LWG8LU;Avf=!?` zZFX&u6iDid%Cv|_^!UBE?DbQB(!(YMt&Ookk6#YBWS{FYEslZIoO{I-5{^~pv3Da( zwi>NPQ~8Mnk@896)E_+PTj`A13>HyhlDy>n-r_I85 z3b^8{G-4@Ds{s$W5`*0E&x52;4p_bZC@i~md|xy>RFEwH$q>^x#Is_nH75B^$?O8J zCzsYU@}r?&BVB+xf-BdCDPg)cXAk90L;wkOBiB_gdj@KT-z?jJ0+pd-_>y)ZQR{(~ zO_3Fqy(OIrq4Il!#et?g;5P-~<8R&`u#TW~)~R3QKC}P`G!*?H3_|rq zyU>ReEjEoJJ9>mx;dI(oDEo7Gw(@C@4|A+#!vkWjtUNfg6!XHggrrxc7pxf-nxL37 zFnigADUrdmsm+1!3~)z(TxKG@QkG!er8h8a7tkE8&T!f2=`&2ZAcHE2*2u2b8pm{3 z@Y>v3flgx2@MiAJKj}GkxmWk1HJhUNK860Cvd9^LQ7NtP-R}1a!Z)8J12IUn1(BMI zx)C3Ch0xecD^T(ya8+mz1rgXJTu`b(r}JYozvXAp&CdLZ%#eM}G(HrE8I=1v6YRjp zE3!uqHb}1|W4zI}mW2&CGS8&2BBiHg@xkQE7R%sfE|-A_MjdbjPcG$P&zO>5dFH9?D3g!e6pq<{aPH$gW>+NvFtfR?SOmKbsCDN0Qm84}rl zBf9{cN`u`(7$h>A^ob8tnYN)By$A}qw?q@*T=k3+A;Gds5!o3iJ?#NY8CRP;7O&lj z|KDvwXP6P%7QJn#FRYb-y-Xu9XWSh?n+~~PDCNfT3eCJJHk8&5-rRLjD?jEUQmeNH zjxe2owY&lz4G=h7Gu0bedpqaXSCu&#+OYrw8s32VC#(Ia)WJ3dvrv|pAv1E4qB${B zXwa612BJe6z4jp_qK@IW`5wHRFQ%Kd{iiL4dbPLFiP!bILn4vAP;Z-kOl2LNgSxd^ zhPBd_OliZ9m38}NR`+@0=E2VzX`owx6~FMBs{!lAC$MU7EjwP#-T3^s7O>#dwS z?6OAT|7<~Mcmx<}*vd))CBXUq@z z4VYs)QOB~L!k+Hr^0b?J`!?5)pdU{7#JBZdQw~GDPmT?q?b>clt&w>@MBhZ0Z~c=5 z^_1OBZDN0MYkP6X;qw1ANQ!qHb4ddU38l@l0x z;dDm}50I7B973d6wNb``AoAgkD!Fyz(U_5%TZ(TR5kBFOJKW(HsZ$O@?ZD?fY*nhq zJH$^w?Hd;7Dr4mkbKOV_0^lUV{w~`_t)e;+QM4Giv{JR2!66CCKo&2A&6Pm5Z>^@N z*GV>H`^mmeiM4C1^k~aHD|RTm{;qp~*laT~t;^Xi!rd%SF|%=_$9%clnWZOh+b#Etb=L=Bt+*^-)dH#w z;ijw~W;7jKKlt#=j(>2YmK}?AA}l}f_?l(Rog-vSuUyIGvH`t|3y9&42bpAL)bTUU zU`FSgi#$xUMCc3060`#!LFlhYq$`*JRirvGX1IeN7e15$C=cU+^Ss+>Vln?1GU=`V zyes{7h@UopF6Q6F|GIvmpA z_0uXa2xc_G92T|X*MM;4cS=;e2{93w1>T)7DSy{-4Owk{Q&`=5O~P${(VNtn$*9UF zrFm$O)uj?iL0R`$l2n|*(-Doy6_IG)7yv88DJ-n&$HYjnC4*uyTpN^QQ4b?)rlb(~ z2wOCVx&~#HXHooL-%jJ5UOuXyc;a?3cy)`WGPOl!W)fv~2)c4u>nHId6ByJpmt-+c zv6gti1#oukpul0BXAVGW=p|ekOAP;axiBCU@m=W%!}sxOL9kgrAHsn^ENZ9aLO*ro z$cQQhLsz?VO5SZ$=k*o{hhAeILm8(Fg`hob1jEjV_L7xNgM2B~V4QG3#G|_19?WSy z>{PTlF@P*(Cvl7fzm0jN9<%wj3unKk8Xl5fga^4=!KFf@fY%#gxd=pG++2>}NLc^t zhQPZ?Um~nz2lUZ)Bh8H&SSMQBP*Du!9Y~m6Q+1%*YOX8X6?Y&{G-dW3&jlVkdC)oI z8ClSga|#{$V4R{oTJComgmDaTSnfu0WX&sWB+m-&8Wx`zA0-_Q3)=?#DU|k7r8)zv zrXp~a^jh<0g^+Bey%Z20dw~~o5RhzZ@;iYNDD2I$h<6@xux~)8!F*&Bq+lT6UAwq4 zbmW(mRZR{f$TJd-MnnUHmJ}$TNQ)ZYg6==obO13d$@j2qXjcYb+_8PG0O^lhSCG9C z)GO}Xr5YqKh0b$jFf)3$&qrLp#Xf&x07i^kREu;yC|pAUK~)?pJ%ru?Er*Kvm99W_ zG;%@IL&fkFKpFThIgsBR0U1vMpHTTFycw0C1DQC2J=t&26YS~^bauduPHeW4n*P;~ za{puci4(9_CX-n#cVVMc@3rx)zX<+cnd|*;xy{yB1ocmTINb;IS_iG0zoW2!MPZ}< zZ^OSLya=i(;8#0m(gPslZyRnyhAdPVYA{DLYCym(c|`w_X-zNu8h5+O%d%1qF#_aZ z6y)awz_wn6dJcV}a&(TYlBIjgyN>n`RTeDixN@&}`koqM2(B^%HN%`)#U(k6X}O+M zm^4bpuv2OX)|6Yopj(t*s!H@M^*_^lE%vU99iT|D))0YO0%v_Q_$B;!%BBagEaBd$ zHsr^7jmb~V)PSFet4|;8^s63ahN|WXqHeSp>01O3ahUfXQ(E$)uL*_uFE$izk%^oU z48|cHED-%+m8TmMt?>Fja&6w~^To^jHAjh0Q$lNVJAt(zkAcZFeJTvit8lP%Zne4q05Z z(LsjIt#7z$w%Uns?6pQ~nwOb>1;%D|+L@Z^LewNy^bcrBl%5fx7b}~8oa|Xt5bMoW z^IStE4XGVuE8TE)*&)ZkuWF+|9X0<{rK$MM61t-Q=^Oz5hz7^mZ16U6hE@8nO(7t$ z+SS9Bs6kC48!%|fN0GYKa?>Gw7w5GvuCTjxK z`uVF$tWdR?%aA1Rp_`p3Ocp}IgJwj0EL(kDIZ!)%oN@ovln{UKrL_E}z$b3HHQW&{ z3m2oZ$x+G;O<0*@sasC1dw>GSVu{|l-Vn{N`xqk{(z#-V0?D|F=#i0t8$}-fab*=Y z&5z8i;>%i%k9?a|UlFX?9HiUp0J} z@|jbAlGpXv)n}Rg$^i`K$=Q*{zUupAa#3e4TIp`G-uRXBE>Lt;I5ntJg<*?ObFk(| zsSSqkZ2rn$d` zR;`$+?;$^=IQMbXmq1SF09Q;c{Q>kedD-oTAUiE*peCn>H;C_!fpDG`{9d7-UpDN+9)yhi!xE#pVf>gMq9|<9Wu?5b zQV3{jx8~_*&j`(2ESK$_+Z}CNW>;;wK#(+|d137Ou`RR{*59uq!DKRSt2s@A98m9p zD4xKoPy0; zXRPXx*77WE{SEhViM{8Q-RBiMf!?d^4Yw63htknRir*OHW)LWBdG~RVQoSB?I)fx! z$U{*sIG1ZFjY16wv;0T6utVlLF71BbtCY>%Z7> zvJ#Gm`tw6w64Cc!bIYP@5ZLc_{~}U84x)NY*+TMbJ9ZrTAiSbylG0j9cFDQ(uU}e^ zRq6a$dIy*CTf6h<>|Jo!U2i-EBQCDL5+_V|Kt|`0I%Zg<(I&H++-G%S z^>``-VVpDDC%S+c#$V;boK^{&01iKWDSLLnXrwof=`LxcvLLEau)0>5FCc*X?bAS#oGU+~OP?Eieed);|UQG4gM7_~Wth z*SFB8$C0+x=J%OjZeh8+!uj`?S*%&x93vb9d%V8o>-_!~v|nP^_sWPMsOzG) z+#2(&^oe>yurT*hmV4*=au588NRW`{rD?c5N-FV+1I+fb0~oTsb#6sf6G!$tz1@NG(a} z&RtsutaOW^d~Lz>U(a2YJ4H5S1%4OtH)E^9O=-G}r|Wo*&11{!Gur&Lwhql5 zq`&OM9NV$$F|))K^QITEId)}s>MDIoA-f%4<0$sbiOJNfs0v<%wpIZsUO`D}H)3f5 zN<31FZ09tRWy`Pn|vLX~UDg2c6lM z`0n4>hW9_&hS$TFL|!#~Nz$)Mbw1i@;ZHk$`1HrgNATf+KMh*Rfp|Vg5=e`Y;qANkPZtPkyg9DNAaylW$5H;+96jz|_n672^xcq^zNtkG8UXr7g~e z+qJ5%?au0bqa{0AYOnR`+7jp0@7gr`R%6*)NG)KW7yWKHm>U~dUp;WQHWHt!^^)DqqhXX3sbITJ*l3Dez46rZxH!%zPFxFJ7Lj!qZgyaPJypwXcn$y*NKS z4H}BzO2!&vRjUf%keB1Gk(3bB^|Mj8P(SRY5%N9dwA5;y=0vgX)hf<=hjbwoV~xlW0&TDN_`8%>N-NyAJZC&B zW-E3j4j4?X&;H%ePD9hB^clQk?xf95-v@*5dsuGkva8oa-X%=`yO4LgLf-8@=y zq75t7SfNYlZ5-}qeWM=V(8Eh(r+z3|rpU7z>CpJsH{uAxk7~3rt9S<127M-1)aIX} zKS^C9B&J;#*C}ii8z<+LeQ3Wlh4tU5gXk?Hh15QN%T2qLEA@Hkas>Fe>&IL5Q>E!Q z`{P;t)ca%%kj>5f@e0;qv{u8pWDIGwum_ScpoW=ODAU?o`d-Nxj!ZZ2Q!?=H9Vr} z1Ij3sT}ja-U0|3Ss0Kx==EM2qp@+PaXIo}r8Eas_sp6FF9@==WE*o7d9%eB7oHf&4 zg8W%>sckK>x6|;=FM~IWw_!MP%kEzTfs{2GL zL1%Wqrcup9W$6vXq|gj`;NLKOV_4bT%b<`1*^LtkVJHjhK%Nd)p-14Fn_17BA~Dj3 zObqJwlku6IPNs1PDrttx%%{&#soSeE!YtAAI1q|5pXqovOs#PoxI%&CIxf`nU^vjE z=K&^h&!A!(I@&#<3E8VSl;lggjEHp{<1(=)G`C$%beut9iK(PvTo)^jbIlRn_Esa$LgC*2(?#- zz3unm1W$w2X{b0*!8opBLer%u{N3F;ioaV2Y|mnB5*L()9}F9i46V_X)X1QLRrB|K zAp_FaZN{~n^OuB`+f0dUiszB9AadLYytoO9m%D->}YzD@iI z$Dg6K;pUiu@^6lws0JT3`d)5Fzd{^6v7n(}%_l0k1=HC1*|-+BJ7=hc8OIkFoTNBi zVE2?TcK+Qpx$zhU!MmRGWRKg6v)jyota*>yf{onU_e{f)DVZ7+zqwXLd~`qs8&{fC zXgvDZk94?k?BYj(PjzOcRGG1g?PgD_)Nc3mOV6H38=HB{UM?zx1q%67$v$O%78*#a z@ERIuG8vJd$Y>r`8v*tO-gg>CN|-{NBx0s{qMSqq3*w#)52;`274(Ds0NTsXFp$K? zq28GJiz8T@^Xm((Vg}(op{0j5?(g+_3nROI6250|X?3pT;A*fr!=o&5*ELg{WP41X zq|5VWS8VAwBA!^ycC(tW06gA!KQXVcvP7%8tReo!R%3uB13USTMzpshY!mChTg{4& zXBjLH(wHtEWRcO9m@HIr#}L@p)_IgHrtA)5L^9t=p=Yv357}dkUEBbaPpLjQ2rRnW zx!U#>KVyi-%Wb~47CZD26a7`@fjJw-WMSUbMcO>j#jO7s zFv1G$i}Kpkd90+uN`RQ(=O?YdBfT-P3Ll5@(@=|Jbv|90^ItW8oj6^hw* z%Cqg1@7h)~$b@>YSm7kwvA(cH?X>JZj{G-jwlmS6E6YRZFj6?eNUEF^J1Ei?v-V}5 zVGV1xx8R0bl$$5o+8#e?V4L;(qjz4}{?Sqhj9t2*C=I9DHv8_C*>^fzzsdL5VbuB4 z9EY_RM@3wrasHAB^FoVG-nKXbVD4I+P6&SL%aHhCe%gnXG2N9Q@H8CPx1Xu`Sq#IK zef!x&0(5EE4R@c71N-Ujy!>=qA|!fW;tI}U3uWLo`94G7X6O(PkEK{as*XXk&v%DX z54+-VOW z22H0Q9fIbvPGKbMj^FHQyI7%=g6kXT{Yo9%Lg8lUPJx9!!W2kW+y2967uUt$?)0## zizY`)dr{B$WQ6Gt>->8Cy5hl`B}cdSGqsIy^EY9i*ZVP}{W{{6NXIGzm(gvr8>lf| z?psbpJ;Dqb9Im;bpzDU#`2Eb^Jdcl9hSH$3l_*i+-0+XhwOq7N|^3g@&X=m3-$*WN;G9b7croOS8H{uRXyQ` zR4O}3zVG1x1*#j$e6GZCTvyv_$x{j{J^Lbi-TEzIdE~M{knC;y&=`YPwa&5Dd4h1g zI$8>5UF}9`B27-Xq%|rnb!pc5TM#AP3rk2WxFn{w?w=}DC6Z`gut=-_FqUg=8^n^{ zu14FE&+DpL?UOh}-FoQBsq%%2r*+J;vKpV+us1sA-D%Pt68*!8+yp|BW&qSQvulZc zUWupRI%V+{sOS7?FW^4^KYMSsBinLhiT#q#!=w74)dB>e06|ZJ9(?R3npFfyR-^mx zWy{=sPq8yIA~P@ZvhE4;o{ZR|59a1JE)#hJ=h6OfKMP#;gn=)Yb&M#>9Fm;xuG!uc zH*h9V0<15D7cReUsmnD>E~ zpZDjvZHM`UCE>T|t+MHE=IaK%#FV7ld480)cX#zgRl`aT3x_*`v0P}`FpBJe%Q7XQ(lXN2xxtTy9xXja!(;YzqU#@LRy4*~-9gc2lN5*mw3+I@-6`PKI7hYY;7+ zvov*;K=iV#L9*aG%m#gy>}FN6Wlx5X*8ooTQS26IU1su|sG?5`^r4kZ*^t5*!B^QS zx2-cIyXfJ?IJBfKpVNN*ks{1JMcj?Y?8nrSzrK=xbDxTF-{(^?Vfj>$u($K}7riOp z!T+W=Mqj)!`s|JQv@gaA^KGL>?BYCX(drQfEmtzkMvtIcSdjFz+CiR-XWjbIZW`n> zoDnGBBcu~M6SyMlBAXQo7OI1uZ?oBLtiBeX1Z>G`e(x-!uh>}M@Iq26pdI0V@DM!o z|2=CoXKAk%8`6h;Bi(Lm@uMfy4Qt-pzEvmFYrfUP6OS_qz2e>XnD*(GPwocW$2;#a z<2^wg`r{Dq^mKpj$K0iZ_t_+Agv;`iW6pT=PtdVH)A2|c3r8w`pw4@rlJ6vIy@*2WKASsNIvFMr!TNN_o$dA7Xjg(;KpT z(UTvMJoB_CpM;XllRtaFI`KRAC>KkV5EQS$!A)1yC*+3`)N;pZ2i0{=4{S){8M`JcUd6WzUf?vF`=IGshy8M->G7RIce*6Syyf@SRUW5Tv%^6VwZmF)@e>`hCP=7darl<5Mu-JPol0ZVFssvb|(>h#oe>KZ_6F6Sfm zJG09W3#?JwH~lWf>7$?Gn1XtDHP?gdd!id#O#hyy~d%^}biV08#H#UiByasgDV4!H)zs00Ptd^Bdz`wYtQ!JdF37e!Dl` z!8Wc6Z_=xNyZfZA6q}Z)EJ;_$rd+N@sp4F>D4|7iJIO4l(PB+%nnZZWoVFxKo1zv~ z_J^dcgru#nN$;P=4j7+z5(sG$O$jLzpP>yt=gtsApzp4A6Ug78KIu2%!@qie$W@WJ z(2-q|Gm5~MOtWZJw+Ir-&9y{h{N?~W1q(MaA;0p;BpX=3K$`4P!B-&=-+ekU^W~2r z%_oz1`e~RwPK)^KOo=Fs`jaifU=#`0-XjsAHDKG2Kq!fIRj+&2rPywerD#h{g*Ki{ zT0^y#AGWeG3^96QE8VmzwCJtDBg!@{X}bV8psL8FrlTYw7>rEdnOPn z0c~gh<*$GG%U}QW_y2XhiUhAD_G}>>3ehL6X?VIJjW;I|eI}g;2*RFLHB{2(QmKO| zAM%P)DeZ56`#=BmH+1*^{r@hRMyrI6ZqF6-8{li~QAJp&d4!Dxtd{N~3#7A|}N#kiGZ`yzS=#y$;fIU=`rB&FTl&ik+ zBXv!p7FE+pRxzoPUIx|A2ip5(F0pso{i`QE9|nkGb5bhOuR@||LzZHwq+Vsja)-01 zLi{_EFyjgd53v!cWdY=88(Y_{x7uTqBn7ZWV7v?3kRU|boe8oko%vz^>t7N!o4u^# zL41){*#*S|B}q>OP-7KOaX{4$HJP4l(K0C7jD#<}mljB9a(JtXEN@g3e_ygoyNWK) zLTTS+_(JjH%=8b18rYEopb2GzK}Wl&IO03C8dV`?ooM1hMI9iZk)r+y$umeT)W~^# zRjAS8ZK3ArtY&f7GwxNb|LYe<{ZRl3|-%-sn4q3(rnY!wu( zZc$0srx&>>bJD5>6#}u)N(xE?n>4B@=M@RSJq0ha@d&2Ad+_C)tE^9PscNNBDFar%MT3a9=-`4gaBo?FzM$e?XC2147a-~0@>SdbqRi*lA^UC;qUG@4w(bv^p^UtY%15xQmRUafJ{*dTT zya(P<@nfQQmR*s6`a`M*+yf-H_wbXdCs2fb%JiUf6FQIr^K+&L(n;dkOstFxsfpY> z{Bj~_-AR~qff|^cil$^s`)AZ7Xc*d>3&|crx27AMl%c1SsA5TqARWgw>9gt;C2v|x zPLl9YNLbO8q?j55QDUXdfixs1t*&%!rgxiAZgekk5?>ze#rC1LZQLPblS82XDH1^v?N+z43sfA^GO2lic61-IrAXaV})l)H!L8R+35cuGt0O zeV0iw?x!^*!BRU&RcoRKI%8ueDN(#C)|dwg?{Ah414 z6F^^ik%A*xtM;Htqo-DCg)7Do^`UcR9E?%}ud@vy&?n2GB?(M3%}Ym=t` zlgw&b?uC6JBtLseDmQK2KxYr3Ar(@oo`7LR^xSh4?PVQUJp)}fIv}e4=s%=;J<&x2 z0Z!4sRhH;mlMbF?8a&c_NO8$XZjdRK?3F~m3xEohO{7RENkaUqID+a01iBE5)5qTm zd7AhH_`G~k#A2T5VkHjbRB_dkoJV?KkCoIX#9KXxo1k^V;89*HH1G!i6CR$9iiGl$rn zZnQr%{WAQhc_6stM0;j35Z9DRJn_VFZPTO)L3uPTYgLIhG^zOs@CP*`zoA-U&^byv zlQ=W>#fF|3@SXMRZ+}mJ|M$P04wZlZ*MI+W`um^$M6N+*IRG4;># z(n8t14)a}n^3cOVjC!8<&u!g6%4S7f*?HnWrN1>g`G1%6=d^~*+OW?4`7e21vBckQ za@NkP$16_HcuLB3#Mf<2-6MvMn5iOmkD|&_hZKa8!6%y>DF~#KblCN19D0xlVMNWd zm-oAeo%v=_d<;d+inFM|EqY=bAxX+q(hx}R(loIuIokf5CE-dyexZ%C5G|)lVtZ>g zRb0Mwv<>j`$I_00I}N*ti0PYEMT8=IF$aAetkA z-VgKym=O3V{iQW*CZ>W6962alomtQ5I4#%a#$A5lKXlpXar0=j$Gfzf-^Ve|0UpvEQ$5?&mgBLuaHcW06=A zl1R@Yg*Zu_iH~bk)p5Qc3^l5X6oZm9An|Eo0aP(XMPp230_2Q9EWfPG2}H#{KMI^- z1=NHsBhz<5dxQT%K+kO+hCSwCa7pX#VTMUuteW&(WMjBW|202KT+brfuF;idl(a^U z2K|-pP&ymg8$Kad<8!UOW@+5zyYgIvan0GBzl}|#;%&uqInG(hwhHRbSnly%qcldg z#QMAoKwAu16SbyB6COLj&LrIKx;dR)3#3R&Pl`(N5~x)u@eL$S^Gc~;luLg8YUCE7 z{hl$AwBB-hR2XuC-gN>BMN_%D&vAW17Ybm<^(#v!wHac1wP?M6YxdLNSyESMG|Nab8n6OCz^Y%bbCMd-`0mOQxRkM z*jMGeksD6^C>}XT#ARAT{py%%%PAvkR}%k+-LU53a_=dkRJ&5N@5h={vqe?Sg}5R8 z&H}yrOcRUw&Qbd6lejFqT_{j5j1!|ei~_wA9cm!vZ-sz;_stnoeKbm;M3a!s(2Qmj z8@3@-BA=Ffq6d(|X;f-Kt6FRXI$iWJ^tf=D9?Pz}QHt(0$#tRR)>Fy0bBVf#Y%bUO zrCbG2(2IJx*N?Ja>)z)J+i_KmCkorq*7r`yMFBnHrm)?yq#a_Z zUN334jU6zr6qgyC&mAQSPkEz0L4&m;tCulW?UF?py>VoX`hggBY<-?u-ONt(Ba3w4 zPXXiW>*XR*XJnwej@Df8Hi6%)hFl+|n0R`Nf_h62<;(=ab)GSRdy+z0szj*8M^CGP zT7&3LDU(BbZX!t}Ih9DTQK%T%5qFxB1(lSlU$$I4RMGlnx%g$d_+`2Hs=o5ea`8_h zq5ra6{IXp9vRwSKT>P?JoJZ%E<>Hs+Lj1B^{IXp9vRwSKTs&pD2&S1vGJv$jpaic% zF<%3aJ6lPxA|a+@NzlFM=q;Ary$ z@SIxZQUof!bCIZV)Je%q4M)$%BEwnkQ20DMe82S71gwVp`V+GpyT$IxPkNz!C? ztyI=TtOc2P$PJ|p#>HX~1=O-kDAZL&NyPl5^Ry?%{A%F5eu(Y=q)K6mPd@fspXg$w z#cLu;9V^;u8pf-X=CV_^blSRT+>+@nWYK6fR$_akAgJ1HBl0Tg#V*DN)8S3?S0Gma zg!ADyF#x}3K0RO8JSs^$YbE&!QOYROUTLZKL>n-vh*c!gl4Q;y0Vjdjk@H+b2vgZ} zdv!H@XMAuDv}(Km6(9WT^X})J_+TVYmJDP`22aYGgrV#rk-F>!D=J!7Nho$H*5FmE zNqUG_CeuErOL;dw_#Qj#x6c7P#|LMq0?of3AKZE+MXoj1JSje&w3m{V7KxO;R;hVN z?nvJBC*y82p_hIuSW<+U>O_NDru|65lf}x_6ogp$4E;d)b|nwfLZX%!)Ln>UH!l zM+m#OT~ODOYzx%#=$dl=<*vNECO~;q0C6Z2>#?dgJWKFNsg6a4|akO2U;p^ORKW*+6y}6c%ijIz} z^iDy6XTcqgWy?PAI=U@rHWly*m#SE4;!0yRz!B@1UN|2r&Atv_1Cm2u{tdUJhCyt5 zst@O<26OXNdQ~)BK--?z)LYQ(VSD@NGaYMpaO+<;G<^)!eglfFyFOCG8M1s{e}o%1 zA|K{Sy*r{t$`PyUde2&edr|2d}}1vI%bh_;aZkGm7sSF}@M+I2FzI@F>7! zsK?$CHD9M!f#{8Q7^?J(CWvsQ`p zA`$DWHf;H@8f48xB?$$EGO}q&H0+`Tok>CrRSV~3qtQq0%??vLY3I2Rn*neOt@vl` zn#BOLR7Xh*U;9pqa;&N;6pJBwT9;r;uYl8Wo#-Su+DD5_xik_QT6~pIrLqr3aH6=L zT!^H~0Sx1rHWPPurj*iS5heE;b5iI+cHSpx3y>uQuniRLP_@y1lDe*%G*NhJU!}Zf z%+1f>w1Dy|)a&QESW}b5W|~Mu_cR%3W>9XOHo}ft2sAkmqTPcinsj-Msz}uqooJV%{jjK{oh_stHtS6m zDH)P1Xf8WZy_M=)KuppzjIJgkq5|ie$N`lf1+n%-t)qEyv?801H7ns23LMCc7-Qg{ zNyJ2`@T%$m5#QrkUVfRY4ddjAE;fX{&~D{YRt>IA60!hlMeH3&GQAT+Q#$ETXblm! z1GH6j3OWy*1+Q`e`3QaVOe1~IK4%@XMp+e^0Kx^Chot0|Nhj2aq*ZDrEQ>kQ{-j_w z&4&1-k*_ZM?GKO26Ck#MR?inP3BLgu^A8M)&&Y&7gv_|mt$q+0ls3)Mei9k6 z)vA9Gnbng3(SHt^G?+-@-_IeFlBnTA`Y~kA?nqLXXAtiMARrP@X&WmcH>^w&u9Nnu zk^YK6DA$yAadAN&5iTo+tUf=MquB z5K#&w>Z3|Ygwe>7<`UiO$e1!lsZj*9MyP&T#X=$*?EEO>D+PRvF*@Uca!0bd5eU96 zEgP+BR0jv8BqEAVi=&y^lkh&E8f;9|y%I^S zA&u^3@Xtp@`li_ngAqcBJN^~|VN(3rxYjd$q^^-h;6%zOSSxfvrk;=9pct--)DuSw zk@`a%*tdvDC)HJ^7+@B9v5)o{se)u_5+M_b zAf)-}%=!I2BKa}(N)o~E44VES@G$2Jc<%0zV+CKIueMS*q)FWt5@blHuU>vp0`Dl! zMoTh5_^T+7t12)-s~~B{6R}z*-k^=w%KMn3Nl3|4I=YfyuehV$7HfXh1$+7Phpr12 zW$f|9(MW$xx(gK4pbe+n2#cc&HbWLiv`R;?qsa)#GfjY)MRYxpuu5dF{l%o91+j}B zVRSb!(#Dy)mngAKoDlUKqDUlO`wZ4dQbi7M61dhPbklU=Ubi)wNgpXCAjpcEk~N9c zq}}#zsO=$GVFESK(XzCy*{rfA8J3#ll_9neTDWCkKW0IwSN`>lb-@yC;jx0>M*!0h z|3*R(8zEv!q_NUk6fP8E_*(m#6QCn3olV`vZeP`*?L*-3n^fo=kw-rw=|je!dde<$ ze5_w4L44On#kE$NtsRvr1n8_H6*-F(X+o2v-OMBMG$%vcrwX{pX+U4E5#ruv#4nUS z^D6ZV% z>8DE1q3U&ywT6>4){H1c$xNFt^4J||z_9@TKiXm{cs43Jlf0G#IESgUxsiyFV~LKW zk#pH;84$%->6uCf=sg@>xRk zIix%ird^S!KkkT5E9uduDw4!iPg(g83oA`_9j-_j99#IG|armWwo zl3cxAH4Cdg)5j2?O$GK_O3gJFrEG%cu^0rt8i3~5=Bm@cc_C_4nV5vDq8nol2YBk>WLcK9e#46Kog0@)Kqo%#P}rpI!WyjMHU(nC-ke~jploT z)w+%0mo5W0a7X;(C+1$sa5uzQSG(x{L#= zSeK%@+@%hZc3S!li8dg%Akg04lulMUQBow8M3|hiYa~ijPr5njGLXQ)_ES#EA{5Q5 zMEusp5HxZ95)EmyK8G5k!T#K1`jfg4!=gu@FZ8&P>hfs>=itU~5K=+eW%l>WMh0hZ$Ctpd1ac zB}un-_cdKcCz_QAqyrKcNh`C?ic%AC3({gjAkDT#lDZO#;E^Ov%h!>h(`qRmKoBn{ zM7oLk!lb{`tmTg16NhUTB zGr;VOAx=jOubdc~Nevc+;y4#oR@%0Red|UVrf;@;dO2nNZkmtD5>*pwQAchYTHOI`J=`C9ySe%fEii*KdDI zR~dDQ1UOB-GO?P(a4e+lN~3DjI_N^&x{)R*G9a13A9EAQ5RWRPH@xc5*WYN!7@sG! zOA9z|>WCs8#i(A5fX%QP37c8Q)Tok5cBK7UYf>d;v6{(S4!eEdCZDOtUSDS3!DH}Qp zSx7w!SqvnjQAA0IS5@B0F+Dp@icTa(M3vtIWwbzMjzH@d>jK$UqHJhTc>V zXAS(48=!cjPC)Sk!o7LzX-vHuQ18fscb?AxE}otWVD}5CJkHeDgVO=V5DmdZUm8^;0es7ehPn`0HbjnN+`(K zfXTC-v8ZyO{LnM}3Ac#>eo1_l2sNNtDUYD^3-#$?MPzD}-Ohdow|>AUlW6}!M;fDyA{Riu#rTf?PNbY; z54Cvf{YI&ULN->8^rxdn%k!)S`$&t&k{mkDaS38C%JXzV4sX6%ki*Bf3UV+CQldQ; zpzhpD%Sc<%9ng;!me$~lM!tj8VEU|++oNX_w@ zce=-NDJX(cuSHO%lC|xK*v|%voo!>(Mk$!G@cQW6qz2mt%)PmwMS;G`EQ8SwyZp7^ z>NhCHaI>q;vQ}Ky;@Qzv?lij82t!m%qB0E`kM1fyGvtXOLqxB6w298?Z=x#{kNlxT z6$;Jl^F&gTfUNLXYG#am_?G>bR?_1R9I<+?s|$!+V{eZ1sEAaPgCy|Pk{yhon8!Y3 z2PzqH2w)Qq?mBk8i6i>vJc(_D7N=JFty$+)<$Qi&-s_X|{(hMlsw|N-@k(!u=_4xmhOxH`J96xU!A@j2H}E#Mq7gc{?aZk-2IH$KMsAag-KH0| zS`hkSUp9T7<`sI=wWfaxX3(0Nh7hHhig3Pw_VKKGF6KoMGAg4zm+o-q;vARK9e(In^cVjMedL?;6yD(=PXMD8 zYgc!cDMpzJ@KOnB3S0Cn!Xd`-Z7w`aRo+Rm_ zRenAtk9iz8fJfxTg%7#zKO#ipSqQjT_>hlJ9HmIcTr6I>l@r8J)!{ z6gPV;a(3LekHyLEFa{+pY!a}X>2Qa{1fztm#hYG8a%)CFXT-Ko!m@N3TSLvH$TyKl zJWBncm`QvNW001os7SkVF11o5DX*jn(db(eyQ=WTcNKevDOGAm8#M7kx`-rCkG!i_ zoe1V3lKrVBCZ$s(;nZg8e?nxAlk}Ga2%Qq~3at#unyIQRYj32;A$bBbh0+$UR(p6Q zOo$&)Ww(nzUG0Io*pq?aj8OeV>nM^cXuCHgUmN03v|o_Wgley~sAj52r8S>A zsrXI2p7nHtqV_~1kzgW~$b|$wFOzH*kQrS7(u46I zKxw+CioCaGdw4lKP(SqXnKqV40wZ$AD{&C)GSIc8t=Z8&;c7<$NY{Yn>YgMUTS#9s zsYd5XlY80PY)>&|zubObyZw*t_9FtE_-o5KHb+zaR#u3=l17TWwK8sU?}$ zC|x;4*Xg27ltDF8olFnQE5C&?DB8wbCk-^_YQY{$A{Ci7a8%DBl|RrfYil=#XsQO{ zPKhM&TzasLbOQ6C*N?W~T1Z``=VlEK*j-SjAP$#UqEkAS5=p%!F|v}%hKfDJjAC}h zs^V+@eR=)zZ|U`ORg&}z=x-85LI)zI$`#;+p2QuRIaF^dO8G21l%hhzNV{4RtwrNr zt?nKkfK1;e!Vf}wCoshmd^k9yb4rqsa<-XNb!VZ+M@Y~_$G|xhwG7uL5KtSvh?KzM zN$ijMRREUotXT394kf)qk^Du4~y=q5Tb4;eKLta}y8);=?h{7H(~q4P9d;1ab9X-W4bF4;wH zrfX8WjU>U~nJ%JS5&fUYRmp5@qx(^b&yxa|q(x~QZ?!>2Xalrb5~$Lr8wyU5CiUB` z#&@-%J*@hfE>=?Ud?*FAFjR*9LbI7FlCV%T&1&hDq?1f4lSZv+Qi-Z6v?JD3?W?GG zPl4h+k;L*6!^xhmg@wa;B32KNo>B~ugDRcWF_H>`B>K!Uc^bPML9y_DnO5!~hBT0~^N#x?Ih;>!L`xVWcf0-$&&)`C*oaz0@ zLi5dcA`=qRQMaZz31N^lQ(c{+6_7&KrB)Ow0B!~e?45SBSF?d=eSJf!O>(p_er4J~ zLqihi4{OLG{x~Yw52MokBq|v48h;uU@TXD9ei{|;=QGWID$`4J(wBc6mGNYuk|{-# zqD3|gSMW9@)8rI;M4k881*yF*SuIkLjpX;*PzZ$dZ5O?9=eTs*QFKclIkqM!o{db|hOGTcQ=z-`5{|gn6K$Hv4p6YxhFFu0mDCrKbgd7tifK}; zoNpvXz<8fs_P{<40#&EA4K=3Um&r;#WZIA=-m#K1xzmO$!_)1uRfB$Y~CIoVBufi&exWTFHz z6r=^!&^_$bSd)qxElKMpg+5Vte9i&3Ay*}0^?zTOC;qpzxL2u@cQJ}; zeC#TB+MQ&iKE>J#<1u6!6l-Mkm{C&W1Iih+NFDYsYp|ax8U%<1nq-e`)~rZJ#7IZ# zFHK>hxuk7X_T+NuTx6{U1eGP4RAUQmyDwKZ`c$>RRDD5_>e4DmtGGr4Y(%B~OgKvn zEeawC8ebhE(GLw|C9Y46%?KBI-538@75wvz!eTpr2709_5;r2P%8+zv)CUu2_4X`j zshdEOOGg-IEv6C8%hriM2;|hk!(7$Nes23_jpFqPJj)=Rx91}V(3qmW05uNps|HnYLILToLh!*5!KXKza&l6k6& zmHx}UXCkMpEr}ZjWkfsxf`w10s1D#565c%VS<`ehNG?t)(zP#Fqc`$xDdQ#6iMa?B ze^AqjL2komSMKE@?Z_a8Q${W4wsu}xPeO(PpgI<{5CA`HDZEx6J< zG0d#cNLh1=G0HA|O_oH!OlpVK9M0ZJRy?U1GFrgvN2fD1eI)dgVnG^)ryW<*{j^uc z+C3P-hy(!Xgkl|8n?$$a%I)edpi4_{T z8GsTbv9%SYX}2lAFWX7@8PiF(BtHWUGkGiNWx% zO^kF_-<1@v%5C3-yhanKehD=!+xGHRcK!{>Yd?(Q{BtP2qeS&j3Kh^@ejG*lr;*ow z4#k+%?eueHI%5I0@ed-;q32Ef7>YP;P>puMlTZT<-A0j}N0| z{@q0FVpb0JJYAbbRYekzH0=@cB-8*OH30Y+3CATTM{)+q0;JW;R!RrJ7pWDdRNHE( z*dm>+f}oKPf1SJi7tO*ikN?#Z~u znYl&Y$1wkxMfHb}*E)?2iLkULGjRv2vI?c6R;WeJRFbVK-HBy+A`B8xNoFJ(>6)fd zQukj@ntCusSu^ZE(r2aH6Cqp~xDUA&60>Y>F;MG?254KAB%@mrk_CdVGtq-1aU(W? zGO{EH=7RJz_i6~&PoOYmK~x%L)+96+8A)cZQlZ`oX(xszmb46%a@9_DQbJ%3&$PiO zl)eqM*q3dZ;+49l@8>5?fXiq4*Z|m6kg(_jsR$~1p*;}bn+(As(cY&a4=A!#E0 zf+|^1I=Xq)5g8uS#O}KYNf9xD=J4ARm)RxHr(@2ki=~DDObDGMq+h2+GPYy6|tIEBk?|y2?rZAt?wvElapR{;+_RfF%?m60GL+3l3eei zm!4D^NzB}O*vDw@Nv?mH)IQTj(qdeffC!;&q;x?fM*9#oI00IqA$q|PBkMG>-8V&Q zxPqGx8;vmPRpfCgpO1tOJtF(&_}e?PXyt`6znMW%VBjyBx*KJ2S@qVN0@4m-M+>N{ zhS+7&oq;beJV{nvi|%L)Ugc^hpVu$9{MT;(32aHQNkyD?#g?*|6j@a> zMtB67({5%Z%~kgDAY0Y&Z84Mky^lLDUP&)_=#3#f}3b%j*;;24&uSU z(PGq*y-kc8HtQ4?kxdzv~g&L~Wl~ ze2A&x`JA)1Uy1SoW2SM0UXY1qN%Il_1b$*pyFJY*O7AEdhr?Qt0~<14<{o%;+$X8$ zu0WFv{2(XI@vIt%i0XX*S{aJxZJ=dro}vjY8^c!{RpKWiX1;oXl)jBjzy<;KW&%#a*yb2TW!vIw@5TPEfzInn#{nhnxe#G7N;bEPu8SaQ)A`Z&m87CFK zO`tY1&eXt#dO#Lr{6~K~ALf<0WnMe5MBAk&7p&7Y%iD8LtC?q7V+R1<<5XJbR%&JF!>yb`dWLk#{R#O{4Fc z+9ttswzf4s>-p^9KyJ`&3~2Ww=Zlg1!yJzKV}s(m&Eaksq}1WJT)*im?ms_YM6}HC zbBL7JP8}k1lYs`R7<~&A0chq!@);jSj;+Z|~-zS+P@jeP#dYF077<^+c{orVdcM@EWy@PrdFtU(J(8pa8 zL3H<&MnnkfdLH$}X&-_W-s> zpE+(%eik+IK4eMLygc)@r=6e4I~j0`!G9RWID#@XQ>Q|`MMksqL@DSKXj@|hH|pdl z)Hu2g^6VtG<6Sj=Q<aG1;Z8IGO`7=#>K`!Lcfs$mA~Jrb#W&QFm00>6dfUYN9zE)Q zj3ob(qVs#{yUK(24;*{+;MkrZjx7ci%;(8%KV%t;+c5TWAUVb6ff+fjq-E=SBmLJ3 zHRTD$qrKb_*7dNpK3u;fY3Ou%N!8m5<10vjy3B!VfIJqJiqCR5UO!EsfV*|O8L5aMl$U>pF6*4ju)s{6Hj{0 zG9^=cv`l$Xy0<0`%M^_|+k%R*3G_6MMn8^L;Qw}`$5AgYME$cW6d(>?0v|??oF^rG8-YZJfu$7IYaUNyN(&Wd{9;nyDOSDYJdrh53 zX(K8sPH8Z`-5Rb!uv()_NTmPGquI>&LNf!ux`WkG}oAFY197ij8Y%qa>8Ty38+XI|st zi#9}ssxZ4|

ecCh7moqYfH2jd3@} zF^2tTpf;mln>@?4FP1#Ck8U72O(F~ac9z1^vxItU*iequQi6Vv2Hai`=V-5ZJi9cw zOLd<4HgNJ=yRz#3!LCg0s_xW3y=38`DPZrSd)$zY#K57p)7E>XSGik+p<%b2 z8s+?rUtt>o9wj*>#(~8>`dx1N-JV;yK*0ZC<(_TP2fNQ$y?R-w>Bs}Ua$nS1(=V;) zi)HF|MVqr_@>46?nEdvNrm=RbUI%J&(PB`i6)xXhix+we`?#CnUK{r)54`fg9m5<; z_wHEc`tewn{`Ij$AmE#0N#s4p5{peHfX8KjwiV#~8^2MT)m6a(P*T81zXk ztU=EfT9@A76xQh*zSiUWe0%eAZI!doLde>Ne=)xscEia5%mHGQZy9ZwxYT8`pX!yx?zttL11)2VK&JSKOVqnkC`E@SUWiHMB zW+nQ3vt!@W-N-m|<{xuSjTs7WttRKphpWkX_x@^Pnc)4^&b`m@jJ&Gv1XNP zS4rLtJF3XT7kVezAL@7L-Cm3OdRNK89k~kY4y4Wd)eBNfV7Itf*q@CF;HO2oRwhht zuw$Gf4AdqOekAMio=n2kY8{*DZ&!NIkMiC0_Ckat0QogbF%4w7Dbl=RqM^^}You8}GCkuEBZDlTvPW}f}* z9@1fD>Yc4YqaG4k6tx?5r@~W}r^KPcDLYR!0#IS;Ts5T5`y=>jobuJ0+jjH?-B|M3 zAGCRj*gGP{!P~hkw0m8>A<4VV^Fb{!ydSY{a-ah`U&46aumd0BSR|?hk8{^OAG@pD zEYFA;2ulYz+>LSI6*#>Pn`WRk|C5B*?4|M6hw)aXm~QfD#m6Jg^%s*KRuB+_eAJQN z@Mz0v4@{bCEdpM=*ASgBW z?nivY*r}BtGR5QDMos0E^ykN!bYvr;!KLRYdwHZlC7=S>`G*&A(R?}7Y9*>dq(Z7= zpEHm}(i{UaLyYsfafW7Lohm$w}3BlaDtIbEgJ zRqEL0fDz5p8>Ot_Uv5PrSYzMp-;J%t9Nuzs)#x)8R58W1em}pS!W;pG2<0|&QdM^wi86PE zZKZ5uS3$i2)hY|Lxo~r5n6lgLJo5Mt+uO_c4h~wN^#!$Eha>ZO=tmjS3u1od*JsyB zEmpnfvf00u)ypjp?yVfP=pc;Qt$tLRuLcG=IN#7pHS~T9+lkchuyWj%om$JXQH`;8 znb#1iP>TKPO`?|LX!6_9DHj04YD%Vc%>U%0=QmDiq^vitouA*G5=Syrz|I}pdVPD* z=eI+LS{!?fLth%H*}2)!0k%=$H&rn>`;t9}C7UVTEW7oHXG#0{S>f->ks|w==d;v3 z*W%V8cNVkM)M1wP>(J&R?(awMYh1owUh7+n`2BfRgTLic>V579&S!_mYhE{wvFA$f zGb&mh{ISIv%;A}&4Xkw!?rXMh^H@5xjkF=H5wnH(4%7^e%wvwb<6B(gpjo&*=90O| z!G6AUj4f`li&?*t+k!}uRtKF(-ly8IGN4Cey&#nV#~VCob{*v-bG>4Rn{yGj*ow5p zTjbeSd+H&qHd~$PP{tT93$PfPO;@Nty02iA3h3wY*yW9-H}PJ`cbnne+juY?*?92I zoB3U*xYdK4j$JT*C`Z0aQS%v~lpf0AS`zuviu%2MJ6p!+Xd3aI`YLT4=m@d9^d`yZ zcANgyTMcfbKzBonH&hw&HphcLW0fD%rB_=Jt7>-i+Hqk zWjN$hU3k;n1)^1H54@=Tj=@yZj`T=_6lOlK)|--}>LxRL1NV?@qvTaXkXr8NB1 zus@aclXUp~xF<-CAv#^9@7pq?iqEZ)qjq}a#HDkq@azz_9b-LRJ=3NTo@o=;K6d*N zc!DEqeB35jeceuPbF3umEiBFQT*$=UlqGqx6%|%*oM~q+MEq6nJdESMoke7&m@}Z- z@}}7#^-9^f`Z4}~gEU^l}lRD{}x>U2cyMEA|X`~npLpIv|@F}D*4IRy% ztzk_ZYn*e9?t^A@_`a;*>R^X9_;vPc2d=9<>NTS}^4KX+w|G^QW*KJ670VNTj=el; zl!$8!bJssaInwz1i~Z52F88zaSaZTID5x-NQwns#A3Eszs=RtCe$){~R#2Qf} z{5cyBx68=g%I79+Y1x>z==qIS2K2pU!J#G_v|$hikl`_6o_#My2~TR7Nlsf*Gn-S|cDm4;s7xo8Uy4d3*@Tvzf$&8`12CJOOG(a0 zdG_OujH69XI@$9^Yx&odE2y0To`@^OtZcSBE8J#}*Kh4u@o)_L72CqcJ&5dOi46V( z&_X-KXRyAot*7CS-Rz}f9S=LY+r9EOY}WP5nq61)nG-iPRdde3TUPIN9_Cjk{!te0 z`U7^2_Zb^-@D*&nhsrYYQJP)3 z7IE;RCEwZJ4&$?)YlqJxoGi7FH+)aUL>j%&nLtVRvz)6=jWE=?WKlGLl=tSn@yf?s zbc(u?E*yx~l}%ACI^S}GE60-ml% z`03;LqdkYHXO3zYgP+r};YsHR2;M8H-x|9oe$5sS8D#8Lp0x-3J<6Yw?Ez+7VTP3A zX=G1Qgw6MIMAatv7(G0`f$gaOq2vf(#XE9@hyC~cES$sW2OvV=t{zWPYv3%eZj<&50(1{3jBH2ZujAyQph3kQWMYxFsIQyL5@kr@o zB=LcE-;BZ;*`ePg=_qjB59NaB%I|>tO3tS`$_4jw+Q|5M&4(-xEk?;z(VzAo<87T5 zek;Kv)% z;6C*CTi%fNH{9=BEtZ^m#$IGY!mj%tYUj$TzO!=yi}S;si?8dqbOQh8nNA?RWCZu$ z>|{B;rh!~`GWYyW#%)WqquuORxxL0c&ij>nVZTB|rk?+fcNf3KP%iG)Ha$+T#!j!A))MEHX=f6&64g=S zjM!k#va4DWlKwjV&kbLfeUGEPF2lTJt$E{r&uNZzV8;xJ!pU!)dD+ih3uqbC)&xsqZT*cYeBdT2k- z@eTH~vpf{_g=er|aKtd5(aP2gjupc_)AQTNsk~Wb6Q@kWRv6^b&ULT@ul>fo_2#9A zeP`VGTlyT{_B+t$wk+0#aJkFS@rFLXJ~BKr+uJs@6+O^alo;_CZ(yyUPLO<%lhb|{ zF{01mtn66@O0-M<*~fExyg)Eic?zlA;kgF@og`r+|I=17kEZiH0v$M z{zGJ;(#~Dq@~)hCr-4lgOX=w;z4N*bAC_FPriw(0F*-vlWvwZ{rM6@c)E1%kr2ihZ{kT=P1m)-q|E0XkRK^)n@ zGc$OzBSy857VV&gVL=ZgB6Y1o_6DoX8w{=S>JQc{OYcr&jTcYx*ma38mGR4u$p5i7 zGCr-#D*&ARye_}dXV|hK^|XL2P3W-O4~$)T!epFpL^iUBQH&gFcy*QS<$>CQHb)0D|9IrV2OHU2>pq;ZA26cqMF%Z&j>M*g#!U;&M=ey4u8B%wA)>y7F_t7f z+O<=wF2xjTshOAnbl1|W^NpTVs&%NHOpjFMUJHG}2Y1T{md~AY((8#AxbCs`X~isD zbo9>dSy@*M;F<3GVWDg+g#7my-(f2XV zN3q3soa2trLvQ<4ZrOC~)gEHYH!Zr4aB_?9+_q=qWOpq|7yOMOm3?1+^S1uYo;!Tu zo@R1S3v2P7`@Z-kbJwL69P&@hEgIixBKIuT#WUp*S#<1BD-R;- zIk${lA0g3(NV*sAVZ&)8WONw3GI6`%87DzKG8OO|bqhUz^%C}K!8pvL zhx-bicj$5Dc{EPk_#3S{|52ZVnL0!w*zdH@4eQ)VPS(ShHuj1G2YR0WyvKd+c~7`n z<&Mwem62rog4a=+_&lBiufC)r=Yh@4sXf~p=b%U^+Afdl6I(vL$+&cu(@$LUK@JaM zo;k`!2aNC`lOVSE$Sp(M@WI166Oo5L_}+Gmxg37!J#x7SUe|J;ta6THo!uP3Q=>$5xbw~!Vx`@4;eTf6(TffeT%E8+HjyYBb%F1rvCRPo|y zuLyb^ZChLMi!@mc4wN^m$w2+SO2x!sYP?k3AfBQ(OGX~?xZ`cZUKTj{izDR#4!z-p z1#SmsI-qdn;O0cmf;IJl0Eb8x`p$W^#*{F!e#Pl(3l!_!9}W})eHe4Pzng(H$`dvF zxO8Ra*SBMh)+yGx|QNe@vRiSQzR*ty}JKHE!?m-jmTj z!O`=o+L7NJ*tS@KFy`Qt=KkN^a~^{sL$MsCt}A+9rJyZ5Jxl}zF>t4981LS4O$28; z^H^*vdc^anS7QL=DH_ne6b&e*R*X_0BWrE0ejJ$^9+*QR1x)AadaBx|q$)M6%48$w z6=(khpB{6^oNQT38~i-{{OD2V+iSq#C>xV;ppzh)k~#@SaFAt0QSNseJ?~b{iVj$6 z5}j1*Fz))&b@9EHa=JV0cQd*#-P~+RCHW>rGtJ4OX~-K2lN(v8INmMAes_@LZW@?i zU?T61OJLP(iN0(h4b&NPyjvaa=D0ELw$4U&(%g}SYdp0oL|1!O)|At`&EalFAK-2$ ziE5Z7qFS-ht@x0&0@! z_m|v)E07>k2&vUVhXVD+$CPw3r~B36{w~M;G5F+_)u{k~n~o^rv!--ZMQQeYzd7Eo z$NeyuW|$C&_<%5AMDZj6hGsyG=k|x={g&^i9hdI$HqizXvo}U(6^USyqm>wm9Utz8 zkiFl}*`1wqE8l5F^r{54A<%kH$d8FY11BNlc)vQ`ANl?iyEim>#2Ki$wUZudYmq&Y zeClL7-EU6!XTIMP({K^?mP(>{CRsOUUAt2`bxn;t-XBi)kNMX!H>oXAU=^WPl0KA% zsA7%kg+}3UzZzP`{)Eo-CupLH`(|4G2?;>Oc~8$aG(j=u5>HQ1$0v|rgeMRMy3|8$ z75UK7(ULD(0d==E5d86ebH1Nxpep*^Y%m@6v6dL5n7UPaBYFCGcR1e-M0l4`F-m&) zP`aD0yZ!4jUeA*$H&G;%@(e`g4hq_-=MiedY(Ib;k17XF7#1=97CD*->aY}&93d=iP z!FRf`>*Bl{S@0M?Ir2^DuS3Ljm>S}GAI_Mte+vmuUe}Cvv-L=?T;tI)i`cOk?eTs5 z4e#JMYFx{9PCw9F6z z96?LIv1@DyXtaqUZqcIF(eNl~6;_XxN1SQg*AG4aZdL9133|}+ig@4uJ6qVN*5BdJ zk^^Vz;=dny4|@}aM-^O&Wlz{Ep)YB5#{Z6Lf5$-9<@?k}_br|?IUOt>5z)0kU;%Hy zdPZG5)^qig%B9v~S&Q0V@$(#wY*j?Wti6xU>bm?z&QwPt6Ui24>fzXOG{hF8SHZ0|va{6Z-VX(Z0bt zJ=o#xw9(&K6%DN3tE$`4iWudxBc5wD3>}sCDo`BB*US+Nc2eS+5qc>xOtt4`?ifxx za^ap8%1Ba!vOyzy>87a0&b!t47rPccx2mItI(?4vSeJq^a}K{oY=So4j|lzQ61BC& z42^K<$Xjbp)h>cL$9-r6bj91Az4g3_2(#NFS)&_>aB{4{sGzq;d75`KY+!-w3-I{3 zVcXEA9BOv$IFqcgpD?3OzitqvR!qi`GO!Zyj=tcq~Ho~=*ptJ_A0XY1JedvpP7 zZz7|HPycMy?8|eeon01?i)V|EVeG*(*Gjxx!<^2U*APg~9)|J)V)WSkmVM@-z0Bv{ zZ0V2}@@pCYm$n-Dk~ic%Wa16Um^A3gxN~&>Gq1N@vPI8WlJ@ z&rKYJ2qjpJyS7&A(I$RN39~MP1dI*+yB#rn#2k)TK4K3?93OFqBc6}=!;!#8!r@5d zBk^z~@sV^mlKDtJ94UOH9F9~zQV&NOa_B%?_)GnpXXsevV=Y#U1xX%#M5ciQB3SVcYUyQC3 z;o-W;-%(s;;vC{S^Kjk5P87xMJr(MD9be%OSIFBd!ghrVcjx1N-rSu}`+0SDKJVw{ z-TAVgKaa^gUk~Ta!}&<6noks})Y}3ow{yq0{mfRS<6ZH(I7JHGUa4tEH zQ+^XidiV`=%$em>e5+Fq&HQXLrO1pTpjO@7OB{K1##~s-u;i&gVCN5KpxUJ>v+jXIy0H$%i*ydXSQ51+OiR@=cxQ zjf}<_2S+l~#5*@L*8>smh$H8$W^=5K4!LQpyRG`<&}a@%ZyPcMZBI6qJhN7~{yeNX zSih_d133>!zid<1GmZh03Ri1dF>g#?MSx!kJL72N7$;N$XDu~y%+o|M&s*;Btev%0 zvPm3c5b=J_(lq=RT*C*hcf9{^R(slVi*9ft9FQQ!6}nDd##L2yaWjL^%Hf?JxD>kU zDmX@R$N*N+eTa{);|*iAXvX54wVkk9j9)77b`TZ*oWK0Zy+wW<#(x;8iVI?StO znq}&&?NUGTP5N$E`>);oTe~fL*rZRI^lk(Gqfz9S>5-qa%Pp9N{snv~j?{dv4wTwt z7{i>6MgP@NgAzdtZ996jQ=jv=rTgZrFv@6fp zXmPgA^IGd8;%=-smD{6lVrE3>?%gpXLU)oZH=F8i`#fey{3bQZMijq#FN%M}uAbue z&5FQhdETFJaaYu^U2*t)jk}5Cw}&|X!y9HCzv27!A&#FVeGWJd( zLeU->M{PA;D;(9EMkPx2CHHQAoar+VT4ck*TE0HuG=*) zoaY)J=5hwd+?wMuuxp3h&aD_a#a=1m?RH2%@<%z3;IZD)DQ68&7gdkH*QE z(UCT)?Otu=aG##Nj_ckL-?(N2y!|*^hq@xqS3kJnDaX0SHz2qhdea+p1Zw|~j%LfZ z-Fb}Z!Cx!^9>#cV<{Z1hGpPb!QD%G!7tD;)R-k8vGLB-Ad|izf{x)hAt4HaOWrm>- zWtZR5{>I3u$uEvr+n^u8xsAgLN^*-?RnUhfH=^o_Ju0o89tus=;}`3> zj8ZUWM~%E25$02|Ybmk(6>g8&JnU0O1^*A(C!Lk|cIY8@85R3kYTEK>IXmH;W4p_G z>+TYFf92E(72KmiYd@mCm$dhBOCIY#zt9pb+EXo=6bt&t`jqo-){W>d?Wc+VXND(~ z=~b`&|JVZFZ^yN1&!3|E^zlwqfq%S{*U{lBd;9#OhiI?J^blPlEk{W>_PDyOuHjfq zE0SxI{ON529qVnY0pFp-7D0?CW23)P|G^qlAPYU2VSK2WC)BiDjy=n$FW~c_6{fz} z3a-e&zBst|=$Qc)CA(Y?aLuBVk~eJk})lgiax`yg*MVv z;p!FO2uoZK$%E(Obp6r(JFk;=)Vw)ew?~`uHHta1JYTaSlKBcfkR!tBiV>^L=gl~8 z&ga>0&*$wpuP*1+IKRpLGiS&?j-Nf(e;>F+SD9f&pBlt``QdxwNIDz$dUk60^62*? zSKgkGdGFTS6Y`$nc6@?eF%{&v%K#J#Om#e%|B6 zLUIY`d&J?Mkx>YlYuuyUCYhb@*+`%tzt+?Y|%GvR8FP8>G-TImVmQs-r7MN=Ur@wnuv{5`?t?N?}(C&$jS$NxtDtl z1fbPx|EB!6K5yASSPC;&ul>||xnFx6En;rfNj~M=9d9A&-0Z(NM_L0gcjDdIHO4$= z-s&A~)p$QE*5e!hE8c$28ryr%9kLmCBvKF#~JTU7yPsti+-Viacw@~myKSP zxKW~Id zALrmY1M-;}*vI@lBackm&l`X{u1(iJdGi@`#b8JTVyP` zm2qQoPW-m=@V0NMroO$6ZR0#MxpDO){KYB>^;_Pf)yPWHbN0E$DE{(0%6GnF#feY4 zqqk@`$FR)oEV?%2hnXz~)SQNe)Ed3hBjmX)TGgCh$(*;DUD~OG@h0C$w{1KvO{)Ml$(5H=eLsU<(`4IcFe2dW4BWOr5(+@ z*N&C%_5M5U%ID!c7$!z5GaaPQruSZ>Ek>S+B!i{r|J~ZavZ*NtW2Ve#L;# z1Htc?eN#)=)dCA?AYq@>`{37UYT0Cqm)RaI=D+J4caO}7%&e-&s*hc4@hgH~RYpd5 z_|@!~yViG8#%@1Z6?Jk)& zINtrbJ1=&l@@lJQq@Ss=z1^zKw&49aUH`haUzzyHd_0{Gc$$gEuLpx4BZ5S~oreVphQCui&#P#nY-Nd*y zWmB{6u_ZTJvX9mi$5CCnzCo|5@tg6LkD>}ET5G;sV}kdy<}x*6mQEopUfm4suxtTph~Ey_`h#@~ADz_?U4#w9Y2K0Ns!UQ7fInDO=M zDKJ|6FiYb7wW;@E>^*lvb{iempNI~-Idz!a+;OpA(3eRLIWy`-%i51fe4Z>ex0o*Y zja_^dqt;l3qGil{QwUDskg++{b6` zr3I5Zi(y9;y?LoEj%GaH&GN}J^V9RCrlslZyl2L~-_6H;-i&eOl6Q>xtH-WDi3c0% zt#j}pd$l&VOl`bs)ZI??uqkcbxVnbUyp&?E{CEe}w&7<vPA<&NXAlx%)96z}5P*Fv#DpGX$aT$=s4NVu5dLb3aF} zPn=`!sQ0$LLN3+i^*-1D#I(}IBX7n1dcXHlV_B+Sy(33i_W81goxAh?v}}t9{yryT zN8TMQFuTr*!(wZnS}&{^oco;WYT~9LS8OmBhWdwCU|}Cy)Z(lU@5m*$GB!#-qw98a zZnBO}K6|_I*uW;J@8jQp_=j))Yx~!4{@p)(BmSC|tiF`&R4Bb+y637@&Om40 zOVNsDw+ga%)f8GN#-x~KBaxOVUgvNAaVzp)(s#c!7W&`+_|5Z~omM zzfu3A_~SSK_{VR)t@wzM!=FF4pMLr7XMDu1KmY4r+PF{rRgL?~pVGJA|Hrznp$=|b z{{DyW%XXtQN*OsW|8x3jyO^1cYFzwt`s=@M?Vs9@|Kaj2WJp$*Z~y7XZ-4ml+n@jS z_AV^PG;jL(hjl4LpR`UQ0lj3M6ZAx`bUcA5uTCk9Vsoj~@zm&p_24WgmC}Ct=|6q@ z6Px>g{TKGb_ka22=XFm^4M0i0XAhf{jF64Z^;+^fp5Oi4#+X0-6OA&>+=ih4>OWG~ zBx=FT1jd9N_X$7QcikZfwdthN{2TwMF1m`gRmeka%vkw<+aKF`uw4v2EdSPThxj+- zS;X+8?cI3u2~z-uM9F>jzy0t+din?bFLf()Ej71f62JP78R?voFVVn~F?XJte}f+d z$?b%Z`T1U+sQbh+sfmE%iO}l7XfPyOk%cMs&J|&2r^mQ~K#o+XW(a)Smsqan4m3a#6ewz+4Y1Ky*1$1A$Vn@OHtWXjOkqrCjx z59Po9uh&AAOfsQ`1-d;Mh0XzB)43$)GC-TvFraLOO^I8YsiA0{iQ;4Fr34eMh5m29 z{5iL+t2L_E^;DQYrtg1B<>zmI`2IhBU)x_V`g?rafBfOk>q^cwh}^pOy+)S-KdP;H z9_QPFrooo1UiMABLswrFzbK)h#D9Am*5qECuO9jVG8)K5cw2qT-HV)6H>9w2-UZhR zhKoW}Ihf@hpazgMJ_&1--lH1U^aS#Pyo^ot4h5z$xoUENT0BJp$b$WvgU?BJ6H76G zT-`*n&vAE<(8|~Fm;DzKp1x> z9H}%D1n{%NsQqml<=D47QB!Nt!2#9jqQUA@wz_IOgan2+X-K^!i^4o3$2!KVD(JS* ztTG8VyYlpeyC01cm2@_=swGU)(LouChup^r^F#O+3+5%`+ap=34D42zL1Z>?zBKwe zJXuNi%%XUE#^fMRG!eQ*Ctx381r|tXMu)-0feD`zkN+X>xFLgwc z>U~S1Rrl;vyB|izDy!nTU2Lp6QDZdRO6XH-VU*xOz#`F8cUD97c!u-Fh8YML=cUr! zq#AAPEkW;8tn+6j)tPxWe~K$!Lvk2}3Sc@A!$}VXn4#MNOclv2RXo!~k50*{c?<>Q zRfq1ubZeq`z(&J;@NuvU?9c&2UC63(x5iKeglwK{qPZ7TXX*&4qhK3Y&mF_kR0@!% zS(j`z8Y{74n$8xYR3$-_P$4fo#w6w5w0d@J+|}`wwGm=*djDAoV9M(_L2gxjqKO5T z0W>%)3z)vfJI%BZNhbw$U^^Ppusp1S4sC6S=ROM}oVhDmn;&N~BTlrY)@d_Tr-s#ZcrWK2QM!L;KD;0KQxh@^9CR=4A(W32I+*pFcr(0TCgWr zFVq~cBV(~XN)HJ@=hWI<7sYbJ8MwUAQ?9;N|gVn@#~+lyzcdj*gJ=<~vt zj=q)kj2&DGM$=@gimfVX`Zd77Pp(1lY1k(M=5K%igFy-S(tMMY$<|{Epsmm9L+R|T zB%UM)W+oyudQtBPFSB)r=L8tSoQuFFY*)*=ow(gC#hM3K| z8rHoV;oZ0@nApPUS}|#=Zu1%mAGWsYdDo(V7!Kwm6qU#5>$)Vn7x|F4kThmL;%B)f z%sd==%_BS8FX@lx!P+HBC=Za;O3XSBq8TM4}bHVQ22 zej%~=F-c0N0JUr~6))8praa%u zTv18SBXhH;z_wn!_9o`~qrO~En0e?o9c{&f@N7&;65^2K)z04|vF}=6BlHeZ{&FC& zk<6=$d%z@f3nLJnE2*x;blg*7I5!Y8!Ks{NWwTew!y^U>sxr3U&JOi>cHBA(%9*5Y zn=PEN#}Qlo)@N!$bC{KnX#)^~G`RGNzzHf#sjX3RJh)WaFaZ~7qI z+kJyF4s$VSI>x>|Uv{(1J?bU*VlS0;kS(gZS7gg@FR_Ep`(@d}@QLBrDklNX7(ezJ zkNp{32HBk2to6r7g>I(X+!;NO(v6q|XpiKJ9mz44tBu|}J83=SpqaIvq*R=#34c}aj*RJk9lN$CY5ni6LX#e|Gwi&4g*PNxkczr%Tn8kLZh>m6k z42Lo1o6Z^z7$nVD$$Nbd#pWiPZMhYX_EkFLOX5=rK-Hn)iBN!SeO4&0*$e|Il~4c+ z8F>-qC+mVZ`XrdjJvfKjU3Ef!MJG(6j^S)L9P=kF*7j{sla_B183`s)>|r>rZ7$++ zP~FSnFuT(KE_x02^PR(v8Z@H%(jz+@apvB+&b8g1Jh>XuJ63}l>)~c4ytEpId!$;} zjPlQht#uDIti02(s|3i9cqpN_L=7LkfWe|I#F&I6V~$=24c=fdScU<3U2|dXHdE>Q zX-jLS0$Hchd?}WUd|rM`&_>o7KmArgU>Ug#;~T5|MK z=@{8yF#{PgVp@oG5IsEG#Q3tAr&r7v>fb5i=5!t&i^KfmH;jEf8ht?}rWZ|Pssoiy z)9P|{h|InWIYz2wkLf+kFtLoY9K_goo>%qm-NPgd34l<~X8<;qNytpNqA$}ekv}$X zPYuJZe>lE41i2GUdj@#9=g!#f2qQ1t0^<~Puf&F*-IUwc;K~T{*3WGXr&+o{qhvA` z#&Qh>Hel28WT(M31l`Cx%8~d2@#gy$95Qwv>a-FineDTcR8IWMZSj?Ej_>$({#M4$ zSjl|`!|_ZXznK=3k*BBo?c?J$n1yq}*lRn~I?2PnK%BeKTeP;>JYh%fo-49=Z`s>C zAydLW0!)q1GqV8IBzyhHK8_K>O4NCa^b|^^z0@Gv{`EUuEEF5Rg{G^Qdv5%V?HoBZ zxu8F51bu@Qlam`;Q0~_l!DIaEgcI91jJ2UEV>K?TWrJcHD7`SiGp9ki_A+_bU+U%g zB$?hP7rf;8$>M_FG=N>eb@csW-sKlF?)kv=s^9MOlRtDypdnZmcjl$9hQ^^RSistM zn2@H1_xr2v=lb&2{<7`Z^SyE;kD6g@m`?akuG$$FgVwtJx&eT`aLz6E`w_c6?xpPW zzHdPEy;Xe)x<05KbKdS(^@S1dF`QL{thq!?LuE2a_pp4E%bH2-GkV7+TeLECP-~4R zyUc(PEgf2bZo<3aY;DZo<$3Y}$l!dgxdS|=E;`eD#*$r)RpKv|$ctU@N7l^ojs~cw zk)kzU5A}9GUk~-(@yvFpmw0A$IMmaC*`np3e38J#O<|p66;14F7=;z^S5=6CsI4?~ zA*xO$($A3cC>$MOU8yzE@zoZSizY=yXIvnRNlhoe4wD<8#dfS5?P{xnYX_?rMsex( z%`-g}AczvcJ>)@VI2;Lt1hd1}tf8JN7NF4H;n`rFiYPsGn_#T_*i`+Eo$2*o^Gpvz zCYvO6my*epPPzsN|J=|t!-HGhV2LZnth5Gw)X{6Mp@|jmD9?IT2KR-WDlgvZC**L` z7Am9Ap}DJ`nOHMOx|)*~R7hNeE)X6qjYjy|nNK0u5`*XI_OngYcka9IUBIZCr!&u) zCMKwp0GJQerbtW)P1&;RXorQ);VvTvjZ`i@rYKd{#+GV2mIo&1(|)7cy{u}U2)vp; ziof^3E6Q62m48rSeLV2$Qv*+I6Q3D)Y^mmY)otiUs9Fkoye zJ}dA@=SHp$@ri+7p{6K4E$}hRuEKySpB4D%nlAc)H&ut(NxL=p0N@cB_`v+Vn&D7D zfuWTiyl%>t0ubq z4demT?a5ANg!yeDQ1>};R}H;nfc;@@b|`6+09in$zZ9L%M70(Mgdit@NDMacx z%E(4d@SD&R>FMBvujdeFf4-hWd_9MFa=gBtLwr4lc=-0O=MXRdd_9NwdJggNbBM0+ zs-^-!VrgWY0#3+)*B)gi;Lq602((o;Xm6764Zt8eTamHCA_*tKweVn@yB}f>Jy0QX`KQx*qGc6!bKD5l}IPU}_>i z1zOf)Ye5@~8Qt7V{u?=9m%)bJeX+n~QEEn0$jVGm8NJ(YIb z&QxRcvuq&Ziyh};N=N6LB-` z9G(;6*^SR|huB}7YR*9}q_q27TPRm4u}<<%G*L%A()q|xRVsuR#^GP?BzLKN)>zL_ zKFYZ2Dro@~-lY}!A8M&jlRA7kzk}0+CnPdN3v6sKd_=L#U0o9l3g(1F^K-YUVwH8Y zwC3J1pg|S`bt`gnwmwT6c0YIi+@>BJwIQ&~Z3L8#@HY}GC55<=G1ry5~8iZ0O) z7!(NJT2=EVK0VWrKRCxf{GQZ) zl_clMCgVj)2Id-OW_~uR#N0K}s#bjz&WuKldcdd_0Rn&ql!x+p+N;KV5w9BbT<%}JY6dk)i0y;^3}3x!KJxzIRr5Rd z>iA5rirQ=b-Fp>%vRBRLd6kLEGy(^&8bHq$T?1%0u;DzdC74PUklqE~JPhU9tdTN8 zD?6R7>cFca7?>BOWp|`z(#gPxmOcO zDgxZN$c3MxGgdnEY()k>wKg#iBaCoVxhq&q#n`GUKx!|w%2=%da&0Qc*r~yL(fzmO zRg-SLY8=`kv+BjAK(c1&*Vswe5K06%L^lU=ahMNz)fC@cX{>q%bL&^e)bcIEO3HfqF~ZIBbBsE~`X$RO(({2jP& z?HCGFusCZ_P$p!d4UIHgrW6=$DfYy4G{B0GI7}syYe-CS^mH8hJ>m1YI(7v+)Ft1G zGEn*22$(r0iKfU-m%OMdyR3K`R+J|_n(h)a4ukgeIOsbo|5P0dx!b6RCFXutx!X`- zhb3V$)-*tJh(RjNoJ3JFFbxs{uwIa}@97o)%ytsYhOsTaeNHuIeXqjaZlZ^&p`rBC zAl85YLq|(%l+U7Jju?%G@inH)Ts=mG+(BM6o3v=p?zP;H0$mi;zTA!kb(;Yx+0+$3Ra1gMv!P8>hf9ITdD6hx=r)YMw&hh+IDxQ%29Qx zopLqn2Jr&C1Aqy+M><{!p$C@>`4~< zjz!EvQB`BYJgXAOs|%L^!AfSbFqkt@w+gGi09m=zWdyuyOqY?eN*J|g`Ihe{-Xlj9 z@^D=|oyNN%MKXB7__rtBn!RB}1C z^sLZWofoxaIOfaVy?3~{#6@Wt>E(m9Ij@%GIuo(FfvfZ)q!hPhL2M=|%W}BMi<->Wv4 zeSZxSJGEtm-g5W7(*D)5nje(y;>ES64_?@|Ykyu@W0?eJ=j1ZVo-PQ|T%IR)I#=WM zdFYDm`LcIkT6Ze_;7w%m-}eY+*@Ifgb{UyTLN_F*?!8!)D_d2bk;^$}XYlG6O-mLn z^F_jL!RS?}U6S0oEO_FqU42Ah_wn(bV?&xlYr;5z7qH?dt=i~6o6^d}IY;#1>SHGP z+L$T+L}2>J+k3Z?ur_v%Sbvv&rpfR7e4oAD7h7J)EqCjrrW~0e^+ocS%_Sv{47W2+ zWU#XjU8$O>J*j9-ghF*NyiQQTEUeHm~a6dbk51ukKj@;If1dO=z>^z#TiZdJL*1JcSzC-Vw z%Bs1QGjgViK_X~uygCuV{NO&uyEnZLPDExp$SLzclWjSo^{*q?)m-|*zI*@uP*ZoP z>CSQSP&cuys4pRvmPC@2Pi&hF7;FDPc8FuXXCr2w+bKb2#^yD}Qdq19XNmXyu zAlt=n;=&LQ7R=B;v`E#WCKf}BP78I}r{2+)l*{FP>MDGvv5vJD(~&ZBG-sA?v=>$i z-oKtMa=t$nQsc9>{Ek2{_v*K(Vf~_y z`5&F6XR9&IMyv7r+UmF*qlKEIG3vgw__A@|z{ zvMht=CrX?+`g*({W%|l`Ya57Mil`Uz~dz9w3)wxk!2l{xBSZPu?)0P{* zU#i_1X(ZI$!*|XfcQ@R=&5jK74_54>(%*X919@pTX~}%{Q>tHnyP3Cs_G4tPF%YJy zv;s`TWv&O&wrjU~a3M&>wAb=(r;$fQ)#iq%+A~KOM31AJC0(p7>8SBJy6&3d1`l`i zB(89Whrgw9G@yqI^@XS!WjWGW(=_%PlgC$#xC;uk1Io-LGeN(M(TCDe|Ec<7R9U!`d5i=U;=Q(Zg&!! zuXPKRd&Rt_YZ-fgs@Lx4&Ixp`H*~(o>fx@>p@_8f#AUI`&|_+zNAAu8uT+>xvb2%Q z&Y9MSEtN_hEf-vIEQKYUWt^BLw6aQ&*AmX2u8$J3sH+5ZEy4J}guhstD(Y;qZFKSx z#3QMsj3#Np^MP3cpvWviUrPwXn^P2(8!5iUrp)_N}+)ZM6geKMd)oe!R3=+hR8xpbg>HB9NPaHQM}!;$97W4+iI$@J#Jg(aIG1Ox%jK1Au%|aOiy_F!g_PQ4(-NHPj zh}*Vxjb6fcG4t5A>R`3On3U$C_X4f0%=0TkDqfM<-x+IC>BQ7@E6H2iSo+!%-Q?WD z;Io!hzg^3qF70~MhKaYUQh$Rsa5kDRheU5}Bj5@{%Vs9YnW<*tOqnro^?7?`Vl(cK zjh&O4`=xixm+hVV**n(Ku$8VeIILt%vn}`aJ#W(vYj|r{ll8V;RrOZ&z&j-ytTrak zkRw(|loV@>&eblZ;yj54`ZmT=V`Aj5ZQ1Tdy{5YDN-f*(HjoiD(8IpdH#lVcZ7)_u zw{@M9$zO5yq|5Btu)NysL6{HkL5x%`&+QLIId&n`ki*MzpE7<(<<=(>y04Sp$u+f; zYps=Ya&6(Kjvn+w!atHMmg%78IPqRthLwlkx0W$?P}tiXdDQ2jo;V?;`Z`5jcTVeb zVi~3i$LGAWB>d#NIwi^HUhRvK{6xbKt>HV7AZzqgs-a zo+sP>75xj$qQ4N+LXvW%LVc+$_J-S)s;}jma4E^olGV#B)(V4CD-)a)4A@O^IOe5NqzX1a!Ial`^~ct&qa9WT&0I{ zAiG!=zjgYxw|JiMn!9IL4H+DN^Q>9Va<7*?t8mBelx^00)2~JE>zM-gL=UC$EWylo zW&c`V74D|myJuG$Np$jZE82_w!`+|(1XcHA??uo0&mEMJ43AIQ@twMU$6E9Go3%8k z3#OiyTCLrjDAu!D#rf>eE}TVq>?x`1j`p_mn|3?DTnQ6V?sxCUPWX+-!!3D&)b7SV zZBOp7Ax-WZVKrTz@#**swG|U=PQ1vyVwN(?_&62BK)TdEJ2HsPPQSG2riR=B+Ifu8 zJa~*q*GM#dxL;Cs3@vPj&97%lM5((!-ly%@y`QB(JrddR_w_Vpj*h%YmXG~3rc#~t zaYMF@vjcieF#Y1Jg(5&yaaE1kd9C}XSwU~B(auc+;Oiswn{>LD@hy0~^>qu3-luZ)qATMb~u_<=$6 z=8?kLK7Pv$-Aa?iT8$VKuIXbvIBy@GuMwq9->t*?xL+IdG=>uHGW)DJ^dGNa97bs~ z!SRqfOIN)yGp!akGt;e^=?*i~8uQA`wDvf^{geZaLN~K_#pOD&=etS9vz2 z()j<2u5qazukj%rso?zl&3}EKBP(3|!jeWlkTiEhH$*gk{vLH|U$zeVg`T*R;IL_q zvyHnUqrcW{d;F9wH!@uA=JtaXG1kODy_YnwbH;Et$)K=wxZ!m)ejjGc*bje94+P^X^rlKtu+N*0iA+BwYM5eK`lg-ATALWj z+Y`nKP62pt+=R`tlO*Pb9!Q>~2VzKLExw2zvKJ&GXKHSh4eVx(DZq@Yy>(+CZ3-Zp z>N0UJhQbA7{)t&eT<+P06HP=Nlg(aq%C$+S^S*Uh5a1=<8X2V2E4+XxR^ zOvnuj;AuVtaV9ckBL@i)L$?3SFtN*B=gC5f%M9tlWMmeVnjv4cr;0X7DP28A(#JBQ zx*&Y9wo8VY(IGh+CehPmnWqU+7)R#})X?s;1?;&Ryviu!>zQ%v);BD*8d~h$#86Dq zHN+15?Xxne6{B6la!MlSj`d)&qT6+;UX=E9T%9;ezVmtr=u>U1WaHpLB`HCRVXzPE zxDJI~Yf;I|0;Ae=*}WD>#vlBvMT^iBdiYVr1~ijx)D4oVpp>FQC`B*&flE#jm;jjg zSQR;$s;XxXiEU^~jn@RAZ4Wb0wpvRiGm34v~ zgwS`~5z-n`zco=RCz|2Om#mm?5cvW)3H5$RmY^i6M&++Es;#O`(np<3awtAC83!Ou z6}{t@1^`=l@({?y2yq4^@FH0P0GNJ_EFu1iN${%dQ0>WLEIkmRZ;{xWMzQG%b$6)- z*e+E!T$c{I(gRoaRBR0<>u0f>ya#V2Nda1tEp={3pAf9vlx*DWia;blo8&O_x-0Jm z0T_>N0ZJMvbY0TD3*U0-EpYRx*=J+A<$aXynL1Wj_pxT%(ZAg_7hO$NB=1`8#b;*B|(<~X$;dvD3@c`UzfUDY)2)0G z8PkQk=F`ZiB1FGRpO|8hC?AmHlgQwBu$w%&sXmR24f@!H&mu$XqN~XByH6wooEGDr zj4bG6UGWi?R1HEEuQCRyL8St~R0{=1XIIp`lLbYN;FTV_z;ES~Jiuw#^nhphmmIIu z&%l4sjag`w4O9JT9x$iqy6r{BI&eqR8s3rM zn6J_sPqJkm3p7qh*>Z}Ql=L0fq1GZjOf*cQ5`;Fxc{n_~%VYEx!EF;b|K4#MhaGl{ z{(Ri_*(8>qof`L(Nn}5p#PoMhBK&(#&HYLK7mX(>rqKvMf7ZgZmh+@%6e^3gdZ$F4;XTzCS~9ZFwB-SHM!Z&LO& zoc^#Hm3PI8=SgIHWi|;3_+XPL9!gDXy7&jnEjY{LFsbNqqZv-?BS22eK&qPRTEBqX zK8-|HeGYCzf!#|N8US>Nr5NwDEj)&`=qxPQsEVr551TVPSB>$(Mq(N%a{SfkM_*4- zvRlz-fLo4Nut z-b%+Tt#L{AjH&`Y!2|)Q^gu5%hiII^ior_uFoY_WZELQoDxN;CcF*KdDQ!C7UOopk zTvo<7VAU7ZRX)>1t$>;{C5g^e=;UA=a#J=s&xq32id3OFm}BJm&?dvggUN3TR=Rn% ziT)YF;#Zh%#WTmGoTnH1X(dDNOR9N zF`O}jPX3JaAE!pKIgMqhGoxTNb7P6>N>d3Yx=YSxRoGpa!3DB{;bm2~95MQ$K=wYJ zTk2h#>|P6pT2)04As1?C9R^bHTKU{!fw+zUVWvATOY~B=u4)d}I+HM5&t@f(50Xor zsbpxWs@SApZZ8)bT~UC=_{i z)k;icT~dM}QRxjpLOs`|3|0CHZGdVS;evKiH3KU@XOUWcnxoo(sWes>)P1Ij7Un&3 zO12~m6D|OfFs>ttOqaA~6_b_eT)cL$pV1ST@T58DJp}i(9(3`Y5yVp^q+rf+wMdLI0Bj(_k8!mKmWAZ`2711-~Dg_J@1!aUw{7p@N`Oi))zx0X!SnL9n&nLmFK(#Ym%A1!dhM=n{(ZadQW6I_ zFV;7INIx}v`+t`7UD`U$+O0&0i@fgGJaCF<^7CBp?>Llk${-5C#{w%=aEvPuH;{Jd zR$)fTkW9?o3)N|tS%C1aQp??>YK82C9M8HC+{oCSd22Ew_O)}iNMjvLfmSiBbP$i$ z6NHg~gCw*t5HZS{JQVoa8tEYZ=Q+S%(vR)?pZ9Ybrf=w(VXCf$43po63?SY!W9Lg4 zT+*6h^08pbci$L9?vo@7K#tQ&{pn9h|L5<^ckQwr%p`kIy|on>#y%xrsf^W{T`AR& zMHmYM_yJvsc?jIWbnmKHBdUqo%qjopZ@<5+k#l>bQeN~>`#*m8VN<2h%(cjYsr!)#0c>~U%`qEOVI!P?VlT#;h+DqeFedpLz!-u z?XOr&|4aHg-Av1ID#stW*8g93f{HHrzB4ln%%YwkO_J3Bxman5ep^nJ04d0LxvCXR z5(Ql^x`w3(A(Yg;H}Ip*$x}@n&bF>9_P>4m^PlRE>A$7#_Dj{m7=rwCI%X9W3DQF< zt~J<3OeG0bhLx{{EwGqio|qh9gFvk?BY8t(Hdv64x>2mY;xX7C=C_A$rYQddki`$* z|AC-lS0!eytkx7rRG2)jstkpY5;+H$tdpCCl1FHg;jwd?O)_>bbI1J1 zDMWhw&fQ({*WfGDdaD$nhnfdC(Ym0LVQ^74h}L3ABUY5sQFxLzg{CpJpa;^XnZ#cc zN%p+;xp3D;ke?9BOu=~1hEXIbf3QZYmmbC;4=z_Ro;4Mi39$jHl5Qta zu3Jo43|dT7pw#FIw*Xf#URKlJpl6nixu!gH-O@lTbA6m zg@TjS^N!YddWugmUW_tqgjd`)O!Sbn0?IL9JUvEHEUgW>0kgaZ0j*ex$C$vq1WZLy zT`sZJlvJ@10NH8iR7@p|VJVJ55~-_&k(r?m%G(4lWgV-?V``fQq=yj>K!(g_;L&~L zkF%k~MCY84V)SS)aKP3#c$m8^$)cB>NKQ#p(dg=Ia7`F8;-J~U28Fb?n8Mu#NVwc# z#@KQx#tI-M(yIXSmCjRCO{zfP3lRViVlF)czMzEwSsm+Ja)6S883Bxtw!UF*nX#iG zn4P6Sv}4ww1w}Jc9K%q3z(WD{Q>_w~4L}+IoK-Gou@e1Z-k@tTs*V;w_+mno6d(X? z143*7Ql;XK8U+@vgp6+FpNJiTf`{Kui>-?bY$;|sYqd$dttu>JG(snZj&E8+Ksho1 zp>~a=1u(dBaUF9X-}Gl1JQlxsF8iN<_~rYHeodP>Fu(iXd;h6j_bg$QVjz}g`RDJ`&%gY5F^-u^ z50fp4j)Buy<>+s2Lt{R|CaoAU8nFTbKQe~iU^pSEOINUHBYK9pH0kq|xwtcg2&Rkd z5Nm-ghUSdz8F|L$3GOiqomCR% zaTze%9o7_eXBtn& z@|A>IGg41L5c?7}1Su3I=3Zj@7oZ3Y?bRSTi*BU5O3$7i-0iZSiO8VT(-_n`C`6c- z)Wiw_mLVKHH8O-ehPyzGh>}BOaB&4Th4_W0c>3%>--R5)G#&`_L`?&P6_cr|kgSC{ z$gtf`_P)q%9|t?wQbq~gTld2178!;OiOa3|r)&B(C~yDsH7M^L&rJCdp|5}r4&Eqq zPa5>Q^h5D5p|H#ZgkfaRk;*1Vh+Cbj4_N~qKvzf1`aJAI0B(KoMgcNuNf0y$fSU>} zSs{cS-9&8l7Mf0x8UjkxkxYZg7~qrPGoqKWoGJh5c`PyQBOigYqzNFqwf#lLqqjBZ+Ov zKxd5-D_~@J#tB5!6^0#VEMxwi6$PLj()Z~D6sE9qH`yN^(b`3!o`hlDn|d`vua!nR!*pFsMmjh%5>s4Kp$pNa*>LUr7OhP+<>- zbv4@kQ`)V+=e3cFOx0cbvct7&!owhs#3GkFVF7F@%usCI(xs^_aT-*}_*Y-=>RUQA z6>k!+WjhJPg!j9&eft!B*BX^{U!yvbF3DZOjPo?qGbPlO$a2=_&0bOi6K?KSPdq^*CHUTcgs{gu{e zuFuB8e7h2l55j(k`;ClzQL)XxHED&i)sn|_q!ouE!fZ`I{K#W=M~bScG#o>6Nx(t?ICoAS7z! zwIaSmYZoQduioZK3b<>t8RTE6gRnC}M7pMz5fIByykimvFNvpB-^eP3zovb>p|`i3 zSCe>2*GW${iOlFDwXM@h&NvC}b=P$xv|p)fD{0y0+`EvJwNXOf#%a7EPnyR;MmC5o ze{D$s!}?`Oz+@kk1AZn5ahfSbTp0;U`j7<>l?REjmjsWb2Wgye4R4W5Yr+&K_$!WQ zZzRU8B;c%`x@(eXLepF+9LJ~YyuTs)7DKKZqyLUpfgvH}aTTONi23Do(6Ry{436jt1z25;N-bzri;WGI z8eLL*lrK_NFB;G1kn7Tlu7;QCJ#R@9p+x!()gmN0i3S1f&FiF~!i#$Ite=#L+q36M zK>^Zl>1rR6f}NW_Nd+mp_ySbBBOT*NY&mOl)YO5gJ=DbY zaZpbZvurb{w{%AZDY$WcX0y5|rnoA#G_2sFVkxpJIy@_%C6NXo5A64DYs?u#>>cTt ztYer%e13oRG*9B&#BS&L^-5wk1ISYVS6N%KfVZ02&`b#8y%wsS-j`vKG^ml%+tLriKB=&|&a3$$sp! z<8T*M@q|Kdl9_LYh#54ghEUlu9<2{!5>*;o3&M~?m%A225RyBWIUIPuF4z0FJMf6X zwV5HK(-ghEgSgl*ccCl|UT4LIc8j2^nuY3n{ z1kpB80PH+i2v1LTe;>PYqLQeS1@h3)o~EXnH7dluPL|Gfz0p&oOK#-~UmAn?DznT1 z9l@?>PqSs&TmQ_jCmtX9`Q4s)kT=Q#9QebDhpV<$0O@QCP;GABWBNc{CT@r--GXgJ zGdFq#vTdo@5)!u+$az8v&clfZu^nJTAf{|JfYd>Hk%^sEavfkc8B)vY>}wx8)(}vpRPA6C82j+V zaTg?(da~#eW5p6qFG=}52od)l|eHyK(%i%GW{l1cq{a-$&3%I#FkH!rQRu3 z2h~@5B2Ed043KYE*+GhS8c+^+O2uS^_a^t6=?PB@Sr*jPhe(fRU=6^o)-kRnH&|p( zA0>YoR=SkDyAhg=d6gI%?kI~tb7M_#nXG>6a`Ba2COSl6H?Fx8Yo2tU1cnqCK5?;An#49 zFrpI!mO>TzNb*dYXz0fbYbOFJpClQyg;dJ%@a)-K9%o23 z=6sEzxGYm(l353E3FO>lm#Oq5xiuU_3_YD;rR#Lp#$-grVgEbE;JaR>q62j#nL$TM z&BQhs>Y`%sX4YDAe;lU%--187iGo#=hPygL88eV+@s$8sTCyo(K!9VgHf?Hpq$pOY zkyD+M+1976_B%W*&52q9&|2oL2*xDRRRa7=AVKB0_fn*T#T8P{0QKBqW%_bJN$+8~ z<3I82-h#cOCpqDp+LWkDV_HK@b;oc31I@?1DV4da9~gteZKp^IY40Pw-aOWM?zuPj zY#r5mcI&t=)MW@cKqt6bg)};Hw7uj`ZI%qsaT3!#76>^JAhrTfI~~HR4$s!{0o%^~ z&l#GR3yM6^M5Z6%(Z7JCEu`dyK>Gb`#n3W_^5?ViN z5Qa0Oc)p3=*Aq=-vQnp7#D9!Ef~Ct46gv3DR&#|ikT8KUY{@`zz}mzLszA8hGjC6) zL5gQ5>KpF%{9)K{J(0|+Su8Qzlq3spsUwu6LhG2W8B%UBF)bNGcxb7ZL#T5fLRw&{ zqMCYE^2W22OJr(+747@(GZWKak?Uz%*5(w?FnOS58N($@R<$K<5;;FJL?uuEQh_4z z)##L=a+`G3V?|o?{2|SmCZ75kA+pM!IEQKuXvG4j$cUEWmxZwB{)`$?gR(2ZoW~@! zON_NQA~b23$HW?_)@MiYGhj?}EovTOaUz)7=xu15tXA@T(&eUY)fiIOgGXrGCa<;^**f_QBcuO1V-OvKx~P!=k2^-IhG!wKz7HaD zj#0pkZVNf0zhOf%H8%{WSS`IJfXYN9m!?Lm=*h;&KVUT3U~`-q#UvtTwTP7`ttb&9 z%hK;y+vc?rxoR{J!lR3k7M2l9iX7`cGg;cxi1~d}En8iJOdkD_iw6vaG=O?+jO)b8 z%vLeOG8VCL(5J)Ngn(~WH^m_FUyl}yAyB!6{bv#Fnren*Yq=_U1_c!?p$O3{Vd(?v z8_T%RAn*w$7G@h=k33_-BSWwhRjX#cwh+wHRq{O>ajn_`s& zYRGieCXlf~qWvx#vIjzE0YW6|t3u8ythngN)RP8%?z79u+(%ZPsUv0@eU(Z0L@^FS zAXSsJE3Kf`Jv87}@6n|mQw3E9%n5WjN?bJ0D$twFimxsPAA9`w>tX=$=$ZSq9yY;= zKN!R;06(iV5ld@01{(3i3Pk#Q^e~A@r8|S9CGJ;K6!NQZW=g2*(V zWCqfk;7PX}YiGnE)6c?$p}WDelJs=S**Ad6GgJYc%q&3IBzJ-21)!=1<_l1jEg5Z+ zMxYN#x@Lot29VN_Z;e(pjN|k)C9Anp+McY6#U&XaE=4tU1t$Zn!E# z;lc!wB!JGaRe*lGKueko0d|nh1yzqv>$f|&c&3V>R7`xRVN3%|x!Rp;f@w|3ms_)y z0rOsaD~QJA7VS@QlCRWxPpS~1;ZCBIa|bS6DZ6Ii^tY5Yk_XN&p(&yaCO zI+^wA>qA>>15_M&6VF||9(i&7o^>|f8BudCowRmC7S;0nhUPj2P>I6OEp;|&!+*QFJ3JO~?sqSx! z)*%0i8n`>9#=c${4)opQ0tCONJAj5A2h2qh0LI-gEqvygduM>21ZeT%*)5vW$6Rlb zqKhpR(+NS$Aj?DsI;Or$(n5Y2(jC>70|8jFHqsG+>Rq7=pzBkS0Ei}IdOy408K2=J zdIe+%X4`?;PBHv;6BBg}W}IT0n2^Q=G1zv4F`f}ImizBOAxaZCYhf|Zi&0RI_UR3t zGZ7vqf6g9rbyo;D22);w;mSmO(hkZLol?O50T|GQ-a1({Un}gKg#|t6+hpo*c#}V!UjSSwfpsnOjS}<9lzRMPS9I z_Uwj~d`}7v)Dh||R8y<+u#Yq>36h&DXTs2H2Ewnys;V|=*9*^rb%uZdQm$Ax&u*IC zM;0yO-M1?WLp!5?T+N0@)W$()7AEEB0_$4BLhi7EQLBc3Vdut}oY&OKrT3?IrOe$i zXCnXfnTkTI`qUyuUquWw)_64z^WFB01rOs4);C&V4f?b}PgzF{gPBoDk@@k1yoYQ* zD#UxXAE&KSP*;QPhi5f|izi{t0YQby0aWt5TD2iLx1U!@x5uk3LOaA_=T&EJu1@Zp z#qk5gfrSGQWk$^Dc9on_!OeRCMiytqg)ivx-cc4w=f~BqQnq0P`ZQE$UL~vOxXQFeSg7-Ax_iT=m&W&cm4!7ou18% zK(_g=wwL)azE`UTt_X(n%)Yl~UG*Y~p!6zhn+f!^gi2fYGIz!C4GKNxNwq3B2A{zO zr6SL_y?_GHBy%oV(Qj*WdGC0tY_)MV zlKggFt#@tEu^&w7#G<8XC6fZ3M zbzZf1V@^ubQu18g$}WB`sT(Om)x4_f&DFus($;#BK{B#pz8C9vy;sI=0gmQ-rG4?e z2cOKi8tHqG*S+!3 zAA>`Te|wrHjHBlNSoRspBJAK85w-rbi08wPx zZ|Ycb()T5cJZ7=FlnmQ-E7@)D04f^)Uf+)aT;E0A?%u!nlWaC>D`C8+5)I7Hvqi04 zN@P#X?`kf`diQ=;Z%jgNhL`wKud2AQ*^n#)|LSYq<6^>Nh3=l$m)fU(WZX06rT#Pa z+JDlPRl1S-_`JJ+ezXJf^FjY(&86e>;f3d=iAPpE6T?e)=6Jj_)tA}?`}e+0*89VV zn$qv*B)k83SKGC(Ze=+qk9YOm{H|LscxetbMin2g4;kXkoie}B-&UmuwGQ*I>+|~l z`8jW`dH8&3$LH;X=P3;&EWzWQUc6LOf9+q9`u3)t4?Tzee$cU8Zf(6kmXIQ%7gxAtCaRfZbe(~+izPR|32tH?&;jsq{JLkj7$R&U9`uV+2i|| z*XZ$Fc52r@X7$SODBbrWy6e6d6&a!DUCz|;Z@6j=?YFzCeL)h1bW=JpZ9X>sVv&D zJzZSe6Z3XgIO+f|?%7VK7WeY{JgvF;d|4cN+gtoj{$G2o#ElnpYB;amRZF<-uByW1 zecFDWLoU~!!Q?5(P_^MDq8R>Va1GeVI6=m*8-64g#y1Oh4U{Fxaz8kJBsY*~o8r$v zk~8}I;OU54v04Pphp5SQG!T;sw3QV7EPAH>9~Q>2}V#-ROxj z$XfQJS6)6_V&C5)bF!Z8>+Ln`z3CCQYj9lMJQMB^KJob+rzx9XYiAf`jy}e?Lnf#( zCi2R%OE&}C+PE5-yLoal4HmHbK6z9_)}#iGfZaGb@9H}^?LwzrxBeOoeSV1X?Kz-J z)!)}59d291@zUY#77a`rgGN8ICPQu=c_YcPrP<%nRITS+tD8oqntEccg@IW(rh)Dg z!^uNLmWPly=kwl?8kc;6S!&Ky%FBBLoo>!jhbnpQp+q_E`I%_&NjN>l!xKehhHyaUEtYc8HQhiR;<0WBVN=4gIIHv`ZsqbAzv7Ez0Y+wr&j5 zWS!n~P0#;k?Rb5%152K7E_UE;ciW1oqgh^%tA?&m4nqM8PjH8J)HM#t5vX#dolnF@ z$r^$iqkwNcr{|DRIj1KiRFdH`wnof3Ddsy`ihx%i8%1rVl#60&SN8b>jiN0Xl1%Qw ztdUi3jBJtlW-kDx-LN>j!(GE=oYN_&(}aq*2DA~nVyZkv$|jy|#K?%H z00S$grgHBZQ^EI8OOF*cw9jU^;D}Z8ZN5|{eVg-TwW9LAWpCDlWxRH-Zg^bP6*FJ3 z^W+uBy~dAOUe_rJerqgEHxS8~og4OfusX?}euwZzoom^zsE=!#N0I?!gPw-|b^Py) z+{b)-fW>!s@lhME->R!NyVs!`ZPA31-?&nQ}?gqXwS2 zBsxz(ex1=sbR&bn{q2s%18t5q_=+~~#=K)hx=ptua$$V#9U$E6K@Rt>pVfQEv9)V| z!Ayh~xr?-zp~J+#dVBA%(@hxs(E~d#yPwJbi0mmz$@P2erE;$ycfC~K(vRG)mlruA zAI%;#LX=6o#u1aR>Uhy!U1#HH^~H6@loDr0wOvM)Tm9$P89)zDj~}*l8;Nsxh9mXb zD%fn<+dLKbbDiEsFOMQ>oL=vh@mHp{xm>yIV9$EcLQgIBYpLEy7^Q;2j)!$(J2Co(-NmbT3+hda%E9myg(G ztaE6>xUpn!vhiRKs)?!aw#$7X3>1NWAJ%8Gb>{Hqp;h}499{)81Zvz6`a zBfqR=xmik5`+?D!=^*ajH+Qsz8M`B=>UaHnjHy|*HPcc<`Fsm)wu{9yj*)0H(FQYn z8;(}i_#`*`jQJ^uB2ZGB?H@fcCC#DKE!RiLq$FpQp!0Y#x7M`G%4RiSSr4J65||Az z!?19vD>7E8>`V8?>tq=WsL8#@Ug{MqXZQbEizdo>f^j+9G*lgOgk^1e%E#SWNz(QA z?JwW%nDZD5SD)Em9(!lxIC-!~&JjK67X17BlBq)SwGgFAJW#fc0rDC0w51euQnklKGWeyvVWVcRi!Lta96BU{ zB|%~_7(*LN@li1ud&a!TqW4r%ipkq*mC71ZQ!s^e)QLW9BqeC6qN|cu6@9>rGNH-b zqE>^+1S_ufmQ-w7m4+z}waz5eCD8mx$(jN?1z<7N?j)wQqGdv-x5T3_?Hfe2FSTR? z^ICNkLk_tRm7x>aXKKh_y+-JA?zFcp?Cx)~w@425$7{IE?&s(AO+?pX$l`hSMHv|e z+L+I&{i)V|zO31_?GMc7+_DHLJ-74Xfvs%3JD-r4AkUvqj5nyq=U(oz55JRW59RVj zr`UDwGtVRCy?h&+HF@3b=q-1D_iOK&{w1nCv(Fjab&un|rCf7ccB9$e%g&QJeA0(c z#2&AhwKPt+9y7vIUNFfGijJ#qT_4oteUF<-SXC3m-9woUv<>-U;%vj6WQwO;DE zv)+%7P}$}Uyh)EQW4*n;ZzTP#|JMPh`FMJN`npRY{jLAkv}d{2*BwE)r))8G@cN9j z!yst+6CHr|%>Q@R5A=jBHg_MezFq;dC`{Me(fY_(t0t4LrUrF(tlwpq2d`e4`+@X! zukWr588fnEZF{yK?~kkYANqu0sFZiy9ZWOtq7{$wDgvAIOMeNs_D42t1zo($>1&&7 zyf%@OSbNy$nI(_i-n-)wN|G`f^-jNsbn7Kxn9wJkKygptwV@?PpG<3W?u}3%NM^ab z-TjEUKjmlLj@ddfQIxC9!^+)4#|5x zxw4#bx01GqyWjm>`xBR5nfXDl=iy;H_on0?e}_a}##49hK#vlr;im4Jcc=}Ou`JRk z;m;36uaA8SGwadR%<#sVkMnqXW{Lf^>lyU2?bdxt#gz$d)<4P(F|M&mOhkXelPKI@ zo1R26wfxQ%9zndbccJeZg!Oi#6eXy~8!k+I?EkgFO) zoVk|WnlWe>Wf$>^nYd-lulZ)deLKtxr(90Hz5R~IKMSAP*_!zVSRb6hX6y_O-w>F| zV~=Zox{*H~_{;J%Y}O=Nw1m0aIh*a8FT*<{j`=@!o9Kb3FPc7QGUiS!KXUGgos^b& zMo1jJ?$)l(^vqkOv=LE22%Q`dHW0nu8K5}(m=SEepoUd+W$RX+N6~8?? z-KkMOg^cALGIk?(c689xp4ixrQKF*>^z}zauW;P-NGS8LGQyTiJ#p&sPIG@+znzwl30Y2aTKxpH6w>_`hwj-lN zKD@7sK5$%~yFRbq&laxp#>2Jp;99%CHg(pzYImnaz?ObwUA9wm#d&(sGm%$!YK_4#!&b<^<#y)lKrG$FLKLGIMw12>vb1%hs^w-Y4MlUM? z(`Y+Wxv`r=<2}cfO3$|QE0rbOu62dyo-4uJl~BdzoqD_C2dB%iNdjc z_s_lj{g_2%%-FKSN#gkIjK3KYSk9fORp!wP^2M>(`9r=kURTe^i@e{zt-R!OnO^j{ z+(vp|ve}m`AGYJ+T-cTW!d%(E#ofOp-MmHJl{@CbbtUY}3-{&KkCd10%KI`ttrtIw zS+W{u8Qj6RUeoJ*Lmp<7c{md9tHactH)VgB_KovWXy}ihTZhE|Hg8;|A?u#<#*M>c zar`^wi2wVJ`J2|>Yu-^~598`_w8gu;C{eb!ms@%8ddwYTUxsHnTh5m1S;A|vI?Eg0 zY^Po4ynL>HO_*s;9Q>Jrn%+zY->2O@kNi3=-tPn5B{4m%R`iY;D|0JBZ`a!O+K7p| z`h=}<8{e`=_P$U-4gZr<{|`NB>QnI}rtfwIE!wdg6Pacp&LI~;bSC8Gl8((^5 zZ)MH4T9*2fv_EIDv6w$VJB_yAh*ljZ;GbGRfL$z0{mAK)UI&A54|cX^?3oLuC17b_ zaq0Ff_gKKD71tw<<2cC>FDZqb8;yRqdhDfIad4hv*ROTt%+5+X+wm_sS| zcFL{NY2j*2Iz)=puc(9f#>S$H4XJ?apkYzPsucUKhKybmK|4%es^#t#<7otE>+3{! z^{yxCJ{T}Utfus4{p>v-jOHr=gXtcBgB~%k&qRMeK4Av0e0AlOKg+PrAMN*}4C^r0 z)~kAOg}L$%dhmWH=#tjh_!;9athu^#hB(3nVntD3!-_#U<4 zLY5e*wWl0&jRWcL+iAo?Kl!ZS)0iPEG%xn9fk2(<)k zSk*j#h$ecwWhJRoJ{1KxbK?YEQ9CW1G!v+!HVm)2bdN67%qkx?OB{P4v$jF6x^!&= zuL!hL_>+5AWM>4y6nco;UTl^Yj{9n>QaL*U%A1JeL&%@g=TBWo1)aoy7te-Pk#9#xG5a-|4g`vqW)jTO#sTT;-;SEoZoe z7;m_)dDF+%r5~b;CWLdCas>R!A$GJyP*= zcpb;AHQBH#N7aISMr}N0n#A8bw?6Xf{7q}p&9}A}maj`~`oF?oQKh@*KfUSv=kxiW z{S{TVukrU!`YVz>FX3PJ&g!%NiirJbx0woXBmmYNz2fub$+yg2X;eb@f$ID{+}(3Z zg(FSD?a28ZxvbvmcicUVcVDeIc^c;@J&pSekTTVG`5E&!&gr87IUarHRk8YQKJ(ZX zV5aGWX!V_VkP*jItI&Pk=`WVSxsTr*h@c8(OX$g1pW1ZfiqPPGhbd z(uwx7)MKe(mURgqo;k;C*;RTD9U0WaU(DCY4f58S;1hB5zz;)MoQ#-eXHGYdpl{+uN`N>G-w4934xu zR-03XtTWxfHk3mUe=~+Ht?iUShwgjkUO9W+u`kR`!&lU>i`C9KY;KG}3d*0ZF|{Ys ziQ4GidKiNgv5$dHTNv@2S+SC3d2_ik&p6+ZkI#gO2aj8cS@-!pfky5MP>uXvyyrdt z-!LDb_~$U6hQo9l^Jy6$EHUfV+Vs1RDEJ6;s`teclK zc+ipc_3^w@DylKN=$X%RtIsx7S#3=Qh~ot-6=boxq!ebuPaqA( z?6b={Vao63Jv-^!%^Vq|fq9zsCR(dE>fcm8=-&!vgi-ma=p8|C&J>07C{E(Sj>j2H zK>~7=XYWethPyVC2kMri|J+N9)Wys=uII=>eej;c*r*YXx*p(Sl)Faa{2m(XG5`M% zQK7jVKqdJNpe69!55^);4iL_?p*b6X*Dd$QZi8eW7qj!u$gi_MHv1v8h#b?ncb1hk z$Jo3-iX--1AO>SO?wqecwsqMfgOVk=pxMyeS`oL0kYq?Inbo)zK*0d;0-CWH_feFA z6XGMrtk>vKRknJgSMQQfNQsJVa%o=&HAM7m(MD^J&){do5YaQnK;4-KOPD=!WwQqE zp27;XN@#tro@5~ShUG?@Pl08sLs#7x^&;aO^`aw7Qq7BG8!7v{9DT6aj)6&3fL{ry zjl~=iU0Z`YP7=Cq=S~7!)h$nk_yDu!1H>xkaEbpiId-c7J#D2v$-4&L>ah}upveA4 zwHMS(KtB}CNc>blTMj0`z>@FC%TX-W8@|&{YRp%;eiYdQBc;5py7}oauEsb8 z)#!i6-E8M*wv6NIcE2FetYlrwIA(txD1%{NsIYVNQ?`K~5yW;DuQ`=|&aA`a{v~HP zq3qiv$4kz{EokLq>Gw~PT=X^ngRoDg%WY^l+2MYe*j+jLlLQbZy!)ztfL#+SX ztYla{@MtA#z7wLy7rzi6F~4W`V8ufB+&Me}cT?otBIpy{e0oav>4B$B;Y@C>zxzh!zU-qbo-~z< z=-xujZ)khF4?7Hw-rs)6)_$&FwR<=3nOYD|Fv|q{n5DtPk|oY_ewFs-<-R5a79t($ zjJS1qQugx4`cISWh0X*Y2_fF+S?koi{5*MG$sU&i*kAO#(AvjD{Hopkvk>2N#uNYn z_j+dNqxb6NNrb-Z1NnpScu)^Va6=Xd2U(yd^xO(>jO52w4{Ju;7>PMbCCFze^Nd%3 z4$k$=P(*WBLJG{=v)p9e)%tn;DF1-QDu2|6v7up<%eg=3CA;2B46PTqT1czmDK`WN zHMZG9uPuoe@)UYGtgwC&i^YOi7Sa;3N@DSA2A;G;c$rEO4c9swEPSc_YHdV|w$e%0 z^^Wv8XQs%!zQ%I37&BGjz_hl(4Nw7BQ`+p>;*mK2w}#uzB&r{jxz9icm(s$~hgC7XVF@>q?OF9f|-<`+L4O>fP#2Hm`%RGgxQ8GOxQWl zTc?KK^&w|R9nQ}EdBqckx@CoJ5=OI2mn|>l<@0WNEmI0;V65z@2d~%&#?j) zU>eS5E3r-t&L(iaUO8&&wslcg@8&+AUj2nVP)>P5HSf!Y0VfYlUDgdZZ`m;M>fj$~ zoZtL8$#do{-kR#jmm#OE)zjMIYPzRRmpxGragtVO{Y^hByEO16vXkA?OcN%?66MJ2 z3)5=9(A5>QynLBVX+=~mN)pdyO1sqY_(`6sNB!Rl(Y7$^)Rz_k$2ga@qvZK^lVf}ILG0Xj z{wdhG_(;cAWzwqBj-pKkwGHHN&Do*Zd+nr=HJEAQNsX)#n|1PwX=DjT7ShRH#4Q-H zy7y^hp@)1sZh>Lkpx3oyOO~VzFZ-1-N@0X>c;Xb_V$YRfg>G`L0uD1v9%gnQw%UoTPGs{G5QQU1ma^|JfQbADa_D4wl+g+$~HyQjwp35IV=$ny{_5$ zq}7rvRc#+y`@OZMea2~*T7_9+2oV!`$_xOCNG4=0tR7mo`~|Ko z2^fQ?kW0UUXbl(wQ1aMbP3;S{k6{Jj+brmv499Wxaj_*!kijb8sXDbcQ~QXrEh+x)4i z2ZNqztcM)?Zg*{jyMgaIf8|@W`AMqtHRATHqdc1Qjre9SS+{AYInXyr^EC6@EZMQn z5keCWfjK{~uIDj^p=(c1oVg{sn^{xtcTZp$?$g`KJ1Ns3uyaTcLZPe&RW8$Q73k@5p0)9Y5xLRlMsLSbofdcK5EH&dJ9? z53sm*pvTMd6!iE-y%~C5s0|auJMtbS*=zxK=@M|ohh@zg=AsE_wH&U^C%si%(c|cPo?P6GVJ4x?guMRn35EO=uBK|WD$&bc5+qozUrV@# zcXgpia;kU9yU!MDg*&jv6(~CKorl*G4`0mkEDukuE*_q^g7%~S_T56t5W8$%bX*yS-l>}0 z&!Z4q775Nt2kbt38N&$FcqIKgXS_^JjY#1}IkCQ0|3Q|?YisfxBX6^o0ddQSSg9&P zf$J;!Y*Ebw?aV8j0Or5AqRL&`d?u>JtlB-zQLfT|X^!%KREt?dcqd1>wv6l-93|e3 zl`{UK7H>_>&mZeuPybqb&QoO8?3uERn9?%Wv7%+5#kd*Xg*_A2BM(N#G z#ub(%rQgIo)Ux@0-NR4~hA@AYmWoG~$!g#wSC-L@PL3ns9tekE*v;0ml%XZ%*s-Ge zu_ePc=OgC(tDooAv|wk=r074XijdphD$zy+kf7+ThI*AsPjA)jt+u@tb8C(FsoIB? zhRzseyup)`_Yp5cTPx?j1A4(xj`1*L9NiF_g5_E^9W75e^{f-9H9K>T8B4d06#{)Q2r>X@%}pAC+N3| zj;#_I{cmk<-Y)RXD%RB#HsNYG6rU{m+mu4RAMgvkMiQ^+27j*SoMUe+<`W3+Fn2}w zG`|h+(ETu4hId6-P@jPJ67Kcv)wyS1rk)AwvcDzmfgW-AP3Ti`JwcyJ@iz4F49pIT zs>s_Xmi}MFBjV4Dl2O=4f|@Yvi76_bOIMn;LBfwgmMa zJ6_#n)P|+-8Cl0?yxJ>!;$6;&IaBw&TC8tR8Qm!vacND{(d(F<_557>WO=wZ2l%y4 z--i2jxkDR}!Vl0)&}u`PTk^VC+430;bZ72+rxQ_7{)Fu<7?A`c)=a7SExC?;_-5DY z&}}(eR$F7+a@w^T8PvYkc?@2(sD(aC+wuSQhDKWmpCMgloOXl4w5q!v^9Z$hEa(Bc z;ZjHAShSkCSw*$mXdKM*FGi0Ti0BaNMEPt4h7BK?4!3(N&2_$~`QE~H)aHAt)${Rf zbrv_#FFMU(TAjt$k#`tZ2WXw#sofoXzl`+1`97%eJY2QOVZ=+?9=v_dFuK$O|MH0M zztN8q%ifO_dmbC6-@G<~*Wm;c%gpsU)c=oM@66-Q9PM2SPyP=Bg?Ej@yNSY+?f(Xa zcUS>_gG%34e55C+0oN@nXBmLSq!VGah%da0%rU4-1ZeCUH>$Jn>eQ~p7CeyE%P zhZ!OGMGN!slMzoZly8Wk)?>C{%J#bL1L94C_+H@s8w|Gme&UvNDBIIe0P&Qg* zZmc`U6z=s_h6<)ziv|aNf`I+m!)_70KiDCB8{y*)<1naGD}|U&y&Vk9^ri?k`6! z%|3FAqr@!Vmg+BOcsq=I0Uj5xz;%z=0GFH5K9ARlvRsY1#m9=zLu|7N=Hc!NRGSib9?CoP{GMf0|7aiot@EzjJZVVQFDcP$I$XqNq4 zT<_%J3{BWj9kQNX%>Ym6#aV2pd&iw41-Tl@AZwg4!R;z`rBk>AMa#_tsME3JJ8s~= z{qXoSZ6Id$NS%qnX$<0m9&CBLrK>mKSlz(S*ev^-9 z4(`Z9jlPdICw<#a0vuthOVK$~j=Am@tiZK*E@!7=68l@7U1RuP5K@ywqqrGs@%T7Po!g5-`+TV&Q7t_ zeqXTPgSAtP&s#;b!~%_J&%!$kP#eU~g()6hpV|l-ZGkrCL|NrvT}7j&A`D-7eFB9-#`7~t54#WHKLrL zoCDI3j5IMQz}Kim3L4XrC9P|532<|bHdG|%My*CykKu1FefoXx@^A6mpDT9ykFP)d z-;dvY{^9%l^`}34{Ym}1`1;fDzy9=1fj@zSz}Fw@k3WC=6WtZRETZR&L;JdJ5lm?8 z20XqtFVIy}HD{UOSq)24O2z!n7_I0%#U5!8a%*&{T@@=|CBu0sg|6yV^VRMtOjlL1 zd=+*F!-E`O4Uw-Jk)=R7MOPJ98s4fD(N$U7v%wZ0U>W(UZ_oBfe7Y*A($H3`jIVjO z&sFnNEh1f`jb8(CB(9)cm+003KW(ccg@0oVs@=UnW_<;6w|3q9Q`I6aYVw%IpAvV= zRgT$fwR^f)1?!hy8|P|@IdcfFD*7D=l2riNtw-ok*U)i*Qi@zoJG3Cg&O6rDZB?RIPS~OHDQJYP*!#EZ z$Lwl@YJGw^^qp?dJe`Fh@=tK-7)v@xZKOnCd#+9tY%0usl?AO=$ z1rwq7(Qi6M^0m~O@Sx#QlP?z5`H<|tI^&$fJ9qq|9p)2%jH7UkZuAi1BMfy5uRP<= zM%te#w_V2$54dOMS&%2E)EuHmd8Y~K4ozUqLer5^1eYJ~vM0V} zUwG6`RZCWP_OE9@)?M-!a*+FJolkN)>9L7A@QMt4#52$GImh@(Zu#VCh;KLzlT3Bc zPo7Q#SN&_e8R9?d`|K_$tUOWJ)&u%WPd%(PB83ki0xV8jKw}bXD+ag)6ujzyp5O^? zm6VC;FIaBD=2}ew&N}AmaxRskCQXnjI4SjdkT1Nf0G9e0`NBjiu{`nDP*8Mw%C$aV`a1%SmYPw3uTYug?MI+4mMXr6|)P@B%pT&q#p|h zq=evY1%a``&MLX48VX>En3jr;M>fLI39rgPS|0}q@RhB%CPxMMK|V2LKoZff686NR zMQmBTuCBt^1o~ThQ8j=p%t<%^3u8gPlcm7o17@ow_!oy9{ApP))bIw@4c00oOy@TI z4d;h_vB*0^6x8fKGsOgWIoDOXvcjTZof65@Xh&R`x~{i ziZA~3U6+FOQ(68bwW0}4oqzNB`=7V1kvZ0kb?Dynd1po}@#b1){qs^v)-UjR%ljGR z10d?$w(5trDN5T4HvyLeB>~dEgy>uXOc=e+MFA55(C&eC05H(5Xyd9 z@*jJrP8AUWx4UdBnwuSPNgfa@`Zu~z)e>d45k6REM?&v1N@b$1CDoX-*gO5tpZ}U_ zUs*teIr2&T>4*6F$C!Wm=DW{-{9NiU{l#>1{o%X6whPgFS8@sk_^VJcjg$i>Mizn9 z8CEA;2INyDtkbU#u~Y`m1JmzYABuv_O+YbXibZ?B8jZf`(IdH7G2;=^L0J=sFbOu- z+2mftQfKwqO(^th@(V?tZGQ`~h6da~{gtbWe?~=^VMSKp6PB?0T7c#NgWzxCXVT%_E zs9`V)2rW{;47?!7zZF#)Kn>v)Ru(u_88|Eu7|mL=cy$|2kcJxUr>&VN#+9!8!M`xk z6|k~vSLaf&s6fZ`g<`MJASPH*XYhHQ_viZ)7^=zaFp7$-jGUi9KHN}XCB7qwp(U{7 zPb~6^76g||_A1rlYh3{&WUmc~Zg7x6iUGUhfpe?jH+t@)Mc3w}EJ&xoe;MGi zWQqoaW8pM;j=wrbp`MuIlOGH6mRt8Z!p}e9a6Z*veyTrwj^F-I{roo=5zqQl`Z@ol x{`5aNejoEUKYdewgj@gpr{7`2U;pj5&u^dKKEHi_`+V8Y{{y+K)QkW`2mqk$(Bc39 literal 0 HcmV?d00001 diff --git a/bin/mega-evme/tests/replay_halt_logs.rs b/bin/mega-evme/tests/replay_halt_logs.rs new file mode 100644 index 00000000..c4df5bf7 --- /dev/null +++ b/bin/mega-evme/tests/replay_halt_logs.rs @@ -0,0 +1,94 @@ +//! Mainnet regression: a halted transaction's receipt must carry no logs. +//! +//! revm 27 gave `ExecutionResult::Halt` no `logs` field, so "a failed transaction's receipt has no +//! logs" was guaranteed by the type. revm 40 puts a log list on every variant and fills it from +//! `journal.take_logs()`. `MegaETH` rewrites an already-committed frame result into a failure — +//! pre-REX5 a CREATE's code-deposit compute gas is recorded once the constructor's checkpoint is +//! committed — so the committed logs reached the receipt and changed its logs root. +//! +//! A full-history replay of the pre-REX4 range caught three mainnet transactions doing exactly +//! that. They are captured here with their on-chain receipts, so the regression is pinned against +//! the chain rather than against a hand-written expectation: `--verify-receipt` compares status, +//! gas and logs, and fails the run on any difference. +//! +//! Runs fully offline — `--rpc.replay-file` never falls back to the network, and a cache miss is a +//! hard error. The unit-level coverage of the same defect lives in the `mega-evm` crate's +//! per-spec test suites; this file is the end-to-end half. + +use std::{path::PathBuf, process::Command}; + +mod common; + +/// Offline RPC capture: the three transactions, their on-chain receipts, the state their blocks +/// need, and the external-env snapshot. Stored compressed; resolved through the shared helper. +const CACHE: &str = "halt_logs_repro.cache.json"; + +/// The captured transactions. All three are large mainnet CREATEs on the `Rex` spec whose +/// constructor emitted a log before the post-commit code-deposit charge halted the transaction; +/// each on-chain receipt records zero logs. +const TXS: [&str; 3] = [ + "0x002ecbc328e5259b3756b69a221fc7ff7956dd616a9d872eda1701914bb6f3cc", + "0x0a85678457f7b5db647f6ecd05f1ccaf17c5ef2df771d02126a73fa8b41865bb", + "0xac0ae5fc76d7939fc55015d8865799412235387926dcf1444084c63e07ddf565", +]; + +fn cache() -> PathBuf { + common::fixture(CACHE) +} + +fn replay(tx: &str, args: &[&str]) -> (bool, String, String) { + let output = Command::new(env!("CARGO_BIN_EXE_mega-evme")) + .args(["replay", "--rpc.replay-file", cache().to_str().expect("cache path is utf-8")]) + .args(args) + .arg(tx) + .output() + .expect("failed to run mega-evme"); + ( + output.status.success(), + String::from_utf8(output.stdout).expect("stdout is utf-8"), + String::from_utf8(output.stderr).expect("stderr is utf-8"), + ) +} + +/// Each captured transaction replays to its on-chain receipt exactly. Before the log strip these +/// exited 2 with `logs_count: onchain 0 vs replay 1`. +#[test] +fn test_halted_mainnet_creates_reproduce_their_onchain_receipts() { + for tx in TXS { + let (success, stdout, stderr) = replay(tx, &["--verify-receipt", "--json"]); + + assert!(success, "{tx} must verify against its on-chain receipt.\nstderr: {stderr}"); + let result = common::json_values(&stdout) + .pop() + .unwrap_or_else(|| panic!("{tx} produced no JSON result")); + assert_eq!( + result["verification"], + serde_json::json!({ "match": true }), + "{tx} must report a receipt match, got: {result}", + ); + } +} + +/// The receipt each of them replays to reports failure and carries no logs — the window this +/// regression is about. Asserted separately from the match above so a capture that somehow lost +/// its on-chain receipts cannot let the previous test pass vacuously. +/// +/// The assertion reads the emitted receipt, not the summary's `logs_count`: that field is only +/// populated on the success arm of the outcome builder and reports zero for every failed result, +/// so it cannot distinguish a leaking replay from a clean one. +#[test] +fn test_halted_mainnet_creates_report_failure_with_no_logs() { + for tx in TXS { + let (success, stdout, stderr) = replay(tx, &["--json"]); + + assert!(success, "{tx} must replay.\nstderr: {stderr}"); + let result = common::json_values(&stdout) + .pop() + .unwrap_or_else(|| panic!("{tx} produced no JSON result")); + assert_eq!(result["success"], serde_json::json!(false), "{tx} halted on-chain"); + let logs = result["receipt"]["logs"] + .as_array() + .unwrap_or_else(|| panic!("{tx} produced no receipt logs array: {result}")); + assert!(logs.is_empty(), "{tx} must replay with an empty receipt log list, got: {logs:?}"); + } +}