Shared-memory failure hardening: crash isolation, strict decoding, recovery, re-upgrade, soak

.config/nextest.toml

@@ -0,0 +1,24 @@
+# Serialize the shared-memory crash tests. commentlint: allow(JUDGE)
+[test-groups]
+shm-crash = { max-threads = 1 }
+
+[[profile.default.overrides]]
+filter = 'package(mc-host) and binary(shm_failure_modes)'
+test-group = 'shm-crash'
+
+# The soak binary uses shm-crash serialization. commentlint: allow(JUDGE)
+[[profile.default.overrides]]
+filter = 'package(mc-host) and binary(shm_soak)'
+test-group = 'shm-crash'
+slow-timeout = { period = "120s", terminate-after = 5 }
+
+# Run the ignored full soak with:
+#   cargo nextest run -P shm-soak --run-ignored ignored-only
+# commentlint: allow(JUDGE)
+[profile.shm-soak]
+default-filter = 'package(mc-host) and binary(shm_soak) and test(full_soak_cycles_conserve_resources)'
+
+[[profile.shm-soak.overrides]]
+filter = 'package(mc-host) and binary(shm_soak)'
+test-group = 'shm-crash'
+slow-timeout = { period = "300s", terminate-after = 12 }

.github/workflows/ci.yml

@@ -38,6 +38,65 @@ permissions:
 # gauntlet.
 
 jobs:
+  shm-hardening-gate:
+    name: SHM failure-hardening matrix gate
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Matrix validator unit tests
+        working-directory: packages/e2e-tests
+        run: bun test scripts/validate-shm-hardening-matrix.test.ts
+
+      # --allow-unresolved keeps an unresolved manifest loud but
+      # non-fatal; a FROZEN manifest is validated strictly with no
+      # workflow edit. commentlint: allow(JUDGE)
+      - name: Validate hardening matrix (strict when frozen)
+        working-directory: packages/e2e-tests
+        run: bun scripts/validate-shm-hardening-matrix.ts --allow-unresolved
+
+  shm-crash-recovery:
+    name: SHM crash isolation + recovery (Linux, provisional ring tuple)
+    runs-on: ubuntu-latest
+    needs: [shm-hardening-gate]
+    timeout-minutes: 45
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: taiki-e/install-action@nextest
+
+      - name: Provision metadata-only sibling stubs
+        run: sh scripts/provision-rust-ci-stubs.sh
+
+      # Serialized shm-crash nextest group; the ignored full soak
+      # stays opt-in via shm-hardening-optin.yml; no artifact upload.
+      # The --lib pass runs the provider_recovery controller unit
+      # suite (stale retries, deadline isolation, wedged cleanup,
+      # late-result fencing, inbox bounds, custody accounting), which
+      # no integration binary exercises. commentlint: allow(JUDGE)
+      - name: Crash, recovery, and soak-smoke suites
+        run: |
+          cargo nextest run -p mc-host --lib \
+            --test shm_failure_modes --test shm_soak
+
   shm-source-build:
     name: Shared memory source build (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
@@ -79,10 +138,15 @@ jobs:
           cargo nextest run -p mc-host \
             --test shm_transport --test transport_negotiation
 
-      - name: Rust contracts and clean host omission (macOS)
+      # No retained macOS provider: Linux-gated crash/soak harnesses
+      # are absent by cfg, so this proves side-effect-free omission
+      # (R15), not active parity. commentlint: allow(JUDGE)
+      - name: Contracts, observer self-tests, and omission proof (macOS)
         if: runner.os == 'macOS'
         run: |
-          cargo nextest run -p mc-shm-transport --test contract
+          cargo nextest run -p mc-shm-transport \
+            --test contract --test fuzz_corpus
+          cargo nextest run -p mc-host --test shm_soak
           cargo test -p mc-host --lib \
             shm_provider::tests::platform_preflight_is_side_effect_free
 

.github/workflows/shm-hardening-optin.yml

@@ -0,0 +1,81 @@
+name: SHM hardening opt-in
+
+on:
+  workflow_dispatch:
+    inputs:
+      soak_cycles:
+        description: Measured soak cycles (MC_SHM_SOAK_CYCLES)
+        required: false
+        default: "1000"
+      fuzz_seconds:
+        description: Per-target libFuzzer -max_total_time seconds
+        required: false
+        default: "60"
+
+permissions:
+  contents: read
+
+jobs:
+  full-soak:
+    name: Full resource soak (Linux, provisional ring tuple)
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: taiki-e/install-action@nextest
+
+      - name: Provision metadata-only sibling stubs
+        run: sh scripts/provision-rust-ci-stubs.sh
+
+      - name: Ignored full soak (shm-soak nextest profile)
+        env:
+          MC_SHM_SOAK_CYCLES: ${{ inputs.soak_cycles }}
+        run: cargo nextest run -P shm-soak --run-ignored ignored-only
+
+  bounded-fuzz:
+    name: Bounded libFuzzer campaign (nested fuzz workspace)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: dtolnay/rust-toolchain@nightly
+
+      - name: Install cargo-fuzz
+        run: cargo install cargo-fuzz --locked
+
+      - name: Nested fuzz workspace fmt and build
+        working-directory: crates/mc-shm-transport/fuzz
+        run: |
+          cargo fmt --check
+          cargo check --locked
+
+      - name: Bounded fuzz per target
+        working-directory: crates/mc-shm-transport
+        env:
+          FUZZ_SECONDS: ${{ inputs.fuzz_seconds }}
+        run: |
+          case "$FUZZ_SECONDS" in
+            ''|*[!0-9]*)
+              echo "fuzz_seconds must be an unsigned integer, got: $FUZZ_SECONDS" >&2
+              exit 1
+              ;;
+          esac
+          # libFuzzer's -max_total_time bounds the run only when positive;
+          # zero would run the first target until the workflow timeout and
+          # starve the remaining targets.
+          if [ "$FUZZ_SECONDS" -eq 0 ]; then
+            echo "fuzz_seconds must be a positive number of seconds" >&2
+            exit 1
+          fi
+          for target in frame_descriptor provider_grant provider_sample; do
+            cargo +nightly fuzz run "$target" -- \
+              -max_total_time="$FUZZ_SECONDS"
+          done

Cargo.toml

@@ -3,7 +3,9 @@ resolver = "2"
 members = ["crates/mc-core", "crates/mc-store", "crates/mc-host", "crates/mc-module", "crates/mc-tokenizer", "crates/mc-shm-transport", "packages/mc-shm-native"]
 # The evidence probe depends on the PUBLISHED subc crates from crates.io rather than the
 # sibling sources, so it must never join this workspace's dependency graph.
-exclude = ["packages/dashboard/src-tauri", "docs/evidence/subc-surface-probe"]
+# The cargo-fuzz workspace under crates/mc-shm-transport/fuzz declares its own
+# [workspace] and stays excluded here.
+exclude = ["packages/dashboard/src-tauri", "docs/evidence/subc-surface-probe", "crates/mc-shm-transport/fuzz"]
 
 [workspace.dependencies]
 # CortexKit cache-stability core + storage substrate. Sibling path-deps for local

crates/mc-host/Cargo.toml

@@ -25,6 +25,7 @@ getrandom = "0.2"
 mc-shm-transport = { path = "../mc-shm-transport", default-features = false }
 
 [dev-dependencies]
+libc = "0.2"
 tempfile = "3"
 tokio = { workspace = true, features = ["test-util", "signal"] }
 # `stdio` only serves the broca_subprocess fixture that replaces its own

crates/mc-host/src/connection.rs

@@ -36,7 +36,7 @@ use crate::transport_negotiation::{
 };
 use crate::transport_provider::{
     fresh_activation_token, Candidate, GrantBinding, GrantRecord, InjectedProvider,
-    PreparedCandidate, ProviderContext, TCP_CAPABILITY_VERSION,
+    PreflightEligibility, PreparedCandidate, ProviderContext, TCP_CAPABILITY_VERSION,
 };
 use crate::wire::{encode_owned_frame, pure_header_flags, response_flags, FrameId};
 
@@ -863,59 +863,73 @@ async fn handle_negotiate<H: McHostHandler>(
     }
     // The first serveable offer in client preference order wins.
     let mut capability_mismatch = false;
-    let mut non_tcp_offered = false;
+    let mut dynamically_unavailable = false;
     for offer in &request.offers {
         if offer.transport == TRANSPORT_TCP {
             if offer.capability_version == TCP_CAPABILITY_VERSION {
                 break;
             }
             continue;
         }
-        non_tcp_offered = true;
         // Provider identity is `(transport, capability_version)`: a
         // name-only lookup would hide a serveable provider behind a
         // mismatched sibling at the same name.
         match shared
             .providers
             .find(&offer.transport, offer.capability_version)
         {
-            Some(provider)
-                if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            Some(provider) => {
+                // A panicking preflight fails toward static omission: reasonless TCP and no client probe (KTD6). commentlint: allow(JUDGE)
+                let eligibility = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                     crate::panic_boundary::redact_sync(|| {
                         provider.preflight(offer.parameters.as_ref())
                     })
                 }))
-                .unwrap_or(false) =>
-            {
-                let provider = Arc::clone(provider);
-                let selected = SelectedTransport {
-                    transport: offer.transport.clone(),
-                    capability_version: offer.capability_version,
-                };
-                return grant_candidate(
-                    shared,
-                    gen,
-                    corr,
-                    selected,
-                    provider,
-                    offer.parameters.clone(),
-                    setup,
-                )
-                .await;
+                .unwrap_or(PreflightEligibility::StaticallyOmitted);
+                match eligibility {
+                    PreflightEligibility::Serveable => {
+                        let provider = Arc::clone(provider);
+                        let selected = SelectedTransport {
+                            transport: offer.transport.clone(),
+                            capability_version: offer.capability_version,
+                        };
+                        return grant_candidate(
+                            shared,
+                            gen,
+                            corr,
+                            selected,
+                            provider,
+                            offer.parameters.clone(),
+                            setup,
+                        )
+                        .await;
+                    }
+                    // Exact `unavailable` is reserved for an installed, statically eligible provider's dynamic readiness or admission pressure (KTD6). commentlint: allow(JUDGE)
+                    PreflightEligibility::DynamicallyUnavailable => {
+                        dynamically_unavailable = true;
+                    }
+                    PreflightEligibility::StaticallyOmitted => {}
+                }
             }
-            Some(_) => {}
             // Known transport at another version: name the real cause
             // (§7.7.3) rather than reporting it as unavailable.
             None if shared.providers.serves_transport(&offer.transport) => {
                 capability_mismatch = true;
             }
+            // Permanent absence selects reasonless TCP, never `unavailable`, so a client cannot probe for a provider that cannot appear (KTD6). commentlint: allow(JUDGE)
             None => {}
         }
     }
-    let reason = if capability_mismatch {
-        Some(FallbackReason::CapabilityVersionMismatch)
-    } else if non_tcp_offered {
+    // `unavailable` outranks `capability_version_mismatch` across the
+    // evaluated offers: it is the only reason that authorizes a client
+    // re-upgrade probe (§7.7.3), and a dynamically unavailable eligible
+    // offer is transient — a later probe can succeed. Reporting a static
+    // mismatch from a lower-preference sibling would permanently suppress
+    // recovery of the unavailable transport.
+    let reason = if dynamically_unavailable {
         Some(FallbackReason::Unavailable)
+    } else if capability_mismatch {
+        Some(FallbackReason::CapabilityVersionMismatch)
     } else {
         None
     };

crates/mc-host/src/lib.rs

@@ -16,6 +16,8 @@ pub mod connection_file;
 pub mod handler;
 pub mod lifecycle;
 #[doc(hidden)]
+pub mod provider_recovery;
+#[doc(hidden)]
 pub mod shm_provider;
 pub mod synapse;
 #[doc(hidden)]