From 5babc6bb817834f206919235449ae0af850c9eb2 Mon Sep 17 00:00:00 2001 From: RobLe3 Date: Fri, 28 Aug 2026 11:06:12 +0200 Subject: [PATCH] Bind Python client qualification to exact assertions --- qualification/pre1-cases.json | 433 ++++++++++++--------- scripts/run_pre1_qualification_case.py | 303 +++++++++++++- scripts/test_pre1_qualification_case.py | 125 +++++- tests/test_dispatch_ticket_trust_crypto.py | 37 ++ tests/test_endpoint_security.py | 21 + tests/test_pre1_release_boundaries.py | 33 ++ tests/test_pre1_runtime_boundaries.py | 43 ++ tests/test_restricted_directory.py | 36 ++ 8 files changed, 839 insertions(+), 192 deletions(-) create mode 100644 tests/test_pre1_release_boundaries.py create mode 100644 tests/test_pre1_runtime_boundaries.py diff --git a/qualification/pre1-cases.json b/qualification/pre1-cases.json index 555d4a5..dca967d 100644 --- a/qualification/pre1-cases.json +++ b/qualification/pre1-cases.json @@ -3,184 +3,267 @@ "evidence_policy": "digest-only", "network_policy": "isolated-fixtures-only", "non_authorizing": true, - "scenario_commands": { - "ciphertext-tamper": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_confidentiality.py" - ], - "confidentiality-downgrade": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_confidentiality.py" - ], - "confidentiality-key-missing": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_confidentiality.py" - ], - "config-malformed": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_cli_ux.py" - ], - "config-missing": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_cli_ux.py" - ], - "config-permission-denied": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_operator_identity.py" - ], - "credential-expired": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_dispatch_ticket_trust_runtime.py" - ], - "credential-missing": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_current_node_token.py" - ], - "credential-replayed": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_dispatch_ticket_trust_store.py" - ], - "credential-rotated": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_dispatch_ticket_trust_store.py" - ], - "cross-flavor-equivalence": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_conformance.py" - ], - "directory-unavailable": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_restricted_directory.py" - ], - "dns-failure": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_endpoint_security.py" - ], - "duplicate-registration": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_register_operator_wiring.py" - ], - "minimum-version": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_cli_ux.py" - ], - "offline-locked-install": [ - "@python", - "scripts/run_sdk_quality.py" - ], - "package-version-self-report": [ - "@python", - "scripts/run_sdk_quality.py" - ], - "partial-response": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_conformance.py" - ], - "process-crash-restart": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_service_lifecycle_identity.py" - ], - "rate-limit": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_tunnel.py" - ], - "request-timeout": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_client.py" - ], - "rollback-last-supported": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_updater.py" - ], - "signature-mismatch": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_dispatch_ticket_trust_crypto.py" - ], - "tls-failure": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_endpoint_security.py" - ], - "unsupported-version": [ + "scenarios": { + "ciphertext-tamper": { + "assertion": "test_tampered_ciphertext_fails", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_confidentiality.py::test_tampered_ciphertext_fails" + ] + }, + "confidentiality-downgrade": { + "assertion": "test_p0a_no_key_is_refused_by_default", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_client.py::test_p0a_no_key_is_refused_by_default" + ] + }, + "confidentiality-key-missing": { + "assertion": "test_p0a_skips_keyless_node_and_uses_keyed_candidate", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_client.py::test_p0a_skips_keyless_node_and_uses_keyed_candidate" + ] + }, + "config-malformed": { + "assertion": "test_malformed_node_configuration_fails_closed", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_pre1_runtime_boundaries.py::test_malformed_node_configuration_fails_closed" + ] + }, + "config-missing": { + "assertion": "test_missing_node_configuration_is_explicitly_absent", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_pre1_runtime_boundaries.py::test_missing_node_configuration_is_explicitly_absent" + ] + }, + "config-permission-denied": { + "assertion": "test_permission_denied_config_write_leaves_no_file", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_pre1_runtime_boundaries.py::test_permission_denied_config_write_leaves_no_file" + ] + }, + "credential-expired": { + "assertion": "test_expired_dispatch_ticket_key_fails_closed", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_dispatch_ticket_trust_crypto.py::test_expired_dispatch_ticket_key_fails_closed" + ] + }, + "credential-missing": { + "assertion": "test_missing_restricted_credential_fails_before_network", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_restricted_directory.py::test_missing_restricted_credential_fails_before_network" + ] + }, + "credential-replayed": { + "assertion": "test_replayed_dispatch_ticket_fails_closed", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_dispatch_ticket_trust_crypto.py::test_replayed_dispatch_ticket_fails_closed" + ] + }, + "credential-rotated": { + "assertion": "test_revoked_dispatch_ticket_key_fails_closed_after_rotation", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_dispatch_ticket_trust_crypto.py::test_revoked_dispatch_ticket_key_fails_closed_after_rotation" + ] + }, + "cross-flavor-equivalence": { + "assertion": "test_profile_fixture_native_policy_scenarios_use_real_routing_gate", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_policy_manifest.py::test_profile_fixture_native_policy_scenarios_use_real_routing_gate" + ] + }, + "directory-unavailable": { + "assertion": "test_restricted_directory_failure_does_not_fall_back", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_restricted_directory.py::test_restricted_directory_failure_does_not_fall_back" + ] + }, + "dns-failure": { + "assertion": "test_dns_error_detection_covers_macos_resolver_wording", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_tunnel.py::TestSetup::test_dns_error_detection_covers_macos_resolver_wording" + ] + }, + "duplicate-registration": { + "assertion": "test_e050_client_credential_lifecycle_fixture", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_current_node_token.py::test_e050_client_credential_lifecycle_fixture" + ] + }, + "minimum-version": { + "assertion": "test_minimum_python_version_is_declared_and_candidate_remains_pre1", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_pre1_release_boundaries.py::test_minimum_python_version_is_declared_and_candidate_remains_pre1" + ] + }, + "offline-locked-install": { + "assertion": "test_offline_candidate_contract_pins_locked_release_inputs", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_pre1_release_boundaries.py::test_offline_candidate_contract_pins_locked_release_inputs" + ] + }, + "package-version-self-report": { + "assertion": "test_package_version_self_report_matches_candidate_contract", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_pre1_release_boundaries.py::test_package_version_self_report_matches_candidate_contract" + ] + }, + "partial-response": { + "assertion": "test_returns_fail_when_required_field_missing", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_conformance.py::TestConfHealth01::test_returns_fail_when_required_field_missing" + ] + }, + "process-crash-restart": { + "assertion": "test_sqlite_two_process_crash_recovery_and_single_terminal_winner", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_service_lifecycle.py::test_sqlite_two_process_crash_recovery_and_single_terminal_winner" + ] + }, + "rate-limit": { + "assertion": "test_relay_bind_rate_limit_is_per_source_and_recovers", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_relay_session.py::test_relay_bind_rate_limit_is_per_source_and_recovers" + ] + }, + "request-timeout": { + "assertion": "test_timeout_is_classified_deterministically", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_backends.py::test_timeout_is_classified_deterministically" + ] + }, + "rollback-last-supported": { + "assertion": "test_auto_update_tick_failed_upgrade_does_not_reexec", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_updater.py::test_auto_update_tick_failed_upgrade_does_not_reexec" + ] + }, + "signature-mismatch": { + "assertion": "test_tampered_dispatch_ticket_signature_fails_closed", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_dispatch_ticket_trust_crypto.py::test_tampered_dispatch_ticket_signature_fails_closed" + ] + }, + "tls-failure": { + "assertion": "test_tls_handshake_failure_is_transient_and_bounded", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_endpoint_security.py::test_tls_handshake_failure_is_transient_and_bounded" + ] + }, + "unsupported-version": { + "assertion": "test_parse_version_rejects_non_stable_candidates", + "command": [ + "@python", + "-m", + "pytest", + "-q", + "tests/test_updater.py::TestVersionCompare::test_parse_version_rejects_non_stable_candidates" + ] + } + }, + "schema": "iicp.pre1-component-case-map.v2", + "support": { + "assertion": "test_candidate_ranker_runs_after_policy_eligibility_and_records_exploration", + "command": [ "@python", "-m", "pytest", "-q", - "tests/test_updater.py" + "tests/test_client.py::test_candidate_ranker_runs_after_policy_eligibility_and_records_exploration" ] - }, - "schema": "iicp.pre1-component-case-map.v1", - "support_command": [ - "@python", - "-m", - "pytest", - "-q", - "tests/test_client.py", - "tests/test_conformance.py" - ] + } } diff --git a/scripts/run_pre1_qualification_case.py b/scripts/run_pre1_qualification_case.py index a5feae2..67f7e65 100755 --- a/scripts/run_pre1_qualification_case.py +++ b/scripts/run_pre1_qualification_case.py @@ -22,15 +22,54 @@ CASE_MAP_PATH = ROOT / "qualification/pre1-cases.json" CASE_MAP = json.loads(CASE_MAP_PATH.read_text(encoding="utf-8")) if ( - CASE_MAP.get("schema") != "iicp.pre1-component-case-map.v1" + CASE_MAP.get("schema") != "iicp.pre1-component-case-map.v2" or CASE_MAP.get("component") != COMPONENT or CASE_MAP.get("network_policy") != "isolated-fixtures-only" or CASE_MAP.get("evidence_policy") != "digest-only" or CASE_MAP.get("non_authorizing") is not True ): raise RuntimeError("invalid component qualification case map") -SUPPORT_COMMAND = CASE_MAP["support_command"] -SCENARIO_COMMANDS = CASE_MAP["scenario_commands"] +SUPPORT_CASE = CASE_MAP["support"] +SCENARIO_CASES = CASE_MAP["scenarios"] + + +def _case_command(value: object, label: str) -> list[str]: + if not isinstance(value, dict) or set(value) != {"assertion", "command"}: + raise RuntimeError(f"invalid exact qualification case: {label}") + assertion = value.get("assertion") + command = value.get("command") + exact_pytest = ( + isinstance(command, list) + and len(command) == 5 + and command[:4] == ["@python", "-m", "pytest", "-q"] + and isinstance(command[4], str) + and command[4].startswith("tests/") + and command[4].split("::", 1)[0].endswith(".py") + and command[4].endswith(f"::{assertion}") + ) + if ( + not isinstance(assertion, str) + or not assertion + or not isinstance(command, list) + or not command + or not all(isinstance(row, str) and row for row in command) + or not exact_pytest + ): + raise RuntimeError(f"qualification case is not an exact pytest assertion: {label}") + return command + + +SUPPORT_COMMAND = _case_command(SUPPORT_CASE, "support") +SCENARIO_COMMANDS = { + scenario: _case_command(value, scenario) + for scenario, value in SCENARIO_CASES.items() +} +_assertions = [ + SUPPORT_CASE["assertion"], + *(row["assertion"] for row in SCENARIO_CASES.values()), +] +if len(_assertions) != len(set(_assertions)): + raise RuntimeError("qualification cases must bind unique exact assertions") def canonical_sha256(value: object) -> str: @@ -51,6 +90,21 @@ def description() -> dict: "evidence_policy": "digest-only", "artifact_consumption": "verified-candidate-root", "source_commit_binding": True, + "semantic_binding": { + "contract": "iicp.pre1-semantic-assertion-binding.v1", + "exact_assertion_per_scenario": True, + "cell_dimensions_consumed": [ + "runtime", + "target", + "directory_flavor", + "mode", + "cell_id", + "scenario_id", + ], + "negative_controls_passed": True, + "packaged_artifact_gate": "required-per-execution", + "offline_environment_gate": "required-per-execution", + }, "commands_sha256": canonical_sha256({"support": SUPPORT_COMMAND, "scenarios": SCENARIO_COMMANDS}), "non_authorizing": True, } @@ -90,8 +144,154 @@ def load_json_environment(name: str) -> tuple[Path, dict]: return path, value -def validate_context(cell: str, scenario: str | None) -> tuple[str, dict, dict]: - _component, runtime, target, _directory, _mode = parse_cell(cell) +def valid_digest(value: object) -> bool: + return isinstance(value, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", value) is not None + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def artifact_materialization_sha256(manifest: dict, artifact_root: Path) -> str: + records: list[dict] = [] + expected_directories = {row["id"] for row in manifest.get("components", [])} + if {entry.name for entry in artifact_root.iterdir()} != expected_directories: + raise ValueError("qualification artifact materialization component set differs") + for component in manifest.get("components", []): + component_root = artifact_root / component["id"] + expected = {row["name"] for row in component.get("artifacts", [])} | { + "package-manifest.json", + "build-receipt.json", + } + if ( + not component_root.is_dir() + or component_root.is_symlink() + or {entry.name for entry in component_root.iterdir()} != expected + ): + raise ValueError("qualification component artifact set differs") + for artifact in component.get("artifacts", []): + path = component_root / artifact["name"] + if not path.is_file() or path.is_symlink(): + raise ValueError("qualification artifact is unsafe") + digest = file_sha256(path) + size = path.stat().st_size + if digest != artifact.get("sha256") or size != artifact.get("size_bytes"): + raise ValueError("qualification artifact digest or size differs") + records.append( + { + "component": component["id"], + "name": artifact["name"], + "kind": artifact["kind"], + "target": artifact["target"], + "size_bytes": size, + "sha256": digest, + } + ) + for name, kind, digest_field in ( + ("package-manifest.json", "package_manifest", "package_manifest_sha256"), + ("build-receipt.json", "build_receipt", "build_receipt_sha256"), + ): + path = component_root / name + if not path.is_file() or path.is_symlink(): + raise ValueError("qualification artifact companion is unsafe") + digest = file_sha256(path) + if digest != component.get(digest_field): + raise ValueError("qualification artifact companion digest differs") + records.append( + { + "component": component["id"], + "name": name, + "kind": kind, + "target": "any", + "size_bytes": path.stat().st_size, + "sha256": digest, + } + ) + return canonical_sha256(sorted(records, key=lambda row: (row["component"], row["name"]))) + + +def semantic_execution_context( + cell: str, + scenario: str | None, + candidate_digest: str, + materialization_digest: str, + runtime_map_digest: str, + environment_digest: str, +) -> dict: + component, runtime, target, directory, mode = parse_cell(cell) + return { + "schema": "iicp.pre1-semantic-execution-context.v1", + "component": component, + "runtime": runtime, + "target": target, + "directory_flavor": directory, + "mode": mode, + "cell_id": cell, + "scenario_id": scenario or "support", + "candidate_manifest_sha256": candidate_digest, + "artifact_materialization_sha256": materialization_digest, + "runtime_map_sha256": runtime_map_digest, + "qualification_environment_sha256": environment_digest, + "network_policy": "isolated-fixtures-only", + "evidence_policy": "digest-only", + } + + +def _validate_environment_manifest( + value: dict, + *, + target: str, + runtime: str, + candidate_digest: str, + materialization_digest: str, + runtime_map_digest: str, +) -> str: + if ( + value.get("schema") != "iicp.pre1-qualification-environment.v1" + or value.get("status") != "READY" + or value.get("target") != target + or value.get("content_free") is not True + or value.get("secrets_present") is not False + or value.get("non_authorizing") is not True + ): + raise ValueError("qualification environment boundary differs") + bindings = value.get("bindings", {}) + expected = { + "candidate_manifest_sha256": candidate_digest, + "artifact_materialization_sha256": materialization_digest, + "runtime_map_sha256": runtime_map_digest, + } + if any(bindings.get(name) != digest for name, digest in expected.items()): + raise ValueError("qualification environment immutable binding differs") + runtime_row = value.get("runtimes", {}).get(runtime) + if not isinstance(runtime_row, dict) or any( + runtime_row.get(field) != "PASS" + for field in ( + "online_prepare_status", + "offline_install_status", + "package_artifact_smoke_status", + ) + ): + raise ValueError("qualification package or offline environment gate differs") + if ( + runtime_row.get("egress_disabled_during_offline") is not True + or runtime_row.get("empty_volatile_cache_at_start") is not True + ): + raise ValueError("qualification offline safety boundary differs") + claimed = value.get("environment_sha256") + copy = json.loads(json.dumps(value)) + copy["environment_sha256"] = None + if not valid_digest(claimed) or claimed != canonical_sha256(copy): + raise ValueError("qualification environment digest differs") + return str(claimed) + + +def validate_context(cell: str, scenario: str | None) -> tuple[str, dict, dict, dict]: + _component, runtime, target, directory, mode = parse_cell(cell) if detected_target() != target: raise ValueError("qualification target differs from the actual host") if os.environ.get("IICP_PRE1_CELL_ID") != cell: @@ -111,7 +311,7 @@ def validate_context(cell: str, scenario: str | None) -> tuple[str, dict, dict]: ): raise ValueError("qualification HOME is not isolated") - manifest_path, manifest = load_json_environment("IICP_PRE1_CANDIDATE_MANIFEST") + _manifest_path, manifest = load_json_environment("IICP_PRE1_CANDIDATE_MANIFEST") candidate_digest = os.environ.get("IICP_PRE1_CANDIDATE_DIGEST") if manifest.get("status") != "FROZEN" or manifest.get("immutable") is not True or manifest.get("manifest_sha256") != candidate_digest: raise ValueError("qualification candidate binding is not immutable") @@ -126,9 +326,9 @@ def validate_context(cell: str, scenario: str | None) -> tuple[str, dict, dict]: component_root = artifact_root / COMPONENT if not artifact_root.is_dir() or artifact_root.is_symlink() or not component_root.is_dir() or component_root.is_symlink(): raise ValueError("qualification artifact root is unsafe") - expected = {row["name"] for row in component.get("artifacts", [])} | {"package-manifest.json", "build-receipt.json"} - if {entry.name for entry in component_root.iterdir()} != expected or any(entry.is_symlink() for entry in component_root.iterdir()): - raise ValueError("qualification component artifact set differs") + materialization_digest = artifact_materialization_sha256(manifest, artifact_root) + if os.environ.get("IICP_PRE1_ARTIFACT_MATERIALIZATION_SHA256") != materialization_digest: + raise ValueError("qualification artifact materialization binding differs") _runtime_path, runtime_map = load_json_environment("IICP_PRE1_RUNTIME_MAP") if runtime_map.get("schema") != "iicp.pre1-runtime-map.v1" or runtime_map.get("target") != target: @@ -136,14 +336,53 @@ def validate_context(cell: str, scenario: str | None) -> tuple[str, dict, dict]: runtime_row = runtime_map.get("runtimes", {}).get(runtime) if not isinstance(runtime_row, dict): raise ValueError("qualification runtime is unavailable") - return runtime, runtime_row, manifest + runtime_map_digest = runtime_map.get("map_sha256") + if ( + not valid_digest(runtime_map_digest) + or os.environ.get("IICP_PRE1_RUNTIME_MAP_SHA256") != runtime_map_digest + ): + raise ValueError("qualification runtime map binding differs") + _environment_path, environment = load_json_environment( + "IICP_PRE1_ENVIRONMENT_MANIFEST" + ) + environment_digest = _validate_environment_manifest( + environment, + target=target, + runtime=runtime, + candidate_digest=str(candidate_digest), + materialization_digest=materialization_digest, + runtime_map_digest=str(runtime_map_digest), + ) + if os.environ.get("IICP_PRE1_QUALIFICATION_ENVIRONMENT_SHA256") != environment_digest: + raise ValueError("qualification environment binding differs") + context = semantic_execution_context( + cell, + scenario, + str(candidate_digest), + materialization_digest, + str(runtime_map_digest), + environment_digest, + ) + expected_environment = { + "IICP_PRE1_RUNTIME": runtime, + "IICP_PRE1_TARGET": target, + "IICP_PRE1_DIRECTORY_FLAVOR": directory, + "IICP_PRE1_MODE": mode, + "IICP_PRE1_CONTEXT_SHA256": canonical_sha256(context), + } + if any(os.environ.get(name) != value for name, value in expected_environment.items()): + raise ValueError("qualification semantic context differs") + return runtime, runtime_row, manifest, context def expected_runtime_version(runtime: str, manifest: dict) -> str: if runtime.startswith("cpython-") or runtime.startswith("node-") or runtime.startswith("php-"): return runtime.split("-", 1)[1] - if runtime == "candidate-stable-pinned": - return str(manifest["toolchains"]["rust_stable"]) + if runtime.startswith("rust-"): + expected = str(manifest["toolchains"]["rust_stable"]) + if runtime != f"rust-{expected}": + raise ValueError("qualification stable Rust runtime differs from the candidate") + return expected field = {"client-rust": "rust_client_msrv", "directory-rust": "rust_directory_msrv", "management": "management_msrv"}[COMPONENT] return str(manifest["toolchains"][field]) @@ -172,7 +411,25 @@ def validate_runtime(runtime: str, runtime_row: dict, manifest: dict) -> None: def command_environment(runtime_row: dict, runtime: str) -> dict[str, str]: allowed = {"PATH", "HOME", "IICP_HOME", "LANG", "LC_ALL", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT", "TEMP", "TMP", "TMPDIR", "RUSTUP_HOME"} env = {key: value for key, value in os.environ.items() if key.upper() in allowed} - for name in ("IICP_PRE1_CELL_ID", "IICP_PRE1_SCENARIO_ID", "IICP_PRE1_NETWORK_POLICY", "IICP_PRE1_EVIDENCE_POLICY", "IICP_PRE1_ARTIFACT_ROOT", "IICP_PRE1_CANDIDATE_MANIFEST", "IICP_PRE1_CANDIDATE_DIGEST", "IICP_PRE1_RUNTIME_MAP"): + for name in ( + "IICP_PRE1_CELL_ID", + "IICP_PRE1_SCENARIO_ID", + "IICP_PRE1_NETWORK_POLICY", + "IICP_PRE1_EVIDENCE_POLICY", + "IICP_PRE1_ARTIFACT_ROOT", + "IICP_PRE1_CANDIDATE_MANIFEST", + "IICP_PRE1_CANDIDATE_DIGEST", + "IICP_PRE1_RUNTIME_MAP", + "IICP_PRE1_ENVIRONMENT_MANIFEST", + "IICP_PRE1_RUNTIME", + "IICP_PRE1_TARGET", + "IICP_PRE1_DIRECTORY_FLAVOR", + "IICP_PRE1_MODE", + "IICP_PRE1_ARTIFACT_MATERIALIZATION_SHA256", + "IICP_PRE1_RUNTIME_MAP_SHA256", + "IICP_PRE1_QUALIFICATION_ENVIRONMENT_SHA256", + "IICP_PRE1_CONTEXT_SHA256", + ): env[name] = os.environ[name] env.update(runtime_row.get("env", {})) cache = Path(env["IICP_HOME"]) / "qualification-cache" / COMPONENT / runtime @@ -184,6 +441,22 @@ def command_environment(runtime_row: dict, runtime: str) -> dict[str, str]: env["npm_config_audit"] = "false" env["npm_config_update_notifier"] = "false" env["NO_COLOR"] = "1" + manifest = json.loads(Path(env["IICP_PRE1_CANDIDATE_MANIFEST"]).read_text()) + component = next(row for row in manifest["components"] if row["id"] == COMPONENT) + release_manifest = next( + ( + row + for row in component["artifacts"] + if row["kind"] == "release-manifest" + ), + None, + ) + if release_manifest is not None: + env["IICP_RELEASE_MANIFEST"] = str( + Path(env["IICP_PRE1_ARTIFACT_ROOT"]) + / COMPONENT + / release_manifest["name"] + ) return env @@ -213,7 +486,9 @@ def main() -> int: if args.scenario is not None and args.scenario not in SCENARIO_COMMANDS: parser.error("scenario is not owned by this component") try: - runtime, runtime_row, manifest = validate_context(args.cell, args.scenario) + runtime, runtime_row, manifest, _context = validate_context( + args.cell, args.scenario + ) validate_runtime(runtime, runtime_row, manifest) template = SCENARIO_COMMANDS[args.scenario] if args.scenario else SUPPORT_COMMAND argv = expand_command(template, runtime_row) diff --git a/scripts/test_pre1_qualification_case.py b/scripts/test_pre1_qualification_case.py index 7298620..73b4c59 100755 --- a/scripts/test_pre1_qualification_case.py +++ b/scripts/test_pre1_qualification_case.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from __future__ import annotations +import copy import importlib.util import unittest from pathlib import Path @@ -23,6 +24,18 @@ def test_description_is_complete_and_content_free(self) -> None: self.assertEqual(value["component"], module.COMPONENT) self.assertEqual(value["scenarios"], sorted(module.SCENARIO_COMMANDS)) self.assertTrue(value["commands_sha256"].startswith("sha256:")) + self.assertTrue(value["semantic_binding"]["exact_assertion_per_scenario"]) + self.assertEqual( + set(value["semantic_binding"]["cell_dimensions_consumed"]), + { + "runtime", + "target", + "directory_flavor", + "mode", + "cell_id", + "scenario_id", + }, + ) self.assertTrue(value["non_authorizing"]) def test_cell_parser_rejects_wrong_component_and_boundary(self) -> None: @@ -44,9 +57,115 @@ def test_cell_parser_rejects_wrong_component_and_boundary(self) -> None: def test_referenced_test_files_exist(self) -> None: commands = [module.SUPPORT_COMMAND, *module.SCENARIO_COMMANDS.values()] for command in commands: - for token in command: - if token.startswith(("tests/", "scripts/")) and "." in Path(token).name: - self.assertTrue((ROOT / token).is_file(), token) + node_id = command[4] + assertion = node_id.rsplit("::", 1)[-1] + source = ROOT / node_id.split("::", 1)[0] + marker = f"def {assertion}(" + self.assertTrue(source.is_file(), source) + self.assertIn(marker, source.read_text()) + + def test_every_scenario_has_one_unique_exact_assertion(self) -> None: + self.assertEqual(set(module.SCENARIO_CASES), set(module.SCENARIO_COMMANDS)) + assertions = [row["assertion"] for row in module.SCENARIO_CASES.values()] + self.assertEqual(len(assertions), len(set(assertions))) + self.assertNotIn(module.SUPPORT_CASE["assertion"], assertions) + + def test_semantic_context_negative_controls_change_the_binding(self) -> None: + base = ( + "client-python|cpython-3.11|macos-arm64|rust|restricted", + "rate-limit", + "sha256:" + "a" * 64, + "sha256:" + "b" * 64, + "sha256:" + "c" * 64, + "sha256:" + "d" * 64, + ) + expected = module.canonical_sha256(module.semantic_execution_context(*base)) + mutations = [ + (base[0].replace("cpython-3.11", "cpython-3.14"), *base[1:]), + (base[0].replace("macos-arm64", "windows-x86_64"), *base[1:]), + (base[0].replace("|rust|", "|php|"), *base[1:]), + (base[0].replace("restricted", "public"), *base[1:]), + (base[0], "disk-full", *base[2:]), + (*base[:2], "sha256:" + "e" * 64, *base[3:]), + (*base[:3], "sha256:" + "e" * 64, *base[4:]), + (*base[:4], "sha256:" + "e" * 64, base[5]), + (*base[:5], "sha256:" + "e" * 64), + ] + for mutation in mutations: + with self.subTest(mutation=mutation[:2]): + try: + observed = module.canonical_sha256( + module.semantic_execution_context(*mutation) + ) + except ValueError: + continue + self.assertNotEqual(observed, expected) + + def test_environment_manifest_requires_offline_package_smoke_and_digest(self) -> None: + candidate = "sha256:" + "a" * 64 + artifacts = "sha256:" + "b" * 64 + runtime_map = "sha256:" + "c" * 64 + value = { + "schema": "iicp.pre1-qualification-environment.v1", + "status": "READY", + "target": "macos-arm64", + "bindings": { + "candidate_manifest_sha256": candidate, + "artifact_materialization_sha256": artifacts, + "runtime_map_sha256": runtime_map, + "runner_inventory_sha256": "sha256:" + "d" * 64, + }, + "network": {}, + "source_state": {}, + "runtimes": { + "cpython-3.11": { + "lock_inputs_sha256": "sha256:" + "e" * 64, + "dependency_cache_sha256": "sha256:" + "f" * 64, + "online_prepare_status": "PASS", + "offline_install_status": "PASS", + "package_artifact_smoke_status": "PASS", + "egress_disabled_during_offline": True, + "empty_volatile_cache_at_start": True, + } + }, + "content_free": True, + "secrets_present": False, + "non_authorizing": True, + "environment_sha256": None, + } + value["environment_sha256"] = module.canonical_sha256(value) + self.assertEqual( + module._validate_environment_manifest( + value, + target="macos-arm64", + runtime="cpython-3.11", + candidate_digest=candidate, + materialization_digest=artifacts, + runtime_map_digest=runtime_map, + ), + value["environment_sha256"], + ) + for mutation in ( + ("offline_install_status", "FAIL"), + ("package_artifact_smoke_status", "FAIL"), + ("egress_disabled_during_offline", False), + ): + changed = copy.deepcopy(value) + changed["runtimes"]["cpython-3.11"][mutation[0]] = mutation[1] + changed["environment_sha256"] = None + changed["environment_sha256"] = module.canonical_sha256(changed) + with self.assertRaises(ValueError): + module._validate_environment_manifest( + changed, + target="macos-arm64", + runtime="cpython-3.11", + candidate_digest=candidate, + materialization_digest=artifacts, + runtime_map_digest=runtime_map, + ) + + def test_python_runtime_is_exactly_bound_to_cell(self) -> None: + self.assertEqual(module.expected_runtime_version("cpython-3.11", {}), "3.11") if __name__ == "__main__": diff --git a/tests/test_dispatch_ticket_trust_crypto.py b/tests/test_dispatch_ticket_trust_crypto.py index a36a095..4fbfbaf 100644 --- a/tests/test_dispatch_ticket_trust_crypto.py +++ b/tests/test_dispatch_ticket_trust_crypto.py @@ -32,6 +32,43 @@ def _decision(vector: dict, keys: dict[str, dict], signature_valid: bool) -> str return "accept_anchored" +def _assert_fixture_decision(vector_id: str, expected: str) -> None: + fixture = json.loads( + (Path(__file__).parents[1] / "parity" / "dispatch-ticket-trust-v2-crypto.json").read_text() + ) + domain = _decode(fixture["domain_separator_b64url"]) + keys = {key["key_id"]: key for key in fixture["keys"]} + vector = next(value for value in fixture["vectors"] if value["id"] == vector_id) + public_key = Ed25519PublicKey.from_public_bytes( + _decode(keys[vector["claims"]["key_id"]]["public_key_b64url"]) + ) + try: + public_key.verify( + _decode(vector["signature_b64url"]), + domain + _canonical(vector["claims"]), + ) + signature_valid = True + except InvalidSignature: + signature_valid = False + assert _decision(vector, keys, signature_valid) == expected + + +def test_expired_dispatch_ticket_key_fails_closed() -> None: + _assert_fixture_decision("expired_key_refused", "reject_key_expired") + + +def test_replayed_dispatch_ticket_fails_closed() -> None: + _assert_fixture_decision("local_replay_refused", "reject_local_replay") + + +def test_revoked_dispatch_ticket_key_fails_closed_after_rotation() -> None: + _assert_fixture_decision("revoked_key_refused", "reject_key_revoked") + + +def test_tampered_dispatch_ticket_signature_fails_closed() -> None: + _assert_fixture_decision("tampered_claim_refused", "reject_signature") + + def test_dispatch_ticket_v2_signed_vectors_are_portable() -> None: fixture = json.loads((Path(__file__).parents[1] / "parity" / "dispatch-ticket-trust-v2-crypto.json").read_text()) domain = _decode(fixture["domain_separator_b64url"]) diff --git a/tests/test_endpoint_security.py b/tests/test_endpoint_security.py index cd86fc1..eaf026b 100644 --- a/tests/test_endpoint_security.py +++ b/tests/test_endpoint_security.py @@ -82,6 +82,27 @@ async def test_private_provider_requires_opt_in_and_uses_pinned_transport(monkey thread.join(timeout=2) +@pytest.mark.asyncio +async def test_tls_handshake_failure_is_transient_and_bounded() -> None: + server = ThreadingHTTPServer(("127.0.0.1", 0), _ProviderHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with pytest.raises(IicpError) as failure: + await post_json( + f"https://127.0.0.1:{server.server_port}/task", + {}, + timeout_ms=2_000, + tls_verify=True, + ) + assert failure.value.code == "IICP-E004" + assert failure.value.retryable is True + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_shared_fixture_matches_python_policy() -> None: fixture = json.loads((Path(__file__).parent / "fixtures" / "endpoint-security-v1.json").read_text()) for vector in fixture["address_vectors"]: diff --git a/tests/test_pre1_release_boundaries.py b/tests/test_pre1_release_boundaries.py new file mode 100644 index 0000000..b143976 --- /dev/null +++ b/tests/test_pre1_release_boundaries.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import sys +import tomllib +from pathlib import Path + +import iicp_client + +ROOT = Path(__file__).parents[1] +PROJECT = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] +QUALITY_RUNNER = (ROOT / "scripts" / "run_sdk_quality.py").read_text(encoding="utf-8") +RELEASE_WORKFLOW = (ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8") + + +def test_minimum_python_version_is_declared_and_candidate_remains_pre1() -> None: + assert PROJECT["requires-python"] == ">=3.11" + assert sys.version_info >= (3, 11) + assert PROJECT["version"].split(".", 1)[0] == "0" + + +def test_package_version_self_report_matches_candidate_contract() -> None: + assert PROJECT["name"] == "iicp-client" + assert iicp_client.__version__ == PROJECT["version"] + + +def test_offline_candidate_contract_pins_locked_release_inputs() -> None: + lock = (ROOT / "uv.lock").read_text(encoding="utf-8") + assert 'name = "iicp-client"' in lock + assert f'version = "{PROJECT["version"]}"' in lock + assert '"uv", "run", "--isolated", "--python", runtime, "--locked"' in QUALITY_RUNNER + assert "python -m build" in RELEASE_WORKFLOW + assert "python -m venv /tmp/iicp-release-smoke" in RELEASE_WORKFLOW + assert "pip install dist/*.whl" in RELEASE_WORKFLOW diff --git a/tests/test_pre1_runtime_boundaries.py b/tests/test_pre1_runtime_boundaries.py new file mode 100644 index 0000000..c477469 --- /dev/null +++ b/tests/test_pre1_runtime_boundaries.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from iicp_client.identity import NodeIdentity, load_node, node_path, save_node + + +def test_malformed_node_configuration_fails_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("IICP_HOME", str(tmp_path)) + path = node_path("malformed") + path.write_text("{not-json", encoding="utf-8") + with pytest.raises(json.JSONDecodeError): + load_node("malformed") + + +def test_missing_node_configuration_is_explicitly_absent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("IICP_HOME", str(tmp_path)) + assert load_node("missing") is None + + +def test_permission_denied_config_write_leaves_no_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("IICP_HOME", str(tmp_path)) + node = NodeIdentity.generate( + operator_id="operator-test", + name="permission-denied", + backend_url="http://127.0.0.1:11434", + model="test-model", + ) + destination = node_path(node.name) + original = Path.write_text + + def refuse(path: Path, *_args: object, **_kwargs: object) -> int: + if path == destination: + raise PermissionError("simulated permission denied") + return original(path, *_args, **_kwargs) + + monkeypatch.setattr(Path, "write_text", refuse) + with pytest.raises(PermissionError, match="permission denied"): + save_node(node) + assert not destination.exists() diff --git a/tests/test_restricted_directory.py b/tests/test_restricted_directory.py index ed3a40c..6ece400 100644 --- a/tests/test_restricted_directory.py +++ b/tests/test_restricted_directory.py @@ -35,6 +35,12 @@ def test_context_and_decision_fail_closed(tmp_path): SecretRef("file", str(secret)).resolve() +def test_missing_restricted_credential_fails_before_network(monkeypatch): + monkeypatch.delenv("IICP_TEST_MISSING_RESTRICTED_MEMBER", raising=False) + with pytest.raises(IicpError, match="credential is unavailable"): + SecretRef("environment", "IICP_TEST_MISSING_RESTRICTED_MEMBER").resolve() + + @pytest.mark.asyncio async def test_restricted_discovery_sends_membership_and_requires_decision(monkeypatch): seen: dict[str, str] = {} @@ -52,6 +58,36 @@ async def handler(request: httpx.Request) -> httpx.Response: assert seen["x-iicp-subject-id"] == "client-a" +@pytest.mark.asyncio +async def test_restricted_directory_failure_does_not_fall_back(monkeypatch): + calls = [] + + async def handler(request: httpx.Request) -> httpx.Response: + calls.append(str(request.url)) + raise httpx.ConnectError("isolated directory unavailable", request=request) + + original = httpx.AsyncClient + monkeypatch.setattr( + httpx, + "AsyncClient", + lambda *args, **kwargs: original( + transport=httpx.MockTransport(handler), + **{key: value for key, value in kwargs.items() if key != "transport"}, + ), + ) + client = IicpClient( + ClientConfig( + directory_url="https://directory.test", + route_discovery_mode="ticketed", + restricted_directory=context(), + ) + ) + with pytest.raises(IicpError, match="Network error"): + await client.discover_async("urn:iicp:intent:llm:chat:v1") + assert len(calls) == 1 + assert calls[0].startswith("https://directory.test/v1/discover?") + + def test_restricted_mode_refuses_legacy_fallback(): with pytest.raises(ValueError, match="legacy"): IicpClient(ClientConfig(route_discovery_mode="legacy", restricted_directory=context()))