From a8f15a1875163c397390443b17a7bd27a21891c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:11:10 +0300 Subject: [PATCH 001/119] Recover minimal catalog and Dolt lifecycle checkpoint --- .github/workflows/ci.yml | 6 +- CHANGELOG.md | 19 + pyproject.toml | 2 +- src/openstatspec/__init__.py | 23 +- src/openstatspec/api.py | 101 +- src/openstatspec/cli.py | 12 +- src/openstatspec/core/__init__.py | 14 + src/openstatspec/spss/__init__.py | 22 +- src/openstatspec/spss/sav.py | 570 ++++- src/openstatspec/sql/__init__.py | 62 +- src/openstatspec/sql/capabilities.py | 585 +++-- src/openstatspec/sql/dolt_conformance.py | 199 ++ src/openstatspec/sql/profiles.py | 141 +- src/openstatspec/sql/wide.py | 2593 +++++++++++++++++++--- src/openstatspec/sql/workflow.py | 5 +- tests/test_catalog_lifecycle.py | 336 +++ tests/test_dolt_conformance.py | 311 +++ 17 files changed, 4343 insertions(+), 658 deletions(-) create mode 100644 src/openstatspec/sql/dolt_conformance.py mode change 100755 => 100644 src/openstatspec/sql/profiles.py create mode 100644 tests/test_catalog_lifecycle.py create mode 100644 tests/test_dolt_conformance.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e31ca4e..5502f41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v7 with: &specification-checkout repository: OpenStatSpec/specification - ref: 79339ec3d8f8aa81789b7e85f6b8afa6f1374e50 + ref: agent/dolt-declaration-validator path: openstatspec-specification - name: Checkout required SPSS engine uses: actions/checkout@v7 @@ -32,6 +32,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - run: python -m pip install --upgrade pip + - run: python -m pip install ./openstatspec-specification - run: python -m pip install ./openstatspec-pyspssio - run: python -m pip install -e ".[dev]" - run: python -m pytest -m "not services" @@ -53,6 +54,7 @@ jobs: - run: python -m pip install --upgrade pip build - run: python -m build - run: python -m venv /tmp/openstatspec-wheel-smoke + - run: /tmp/openstatspec-wheel-smoke/bin/python -m pip install ./openstatspec-specification - run: /tmp/openstatspec-wheel-smoke/bin/python -m pip install ./openstatspec-pyspssio - run: /tmp/openstatspec-wheel-smoke/bin/python -m pip install dist/*.whl - run: /tmp/openstatspec-wheel-smoke/bin/openstatspec capabilities @@ -90,6 +92,7 @@ jobs: with: python-version: "3.12" - run: python -m pip install --upgrade pip + - run: python -m pip install ./openstatspec-specification - run: python -m pip install ./openstatspec-pyspssio - run: python -m pip install -e ".[dev,sql]" - run: python -m pytest -m services @@ -192,6 +195,7 @@ jobs: --workdir /var/lib/dolt \ "$OPENSTATSPEC_DOLT_IMAGE" sql-server -H 0.0.0.0 -l warning - run: python -m pip install --upgrade pip + - run: python -m pip install ./openstatspec-specification - run: python -m pip install ./openstatspec-pyspssio - run: python -m pip install -e ".[dev,sql]" - name: Wait for Dolt diff --git a/CHANGELOG.md b/CHANGELOG.md index edb3058..74e0b61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,17 @@ SPSS profile. - Export of supported dataset semantics to SAV and ZSAV. - SQLite, PostgreSQL, MySQL, and MariaDB profiles, including service-backed PostgreSQL 17/18, MySQL 8.4/9.7, and MariaDB 11.4/11.8/12.3 CI coverage. +- Read-only positive Dolt identity and working-set inspection. Operational + Dolt writes load the shared `openstatspec-specification` declaration + validator and remain fail-closed because the packaged concrete declaration + set is empty. +- Added explicit packaged/directory conformance-source injection and exact + single-match binding across active Dolt product version, adapter + implementation and version, and specification commit before mutation. +- Explicit catalog initialization/migration through `initialize_catalog` and + `openstatspec init`; data operations never auto-create catalog relations. +- Read-only Dolt working-set evidence through `dolt_state_snapshot` and + `openstatspec dolt-state`; core performs no Dolt version-control mutations. - Preflight checks for target profile limits, atomic imports, validation, a command-line interface, and machine-readable capability and loss reports. @@ -121,3 +132,11 @@ SPSS profile. - Multiple-response sets, variable alignment, variable sets, and custom attributes have explicit capability diagnostics; see the SAV profile. - Encrypted SPSS files are not supported. +- Dolt adapter-envelope values are not claims about Dolt server limits. Failure + recovery is compensating and reports deterministic residual inventory; a + verified catalog receives a best-effort failed-operation audit when cleanup + itself fails. +- Export publication uses a durable, audit-recorded prior-file backup through + SQL finalization. Post-success backup retention is reported separately and + does not rewrite a successful export as failed; pre-success failures restore + the prior destination and close the running operation as failed. diff --git a/pyproject.toml b/pyproject.toml index d16b5f5..9309c07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.11" license = {text = "Apache-2.0"} authors = [{name = "OpenStatSpec"}] -dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "sqlglot>=30.14", "rfc8785>=0.1.4"] +dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "openstatspec-specification==0.2.0", "sqlglot>=30.14", "rfc8785>=0.1.4"] [project.optional-dependencies] dev = ["pytest>=8"] diff --git a/src/openstatspec/__init__.py b/src/openstatspec/__init__.py index d9754f0..2cc1a7b 100644 --- a/src/openstatspec/__init__.py +++ b/src/openstatspec/__init__.py @@ -2,17 +2,17 @@ from .api import ( apply_spss_in_place, apply_transformation_plan_in_place, - capabilities, capability_matrix, derive_sql_dataset, - execute_sql_transformation, - export_sav, get_dataset, import_sav, inspect, - install_in_place_transformation_schema, list_datasets, + capabilities, capability_matrix, derive_sql_dataset, dolt_state_snapshot, execute_sql_transformation, + export_sav, get_dataset, import_sav, initialize_catalog, inspect, list_datasets, + install_in_place_transformation_schema, register_sql_transformation, reconcile_derived_removals, reconcile_sql_transformation_runs, remove_derived_physical_relation, retire_derived, validate, validate_derived, ) from .core import CapabilityDeclaration, LossReport, UnsupportedOperationError -from .sql.workflow import TransformationError from .frontends.spss import SpssFrontendCompilation, compile_spss_syntax +from .sql import DoltConformanceSource +from .sql.workflow import TransformationError from .transform import ( RecodeMatch, RecodeOperation, RecodeResult, RecodeRule, ReplaceValueLabelsOperation, SetVariableLabelOperation, @@ -21,18 +21,17 @@ ) __all__ = [ - "CapabilityDeclaration", "LossReport", "SpssFrontendCompilation", - "TransformationError", "TransformationFrontendError", + "CapabilityDeclaration", "DoltConformanceSource", "LossReport", "TransformationError", + "SpssFrontendCompilation", "TransformationFrontendError", "RecodeMatch", "RecodeOperation", "RecodeResult", "RecodeRule", "ReplaceValueLabelsOperation", "SetVariableLabelOperation", "TransformationPlan", "TypedValue", "ValueLabel", "VariableDefinition", "VariableSchema", "transformation_plan_from_dict", - "UnsupportedOperationError", "capabilities", "capability_matrix", "apply_spss_in_place", "apply_transformation_plan_in_place", - "compile_spss_syntax", "derive_sql_dataset", - "execute_sql_transformation", "export_sav", - "get_dataset", "import_sav", "inspect", - "install_in_place_transformation_schema", "list_datasets", + "compile_spss_syntax", "install_in_place_transformation_schema", + "UnsupportedOperationError", "capabilities", "capability_matrix", + "derive_sql_dataset", "dolt_state_snapshot", "execute_sql_transformation", "export_sav", "get_dataset", + "import_sav", "initialize_catalog", "inspect", "list_datasets", "register_sql_transformation", "reconcile_derived_removals", "reconcile_sql_transformation_runs", "remove_derived_physical_relation", "retire_derived", "validate", diff --git a/src/openstatspec/api.py b/src/openstatspec/api.py index 9de2e21..ba0efc8 100644 --- a/src/openstatspec/api.py +++ b/src/openstatspec/api.py @@ -7,7 +7,13 @@ from .core.results import result from .spss import export_dataset, import_dataset, inspect_source from .spss.sav import engine_identity -from .sql import declared_profiles, validate_dataset +from .sql import ( + DoltConformanceSource, + declared_profiles, + dolt_state_snapshot as _dolt_state_snapshot, + initialize_catalog as _initialize_catalog, + validate_dataset, +) from .sql.catalog_api import catalog_dataset as _catalog_dataset, catalog_datasets as _catalog_datasets from .sql.workflow import ( derive_dataset as _derive_dataset, @@ -33,7 +39,11 @@ -def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]: +def capability_matrix( + database_url: str | None = None, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> Mapping[str, Any]: """Return the pyspssio-backed SAV/ZSAV fidelity boundary. supported means that the adapter has a tested faithful path. @@ -86,9 +96,18 @@ def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]: "maximum_source_file_bytes": None, "limit_basis": "runtime memory and active SQL connection limits", }, - "active_connection": active_connection(database_url) if database_url else None, + "active_connection": ( + active_connection( + database_url, + dolt_conformance_source=dolt_conformance_source, + ) + if database_url else None + ), "catalog_binding": catalog_binding(database_url) if database_url else None, - "sql_profiles": declared_profiles(database_url), + "sql_profiles": declared_profiles( + database_url, + dolt_conformance_source=dolt_conformance_source, + ), "optional_profiles": { "sql_transformation_workflow": transformation_capabilities(database_url), "spss_in_place_transformation": ( @@ -98,26 +117,84 @@ def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]: } return declaration -def capabilities(database_url: str | None = None) -> Mapping[str, Any]: - return capability_matrix(database_url) +def capabilities( + database_url: str | None = None, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> Mapping[str, Any]: + return capability_matrix( + database_url, dolt_conformance_source=dolt_conformance_source, + ) def inspect(source: str | Path, /, **options: Any) -> Mapping[str, Any]: return result(inspect_source(source, **options)) -def import_sav(source: str | Path, /, *, database_url: Any, dataset_id: str, **options: Any) -> Mapping[str, Any]: +def dolt_state_snapshot( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> Mapping[str, Any]: + """Return read-only Dolt branch, HEAD, status, and diff-summary evidence.""" + return result(_dolt_state_snapshot( + database_url=database_url, + dolt_conformance_source=dolt_conformance_source, + )) + + +def initialize_catalog( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> Mapping[str, Any]: + """Install or explicitly migrate a dedicated OpenStatSpec catalog.""" + return result(_initialize_catalog( + database_url=database_url, + dolt_conformance_source=dolt_conformance_source, + )) + + +def import_sav( + source: str | Path, /, *, database_url: Any, dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> Mapping[str, Any]: """Import one source file into one dedicated wide SQL table.""" - return result(import_dataset(source, database_url=database_url, dataset_id=dataset_id, **options)) + return result(import_dataset( + source, database_url=database_url, dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + **options, + )) -def export_sav(*, database_url: Any, dataset_id: str, destination: str | Path, **options: Any) -> Mapping[str, Any]: +def export_sav( + *, database_url: Any, dataset_id: str, destination: str | Path, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> Mapping[str, Any]: """Export one database-resident conforming dataset to SAV/ZSAV.""" - return result(export_dataset(database_url=database_url, dataset_id=dataset_id, destination=destination, **options)) + return result(export_dataset( + database_url=database_url, dataset_id=dataset_id, + destination=destination, + dolt_conformance_source=dolt_conformance_source, + **options, + )) -def validate(*, database_url: Any, dataset_id: str, **options: Any) -> Mapping[str, Any]: - return result(validate_dataset(database_url=database_url, dataset_id=dataset_id, **options)) +def validate( + *, + database_url: Any, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> Mapping[str, Any]: + return result(validate_dataset( + database_url=database_url, + dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + **options, + )) def list_datasets(*, database_url: Any, kind: str | None = None) -> Mapping[str, Any]: diff --git a/src/openstatspec/cli.py b/src/openstatspec/cli.py index cc9d5c7..16f026b 100644 --- a/src/openstatspec/cli.py +++ b/src/openstatspec/cli.py @@ -6,9 +6,9 @@ from .api import ( apply_spss_in_place, apply_transformation_plan_in_place, - capability_matrix, derive_sql_dataset, + capability_matrix, derive_sql_dataset, dolt_state_snapshot, execute_sql_transformation, - export_sav, get_dataset, import_sav, inspect, list_datasets, + export_sav, get_dataset, import_sav, initialize_catalog, inspect, list_datasets, install_in_place_transformation_schema, register_sql_transformation, validate, validate_derived, ) @@ -29,6 +29,10 @@ def main(argv: Sequence[str] | None = None) -> int: commands = parser.add_subparsers(dest="command", required=True) capability_parser = commands.add_parser("capabilities", help="show supported and lossy feature matrix") capability_parser.add_argument("--database-url", help="include active connection limits") + dolt_state = commands.add_parser("dolt-state", help="show read-only Dolt branch, HEAD, status, and diff evidence") + dolt_state.add_argument("--database-url", required=True) + initializer = commands.add_parser("init", help="initialize or migrate a dedicated catalog") + initializer.add_argument("--database-url", required=True) importer = commands.add_parser("import", help="import one SAV/ZSAV file") importer.add_argument("source") importer.add_argument("--database-url", required=True) @@ -114,6 +118,10 @@ def main(argv: Sequence[str] | None = None) -> int: args = parser.parse_args(argv) if args.command == "capabilities": output = capability_matrix(database_url=args.database_url) + elif args.command == "dolt-state": + output = dolt_state_snapshot(database_url=args.database_url) + elif args.command == "init": + output = initialize_catalog(database_url=args.database_url) elif args.command == "import": output = import_sav(args.source, database_url=args.database_url, dataset_id=args.dataset_id) elif args.command == "export": diff --git a/src/openstatspec/core/__init__.py b/src/openstatspec/core/__init__.py index 12d26cf..d032ac4 100644 --- a/src/openstatspec/core/__init__.py +++ b/src/openstatspec/core/__init__.py @@ -1,6 +1,7 @@ from .results import Diagnostic, OperationResult """Pure OpenStatSpec concepts; no file or database adapter code.""" +import hashlib from dataclasses import dataclass, field from typing import Any @@ -9,6 +10,19 @@ class UnsupportedOperationError(NotImplementedError): """Raised when faithful support for a requested operation is unavailable.""" +def safe_error_identity(error: Exception, *, phase: str) -> dict[str, Any]: + """Return a path-free, stable identity for an exception.""" + code = getattr(error, "code", None) + if code is not None and not isinstance(code, (str, int, float, bool)): + code = type(code).__name__ + return { + "type": type(error).__name__, + "code": code, + "phase": phase, + "message_sha256": hashlib.sha256(str(error).encode("utf-8")).hexdigest(), + } + + @dataclass(frozen=True) class LossReport: """Machine-readable fidelity outcome for a completed future operation.""" diff --git a/src/openstatspec/spss/__init__.py b/src/openstatspec/spss/__init__.py index cfcc3aa..cc06cb7 100644 --- a/src/openstatspec/spss/__init__.py +++ b/src/openstatspec/spss/__init__.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Any +from ..sql.dolt_conformance import DoltConformanceSource from .sav import export_sav_dataset, import_sav_dataset, inspect_sav @@ -10,13 +11,26 @@ def inspect_source(source: str | Path, **options: Any) -> dict[str, Any]: return inspect_sav(source) -def import_dataset(source: str | Path, *, database_url: Any, dataset_id: str, **options: Any) -> dict[str, Any]: - return import_sav_dataset(source=source, database_url=str(database_url), dataset_id=dataset_id) +def import_dataset( + source: str | Path, *, database_url: Any, dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> dict[str, Any]: + return import_sav_dataset( + source=source, database_url=str(database_url), dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + ) -def export_dataset(*, database_url: Any, dataset_id: str, destination: str | Path, **options: Any) -> dict[str, Any]: +def export_dataset( + *, database_url: Any, dataset_id: str, destination: str | Path, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> dict[str, Any]: return export_sav_dataset( - database_url=str(database_url), dataset_id=dataset_id, destination=destination, + database_url=str(database_url), dataset_id=dataset_id, + destination=destination, allow_loss=tuple(options.get("allow_loss", ())), legacy_locale=options.get("legacy_locale"), + dolt_conformance_source=dolt_conformance_source, ) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index 7e51cfa..b4cae12 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -7,12 +7,13 @@ """ import hashlib +import os import json import math -from contextlib import ExitStack +from contextlib import ExitStack, contextmanager from datetime import UTC, datetime from pathlib import Path -from tempfile import TemporaryDirectory +from tempfile import mkstemp, TemporaryDirectory from typing import Any import pandas as pd @@ -37,18 +38,35 @@ set_variable_attribute_pairs, variable_attribute_pairs, ) -from ..core import UnsupportedOperationError -from ..sql.capabilities import effective_profile -from ..sql.profiles import preflight +from ..core import ( + UnsupportedOperationError, + safe_error_identity as _export_error_identity, +) +from ..sql.dolt_conformance import DoltConformanceSource from ..sql.wide import ( create_wide_dataset, + fail_export_operation, + finish_export_operation, physical_name, + read_export_operation_state, read_fidelity_events, read_wide_dataset, + record_export_backup_retained, + record_export_cleanup_failure, record_export_operation, validate_spss_catalog, ) + +class ExportRecoveryError(UnsupportedOperationError): + """Export publication could not restore the destination's prior state.""" + + def __init__(self, code: str, detail: str, *, details: dict[str, Any]) -> None: + super().__init__(f"OpenStatSpec export recovery failed [{code}]: {detail}") + self.code = code + self.details = {"reason": code, **details} + + _UTF8_ENCODINGS = {"UTF-8", "UTF8"} def engine_identity() -> dict[str, str]: """Return the exact pinned SPSS engine identity for audit records.""" @@ -82,10 +100,16 @@ def _dictionary(source_path: Path) -> tuple[dict[str, Any], dict[str, Any]]: ) except RawDictionaryError as error: metadata["_documents"] = None - metadata["_documents_error"] = str(error) + metadata["_documents_error"] = _export_error_identity( + error, + phase="read_sav_documents", + ) except Exception as error: # The source stays importable, but not silently faithful. metadata["_var_sets"] = None - metadata["_var_sets_error"] = str(error) + metadata["_var_sets_error"] = _export_error_identity( + error, + phase="read_sav_variable_sets", + ) return metadata, _engine_loss_report(metadata) @@ -217,7 +241,10 @@ def inspect_sav(source: str | Path) -> dict[str, Any]: } -def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str) -> dict[str, Any]: +def import_sav_dataset( + *, source: str | Path, database_url: str, dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: source_path = Path(source) _require_source(source_path) frame, metadata = pyspssio.read_sav( @@ -233,6 +260,7 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str result = create_wide_dataset( database_url=database_url, dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, source_name=source_path.name, source_format=source_path.suffix[1:].upper(), rows=_rows(frame, variables), @@ -257,24 +285,256 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str return {**result, "loss_report": loss_report} +def _path_reference(path: Path, *, role: str) -> dict[str, str]: + """Return an opaque path identity without disclosing any path component.""" + absolute_path = os.path.abspath(os.fspath(path)) + return { + "role": role, + "path_sha256": hashlib.sha256( + absolute_path.encode("utf-8") + ).hexdigest(), + } + + +def _path_reference_text(path: Path, *, role: str) -> str: + """Serialize a redacted path identity for legacy text audit columns.""" + return json.dumps( + _path_reference(path, role=role), + sort_keys=True, separators=(",", ":"), + ) + + +def _reserve_export_backup(destination: Path) -> Path: + descriptor, name = mkstemp( + dir=destination.parent, prefix=f".{destination.name}.", + suffix=".previous", + ) + os.close(descriptor) + return Path(name) + + +def _restore_export_destination( + *, destination: Path, backup: Path, had_previous: bool, +) -> None: + if had_previous: + os.replace(backup, destination) + else: + destination.unlink(missing_ok=True) + + +def _raise_export_cleanup_failed( + *, original_error: Exception, cleanup_error: Exception, + phase: str, destination: Path, backup: Path, staged: Path, + had_previous: bool, database_url: str, operation_id: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + inventory = { + "destination": _path_reference(destination, role="destination"), + "destination_exists": destination.exists(), + "backup": _path_reference(backup, role="durable_backup"), + "backup_exists": backup.exists(), + "staged_export": _path_reference(staged, role="staged_export"), + "staged_export_exists": staged.exists(), + } + inventory_sha256 = hashlib.sha256( + json.dumps(inventory, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + action_id = hashlib.sha256( + json.dumps( + { + "destination_path_sha256": inventory["destination"]["path_sha256"], + "phase": phase, + }, + sort_keys=True, separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + recovery = { + "procedure_id": "openstatspec.export-destination-restore.v1", + "action_id": action_id, + "targets": { + "destination": inventory["destination"], + "durable_backup": inventory["backup"], + "staged_export": inventory["staged_export"], + }, + "residual_inventory_sha256": inventory_sha256, + "cleanup_attempted": True, + "cleanup_succeeded": False, + "previous_destination_existed": had_previous, + "durable_backup_survives_staging_cleanup": backup.exists(), + } + cleanup_audit_operation_id = None + cleanup_audit_fault = None + try: + cleanup_audit_operation_id = record_export_cleanup_failure( + database_url=database_url, + destination=_path_reference_text( + destination, role="destination", + ), + original_error=original_error, cleanup_error=cleanup_error, + residual_object_inventory=inventory, + deterministic_recovery_evidence=recovery, + operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as audit_error: + cleanup_audit_fault = _export_error_identity( + audit_error, phase="cleanup_failed_audit", + ) + exception_recovery = { + **recovery, + "cleanup_failed_audit_persisted": cleanup_audit_fault is None, + "cleanup_failed_audit_operation_id": cleanup_audit_operation_id, + "terminal_reporting": ( + "catalog_and_exception" if cleanup_audit_fault is None + else "out_of_band_exception" + ), + } + raise ExportRecoveryError( + "cleanup_failed", + "Export failed and the destination's prior state could not be restored.", + details={ + "subcode": "export_destination_restore_failed", + "original_cause": _export_error_identity( + original_error, phase=f"export_{phase}", + ), + "cleanup_fault": _export_error_identity( + cleanup_error, phase="export_destination_restore", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": exception_recovery, + "audit_fault": cleanup_audit_fault, + "success_forbidden": True, + }, + ) from cleanup_error + + +def _mark_export_failed_after_restore( + *, database_url: str, operation_id: str, error: Exception, phase: str, + destination: Path, backup: Path, had_previous: bool, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + """Close a running audit after the old destination has been restored.""" + failure = { + "phase": phase, + "cause": _export_error_identity(error, phase=f"export_{phase}"), + "destination": _path_reference(destination, role="destination"), + "durable_backup": _path_reference(backup, role="durable_backup"), + "previous_destination_existed": had_previous, + "destination_restored": True, + } + try: + fail_export_operation( + database_url=database_url, operation_id=operation_id, + failure_details=failure, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as audit_error: + raise ExportRecoveryError( + "failure_audit_failed", + "The destination was restored, but the running export audit could not be closed.", + details={ + "subcode": "export_failure_audit_failed", + "original_cause": failure["cause"], + "audit_fault": _export_error_identity( + audit_error, phase="export_failure_audit", + ), + "residual_object_inventory": { + "destination": _path_reference( + destination, role="destination", + ), + "destination_exists": destination.exists(), + "backup": _path_reference( + backup, role="durable_backup", + ), + "backup_exists": backup.exists(), + "operation_id": operation_id, + }, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.export-audit-reconciliation.v1", + "action_id": operation_id, + "phase": phase, + "destination_restored": True, + "operation_terminal_state_verified": False, + "terminal_reporting": "out_of_band_exception", + }, + "success_forbidden": True, + }, + ) from audit_error + + +@contextmanager +def _export_staging_directory( + *, destination: Path, database_url: str, + publication_state: dict[str, Any], + dolt_conformance_source: DoltConformanceSource | None = None, +): + """Compensate a published file if staging-directory cleanup fails.""" + try: + with TemporaryDirectory( + dir=destination.parent, + prefix=f".{destination.name}.staging.", + ) as export_directory: + yield export_directory + except Exception as staging_error: + if not publication_state.get("published"): + raise + backup = publication_state["backup"] + staged = publication_state["staged"] + had_previous = publication_state["had_previous"] + operation_id = publication_state["operation_id"] + try: + _restore_export_destination( + destination=destination, + backup=backup, + had_previous=had_previous, + ) + except Exception as restore_error: + _raise_export_cleanup_failed( + original_error=staging_error, + cleanup_error=restore_error, + phase="staging_cleanup", + destination=destination, + backup=backup, + staged=staged, + had_previous=had_previous, + database_url=database_url, + operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + _mark_export_failed_after_restore( + database_url=database_url, + operation_id=operation_id, + error=staging_error, + phase="staging_cleanup", + destination=destination, + backup=backup, + had_previous=had_previous, + dolt_conformance_source=dolt_conformance_source, + ) + raise + + def export_sav_dataset( *, database_url: str, dataset_id: str, destination: str | Path, allow_loss: tuple[str, ...] = (), legacy_locale: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> dict[str, Any]: destination_path = Path(destination) if destination_path.suffix.lower() not in {".sav", ".zsav"}: raise UnsupportedOperationError("Export destinations must use the .sav or .zsav extension.") - profile, _active = effective_profile(database_url) dataset, variables, rows = read_wide_dataset( - database_url=database_url, dataset_id=dataset_id, profile=profile, + database_url=database_url, dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, ) - preflight(profile, variables, rows=rows) validate_spss_catalog( variables, case_weight_variable=dataset.get("case_weight_variable"), multiple_response_sets=dataset.get("multiple_response_sets"), ) - persisted_events = read_fidelity_events(database_url=database_url, dataset_id=dataset_id) + persisted_events = read_fidelity_events( + database_url=database_url, dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + ) if legacy_locale is not None: persisted_events = tuple( event for event in persisted_events @@ -300,22 +560,244 @@ def export_sav_dataset( ], columns=[variable["source_name"] for variable in variables], ) - try: + accepted_events = tuple( + event for event in loss_report if event["code"] in allow_loss + ) + operation_id = None + publication_state: dict[str, Any] = {"published": False} + with _export_staging_directory( + destination=destination_path, database_url=database_url, + publication_state=publication_state, + dolt_conformance_source=dolt_conformance_source, + ) as export_directory: + staged_destination = Path(export_directory) / destination_path.name _write_with_dictionary_bridge( - destination_path, frame, dataset, variables, legacy_locale=legacy_locale, + staged_destination, frame, dataset, variables, + legacy_locale=legacy_locale, ) - except Exception: - destination_path.unlink(missing_ok=True) - raise - operation_id = record_export_operation( - database_url=database_url, - dataset_id=dataset_id, - destination=str(destination_path), - allowed_fidelity_events=tuple( - event for event in loss_report if event["code"] in allow_loss - ), - operation_details={"engine": engine_identity(), "legacy_locale": legacy_locale}, - ) + had_previous = destination_path.exists() + backup = _reserve_export_backup(destination_path) + if not had_previous: + backup.unlink() + try: + operation_id = record_export_operation( + database_url=database_url, + dataset_id=dataset_id, + destination=_path_reference_text( + destination_path, role="destination", + ), + allowed_fidelity_events=accepted_events, + operation_details={ + "engine": engine_identity(), "legacy_locale": legacy_locale, + "recovery": { + "procedure_id": "openstatspec.export-destination-restore.v1", + "phase": "prepared", + "destination": _path_reference( + destination_path, role="destination", + ), + "durable_backup": _path_reference( + backup, role="durable_backup", + ), + "previous_destination_existed": had_previous, + "publication_finalized": False, + }, + }, + terminal=False, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as audit_error: + try: + backup.unlink(missing_ok=True) + except Exception as cleanup_error: + _raise_export_cleanup_failed( + original_error=audit_error, cleanup_error=cleanup_error, + phase="audit_start_placeholder_cleanup", + destination=destination_path, backup=backup, + staged=staged_destination, had_previous=had_previous, + database_url=database_url, + dolt_conformance_source=dolt_conformance_source, + ) + raise + backup_installed = False + try: + if had_previous: + os.replace(destination_path, backup) + backup_installed = True + os.replace(staged_destination, destination_path) + publication_state.update({ + "published": True, + "backup": backup, + "staged": staged_destination, + "had_previous": had_previous, + "operation_id": operation_id, + }) + except Exception as publish_error: + if backup_installed or not had_previous: + try: + _restore_export_destination( + destination=destination_path, backup=backup, + had_previous=had_previous, + ) + except Exception as cleanup_error: + _raise_export_cleanup_failed( + original_error=publish_error, cleanup_error=cleanup_error, + phase="publish", destination=destination_path, backup=backup, + staged=staged_destination, had_previous=had_previous, + database_url=database_url, operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + else: + try: + backup.unlink(missing_ok=True) + except Exception as cleanup_error: + _raise_export_cleanup_failed( + original_error=publish_error, + cleanup_error=cleanup_error, + phase="publish_placeholder_cleanup", + destination=destination_path, backup=backup, + staged=staged_destination, + had_previous=had_previous, + database_url=database_url, operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + _mark_export_failed_after_restore( + database_url=database_url, operation_id=operation_id, + error=publish_error, phase="publish", destination=destination_path, + backup=backup, had_previous=had_previous, + dolt_conformance_source=dolt_conformance_source, + ) + raise + assert operation_id is not None + finalization_state = None + try: + finish_export_operation( + database_url=database_url, operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as finalization_error: + state_read_fault = None + try: + finalization_state = read_export_operation_state( + database_url=database_url, operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as state_error: + state_read_fault = _export_error_identity( + state_error, phase="export_finalization_state_read", + ) + if ( + finalization_state is not None + and finalization_state["classification"] == "succeeded" + ): + pass + elif ( + finalization_state is not None + and finalization_state["classification"] == "running" + ): + try: + _restore_export_destination( + destination=destination_path, backup=backup, + had_previous=had_previous, + ) + except Exception as cleanup_error: + _raise_export_cleanup_failed( + original_error=finalization_error, + cleanup_error=cleanup_error, + phase="audit_finalization", + destination=destination_path, + backup=backup, + staged=staged_destination, + had_previous=had_previous, + database_url=database_url, + operation_id=operation_id, + dolt_conformance_source=dolt_conformance_source, + ) + _mark_export_failed_after_restore( + database_url=database_url, operation_id=operation_id, + error=finalization_error, phase="audit_finalization", + destination=destination_path, backup=backup, + had_previous=had_previous, + dolt_conformance_source=dolt_conformance_source, + ) + raise + else: + raise ExportRecoveryError( + "audit_finalization_ambiguous", + "The published export and durable backup were preserved because " + "the operation catalogs do not prove whether finalization committed.", + details={ + "subcode": "export_finalization_commit_ambiguous", + "operation_id": operation_id, + "finalization_cause": _export_error_identity( + finalization_error, phase="export_audit_finalization", + ), + "state_read_fault": state_read_fault, + "observed_operation_state": finalization_state, + "residual_object_inventory": { + "destination": _path_reference( + destination_path, role="published_destination", + ), + "destination_exists": destination_path.exists(), + "backup": _path_reference( + backup, role="durable_backup", + ), + "backup_exists": backup.exists(), + }, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.export-audit-reconciliation.v1", + "operation_terminal_state_verified": False, + "automatic_filesystem_recovery_performed": False, + "published_file_preserved": destination_path.exists(), + "durable_backup_preserved": backup.exists(), + "manual_reconciliation_required": True, + "terminal_reporting": "out_of_band_exception", + }, + "success_forbidden": True, + }, + ) from finalization_error + if backup.exists(): + try: + backup.unlink() + except Exception as cleanup_error: + audit_fault = None + try: + record_export_backup_retained( + database_url=database_url, operation_id=operation_id, + destination=_path_reference_text( + destination_path, role="destination", + ), + backup=_path_reference_text( + backup, role="durable_backup", + ), + cleanup_error=cleanup_error, + dolt_conformance_source=dolt_conformance_source, + ) + except Exception as warning_error: + audit_fault = _export_error_identity( + warning_error, phase="backup_retained_warning_audit", + ) + raise ExportRecoveryError( + "backup_retained", + "The export succeeded, but its durable prior-file backup could not be removed.", + details={ + "subcode": "post_success_backup_retained", + "operation_id": operation_id, + "operation_status": "succeeded", + "destination": _path_reference( + destination_path, role="destination", + ), + "durable_backup": _path_reference( + backup, role="durable_backup", + ), + "cleanup_fault": _export_error_identity( + cleanup_error, phase="post_success_backup_disposal", + ), + "warning_audit_persisted": audit_fault is None, + "audit_fault": audit_fault, + "success_forbidden": False, + }, + ) + return { "dataset_id": dataset_id, "destination": str(destination_path), @@ -335,10 +817,21 @@ def _write_with_dictionary_bridge( pyspssio engine then copies it into the real SAV or ZSAV writer, which keeps any ZSAV internal dictionary offsets valid. """ + compatible_names = { + str(variable["source_name"]): str(variable["compat_name"]) + for variable in variables + if variable.get("compat_name") + and str(variable["compat_name"]).casefold() + != str(variable["source_name"]).casefold() + } + variable_sets = (dataset.get("source_extensions") or {}).get( + "spss.variable_sets" + ) metadata = _writer_metadata(dataset, variables) documents = _json_load(dataset.get("documents"), []) source_encoding = str(dataset.get("source_encoding") or "UTF-8") legacy_output = _is_non_utf8_encoding(source_encoding) and legacy_locale is not None + output_encoding: str | None = None with ExitStack() as stack: document_source = None if documents: @@ -393,24 +886,18 @@ def _write_with_dictionary_bridge( set_variable_attribute_pairs( writer, variable["source_name"], attribute_pairs(values), ) - variable_sets = (dataset.get("source_extensions") or {}).get("spss.variable_sets") if variable_sets: writer.var_sets = variable_sets if document_source is not None: writer.copy_documents_from(document_source) writer.commit_header() - _require_matching_legacy_encoding(source_encoding, str(writer.file_encoding), legacy_output) + output_encoding = str(writer.file_encoding or "") + if not output_encoding: + raise RawDictionaryError("Writer did not expose its output file encoding.") + _require_matching_legacy_encoding(source_encoding, output_encoding, legacy_output) writer.write_data(frame) - write_compatible_names( - destination, - { - str(variable["source_name"]): str(variable["compat_name"]) - for variable in variables - if variable.get("compat_name") - and str(variable["compat_name"]).casefold() != str(variable["source_name"]).casefold() - }, - encoding="UTF-8", - ) + if output_encoding is None: + raise RawDictionaryError("Writer output encoding was not captured.") extended_labels = { str(name): str(definition["label"]) for name, definition in _json_load(dataset.get("multiple_response_sets"), {}).items() @@ -420,7 +907,12 @@ def _write_with_dictionary_bridge( and definition.get("label") } write_extended_mrset_labels( - destination, extended_labels, encoding=source_encoding, + destination, extended_labels, encoding=output_encoding, + ) + write_compatible_names( + destination, + compatible_names, + encoding=output_encoding, ) def _catalog_format(variable: dict[str, Any], key: str) -> tuple[int, int, int]: diff --git a/src/openstatspec/sql/__init__.py b/src/openstatspec/sql/__init__.py index 3a841fe..2c74007 100644 --- a/src/openstatspec/sql/__init__.py +++ b/src/openstatspec/sql/__init__.py @@ -3,12 +3,64 @@ from typing import Any from .capabilities import profile_declarations -from .wide import validate_wide_dataset +from .dolt_conformance import DoltConformanceSource +from .wide import ( + dolt_state_snapshot as _dolt_state_snapshot, + initialize_wide_catalog, + validate_wide_dataset, +) -def declared_profiles(database_url: str | None = None) -> dict[str, dict[str, object]]: - return profile_declarations(database_url) +def declared_profiles( + database_url: str | None = None, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, dict[str, object]]: + return profile_declarations( + database_url, dolt_conformance_source=dolt_conformance_source, + ) -def validate_dataset(*, database_url: Any, dataset_id: str, **options: Any) -> dict[str, Any]: - return validate_wide_dataset(database_url=str(database_url), dataset_id=dataset_id) +def dolt_state_snapshot( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + return _dolt_state_snapshot( + database_url=str(database_url), + dolt_conformance_source=dolt_conformance_source, + ) + + +def initialize_catalog( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + return initialize_wide_catalog( + database_url=str(database_url), + dolt_conformance_source=dolt_conformance_source, + ) + + +def validate_dataset( + *, + database_url: Any, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, + **options: Any, +) -> dict[str, Any]: + return validate_wide_dataset( + database_url=str(database_url), + dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, + ) + + +__all__ = [ + "DoltConformanceSource", + "declared_profiles", + "dolt_state_snapshot", + "initialize_catalog", + "validate_dataset", +] diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index 2701660..fdfd20c 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -5,20 +5,74 @@ import re from dataclasses import replace from pathlib import Path -from typing import Any +from typing import Any, Mapping from sqlalchemy import MetaData, create_engine, text from sqlalchemy.engine import make_url +from .dolt_conformance import DoltConformanceSource, effective_limits as dolt_effective_limits from .normative import catalog -from .profiles import DOLT, MYSQL, POSTGRESQL, SQLITE, SqlProfile -from .profiles import profile_for_url, validate_connection_url +from .profiles import DOLT, MYSQL, POSTGRESQL, SQLITE, MYSQL_WIRE_PROFILES, SqlProfile +from .profiles import profile_for_url from ..core import UnsupportedOperationError -SPECIFICATION_COMMIT = "79339ec3d8f8aa81789b7e85f6b8afa6f1374e50" -SPECIFICATION_RELEASE: str | None = "v0.2.0" +# Release/build automation must bind this to the exact commit used to build +# openstatspec-specification. An uncommitted source tree has no truthful pin. +SPECIFICATION_COMMIT: str | None = None +SPECIFICATION_RELEASE: str | None = None + +DOLT_WRITE_CONFORMANCE = { + "declaration_schema_id": "openstatspec-dolt-adapter-declaration-v1", + "write_enabled": False, + "status": "packaged_concrete_declarations_required", +} + + +def _conformance_source( + source: DoltConformanceSource | None, +) -> DoltConformanceSource: + return source or DoltConformanceSource.packaged() + + +def _validated_dolt_declarations( + source: DoltConformanceSource | None, +) -> tuple[dict[str, Any], ...]: + try: + return tuple( + dict(item) + for item in _conformance_source(source).validated_declarations() + ) + except UnsupportedOperationError: + return () + + +def _bound_specification_commit() -> str: + if ( + not isinstance(SPECIFICATION_COMMIT, str) + or re.fullmatch(r"[0-9a-f]{40}", SPECIFICATION_COMMIT) is None + ): + raise UnsupportedOperationError( + "The Python adapter is not bound to an exact " + "openstatspec-specification commit; Dolt write rejected before " + "mutation." + ) + return SPECIFICATION_COMMIT + + +def _dolt_write_enabled( + source: DoltConformanceSource | None = None, + *, + active_product_version: str | None = None, +) -> bool: + conformance = _conformance_source(source) + if active_product_version is None: + return bool(conformance.status()["write_enabled"]) + conformance.require_exact_match( + active_product_version=active_product_version, + specification_commit=_bound_specification_commit(), + ) + return True -_DOLT_2_2_STABLE_VERSION = re.compile(r"2\.2\.(0|[1-9][0-9]*)") SERVER_POLICIES = { "sqlite": { @@ -27,72 +81,61 @@ }, "mysql": { "claimed": ["MySQL 8.4.x", "MySQL 9.7.x"], - "ci": ["MySQL 8.4.11", "MySQL 9.7.2"], - }, - "dolt": { - "claimed": ["Dolt 2.2.x"], - "range": { - "minimum_inclusive": "2.2.2", - "maximum_exclusive": "2.3.0", - }, - "ci": ["Dolt 2.2.2", "Dolt 2.2.3"], - "exact_ci": ["2.2.2", "2.2.3"], + "ci": ["MySQL 8.4.x", "MySQL 9.7.x"], }, "mariadb": { "claimed": ["MariaDB 11.4.x", "MariaDB 11.8.x", "MariaDB 12.3.x"], - "ci": ["MariaDB 11.4.12", "MariaDB 11.8.8", "MariaDB 12.3.2"], + "ci": ["MariaDB 11.4.x", "MariaDB 11.8.x", "MariaDB 12.3.x"], + }, + "dolt": { + "claimed": [], + "ci": [], }, "postgresql": { "claimed": ["PostgreSQL 17.x", "PostgreSQL 18.x"], - "ci": ["PostgreSQL 17.10", "PostgreSQL 18.4"], + "ci": ["PostgreSQL 17.x", "PostgreSQL 18.x"], }, } -def profile_declarations(database_url: str | None = None) -> dict[str, dict[str, Any]]: +def profile_declarations( + database_url: str | None = None, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, dict[str, Any]]: """Declare every profile and, optionally, one active connection.""" - active = active_connection(database_url) if database_url else None + source = _conformance_source(dolt_conformance_source) + active = ( + active_connection(database_url, dolt_conformance_source=source) + if database_url else None + ) + dolt_declaration = None + if active and active["profile"] == "dolt": + dolt_declaration = source.require_exact_match( + active_product_version=active["raw_product_version"], + specification_commit=_bound_specification_commit(), + ) return { - "sqlite": _profile("sqlite", SQLITE, active), - "mysql": _profile("mysql", MYSQL, active), - "mariadb": _profile("mariadb", MYSQL, active), - "dolt": _profile("dolt", DOLT, active), - "postgresql": _profile("postgresql", POSTGRESQL, active), + "sqlite": _profile("sqlite", SQLITE, active, source), + "mysql": _profile("mysql", MYSQL, active, source), + "mariadb": _profile("mariadb", MYSQL, active, source), + "dolt": _profile("dolt", DOLT, active, source, dolt_declaration), + "postgresql": _profile("postgresql", POSTGRESQL, active, source), } -def _required_text_probe(connection: Any, statement: str, label: str) -> str: - """Return one required identity value without normalizing absence into text.""" - try: - value = connection.execute(text(statement)).scalar_one() - except Exception as error: - raise UnsupportedOperationError( - f"Active SQL server identity probe {label} failed." - ) from error - if value is None or value is False: - raise UnsupportedOperationError( - f"Active SQL server identity probe {label} returned no value." - ) - raw = str(value) - if not raw.strip(): - raise UnsupportedOperationError( - f"Active SQL server identity probe {label} returned no value." - ) - return raw - - -def active_connection(database_url: str) -> dict[str, Any]: +def active_connection( + database_url: str, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: engine = create_engine(database_url) with engine.connect() as connection: dialect = connection.dialect.name - raw_comment: str | None = None if dialect == "sqlite": profile_name = "sqlite" - raw_wire_version = _required_text_probe( - connection, "select sqlite_version()", "sqlite_version()", - ) - raw_product_version = raw_wire_version - identity_source = "SELECT sqlite_version()" + raw_version = str(connection.execute(text("select sqlite_version()")).scalar_one()) + identity_source = "select sqlite_version()" compile_options = { name: int(value) for option in connection.exec_driver_sql("pragma compile_options").scalars() @@ -103,107 +146,138 @@ def active_connection(database_url: str) -> dict[str, Any]: observed = {"compile_options": compile_options} elif dialect == "postgresql": profile_name = "postgresql" - raw_wire_version = _required_text_probe( - connection, "show server_version", "server_version", - ) - raw_product_version = raw_wire_version + raw_version = str(connection.execute(text("show server_version")).scalar_one()) identity_source = "SHOW server_version" observed = {} elif dialect in {"mysql", "mariadb"}: - raw_wire_version = _required_text_probe( - connection, "select @@version", "@@version", + wire_version = _required_identity_text( + connection.execute(text("select @@version")).scalar_one(), "@@version", ) - raw_comment = _required_text_probe( - connection, "select @@version_comment", "@@version_comment", + comment = _required_identity_text( + connection.execute(text("select @@version_comment")).scalar_one(), + "@@version_comment", ) - identity_text = f"{raw_wire_version} {raw_comment}".casefold() - if raw_comment.strip().casefold() == "dolt": - profile_name = "dolt" - raw_product_version = _required_text_probe( - connection, "select DOLT_VERSION()", "DOLT_VERSION()", - ) - if not _dolt_version_supported(raw_product_version): + normalized_comment = comment.strip().casefold() + product_version = wire_version + if normalized_comment == "dolt": + if "mariadb" in wire_version.casefold(): raise UnsupportedOperationError( - "The active Dolt product version must be a canonical stable " - "release in the supported range >=2.2.2,<2.3.0." + "Conflicting Dolt and MariaDB active-server identity." ) - identity_source = "SELECT @@version, @@version_comment, DOLT_VERSION()" - elif "mariadb" in identity_text: - profile_name = "mariadb" - raw_product_version = raw_wire_version - identity_source = "SELECT @@version, @@version_comment" - elif "mysql" in raw_comment.casefold(): - profile_name = "mysql" - raw_product_version = raw_wire_version - identity_source = "SELECT @@version, @@version_comment" - else: - raise UnsupportedOperationError( - "The active MySQL-wire server product is unknown or unsupported." + product_version = _required_identity_text( + connection.execute(text("select DOLT_VERSION()")).scalar_one(), + "DOLT_VERSION()", ) - packet_text = _required_text_probe( - connection, "select @@max_allowed_packet", "@@max_allowed_packet", - ) - try: - packet = int(packet_text) - except ValueError as error: - raise UnsupportedOperationError( - "Active SQL server returned an invalid @@max_allowed_packet." - ) from error - if packet <= 0: - raise UnsupportedOperationError( - "Active SQL server returned an invalid @@max_allowed_packet." + profile_name, product = "dolt", "Dolt" + active_branch = _required_identity_text( + connection.execute(text("select ACTIVE_BRANCH()")).scalar_one(), + "ACTIVE_BRANCH()", ) - observed = {"max_allowed_packet": packet} + identity_source = ( + "SELECT @@version, @@version_comment, DOLT_VERSION(), ACTIVE_BRANCH()" + ) + elif "mariadb" in (wire_version + " " + comment).casefold(): + profile_name, product = "mariadb", "MariaDB" + identity_source = "SELECT @@version, @@version_comment" + else: + profile_name, product = "mysql", "MySQL" + identity_source = "SELECT @@version, @@version_comment" + raw_version = product_version + packet = int(connection.execute(text("select @@max_allowed_packet")).scalar_one()) + observed = { + "max_allowed_packet": packet, + **( + {"active_branch": active_branch} + if profile_name == "dolt" else {} + ), + } else: # pragma: no cover - validate_connection_url rejects this first raise ValueError(f"Unsupported active SQL dialect {dialect!r}.") return { "dialect": dialect, "profile": profile_name, - "engine": profile_name, - "product": profile_name, - "transport": ( - "mysql_compatible" if profile_name == "dolt" - else "mysql" if dialect in {"mysql", "mariadb"} - else dialect - ), - "driver": engine.dialect.driver, - "server_version": _normalized_version(profile_name, raw_product_version), - "raw_server_version": raw_product_version, - "raw_wire_version": raw_wire_version, - "raw_product_version": raw_product_version, - "raw_version_comment": raw_comment, + "product": product if dialect in {"mysql", "mariadb"} else profile_name, + "server_version": _normalized_version(raw_version), + "raw_server_version": raw_version, + "raw_wire_version": wire_version if dialect in {"mysql", "mariadb"} else raw_version, + "raw_product_version": raw_version, + "raw_version_comment": comment if dialect in {"mysql", "mariadb"} else None, "identity_source": identity_source, - "claimed_supported": server_version_supported(profile_name, raw_product_version), - "matched_claim": _matched_claim(profile_name, raw_product_version), + "claimed_supported": server_version_supported( + profile_name, + raw_version, + dolt_conformance_source=dolt_conformance_source, + ), + "matched_claim": _matched_claim( + profile_name, + raw_version, + dolt_conformance_source=dolt_conformance_source, + ), "catalog_binding": catalog_binding(database_url), + "working_set_binding": ( + { + "database": catalog_binding(database_url)["namespace"], + "active_branch": active_branch, + } + if profile_name == "dolt" else None + ), "observed": observed, } -def effective_profile(database_url: str) -> tuple[SqlProfile, dict[str, Any]]: +def _required_identity_text(value: Any, source: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise UnsupportedOperationError( + f"Active-server identity probe {source} returned no non-empty text." + ) + return value.strip() + + +def effective_profile( + database_url: str, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> tuple[SqlProfile, dict[str, Any]]: """Resolve and enforce the profile used by import preflight.""" - configured = validate_connection_url(database_url) - active = active_connection(database_url) + source = _conformance_source(dolt_conformance_source) + configured = profile_for_url(database_url) + active = active_connection(database_url, dolt_conformance_source=source) if configured is not MYSQL and active["profile"] != configured.name: raise UnsupportedOperationError("The active SQL server does not match the configured profile.") - if configured is MYSQL and active["profile"] not in {"mysql", "mariadb", "dolt"}: - raise UnsupportedOperationError("The active SQL server is not MySQL, MariaDB, or Dolt.") - if active["profile"] == "dolt" and make_url(database_url).drivername != "mysql+pymysql": - raise UnsupportedOperationError("Dolt requires an explicit mysql+pymysql URL.") + if configured is MYSQL and active["profile"] not in MYSQL_WIRE_PROFILES: + raise UnsupportedOperationError("The active SQL server is not a claimed MySQL-wire product.") + dolt_declaration = None + if active["profile"] == "dolt": + dolt_declaration = source.require_exact_match( + active_product_version=active["raw_product_version"], + specification_commit=_bound_specification_commit(), + ) if not active["claimed_supported"]: raise UnsupportedOperationError( f"Active {active['profile']} server version {active['server_version']} is not claimed supported." ) - selected = DOLT if active["profile"] == "dolt" else configured - declaration = _profile(active["profile"], selected, active) + if ( + active["profile"] in MYSQL_WIRE_PROFILES + and int(active["observed"]["max_allowed_packet"]) <= 131_072 + ): + raise UnsupportedOperationError( + "Active @@max_allowed_packet is too small for the SQL adapter safety reserve." + ) + configured = DOLT if active["profile"] == "dolt" else configured + declaration = _profile( + active["profile"], configured, active, source, dolt_declaration, + ) limits = declaration["effective_limits"] assert limits is not None return replace( - selected, + configured, name=active["profile"], - max_physical_variables=int(limits["maximum_source_variables"]), + max_source_variables=int(limits["maximum_source_variables"]), max_text_value_bytes=int(limits["maximum_value_bytes"]), - max_row_bytes=int(limits["maximum_row_bytes"]), + max_row_bytes=( + int(limits["maximum_row_bytes"]) if limits["maximum_row_bytes"] is not None else None + ), + max_statement_bytes=limits.get("maximum_statement_bytes"), ), active @@ -231,68 +305,75 @@ def catalog_binding(database_url: str) -> dict[str, Any]: def _profile( - name: str, profile: SqlProfile, active: dict[str, Any] | None, + name: str, + profile: SqlProfile, + active: dict[str, Any] | None, + dolt_conformance_source: DoltConformanceSource | None = None, + dolt_declaration: Mapping[str, Any] | None = None, ) -> dict[str, Any]: + dolt_envelope = name == "dolt" identifier = { "value": profile.identifier_limit, "unit": "characters" if name in {"mysql", "mariadb"} else "bytes", "source": ( "MySQL/MariaDB native identifier limit" if name in {"mysql", "mariadb"} - else "Dolt 2.2.2 observed 64-byte ASCII identifier limit" if name == "dolt" else "PostgreSQL NAMEDATALEN minus one native byte limit" if name == "postgresql" + else "OpenStatSpec Dolt adapter envelope pending pinned live boundary evidence" + if dolt_envelope else "OpenStatSpec profile boundary; SQLite has no fixed native identifier limit" ), "repertoire": "generated ASCII [a-z0-9_] identifiers", } - declared = { - "maximum_physical_columns": profile.max_physical_variables + 1, - "maximum_source_variables": profile.max_physical_variables, + profile_limits = { + "maximum_physical_columns": profile.max_source_variables + 1, + "maximum_source_variables": profile.max_source_variables, + "maximum_statement_bytes": profile.max_statement_bytes, "identifier_limit": identifier, "maximum_value_bytes": profile.max_text_value_bytes, "maximum_row_bytes": profile.max_row_bytes, } - theoretical = ( - {"maximum_value_bytes": 4_294_967_295} - if name == "dolt" else declared - ) - proposed = ( - { - "maximum_physical_columns": declared["maximum_physical_columns"], - "maximum_source_variables": declared["maximum_source_variables"], - "maximum_value_bytes": declared["maximum_value_bytes"], - "maximum_row_bytes": declared["maximum_row_bytes"], + adapter_envelope = None + if dolt_envelope: + adapter_envelope = { + **profile_limits, + "limit_basis": "proposed_adapter_envelope", + "evidence_status": "pending_pinned_live_conformance", } - if name == "dolt" else None - ) - observed_limits = ( - { - "minimum_observed_physical_columns": 307, - "identifier_limit": identifier, - "rejected_identifier_bytes": 65, + theoretical = { + key: None for key in ( + "maximum_physical_columns", "maximum_source_variables", + "maximum_statement_bytes", "identifier_limit", + "maximum_value_bytes", "maximum_row_bytes", + ) + } + theoretical["limit_basis"] = "server_limits_not_claimed" + else: + theoretical = { + **profile_limits, + "limit_basis": "profile_theoretical_engine_ceiling", } - if name == "dolt" else None - ) effective = None status = "not_connected" - if active and active["profile"] == name: - effective = dict(declared) - sources = ( - { - "maximum_source_variables": "proposed Dolt adapter envelope", - "maximum_physical_columns": "proposed Dolt adapter envelope", - "identifier_limit": "observed on exact Dolt 2.2.2", - "maximum_value_bytes": "observed Dolt adapter value envelope", - "maximum_row_bytes": "proposed Dolt adapter envelope", - } - if name == "dolt" - else { - "maximum_source_variables": "profile theoretical engine ceiling", - "maximum_physical_columns": "profile theoretical engine ceiling", - "identifier_limit": identifier["source"], - "maximum_value_bytes": "profile theoretical engine ceiling", - "maximum_row_bytes": "profile theoretical engine ceiling", - } + if active and active["profile"] == name and ( + not dolt_envelope or dolt_declaration is not None + ): + effective = ( + dolt_effective_limits(dolt_declaration) + if dolt_envelope and dolt_declaration is not None + else dict(theoretical) + ) + default_source = ( + "proposed adapter envelope pending pinned live conformance" + if dolt_envelope else "profile theoretical engine ceiling" ) + sources = { + "maximum_source_variables": default_source, + "maximum_statement_bytes": default_source, + "maximum_physical_columns": default_source, + "identifier_limit": identifier["source"], + "maximum_value_bytes": default_source, + "maximum_row_bytes": default_source, + } observed = active["observed"] if name == "sqlite": options = observed["compile_options"] @@ -315,15 +396,11 @@ def _profile( sources["maximum_value_bytes"] = "active PRAGMA compile_options MAX_LENGTH" sources["maximum_row_bytes"] = "active PRAGMA compile_options MAX_LENGTH" status = "active_connection_mixed" - elif name in {"mysql", "mariadb", "dolt"}: + elif name in MYSQL_WIRE_PROFILES: packet = int(observed["max_allowed_packet"]) payload = max(0, (packet - 131_072) // 2) effective["maximum_value_bytes"] = min( - ( - declared["maximum_value_bytes"] - if name == "dolt" else theoretical["maximum_value_bytes"] - ), - payload, + effective["maximum_value_bytes"], payload, ) effective["maximum_statement_bytes"] = payload sources["maximum_value_bytes"] = "active @@max_allowed_packet worst-case payload" @@ -332,110 +409,129 @@ def _profile( else: status = "profile_theoretical_fallback" effective["sources"] = sources + elif active and active["profile"] == name and dolt_envelope: + status = "blocked_pending_pinned_live_conformance" policy = SERVER_POLICIES[name] + dolt_declarations = ( + _validated_dolt_declarations(dolt_conformance_source) + if dolt_envelope else () + ) + dolt_claimed_versions = sorted({ + version + for declaration in dolt_declarations + for version in declaration["claimed_product_versions"] + }) + dolt_tested_versions = sorted({ + version + for declaration in dolt_declarations + for version in declaration["tested_product_versions"] + }) + dolt_status = ( + _conformance_source(dolt_conformance_source).status() + if dolt_envelope else None + ) return { - "profile": name, - "engine": name, - "dialect": "mysql" if name == "dolt" else name, - "transport": "mysql_compatible" if name == "dolt" else name, - "specification_commit": SPECIFICATION_COMMIT, - "specification_status": "released", - "specification_release": SPECIFICATION_RELEASE, - "driver": "psycopg" if name == "postgresql" else "PyMySQL" if name in {"mysql", "mariadb", "dolt"} else "sqlite3", - "claimed_server_versions": policy["claimed"], - "claimed_version_range": policy.get("range"), - "ci_tested_server_versions": policy["ci"], - **( - {"exact_ci_tested_versions": policy["exact_ci"]} - if "exact_ci" in policy else {} + "driver": "psycopg" if name == "postgresql" else "PyMySQL" if name in MYSQL_WIRE_PROFILES else "sqlite3", + "claimed_server_versions": ( + dolt_claimed_versions if dolt_envelope else policy["claimed"] + ), + "ci_tested_server_versions": ( + dolt_tested_versions if dolt_envelope else policy["ci"] + ), + "write_conformance": ( + { + "declaration_schema_id": DOLT_WRITE_CONFORMANCE[ + "declaration_schema_id" + ], + **dict(dolt_status or {}), + "active_declaration_id": ( + dolt_declaration["declaration_id"] + if dolt_declaration is not None else None + ), + } if dolt_envelope else { + "write_enabled": True, + "tested_server_versions": list(policy["ci"]), + "status": "profile_claimed", + } + ), + "operational_write_enabled": ( + dolt_declaration is not None + if dolt_envelope and active and active["profile"] == name + else bool((dolt_status or {}).get("write_enabled")) + if dolt_envelope else True ), "theoretical_limits": theoretical, - "proposed_adapter_limits": proposed, - "observed_limits": observed_limits, + "adapter_envelope": adapter_envelope, "effective_limits": effective, "effective_limits_status": status, - "numeric_type": "DOUBLE PRECISION" if name == "postgresql" else "DOUBLE" if name in {"mysql", "mariadb", "dolt"} else "REAL", + "numeric_type": "DOUBLE PRECISION" if name == "postgresql" else "DOUBLE" if name in MYSQL_WIRE_PROFILES else "REAL", "numeric_value_policy": { - "finite_binary64": "supported", - "nan": "rejected_before_ddl", - "positive_infinity": "rejected_before_ddl", - "negative_infinity": "rejected_before_ddl", - }, - "text_type": "LONGTEXT" if name in {"mysql", "mariadb", "dolt"} else "TEXT", - "ddl_atomic": name not in {"mysql", "mariadb", "dolt"}, - "failure_cleanup": "compensating_cleanup" if name in {"mysql", "mariadb", "dolt"} else "transaction_rollback", - "limit_bases": { - "maximum_physical_columns": ( - "proposed_adapter_envelope" if name == "dolt" else "theoretical_engine_limit" - ), - "maximum_source_variables": ( - "proposed_adapter_envelope" if name == "dolt" else "theoretical_engine_limit" - ), - "identifier_limit": "observed_exact_version" if name == "dolt" else "theoretical_engine_limit", - "maximum_value_bytes": ( - "observed_exact_version" if name == "dolt" else "theoretical_engine_limit" + "sql_null": "canonical_system_missing", + "spss_nan": "canonicalize_to_sql_null_during_spss_decode", + "adapter_input": "finite_binary64_or_null", + "positive_infinity": "reject_before_mutation", + "negative_infinity": "reject_before_mutation", + "live_bit_exact_evidence": ( + "pending_pinned_live_conformance" if dolt_envelope + else "profile_conformance_claim" ), - "maximum_row_bytes": ( - "proposed_adapter_envelope" if name == "dolt" else "theoretical_engine_limit" - ), - "maximum_statement_bytes": "active_connection_observation", }, - "storage_evidence": ( - { - "binary64": { - "type": "DOUBLE", - "classification": "observed_exact_version", - "source": "Dolt 2.2.2 interoperability verification", - "version": "2.2.2", - "maximum_finite_round_trip_exact": True, - }, - "text": { - "type": "LONGTEXT NOT NULL", - "classification": "observed_exact_version", - "source": "Dolt 2.2.2 interoperability verification", - "version": "2.2.2", - "observed_value_bytes": 65_504, - "unit": "bytes", - }, - } - if name == "dolt" else None - ), - "transformation_workflow": "unsupported" if name == "dolt" else None, + "text_type": "LONGTEXT" if name in MYSQL_WIRE_PROFILES else "TEXT", + "ddl_atomic": name not in MYSQL_WIRE_PROFILES, + "failure_cleanup": "compensating_cleanup" if name in MYSQL_WIRE_PROFILES else "transaction_rollback", "physical_table_mapping": "dataset.physical_table_schema + dataset.physical_table_name", "identifier_policy": "deterministic ASCII mapping; source name remains authoritative", } -def server_version_supported(profile: str, raw_version: str) -> bool: - if profile == "dolt": - return _dolt_version_supported(raw_version) +def server_version_supported( + profile: str, + raw_version: str, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> bool: version = _version_tuple(raw_version) + if profile == "dolt": + try: + return _dolt_write_enabled( + dolt_conformance_source, + active_product_version=raw_version.strip(), + ) + except UnsupportedOperationError: + return False if profile == "sqlite": return (3, 24) <= version[:2] < (4, 0) allowed = { "postgresql": {(17,), (18,)}, "mysql": {(8, 4), (9, 7)}, "mariadb": {(11, 4), (11, 8), (12, 3)}, + "dolt": {(2, 2, 2)}, }[profile] width = len(next(iter(allowed))) return version[:width] in allowed -def _dolt_version_supported(raw_version: str) -> bool: - """Accept only canonical stable Dolt 2.2 patches at or above 2.2.2.""" - match = _DOLT_2_2_STABLE_VERSION.fullmatch(raw_version.strip()) - return match is not None and match.group(1) not in {"0", "1"} - - -def _matched_claim(profile: str, raw_version: str) -> str | None: - if not server_version_supported(profile, raw_version): +def _matched_claim( + profile: str, + raw_version: str, + *, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> str | None: + if not server_version_supported( + profile, + raw_version, + dolt_conformance_source=dolt_conformance_source, + ): return None + if profile == "dolt": + return raw_version.strip() version = _version_tuple(raw_version) for claim in SERVER_POLICIES[profile]["claimed"]: numbers = _version_tuple(claim) if profile == "sqlite" or version[:2] == numbers[:2] or version[:1] == numbers[:1]: return claim - return SERVER_POLICIES[profile]["claimed"][0] + claims = SERVER_POLICIES[profile]["claimed"] + return claims[0] if claims else None def _version_tuple(raw_version: str) -> tuple[int, ...]: @@ -445,8 +541,5 @@ def _version_tuple(raw_version: str) -> tuple[int, ...]: return tuple(int(part) for part in match.groups(default="0")) -def _normalized_version(profile: str, raw_version: str) -> str: - """Normalize a product version using that product's stable release width.""" - version = _version_tuple(raw_version) - width = 2 if profile == "postgresql" else 3 - return ".".join(str(part) for part in version[:width]) +def _normalized_version(raw_version: str) -> str: + return ".".join(str(part) for part in _version_tuple(raw_version)) diff --git a/src/openstatspec/sql/dolt_conformance.py b/src/openstatspec/sql/dolt_conformance.py new file mode 100644 index 0000000..fa0e5d8 --- /dev/null +++ b/src/openstatspec/sql/dolt_conformance.py @@ -0,0 +1,199 @@ +"""Thin adapter glue for the shared OpenStatSpec Dolt declaration validator.""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version as distribution_version +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from ..core import UnsupportedOperationError + + +def _adapter_version() -> str: + source_project = Path(__file__).resolve().parents[3] / "pyproject.toml" + if source_project.is_file(): + project = tomllib.loads(source_project.read_text(encoding="utf-8")) + value = project.get("project", {}).get("version") + if isinstance(value, str) and value: + return value + try: + return distribution_version("openstatspec") + except PackageNotFoundError as error: + raise RuntimeError( + "The openstatspec adapter version is unavailable from both the " + "source project and installed distribution metadata." + ) from error + + +ADAPTER_IMPLEMENTATION_ID = "openstatspec-python" +ADAPTER_VERSION = _adapter_version() + + +def _shared_api() -> tuple[Any, Any, Any, Any]: + try: + from openstatspec_specification.dolt import ( + DoltDeclarationError, + DoltDeclarationSource, + load_validated_dolt_declarations, + select_dolt_declaration, + ) + except (ImportError, ModuleNotFoundError) as error: + raise UnsupportedOperationError( + "The openstatspec-specification companion distribution is not " + "installed; Dolt writes remain disabled." + ) from error + return ( + DoltDeclarationError, + DoltDeclarationSource, + load_validated_dolt_declarations, + select_dolt_declaration, + ) + + +@dataclass(frozen=True) +class DoltConformanceSource: + """Explicit source for validated packaged or directory declarations.""" + + directory: Path | None = None + + @classmethod + def packaged(cls) -> "DoltConformanceSource": + return cls() + + @classmethod + def from_directory(cls, root: str | Path) -> "DoltConformanceSource": + return cls(directory=Path(root)) + + def validated_declarations(self) -> tuple[Mapping[str, Any], ...]: + ( + shared_error, + shared_source_type, + shared_loader, + _shared_selector, + ) = _shared_api() + try: + shared_source = ( + shared_source_type.packaged() + if self.directory is None + else shared_source_type.from_directory(self.directory) + ) + return tuple(shared_loader(shared_source)) + except shared_error as error: + raise UnsupportedOperationError( + "The Dolt declaration source failed shared semantic or " + "resource-integrity validation: " + str(error) + ) from error + + def status(self) -> Mapping[str, Any]: + try: + declarations = self.validated_declarations() + except UnsupportedOperationError as error: + return { + "write_enabled": False, + "declarations_available": False, + "declaration_count": 0, + "status": "blocked_invalid_or_unavailable_declaration_source", + "reason": str(error), + } + return { + "write_enabled": False, + "declarations_available": bool(declarations), + "declaration_count": len(declarations), + "status": ( + "validated_concrete_declarations_available" + if declarations + else "blocked_no_concrete_declarations" + ), + } + + def require_exact_match( + self, + *, + active_product_version: str, + specification_commit: str, + ) -> Mapping[str, Any]: + if ( + not isinstance(specification_commit, str) + or len(specification_commit) != 40 + or any(character not in "0123456789abcdef" for character in specification_commit) + ): + raise UnsupportedOperationError( + "Dolt conformance selection requires an exact lowercase 40-hex " + "specification commit." + ) + declarations = self.validated_declarations() + if not declarations: + raise UnsupportedOperationError( + "The validated Dolt declaration source contains no concrete " + "declarations; write rejected before mutation." + ) + shared_error, _source_type, _loader, shared_selector = _shared_api() + try: + return shared_selector( + declarations, + active_product_version=active_product_version, + adapter_implementation_id=ADAPTER_IMPLEMENTATION_ID, + adapter_version=ADAPTER_VERSION, + specification_commit=specification_commit, + ) + except shared_error as error: + raise UnsupportedOperationError( + "No unique exact Dolt conformance declaration matches the " + "active product, adapter, and specification binding: " + + str(error) + ) from error + + +def effective_limits(declaration: Mapping[str, Any]) -> dict[str, Any]: + """Project the shared declaration's effective layers into adapter limits.""" + + dimensions = declaration["limit_declarations"] + + def effective(name: str) -> Mapping[str, Any]: + matches = tuple( + row for row in dimensions[name] if row.get("basis") == "effective" + ) + if len(matches) != 1: + raise UnsupportedOperationError( + "Validated Dolt declaration has no unique effective " + name + " layer." + ) + return matches[0] + + physical = effective("physical_columns") + source = effective("source_variables") + identifier = effective("identifier") + value = effective("value") + structural = effective("structural_row") + statement = effective("emitted_statement") + return { + "maximum_physical_columns": physical["value"], + "maximum_source_variables": source["value"], + "maximum_statement_bytes": statement["value"], + "identifier_limit": { + "value": identifier["value"], + "unit": identifier["unit"], + "source": "validated concrete Dolt declaration", + "repertoire": declaration["identifier_limit"]["repertoire"], + }, + "maximum_value_bytes": value["value"], + "maximum_row_bytes": ( + structural["value"] + if isinstance(structural["value"], int) + and not isinstance(structural["value"], bool) + else None + ), + "limit_basis": "validated_concrete_dolt_declaration", + "sources": { + name: "declaration:" + str(declaration["declaration_id"]) + for name in ( + "maximum_physical_columns", + "maximum_source_variables", + "maximum_statement_bytes", + "identifier_limit", + "maximum_value_bytes", + "maximum_row_bytes", + ) + }, + } diff --git a/src/openstatspec/sql/profiles.py b/src/openstatspec/sql/profiles.py old mode 100755 new mode 100644 index 84142c4..e1800f4 --- a/src/openstatspec/sql/profiles.py +++ b/src/openstatspec/sql/profiles.py @@ -4,10 +4,10 @@ Importers use this information for preflight checks before creating a dataset. """ -import math - from collections.abc import Iterable, Mapping from dataclasses import dataclass +import math +from numbers import Real from typing import Any from urllib.parse import urlparse @@ -18,23 +18,30 @@ class SqlProfile: name: str url_schemes: tuple[str, ...] - max_physical_variables: int + max_source_variables: int identifier_limit: int binary64_numeric: bool lossless_text: bool max_text_value_bytes: int - max_row_bytes: int + max_row_bytes: int | None tested_reference: bool = False driver_packages: tuple[str, ...] = () + max_statement_bytes: int | None = None + + @property + def max_physical_variables(self) -> int: + """Compatibility alias for the historical, source-count-named field.""" + return self.max_source_variables def as_dict(self) -> dict[str, object]: return { - "max_physical_variables": self.max_physical_variables, + "max_source_variables": self.max_source_variables, "identifier_limit": self.identifier_limit, "binary64_numeric": self.binary64_numeric, "lossless_text": self.lossless_text, "max_text_value_bytes": self.max_text_value_bytes, "max_row_bytes": self.max_row_bytes, + "max_statement_bytes": self.max_statement_bytes, "tested_reference": self.tested_reference, "driver_packages": list(self.driver_packages), } @@ -48,20 +55,16 @@ def as_dict(self) -> dict[str, object]: "postgresql", ("postgresql", "postgres"), 1_599, 63, True, True, 1_073_741_823, 1_073_741_823, True, ("psycopg",), ) -# SQLAlchemy's generic Text column compiles to MySQL TEXT, not MEDIUMTEXT. -# The contract therefore declares TEXT's 64 KiB payload limit rather than -# promising an unimplemented wider physical type. MYSQL = SqlProfile( "mysql", ("mysql", "mariadb"), 1_016, 64, True, True, 4_294_967_295, 65_535, True, ("PyMySQL",), ) DOLT = SqlProfile( "dolt", (), 305, 64, True, True, - 65_504, 65_504, True, ("PyMySQL",), + 4_294_967_295, None, False, ("PyMySQL",), ) -# Dolt deliberately has no URL scheme. It is selected only after the server -# reached through the MySQL wire family positively identifies itself as Dolt. -PROFILES = (SQLITE, POSTGRESQL, MYSQL, DOLT) +PROFILES = (SQLITE, POSTGRESQL, MYSQL) +MYSQL_WIRE_PROFILES = frozenset({"mysql", "mariadb", "dolt"}) def profile_for_url(database_url: str) -> SqlProfile: @@ -108,12 +111,15 @@ def preflight( """Validate strict target capabilities before any source dataset is created.""" variables = None if isinstance(variables_or_count, int) else list(variables_or_count) variable_count = variables_or_count if isinstance(variables_or_count, int) else len(variables) - if variable_count > profile.max_physical_variables: + if variable_count > profile.max_source_variables: raise _exceeded( - "physical_variable_limit", - f"{profile.name} supports at most {profile.max_physical_variables} " + "source_variable_limit", + f"{profile.name} supports at most {profile.max_source_variables} " "source variables in one strict wide table.", - variable_count=variable_count, maximum=profile.max_physical_variables, + source_count=variable_count, + max_source=profile.max_source_variables, + physical_count=variable_count + 1, + max_physical=profile.max_source_variables + 1, ) if variables is None: return @@ -167,7 +173,7 @@ def preflight( maximum=profile.max_text_value_bytes, ) declared_row_bytes = 8 + sum(_row_storage_bytes(profile, variable) for variable in variables) - if declared_row_bytes > profile.max_row_bytes: + if profile.max_row_bytes is not None and declared_row_bytes > profile.max_row_bytes: raise _exceeded( "declared_row_size_limit", f"the declared SQL row requires {declared_row_bytes} bytes; " @@ -177,20 +183,80 @@ def preflight( for row_ordinal, row in enumerate(rows or (), start=1): row_bytes = 8 for variable in variables: - if variable.get("storage_kind") != "string": - value = row.get(str(variable["physical_name"])) + if variable.get("storage_kind") == "numeric": + physical_name = str(variable["physical_name"]) + if physical_name not in row: + raise _exceeded( + "numeric_value_missing", + f"row {row_ordinal} has no value for {variable['source_name']!r}.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + ) + value = row[physical_name] + if value is not None and ( + isinstance(value, bool) or not isinstance(value, Real) + ): + raise _exceeded( + "numeric_value_type", + f"row {row_ordinal} value for {variable['source_name']!r} " + "is not a binary64 number or SQL NULL.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + value_type=type(value).__name__, + ) + if isinstance(value, int) and value is not None: + try: + converted = float(value) + except OverflowError as error: + raise _exceeded( + "numeric_value_not_binary64_exact", + f"row {row_ordinal} integer for {variable['source_name']!r} " + "cannot be represented exactly as binary64.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + ) from error + if not math.isfinite(converted) or int(converted) != value: + raise _exceeded( + "numeric_value_not_binary64_exact", + f"row {row_ordinal} integer for {variable['source_name']!r} " + "cannot be represented exactly as binary64.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + ) if value is not None and not math.isfinite(float(value)): raise _exceeded( - "nonfinite_numeric_value", + "numeric_value_not_finite", f"row {row_ordinal} value for {variable['source_name']!r} " - "is not a finite binary64 value.", + "is not finite; non-finite adapter inputs are rejected.", row_ordinal=row_ordinal, source_name=variable["source_name"], + classification=( + "nan" if math.isnan(float(value)) + else "positive_infinity" if float(value) > 0 + else "negative_infinity" + ), ) row_bytes += 8 continue - value = row.get(str(variable["physical_name"]), "") - encoded_bytes = len(str(value).encode("utf-8")) + physical_name = str(variable["physical_name"]) + if physical_name not in row: + raise _exceeded( + "string_value_missing", + f"row {row_ordinal} has no value for {variable['source_name']!r}.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + ) + value = row[physical_name] + if not isinstance(value, str): + raise _exceeded( + "string_value_type", + f"row {row_ordinal} value for {variable['source_name']!r} " + "is not a string; SPSS string missing values must be empty strings.", + row_ordinal=row_ordinal, + source_name=variable["source_name"], + value_type=type(value).__name__, + ) + encoded_bytes = len(value.encode("utf-8")) if encoded_bytes > profile.max_text_value_bytes: raise _exceeded( "text_value_limit", @@ -200,8 +266,8 @@ def preflight( row_ordinal=row_ordinal, source_name=variable["source_name"], encoded_bytes=encoded_bytes, maximum=profile.max_text_value_bytes, ) - row_bytes += 20 if profile.name in {"mysql", "mariadb", "dolt"} else encoded_bytes - if row_bytes > profile.max_row_bytes: + row_bytes += 20 if profile.name in MYSQL_WIRE_PROFILES else encoded_bytes + if profile.max_row_bytes is not None and row_bytes > profile.max_row_bytes: raise _exceeded( "row_size_limit", f"row {row_ordinal} requires {row_bytes} bytes; " @@ -209,16 +275,39 @@ def preflight( row_ordinal=row_ordinal, row_bytes=row_bytes, maximum=profile.max_row_bytes, ) + statement_bytes = statement_payload_bytes(row, variables) + if profile.max_statement_bytes is not None and statement_bytes > profile.max_statement_bytes: + raise _exceeded( + "statement_payload_limit", + f"row {row_ordinal} requires {statement_bytes} payload bytes; " + f"{profile.name} permits {profile.max_statement_bytes} per bounded statement.", + row_ordinal=row_ordinal, statement_bytes=statement_bytes, + maximum=profile.max_statement_bytes, + ) def _row_storage_bytes(profile: SqlProfile, variable: Mapping[str, Any]) -> int: if variable.get("storage_kind") == "numeric": return 8 - if profile.name in {"mysql", "mariadb", "dolt"}: + if profile.name in MYSQL_WIRE_PROFILES: return 20 return int(variable.get("string_width") or 0) +def statement_payload_bytes( + row: Mapping[str, Any], variables: Iterable[Mapping[str, Any]], +) -> int: + """Count raw payload bytes for packet-safe bounded insert batches.""" + size = 32 + for variable in variables: + if variable.get("storage_kind") == "numeric": + size += 8 + else: + value = row.get(str(variable["physical_name"]), "") + size += len(str(value).encode("utf-8")) + 8 + return size + + def _physical_name(source_name: str, used: set[str]) -> str: """The profile-independent deterministic OpenStatSpec SQL-name mapping.""" diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index a9e3bad..bd21beb 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1,23 +1,27 @@ """SQLite reference SQL profile for the strict OpenStatSpec wide-table contract.""" +import hashlib import json import math import re import sys +from contextlib import contextmanager from datetime import UTC, datetime from uuid import uuid4 from collections.abc import Iterable, Mapping from typing import Any -from sqlalchemy import delete, BigInteger, Boolean, Column, Float, Integer, MetaData, String, Table, Text, create_engine, insert, inspect, select, text, update -from sqlalchemy.dialects import mysql, postgresql, sqlite -from ..core import UnsupportedOperationError -from .capabilities import effective_profile -from .profiles import preflight, validate_connection_url +from sqlalchemy import delete, BigInteger, Boolean, Column, DateTime, Float, Integer, MetaData, String, Table, Text, create_engine, insert, inspect, select, text, update +from ..core import UnsupportedOperationError, safe_error_identity as _safe_error_identity +from .capabilities import active_connection, effective_profile +from .dolt_conformance import DoltConformanceSource +from .profiles import preflight, statement_payload_bytes, validate_connection_url from .normative import ( + binary64_type, + CATALOG_CONTRACT_ID, + CATALOG_SCHEMA_VERSION, catalog as normative_catalog, create as create_normative_catalog, - delete_dataset_representation as delete_normative_dataset, dataset_id_for_name as normative_dataset_id_for_name, finish_operation as finish_normative_operation, record_fidelity_events as record_normative_fidelity_events, @@ -37,29 +41,257 @@ def __init__(self, code: str, detail: str, *, details: Mapping[str, Any]) -> Non self.details = {"reason": code, **details} +class ImportRecoveryError(UnsupportedOperationError): + """Import recovery could not establish the promised terminal state.""" + + def __init__(self, code: str, detail: str, *, details: Mapping[str, Any]) -> None: + super().__init__(f"OpenStatSpec import recovery failed [{code}]: {detail}") + self.code = code + self.details = {"reason": code, **details} + + def _catalog_error(code: str, detail: str, **details: Any) -> CatalogPreflightError: return CatalogPreflightError(code, detail, details=details) -def string_type(profile: Any) -> Text: - """Use Dolt's tested LONGTEXT storage without changing MySQL/MariaDB DDL.""" - return mysql.LONGTEXT() if profile.name == "dolt" else Text() +def _canonical_sha256(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") + ).hexdigest() -def binary64_type() -> Float: - """Return the required IEEE-754 binary64 SQL type for every profile. +def _verification_fault_identity( + code: str, *, phase: str, evidence: Any, +) -> dict[str, Any]: + return { + "type": "InvariantVerificationError", + "code": code, + "phase": phase, + "message_sha256": _canonical_sha256(evidence), + } + - ``Float()`` is not adequate as a portable declaration: SQLAlchemy compiles - it to ``FLOAT`` for MySQL, which is single precision there. The strict - profile therefore declares the physical type explicitly for every target. - """ - return ( - Float(precision=53) - .with_variant(mysql.DOUBLE(asdecimal=False), "mysql") - .with_variant(mysql.DOUBLE(asdecimal=False), "mariadb") - .with_variant(postgresql.DOUBLE_PRECISION(), "postgresql") - .with_variant(sqlite.REAL(), "sqlite") +def _normalized_dolt_rows( + rows: Iterable[Mapping[str, Any]], *, expected_keys: tuple[str, ...], +) -> list[dict[str, Any]]: + normalized = [] + for row in rows: + raw = dict(row) + if set(raw) != set(expected_keys): + raise UnsupportedOperationError( + "Dolt state probe returned an unexpected column shape." + ) + if expected_keys == ("table_name", "staged", "status"): + if ( + not isinstance(raw["table_name"], str) + or not raw["table_name"].strip() + or raw["staged"] not in {False, True, 0, 1} + or not isinstance(raw["status"], str) + or not raw["status"].strip() + ): + raise UnsupportedOperationError( + "Dolt status probe returned an invalid row value shape." + ) + raw["staged"] = bool(raw["staged"]) + else: + relation_names = (raw["from_table_name"], raw["to_table_name"]) + if ( + not any(isinstance(name, str) and name.strip() for name in relation_names) + or any(name is not None and not isinstance(name, str) for name in relation_names) + or not isinstance(raw["diff_type"], str) + or not raw["diff_type"].strip() + or raw["data_change"] not in {False, True, 0, 1} + or raw["schema_change"] not in {False, True, 0, 1} + ): + raise UnsupportedOperationError( + "Dolt diff-summary probe returned an invalid row value shape." + ) + raw["data_change"] = bool(raw["data_change"]) + raw["schema_change"] = bool(raw["schema_change"]) + normalized.append({key: raw[key] for key in expected_keys}) + return sorted(normalized, key=lambda row: json.dumps(row, sort_keys=True, default=str)) + + +def _dolt_evidence_block( + rows: Iterable[Mapping[str, Any]], *, audit_relations: set[str], + expected_keys: tuple[str, ...], +) -> dict[str, Any]: + normalized = _normalized_dolt_rows(rows, expected_keys=expected_keys) + + def is_audit_row(row: Mapping[str, Any]) -> bool: + if expected_keys == ("table_name", "staged", "status"): + return row["table_name"] in audit_relations and ( + str(row["status"]).strip().casefold() == "modified" + ) + from_name = row.get("from_table_name") + to_name = row.get("to_table_name") + return ( + isinstance(from_name, str) + and from_name == to_name + and from_name in audit_relations + and str(row.get("diff_type") or "").strip().casefold() == "modified" + and bool(row.get("data_change")) + and not bool(row.get("schema_change")) + ) + + audit = [row for row in normalized if is_audit_row(row)] + non_audit = [row for row in normalized if not is_audit_row(row)] + return { + "rows": normalized, + "sha256": _canonical_sha256(normalized), + "audit_catalog_rows": audit, + "audit_catalog_sha256": _canonical_sha256(audit), + "non_audit_rows": non_audit, + "non_audit_sha256": _canonical_sha256(non_audit), + } + + +def _capture_dolt_state( + connection: Any, *, profile_name: str, audit_relations: set[str], +) -> dict[str, Any] | None: + """Capture read-only Dolt version-control state without changing branches or HEAD.""" + if profile_name != "dolt": + return None + identity = connection.exec_driver_sql( + "SELECT DATABASE() AS database_name, ACTIVE_BRANCH() AS active_branch, " + "DOLT_HASHOF('HEAD') AS head_hash" + ).mappings().one() + summaries = {} + for label, left, right in ( + ("head_to_working", "HEAD", "WORKING"), + ("head_to_staged", "HEAD", "STAGED"), + ("staged_to_working", "STAGED", "WORKING"), + ): + rows = connection.exec_driver_sql( + "SELECT from_table_name, to_table_name, diff_type, " + "data_change, schema_change " + f"FROM DOLT_DIFF_SUMMARY('{left}', '{right}') " + "ORDER BY from_table_name, to_table_name, diff_type" + ).mappings().all() + summaries[label] = _dolt_evidence_block( + rows, audit_relations=audit_relations, + expected_keys=( + "from_table_name", "to_table_name", "diff_type", + "data_change", "schema_change", + ), + ) + status = _dolt_evidence_block( + connection.exec_driver_sql( + "SELECT table_name, staged, status FROM dolt_status " + "ORDER BY table_name, staged, status" + ).mappings().all(), + audit_relations=audit_relations, + expected_keys=("table_name", "staged", "status"), ) + for key in ("database_name", "active_branch", "head_hash"): + if not isinstance(identity[key], str) or not identity[key].strip(): + raise UnsupportedOperationError( + f"Dolt state probe returned no non-empty {key}." + ) + result = { + "database": identity["database_name"].strip(), + "active_branch": identity["active_branch"].strip(), + "head": identity["head_hash"].strip(), + "status": status, + "diff_summaries": summaries, + } + result["snapshot_sha256"] = _canonical_sha256(result) + return result + + +def _require_dolt_working_set_binding( + snapshot: dict[str, Any] | None, active: Mapping[str, Any], *, phase: str, +) -> None: + if snapshot is None: + return + binding = active.get("working_set_binding") + if ( + not isinstance(binding, Mapping) + or snapshot["database"] != binding.get("database") + or snapshot["active_branch"] != binding.get("active_branch") + ): + raise UnsupportedOperationError( + f"Dolt database/branch working-set binding mismatch during {phase}." + ) + + +def _require_dolt_success_identity( + before: dict[str, Any] | None, after: dict[str, Any] | None, *, phase: str, +) -> None: + if before is None and after is None: + return + if before is None or after is None or any( + before[key] != after[key] for key in ("database", "active_branch", "head") + ): + raise UnsupportedOperationError( + f"Dolt database/branch/HEAD changed during {phase}." + ) + + +def _dolt_failure_boundary_evidence( + before: dict[str, Any] | None, after: dict[str, Any] | None, +) -> dict[str, Any]: + if before is None and after is None: + return {"applicable": False} + if before is None or after is None: + return {"applicable": True, "verified": False, "reason": "snapshot_missing"} + invariant_failures = [] + for key in ("database", "active_branch", "head"): + if before[key] != after[key]: + invariant_failures.append(f"{key}_changed") + if before["status"]["non_audit_sha256"] != after["status"]["non_audit_sha256"]: + invariant_failures.append("non_audit_status_changed") + for label in sorted(before["diff_summaries"]): + if ( + before["diff_summaries"][label]["non_audit_sha256"] + != after["diff_summaries"][label]["non_audit_sha256"] + ): + invariant_failures.append(f"non_audit_{label}_changed") + return { + "applicable": True, + "verified": not invariant_failures, + "invariant_failures": invariant_failures, + "before": before, + "after": after, + "permitted_delta": "failed-operation audit catalog relations only", + "prohibited_vc_actions": [ + "DOLT_ADD", "DOLT_COMMIT", "checkout", "reset", "branch_change", + ], + } + + +@contextmanager +def _bound_catalog_transaction( + *, engine: Any, profile_name: str, active: Mapping[str, Any], + audit_relations: set[str], phase: str, +) -> Iterable[Any]: + """Bind a write transaction to one Dolt database/branch/HEAD identity.""" + with engine.connect() as connection: + before = _capture_dolt_state( + connection, profile_name=profile_name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding(before, active, phase=f"{phase} preflight") + if profile_name == "dolt": + raise UnsupportedOperationError( + "Dolt audit-only mutation is blocked until operation-scoped " + "append/update and immutable-history evidence is implemented." + ) + connection.rollback() + with connection.begin(): + yield connection + after = _capture_dolt_state( + connection, profile_name=profile_name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding(after, active, phase=f"{phase} completion") + _require_dolt_success_identity(before, after, phase=phase) + boundary = _dolt_failure_boundary_evidence(before, after) + if boundary.get("applicable") and not boundary.get("verified"): + raise UnsupportedOperationError( + f"Dolt non-audit working-set state changed during {phase}; " + "the transaction was rolled back." + ) def catalog(metadata: MetaData) -> tuple[Table, Table, Table, Table]: @@ -144,14 +376,17 @@ def _migrate_catalog_columns( the newly observable metadata alongside it. """ inspector = inspect(connection) + text_declaration = str( + Text.compile(dialect=connection.dialect) + ) additions = { datasets.name: { - "file_attributes": "TEXT NOT NULL DEFAULT '{}'", + "file_attributes": f"{text_declaration} NOT NULL DEFAULT '{{}}'", "case_weight_variable": "VARCHAR(255)", }, variables.name: { "role": "VARCHAR(32)", - "attributes": "TEXT NOT NULL DEFAULT '{}'", + "attributes": f"{text_declaration} NOT NULL DEFAULT '{{}}'", "compat_name": "VARCHAR(255)", "print_format": "VARCHAR(64)", "write_format": "VARCHAR(64)", @@ -162,7 +397,7 @@ def _migrate_catalog_columns( "use_first_var_label": "BOOLEAN", "counted_value_type": "VARCHAR(16)", "counted_numeric": "DOUBLE", - "counted_text": "TEXT", + "counted_text": text_declaration, }, } preparer = connection.dialect.identifier_preparer @@ -205,14 +440,11 @@ def _record_failed_preflight( multiple_response_catalog: Table, fidelity_event_catalog: Table, operation_catalog: Table, normative: Any, operation_id: str, source_name: str, source_format: str, variable_count: int, profile_name: str, error: Exception, + legacy: tuple[Table, ...], ) -> None: """Persist a failed preflight without creating any source dataset state.""" with engine.begin() as connection: - create_normative_catalog(connection, normative) - metadata.create_all(connection, tables=[ - datasets, variable_catalog, multiple_response_catalog, - fidelity_event_catalog, operation_catalog, - ]) + _require_verified_catalog(connection, normative, legacy) failed_at = datetime.now(UTC).replace(tzinfo=None) record_normative_operation( connection, normative, operation_id=operation_id, @@ -745,6 +977,1338 @@ def data_table_name(dataset_id: str) -> str: return f"data_{stem[:48]}" +def _catalog_layout(metadata: MetaData) -> tuple[tuple[Table, ...], Any]: + datasets, variables, fidelity_events, operations = catalog(metadata) + multiple_response = multiple_response_set_catalog(metadata) + source_extensions = source_extension_catalog(metadata) + documents, value_labels, missing_rules, attributes = normalized_metadata_tables(metadata) + return ( + datasets, variables, multiple_response, source_extensions, documents, + value_labels, missing_rules, attributes, fidelity_events, operations, + ), normative_catalog(metadata) + +def _normalized_sql_type(inspector: Any, value: Any) -> str: + compiled = " ".join( + str(value.compile(dialect=inspector.bind.dialect)).strip().upper().split() + ) + if inspector.bind.dialect.name in {"mysql", "mariadb"}: + if compiled in {"BOOL", "BOOLEAN", "TINYINT(1)"}: + return "BOOLEAN/TINYINT(1)" + return compiled + + +def _normalized_default(value: Any) -> str | None: + if value is None: + return None + result = " ".join(str(value).strip().split()) + while result.startswith("(") and result.endswith(")"): + result = result[1:-1].strip() + return result + + +def _expected_unique_constraints(table: Table) -> set[tuple[str, ...]]: + return { + tuple(column.name for column in constraint.columns) + for constraint in table.constraints + if getattr(constraint, "__visit_name__", "") == "unique_constraint" + } + + +def _actual_unique_constraints( + inspector: Any, table_name: str, +) -> set[tuple[str, ...]]: + constraints = { + tuple(str(name) for name in item.get("column_names") or ()) + for item in inspector.get_unique_constraints(table_name) + } + constraints.update( + tuple(str(name) for name in item.get("column_names") or ()) + for item in inspector.get_indexes(table_name) + if item.get("unique") + ) + constraints.discard(()) + return constraints + + +def _expected_foreign_keys(table: Table) -> set[tuple[Any, ...]]: + return { + ( + tuple(column.name for column in constraint.columns), + next(iter(constraint.elements)).column.table.name, + tuple(element.column.name for element in constraint.elements), + ) + for constraint in table.foreign_key_constraints + } + + +def _actual_foreign_keys( + inspector: Any, table_name: str, +) -> set[tuple[Any, ...]]: + return { + ( + tuple(str(name) for name in item.get("constrained_columns") or ()), + str(item.get("referred_table") or ""), + tuple(str(name) for name in item.get("referred_columns") or ()), + ) + for item in inspector.get_foreign_keys(table_name) + } + + +def _normalized_check_sql(value: Any) -> str: + result = " ".join(str(value).strip().casefold().split()) + result = result.replace("`", "").replace('"', "") + result = re.sub(r"\[([^]]+)\]", r"\1", result) + while result.startswith("(") and result.endswith(")"): + result = result[1:-1].strip() + return result + + +def _expected_check_constraints(table: Table) -> set[str]: + return { + _normalized_check_sql(constraint.sqltext) + for constraint in table.constraints + if getattr(constraint, "__visit_name__", "") == "table_or_column_check_constraint" + } + + +def _actual_check_constraints(inspector: Any, table_name: str) -> set[str]: + return { + _normalized_check_sql(item.get("sqltext") or "") + for item in inspector.get_check_constraints(table_name) + } + + +_MIGRATED_SERVER_DEFAULTS = { + ("dataset_catalog", "file_attributes"): {None, "'{}'", '"{}"', "{}"}, + ("variable_catalog", "attributes"): {None, "'{}'", '"{}"', "{}"}, +} + + +def _catalog_table_shape_valid( + inspector: Any, table: Table, *, allow_missing: bool, +) -> bool: + actual = { + str(column["name"]): column + for column in inspector.get_columns(table.name) + } + expected = {column.name: column for column in table.columns} + if set(actual) - set(expected): + return False + if not allow_missing and set(actual) != set(expected): + return False + for name in set(actual) & set(expected): + expected_column = expected[name] + actual_column = actual[name] + if ( + _normalized_sql_type(inspector, expected_column.type) + != _normalized_sql_type(inspector, actual_column["type"]) + ): + return False + if bool(actual_column.get("nullable")) != bool(expected_column.nullable): + return False + actual_default = _normalized_default(actual_column.get("default")) + if (table.name, name) in _MIGRATED_SERVER_DEFAULTS: + if actual_default not in _MIGRATED_SERVER_DEFAULTS[(table.name, name)]: + return False + else: + expected_default = _normalized_default( + expected_column.server_default.arg + if expected_column.server_default is not None else None + ) + if actual_default != expected_default: + return False + if actual_column.get("identity") is not None or actual_column.get("computed") is not None: + return False + if expected_column.autoincrement is True and actual_column.get("autoincrement") is not True: + return False + if expected_column.autoincrement is False and actual_column.get("autoincrement") is True: + return False + expected_pk = tuple(column.name for column in table.primary_key.columns) + actual_pk = tuple( + str(name) for name in ( + inspector.get_pk_constraint(table.name).get("constrained_columns") or () + ) + ) + if actual_pk != expected_pk: + return False + if _actual_unique_constraints(inspector, table.name) != _expected_unique_constraints(table): + return False + if _actual_foreign_keys(inspector, table.name) != _expected_foreign_keys(table): + return False + if _actual_check_constraints(inspector, table.name) != _expected_check_constraints(table): + return False + return True + + +def _identity_shape_valid( + inspector: Any, table: Table, *, key_name: str, +) -> bool: + try: + return ( + [column.name for column in table.primary_key.columns] == [key_name] + and _catalog_table_shape_valid(inspector, table, allow_missing=False) + ) + except Exception: + return False + + +def _catalog_existing_shapes_valid( + inspector: Any, tables: Iterable[Table], +) -> bool: + """Require exact existing shape while allowing only absent migration columns.""" + try: + return all( + _catalog_table_shape_valid(inspector, table, allow_missing=True) + for table in tables + ) + except Exception: + return False + + +_MIGRATABLE_CATALOG_COLUMNS = { + "dataset_catalog": {"file_attributes", "case_weight_variable"}, + "variable_catalog": { + "role", "attributes", "compat_name", "print_format", "write_format", + }, + "multiple_response_set_catalog": { + "is_dichotomy", "use_category_labels", "use_first_var_label", + "counted_value_type", "counted_numeric", "counted_text", + }, +} + + +def _catalog_missing_columns( + inspector: Any, tables: Iterable[Table], +) -> dict[str, set[str]]: + missing = {} + for table in tables: + absent = {column.name for column in table.columns} - { + str(column["name"]) + for column in inspector.get_columns(table.name) + } + if absent: + missing[table.name] = absent + return missing + + +def _catalog_missing_columns_are_migratable( + missing: Mapping[str, set[str]], +) -> bool: + return bool(missing) and all( + columns <= _MIGRATABLE_CATALOG_COLUMNS.get(table_name, set()) + for table_name, columns in missing.items() + ) + + +def _registered_physical_relations( + connection: Any, *, existing_tables: set[str], normative: Any, + legacy: Iterable[Table], +) -> tuple[set[str], tuple[Table, ...], set[str], set[str], str]: + """Return owned relations and the optional workflow identity state.""" + legacy = tuple(legacy) + declared_tables = legacy + normative.all() + static_tables = {table.name for table in declared_tables} + physical_tables: set[str] = set() + physical_views: set[str] = set() + inspector = inspect(connection) + if ( + legacy[0].name in existing_tables + and "data_table" in { + str(column["name"]) for column in inspector.get_columns(legacy[0].name) + } + ): + physical_tables.update(str(name) for name in connection.execute( + select(legacy[0].c.data_table) + ).scalars() if name) + if ( + normative.dataset.name in existing_tables + and "physical_table_name" in { + str(column["name"]) for column in inspector.get_columns(normative.dataset.name) + } + ): + physical_tables.update(str(name) for name in connection.execute( + select(normative.dataset.c.physical_table_name) + ).scalars() if name) + + # The optional workflow is another OpenStatSpec-owned relation profile in + # the same dedicated namespace. Import locally to avoid its documented + # dependency on this module during module initialization. + from .workflow import ( # pylint: disable=import-outside-toplevel + PROFILE_ID, PROFILE_SCHEMA_VERSION, workflow_catalog, + ) + workflow = workflow_catalog(MetaData()) + workflow_tables = {table.name for table in workflow.all()} + workflow_identity = workflow.transformation_profile_identity + profile_present = workflow_identity.name in existing_tables + if profile_present: + declared_tables += workflow.all() + static_tables.update(workflow_tables) + if not workflow_tables <= existing_tables: + return static_tables, declared_tables, physical_tables, physical_views, "foreign" + if not _identity_shape_valid( + inspector, workflow_identity, key_name="profile_identity_key", + ): + return static_tables, declared_tables, physical_tables, physical_views, "foreign" + identities = connection.execute(select(workflow_identity)).mappings().all() + if len(identities) != 1: + return static_tables, declared_tables, physical_tables, physical_views, "ambiguous" + if ( + identities[0]["profile_identity_key"] != 1 + or identities[0]["contract_id"] != PROFILE_ID + or identities[0]["schema_version"] != PROFILE_SCHEMA_VERSION + or identities[0]["core_contract_id"] != CATALOG_CONTRACT_ID + ): + return static_tables, declared_tables, physical_tables, physical_views, "foreign" + for row in connection.execute(select( + workflow.derived_dataset.c.physical_relation_name, + workflow.derived_dataset.c.output_mode, + )).mappings(): + name = str(row["physical_relation_name"]) + if row["output_mode"] == "view": + physical_views.add(name) + else: + physical_tables.add(name) + elif workflow_tables & existing_tables: + return static_tables, declared_tables, physical_tables, physical_views, "foreign" + return static_tables, declared_tables, physical_tables, physical_views, "valid" + + +def _catalog_dataset_bijection_state( + connection: Any, *, normative: Any, legacy: Iterable[Table], +) -> str: + """Require one exact legacy-to-normative dataset/table mapping per row.""" + datasets = tuple(legacy)[0] + legacy_rows = [ + (row["dataset_id"], row["data_table"]) + for row in connection.execute(select( + datasets.c.dataset_id, datasets.c.data_table, + )).mappings() + ] + normative_rows = [ + (row["dataset_name"], row["physical_table_name"]) + for row in connection.execute(select( + normative.dataset.c.dataset_name, + normative.dataset.c.physical_table_name, + )).mappings() + ] + for rows in (legacy_rows, normative_rows): + if any( + not isinstance(dataset_name, str) or not dataset_name.strip() + or not isinstance(table_name, str) or not table_name.strip() + for dataset_name, table_name in rows + ): + return "unverified" + dataset_names = [dataset_name for dataset_name, _table_name in rows] + table_names = [table_name for _dataset_name, table_name in rows] + if ( + len(set(dataset_names)) != len(dataset_names) + or len(set(table_names)) != len(table_names) + or len(set(rows)) != len(rows) + ): + return "ambiguous" + return "valid" if set(legacy_rows) == set(normative_rows) else "unverified" + + +def _catalog_state(connection: Any, normative: Any, legacy: Iterable[Table]) -> str: + inspector = inspect(connection) + existing_tables = set(inspector.get_table_names()) + existing_views = set(inspector.get_view_names()) + if existing_tables & existing_views: + return "ambiguous" + existing_relations = existing_tables | existing_views + if normative.catalog_identity.name not in existing_tables: + return "absent" if not existing_relations else "foreign" + try: + identities = connection.execute( + select(normative.catalog_identity) + ).mappings().all() + except Exception: + return "foreign" + if len(identities) != 1: + return "ambiguous" + if not _identity_shape_valid( + inspector, normative.catalog_identity, key_name="catalog_identity_key", + ): + return "foreign" + identity = identities[0] + if ( + identity["catalog_identity_key"] != 1 + or identity["contract_id"] != CATALOG_CONTRACT_ID + or identity["schema_version"] != CATALOG_SCHEMA_VERSION + ): + return "foreign" + static_tables, declared_tables, physical_tables, physical_views, profile_valid = ( + _registered_physical_relations( + connection, existing_tables=existing_tables, + normative=normative, legacy=legacy, + ) + ) + if profile_valid != "valid": + return profile_valid + if ( + physical_tables & physical_views + or static_tables & physical_tables + or static_tables & physical_views + ): + return "ambiguous" + owned_relations = static_tables | physical_tables | physical_views + if existing_relations - owned_relations: + return "foreign" + if physical_tables - existing_tables or physical_views - existing_views: + return "unverified" + if not static_tables <= existing_tables: + return "unverified" + if not _catalog_existing_shapes_valid(inspector, declared_tables): + return "foreign" + missing_columns = _catalog_missing_columns(inspector, declared_tables) + if missing_columns: + return ( + "migration_required" + if _catalog_missing_columns_are_migratable(missing_columns) + else "unverified" + ) + mapping_state = _catalog_dataset_bijection_state( + connection, normative=normative, legacy=legacy, + ) + if mapping_state != "valid": + return mapping_state + return "verified" + + +def _require_verified_catalog( + connection: Any, normative: Any, legacy: Iterable[Table], +) -> None: + state = _catalog_state(connection, normative, legacy) + if state != "verified": + raise UnsupportedOperationError( + f"The selected OpenStatSpec catalog is {state}; run explicit catalog initialization first." + ) + + +def _catalog_snapshot( + connection: Any, +) -> tuple[set[str], dict[str, set[str]]]: + inspector = inspect(connection) + tables = set(inspector.get_table_names()) + return tables, { + table_name: { + str(column["name"]) for column in inspector.get_columns(table_name) + } + for table_name in tables + } + + +def _compensate_catalog_initialization( + connection: Any, *, metadata: MetaData, before_tables: set[str], + before_columns: Mapping[str, set[str]], +) -> None: + """Restore the pre-initialization relation/column inventory after DDL failure.""" + current_tables = set(inspect(connection).get_table_names()) + for table in reversed(metadata.sorted_tables): + if table.name in current_tables and table.name not in before_tables: + table.drop(connection, checkfirst=True) + inspector = inspect(connection) + preparer = connection.dialect.identifier_preparer + for table_name, original_columns in before_columns.items(): + if not inspector.has_table(table_name): + raise RuntimeError( + f"Pre-existing catalog table {table_name!r} disappeared during initialization." + ) + current_columns = { + str(column["name"]) for column in inspect(connection).get_columns(table_name) + } + for column_name in sorted(current_columns - original_columns): + connection.execute(text( + f"ALTER TABLE {preparer.quote(table_name)} " + f"DROP COLUMN {preparer.quote(column_name)}" + )) + + +def _catalog_residual_inventory( + engine: Any, *, before_tables: set[str], + before_columns: Mapping[str, set[str]], +) -> dict[str, Any]: + try: + with engine.connect() as connection: + inspector = inspect(connection) + current_tables = set(inspector.get_table_names()) + added_columns = { + table_name: sorted( + { + str(column["name"]) + for column in inspector.get_columns(table_name) + } - original_columns + ) + for table_name, original_columns in before_columns.items() + if table_name in current_tables + } + return { + "new_tables": sorted(current_tables - before_tables), + "missing_preexisting_tables": sorted(before_tables - current_tables), + "added_columns": { + name: columns for name, columns in added_columns.items() if columns + }, + "views": sorted(inspector.get_view_names()), + } + except Exception as inventory_error: + return {"inspection_error_type": type(inventory_error).__name__} + + +def dolt_state_snapshot( + *, + database_url: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + """Return a read-only, digest-bound snapshot of one active Dolt database.""" + validate_connection_url(database_url) + active = active_connection( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + if active["profile"] != "dolt": + raise UnsupportedOperationError( + "dolt_state_snapshot requires a positively identified Dolt connection." + ) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + engine = create_engine(database_url) + with engine.connect() as connection: + state = _capture_dolt_state( + connection, profile_name="dolt", audit_relations=audit_relations, + ) + assert state is not None + _require_dolt_working_set_binding( + state, active, phase="read-only state capture", + ) + binding = active["working_set_binding"] + return { + "profile": "dolt", + "server_version": active["server_version"], + "read_only": True, + "operational_write_enabled": bool(active["claimed_supported"]), + "working_set_binding": binding, + "state": state, + } + + +def initialize_wide_catalog( + *, + database_url: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + """Install or explicitly migrate a dedicated catalog after server preflight.""" + validate_connection_url(database_url) + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + datasets, variables, multiple_response = legacy[:3] + with engine.connect() as connection: + state = _catalog_state(connection, normative, legacy) + if state not in {"absent", "verified", "migration_required"}: + raise UnsupportedOperationError( + f"The selected database catalog is {state}; initialization is not permitted." + ) + before_tables, before_columns = _catalog_snapshot(connection) + pre_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + _require_dolt_working_set_binding( + pre_dolt_state, active, phase="catalog initialization preflight", + ) + connection.rollback() + try: + with connection.begin(): + create_normative_catalog(connection, normative) + metadata.create_all(connection, tables=list(legacy)) + _migrate_catalog_columns( + connection, datasets, variables, multiple_response, + ) + _require_verified_catalog(connection, normative, legacy) + post_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + _require_dolt_working_set_binding( + post_dolt_state, active, phase="catalog initialization completion", + ) + _require_dolt_success_identity( + pre_dolt_state, post_dolt_state, phase="catalog initialization", + ) + except Exception as install_error: + try: + with connection.begin(): + _compensate_catalog_initialization( + connection, metadata=metadata, before_tables=before_tables, + before_columns=before_columns, + ) + except Exception as cleanup_error: + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + try: + after_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + dolt_boundary = _dolt_failure_boundary_evidence( + pre_dolt_state, after_dolt_state, + ) + except Exception as snapshot_error: + dolt_boundary = { + "applicable": profile.name == "dolt", + "verified": False, + "snapshot_fault": _safe_error_identity( + snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", + ), + } + raise ImportRecoveryError( + "cleanup_failed", + "Catalog initialization failed and its DDL compensation also failed.", + details={ + "subcode": "catalog_install_cleanup_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _safe_error_identity( + cleanup_error, phase="catalog_compensation", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.catalog-init-compensation.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted( + table.name for table in metadata.tables.values() + ), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "cleanup_attempted": True, + "cleanup_succeeded": False, + "preexisting_unverified_catalog_mutation_forbidden": True, + "dolt_failure_boundary": dolt_boundary, + }, + "success_forbidden": True, + }, + ) from cleanup_error + try: + after_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + dolt_boundary = _dolt_failure_boundary_evidence( + pre_dolt_state, after_dolt_state, + ) + except Exception as snapshot_error: + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + recovery = { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted(metadata.tables), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": { + "applicable": profile.name == "dolt", + "verified": False, + }, + } + raise ImportRecoveryError( + "cleanup_failed", + "Catalog compensation completed but Dolt state could not be verified.", + details={ + "subcode": "dolt_state_capture_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": recovery, + "success_forbidden": True, + }, + ) from snapshot_error + if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + recovery = { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted(metadata.tables), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": dolt_boundary, + } + raise ImportRecoveryError( + "cleanup_failed", + "Catalog compensation did not preserve Dolt failure-boundary invariants.", + details={ + "subcode": "dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="post_catalog_cleanup_dolt_state_verification", + evidence=dolt_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": recovery, + "success_forbidden": True, + }, + ) from install_error + raise + return { + "profile": profile.name, + "server_version": active["server_version"], + "catalog": "verified", + } + + +def _bounded_batches( + rows: list[dict[str, Any]], variables: list[dict[str, Any]], + maximum: int | None, +) -> Iterable[list[dict[str, Any]]]: + if maximum is None: + if rows: + yield rows + return + batch: list[dict[str, Any]] = [] + used = 0 + for row in rows: + size = statement_payload_bytes(row, variables) + if size > maximum: + raise RuntimeError("A preflighted row exceeds the active statement payload limit.") + if batch and used + size > maximum: + yield batch + batch, used = [], 0 + batch.append(row) + used += size + if batch: + yield batch + + +def _delete_normative_import_state( + connection: Any, normative: Any, *, dataset_name: str, + physical_table_name: str, normative_dataset_id: str | None, + normative_dataset_creation_attempted: bool, operation_id: str, +) -> None: + dataset_ids = [] + if normative_dataset_creation_attempted: + dataset_ids = list(connection.execute( + select(normative.dataset.c.dataset_id).where( + normative.dataset.c.dataset_name == dataset_name, + normative.dataset.c.physical_table_name == physical_table_name, + ) + ).scalars()) + if normative_dataset_id is not None and normative_dataset_id not in dataset_ids: + dataset_ids.append(normative_dataset_id) + connection.execute(delete(normative.fidelity_event).where( + normative.fidelity_event.c.operation_id == operation_id + )) + for normative_dataset_id in dataset_ids: + variable_ids = list(connection.execute( + select(normative.variable.c.variable_id) + .where(normative.variable.c.dataset_id == normative_dataset_id) + ).scalars()) + label_set_ids = list(connection.execute( + select(normative.value_label_set.c.value_label_set_id) + .where(normative.value_label_set.c.dataset_id == normative_dataset_id) + ).scalars()) + variable_set_ids = list(connection.execute( + select(normative.variable_set.c.variable_set_id) + .where(normative.variable_set.c.dataset_id == normative_dataset_id) + ).scalars()) + response_set_ids = list(connection.execute( + select(normative.multiple_response_set.c.multiple_response_set_id) + .where(normative.multiple_response_set.c.dataset_id == normative_dataset_id) + ).scalars()) + if variable_set_ids: + connection.execute(delete(normative.variable_set_member).where( + normative.variable_set_member.c.variable_set_id.in_(variable_set_ids) + )) + if response_set_ids: + connection.execute(delete(normative.multiple_response_member).where( + normative.multiple_response_member.c.multiple_response_set_id.in_(response_set_ids) + )) + if variable_ids: + for table in ( + normative.variable_value_label_set, normative.missing_rule, + normative.variable_attribute, + ): + connection.execute(delete(table).where(table.c.variable_id.in_(variable_ids))) + if label_set_ids: + connection.execute(delete(normative.value_label).where( + normative.value_label.c.value_label_set_id.in_(label_set_ids) + )) + for table in ( + normative.dataset_weight_variable, normative.dataset_attribute, + normative.document, normative.variable_set, normative.multiple_response_set, + normative.value_label_set, normative.fidelity_event, normative.variable, + ): + connection.execute(delete(table).where(table.c.dataset_id == normative_dataset_id)) + connection.execute(delete(normative.dataset).where( + normative.dataset.c.dataset_id == normative_dataset_id + )) + connection.execute(delete(normative.operation).where( + normative.operation.c.operation_id == operation_id + )) + + +def _cleanup_import_state( + connection: Any, *, dataset_id: str, operation_id: str, data_table: Table, + state: Mapping[str, Any], normative: Any, legacy: tuple[Table, ...], +) -> None: + ( + datasets, variables, multiple_response, source_extensions, documents, + value_labels, missing_rules, attributes, fidelity_events, operations, + ) = legacy + _delete_normative_import_state( + connection, normative, dataset_name=dataset_id, + physical_table_name=data_table.name, + normative_dataset_id=state["normative_dataset_id"], + normative_dataset_creation_attempted=state["normative_dataset_creation_attempted"], + operation_id=operation_id, + ) + if state["legacy_dataset_created"]: + for table in ( + multiple_response, source_extensions, documents, value_labels, + missing_rules, attributes, + ): + connection.execute(delete(table).where(table.c.dataset_id == dataset_id)) + connection.execute(delete(variables).where(variables.c.dataset_id == dataset_id)) + connection.execute(delete(datasets).where(datasets.c.dataset_id == dataset_id)) + connection.execute(delete(fidelity_events).where( + fidelity_events.c.operation_id == operation_id + )) + connection.execute(delete(operations).where(operations.c.operation_id == operation_id)) + if state["data_table_creation_attempted"]: + data_table.drop(connection, checkfirst=True) + + +def _record_failed_import_audit( + *, engine: Any, operation_id: str, source_name: str, source_format: str, + variable_count: int, profile_name: str, import_error: Exception, + normative: Any, legacy: tuple[Table, ...], +) -> None: + """Persist only a failed operation and NULL-dataset event after cleanup.""" + fidelity_events, operations = legacy[8:] + failed_event = { + "code": "import_failed", + "detail": "Import failed after mutation began; operation-owned state was removed.", + "severity": "error", + "source_item": source_name, + "details": { + "phase": "mutation", + "profile": profile_name, + "variable_count": variable_count, + "error_type": type(import_error).__name__, + }, + } + with engine.begin() as connection: + _require_verified_catalog(connection, normative, legacy) + failed_at = datetime.now(UTC).replace(tzinfo=None) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="import", status="failed", source_format=source_format, + started_at=failed_at, completed_at=failed_at, + ) + connection.execute(insert(operations).values( + operation_id=operation_id, direction="import", status="failed", + dataset_id=None, source=source_name, created_at=_now(), + completed_at=_now(), details=json.dumps({ + "reason": "runtime_failure", + "variable_count": variable_count, + "error_type": type(import_error).__name__, + }, sort_keys=True), + )) + connection.execute(insert(fidelity_events), _event_rows( + operation_id=operation_id, dataset_id=None, direction="import", + fidelity_events=(failed_event,), + )) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="import", events=(failed_event,), + ) + + +def _record_import_cleanup_failure_audit( + *, engine: Any, operation_id: str, source_name: str, source_format: str, + profile_name: str, import_error: Exception, cleanup_error: Exception, + residual_object_inventory: Mapping[str, Any], + deterministic_recovery_evidence: Mapping[str, Any], + normative: Any, legacy: tuple[Table, ...], +) -> None: + """Best-effort immutable audit for verified-catalog cleanup failure.""" + fidelity_events, operations = legacy[8:] + original = _safe_error_identity(import_error, phase="import_mutation") + cleanup = _safe_error_identity(cleanup_error, phase="compensating_cleanup") + event_details = { + "original_cause": original, + "cleanup_fault": cleanup, + "residual_object_inventory": dict(residual_object_inventory), + "deterministic_recovery_evidence": dict( + deterministic_recovery_evidence + ), + } + event = { + "code": "cleanup_failed", + "detail": "Import cleanup failed; terminal recovery requires out-of-band review.", + "severity": "error", + "source_item": source_name, + "details": event_details, + } + with engine.begin() as connection: + _require_verified_catalog(connection, normative, legacy) + existing = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one_or_none() + normative_existing = connection.execute(select(normative.operation).where( + normative.operation.c.operation_id == operation_id + )).mappings().one_or_none() + if (existing is None) != (normative_existing is None): + raise UnsupportedOperationError( + "Import operation catalogs disagree about cleanup-failure state." + ) + if existing is not None: + if ( + existing["direction"] != "import" + or existing["status"] != "running" + or normative_existing["status"] != "started" + ): + raise UnsupportedOperationError( + "Existing import operation is not in an auditable running state." + ) + details = json.loads(existing["details"] or "{}") + details["cleanup_failure"] = event_details + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values( + status="failed", completed_at=_now(), + details=json.dumps(details, sort_keys=True), + )) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="failed", + ) + ordinals = connection.execute(select(fidelity_events.c.ordinal).where( + fidelity_events.c.operation_id == operation_id + )).scalars().all() + event_row = _event_rows( + operation_id=operation_id, dataset_id=None, direction="import", + fidelity_events=(event,), + )[0] + event_row["ordinal"] = max(ordinals, default=0) + 1 + connection.execute(insert(fidelity_events).values(**event_row)) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="import", events=(event,), + ) + else: + failed_at = datetime.now(UTC).replace(tzinfo=None) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="import", status="failed", source_format=source_format, + started_at=failed_at, completed_at=failed_at, + ) + connection.execute(insert(operations).values( + operation_id=operation_id, direction="import", status="failed", + dataset_id=None, source=source_name, created_at=_now(), + completed_at=_now(), details=json.dumps({ + "reason": "cleanup_failed", + "profile": profile_name, **event_details, + }, sort_keys=True), + )) + connection.execute(insert(fidelity_events), _event_rows( + operation_id=operation_id, dataset_id=None, direction="import", + fidelity_events=(event,), + )) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="import", events=(event,), + ) + + +def _import_residual_inventory( + engine: Any, *, dataset_id: str, operation_id: str, data_table: Table, + state: Mapping[str, Any], normative: Any, legacy: tuple[Table, ...], +) -> dict[str, Any]: + try: + with engine.connect() as connection: + inspector = inspect(connection) + tables = set(inspector.get_table_names()) + + def count_rows(table: Table, condition: Any) -> int | None: + if table.name not in tables: + return None + return len(connection.execute(select(table).where(condition)).all()) + + return { + "data_table": { + "name": data_table.name, + "present": data_table.name in tables, + }, + "legacy_dataset_rows": count_rows( + legacy[0], legacy[0].c.dataset_id == dataset_id, + ), + "legacy_operation_rows": count_rows( + legacy[9], legacy[9].c.operation_id == operation_id, + ), + "legacy_fidelity_event_rows": count_rows( + legacy[8], legacy[8].c.operation_id == operation_id, + ), + "normative_dataset_rows": count_rows( + normative.dataset, + ( + normative.dataset.c.dataset_name == dataset_id + ) & ( + normative.dataset.c.physical_table_name == data_table.name + ), + ), + "normative_operation_rows": count_rows( + normative.operation, + normative.operation.c.operation_id == operation_id, + ), + "normative_fidelity_event_rows": count_rows( + normative.fidelity_event, + normative.fidelity_event.c.operation_id == operation_id, + ), + "mutation_markers": dict(state), + } + except Exception as inventory_error: + return {"inspection_error_type": type(inventory_error).__name__} + + +@contextmanager +def _import_cleanup_guard( + *, engine: Any, dataset_id: str, operation_id: str, data_table: Table, + source_name: str, source_format: str, variable_count: int, + profile_name: str, normative: Any, legacy: tuple[Table, ...], + snapshot_connection: Any, pre_dolt_state: dict[str, Any] | None, +) -> Iterable[dict[str, Any]]: + state: dict[str, Any] = { + "data_table_creation_attempted": False, + "legacy_dataset_created": False, + "normative_dataset_creation_attempted": False, + "normative_dataset_id": None, + } + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + + def capture_boundary() -> dict[str, Any]: + after = _capture_dolt_state( + snapshot_connection, profile_name=profile_name, + audit_relations=audit_relations, + ) + return _dolt_failure_boundary_evidence(pre_dolt_state, after) + + try: + yield state + except Exception as import_error: + try: + with engine.begin() as cleanup_connection: + _cleanup_import_state( + cleanup_connection, dataset_id=dataset_id, + operation_id=operation_id, data_table=data_table, + state=state, normative=normative, legacy=legacy, + ) + except Exception as cleanup_error: + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=state, normative=normative, + legacy=legacy, + ) + try: + pre_audit_dolt_boundary = capture_boundary() + except Exception as snapshot_error: + pre_audit_dolt_boundary = { + "applicable": profile_name == "dolt", + "verified": False, + "snapshot_fault": _safe_error_identity( + snapshot_error, phase="post_cleanup_dolt_state_capture", + ), + } + audit_recovery = { + "procedure_id": "openstatspec.import-compensation.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "cleanup_attempted": True, + "cleanup_succeeded": False, + "operation_owned_state_targeted": True, + "dolt_failure_boundary": pre_audit_dolt_boundary, + } + cleanup_audit_fault = None + audit_permitted = not ( + pre_audit_dolt_boundary.get("applicable") + and not pre_audit_dolt_boundary.get("verified") + ) + if not audit_permitted: + cleanup_audit_fault = _verification_fault_identity( + "dolt_state_unverified_before_cleanup_failed_audit", + phase="pre_cleanup_failed_audit_boundary", + evidence=pre_audit_dolt_boundary, + ) + dolt_boundary = pre_audit_dolt_boundary + else: + try: + _record_import_cleanup_failure_audit( + engine=engine, operation_id=operation_id, + source_name=source_name, source_format=source_format, + profile_name=profile_name, import_error=import_error, + cleanup_error=cleanup_error, + residual_object_inventory=inventory, + deterministic_recovery_evidence=audit_recovery, + normative=normative, legacy=legacy, + ) + except Exception as audit_error: + cleanup_audit_fault = _safe_error_identity( + audit_error, phase="cleanup_failed_audit", + ) + try: + dolt_boundary = capture_boundary() + except Exception as snapshot_error: + dolt_boundary = { + "applicable": profile_name == "dolt", + "verified": False, + "snapshot_fault": _safe_error_identity( + snapshot_error, + phase="post_cleanup_audit_dolt_state_capture", + ), + } + raise ImportRecoveryError( + "cleanup_failed", + "Import failed and complete compensating cleanup also failed.", + details={ + "subcode": "import_cleanup_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _safe_error_identity( + cleanup_error, phase="compensating_cleanup", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.import-compensation.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "cleanup_attempted": True, + "cleanup_succeeded": False, + "operation_owned_state_targeted": True, + "cleanup_failed_audit_persisted": cleanup_audit_fault is None, + "terminal_reporting": ( + "catalog_and_exception" if cleanup_audit_fault is None + else "out_of_band_exception" + ), + "dolt_failure_boundary": dolt_boundary, + }, + "audit_fault": cleanup_audit_fault, + "success_forbidden": True, + }, + ) from cleanup_error + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=state, normative=normative, + legacy=legacy, + ) + try: + pre_failed_audit_boundary = capture_boundary() + except Exception as snapshot_error: + raise ImportRecoveryError( + "cleanup_failed", + "Import cleanup completed but its pre-audit Dolt boundary could not be captured.", + details={ + "subcode": "pre_failed_audit_dolt_state_capture_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="pre_failed_audit_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "failed_operation_audit_attempted": False, + "terminal_reporting": "out_of_band_exception", + "dolt_failure_boundary": { + "applicable": profile_name == "dolt", + "verified": False, + }, + }, + "audit_fault": None, + "success_forbidden": True, + }, + ) from snapshot_error + if ( + pre_failed_audit_boundary.get("applicable") + and not pre_failed_audit_boundary.get("verified") + ): + raise ImportRecoveryError( + "cleanup_failed", + "Import cleanup completed but its pre-audit Dolt boundary is unverified.", + details={ + "subcode": "pre_failed_audit_dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="pre_failed_audit_dolt_state_verification", + evidence=pre_failed_audit_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "failed_operation_audit_attempted": False, + "terminal_reporting": "out_of_band_exception", + "dolt_failure_boundary": pre_failed_audit_boundary, + }, + "audit_fault": None, + "success_forbidden": True, + }, + ) from import_error + try: + _record_failed_import_audit( + engine=engine, operation_id=operation_id, + source_name=source_name, source_format=source_format, + variable_count=variable_count, profile_name=profile_name, + import_error=import_error, normative=normative, legacy=legacy, + ) + except Exception as audit_error: + raise ImportRecoveryError( + "failure_audit_failed", + "Import cleanup succeeded but its failed-operation audit could not be persisted.", + details={ + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _safe_error_identity( + audit_error, phase="failed_operation_audit", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.failed-import-audit.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + }, + "success_forbidden": True, + }, + ) from audit_error + try: + dolt_boundary = capture_boundary() + except Exception as snapshot_error: + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=state, normative=normative, + legacy=legacy, + ) + raise ImportRecoveryError( + "cleanup_failed", + "Import cleanup completed but Dolt failure-boundary state could not be verified.", + details={ + "subcode": "dolt_state_capture_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="post_audit_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": { + "applicable": profile_name == "dolt", + "verified": False, + }, + }, + "success_forbidden": True, + }, + ) from snapshot_error + if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=state, normative=normative, + legacy=legacy, + ) + raise ImportRecoveryError( + "cleanup_failed", + "Import cleanup did not preserve the Dolt failure-boundary invariants.", + details={ + "subcode": "dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + import_error, phase="import_mutation", + ), + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="post_audit_dolt_state_verification", + evidence=dolt_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": dolt_boundary, + }, + "success_forbidden": True, + }, + ) from import_error + raise + + def create_wide_dataset( *, database_url: str, dataset_id: str, source_name: str, source_format: str, rows: Iterable[Mapping[str, Any]], variables: list[dict[str, Any]], file_label: str = "", @@ -760,9 +2324,12 @@ def create_wide_dataset( source_extensions: Mapping[str, Any] | None = None, fidelity_events: Iterable[Mapping[str, Any]] = (), operation_details: Mapping[str, Any] | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> dict[str, Any]: validate_connection_url(database_url) - profile, _active_connection = effective_profile(database_url) + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) metadata = MetaData() datasets, variable_catalog, fidelity_event_catalog, operation_catalog = catalog(metadata) @@ -770,269 +2337,286 @@ def create_wide_dataset( multiple_response_catalog = multiple_response_set_catalog(metadata) source_extensions_catalog = source_extension_catalog(metadata) documents_catalog, value_labels_catalog, missing_rules_catalog, attributes_catalog = normalized_metadata_tables(metadata) + legacy = ( + datasets, variable_catalog, multiple_response_catalog, source_extensions_catalog, + documents_catalog, value_labels_catalog, missing_rules_catalog, + attributes_catalog, fidelity_event_catalog, operation_catalog, + ) operation_id = str(uuid4()) fidelity_events = tuple(fidelity_events) source_rows = list(rows) - try: - preflight(profile, variables, rows=source_rows) - validate_spss_catalog( - variables, - case_weight_variable=case_weight_variable, - multiple_response_sets=multiple_response_sets, - ) - except Exception as error: - _record_failed_preflight( - engine=engine, metadata=metadata, datasets=datasets, - variable_catalog=variable_catalog, multiple_response_catalog=multiple_response_catalog, - fidelity_event_catalog=fidelity_event_catalog, operation_catalog=operation_catalog, - operation_id=operation_id, source_name=source_name, - source_format=source_format, variable_count=len(variables), - profile_name=profile.name, error=error, normative=normative, - ) - raise data_table = Table( data_table_name(dataset_id), metadata, Column("__case_ordinal", BigInteger, primary_key=True, nullable=False), - *(Column(item["physical_name"], binary64_type() if item["storage_kind"] == "numeric" else string_type(profile), - nullable=item["storage_kind"] == "numeric") for item in variables), + *(Column( + item["physical_name"], + binary64_type() if item["storage_kind"] == "numeric" else Text, + nullable=item["storage_kind"] == "numeric", + ) for item in variables), ) - materialized = [ - {"__case_ordinal": ordinal, **row} - for ordinal, row in enumerate(source_rows, start=1) - ] - normative_dataset_id = str(uuid4()) - namespace_owned = False - data_table_was_absent = False - try: - with engine.begin() as connection: - create_normative_catalog(connection, normative) - namespace_owned = True - metadata.create_all(connection, tables=[ - datasets, variable_catalog, multiple_response_catalog, - source_extensions_catalog, documents_catalog, value_labels_catalog, - missing_rules_catalog, attributes_catalog, fidelity_event_catalog, - operation_catalog, - ]) - _migrate_catalog_columns( - connection, datasets, variable_catalog, multiple_response_catalog, + audit_relations = { + fidelity_event_catalog.name, operation_catalog.name, + normative.fidelity_event.name, normative.operation.name, + } + preflight_state = { + "data_table_creation_attempted": False, + "legacy_dataset_created": False, + "normative_dataset_creation_attempted": False, + "normative_dataset_id": None, + } + with engine.connect() as preflight_connection: + _require_verified_catalog(preflight_connection, normative, legacy) + preflight_dolt_state = _capture_dolt_state( + preflight_connection, profile_name=profile.name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding( + preflight_dolt_state, active, phase="import preflight", + ) + preflight_connection.rollback() + try: + preflight(profile, variables, rows=source_rows) + validate_spss_catalog( + variables, + case_weight_variable=case_weight_variable, + multiple_response_sets=multiple_response_sets, ) - if connection.execute( - select(datasets.c.dataset_id).where( - datasets.c.dataset_id == dataset_id + except Exception as error: + try: + _record_failed_preflight( + engine=engine, metadata=metadata, datasets=datasets, + variable_catalog=variable_catalog, + multiple_response_catalog=multiple_response_catalog, + fidelity_event_catalog=fidelity_event_catalog, + operation_catalog=operation_catalog, operation_id=operation_id, + source_name=source_name, source_format=source_format, + variable_count=len(variables), profile_name=profile.name, + error=error, normative=normative, legacy=legacy, ) - ).first(): - raise ValueError( - f"Dataset {dataset_id!r} already exists; imports never overwrite a dataset." + except Exception as audit_error: + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=preflight_state, + normative=normative, legacy=legacy, ) - if connection.execute( - select(datasets.c.dataset_id).where( - datasets.c.data_table == data_table.name + raise ImportRecoveryError( + "failure_audit_failed", + "Preflight failed and its failed-operation audit could not be persisted.", + details={ + "original_cause": _safe_error_identity( + error, phase="import_preflight", + ), + "cleanup_fault": _safe_error_identity( + audit_error, phase="failed_preflight_audit", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.failed-preflight-audit.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + }, + "success_forbidden": True, + }, + ) from audit_error + try: + post_preflight_dolt_state = _capture_dolt_state( + preflight_connection, profile_name=profile.name, + audit_relations=audit_relations, ) - ).first(): - raise ValueError( - f"Dataset ID {dataset_id!r} collides with an existing physical " - "data-table name; import was not started." + dolt_boundary = _dolt_failure_boundary_evidence( + preflight_dolt_state, post_preflight_dolt_state, ) - if inspect(connection).has_table(data_table.name): - raise ValueError( - f"Physical data-table name {data_table.name!r} is already occupied." + except Exception as snapshot_error: + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=preflight_state, + normative=normative, legacy=legacy, ) - data_table_was_absent = True - record_normative_operation( - connection, normative, operation_id=operation_id, - operation_kind="import", status="started", - source_format=source_format, - ) - connection.execute(insert(operation_catalog).values( - operation_id=operation_id, direction="import", status="running", - dataset_id=dataset_id, source=source_name, created_at=_now(), - details=json.dumps({ - "variable_count": len(variables), - **dict(operation_details or {}), - }, sort_keys=True), - )) - data_table.create(connection) - connection.execute(insert(datasets).values( - dataset_id=dataset_id, data_table=data_table.name, - source_format=source_format, source_name=source_name, - source_encoding=source_encoding, case_count=len(materialized), - source_table_name=source_table_name, source_sha256=source_sha256, - source_created_at=source_created_at, - source_modified_at=source_modified_at, imported_at=imported_at, - file_label=file_label, documents=documents, - file_attributes=file_attributes, - case_weight_variable=case_weight_variable, - multiple_response_sets=multiple_response_sets, - )) - connection.execute( - insert(variable_catalog), - [dict(dataset_id=dataset_id, **item) for item in variables], - ) - docs_rows = document_rows(dataset_id, documents) - if docs_rows: - connection.execute(insert(documents_catalog), docs_rows) - labels_rows = value_label_rows(dataset_id, variables) - if labels_rows: - connection.execute(insert(value_labels_catalog), labels_rows) - missing_rows = missing_rule_rows(dataset_id, variables) - if missing_rows: - connection.execute(insert(missing_rules_catalog), missing_rows) - attributes_rows = attribute_rows( - dataset_id, variables, - file_attributes=file_attribute_values, - variable_attributes=variable_attribute_values, - ) - if attributes_rows: - connection.execute(insert(attributes_catalog), attributes_rows) - mrset_rows = multiple_response_set_rows( - dataset_id, multiple_response_sets, - ) - if mrset_rows: - connection.execute(insert(multiple_response_catalog), mrset_rows) - extension_rows = source_extension_rows( - dataset_id, source_extensions or {}, - ) - if extension_rows: - connection.execute(insert(source_extensions_catalog), extension_rows) - event_rows = _event_rows( - operation_id=operation_id, dataset_id=dataset_id, - direction="import", fidelity_events=fidelity_events, - ) - if event_rows: - connection.execute(insert(fidelity_event_catalog), event_rows) - store_normative_dataset( - connection, normative, dataset_name=dataset_id, - source_format=source_format, physical_table_name=data_table.name, - dataset_label=file_label, source_encoding=source_encoding, - source_hash=source_sha256, source_case_count=len(materialized), - imported_at=imported_at or None, variables=variables, - documents=docs_rows, value_labels=labels_rows, - missing_rules=missing_rows, attributes=attributes_rows, - multiple_response_sets=mrset_rows, - source_extensions=source_extensions or {}, - case_weight_variable=case_weight_variable, - dataset_id=normative_dataset_id, - ) - record_normative_fidelity_events( - connection, normative, operation_id=operation_id, - dataset_id=normative_dataset_id, direction="import", - events=fidelity_events, - ) - if materialized: - connection.execute(insert(data_table), materialized) - connection.execute(update(operation_catalog).where( - operation_catalog.c.operation_id == operation_id - ).values(status="succeeded", completed_at=_now())) - finish_normative_operation( - connection, normative, operation_id=operation_id, - status="succeeded", - ) - except Exception as error: - if ( - profile.name in {"mysql", "mariadb", "dolt"} - and namespace_owned - and data_table_was_absent - ): - try: - with engine.begin() as cleanup: - delete_normative_dataset( - cleanup, normative, normative_dataset_id, - ) - cleanup_inspector = inspect(cleanup) - for table in ( - multiple_response_catalog, source_extensions_catalog, - documents_catalog, value_labels_catalog, - missing_rules_catalog, attributes_catalog, - fidelity_event_catalog, variable_catalog, - ): - if cleanup_inspector.has_table(table.name): - cleanup.execute( - delete(table).where(table.c.dataset_id == dataset_id) - ) - if cleanup_inspector.has_table(datasets.name): - cleanup.execute( - delete(datasets).where(datasets.c.dataset_id == dataset_id) - ) - data_table.drop(cleanup, checkfirst=True) - metadata.create_all(cleanup, tables=[ - datasets, variable_catalog, multiple_response_catalog, - source_extensions_catalog, documents_catalog, - value_labels_catalog, missing_rules_catalog, - attributes_catalog, fidelity_event_catalog, - operation_catalog, - ]) - failed_event = ({ - "code": "import_failed", - "detail": str(error), - "severity": "error", - "source_item": source_name, - "details": { - "profile": profile.name, - "phase": "post_ddl", - "cleanup": "complete", - "error_type": type(error).__name__, + raise ImportRecoveryError( + "cleanup_failed", + "Preflight audit completed but Dolt state could not be verified.", + details={ + "subcode": "dolt_state_capture_failed", + "original_cause": _safe_error_identity( + error, phase="import_preflight", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="post_preflight_audit_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": { + "applicable": profile.name == "dolt", + "verified": False, + }, }, - },) - normative_operation_exists = cleanup.execute(select( - normative.operation.c.operation_id - ).where( - normative.operation.c.operation_id == operation_id - )).first() - if normative_operation_exists: - finish_normative_operation( - cleanup, normative, operation_id=operation_id, - status="failed", - ) - else: - failed_at = datetime.now(UTC).replace(tzinfo=None) - record_normative_operation( - cleanup, normative, operation_id=operation_id, - operation_kind="import", status="failed", - source_format=source_format, started_at=failed_at, - completed_at=failed_at, - ) - mirror_operation_exists = cleanup.execute(select( - operation_catalog.c.operation_id - ).where( - operation_catalog.c.operation_id == operation_id - )).first() - if mirror_operation_exists: - cleanup.execute(update(operation_catalog).where( - operation_catalog.c.operation_id == operation_id - ).values( - status="failed", dataset_id=None, completed_at=_now(), - )) - else: - cleanup.execute(insert(operation_catalog).values( - operation_id=operation_id, direction="import", - status="failed", dataset_id=None, source=source_name, - created_at=_now(), completed_at=_now(), - details=json.dumps({ - "reason": "post_ddl", - "variable_count": len(variables), - }, sort_keys=True), - )) - cleanup.execute( - insert(fidelity_event_catalog), - _event_rows( - operation_id=operation_id, dataset_id=None, - direction="import", fidelity_events=failed_event, + "success_forbidden": True, + }, + ) from snapshot_error + if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): + inventory = _import_residual_inventory( + engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, state=preflight_state, + normative=normative, legacy=legacy, + ) + raise ImportRecoveryError( + "cleanup_failed", + "Preflight audit changed non-audit Dolt state.", + details={ + "subcode": "dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + error, phase="import_preflight", ), - ) - record_normative_fidelity_events( - cleanup, normative, operation_id=operation_id, - dataset_id=None, direction="import", - events=failed_event, - ) - except Exception as cleanup_error: - raise RuntimeError( - f"OpenStatSpec compensating cleanup failed: {cleanup_error}" - ) from cleanup_error - raise - return { - "dataset_id": dataset_id, "data_table": data_table.name, - "case_count": len(materialized), "operation_id": operation_id, - } + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="post_preflight_audit_dolt_state_verification", + evidence=dolt_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": operation_id, + "targets": { + "dataset_id": dataset_id, + "physical_table": data_table.name, + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": dolt_boundary, + }, + "success_forbidden": True, + }, + ) from error + raise + with engine.connect() as mutation_connection: + pre_dolt_state = _capture_dolt_state( + mutation_connection, profile_name=profile.name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding( + pre_dolt_state, active, phase="import mutation preflight", + ) + mutation_connection.rollback() + with _import_cleanup_guard( + engine=engine, dataset_id=dataset_id, operation_id=operation_id, + data_table=data_table, source_name=source_name, + source_format=source_format, variable_count=len(variables), + profile_name=profile.name, normative=normative, legacy=legacy, + snapshot_connection=mutation_connection, + pre_dolt_state=pre_dolt_state, + ) as mutation: + with mutation_connection.begin(): + connection = mutation_connection + _require_verified_catalog(connection, normative, legacy) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="import", status="started", source_format=source_format, + ) + connection.execute(insert(operation_catalog).values( + operation_id=operation_id, direction="import", status="running", dataset_id=dataset_id, + source=source_name, created_at=_now(), details=json.dumps({"variable_count": len(variables), **dict(operation_details or {})}, sort_keys=True), + )) + if connection.execute(select(datasets.c.dataset_id).where(datasets.c.dataset_id == dataset_id)).first(): + raise ValueError(f"Dataset {dataset_id!r} already exists; imports never overwrite a dataset.") + if connection.execute(select(datasets.c.dataset_id).where(datasets.c.data_table == data_table.name)).first(): + raise ValueError(f"Dataset ID {dataset_id!r} collides with an existing physical data-table name; import was not started.") + mutation["data_table_creation_attempted"] = True + data_table.create(connection) + materialized = [ + {"__case_ordinal": ordinal, **row} + for ordinal, row in enumerate(source_rows, start=1) + ] + connection.execute(insert(datasets).values( + dataset_id=dataset_id, data_table=data_table.name, source_format=source_format, + source_name=source_name, source_encoding=source_encoding, case_count=len(materialized), + source_table_name=source_table_name, + source_sha256=source_sha256, + source_created_at=source_created_at, source_modified_at=source_modified_at, + imported_at=imported_at, + file_label=file_label, documents=documents, + file_attributes=file_attributes, case_weight_variable=case_weight_variable, + multiple_response_sets=multiple_response_sets, + )) + mutation["legacy_dataset_created"] = True + connection.execute(insert(variable_catalog), [dict(dataset_id=dataset_id, **item) for item in variables]) + docs_rows = document_rows(dataset_id, documents) + if docs_rows: + connection.execute(insert(documents_catalog), docs_rows) + labels_rows = value_label_rows(dataset_id, variables) + if labels_rows: + connection.execute(insert(value_labels_catalog), labels_rows) + missing_rows = missing_rule_rows(dataset_id, variables) + if missing_rows: + connection.execute(insert(missing_rules_catalog), missing_rows) + attributes_rows = attribute_rows( + dataset_id, variables, + file_attributes=file_attribute_values, + variable_attributes=variable_attribute_values, + ) + if attributes_rows: + connection.execute(insert(attributes_catalog), attributes_rows) + mrset_rows = multiple_response_set_rows(dataset_id, multiple_response_sets) + if mrset_rows: + connection.execute(insert(multiple_response_catalog), mrset_rows) + extension_rows = source_extension_rows(dataset_id, source_extensions or {}) + if extension_rows: + connection.execute(insert(source_extensions_catalog), extension_rows) + event_rows = _event_rows( + operation_id=operation_id, dataset_id=dataset_id, direction="import", fidelity_events=fidelity_events, + ) + if event_rows: + connection.execute(insert(fidelity_event_catalog), event_rows) + mutation["normative_dataset_creation_attempted"] = True + normative_dataset_id = store_normative_dataset( + connection, normative, dataset_name=dataset_id, + source_format=source_format, physical_table_name=data_table.name, + dataset_label=file_label, source_encoding=source_encoding, + source_hash=source_sha256, source_case_count=len(materialized), + imported_at=imported_at or None, variables=variables, + documents=docs_rows, value_labels=labels_rows, + missing_rules=missing_rows, attributes=attributes_rows, + multiple_response_sets=mrset_rows, + source_extensions=source_extensions or {}, + case_weight_variable=case_weight_variable, + ) + mutation["normative_dataset_id"] = normative_dataset_id + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, + dataset_id=normative_dataset_id, direction="import", + events=fidelity_events, + ) + if materialized: + for batch in _bounded_batches( + materialized, variables, profile.max_statement_bytes, + ): + connection.execute(insert(data_table), batch) + connection.execute(update(operation_catalog).where(operation_catalog.c.operation_id == operation_id).values( + status="succeeded", completed_at=_now(), + )) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="succeeded", + ) + post_dolt_state = _capture_dolt_state( + mutation_connection, profile_name=profile.name, + audit_relations=audit_relations, + ) + _require_dolt_working_set_binding( + post_dolt_state, active, phase="import completion", + ) + _require_dolt_success_identity( + pre_dolt_state, post_dolt_state, phase="import", + ) + return {"dataset_id": dataset_id, "data_table": data_table.name, "case_count": len(materialized), "operation_id": operation_id} def _endpoint_from_row(row: Mapping[str, Any], *, prefix: str) -> Any: @@ -1045,20 +2629,29 @@ def _endpoint_from_row(row: Mapping[str, Any], *, prefix: str) -> Any: def read_wide_dataset( - *, database_url: str, dataset_id: str, profile: Any | None = None, + *, + database_url: str, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]: - """Read a strict dataset only after resolving the active server profile.""" - if profile is None: - profile, _active = effective_profile(database_url) + """Read a strict dataset, preferring normalized metadata with JSON compatibility fallback.""" + profile, _active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) metadata = MetaData() - datasets, variable_catalog, _, _ = catalog(metadata) + datasets, variable_catalog, fidelity_catalog, operation_catalog = catalog(metadata) + normative = normative_catalog(metadata) multiple_response_catalog = multiple_response_set_catalog(metadata) source_extensions_catalog = source_extension_catalog(metadata) documents_catalog, value_labels_catalog, missing_rules_catalog, attributes_catalog = normalized_metadata_tables(metadata) - with engine.begin() as connection: - metadata.create_all(connection, tables=[datasets, variable_catalog, multiple_response_catalog, source_extensions_catalog, documents_catalog, value_labels_catalog, missing_rules_catalog, attributes_catalog]) - _migrate_catalog_columns(connection, datasets, variable_catalog, multiple_response_catalog) + legacy = ( + datasets, variable_catalog, multiple_response_catalog, source_extensions_catalog, + documents_catalog, value_labels_catalog, missing_rules_catalog, + attributes_catalog, fidelity_catalog, operation_catalog, + ) + with engine.connect() as connection: + _require_verified_catalog(connection, normative, legacy) dataset = dict(connection.execute(select(datasets).where(datasets.c.dataset_id == dataset_id)).mappings().one()) data_table = Table(dataset["data_table"], MetaData(), autoload_with=connection) variables = [dict(item) for item in connection.execute( @@ -1130,16 +2723,26 @@ def read_wide_dataset( rules_by_variable.setdefault(item["variable_ordinal"], []).append({"lo": lower, "hi": upper}) for ordinal, rules in rules_by_variable.items(): variables_by_ordinal[ordinal]["missing_ranges"] = json.dumps(rules, ensure_ascii=False) + preflight(profile, variables, rows=rows) return dataset, variables, rows -def read_fidelity_events(*, database_url: str, dataset_id: str) -> tuple[dict[str, Any], ...]: +def read_fidelity_events( + *, + database_url: str, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> tuple[dict[str, Any], ...]: """Read import-time fidelity diagnostics for a catalogued dataset.""" + effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) metadata = MetaData() - _, _, fidelity_event_catalog, _ = catalog(metadata) + legacy, normative = _catalog_layout(metadata) + fidelity_event_catalog = legacy[8] with engine.connect() as connection: - fidelity_event_catalog.create(connection, checkfirst=True) + _require_verified_catalog(connection, normative, legacy) events = connection.execute( select(fidelity_event_catalog) .where(fidelity_event_catalog.c.dataset_id == dataset_id) @@ -1152,32 +2755,155 @@ def read_fidelity_events(*, database_url: str, dataset_id: str) -> tuple[dict[st +def record_export_cleanup_failure( + *, database_url: str, destination: str, original_error: Exception, + cleanup_error: Exception, + residual_object_inventory: Mapping[str, Any], + deterministic_recovery_evidence: Mapping[str, Any], + operation_id: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> str: + """Best-effort immutable export cleanup-failure audit.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + fidelity_events, operations = legacy[8:] + requested_operation_id = operation_id + operation_id = operation_id or str(uuid4()) + original = _safe_error_identity(original_error, phase="export") + cleanup = _safe_error_identity(cleanup_error, phase="export_destination_restore") + event_details = { + "original_cause": original, + "cleanup_fault": cleanup, + "residual_object_inventory": dict(residual_object_inventory), + "deterministic_recovery_evidence": dict( + deterministic_recovery_evidence + ), + } + event = { + "code": "cleanup_failed", + "detail": "Export destination recovery failed; out-of-band review is required.", + "severity": "error", + "source_item": destination, + "details": event_details, + } + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="record export cleanup failure", + ) as connection: + _require_verified_catalog(connection, normative, legacy) + if requested_operation_id is not None: + existing = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one_or_none() + normative_existing = connection.execute(select(normative.operation).where( + normative.operation.c.operation_id == operation_id + )).mappings().one_or_none() + if ( + existing is None or normative_existing is None + or existing["direction"] != "export" + or existing["status"] != "running" + or normative_existing["status"] != "started" + ): + raise UnsupportedOperationError( + "Existing export operation is not in an auditable terminal-transition state." + ) + details = json.loads(existing["details"] or "{}") + details["cleanup_failure"] = event_details + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values( + status="failed", completed_at=_now(), + details=json.dumps(details, sort_keys=True), + )) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="failed", + ) + ordinals = connection.execute(select(fidelity_events.c.ordinal).where( + fidelity_events.c.operation_id == operation_id + )).scalars().all() + event_row = _event_rows( + operation_id=operation_id, dataset_id=None, direction="export", + fidelity_events=(event,), + )[0] + event_row["ordinal"] = max(ordinals, default=0) + 1 + connection.execute(insert(fidelity_events).values(**event_row)) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="export", events=(event,), + ) + else: + failed_at = datetime.now(UTC).replace(tzinfo=None) + record_normative_operation( + connection, normative, operation_id=operation_id, + operation_kind="export", status="failed", source_format=None, + started_at=failed_at, completed_at=failed_at, + ) + connection.execute(insert(operations).values( + operation_id=operation_id, direction="export", status="failed", + dataset_id=None, destination=destination, created_at=_now(), + completed_at=_now(), details=json.dumps({ + "reason": "cleanup_failed", **event_details, + }, sort_keys=True), + )) + connection.execute(insert(fidelity_events), _event_rows( + operation_id=operation_id, dataset_id=None, direction="export", + fidelity_events=(event,), + )) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, dataset_id=None, + direction="export", events=(event,), + ) + return operation_id + + def record_export_operation( *, database_url: str, dataset_id: str, destination: str, allowed_fidelity_events: Iterable[Mapping[str, Any]], operation_details: Mapping[str, Any] | None = None, + terminal: bool = True, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> str: """Persist a completed export and the fidelity loss explicitly accepted by its caller.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) metadata = MetaData() - datasets, variables, fidelity_events, operations = catalog(metadata) - normative = normative_catalog(metadata) - multiple_response = multiple_response_set_catalog(metadata) + legacy, normative = _catalog_layout(metadata) + datasets, variables, multiple_response = legacy[:3] + fidelity_events, operations = legacy[8:] operation_id = str(uuid4()) events = tuple(allowed_fidelity_events) - with engine.begin() as connection: - metadata.create_all(connection, tables=[datasets, variables, multiple_response, fidelity_events, operations]) - create_normative_catalog(connection, normative) + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="record export operation", + ) as connection: + _require_verified_catalog(connection, normative, legacy) normative_dataset_id = normative_dataset_id_for_name(connection, normative, dataset_id) completed_at = datetime.now(UTC).replace(tzinfo=None) + normative_status = "succeeded" if terminal else "started" + legacy_status = "succeeded" if terminal else "running" record_normative_operation( connection, normative, operation_id=operation_id, - operation_kind="export", status="succeeded", source_format=None, - started_at=completed_at, completed_at=completed_at, + operation_kind="export", status=normative_status, source_format=None, + started_at=completed_at, completed_at=completed_at if terminal else None, ) connection.execute(insert(operations).values( - operation_id=operation_id, direction="export", status="succeeded", dataset_id=dataset_id, - destination=destination, created_at=_now(), completed_at=_now(), + operation_id=operation_id, direction="export", status=legacy_status, dataset_id=dataset_id, + destination=destination, created_at=_now(), + completed_at=_now() if terminal else None, details=json.dumps({"allow_loss": [event["code"] for event in events], **dict(operation_details or {})}, sort_keys=True), )) rows = _event_rows( @@ -1197,12 +2923,268 @@ def record_export_operation( ) return operation_id -def validate_wide_dataset(*, database_url: str, dataset_id: str) -> dict[str, Any]: - profile, _active = effective_profile(database_url) + +def finish_export_operation( + *, + database_url: str, + operation_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + """Mark a published export successful only after filesystem finalization.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + operations = legacy[9] + audit_relations = { + legacy[8].name, legacy[9].name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="finish export operation", + ) as connection: + _require_verified_catalog(connection, normative, legacy) + row = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one() + if row["direction"] != "export" or row["status"] != "running": + raise UnsupportedOperationError( + "Only a running export operation can be finalized." + ) + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values(status="succeeded", completed_at=_now())) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="succeeded", + ) + + +def read_export_operation_state( + *, + database_url: str, + operation_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: + """Read both export-operation catalogs without changing either one.""" + validate_connection_url(database_url) + effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + operations = legacy[9] + with engine.connect() as connection: + _require_verified_catalog(connection, normative, legacy) + legacy_row = connection.execute(select( + operations.c.direction, + operations.c.status, + ).where( + operations.c.operation_id == operation_id + )).mappings().one_or_none() + normative_row = connection.execute(select( + normative.operation.c.operation_kind, + normative.operation.c.status, + ).where( + normative.operation.c.operation_id == operation_id + )).mappings().one_or_none() + + legacy_state = ( + None if legacy_row is None else { + "direction": legacy_row["direction"], + "status": legacy_row["status"], + } + ) + normative_state = ( + None if normative_row is None else { + "operation_kind": normative_row["operation_kind"], + "status": normative_row["status"], + } + ) + if ( + legacy_state == {"direction": "export", "status": "succeeded"} + and normative_state == { + "operation_kind": "export", "status": "succeeded", + } + ): + classification = "succeeded" + elif ( + legacy_state == {"direction": "export", "status": "running"} + and normative_state == { + "operation_kind": "export", "status": "started", + } + ): + classification = "running" + else: + classification = "ambiguous" + return { + "operation_id": operation_id, + "legacy": legacy_state, + "normative": normative_state, + "classification": classification, + } + + +def fail_export_operation( + *, + database_url: str, + operation_id: str, + failure_details: Mapping[str, Any], + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + """Close one running export after filesystem compensation succeeded.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + fidelity_events, operations = legacy[8:] + audit_relations = { + fidelity_events.name, operations.name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="fail export operation", + ) as connection: + _require_verified_catalog(connection, normative, legacy) + row = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one() + normative_row = connection.execute(select(normative.operation).where( + normative.operation.c.operation_id == operation_id + )).mappings().one() + if ( + row["direction"] != "export" or row["status"] != "running" + or normative_row["operation_kind"] != "export" + or normative_row["status"] != "started" + ): + raise UnsupportedOperationError( + "Only matching running export operations can be failed." + ) + details = json.loads(row["details"] or "{}") + details["failure"] = dict(failure_details) + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values( + status="failed", completed_at=_now(), + details=json.dumps(details, sort_keys=True), + )) + finish_normative_operation( + connection, normative, operation_id=operation_id, status="failed", + ) + event = { + "code": "export_failed", + "detail": "Export publication or finalization failed after audit start.", + "severity": "error", + "source_item": row["destination"], + "details": dict(failure_details), + } + ordinals = connection.execute(select(fidelity_events.c.ordinal).where( + fidelity_events.c.operation_id == operation_id + )).scalars().all() + event_row = _event_rows( + operation_id=operation_id, dataset_id=row["dataset_id"], + direction="export", fidelity_events=(event,), + )[0] + event_row["ordinal"] = max(ordinals, default=0) + 1 + connection.execute(insert(fidelity_events).values(**event_row)) + normative_dataset_id = normative_dataset_id_for_name( + connection, normative, row["dataset_id"], + ) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, + dataset_id=normative_dataset_id, direction="export", events=(event,), + ) + + +def record_export_backup_retained( + *, database_url: str, operation_id: str, destination: str, backup: str, + cleanup_error: Exception, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: + """Append a warning without rewriting a successfully finalized export.""" + profile, active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) + engine = create_engine(database_url) + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + fidelity_events, operations = legacy[8:] + audit_relations = { + fidelity_events.name, operations.name, + normative.fidelity_event.name, normative.operation.name, + } + with _bound_catalog_transaction( + engine=engine, profile_name=profile.name, active=active, + audit_relations=audit_relations, phase="record retained export backup", + ) as connection: + _require_verified_catalog(connection, normative, legacy) + row = connection.execute(select(operations).where( + operations.c.operation_id == operation_id + )).mappings().one() + normative_row = connection.execute(select(normative.operation).where( + normative.operation.c.operation_id == operation_id + )).mappings().one() + if ( + row["direction"] != "export" or row["status"] != "succeeded" + or normative_row["operation_kind"] != "export" + or normative_row["status"] != "succeeded" + ): + raise UnsupportedOperationError( + "A retained backup warning requires a matching succeeded export." + ) + details = json.loads(row["details"] or "{}") + details["backup_retained"] = { + "destination": destination, "durable_backup": backup, + "cleanup_error_type": type(cleanup_error).__name__, + } + connection.execute(update(operations).where( + operations.c.operation_id == operation_id + ).values(details=json.dumps(details, sort_keys=True))) + event = { + "code": "backup_retained", + "detail": "A successful export retained its durable prior-file backup.", + "severity": "warning", + "source_item": destination, + "details": { + "durable_backup": backup, + "cleanup_error_type": type(cleanup_error).__name__, + }, + } + ordinals = connection.execute(select(fidelity_events.c.ordinal).where( + fidelity_events.c.operation_id == operation_id + )).scalars().all() + event_row = _event_rows( + operation_id=operation_id, dataset_id=row["dataset_id"], + direction="export", fidelity_events=(event,), + )[0] + event_row["ordinal"] = max(ordinals, default=0) + 1 + connection.execute(insert(fidelity_events).values(**event_row)) + normative_dataset_id = normative_dataset_id_for_name( + connection, normative, row["dataset_id"], + ) + record_normative_fidelity_events( + connection, normative, operation_id=operation_id, + dataset_id=normative_dataset_id, direction="export", events=(event,), + ) + + +def validate_wide_dataset( + *, + database_url: str, + dataset_id: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> dict[str, Any]: dataset, variables, rows = read_wide_dataset( - database_url=database_url, dataset_id=dataset_id, profile=profile, + database_url=database_url, + dataset_id=dataset_id, + dolt_conformance_source=dolt_conformance_source, ) - preflight(profile, variables, rows=rows) validate_spss_catalog( variables, case_weight_variable=dataset.get("case_weight_variable"), @@ -1228,11 +3210,6 @@ def validate_wide_dataset(*, database_url: str, dataset_id: str) -> dict[str, An if item["storage_kind"] == "numeric": if not isinstance(column.type, Float) or not column.nullable: raise ValueError(f"Numeric variable {item['source_name']!r} must be a nullable binary64 column.") - elif profile.name == "dolt": - if not isinstance(column.type, mysql.LONGTEXT) or column.nullable: - raise ValueError( - f"String variable {item['source_name']!r} must be a non-null LONGTEXT column." - ) elif not isinstance(column.type, Text) or column.nullable: raise ValueError(f"String variable {item['source_name']!r} must be a non-null text column.") if [row["__case_ordinal"] for row in rows] != list(range(1, len(rows) + 1)): diff --git a/src/openstatspec/sql/workflow.py b/src/openstatspec/sql/workflow.py index 5b2fd97..8d96702 100644 --- a/src/openstatspec/sql/workflow.py +++ b/src/openstatspec/sql/workflow.py @@ -32,10 +32,11 @@ from ..core import UnsupportedOperationError from .capabilities import SPECIFICATION_COMMIT from .normative import ( - CATALOG_CONTRACT_ID, CATALOG_SCHEMA_VERSION, catalog as core_catalog, + CATALOG_CONTRACT_ID, CATALOG_SCHEMA_VERSION, binary64_type, + catalog as core_catalog, ) from .profiles import preflight, validate_connection_url -from .wide import binary64_type, physical_name +from .wide import physical_name PROFILE_ID = "openstatspec-sql-transformation-workflow-v0.1" PROFILE_SCHEMA_VERSION = 2 diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py new file mode 100644 index 0000000..f1b985c --- /dev/null +++ b/tests/test_catalog_lifecycle.py @@ -0,0 +1,336 @@ +import sqlite3 + +import pytest +from sqlalchemy import Table + +import openstatspec +import openstatspec.sql.wide as wide +from openstatspec.core import UnsupportedOperationError +from openstatspec.sql.wide import ( + ImportRecoveryError, + _bounded_batches, + create_wide_dataset, + read_wide_dataset, + record_export_operation, + validate_wide_dataset, +) + + +def _variables(): + return [{ + "ordinal": 1, + "source_name": "name", + "physical_name": "name", + "storage_kind": "string", + "string_width": 8, + "label": "", + "format": "A8", + "measure": "nominal", + "alignment": "left", + "display_width": 8, + "value_labels": "{}", + "missing_ranges": "[]", + }] + + +def _table_names(path): + connection = sqlite3.connect(path) + try: + return { + row[0] for row in connection.execute( + "select name from sqlite_master where type = 'table'" + ) + } + finally: + connection.close() + + +def test_import_requires_explicit_catalog_and_creates_no_relations(tmp_path): + path = tmp_path / "absent.sqlite" + database = f"sqlite:///{path}" + + with pytest.raises(UnsupportedOperationError, match="catalog is absent"): + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + assert _table_names(path) == set() + + +@pytest.mark.parametrize( + "operation", + [ + lambda database: read_wide_dataset(database_url=database, dataset_id="missing"), + lambda database: validate_wide_dataset(database_url=database, dataset_id="missing"), + lambda database: record_export_operation( + database_url=database, + dataset_id="missing", + destination="output.sav", + allowed_fidelity_events=(), + ), + ], +) +def test_read_validate_and_export_require_catalog_without_mutation(tmp_path, operation): + path = tmp_path / "absent.sqlite" + database = f"sqlite:///{path}" + + with pytest.raises(UnsupportedOperationError, match="catalog is absent"): + operation(database) + + assert _table_names(path) == set() + + +@pytest.mark.parametrize( + "foreign_sql", + [ + "create table foreign_relation (value integer)", + "create table view_source (value integer); create view foreign_view as select value from view_source", + ], +) +def test_initializer_rejects_foreign_tables_and_views_without_modification( + tmp_path, foreign_sql, +): + path = tmp_path / "foreign.sqlite" + connection = sqlite3.connect(path) + connection.executescript(foreign_sql) + before = connection.execute( + "select type, name, sql from sqlite_master " + "where name not like 'sqlite_%' order by type, name" + ).fetchall() + connection.close() + + with pytest.raises(UnsupportedOperationError, match="catalog is foreign"): + openstatspec.initialize_catalog(database_url=f"sqlite:///{path}") + + connection = sqlite3.connect(path) + after = connection.execute( + "select type, name, sql from sqlite_master " + "where name not like 'sqlite_%' order by type, name" + ).fetchall() + connection.close() + assert after == before + + +def test_initializer_compensates_partial_catalog_install(tmp_path, monkeypatch): + path = tmp_path / "partial.sqlite" + + def fail_migration(*_args, **_kwargs): + raise RuntimeError("injected catalog migration failure") + + monkeypatch.setattr(wide, "_migrate_catalog_columns", fail_migration) + with pytest.raises(RuntimeError, match="injected catalog migration failure"): + openstatspec.initialize_catalog(database_url=f"sqlite:///{path}") + + assert _table_names(path) == set() + + +def test_post_ddl_failure_removes_dataset_state_and_persists_null_dataset_audit( + tmp_path, monkeypatch, +): + path = tmp_path / "runtime.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + original = wide.store_normative_dataset + + def store_then_fail(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected normative failure") + + monkeypatch.setattr(wide, "store_normative_dataset", store_then_fail) + with pytest.raises(RuntimeError, match="injected normative failure"): + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + assert "data_sample" not in _table_names(path) + connection = sqlite3.connect(path) + assert connection.execute("select count(*) from dataset_catalog").fetchone() == (0,) + assert connection.execute("select count(*) from dataset").fetchone() == (0,) + assert connection.execute( + "select status, dataset_id from operation_catalog" + ).fetchall() == [("failed", None)] + assert connection.execute( + "select code, dataset_id from fidelity_event_catalog" + ).fetchall() == [("import_failed", None)] + assert connection.execute( + "select status from operation" + ).fetchall() == [("failed",)] + assert connection.execute( + "select event_code, dataset_id from fidelity_event" + ).fetchall() == [("import_failed", None)] + connection.close() + + +def test_cleanup_failure_has_machine_readable_error(tmp_path, monkeypatch): + path = tmp_path / "cleanup.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + + def fail_cleanup(*_args, **_kwargs): + raise RuntimeError("injected cleanup failure") + + def fail_mutation(*_args, **_kwargs): + raise RuntimeError("injected mutation failure") + + monkeypatch.setattr(wide, "_cleanup_import_state", fail_cleanup) + monkeypatch.setattr(wide, "store_normative_dataset", fail_mutation) + with pytest.raises(ImportRecoveryError) as error: + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + + assert error.value.code == "cleanup_failed" + assert error.value.details["original_cause"]["type"] == "RuntimeError" + assert error.value.details["cleanup_fault"]["type"] == "RuntimeError" + assert error.value.details["success_forbidden"] is True + evidence = error.value.details["deterministic_recovery_evidence"] + assert evidence["procedure_id"] == "openstatspec.import-compensation.v1" + assert evidence["cleanup_attempted"] is True + assert evidence["cleanup_succeeded"] is False + assert evidence["operation_owned_state_targeted"] is True + assert evidence["cleanup_failed_audit_persisted"] is True + assert evidence["terminal_reporting"] == "catalog_and_exception" + assert len(evidence["residual_inventory_sha256"]) == 64 + connection = sqlite3.connect(path) + assert connection.execute( + "select status, dataset_id from operation_catalog" + ).fetchall() == [("failed", None)] + assert connection.execute( + "select code, dataset_id from fidelity_event_catalog" + ).fetchall() == [("cleanup_failed", None)] + connection.close() + + +def test_bounded_batches_never_exceed_statement_payload_limit(): + variables = _variables() + rows = [ + {"name": "aaaa"}, + {"name": "bbbb"}, + {"name": "cccc"}, + ] + single = wide.statement_payload_bytes(rows[0], variables) + + batches = list(_bounded_batches(rows, variables, single * 2)) + + assert batches == [rows[:2], rows[2:]] + + +def test_duplicate_import_failure_preserves_existing_dataset(tmp_path): + path = tmp_path / "duplicate.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="first.sav", + source_format="SAV", + rows=[{"name": "first"}], + variables=_variables(), + ) + + with pytest.raises(ValueError, match="already exists"): + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="second.sav", + source_format="SAV", + rows=[{"name": "second"}], + variables=_variables(), + ) + + connection = sqlite3.connect(path) + assert connection.execute( + "select dataset_id, case_count from dataset_catalog" + ).fetchall() == [("sample", 1)] + assert connection.execute( + "select dataset_name, source_case_count from dataset" + ).fetchall() == [("sample", 1)] + assert connection.execute( + "select name from data_sample" + ).fetchall() == [("first",)] + connection.close() + + + +def test_missing_additive_column_requires_explicit_migration_without_audit_mutation( + tmp_path, +): + path = tmp_path / "migration-required.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + connection = sqlite3.connect(path) + connection.execute("alter table variable_catalog drop column compat_name") + connection.commit() + connection.close() + + with pytest.raises(UnsupportedOperationError, match="catalog is migration_required"): + create_wide_dataset( + database_url=database, dataset_id="sample", + source_name="fixture.sav", source_format="SAV", + rows=[{"name": "ok"}], variables=_variables(), + ) + + connection = sqlite3.connect(path) + assert "compat_name" not in { + row[1] for row in connection.execute("pragma table_info(variable_catalog)") + } + assert connection.execute("select count(*) from operation_catalog").fetchone() == (0,) + connection.close() + + result = openstatspec.initialize_catalog(database_url=database) + assert result["catalog"] == "verified" + connection = sqlite3.connect(path) + assert "compat_name" in { + row[1] for row in connection.execute("pragma table_info(variable_catalog)") + } + connection.close() + + +def test_multiple_catalog_identities_are_ambiguous_and_never_mutated(tmp_path): + path = tmp_path / "ambiguous.sqlite" + connection = sqlite3.connect(path) + connection.executescript( + "create table catalog_identity (" + "catalog_identity_key integer primary key, " + "contract_id varchar(128) not null unique, " + "schema_version integer not null, created_at datetime not null);" + "insert into catalog_identity values " + "(1, 'openstatspec-strict-wide-table-v1', 1, '2026-01-01')," + "(2, 'conflicting-contract', 1, '2026-01-01');" + ) + before = connection.execute( + "select * from catalog_identity order by catalog_identity_key" + ).fetchall() + connection.close() + + database = f"sqlite:///{path}" + with pytest.raises(UnsupportedOperationError, match="catalog is ambiguous"): + create_wide_dataset( + database_url=database, dataset_id="sample", + source_name="fixture.sav", source_format="SAV", + rows=[{"name": "ok"}], variables=_variables(), + ) + with pytest.raises(UnsupportedOperationError, match="catalog is ambiguous"): + openstatspec.initialize_catalog(database_url=database) + + connection = sqlite3.connect(path) + assert connection.execute( + "select * from catalog_identity order by catalog_identity_key" + ).fetchall() == before + assert _table_names(path) == {"catalog_identity"} + connection.close() diff --git a/tests/test_dolt_conformance.py b/tests/test_dolt_conformance.py new file mode 100644 index 0000000..77a9062 --- /dev/null +++ b/tests/test_dolt_conformance.py @@ -0,0 +1,311 @@ +from __future__ import annotations + +import inspect +import tomllib +from pathlib import Path + +import pandas as pd +import pytest + +import openstatspec.api as api_module +import openstatspec.spss as spss_module +import openstatspec.spss.sav as sav_module +from openstatspec.core import UnsupportedOperationError +import openstatspec.sql.capabilities as capability_module +import openstatspec.sql.wide as wide +from openstatspec.sql.dolt_conformance import ( + ADAPTER_IMPLEMENTATION_ID, + ADAPTER_VERSION, + DoltConformanceSource, +) + + +def test_packaged_source_is_fail_closed_without_concrete_declarations() -> None: + status = DoltConformanceSource.packaged().status() + assert status["write_enabled"] is False + assert status["declaration_count"] == 0 + + +def test_directory_source_is_explicit_and_invalid_root_fails_closed( + tmp_path: Path, +) -> None: + source = DoltConformanceSource.from_directory(tmp_path) + assert source.directory == tmp_path + status = source.status() + assert status["write_enabled"] is False + assert status["status"] == "blocked_invalid_or_unavailable_declaration_source" + + +def test_packaged_companion_missing_is_reported_without_opening_write_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def missing_companion() -> tuple[object, object, object, object]: + raise UnsupportedOperationError("companion missing") + + monkeypatch.setattr( + "openstatspec.sql.dolt_conformance._shared_api", missing_companion, + ) + declaration = capability_module.profile_declarations()["dolt"] + assert declaration["operational_write_enabled"] is False + assert declaration["write_conformance"]["write_enabled"] is False + assert declaration["claimed_server_versions"] == [] + assert declaration["ci_tested_server_versions"] == [] + + +def test_exact_match_binds_active_product_adapter_and_specification( + monkeypatch: pytest.MonkeyPatch, +) -> None: + matching = { + "declaration_id": "dolt-2.2.2-python-0.1.0", + "active_product_version": "2.2.2", + "adapter_implementation_id": ADAPTER_IMPLEMENTATION_ID, + "adapter_version": ADAPTER_VERSION, + "specification_commit": "a" * 40, + } + monkeypatch.setattr( + DoltConformanceSource, + "validated_declarations", + lambda self: (matching,), + ) + selected = DoltConformanceSource.packaged().require_exact_match( + active_product_version="2.2.2", + specification_commit="a" * 40, + ) + assert selected["declaration_id"] == matching["declaration_id"] + + +def test_exact_match_rejects_multiple_adapter_bindings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = { + "active_product_version": "2.2.2", + "adapter_implementation_id": ADAPTER_IMPLEMENTATION_ID, + "adapter_version": ADAPTER_VERSION, + "specification_commit": "a" * 40, + } + monkeypatch.setattr( + DoltConformanceSource, + "validated_declarations", + lambda self: ( + {**binding, "declaration_id": "one"}, + {**binding, "declaration_id": "two"}, + ), + ) + with pytest.raises(UnsupportedOperationError, match="unique exact"): + DoltConformanceSource.packaged().require_exact_match( + active_product_version="2.2.2", + specification_commit="a" * 40, + ) + + +def test_effective_profile_requires_bound_spec_commit_before_mutation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + + class UnexpectedSource: + def require_exact_match( + self, *, active_product_version: str, specification_commit: str, + ) -> None: + calls.append((active_product_version, specification_commit)) + + monkeypatch.setattr( + capability_module, + "active_connection", + lambda database_url, **kwargs: { + "profile": "dolt", + "raw_product_version": "2.2.2", + "server_version": "2.2.2", + "claimed_supported": False, + "observed": {"max_allowed_packet": 1_000_000}, + }, + ) + assert capability_module.SPECIFICATION_COMMIT is None + with pytest.raises(UnsupportedOperationError, match="not bound"): + capability_module.effective_profile( + "mysql+pymysql://example.invalid/catalog", + dolt_conformance_source=UnexpectedSource(), + ) + assert calls == [] + + +def test_every_sql_mutation_entrypoint_accepts_explicit_conformance_source() -> None: + mutation_entrypoints = ( + wide.initialize_wide_catalog, + wide.create_wide_dataset, + wide.record_export_cleanup_failure, + wide.record_export_operation, + wide.finish_export_operation, + wide.fail_export_operation, + wide.record_export_backup_retained, + ) + for entrypoint in mutation_entrypoints: + parameter = inspect.signature(entrypoint).parameters[ + "dolt_conformance_source" + ] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY + assert parameter.default is None + + +def test_adapter_binding_version_matches_distribution_metadata() -> None: + pyproject = tomllib.loads( + (Path(__file__).resolve().parents[1] / "pyproject.toml").read_text( + encoding="utf-8" + ) + ) + assert pyproject["project"]["version"] == ADAPTER_VERSION + + +def test_validate_wide_dataset_propagates_explicit_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + calls: list[object] = [] + + def stop_after_read_preflight(**kwargs: object) -> tuple[object, object, object]: + calls.append(kwargs["dolt_conformance_source"]) + raise UnsupportedOperationError("stop after propagation check") + + monkeypatch.setattr(wide, "read_wide_dataset", stop_after_read_preflight) + with pytest.raises(UnsupportedOperationError, match="propagation check"): + wide.validate_wide_dataset( + database_url="mysql+pymysql://example.invalid/catalog", + dataset_id="synthetic", + dolt_conformance_source=sentinel, + ) + assert calls == [sentinel] + + +def test_public_and_spss_dispatch_propagate_explicit_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + calls: list[tuple[str, object]] = [] + + def capture_api_import(*args: object, **kwargs: object) -> dict[str, bool]: + calls.append(("api_import", kwargs["dolt_conformance_source"])) + return {"ok": True} + + def capture_api_export(**kwargs: object) -> dict[str, bool]: + calls.append(("api_export", kwargs["dolt_conformance_source"])) + return {"ok": True} + + monkeypatch.setattr(api_module, "import_dataset", capture_api_import) + monkeypatch.setattr(api_module, "export_dataset", capture_api_export) + monkeypatch.setattr(api_module, "result", lambda value: value) + api_module.import_sav( + "synthetic.sav", database_url="sqlite://", dataset_id="synthetic", + dolt_conformance_source=sentinel, + ) + api_module.export_sav( + database_url="sqlite://", dataset_id="synthetic", + destination="synthetic.sav", dolt_conformance_source=sentinel, + ) + + def capture_spss_import(**kwargs: object) -> dict[str, bool]: + calls.append(("spss_import", kwargs["dolt_conformance_source"])) + return {"ok": True} + + def capture_spss_export(**kwargs: object) -> dict[str, bool]: + calls.append(("spss_export", kwargs["dolt_conformance_source"])) + return {"ok": True} + + monkeypatch.setattr(spss_module, "import_sav_dataset", capture_spss_import) + monkeypatch.setattr(spss_module, "export_sav_dataset", capture_spss_export) + spss_module.import_dataset( + "synthetic.sav", database_url="sqlite://", dataset_id="synthetic", + dolt_conformance_source=sentinel, + ) + spss_module.export_dataset( + database_url="sqlite://", dataset_id="synthetic", + destination="synthetic.sav", dolt_conformance_source=sentinel, + ) + assert calls == [ + ("api_import", sentinel), + ("api_export", sentinel), + ("spss_import", sentinel), + ("spss_export", sentinel), + ] + + +def test_sav_import_propagates_explicit_source_to_mutation_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + calls: list[object] = [] + monkeypatch.setattr(sav_module, "_require_source", lambda path: None) + monkeypatch.setattr( + sav_module.pyspssio, "read_sav", + lambda *args, **kwargs: (pd.DataFrame(), {}), + ) + monkeypatch.setattr(sav_module, "_dictionary", lambda path: ({}, {})) + monkeypatch.setattr(sav_module, "_sha256", lambda path: "0" * 64) + + def capture_create(**kwargs: object) -> dict[str, str]: + calls.append(kwargs["dolt_conformance_source"]) + return {"dataset_id": "synthetic"} + + monkeypatch.setattr(sav_module, "create_wide_dataset", capture_create) + sav_module.import_sav_dataset( + source="synthetic.sav", database_url="sqlite://", + dataset_id="synthetic", dolt_conformance_source=sentinel, + ) + assert calls == [sentinel] + + +def test_sav_export_propagates_explicit_source_to_read_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + sentinel = object() + calls: list[object] = [] + + def stop_after_read(**kwargs: object) -> tuple[object, object, object]: + calls.append(kwargs["dolt_conformance_source"]) + raise UnsupportedOperationError("stop after SAV read propagation") + + monkeypatch.setattr(sav_module, "read_wide_dataset", stop_after_read) + with pytest.raises(UnsupportedOperationError, match="SAV read propagation"): + sav_module.export_sav_dataset( + database_url="sqlite://", dataset_id="synthetic", + destination="synthetic.sav", + dolt_conformance_source=sentinel, + ) + assert calls == [sentinel] + + +def test_export_recovery_helpers_propagate_explicit_source( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + sentinel = object() + calls: list[tuple[str, object]] = [] + + def capture_cleanup(**kwargs: object) -> str: + calls.append(("cleanup", kwargs["dolt_conformance_source"])) + return "cleanup-operation" + + def capture_failure(**kwargs: object) -> None: + calls.append(("failure", kwargs["dolt_conformance_source"])) + + monkeypatch.setattr( + sav_module, "record_export_cleanup_failure", capture_cleanup, + ) + monkeypatch.setattr(sav_module, "fail_export_operation", capture_failure) + destination = tmp_path / "destination.sav" + backup = tmp_path / "backup.sav" + staged = tmp_path / "staged.sav" + with pytest.raises(sav_module.ExportRecoveryError, match="cleanup_failed"): + sav_module._raise_export_cleanup_failed( + original_error=RuntimeError("synthetic export failure"), + cleanup_error=RuntimeError("synthetic cleanup failure"), + phase="synthetic", destination=destination, backup=backup, + staged=staged, had_previous=False, database_url="sqlite://", + dolt_conformance_source=sentinel, + ) + sav_module._mark_export_failed_after_restore( + database_url="sqlite://", operation_id="synthetic-operation", + error=RuntimeError("synthetic export failure"), phase="synthetic", + destination=destination, backup=backup, had_previous=False, + dolt_conformance_source=sentinel, + ) + assert calls == [("cleanup", sentinel), ("failure", sentinel)] + From b9650a3725ac073d9d4b1ad87b2ae292fab1c962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:16:59 +0300 Subject: [PATCH 002/119] Restore public SQL storage type helper --- src/openstatspec/sql/wide.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index bd21beb..6c7d678 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -12,6 +12,7 @@ from typing import Any from sqlalchemy import delete, BigInteger, Boolean, Column, DateTime, Float, Integer, MetaData, String, Table, Text, create_engine, insert, inspect, select, text, update +from sqlalchemy.dialects import mysql from ..core import UnsupportedOperationError, safe_error_identity as _safe_error_identity from .capabilities import active_connection, effective_profile from .dolt_conformance import DoltConformanceSource @@ -54,6 +55,11 @@ def _catalog_error(code: str, detail: str, **details: Any) -> CatalogPreflightEr return CatalogPreflightError(code, detail, details=details) +def string_type(profile: Any) -> Text: + """Use Dolt's tested LONGTEXT storage without changing MySQL/MariaDB DDL.""" + return mysql.LONGTEXT() if profile.name == "dolt" else Text() + + def _canonical_sha256(value: Any) -> str: return hashlib.sha256( json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") From c527b68c4018c12a9746926cccc608f37bcb8d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:25:24 +0300 Subject: [PATCH 003/119] Bind specification provenance and normalize server versions --- src/openstatspec/sql/capabilities.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index fdfd20c..b9d4b50 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -18,8 +18,8 @@ # Release/build automation must bind this to the exact commit used to build # openstatspec-specification. An uncommitted source tree has no truthful pin. -SPECIFICATION_COMMIT: str | None = None -SPECIFICATION_RELEASE: str | None = None +SPECIFICATION_COMMIT = "28bd6817261d1e55e51bb25076fb93e732ab1c74" +SPECIFICATION_RELEASE = "v0.2.0" DOLT_WRITE_CONFORMANCE = { "declaration_schema_id": "openstatspec-dolt-adapter-declaration-v1", @@ -431,6 +431,9 @@ def _profile( if dolt_envelope else None ) return { + "specification_status": "released", + "specification_release": SPECIFICATION_RELEASE, + "specification_commit": SPECIFICATION_COMMIT, "driver": "psycopg" if name == "postgresql" else "PyMySQL" if name in MYSQL_WIRE_PROFILES else "sqlite3", "claimed_server_versions": ( dolt_claimed_versions if dolt_envelope else policy["claimed"] @@ -542,4 +545,9 @@ def _version_tuple(raw_version: str) -> tuple[int, ...]: def _normalized_version(raw_version: str) -> str: - return ".".join(str(part) for part in _version_tuple(raw_version)) + match = re.search(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?", raw_version) + if not match: + return "" + return ".".join( + str(int(part)) for part in match.groups() if part is not None + ) From 25089a1d5cf29ea769230008559a776a9377a2af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:25:34 +0300 Subject: [PATCH 004/119] Bind specification provenance and normalize server versions --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5502f41..2730de6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v7 with: &specification-checkout repository: OpenStatSpec/specification - ref: agent/dolt-declaration-validator + ref: 28bd6817261d1e55e51bb25076fb93e732ab1c74 path: openstatspec-specification - name: Checkout required SPSS engine uses: actions/checkout@v7 From 17f3c47ac2ae0f0d121e2d988eadff386c8f6c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:25:39 +0300 Subject: [PATCH 005/119] Bind specification provenance and normalize server versions --- tests/test_sql_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 5988265..fe38da4 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -31,7 +31,7 @@ def test_profile_declarations_publish_released_specification_provenance() -> Non assert declaration["specification_release"] == "v0.2.0" assert ( declaration["specification_commit"] - == "79339ec3d8f8aa81789b7e85f6b8afa6f1374e50" + == "28bd6817261d1e55e51bb25076fb93e732ab1c74" ) def test_profile_preflight_fails_without_transforming_a_wide_dataset() -> None: From a650220e84b05d9186b7aaf167c25491ac35d392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:25:43 +0300 Subject: [PATCH 006/119] Bind specification provenance and normalize server versions --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 26d3ff8..2cca423 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -41,7 +41,7 @@ def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> matrix = openstatspec.capability_matrix() assert matrix["specification_status"] == "released" assert matrix["specification_release"] == "v0.2.0" - assert matrix["specification_commit"] == "79339ec3d8f8aa81789b7e85f6b8afa6f1374e50" + assert matrix["specification_commit"] == "28bd6817261d1e55e51bb25076fb93e732ab1c74" assert matrix["directions"] == ["import", "export", "semantic_round_trip"] assert matrix["active_connection"] is None assert matrix["engine"]["package"] == "openstatspec-pyspssio" From 873dc22f1027dc22782f86cc484aaa9f8d9f0015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:26:44 +0300 Subject: [PATCH 007/119] Fix lifecycle runtime regressions --- src/openstatspec/sql/wide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 6c7d678..91a4734 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -383,7 +383,7 @@ def _migrate_catalog_columns( """ inspector = inspect(connection) text_declaration = str( - Text.compile(dialect=connection.dialect) + Text().compile(dialect=connection.dialect) ) additions = { datasets.name: { From 3e129d1fed533fb85666f0e75bd1eb1a6019a89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:26:50 +0300 Subject: [PATCH 008/119] Fix lifecycle runtime regressions --- src/openstatspec/spss/sav.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index b4cae12..d96a399 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -42,6 +42,7 @@ UnsupportedOperationError, safe_error_identity as _export_error_identity, ) +from ..sql.capabilities import effective_profile from ..sql.dolt_conformance import DoltConformanceSource from ..sql.wide import ( create_wide_dataset, @@ -522,6 +523,9 @@ def export_sav_dataset( destination_path = Path(destination) if destination_path.suffix.lower() not in {".sav", ".zsav"}: raise UnsupportedOperationError("Export destinations must use the .sav or .zsav extension.") + effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) dataset, variables, rows = read_wide_dataset( database_url=database_url, dataset_id=dataset_id, dolt_conformance_source=dolt_conformance_source, From d2ec6aada61611fc60360054b5eb824a24ba5f87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:26:55 +0300 Subject: [PATCH 009/119] Fix lifecycle runtime regressions --- src/openstatspec/sql/profiles.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/openstatspec/sql/profiles.py b/src/openstatspec/sql/profiles.py index e1800f4..519ed10 100644 --- a/src/openstatspec/sql/profiles.py +++ b/src/openstatspec/sql/profiles.py @@ -225,16 +225,11 @@ def preflight( ) if value is not None and not math.isfinite(float(value)): raise _exceeded( - "numeric_value_not_finite", + "nonfinite_numeric_value", f"row {row_ordinal} value for {variable['source_name']!r} " "is not finite; non-finite adapter inputs are rejected.", row_ordinal=row_ordinal, source_name=variable["source_name"], - classification=( - "nan" if math.isnan(float(value)) - else "positive_infinity" if float(value) > 0 - else "negative_infinity" - ), ) row_bytes += 8 continue From 5f2d85a7f23dcb62fc80f1640f8621fff2c5b056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:37:00 +0300 Subject: [PATCH 010/119] [skip ci] Migrate workflow tests to explicit catalog initialization --- tests/test_sql_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_sql_workflow.py b/tests/test_sql_workflow.py index a418a70..0b8a7c6 100644 --- a/tests/test_sql_workflow.py +++ b/tests/test_sql_workflow.py @@ -73,6 +73,7 @@ def _variables(): def catalog(tmp_path): path = tmp_path / "workflow.sqlite" url = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=url) create_wide_dataset( database_url=url, dataset_id="source", source_name="source.sav", source_format="SAV", source_sha256="a" * 64, From bc8e187282858f29ca4afc970d33d2e78193bdfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:37:05 +0300 Subject: [PATCH 011/119] [skip ci] Migrate conformance tests to explicit catalog initialization --- tests/test_official_conformance.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_official_conformance.py b/tests/test_official_conformance.py index c2714de..7dc6b6f 100644 --- a/tests/test_official_conformance.py +++ b/tests/test_official_conformance.py @@ -226,6 +226,7 @@ def _assert_expected_catalog(database_url: str, dataset_name: str, expected: dic def _assert_round_trip( *, fixture: dict, source: Path, database_url: str, tmp_path: Path, profile: str, ) -> None: + openstatspec.initialize_catalog(database_url=database_url) assert fixture["directions"] == ["import", "export", "semantic_round_trip"] assert set(fixture["expects"]) <= SEMANTIC_EXPECTATIONS | CATALOG_EXPECTATIONS token = uuid4().hex[:10] @@ -295,6 +296,7 @@ def test_official_manifest_round_trips_through_server_profiles( def _assert_official_preflight_failure( *, database_url: str, profile: str, tmp_path: Path, ) -> None: + openstatspec.initialize_catalog(database_url=database_url) fixture = next( item for item in _manifest()["fixtures"] if item["id"] == "preflight-failure" ) From 4045d4374870b686d162ab3791dae7d1954db7a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:37:11 +0300 Subject: [PATCH 012/119] Migrate service tests to explicit catalog initialization --- tests/test_sql_services.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_sql_services.py b/tests/test_sql_services.py index e2e664f..301bfbd 100755 --- a/tests/test_sql_services.py +++ b/tests/test_sql_services.py @@ -43,6 +43,7 @@ def test_live_profile_import_validate_and_export(environment_name, dataset_id, s database_url = os.environ.get(environment_name) if not database_url: pytest.skip(f"{environment_name} is not configured") + openstatspec.initialize_catalog(database_url=database_url) runtime_dataset_id = f"{dataset_id}_{uuid4().hex[:8]}" imported = openstatspec.import_sav( source_sav, database_url=database_url, dataset_id=runtime_dataset_id, @@ -76,6 +77,7 @@ def test_live_profile_preserves_supported_sav_semantics(environment_name, datase database_url = os.environ.get(environment_name) if not database_url: pytest.skip(f"{environment_name} is not configured") + openstatspec.initialize_catalog(database_url=database_url) runtime_dataset_id = f"{dataset_id}_{suffix[1:]}_{uuid4().hex[:8]}" source = tmp_path / f"{runtime_dataset_id}{suffix}" destination = tmp_path / f"{runtime_dataset_id}-roundtrip{suffix}" @@ -97,6 +99,7 @@ def test_live_dolt_conservative_source_width_envelope(tmp_path) -> None: database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") if not database_url: pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") + openstatspec.initialize_catalog(database_url=database_url) token = uuid4().hex[:8] accepted_id = f"dolt_width_accepted_{token}" rejected_id = f"dolt_width_rejected_{token}" @@ -162,6 +165,7 @@ def test_live_dolt_post_ddl_fault_has_complete_compensating_cleanup( database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") if not database_url: pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") + openstatspec.initialize_catalog(database_url=database_url) dataset_id = f"dolt_cleanup_{uuid4().hex[:8]}" source = tmp_path / f"{dataset_id}.sav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) @@ -204,6 +208,7 @@ def test_live_dolt_adapter_value_boundary_is_atomic() -> None: database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") if not database_url: pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") + openstatspec.initialize_catalog(database_url=database_url) token = uuid4().hex[:8] accepted_id = f"dolt_value_accepted_{token}" rejected_id = f"dolt_value_rejected_{token}" From bdd30dda50dea6d40569f0719c9d29c2266a4e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:42:13 +0300 Subject: [PATCH 013/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_pyspssio_catalog_authority.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_pyspssio_catalog_authority.py b/tests/test_pyspssio_catalog_authority.py index f160b34..6093c0d 100644 --- a/tests/test_pyspssio_catalog_authority.py +++ b/tests/test_pyspssio_catalog_authority.py @@ -25,7 +25,9 @@ def dictionary_with_variable_set(path): return metadata, sav_module._engine_loss_report(metadata) monkeypatch.setattr(sav_module, "_dictionary", dictionary_with_variable_set) - imported = openstatspec.import_sav(source, database_url=f"sqlite:///{database_path}", dataset_id="variables") + database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) + imported = openstatspec.import_sav(source, database_url=database, dataset_id="variables") assert {diagnostic.code for diagnostic in imported.diagnostics} == set() connection = sqlite3.connect(database_path) assert connection.execute( @@ -54,6 +56,7 @@ def test_normalized_mr_catalog_is_authoritative_for_export(tmp_path) -> None: }, }, ) + openstatspec.initialize_catalog(database_url=database) openstatspec.import_sav(source, database_url=database, dataset_id="mr") connection = sqlite3.connect(database_path) rows = connection.execute( @@ -83,6 +86,7 @@ def test_normalized_mr_catalog_is_authoritative_for_export(tmp_path) -> None: def test_fidelity_event_details_survive_reopening_database(tmp_path) -> None: database = f"sqlite:///{tmp_path / 'events.sqlite'}" + openstatspec.initialize_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="events", source_name="events.sav", source_format="SAV", rows=[{"answer": 1.0}], From 83d701df9403785db854556a4e8d0de9f8c996cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:42:45 +0300 Subject: [PATCH 014/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_sav_sqlite.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index 8b99980..1e8a056 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -18,6 +18,7 @@ def test_pyspssio_round_trip_uses_one_wide_table_and_catalog(tmp_path) -> None: source = tmp_path / "tiny.sav" database_path = tmp_path / "dataset.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) exported = tmp_path / "roundtrip.zsav" pyspssio.write_sav( str(source), @@ -83,6 +84,7 @@ def test_file_label_round_trips_through_sqlite_and_export(tmp_path) -> None: source = tmp_path / "label-source.sav" database_path = tmp_path / "label.sqlite" database = "sqlite:///{}".format(database_path) + openstatspec.initialize_catalog(database_url=database) destination = tmp_path / "label-destination.sav" label = "OpenStatSpec label fixture" pyspssio.write_sav( @@ -106,13 +108,15 @@ def test_supported_pyspssio_metadata_round_trips_through_sqlite_for_sav_and_zsav database_path = tmp_path / f"supported-{suffix[1:]}.sqlite" destination = tmp_path / f"supported-roundtrip{suffix}" expected = write_supported_semantics_fixture(source) + database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) - result = openstatspec.import_sav(source, database_url=f"sqlite:///{database_path}", dataset_id=f"supported-{suffix[1:]}") + result = openstatspec.import_sav(source, database_url=database, dataset_id=f"supported-{suffix[1:]}") assert result["case_count"] == 4 - assert openstatspec.validate(database_url=f"sqlite:///{database_path}", dataset_id=f"supported-{suffix[1:]}")["valid"] is True + assert openstatspec.validate(database_url=database, dataset_id=f"supported-{suffix[1:]}")["valid"] is True connection = sqlite3.connect(database_path) assert connection.execute(f"select comment from data_supported_{suffix[1:]} order by __case_ordinal").fetchone() == (expected["long_text"],) - openstatspec.export_sav(database_url=f"sqlite:///{database_path}", dataset_id=f"supported-{suffix[1:]}", destination=destination, allow_loss=_COMPAT_NAME_LOSS) + openstatspec.export_sav(database_url=database, dataset_id=f"supported-{suffix[1:]}", destination=destination, allow_loss=_COMPAT_NAME_LOSS) assert compare_sav_semantics(source, destination) == {"equivalent": True, "differences": []} @pytest.mark.parametrize("suffix", [".sav", ".zsav"]) @@ -123,6 +127,7 @@ def test_import_rejects_physical_table_name_collision_without_partial_catalog(tm source = tmp_path / "fixture.sav" database_path = tmp_path / "dataset.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) openstatspec.import_sav(source, database_url=database, dataset_id="wave-1") with pytest.raises(ValueError, match="collides"): @@ -145,6 +150,7 @@ def test_raw_dictionary_bridge_preserves_distinct_formats_sets_and_attribute_arr source = tmp_path / f"raw-source{suffix}" destination = tmp_path / f"raw-destination{suffix}" database = f"sqlite:///{tmp_path / f'raw-{suffix[1:]}.sqlite'}" + openstatspec.initialize_catalog(database_url=database) with pyspssio.Writer(str(source), mode="w") as writer: writer.compression = 2 if suffix == ".zsav" else 1 writer._add_var("answer", 0) # pylint: disable=protected-access @@ -196,6 +202,7 @@ def test_very_long_string_round_trips_through_sqlite_and_export(tmp_path, suffix destination = tmp_path / f"long-destination{suffix}" database_path = tmp_path / f"long-{suffix[1:]}.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) pyspssio.write_sav( str(source), pd.DataFrame({"comment": [payload, "short"]}), ) From 8d799939ee9fbe1171ca22b291a0f81bf6fc9bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:42:50 +0300 Subject: [PATCH 015/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_spss_catalog_preflight.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_spss_catalog_preflight.py b/tests/test_spss_catalog_preflight.py index 1b2aa64..e618a3a 100755 --- a/tests/test_spss_catalog_preflight.py +++ b/tests/test_spss_catalog_preflight.py @@ -44,10 +44,12 @@ def test_import_catalog_preflight_rejects_invalid_weight_atomically( database_path = tmp_path / "weight.sqlite" variables = _variables() mutate(variables) + database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) with pytest.raises(CatalogPreflightError) as error: create_wide_dataset( - database_url=f"sqlite:///{database_path}", dataset_id="weight", + database_url=database, dataset_id="weight", source_name="weight.sav", source_format="SAV", rows=[{"weight": 1.0, "answer": 1.0, "text": "ok"}], variables=variables, case_weight_variable=weight_name, @@ -87,8 +89,10 @@ def test_import_catalog_preflight_rejects_invalid_weight_atomically( def test_numeric_weight_does_not_require_scale_measurement_level(tmp_path) -> None: variables = _variables() variables[1] = {**variables[1], "measure": "nominal"} + database = f"sqlite:///{tmp_path / 'weight-nominal.sqlite'}" + openstatspec.initialize_catalog(database_url=database) result = create_wide_dataset( - database_url=f"sqlite:///{tmp_path / 'weight-nominal.sqlite'}", + database_url=database, dataset_id="weight_nominal", source_name="weight.sav", source_format="SAV", rows=[{"weight": 1.0, "answer": 2.0, "text": "ok"}], variables=variables, case_weight_variable="answer", @@ -97,6 +101,7 @@ def test_numeric_weight_does_not_require_scale_measurement_level(tmp_path) -> No def _create_valid_dataset(database: str) -> None: + openstatspec.initialize_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="mr", source_name="mr.sav", source_format="SAV", rows=[{"weight": 1.0, "answer": 1.0, "text": "yes"}], From 58f764a715a98ffa045d21517bf9f81201b7917d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:42:55 +0300 Subject: [PATCH 016/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_vls_compatible_names.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_vls_compatible_names.py b/tests/test_vls_compatible_names.py index ff1f48c..7413b26 100644 --- a/tests/test_vls_compatible_names.py +++ b/tests/test_vls_compatible_names.py @@ -61,6 +61,7 @@ def test_vls_custom_compatible_name_round_trips_as_one_variable(tmp_path, suffix source = tmp_path / f"source{suffix}" destination = tmp_path / f"destination{suffix}" database = f"sqlite:///{tmp_path / f'vls-{suffix[1:]}.sqlite'}" + openstatspec.initialize_catalog(database_url=database) _write_vls_source(source) raw_dictionary.write_compatible_names( source, {_SOURCE_NAME: _COMPATIBLE_NAME}, encoding="UTF-8", @@ -115,6 +116,7 @@ def test_malformed_vls_export_removes_output_and_records_no_success(tmp_path, mo destination = tmp_path / "failed.sav" database_path = tmp_path / "failed.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) _write_vls_source(source) raw_dictionary.write_compatible_names( source, {_SOURCE_NAME: _COMPATIBLE_NAME}, encoding="UTF-8", From edbbae071e767410bd30b356b930e91fb94bef85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:43:53 +0300 Subject: [PATCH 017/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_attribute_catalog.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_attribute_catalog.py b/tests/test_attribute_catalog.py index 97df2c1..d0e9c06 100644 --- a/tests/test_attribute_catalog.py +++ b/tests/test_attribute_catalog.py @@ -20,6 +20,7 @@ def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, su imported_again = tmp_path / f"again-{suffix[1:]}.sqlite" database_path = tmp_path / f"attributes-{suffix[1:]}.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) pyspssio.write_sav( str(source), pd.DataFrame({"answer": [1.0]}), metadata={ @@ -62,7 +63,9 @@ def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, su assert exported["var_attributes"] == { "answer": {"Source": "catalog-variable", "Flag": "yes"}, } - openstatspec.import_sav(destination, database_url=f"sqlite:///{imported_again}", dataset_id="again") + imported_again_database = f"sqlite:///{imported_again}" + openstatspec.initialize_catalog(database_url=imported_again_database) + openstatspec.import_sav(destination, database_url=imported_again_database, dataset_id="again") reimported = sqlite3.connect(imported_again) assert reimported.execute( "select attribute_name, attribute_value from attribute_catalog " @@ -74,28 +77,29 @@ def test_attribute_catalog_is_authoritative_for_sav_and_zsav_export(tmp_path, su ).fetchall() == [("Source", "catalog-variable"), ("Flag", "yes")] -def test_attribute_catalog_migrates_old_json_catalog_without_rewriting_it(tmp_path) -> None: +def test_attribute_catalog_falls_back_to_legacy_json_without_rewriting_it(tmp_path) -> None: source = tmp_path / "legacy.sav" destination = tmp_path / "legacy-out.sav" database_path = tmp_path / "legacy.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) pyspssio.write_sav( str(source), pd.DataFrame({"answer": [1.0]}), metadata={"file_attributes": {"File": "legacy"}, "var_attributes": {"answer": {"Var": "legacy"}}}, ) openstatspec.import_sav(source, database_url=database, dataset_id="legacy") connection = sqlite3.connect(database_path) - connection.execute("drop table attribute_catalog") + connection.execute("delete from attribute_catalog") connection.commit() - # Opening an older catalog additively creates the new table and continues to - # use legacy JSON only because that older dataset has no normalized rows. + # An existing verified catalog with no normalized attribute rows falls back + # to legacy JSON without mutating the normalized catalog. dataset, variables, _ = read_wide_dataset(database_url=database, dataset_id="legacy") assert json.loads(dataset["file_attributes"]) == {"File": "legacy"} assert json.loads(variables[0]["attributes"]) == {"Var": "legacy"} assert connection.execute( - "select name from sqlite_master where type = 'table' and name = 'attribute_catalog'" - ).fetchone() == ("attribute_catalog",) + "select count(*) from attribute_catalog" + ).fetchone() == (0,) openstatspec.export_sav( database_url=database, dataset_id="legacy", destination=destination, allow_loss=_REQUIRED_ENGINE_LOSS, @@ -105,6 +109,7 @@ def test_attribute_catalog_migrates_old_json_catalog_without_rewriting_it(tmp_pa def test_attribute_catalog_preserves_ordered_arrays_through_raw_pyspssio_bridge(tmp_path) -> None: database = f"sqlite:///{tmp_path / 'array.sqlite'}" + openstatspec.initialize_catalog(database_url=database) create_wide_dataset( database_url=database, dataset_id="array", source_name="array.sav", source_format="SAV", rows=[{"answer": 1.0}], From 0385537b47bb172f5a10472f59a67ddf2ca1b73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:43:57 +0300 Subject: [PATCH 018/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2cca423..12432c2 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,6 +10,7 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> None: source = tmp_path / "fixture.sav" database = f"sqlite:///{tmp_path / 'dataset.sqlite'}" + openstatspec.initialize_catalog(database_url=database) output = tmp_path / "output.zsav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) @@ -25,7 +26,9 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N assert imported["case_count"] == 1 from openstatspec.core.results import OperationResult - assert isinstance(openstatspec.import_sav(source, database_url=f"sqlite:///{tmp_path / 'typed.sqlite'}", dataset_id="typed"), OperationResult) + typed_database = f"sqlite:///{tmp_path / 'typed.sqlite'}" + openstatspec.initialize_catalog(database_url=typed_database) + assert isinstance(openstatspec.import_sav(source, database_url=typed_database, dataset_id="typed"), OperationResult) assert openstatspec.cli.main(["validate", "--database-url", database, "--dataset-id", "fixture"]) == 0 assert json.loads(capsys.readouterr().out)["valid"] is True @@ -102,6 +105,7 @@ def test_cli_installs_schema_and_applies_plan_or_spss( source = tmp_path / "transform-source.sav" database_path = tmp_path / "transform.sqlite" database_url = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database_url) pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) openstatspec.import_sav( source, From 002b3c029450d776151447b4bcca72af607f79b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:44:02 +0300 Subject: [PATCH 019/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_document_round_trip.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_document_round_trip.py b/tests/test_document_round_trip.py index 40dacf4..1b1bae1 100644 --- a/tests/test_document_round_trip.py +++ b/tests/test_document_round_trip.py @@ -25,6 +25,7 @@ def test_document_lines_round_trip_through_sqlite(destination_suffix: str, tmp_p source = tmp_path / "source.sav" destination = tmp_path / f"destination{destination_suffix}" database = f"sqlite:///{tmp_path / 'documents.sqlite'}" + openstatspec.initialize_catalog(database_url=database) expected = ["Imported from a validated source.", "A second document line."] _source_with_documents(source, expected) @@ -61,6 +62,7 @@ def test_document_lines_import_from_zsav_and_export_to_sav(tmp_path: Path) -> No source = tmp_path / "source.zsav" destination = tmp_path / "destination.sav" database = f"sqlite:///{tmp_path / 'documents.sqlite'}" + openstatspec.initialize_catalog(database_url=database) expected = ["ZSAV document line."] _zsav_with_documents(source, expected, tmp_path / "temporary.sav") @@ -77,6 +79,7 @@ def test_document_and_compatible_name_round_trip_to_zsav(tmp_path: Path) -> None source = tmp_path / "source.sav" destination = tmp_path / "destination.zsav" database = f"sqlite:///{tmp_path / 'combined.sqlite'}" + openstatspec.initialize_catalog(database_url=database) source_name = "long_variable_name" pyspssio.write_sav(str(source), pd.DataFrame({source_name: [7.0]})) write_document_lines(source, ["Combined dictionary fixture."], encoding="UTF-8") @@ -109,6 +112,7 @@ def test_windows_1252_values_and_documents_round_trip_when_locale_is_available( source = tmp_path / "source.sav" destination = tmp_path / "destination.zsav" database = f"sqlite:///{tmp_path / 'cp1252.sqlite'}" + openstatspec.initialize_catalog(database_url=database) value = "Müller €" documents = ["Töö €"] pyspssio.write_sav( From a5f71397e3c2561f825c0e0e74a07f866183f20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:44:06 +0300 Subject: [PATCH 020/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_inplace_transform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 3c4cda4..869e189 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -61,6 +61,7 @@ def _submission(source_text: str) -> InPlacePlanSubmission: def catalog(tmp_path): path = tmp_path / "in-place.sqlite" url = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=url) create_wide_dataset( database_url=url, dataset_id="in_place_source", @@ -286,6 +287,7 @@ def test_string_create_target_is_rejected_before_any_mutation( def test_string_source_can_create_numeric_target(tmp_path) -> None: path = tmp_path / "string-source.sqlite" url = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=url) variables = _variables() variables[0].update({ "source_name": "color", From 558ba5bf2cb4b4cfb0b2dc21cc3d0e5baec2225d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:44:12 +0300 Subject: [PATCH 021/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_loss_reports.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_loss_reports.py b/tests/test_loss_reports.py index f1916f7..48f97e5 100755 --- a/tests/test_loss_reports.py +++ b/tests/test_loss_reports.py @@ -19,6 +19,7 @@ def test_persisted_import_fidelity_events_require_consent_after_reopen(tmp_path) source = tmp_path / "source.sav" database_path = tmp_path / "persisted.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) blocked = tmp_path / "blocked.sav" approved = tmp_path / "approved.sav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) @@ -44,6 +45,7 @@ def test_persisted_import_fidelity_events_require_consent_after_reopen(tmp_path) def test_loss_allowed_export_persists_accepted_diagnostics(tmp_path) -> None: database_path = tmp_path / "accepted-loss.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) source = tmp_path / "source.sav" destination = tmp_path / "accepted.sav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) @@ -59,6 +61,7 @@ def test_loss_allowed_export_persists_accepted_diagnostics(tmp_path) -> None: def test_non_utf8_source_encoding_is_explicit_export_loss(tmp_path) -> None: database_path = tmp_path / "legacy-encoding.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) destination = tmp_path / "legacy-encoding.sav" create_wide_dataset( database_url=database, dataset_id="legacy-encoding", source_name="legacy.sav", @@ -81,6 +84,7 @@ def test_non_utf8_source_encoding_is_explicit_export_loss(tmp_path) -> None: def test_explicit_legacy_locale_selects_the_single_engine_route(tmp_path, monkeypatch) -> None: database_path = tmp_path / "legacy-locale.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) destination = tmp_path / "legacy-locale.sav" create_wide_dataset( database_url=database, dataset_id="legacy-locale", source_name="legacy.sav", @@ -125,6 +129,7 @@ def test_compatible_variable_name_round_trips_from_current_sql_catalog(tmp_path, source = tmp_path / f"compat-source{suffix}" database_path = tmp_path / f"compat-{suffix[1:]}.sqlite" database = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database) destination = tmp_path / f"compat-destination{suffix}" source_name = "long_variable_name" From 2c17f647f23a71d3f55296336f5804823a8772b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:44:17 +0300 Subject: [PATCH 022/119] [skip ci] Migrate tests to explicit catalog initialization --- tests/test_normative_catalog.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_normative_catalog.py b/tests/test_normative_catalog.py index 0b1b64e..6f98071 100644 --- a/tests/test_normative_catalog.py +++ b/tests/test_normative_catalog.py @@ -2,6 +2,7 @@ import sqlite3 from uuid import UUID +import openstatspec import pytest from openstatspec.sql.wide import create_wide_dataset, record_export_operation @@ -72,6 +73,7 @@ def variables(): def test_import_writes_complete_normative_catalog(tmp_path): database_path = tmp_path / "normative.sqlite" database_url = f"sqlite:///{database_path}" + openstatspec.initialize_catalog(database_url=database_url) result = create_wide_dataset( database_url=database_url, dataset_id="wave_1", source_name="fixture.sav", source_format="SAV", From 3705ee3e62d0b9f0ccd9e1483d2018d3b63490e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:45:12 +0300 Subject: [PATCH 023/119] [skip ci] Publish exact SQL test evidence --- src/openstatspec/sql/capabilities.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index b9d4b50..28dc6c8 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -81,11 +81,11 @@ def _dolt_write_enabled( }, "mysql": { "claimed": ["MySQL 8.4.x", "MySQL 9.7.x"], - "ci": ["MySQL 8.4.x", "MySQL 9.7.x"], + "ci": ["MySQL 8.4.11", "MySQL 9.7.2"], }, "mariadb": { "claimed": ["MariaDB 11.4.x", "MariaDB 11.8.x", "MariaDB 12.3.x"], - "ci": ["MariaDB 11.4.x", "MariaDB 11.8.x", "MariaDB 12.3.x"], + "ci": ["MariaDB 11.4.12", "MariaDB 11.8.8", "MariaDB 12.3.2"], }, "dolt": { "claimed": [], @@ -93,7 +93,7 @@ def _dolt_write_enabled( }, "postgresql": { "claimed": ["PostgreSQL 17.x", "PostgreSQL 18.x"], - "ci": ["PostgreSQL 17.x", "PostgreSQL 18.x"], + "ci": ["PostgreSQL 17.10", "PostgreSQL 18.4"], }, } @@ -479,7 +479,7 @@ def _profile( else "profile_conformance_claim" ), }, - "text_type": "LONGTEXT" if name in MYSQL_WIRE_PROFILES else "TEXT", + "text_type": "LONGTEXT" if dolt_envelope else "TEXT", "ddl_atomic": name not in MYSQL_WIRE_PROFILES, "failure_cleanup": "compensating_cleanup" if name in MYSQL_WIRE_PROFILES else "transaction_rollback", "physical_table_mapping": "dataset.physical_table_schema + dataset.physical_table_name", From 8031b395f30aabc7bbdbf4c73010dae20d282ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:46:12 +0300 Subject: [PATCH 024/119] [skip ci] Align atomic tests with explicit lifecycle --- tests/test_atomic_import.py | 53 ++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/tests/test_atomic_import.py b/tests/test_atomic_import.py index 88a98a3..93df530 100755 --- a/tests/test_atomic_import.py +++ b/tests/test_atomic_import.py @@ -5,13 +5,14 @@ from sqlalchemy.exc import IntegrityError import openstatspec.sql.wide as wide -from openstatspec.sql.profiles import DOLT, MYSQL, SQLITE +from openstatspec.sql.profiles import MYSQL, SQLITE from openstatspec.sql.wide import create_wide_dataset def test_failed_row_insert_leaves_no_catalog_or_data_table(tmp_path) -> None: database_path = tmp_path / "dataset.sqlite" database = f"sqlite:///{database_path}" + wide.initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", "storage_kind": "string", "string_width": 8, "label": "", @@ -33,14 +34,19 @@ def test_failed_row_insert_leaves_no_catalog_or_data_table(tmp_path) -> None: assert connection.execute("select count(*) from variable").fetchone() == (0,) -def test_failed_preflight_persists_operation_without_creating_dataset(tmp_path) -> None: +def test_failed_preflight_persists_operation_without_creating_dataset(tmp_path, monkeypatch) -> None: database_path = tmp_path / "preflight.sqlite" database = f"sqlite:///{database_path}" + wide.initialize_wide_catalog(database_url=database) + too_many = SQLITE.max_source_variables + 1 + monkeypatch.setattr( + wide, "effective_profile", lambda _url, **_kwargs: (SQLITE, {}), + ) with pytest.raises(Exception, match="Target capability exceeded"): create_wide_dataset( database_url=database, dataset_id="too-wide", source_name="too-wide.sav", - source_format="SAV", rows=(), variables=[{}] * 2_001, + source_format="SAV", rows=(), variables=[{}] * too_many, ) connection = sqlite3.connect(database_path) @@ -55,12 +61,13 @@ def test_failed_preflight_persists_operation_without_creating_dataset(tmp_path) "select direction, severity, code, details from fidelity_event_catalog" ).fetchone() assert (direction, severity, code) == ("import", "error", "target_capability_exceeded") - assert '"variable_count": 2001' in details + assert f'"variable_count": {too_many}' in details def test_identifier_mapping_preflight_records_failure_before_dataset_creation(tmp_path) -> None: database_path = tmp_path / "identifier.sqlite" database = f"sqlite:///{database_path}" + wide.initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "wrong_name", "storage_kind": "string", "string_width": 8, "label": "", @@ -84,9 +91,10 @@ def test_identifier_mapping_preflight_records_failure_before_dataset_creation(tm def test_declared_string_width_preflight_is_atomic_and_diagnostic(tmp_path, monkeypatch) -> None: database_path = tmp_path / "string-width.sqlite" database = f"sqlite:///{database_path}" + wide.initialize_wide_catalog(database_url=database) monkeypatch.setattr( wide, "effective_profile", - lambda _url: (replace(SQLITE, max_text_value_bytes=3), {}), + lambda _url, **_kwargs: (replace(SQLITE, max_text_value_bytes=3), {}), ) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", @@ -118,6 +126,7 @@ def test_nonatomic_failure_after_normative_write_cleans_both_catalogs_and_data( ) -> None: database_path = tmp_path / "nonatomic-cleanup.sqlite" database = f"sqlite:///{database_path}" + wide.initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", "storage_kind": "string", "string_width": 8, "label": "", @@ -126,7 +135,7 @@ def test_nonatomic_failure_after_normative_write_cleans_both_catalogs_and_data( }] monkeypatch.setattr( wide, "effective_profile", - lambda _url: (replace(MYSQL, name="mysql"), {}), + lambda _url, **_kwargs: (replace(MYSQL, name="mysql"), {}), ) real_store = wide.store_normative_dataset @@ -191,17 +200,8 @@ def test_occupied_foreign_namespace_fails_without_modification(tmp_path) -> None "select name, sql from sqlite_master where type = 'table' order by name" ).fetchall() - variables = [{ - "ordinal": 1, "source_name": "name", "physical_name": "name", - "storage_kind": "string", "string_width": 8, "label": "", - "format": "A8", "measure": "nominal", "alignment": "left", - "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", - }] - with pytest.raises(RuntimeError, match="occupied"): - create_wide_dataset( - database_url=database, dataset_id="foreign", source_name="fixture.sav", - source_format="SAV", rows=[{"name": "ok"}], variables=variables, - ) + with pytest.raises(RuntimeError, match="foreign"): + wide.initialize_wide_catalog(database_url=database) assert connection.execute( "select name, sql from sqlite_master where type = 'table' order by name" @@ -210,12 +210,12 @@ def test_occupied_foreign_namespace_fails_without_modification(tmp_path) -> None @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) -def test_dolt_nonfinite_preflight_creates_no_dataset_or_physical_table( - tmp_path, monkeypatch, value, +def test_nonfinite_preflight_creates_no_dataset_or_physical_table( + tmp_path, value, ) -> None: database_path = tmp_path / "dolt-nonfinite.sqlite" database = f"sqlite:///{database_path}" - monkeypatch.setattr(wide, "effective_profile", lambda _url: (DOLT, {})) + wide.initialize_wide_catalog(database_url=database) variables = [{ "ordinal": 1, "source_name": "value", "physical_name": "value", "storage_kind": "numeric", "string_width": None, "label": "", @@ -246,18 +246,22 @@ def test_dolt_nonfinite_preflight_creates_no_dataset_or_physical_table( ).fetchall() == [(None, "target_capability_exceeded")] -def test_empty_namespace_dolt_width_failure_initializes_identity_and_one_audit( +def test_limited_width_failure_preserves_initialized_identity_and_one_audit( tmp_path, monkeypatch, ) -> None: database_path = tmp_path / "dolt-preflight.sqlite" database = f"sqlite:///{database_path}" - monkeypatch.setattr(wide, "effective_profile", lambda _url: (DOLT, {})) + wide.initialize_wide_catalog(database_url=database) + monkeypatch.setattr( + wide, "effective_profile", + lambda _url, **_kwargs: (replace(SQLITE, max_source_variables=1), {}), + ) with pytest.raises(Exception, match="Target capability exceeded"): create_wide_dataset( database_url=database, dataset_id="too-wide-dolt", source_name="too-wide-dolt.sav", source_format="SAV", - rows=(), variables=[{}] * 306, + rows=(), variables=[{}, {}], ) connection = sqlite3.connect(database_path) @@ -284,6 +288,7 @@ def test_nonatomic_failure_during_final_completion_still_cleans_dataset( ) -> None: database_path = tmp_path / f"{failure_point}.sqlite" database = f"sqlite:///{database_path}" + wide.initialize_wide_catalog(database_url=database) dataset_id = f"cleanup-{failure_point}" variables = [{ "ordinal": 1, "source_name": "name", "physical_name": "name", @@ -293,7 +298,7 @@ def test_nonatomic_failure_during_final_completion_still_cleans_dataset( }] monkeypatch.setattr( wide, "effective_profile", - lambda _url: (replace(MYSQL, name="mysql"), {}), + lambda _url, **_kwargs: (replace(MYSQL, name="mysql"), {}), ) triggered = False if failure_point == "mirror_completion": From 73a3cca122d26970b8fba94319340f211a6b4233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:46:44 +0300 Subject: [PATCH 025/119] [skip ci] Make unbound specification test explicit --- tests/test_dolt_conformance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_dolt_conformance.py b/tests/test_dolt_conformance.py index 77a9062..0722101 100644 --- a/tests/test_dolt_conformance.py +++ b/tests/test_dolt_conformance.py @@ -120,6 +120,7 @@ def require_exact_match( "observed": {"max_allowed_packet": 1_000_000}, }, ) + monkeypatch.setattr(capability_module, "SPECIFICATION_COMMIT", None) assert capability_module.SPECIFICATION_COMMIT is None with pytest.raises(UnsupportedOperationError, match="not bound"): capability_module.effective_profile( From 2ea59fcfdb74d10eea68f90cd6c81f571ef503af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:48:06 +0300 Subject: [PATCH 026/119] [skip ci] Fail closed on SQL identity probe ambiguity --- src/openstatspec/sql/capabilities.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index 28dc6c8..6cf339f 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -151,10 +151,12 @@ def active_connection( observed = {} elif dialect in {"mysql", "mariadb"}: wire_version = _required_identity_text( - connection.execute(text("select @@version")).scalar_one(), "@@version", + _identity_scalar(connection, "select @@version", "@@version"), "@@version", ) comment = _required_identity_text( - connection.execute(text("select @@version_comment")).scalar_one(), + _identity_scalar( + connection, "select @@version_comment", "@@version_comment", + ), "@@version_comment", ) normalized_comment = comment.strip().casefold() @@ -165,12 +167,12 @@ def active_connection( "Conflicting Dolt and MariaDB active-server identity." ) product_version = _required_identity_text( - connection.execute(text("select DOLT_VERSION()")).scalar_one(), + _identity_scalar(connection, "select DOLT_VERSION()", "DOLT_VERSION()"), "DOLT_VERSION()", ) profile_name, product = "dolt", "Dolt" active_branch = _required_identity_text( - connection.execute(text("select ACTIVE_BRANCH()")).scalar_one(), + _identity_scalar(connection, "select ACTIVE_BRANCH()", "ACTIVE_BRANCH()"), "ACTIVE_BRANCH()", ) identity_source = ( @@ -179,9 +181,14 @@ def active_connection( elif "mariadb" in (wire_version + " " + comment).casefold(): profile_name, product = "mariadb", "MariaDB" identity_source = "SELECT @@version, @@version_comment" - else: + elif "mysql" in normalized_comment: profile_name, product = "mysql", "MySQL" identity_source = "SELECT @@version, @@version_comment" + else: + raise UnsupportedOperationError( + "The active MySQL-wire product is unknown; no SQL profile " + "claim can be selected safely." + ) raw_version = product_version packet = int(connection.execute(text("select @@max_allowed_packet")).scalar_one()) observed = { @@ -225,6 +232,15 @@ def active_connection( } +def _identity_scalar(connection: Any, query: str, source: str) -> Any: + try: + return connection.execute(text(query)).scalar_one() + except Exception as error: + raise UnsupportedOperationError( + f"Active-server identity probe {source} failed." + ) from error + + def _required_identity_text(value: Any, source: str) -> str: if not isinstance(value, str) or not value.strip(): raise UnsupportedOperationError( From 92a3d4bf75b6b653a438becbc27e1fb1b2cf1662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:49:40 +0300 Subject: [PATCH 027/119] [skip ci] Align SQL profile tests with fail-closed Dolt evidence --- tests/test_sql_profiles.py | 174 +++++++++++++++---------------------- 1 file changed, 72 insertions(+), 102 deletions(-) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index fe38da4..987cb24 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -92,13 +92,6 @@ def test_normative_catalog_compiles_for_every_sql_family() -> None: ("mysql", "8.4.0", True), ("mysql", "8.4.999", True), ("mysql", "9.7.2", True), ("mysql", "8.0.44", False), ("mysql", "8.5.0", False), ("mysql", "9.8.0", False), - ("dolt", "2.2.2", True), ("dolt", " 2.2.2 ", True), - ("dolt", "2.2.3", True), ("dolt", "2.2.999", True), - ("dolt", "2.2.0", False), ("dolt", "2.2.1", False), - ("dolt", "2.3.0", False), ("dolt", "2.2", False), - ("dolt", "2.2.3-rc1", False), ("dolt", "2.2.3+build.1", False), - ("dolt", "v2.2.3", False), ("dolt", "2.2.03", False), - ("dolt", "garbage", False), ("mariadb", "11.4.0-MariaDB", True), ("mariadb", "11.4.999-MariaDB", True), ("mariadb", "11.8.8-MariaDB", True), @@ -116,6 +109,18 @@ def test_claimed_server_version_policy_is_explicit( assert server_version_supported(profile, version) is supported +@pytest.mark.parametrize( + "version", + [ + "2.2.2", " 2.2.2 ", "2.2.3", "2.2.999", "2.2.0", "2.2.1", + "2.3.0", "2.2", "2.2.3-rc1", "2.2.3+build.1", "v2.2.3", + "2.2.03", "garbage", + ], +) +def test_dolt_without_concrete_declaration_is_not_claimed(version: str) -> None: + assert server_version_supported("dolt", version) is False + + @pytest.mark.services @pytest.mark.parametrize( ("environment_name", "profile"), @@ -123,7 +128,6 @@ def test_claimed_server_version_policy_is_explicit( ("OPENSTATSPEC_POSTGRES_URL", "postgresql"), ("OPENSTATSPEC_MYSQL_URL", "mysql"), ("OPENSTATSPEC_MARIADB_URL", "mariadb"), - ("OPENSTATSPEC_DOLT_URL", "dolt"), ], ) def test_active_server_identity_matches_claimed_ci_profile( @@ -139,6 +143,18 @@ def test_active_server_identity_matches_claimed_ci_profile( assert active["matched_claim"] is not None +@pytest.mark.services +def test_live_dolt_identity_is_observed_but_not_claimed_without_declarations() -> None: + database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") + if not database_url: + pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") + + active = active_connection(database_url) + assert active["profile"] == "dolt" + assert active["claimed_supported"] is False + assert active["matched_claim"] is None + + @pytest.mark.services @pytest.mark.parametrize( ("database_environment", "version_environment", "profile"), @@ -200,17 +216,8 @@ def test_server_policy_distinguishes_claimed_families_from_exact_ci_evidence() - assert declarations["postgresql"]["ci_tested_server_versions"] == [ "PostgreSQL 17.10", "PostgreSQL 18.4", ] - assert declarations["dolt"]["claimed_server_versions"] == ["Dolt 2.2.x"] - assert declarations["dolt"]["claimed_version_range"] == { - "minimum_inclusive": "2.2.2", - "maximum_exclusive": "2.3.0", - } - assert declarations["dolt"]["ci_tested_server_versions"] == [ - "Dolt 2.2.2", "Dolt 2.2.3", - ] - assert declarations["dolt"]["exact_ci_tested_versions"] == [ - "2.2.2", "2.2.3", - ] + assert declarations["dolt"]["claimed_server_versions"] == [] + assert declarations["dolt"]["ci_tested_server_versions"] == [] class _ProbeResult: @@ -256,6 +263,7 @@ def _mock_mysql_probes(monkeypatch, **overrides): "select @@version": "8.0.31", "select @@version_comment": "Dolt", "select DOLT_VERSION()": "2.2.3", + "select ACTIVE_BRANCH()": "main", "select @@max_allowed_packet": 1_073_741_824, } responses.update(overrides) @@ -272,21 +280,22 @@ def test_dolt_identity_is_exact_and_publishes_wire_and_product_versions(monkeypa active = active_connection("mysql+pymysql://user@host/database") assert active["dialect"] == "mysql" - assert active["profile"] == active["engine"] == active["product"] == "dolt" - assert active["transport"] == "mysql_compatible" - assert active["driver"] == "pymysql" + assert active["profile"] == active["product"] == "dolt" assert active["raw_wire_version"] == "8.0.31" assert active["raw_product_version"] == active["raw_server_version"] == "2.2.3" - assert active["raw_version_comment"] == " dOlT " + assert active["raw_version_comment"] == "dOlT" assert active["server_version"] == "2.2.3" - assert active["identity_source"] == "SELECT @@version, @@version_comment, DOLT_VERSION()" - assert active["claimed_supported"] is True + assert active["identity_source"] == ( + "SELECT @@version, @@version_comment, DOLT_VERSION(), ACTIVE_BRANCH()" + ) + assert active["claimed_supported"] is False + assert active["matched_claim"] is None + assert active["working_set_binding"]["active_branch"] == "main" assert connection.calls == [ - "select @@version", "select @@version_comment", - "select DOLT_VERSION()", "select @@max_allowed_packet", + "select @@version", "select @@version_comment", "select DOLT_VERSION()", + "select ACTIVE_BRANCH()", "select @@max_allowed_packet", ] - @pytest.mark.parametrize("comment", [None, False, "", " "]) def test_mysql_wire_identity_requires_a_nonempty_version_comment(monkeypatch, comment) -> None: connection = _mock_mysql_probes( @@ -325,21 +334,23 @@ def test_dolt_comment_requires_a_nonempty_product_version(monkeypatch) -> None: "2.2.3-rc1", "2.2.3+build.1", "v2.2.3", "2.2.03", "garbage", ], ) -def test_dolt_identity_rejects_out_of_range_or_noncanonical_product_versions( +def test_dolt_identity_observes_but_does_not_claim_unbound_versions( monkeypatch, product_version, ) -> None: connection = _mock_mysql_probes( monkeypatch, **{"select DOLT_VERSION()": product_version}, ) - with pytest.raises(UnsupportedOperationError, match=r">=2\.2\.2,<2\.3\.0"): - active_connection("mysql+pymysql://user@host/database") + active = active_connection("mysql+pymysql://user@host/database") + assert active["raw_product_version"] == product_version + assert active["claimed_supported"] is False + assert active["matched_claim"] is None assert connection.calls == [ "select @@version", "select @@version_comment", "select DOLT_VERSION()", + "select ACTIVE_BRANCH()", "select @@max_allowed_packet", ] - def test_unknown_mysql_wire_product_fails_closed_without_dolt_probe(monkeypatch) -> None: connection = _mock_mysql_probes( monkeypatch, @@ -372,77 +383,35 @@ def test_non_dolt_products_never_call_the_dolt_function(monkeypatch) -> None: assert "select DOLT_VERSION()" not in connection.calls -def test_effective_profile_selects_dolt_without_changing_url_profile(monkeypatch) -> None: +def test_effective_profile_fails_closed_without_dolt_declarations(monkeypatch) -> None: active = { - "profile": "dolt", "server_version": "2.2.3", "claimed_supported": True, - "observed": {"max_allowed_packet": 1_073_741_824}, + "profile": "dolt", + "raw_product_version": "2.2.3", } - monkeypatch.setattr(capabilities, "active_connection", lambda _url: active) - - profile, observed = effective_profile("mysql+pymysql://user@host/database") - - assert profile is not MYSQL - assert profile.name == "dolt" - assert profile.url_schemes == () - assert profile.max_physical_variables == 305 - assert profile.max_text_value_bytes == 65_504 - assert profile.max_row_bytes == 65_504 - assert observed is active + monkeypatch.setattr( + capabilities, "active_connection", lambda _url, **_kwargs: active, + ) + with pytest.raises(UnsupportedOperationError, match="no concrete declarations"): + effective_profile("mysql+pymysql://user@host/database") -def test_dolt_declaration_labels_conservative_envelopes() -> None: +def test_dolt_declaration_is_fail_closed_without_concrete_evidence() -> None: declaration = capabilities.profile_declarations()["dolt"] - assert declaration["dialect"] == "mysql" - assert declaration["profile"] == "dolt" - assert declaration["engine"] == "dolt" - assert declaration["transport"] == "mysql_compatible" - assert declaration["claimed_server_versions"] == ["Dolt 2.2.x"] - assert declaration["claimed_version_range"] == { - "minimum_inclusive": "2.2.2", - "maximum_exclusive": "2.3.0", - } - assert declaration["ci_tested_server_versions"] == [ - "Dolt 2.2.2", "Dolt 2.2.3", - ] - assert declaration["exact_ci_tested_versions"] == ["2.2.2", "2.2.3"] - assert declaration["proposed_adapter_limits"]["maximum_physical_columns"] == 306 - assert declaration["proposed_adapter_limits"]["maximum_source_variables"] == 305 - assert declaration["proposed_adapter_limits"]["maximum_value_bytes"] == 65_504 - assert declaration["proposed_adapter_limits"]["maximum_row_bytes"] == 65_504 - assert declaration["theoretical_limits"]["maximum_value_bytes"] == 4_294_967_295 - assert declaration["observed_limits"]["minimum_observed_physical_columns"] == 307 - assert declaration["observed_limits"]["identifier_limit"]["value"] == 64 - assert declaration["observed_limits"]["rejected_identifier_bytes"] == 65 - assert set(declaration["proposed_adapter_limits"]) == { - "maximum_physical_columns", "maximum_source_variables", - "maximum_value_bytes", "maximum_row_bytes", - } - assert declaration["limit_bases"]["maximum_physical_columns"] == "proposed_adapter_envelope" - assert declaration["limit_bases"]["identifier_limit"] == "observed_exact_version" - assert declaration["limit_bases"]["maximum_value_bytes"] == "observed_exact_version" - assert declaration["limit_bases"]["maximum_statement_bytes"] == "active_connection_observation" + assert declaration["claimed_server_versions"] == [] + assert declaration["ci_tested_server_versions"] == [] + assert declaration["operational_write_enabled"] is False assert declaration["effective_limits"] is None + assert declaration["effective_limits_status"] == "not_connected" + assert declaration["write_conformance"]["declaration_count"] == 0 + assert declaration["write_conformance"]["write_enabled"] is False + assert declaration["adapter_envelope"]["limit_basis"] == "proposed_adapter_envelope" + assert declaration["theoretical_limits"]["limit_basis"] == ( + "server_limits_not_claimed" + ) assert declaration["text_type"] == "LONGTEXT" - assert capabilities.profile_declarations()["mysql"]["text_type"] == "LONGTEXT" - assert capabilities.profile_declarations()["mariadb"]["text_type"] == "LONGTEXT" - assert declaration["ddl_atomic"] is False - assert declaration["failure_cleanup"] == "compensating_cleanup" - assert declaration["numeric_value_policy"] == { - "finite_binary64": "supported", - "nan": "rejected_before_ddl", - "positive_infinity": "rejected_before_ddl", - "negative_infinity": "rejected_before_ddl", - } - assert declaration["storage_evidence"]["binary64"]["maximum_finite_round_trip_exact"] is True - assert declaration["storage_evidence"]["binary64"]["source"] - assert declaration["storage_evidence"]["binary64"]["version"] == "2.2.2" - assert declaration["storage_evidence"]["text"]["observed_value_bytes"] == 65_504 - assert declaration["storage_evidence"]["text"]["source"] - assert declaration["storage_evidence"]["text"]["version"] == "2.2.2" - assert declaration["storage_evidence"]["text"]["unit"] == "bytes" - assert declaration["transformation_workflow"] == "unsupported" - + assert capabilities.profile_declarations()["mysql"]["text_type"] == "TEXT" + assert capabilities.profile_declarations()["mariadb"]["text_type"] == "TEXT" def test_dolt_uses_longtext_without_changing_mysql_storage() -> None: dolt_table = Table("dolt_text", MetaData(), Column("value", string_type(DOLT))) @@ -454,17 +423,18 @@ def test_dolt_uses_longtext_without_changing_mysql_storage() -> None: assert "LONGTEXT" not in mysql_ddl -def test_dolt_row_preflight_counts_utf8_values_against_adapter_envelope() -> None: +def test_dolt_row_preflight_enforces_configured_utf8_value_limit() -> None: + limited = replace(DOLT, max_text_value_bytes=3) variables = [{ "ordinal": 1, "source_name": "value", "physical_name": "value", - "storage_kind": "string", "string_width": 65_504, + "storage_kind": "string", "string_width": 3, }] - preflight(DOLT, variables, rows=[{"value": "x" * 65_504}]) + preflight(limited, variables, rows=[{"value": "xxx"}]) with pytest.raises(UnsupportedOperationError) as error: - preflight(DOLT, variables, rows=[{"value": "x" * 65_505}]) + preflight(limited, variables, rows=[{"value": "xxxx"}]) assert error.value.details["reason"] == "text_value_limit" - + assert error.value.details["maximum"] == 3 @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) def test_dolt_numeric_preflight_rejects_nonfinite_values(value) -> None: @@ -484,7 +454,7 @@ def test_dolt_numeric_preflight_rejects_nonfinite_values(value) -> None: def test_validate_identity_failure_happens_before_catalog_access(monkeypatch) -> None: - def fail_identity(_url): + def fail_identity(_url, **_kwargs): raise UnsupportedOperationError("identity unavailable") monkeypatch.setattr(wide, "effective_profile", fail_identity) @@ -502,7 +472,7 @@ def fail_identity(_url): def test_export_identity_failure_happens_before_read_or_destination(monkeypatch, tmp_path) -> None: destination = tmp_path / "blocked.sav" - def fail_identity(_url): + def fail_identity(_url, **_kwargs): raise UnsupportedOperationError("identity unavailable") monkeypatch.setattr(sav, "effective_profile", fail_identity) From f53440f758e58cbfc8f1ee2f9b175d12442be486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:50:57 +0300 Subject: [PATCH 028/119] [skip ci] Normalize reflected SQL catalog shapes --- src/openstatspec/sql/wide.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 91a4734..375fe4f 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1000,6 +1000,11 @@ def _normalized_sql_type(inspector: Any, value: Any) -> str: if inspector.bind.dialect.name in {"mysql", "mariadb"}: if compiled in {"BOOL", "BOOLEAN", "TINYINT(1)"}: return "BOOLEAN/TINYINT(1)" + compiled = re.sub( + r"\b(TINYINT|SMALLINT|MEDIUMINT|INTEGER|INT|BIGINT)\(\d+\)", + r"\1", + compiled, + ) return compiled @@ -1064,6 +1069,13 @@ def _normalized_check_sql(value: Any) -> str: result = " ".join(str(value).strip().casefold().split()) result = result.replace("`", "").replace('"', "") result = re.sub(r"\[([^]]+)\]", r"\1", result) + result = re.sub(r"(? Date: Mon, 3 Aug 2026 18:51:21 +0300 Subject: [PATCH 029/119] Migrate lifecycle tests and normalize reflected catalog shapes --- tests/test_catalog_lifecycle.py | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index f1b985c..460d28a 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -334,3 +334,40 @@ def test_multiple_catalog_identities_are_ambiguous_and_never_mutated(tmp_path): ).fetchall() == before assert _table_names(path) == {"catalog_identity"} connection.close() + + +@pytest.mark.parametrize( + ("reflected", "expected"), + [ + ( + "((contract_id)::text = " + "'openstatspec-strict-wide-table-v1'::text)", + "contract_id = 'openstatspec-strict-wide-table-v1'", + ), + ( + "(`contract_id` = _utf8mb4'openstatspec-strict-wide-table-v1')", + "contract_id = 'openstatspec-strict-wide-table-v1'", + ), + ("((catalog_identity_key)::integer = 1)", "catalog_identity_key = 1"), + ], +) +def test_reflected_check_normalization_removes_only_noop_dialect_syntax( + reflected, expected, +): + assert wide._normalized_check_sql(reflected) == expected + + +def test_reflected_mysql_integer_display_width_is_not_semantic(): + from types import SimpleNamespace + from sqlalchemy.dialects import mysql + + inspector = SimpleNamespace(bind=SimpleNamespace(dialect=mysql.dialect())) + assert wide._normalized_sql_type( + inspector, mysql.INTEGER(display_width=11), + ) == "INTEGER" + assert wide._normalized_sql_type( + inspector, mysql.BIGINT(display_width=20), + ) == "BIGINT" + assert wide._normalized_sql_type( + inspector, mysql.VARCHAR(length=255), + ) == "VARCHAR(255)" From a88f0cc5a380a8897cea7522d187b7a3a89b579d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:56:40 +0300 Subject: [PATCH 030/119] Normalize database DOUBLE wrappers on read [skip ci] --- src/openstatspec/sql/wide.py | 38 +++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 375fe4f..810f50f 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -7,6 +7,7 @@ import sys from contextlib import contextmanager from datetime import UTC, datetime +from decimal import Decimal from uuid import uuid4 from collections.abc import Iterable, Mapping from typing import Any @@ -2646,6 +2647,34 @@ def _endpoint_from_row(row: Mapping[str, Any], *, prefix: str) -> Any: return row[f"{prefix}_numeric"] if endpoint_type == "numeric" else row[f"{prefix}_text"] +def _canonicalize_database_numeric_rows( + rows: Iterable[Mapping[str, Any]], + variables: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + """Convert exact numeric driver wrappers at the SQL read boundary. + + MySQL-family drivers may return DOUBLE columns as Decimal instances. The + public adapter contract remains binary64-only, so database-native decimal + wrappers are converted back to their declared physical representation + before strict preflight validation. Other unexpected values are preserved + so preflight can reject them with its machine-readable diagnostic. + """ + numeric_names = { + str(variable["physical_name"]) + for variable in variables + if variable.get("storage_kind") == "numeric" + } + normalized: list[dict[str, Any]] = [] + for source_row in rows: + row = dict(source_row) + for physical_name in numeric_names: + value = row.get(physical_name) + if isinstance(value, Decimal): + row[physical_name] = float(value) + normalized.append(row) + return normalized + + def read_wide_dataset( *, database_url: str, @@ -2675,9 +2704,12 @@ def read_wide_dataset( variables = [dict(item) for item in connection.execute( select(variable_catalog).where(variable_catalog.c.dataset_id == dataset_id).order_by(variable_catalog.c.ordinal) ).mappings().all()] - rows = [dict(item) for item in connection.execute( - select(data_table).order_by(data_table.c.__case_ordinal) - ).mappings().all()] + rows = _canonicalize_database_numeric_rows( + connection.execute( + select(data_table).order_by(data_table.c.__case_ordinal) + ).mappings().all(), + variables, + ) document_rows_result = connection.execute( select(documents_catalog).where(documents_catalog.c.dataset_id == dataset_id) .order_by(documents_catalog.c.ordinal) From 2c769fa8395623bb2bd22399cf9db16622a56dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:56:57 +0300 Subject: [PATCH 031/119] Cover SQL numeric read normalization [skip ci] --- tests/test_catalog_lifecycle.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 460d28a..733126b 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -1,3 +1,4 @@ +from decimal import Decimal import sqlite3 import pytest @@ -216,6 +217,26 @@ def fail_mutation(*_args, **_kwargs): connection.close() +def test_database_decimal_numeric_wrappers_are_restored_to_binary64(): + variables = [{ + "physical_name": "score", + "storage_kind": "numeric", + }] + rows = wide._canonicalize_database_numeric_rows( + [ + {"score": Decimal("1.5000000000"), "name": "alpha"}, + {"score": None, "name": "missing"}, + ], + variables, + ) + + assert rows == [ + {"score": 1.5, "name": "alpha"}, + {"score": None, "name": "missing"}, + ] + assert isinstance(rows[0]["score"], float) + + def test_bounded_batches_never_exceed_statement_payload_limit(): variables = _variables() rows = [ From af52e51614b3d5cc8887e72c2920ffe305626a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:58:28 +0300 Subject: [PATCH 032/119] Run import preflight before physical table construction [skip ci] --- src/openstatspec/sql/wide.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 810f50f..b1411fa 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -2367,11 +2367,6 @@ def create_wide_dataset( data_table = Table( data_table_name(dataset_id), metadata, Column("__case_ordinal", BigInteger, primary_key=True, nullable=False), - *(Column( - item["physical_name"], - binary64_type() if item["storage_kind"] == "numeric" else Text, - nullable=item["storage_kind"] == "numeric", - ) for item in variables), ) audit_relations = { fidelity_event_catalog.name, operation_catalog.name, @@ -2400,6 +2395,12 @@ def create_wide_dataset( case_weight_variable=case_weight_variable, multiple_response_sets=multiple_response_sets, ) + for item in variables: + data_table.append_column(Column( + item["physical_name"], + binary64_type() if item["storage_kind"] == "numeric" else Text, + nullable=item["storage_kind"] == "numeric", + )) except Exception as error: try: _record_failed_preflight( From df1888e8dff90c498b1a9fee0cf2e477cee919a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:59:10 +0300 Subject: [PATCH 033/119] Distinguish Dolt profile id from product label [skip ci] --- tests/test_sql_profiles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 987cb24..0265698 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -280,7 +280,8 @@ def test_dolt_identity_is_exact_and_publishes_wire_and_product_versions(monkeypa active = active_connection("mysql+pymysql://user@host/database") assert active["dialect"] == "mysql" - assert active["profile"] == active["product"] == "dolt" + assert active["profile"] == "dolt" + assert active["product"] == "Dolt" assert active["raw_wire_version"] == "8.0.31" assert active["raw_product_version"] == active["raw_server_version"] == "2.2.3" assert active["raw_version_comment"] == "dOlT" From 31d0e538fe4fd9588157e646ff13db5dd2ad8f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:59:15 +0300 Subject: [PATCH 034/119] Make mocked SAV writer honor staging contract [skip ci] --- tests/test_loss_reports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_loss_reports.py b/tests/test_loss_reports.py index 48f97e5..2e4100f 100755 --- a/tests/test_loss_reports.py +++ b/tests/test_loss_reports.py @@ -104,6 +104,7 @@ def test_explicit_legacy_locale_selects_the_single_engine_route(tmp_path, monkey observed = {} def writer(destination_path, frame, dataset, variables, *, legacy_locale=None): + destination_path.touch() observed["locale"] = legacy_locale observed["encoding"] = dataset["source_encoding"] observed["values"] = frame["name"].tolist() From 06817f6b1f5ed5082508f687c0f77690ab3cae96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 18:59:19 +0300 Subject: [PATCH 035/119] Align atomic-import tests with strict preflight [skip ci] --- tests/test_atomic_import.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_atomic_import.py b/tests/test_atomic_import.py index 93df530..59fae18 100755 --- a/tests/test_atomic_import.py +++ b/tests/test_atomic_import.py @@ -2,14 +2,12 @@ from dataclasses import replace import pytest -from sqlalchemy.exc import IntegrityError - import openstatspec.sql.wide as wide -from openstatspec.sql.profiles import MYSQL, SQLITE +from openstatspec.sql.profiles import MYSQL, SQLITE, TargetCapabilityExceededError from openstatspec.sql.wide import create_wide_dataset -def test_failed_row_insert_leaves_no_catalog_or_data_table(tmp_path) -> None: +def test_invalid_string_row_is_rejected_before_dataset_or_data_table(tmp_path) -> None: database_path = tmp_path / "dataset.sqlite" database = f"sqlite:///{database_path}" wide.initialize_wide_catalog(database_url=database) @@ -20,7 +18,7 @@ def test_failed_row_insert_leaves_no_catalog_or_data_table(tmp_path) -> None: "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", }] - with pytest.raises(IntegrityError): + with pytest.raises(TargetCapabilityExceededError, match="Target capability exceeded"): create_wide_dataset( database_url=database, dataset_id="broken", source_name="fixture.sav", source_format="SAV", rows=[{"name": None}], variables=variables, From bff1605ddf40075ddef6e79fdd473a722bdb3a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:01:14 +0300 Subject: [PATCH 036/119] Keep Dolt out of operational conformance matrices [skip ci] --- tests/test_official_conformance.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_official_conformance.py b/tests/test_official_conformance.py index 7dc6b6f..57c2d90 100644 --- a/tests/test_official_conformance.py +++ b/tests/test_official_conformance.py @@ -277,7 +277,6 @@ def test_official_manifest_round_trips_through_sqlite( ("OPENSTATSPEC_POSTGRES_URL", "postgresql"), ("OPENSTATSPEC_MYSQL_URL", "mysql"), ("OPENSTATSPEC_MARIADB_URL", "mariadb"), - ("OPENSTATSPEC_DOLT_URL", "dolt"), ], ) @pytest.mark.parametrize(("fixture", "source"), _round_trip_fixtures()) @@ -361,7 +360,6 @@ def test_official_preflight_failure_is_atomic_and_diagnostic(tmp_path: Path) -> ("OPENSTATSPEC_POSTGRES_URL", "postgresql"), ("OPENSTATSPEC_MYSQL_URL", "mysql"), ("OPENSTATSPEC_MARIADB_URL", "mariadb"), - ("OPENSTATSPEC_DOLT_URL", "dolt"), ], ) def test_official_preflight_failure_through_server_profiles( From c6a711f347de5fdf3d00a145073d64c0de1c1153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:01:19 +0300 Subject: [PATCH 037/119] Make Dolt CI assert fail-closed behavior [skip ci] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2730de6..79bd97d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,7 +146,7 @@ jobs: steps: *sql-test-steps dolt-integration: - name: Dolt ${{ matrix.dolt }} integration + name: Dolt ${{ matrix.dolt }} read-only / fail-closed integration runs-on: ubuntu-latest strategy: fail-fast: false @@ -221,7 +221,7 @@ jobs: connection.close() break PY - - run: python -m pytest -m services + - run: python -m pytest -m "services and not candidate_evidence" - name: Show Dolt logs on failure and stop server if: always() run: | From 3fb04c3cc55c82258e257aa86fef228e048a3d91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:01:44 +0300 Subject: [PATCH 038/119] Separate candidate evidence from release gates [skip ci] --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9309c07..87890d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,10 @@ openstatspec = "openstatspec.cli:main" [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-q" -markers = ["services: requires a configured PostgreSQL, MySQL, MariaDB, or Dolt service"] +markers = [ + "services: requires a configured PostgreSQL, MySQL, MariaDB, or Dolt service", + "candidate_evidence: non-claiming manual probe excluded from release gates", +] [tool.hatch.build.targets.wheel] packages = ["src/openstatspec"] From 40595da721c8a3b7ef4e3c3054f261607f2331b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:02:08 +0300 Subject: [PATCH 039/119] Test undeclared Dolt support as read-only and fail-closed [skip ci] --- tests/test_sql_services.py | 181 ++++++------------------------------- 1 file changed, 28 insertions(+), 153 deletions(-) diff --git a/tests/test_sql_services.py b/tests/test_sql_services.py index 301bfbd..fa07a21 100755 --- a/tests/test_sql_services.py +++ b/tests/test_sql_services.py @@ -1,4 +1,4 @@ -"""Real-service conformance checks for PostgreSQL, MySQL, MariaDB, and Dolt.""" +"""Real-service conformance checks plus Dolt fail-closed and candidate probes.""" import os from uuid import uuid4 @@ -10,8 +10,8 @@ from sqlalchemy.exc import DBAPIError import openstatspec -import openstatspec.sql.wide as wide from openstatspec.core import UnsupportedOperationError +from openstatspec.sql.dolt_conformance import DoltConformanceSource from conformance import compare_sav_semantics, write_supported_semantics_fixture @@ -37,7 +37,7 @@ def source_sav(tmp_path): @pytest.mark.parametrize( ("environment_name", "dataset_id"), - [("OPENSTATSPEC_POSTGRES_URL", "profile_pg"), ("OPENSTATSPEC_MYSQL_URL", "profile_mysql"), ("OPENSTATSPEC_MARIADB_URL", "profile_mariadb"), ("OPENSTATSPEC_DOLT_URL", "profile_dolt")], + [("OPENSTATSPEC_POSTGRES_URL", "profile_pg"), ("OPENSTATSPEC_MYSQL_URL", "profile_mysql"), ("OPENSTATSPEC_MARIADB_URL", "profile_mariadb")], ) def test_live_profile_import_validate_and_export(environment_name, dataset_id, source_sav, tmp_path): database_url = os.environ.get(environment_name) @@ -70,7 +70,7 @@ def test_live_profile_import_validate_and_export(environment_name, dataset_id, s @pytest.mark.parametrize( ("environment_name", "dataset_id"), - [("OPENSTATSPEC_POSTGRES_URL", "semantics_pg"), ("OPENSTATSPEC_MYSQL_URL", "semantics_mysql"), ("OPENSTATSPEC_MARIADB_URL", "semantics_mariadb"), ("OPENSTATSPEC_DOLT_URL", "semantics_dolt")], + [("OPENSTATSPEC_POSTGRES_URL", "semantics_pg"), ("OPENSTATSPEC_MYSQL_URL", "semantics_mysql"), ("OPENSTATSPEC_MARIADB_URL", "semantics_mariadb")], ) @pytest.mark.parametrize("suffix", [".sav", ".zsav"]) def test_live_profile_preserves_supported_sav_semantics(environment_name, dataset_id, suffix, tmp_path): @@ -95,168 +95,43 @@ def test_live_profile_preserves_supported_sav_semantics(environment_name, datase ) assert compare_sav_semantics(source, destination) == {"equivalent": True, "differences": []} -def test_live_dolt_conservative_source_width_envelope(tmp_path) -> None: +def test_live_dolt_is_read_only_and_rejects_writes_without_declarations( + tmp_path, +) -> None: database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") if not database_url: pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") - openstatspec.initialize_catalog(database_url=database_url) - token = uuid4().hex[:8] - accepted_id = f"dolt_width_accepted_{token}" - rejected_id = f"dolt_width_rejected_{token}" - accepted_source = tmp_path / f"{accepted_id}.sav" - rejected_source = tmp_path / f"{rejected_id}.sav" - accepted_columns = [f"v{ordinal:03d}" for ordinal in range(1, 306)] - rejected_columns = [*accepted_columns, "v306"] - pyspssio.write_sav( - str(accepted_source), - pd.DataFrame([[float(ordinal) for ordinal in range(1, 306)]], - columns=accepted_columns), - ) - pyspssio.write_sav( - str(rejected_source), - pd.DataFrame([[float(ordinal) for ordinal in range(1, 307)]], - columns=rejected_columns), - ) - - imported = openstatspec.import_sav( - accepted_source, database_url=database_url, dataset_id=accepted_id, - ) - assert imported["case_count"] == 1 - assert openstatspec.validate( - database_url=database_url, dataset_id=accepted_id, - )["variable_count"] == 305 - with pytest.raises(UnsupportedOperationError, match="Target capability exceeded"): - openstatspec.import_sav( - rejected_source, database_url=database_url, dataset_id=rejected_id, - ) + status = DoltConformanceSource.packaged().status() + assert status["status"] == "blocked_no_concrete_declarations" + assert status["declaration_count"] == 0 + assert status["write_enabled"] is False - engine = create_engine(database_url) - with engine.connect() as connection: - assert connection.execute(text( - "select count(*) from dataset where dataset_name = :name" - ), {"name": rejected_id}).scalar_one() == 0 - assert connection.execute(text( - "select count(*) from dataset_catalog where dataset_id = :name" - ), {"name": rejected_id}).scalar_one() == 0 - mirror_event = connection.execute(text(""" - select f.dataset_id, f.direction, f.severity, f.code - from fidelity_event_catalog f - join operation_catalog o on o.operation_id = f.operation_id - where o.source = :source - """), {"source": rejected_source.name}).mappings().one() - normative_event = connection.execute(text(""" - select f.dataset_id, f.direction, f.severity, f.event_code - from fidelity_event f - where f.source_item = :source - """), {"source": rejected_source.name}).mappings().one() - assert tuple(mirror_event.values()) == ( - None, "import", "error", "target_capability_exceeded", - ) - assert tuple(normative_event.values()) == ( - None, "import", "error", "target_capability_exceeded", - ) - assert f"data_{rejected_id}" not in inspect_database(engine).get_table_names() + before = openstatspec.dolt_state_snapshot(database_url=database_url) + assert before["read_only"] is True + assert before["operational_write_enabled"] is False + rejection = "no concrete declarations; write rejected before mutation" + with pytest.raises(UnsupportedOperationError, match=rejection): + openstatspec.initialize_catalog(database_url=database_url) + after_initialize = openstatspec.dolt_state_snapshot(database_url=database_url) -def test_live_dolt_post_ddl_fault_has_complete_compensating_cleanup( - tmp_path, monkeypatch, -) -> None: - database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") - if not database_url: - pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") - openstatspec.initialize_catalog(database_url=database_url) - dataset_id = f"dolt_cleanup_{uuid4().hex[:8]}" - source = tmp_path / f"{dataset_id}.sav" + source = tmp_path / "blocked-write.sav" pyspssio.write_sav(str(source), pd.DataFrame({"answer": [1.0]})) - real_store = wide.store_normative_dataset - - def fail_after_normative_write(*args, **kwargs): - real_store(*args, **kwargs) - raise RuntimeError("injected Dolt post-DDL fault") - - monkeypatch.setattr(wide, "store_normative_dataset", fail_after_normative_write) - - with pytest.raises(RuntimeError, match="injected Dolt post-DDL fault"): + with pytest.raises(UnsupportedOperationError, match=rejection): openstatspec.import_sav( - source, database_url=database_url, dataset_id=dataset_id, + source, database_url=database_url, dataset_id="blocked_write", ) + after_import = openstatspec.dolt_state_snapshot(database_url=database_url) - engine = create_engine(database_url) - with engine.connect() as connection: - assert connection.execute(text( - "select count(*) from dataset where dataset_name = :name" - ), {"name": dataset_id}).scalar_one() == 0 - assert connection.execute(text( - "select count(*) from dataset_catalog where dataset_id = :name" - ), {"name": dataset_id}).scalar_one() == 0 - assert connection.execute(text( - "select status, dataset_id from operation_catalog where source = :source" - ), {"source": source.name}).one() == ("failed", None) - assert connection.execute(text(""" - select f.dataset_id, f.direction, f.severity, f.code - from fidelity_event_catalog f - join operation_catalog o on o.operation_id = f.operation_id - where o.source = :source - """), {"source": source.name}).one() == ( - None, "import", "error", "import_failed", - ) - assert f"data_{dataset_id}" not in inspect_database(engine).get_table_names() - - -def test_live_dolt_adapter_value_boundary_is_atomic() -> None: - database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") - if not database_url: - pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") - openstatspec.initialize_catalog(database_url=database_url) - token = uuid4().hex[:8] - accepted_id = f"dolt_value_accepted_{token}" - rejected_id = f"dolt_value_rejected_{token}" - accepted_value = "é" * 32_752 - rejected_value = accepted_value + "x" - assert len(accepted_value.encode("utf-8")) == 65_504 - assert len(rejected_value.encode("utf-8")) == 65_505 - variables = [{ - "ordinal": 1, "source_name": "value", "physical_name": "value", - "storage_kind": "string", "string_width": 65_504, "label": "", - "format": "A65504", "measure": "nominal", "alignment": "left", - "display_width": 8, "value_labels": "{}", "missing_ranges": "[]", - }] - - imported = wide.create_wide_dataset( - database_url=database_url, dataset_id=accepted_id, - source_name="accepted.sav", source_format="SAV", - rows=[{"value": accepted_value}], variables=variables, - ) - assert imported["case_count"] == 1 - - with pytest.raises(UnsupportedOperationError) as caught: - wide.create_wide_dataset( - database_url=database_url, dataset_id=rejected_id, - source_name="rejected.sav", source_format="SAV", - rows=[{"value": rejected_value}], variables=variables, - ) - assert caught.value.details["reason"] == "text_value_limit" - - engine = create_engine(database_url) - accepted_table = wide.data_table_name(accepted_id) - rejected_table = wide.data_table_name(rejected_id) - quote = engine.dialect.identifier_preparer.quote - with engine.connect() as connection: - assert connection.execute(text( - f"SELECT OCTET_LENGTH(value) FROM {quote(accepted_table)}" - )).scalar_one() == 65_504 - assert connection.execute(text( - "SELECT COUNT(*) FROM dataset_catalog WHERE dataset_id = :dataset_id" - ), {"dataset_id": rejected_id}).scalar_one() == 0 - assert connection.execute(text( - "SELECT COUNT(*) FROM dataset WHERE dataset_name = :dataset_id" - ), {"dataset_id": rejected_id}).scalar_one() == 0 - assert rejected_table not in inspect_database(engine).get_table_names() - engine.dispose() + assert after_initialize["working_set_binding"] == before["working_set_binding"] + assert after_import["working_set_binding"] == before["working_set_binding"] + assert after_initialize["state"] == before["state"] + assert after_import["state"] == before["state"] -def test_live_dolt_published_storage_and_identifier_evidence() -> None: +@pytest.mark.candidate_evidence +def test_live_dolt_candidate_limit_probe_smoke() -> None: database_url = os.environ.get("OPENSTATSPEC_DOLT_URL") if not database_url: pytest.skip("OPENSTATSPEC_DOLT_URL is not configured") From 606f3aa474aed0ecc0a472bccf73172d1e5dde6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:02:32 +0300 Subject: [PATCH 040/119] Verify SQL Decimal normalization preserves nonfinite rejection --- tests/test_catalog_lifecycle.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 733126b..8e35664 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -7,6 +7,7 @@ import openstatspec import openstatspec.sql.wide as wide from openstatspec.core import UnsupportedOperationError +from openstatspec.sql.profiles import MYSQL, TargetCapabilityExceededError, preflight from openstatspec.sql.wide import ( ImportRecoveryError, _bounded_batches, @@ -237,6 +238,24 @@ def test_database_decimal_numeric_wrappers_are_restored_to_binary64(): assert isinstance(rows[0]["score"], float) +@pytest.mark.parametrize( + "value", [Decimal("NaN"), Decimal("Infinity"), Decimal("-Infinity")], +) +def test_database_decimal_nonfinite_wrappers_remain_rejected(value): + variables = [{ + "ordinal": 1, + "source_name": "score", + "physical_name": "score", + "storage_kind": "numeric", + }] + rows = wide._canonicalize_database_numeric_rows([{"score": value}], variables) + + with pytest.raises(TargetCapabilityExceededError) as error: + preflight(MYSQL, variables, rows=rows) + + assert error.value.details["reason"] == "nonfinite_numeric_value" + + def test_bounded_batches_never_exceed_statement_payload_limit(): variables = _variables() rows = [ From 358baef5fb18acb02c3ba3ee53045f48007110a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:09:38 +0300 Subject: [PATCH 041/119] Pin specification validator package 0.2.1 source [skip ci] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79bd97d..1488731 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v7 with: &specification-checkout repository: OpenStatSpec/specification - ref: 28bd6817261d1e55e51bb25076fb93e732ab1c74 + ref: e707636180c3aafd64cc60aa274549212d2c05a7 path: openstatspec-specification - name: Checkout required SPSS engine uses: actions/checkout@v7 From 73813c83013bdb29a49efd4e8e9d62158d3e64af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:09:45 +0300 Subject: [PATCH 042/119] Require specification validator package 0.2.1 [skip ci] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 87890d2..2f44c93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.11" license = {text = "Apache-2.0"} authors = [{name = "OpenStatSpec"}] -dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "openstatspec-specification==0.2.0", "sqlglot>=30.14", "rfc8785>=0.1.4"] +dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "openstatspec-pyspssio==0.5.1.post2", "openstatspec-specification==0.2.1", "sqlglot>=30.14", "rfc8785>=0.1.4"] [project.optional-dependencies] dev = ["pytest>=8"] From 10906aae274571349eb7c55a7d29a994d916bd65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:09:50 +0300 Subject: [PATCH 043/119] Bind Dolt lifecycle to specification validator 0.2.1 commit [skip ci] --- src/openstatspec/sql/capabilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index 6cf339f..dc42fc8 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -18,7 +18,7 @@ # Release/build automation must bind this to the exact commit used to build # openstatspec-specification. An uncommitted source tree has no truthful pin. -SPECIFICATION_COMMIT = "28bd6817261d1e55e51bb25076fb93e732ab1c74" +SPECIFICATION_COMMIT = "e707636180c3aafd64cc60aa274549212d2c05a7" SPECIFICATION_RELEASE = "v0.2.0" DOLT_WRITE_CONFORMANCE = { From 3e06a94a81cb7dfeeea04da5dfef345c473b7355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:09:56 +0300 Subject: [PATCH 044/119] Update CLI specification provenance expectation [skip ci] --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 12432c2..92534d8 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -44,7 +44,7 @@ def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> matrix = openstatspec.capability_matrix() assert matrix["specification_status"] == "released" assert matrix["specification_release"] == "v0.2.0" - assert matrix["specification_commit"] == "28bd6817261d1e55e51bb25076fb93e732ab1c74" + assert matrix["specification_commit"] == "e707636180c3aafd64cc60aa274549212d2c05a7" assert matrix["directions"] == ["import", "export", "semantic_round_trip"] assert matrix["active_connection"] is None assert matrix["engine"]["package"] == "openstatspec-pyspssio" From 63a52ecdab94f8ac15aed90f39d2b04365e4a27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 19:10:01 +0300 Subject: [PATCH 045/119] Verify final specification validator provenance --- tests/test_sql_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 0265698..40601e9 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -31,7 +31,7 @@ def test_profile_declarations_publish_released_specification_provenance() -> Non assert declaration["specification_release"] == "v0.2.0" assert ( declaration["specification_commit"] - == "28bd6817261d1e55e51bb25076fb93e732ab1c74" + == "e707636180c3aafd64cc60aa274549212d2c05a7" ) def test_profile_preflight_fails_without_transforming_a_wide_dataset() -> None: From dd680a1228862d909d305a7129e7d72d8c4ec436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:00:56 +0300 Subject: [PATCH 046/119] ci: pin reviewed specification candidate --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1488731..c96cd3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v7 with: &specification-checkout repository: OpenStatSpec/specification - ref: e707636180c3aafd64cc60aa274549212d2c05a7 + ref: 5b62bce1d2f4d719ac6ca42d73f07e7a127c7093 path: openstatspec-specification - name: Checkout required SPSS engine uses: actions/checkout@v7 From 359e3908fb3fbb6c7f96299b74e117791a786151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:01:31 +0300 Subject: [PATCH 047/119] fix: align specification release provenance --- src/openstatspec/sql/capabilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index dc42fc8..496b584 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -18,8 +18,8 @@ # Release/build automation must bind this to the exact commit used to build # openstatspec-specification. An uncommitted source tree has no truthful pin. -SPECIFICATION_COMMIT = "e707636180c3aafd64cc60aa274549212d2c05a7" -SPECIFICATION_RELEASE = "v0.2.0" +SPECIFICATION_COMMIT = "5b62bce1d2f4d719ac6ca42d73f07e7a127c7093" +SPECIFICATION_RELEASE = "v0.2.1" DOLT_WRITE_CONFORMANCE = { "declaration_schema_id": "openstatspec-dolt-adapter-declaration-v1", From 647d9decbae5d1a31b026bfad033c7af8c45627d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:01:38 +0300 Subject: [PATCH 048/119] fix: align specification release provenance --- tests/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 92534d8..42b19d8 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -43,8 +43,8 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> None: matrix = openstatspec.capability_matrix() assert matrix["specification_status"] == "released" - assert matrix["specification_release"] == "v0.2.0" - assert matrix["specification_commit"] == "e707636180c3aafd64cc60aa274549212d2c05a7" + assert matrix["specification_release"] == "v0.2.1" + assert matrix["specification_commit"] == "5b62bce1d2f4d719ac6ca42d73f07e7a127c7093" assert matrix["directions"] == ["import", "export", "semantic_round_trip"] assert matrix["active_connection"] is None assert matrix["engine"]["package"] == "openstatspec-pyspssio" From 3cf7dd0a1e79ed6623c7cc6f382adcfa12d0e071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:01:44 +0300 Subject: [PATCH 049/119] fix: align specification release provenance --- tests/test_sql_profiles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 40601e9..26de93e 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -28,10 +28,10 @@ def test_profile_detection_tracks_supported_dialect_urls() -> None: def test_profile_declarations_publish_released_specification_provenance() -> None: for declaration in capabilities.profile_declarations().values(): assert declaration["specification_status"] == "released" - assert declaration["specification_release"] == "v0.2.0" + assert declaration["specification_release"] == "v0.2.1" assert ( declaration["specification_commit"] - == "e707636180c3aafd64cc60aa274549212d2c05a7" + == "5b62bce1d2f4d719ac6ca42d73f07e7a127c7093" ) def test_profile_preflight_fails_without_transforming_a_wide_dataset() -> None: From cb4829de7a0cce5911c3be3bb5675cd8b0028b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:05:13 +0300 Subject: [PATCH 050/119] Enforce declared limits and explicit drivers --- src/openstatspec/sql/capabilities.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index 496b584..7d45ff3 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -13,7 +13,7 @@ from .dolt_conformance import DoltConformanceSource, effective_limits as dolt_effective_limits from .normative import catalog from .profiles import DOLT, MYSQL, POSTGRESQL, SQLITE, MYSQL_WIRE_PROFILES, SqlProfile -from .profiles import profile_for_url +from .profiles import validate_connection_url from ..core import UnsupportedOperationError # Release/build automation must bind this to the exact commit used to build @@ -256,7 +256,7 @@ def effective_profile( ) -> tuple[SqlProfile, dict[str, Any]]: """Resolve and enforce the profile used by import preflight.""" source = _conformance_source(dolt_conformance_source) - configured = profile_for_url(database_url) + configured = validate_connection_url(database_url) active = active_connection(database_url, dolt_conformance_source=source) if configured is not MYSQL and active["profile"] != configured.name: raise UnsupportedOperationError("The active SQL server does not match the configured profile.") @@ -288,6 +288,7 @@ def effective_profile( return replace( configured, name=active["profile"], + identifier_limit=int(limits["identifier_limit"]["value"]), max_source_variables=int(limits["maximum_source_variables"]), max_text_value_bytes=int(limits["maximum_value_bytes"]), max_row_bytes=( From 3ee9fba1564c210956054dae14f04cb29d7a5760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:05:20 +0300 Subject: [PATCH 051/119] Use effective string storage in wide tables --- src/openstatspec/sql/wide.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index b1411fa..95b6d23 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -61,6 +61,11 @@ def string_type(profile: Any) -> Text: return mysql.LONGTEXT() if profile.name == "dolt" else Text() +def _wide_column_type(profile: Any, storage_kind: str) -> Any: + """Select the physical value type from the effective SQL profile.""" + return binary64_type() if storage_kind == "numeric" else string_type(profile) + + def _canonical_sha256(value: Any) -> str: return hashlib.sha256( json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") @@ -2398,7 +2403,7 @@ def create_wide_dataset( for item in variables: data_table.append_column(Column( item["physical_name"], - binary64_type() if item["storage_kind"] == "numeric" else Text, + _wide_column_type(profile, item["storage_kind"]), nullable=item["storage_kind"] == "numeric", )) except Exception as error: From c9ff69967b989bec85caca5d76007c8148bc78ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:05:26 +0300 Subject: [PATCH 052/119] Gate schema installation before DDL --- src/openstatspec/sql/inplace_transform.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/openstatspec/sql/inplace_transform.py b/src/openstatspec/sql/inplace_transform.py index e763a02..6e1b556 100644 --- a/src/openstatspec/sql/inplace_transform.py +++ b/src/openstatspec/sql/inplace_transform.py @@ -556,6 +556,9 @@ def _dolt_state(connection: Any) -> tuple[str, str, int]: def install_in_place_transformation_schema(*, database_url: str) -> None: """Install the compact operation audit separately from any data apply.""" + # Resolve the effective profile first so Dolt conformance and explicit + # driver checks fail closed before an engine transaction can execute DDL. + effective_profile(database_url) engine = create_engine(database_url) try: with engine.begin() as connection: From b9241b0a0b787d2879baf974f3d78ea70e66a2ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:05:52 +0300 Subject: [PATCH 053/119] Cover effective profile driver, identifier, and string storage --- tests/test_sql_profiles.py | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 26de93e..4b2a385 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -489,3 +489,59 @@ def fail_identity(_url, **_kwargs): ) assert not destination.exists() + +def test_effective_profile_preserves_explicit_driver_validation(monkeypatch) -> None: + def unexpected_active_connection(*_args, **_kwargs): + raise AssertionError("active connection must not be probed for a rejected driver") + + monkeypatch.setattr( + capabilities, "active_connection", unexpected_active_connection, + ) + + with pytest.raises( + UnsupportedOperationError, + match=r"PostgreSQL requires an explicit postgresql\+psycopg URL", + ): + effective_profile("postgresql+psycopg2://user@host/database") + + +def test_effective_profile_applies_declared_identifier_limit(monkeypatch) -> None: + active = { + "profile": "postgresql", + "claimed_supported": True, + "server_version": "17.10", + "observed": {}, + } + limits = { + "identifier_limit": {"value": 7}, + "maximum_source_variables": 10, + "maximum_value_bytes": 100, + "maximum_row_bytes": 200, + "maximum_statement_bytes": 300, + } + monkeypatch.setattr( + capabilities, "active_connection", lambda _url, **_kwargs: active, + ) + monkeypatch.setattr( + capabilities, "_profile", lambda *_args, **_kwargs: { + "effective_limits": limits, + }, + ) + + profile, observed = effective_profile( + "postgresql+psycopg://user@host/database", + ) + + assert observed is active + assert profile.identifier_limit == 7 + + +def test_wide_string_column_uses_effective_dolt_storage() -> None: + table = Table( + "dolt_wide_text", + MetaData(), + Column("value", wide._wide_column_type(DOLT, "string")), + ) + + ddl = str(CreateTable(table).compile(dialect=mysql.dialect())).upper() + assert "VALUE LONGTEXT" in ddl From 0a7495e1dc0cee691157e4f5b232b719ff7eb895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:05:57 +0300 Subject: [PATCH 054/119] Cover fail-closed schema installation gate --- tests/test_inplace_transform.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 869e189..38bdabd 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -508,3 +508,23 @@ def test_capability_declares_dolt_owned_versioning() -> None: assert declaration["creates_persistent_data_copy"] is False assert declaration["openstatspec_rollback_or_version_history"] is False assert declaration["performs_dolt_commit"] is False + +def test_schema_install_fails_before_engine_or_ddl_when_profile_is_rejected( + monkeypatch, +) -> None: + def reject_profile(_database_url): + raise openstatspec.UnsupportedOperationError("Dolt declaration mismatch") + + def unexpected_engine(_database_url): + raise AssertionError("engine creation would permit DDL") + + monkeypatch.setattr(inplace_transform, "effective_profile", reject_profile) + monkeypatch.setattr(inplace_transform, "create_engine", unexpected_engine) + + with pytest.raises( + openstatspec.UnsupportedOperationError, + match="Dolt declaration mismatch", + ): + inplace_transform.install_in_place_transformation_schema( + database_url="mysql+pymysql://user@host/database", + ) From 9210339d7f20ada5578d70a8fbc0210b87cb24d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:08:51 +0300 Subject: [PATCH 055/119] Report unbound Dolt capabilities as disabled --- src/openstatspec/sql/capabilities.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index 7d45ff3..f67a64e 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -111,10 +111,16 @@ def profile_declarations( ) dolt_declaration = None if active and active["profile"] == "dolt": - dolt_declaration = source.require_exact_match( - active_product_version=active["raw_product_version"], - specification_commit=_bound_specification_commit(), - ) + try: + dolt_declaration = source.require_exact_match( + active_product_version=active["raw_product_version"], + specification_commit=_bound_specification_commit(), + ) + except UnsupportedOperationError: + # Capability inspection is descriptive: preserve the active Dolt + # identity while reporting writes as blocked when no exact + # declaration can be selected. + dolt_declaration = None return { "sqlite": _profile("sqlite", SQLITE, active, source), "mysql": _profile("mysql", MYSQL, active, source), From 0543648daad5a26d6e5a1a2a1544f2927d6f1daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:08:59 +0300 Subject: [PATCH 056/119] Allow conformance-gated Dolt export audits --- src/openstatspec/sql/wide.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 95b6d23..b98787d 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -284,11 +284,6 @@ def _bound_catalog_transaction( audit_relations=audit_relations, ) _require_dolt_working_set_binding(before, active, phase=f"{phase} preflight") - if profile_name == "dolt": - raise UnsupportedOperationError( - "Dolt audit-only mutation is blocked until operation-scoped " - "append/update and immutable-history evidence is implemented." - ) connection.rollback() with connection.begin(): yield connection From 3b678866eff3c897fe5a38a87b10d1f88d73ad17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:09:22 +0300 Subject: [PATCH 057/119] Cover disabled Dolt capabilities and gated audit --- tests/test_sql_profiles.py | 72 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 4b2a385..0ee2744 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -545,3 +545,75 @@ def test_wide_string_column_uses_effective_dolt_storage() -> None: ddl = str(CreateTable(table).compile(dialect=mysql.dialect())).upper() assert "VALUE LONGTEXT" in ddl + +def test_capability_inspection_reports_unbound_dolt_as_disabled(monkeypatch) -> None: + active = { + "profile": "dolt", + "raw_product_version": "2.2.3", + "observed": {"max_allowed_packet": 1_073_741_824}, + } + monkeypatch.setattr( + capabilities, "active_connection", lambda _url, **_kwargs: active, + ) + + declaration = capabilities.profile_declarations( + "mysql+pymysql://user@host/database", + )["dolt"] + + assert declaration["operational_write_enabled"] is False + assert declaration["effective_limits"] is None + assert declaration["effective_limits_status"] == ( + "blocked_pending_pinned_live_conformance" + ) + + +def test_bound_catalog_transaction_allows_gated_dolt_audit(monkeypatch) -> None: + class TransactionConnection: + def __init__(self): + self.rollback_called = False + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def rollback(self): + self.rollback_called = True + + def begin(self): + return self + + class TransactionEngine: + def __init__(self, connection): + self.connection = connection + + def connect(self): + return self.connection + + connection = TransactionConnection() + monkeypatch.setattr( + wide, "_capture_dolt_state", lambda *_args, **_kwargs: {"profile": "dolt"}, + ) + monkeypatch.setattr( + wide, "_require_dolt_working_set_binding", lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + wide, "_require_dolt_success_identity", lambda *_args, **_kwargs: None, + ) + monkeypatch.setattr( + wide, + "_dolt_failure_boundary_evidence", + lambda *_args, **_kwargs: {"applicable": False}, + ) + + with wide._bound_catalog_transaction( + engine=TransactionEngine(connection), + profile_name="dolt", + active={"profile": "dolt"}, + audit_relations={"operation"}, + phase="test export audit", + ) as yielded: + assert yielded is connection + + assert connection.rollback_called is True From 2aa076b765061577e1632c05f99e2d44a4d4ebd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:18:29 +0300 Subject: [PATCH 058/119] Require the claimed driver after Dolt identity --- src/openstatspec/sql/capabilities.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index f67a64e..a8353f1 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -268,6 +268,13 @@ def effective_profile( raise UnsupportedOperationError("The active SQL server does not match the configured profile.") if configured is MYSQL and active["profile"] not in MYSQL_WIRE_PROFILES: raise UnsupportedOperationError("The active SQL server is not a claimed MySQL-wire product.") + if ( + active["profile"] == "dolt" + and make_url(database_url).drivername.lower() != "mysql+pymysql" + ): + raise UnsupportedOperationError( + "Dolt requires an explicit mysql+pymysql URL." + ) dolt_declaration = None if active["profile"] == "dolt": dolt_declaration = source.require_exact_match( From cd09b628f20c57dfa57445eb37e71ae342f2acbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:18:36 +0300 Subject: [PATCH 059/119] Preserve dangling symlinks during export recovery --- src/openstatspec/spss/sav.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index d96a399..bc0613a 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -305,6 +305,11 @@ def _path_reference_text(path: Path, *, role: str) -> str: ) +def _path_entry_exists(path: Path) -> bool: + """Report directory entries without following a possibly dangling symlink.""" + return path.exists() or path.is_symlink() + + def _reserve_export_backup(destination: Path) -> Path: descriptor, name = mkstemp( dir=destination.parent, prefix=f".{destination.name}.", @@ -579,7 +584,7 @@ def export_sav_dataset( staged_destination, frame, dataset, variables, legacy_locale=legacy_locale, ) - had_previous = destination_path.exists() + had_previous = _path_entry_exists(destination_path) backup = _reserve_export_backup(destination_path) if not had_previous: backup.unlink() From ef0e310639814a5aaf0adbe64276ca1da3e34e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:19:06 +0300 Subject: [PATCH 060/119] Cover Dolt driver recheck after identity --- tests/test_sql_profiles.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 0ee2744..32d5b6b 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -617,3 +617,20 @@ def connect(self): assert yielded is connection assert connection.rollback_called is True + +def test_effective_profile_rechecks_driver_after_dolt_identity(monkeypatch) -> None: + active = { + "profile": "dolt", + "raw_product_version": "2.2.3", + } + monkeypatch.setattr( + capabilities, "active_connection", lambda _url, **_kwargs: active, + ) + + with pytest.raises( + UnsupportedOperationError, + match=r"Dolt requires an explicit mysql\+pymysql URL", + ): + effective_profile( + "mariadb+mariadbconnector://user@host/database", + ) From 5d4e46cadc8e4b8bfba53fb128463406159a6d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:19:11 +0300 Subject: [PATCH 061/119] Cover dangling symlink export recovery --- tests/test_sav_sqlite.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index 1e8a056..c4dcbfd 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -7,6 +7,7 @@ import pytest import openstatspec +import openstatspec.spss.sav as sav_module from conformance import compare_sav_semantics, write_supported_semantics_fixture from openstatspec.core import UnsupportedOperationError @@ -222,3 +223,21 @@ def test_very_long_string_round_trips_through_sqlite_and_export(tmp_path, suffix frame, metadata = pyspssio.read_sav(str(destination), convert_datetimes=False) assert metadata["var_types"]["comment"] == payload_width assert frame["comment"].tolist() == [payload, "short"] + +def test_export_recovery_preserves_dangling_destination_symlink(tmp_path) -> None: + missing_target = tmp_path / "missing-target.sav" + destination = tmp_path / "destination.sav" + backup = tmp_path / "destination.previous" + destination.symlink_to(missing_target) + + assert sav_module._path_entry_exists(destination) is True + destination.replace(backup) + destination.write_bytes(b"staged export") + sav_module._restore_export_destination( + destination=destination, + backup=backup, + had_previous=True, + ) + + assert destination.is_symlink() + assert destination.readlink() == missing_target From 795d8de0dc42e771346aa1cbc66c669ac2bcefa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:30:20 +0300 Subject: [PATCH 062/119] Include driver eligibility in Dolt write capability --- src/openstatspec/sql/capabilities.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index a8353f1..351c0cf 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -59,6 +59,10 @@ def _bound_specification_commit() -> str: return SPECIFICATION_COMMIT +def _dolt_driver_eligible(database_url: str) -> bool: + return make_url(database_url).drivername.lower() == "mysql+pymysql" + + def _dolt_write_enabled( source: DoltConformanceSource | None = None, *, @@ -235,6 +239,10 @@ def active_connection( if profile_name == "dolt" else None ), "observed": observed, + "driver_eligible": ( + _dolt_driver_eligible(database_url) + if profile_name == "dolt" else True + ), } @@ -270,7 +278,7 @@ def effective_profile( raise UnsupportedOperationError("The active SQL server is not a claimed MySQL-wire product.") if ( active["profile"] == "dolt" - and make_url(database_url).drivername.lower() != "mysql+pymysql" + and not active["driver_eligible"] ): raise UnsupportedOperationError( "Dolt requires an explicit mysql+pymysql URL." @@ -488,7 +496,7 @@ def _profile( } ), "operational_write_enabled": ( - dolt_declaration is not None + dolt_declaration is not None and bool(active["driver_eligible"]) if dolt_envelope and active and active["profile"] == name else bool((dolt_status or {}).get("write_enabled")) if dolt_envelope else True From cab47e99d2bd0a3e303f23e17165734cc77f902c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:30:26 +0300 Subject: [PATCH 063/119] Align Dolt snapshot write flag with driver eligibility --- src/openstatspec/sql/wide.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index b98787d..141a2c3 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1507,7 +1507,10 @@ def dolt_state_snapshot( "profile": "dolt", "server_version": active["server_version"], "read_only": True, - "operational_write_enabled": bool(active["claimed_supported"]), + "operational_write_enabled": ( + bool(active["claimed_supported"]) + and bool(active["driver_eligible"]) + ), "working_set_binding": binding, "state": state, } From b38350c666d8186d16af2c1604a7fede0ecfaf24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:30:55 +0300 Subject: [PATCH 064/119] Centralize Dolt operational eligibility --- src/openstatspec/sql/capabilities.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index 351c0cf..ad80e52 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -63,6 +63,13 @@ def _dolt_driver_eligible(database_url: str) -> bool: return make_url(database_url).drivername.lower() == "mysql+pymysql" +def dolt_operational_write_enabled( + active: Mapping[str, Any], *, declaration_matched: bool, +) -> bool: + """Require both exact declaration and claimed active driver for writes.""" + return declaration_matched and bool(active.get("driver_eligible", False)) + + def _dolt_write_enabled( source: DoltConformanceSource | None = None, *, @@ -496,7 +503,9 @@ def _profile( } ), "operational_write_enabled": ( - dolt_declaration is not None and bool(active["driver_eligible"]) + dolt_operational_write_enabled( + active, declaration_matched=dolt_declaration is not None, + ) if dolt_envelope and active and active["profile"] == name else bool((dolt_status or {}).get("write_enabled")) if dolt_envelope else True From 2e2e8c9faf1d39328f500e6e5d432530ddf6d9dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:31:00 +0300 Subject: [PATCH 065/119] Reuse Dolt operational eligibility in snapshot --- src/openstatspec/sql/wide.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 141a2c3..a8e942f 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -15,7 +15,9 @@ from sqlalchemy import delete, BigInteger, Boolean, Column, DateTime, Float, Integer, MetaData, String, Table, Text, create_engine, insert, inspect, select, text, update from sqlalchemy.dialects import mysql from ..core import UnsupportedOperationError, safe_error_identity as _safe_error_identity -from .capabilities import active_connection, effective_profile +from .capabilities import ( + active_connection, dolt_operational_write_enabled, effective_profile, +) from .dolt_conformance import DoltConformanceSource from .profiles import preflight, statement_payload_bytes, validate_connection_url from .normative import ( @@ -1507,9 +1509,8 @@ def dolt_state_snapshot( "profile": "dolt", "server_version": active["server_version"], "read_only": True, - "operational_write_enabled": ( - bool(active["claimed_supported"]) - and bool(active["driver_eligible"]) + "operational_write_enabled": dolt_operational_write_enabled( + active, declaration_matched=bool(active["claimed_supported"]), ), "working_set_binding": binding, "state": state, From 0d24d59a3e435e63404360f994e773173c685bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:31:07 +0300 Subject: [PATCH 066/119] Cover Dolt operational driver eligibility --- tests/test_sql_profiles.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 32d5b6b..2406bd4 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -634,3 +634,14 @@ def test_effective_profile_rechecks_driver_after_dolt_identity(monkeypatch) -> N effective_profile( "mariadb+mariadbconnector://user@host/database", ) + +def test_dolt_operational_write_flag_requires_claimed_driver() -> None: + assert capabilities.dolt_operational_write_enabled( + {"driver_eligible": True}, declaration_matched=True, + ) is True + assert capabilities.dolt_operational_write_enabled( + {"driver_eligible": False}, declaration_matched=True, + ) is False + assert capabilities.dolt_operational_write_enabled( + {"driver_eligible": True}, declaration_matched=False, + ) is False From b10926b048457c0f47877cbee527738f8ec31d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:32:01 +0300 Subject: [PATCH 067/119] Keep Dolt driver rejection independent of active metadata --- src/openstatspec/sql/capabilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index ad80e52..c7716bd 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -285,7 +285,7 @@ def effective_profile( raise UnsupportedOperationError("The active SQL server is not a claimed MySQL-wire product.") if ( active["profile"] == "dolt" - and not active["driver_eligible"] + and not _dolt_driver_eligible(database_url) ): raise UnsupportedOperationError( "Dolt requires an explicit mysql+pymysql URL." From 53f05ca26b87ec70444d796dbc035e88ac93d7fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:39:48 +0300 Subject: [PATCH 068/119] Cap MySQL text values at emitted TEXT storage --- src/openstatspec/sql/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openstatspec/sql/profiles.py b/src/openstatspec/sql/profiles.py index 519ed10..235d527 100644 --- a/src/openstatspec/sql/profiles.py +++ b/src/openstatspec/sql/profiles.py @@ -57,7 +57,7 @@ def as_dict(self) -> dict[str, object]: ) MYSQL = SqlProfile( "mysql", ("mysql", "mariadb"), 1_016, 64, True, True, - 4_294_967_295, 65_535, True, ("PyMySQL",), + 65_535, 65_535, True, ("PyMySQL",), ) DOLT = SqlProfile( "dolt", (), 305, 64, True, True, From d9fa47e5f9531e601d5c59cd478d30cf75657331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:40:14 +0300 Subject: [PATCH 069/119] Gate active non-Dolt write capability consistently --- src/openstatspec/sql/capabilities.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index c7716bd..e541c39 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -59,8 +59,23 @@ def _bound_specification_commit() -> str: return SPECIFICATION_COMMIT +def _active_driver_eligible(database_url: str, profile_name: str) -> bool: + driver = make_url(database_url).drivername.lower() + required = { + "postgresql": "postgresql+psycopg", + "mysql": "mysql+pymysql", + "mariadb": "mariadb+mariadbconnector", + "dolt": "mysql+pymysql", + } + return ( + make_url(database_url).get_backend_name() == "sqlite" + if profile_name == "sqlite" + else driver == required[profile_name] + ) + + def _dolt_driver_eligible(database_url: str) -> bool: - return make_url(database_url).drivername.lower() == "mysql+pymysql" + return _active_driver_eligible(database_url, "dolt") def dolt_operational_write_enabled( @@ -246,9 +261,8 @@ def active_connection( if profile_name == "dolt" else None ), "observed": observed, - "driver_eligible": ( - _dolt_driver_eligible(database_url) - if profile_name == "dolt" else True + "driver_eligible": _active_driver_eligible( + database_url, profile_name, ), } @@ -507,6 +521,11 @@ def _profile( active, declaration_matched=dolt_declaration is not None, ) if dolt_envelope and active and active["profile"] == name + else ( + bool(active["claimed_supported"]) + and bool(active["driver_eligible"]) + ) + if active and active["profile"] == name else bool((dolt_status or {}).get("write_enabled")) if dolt_envelope else True ), From f1c56dfb41a52c85d01e70a29c92be1a07809af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:40:19 +0300 Subject: [PATCH 070/119] Use symlink-aware backup cleanup checks --- src/openstatspec/spss/sav.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index bc0613a..d211aeb 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -338,7 +338,7 @@ def _raise_export_cleanup_failed( "destination": _path_reference(destination, role="destination"), "destination_exists": destination.exists(), "backup": _path_reference(backup, role="durable_backup"), - "backup_exists": backup.exists(), + "backup_exists": _path_entry_exists(backup), "staged_export": _path_reference(staged, role="staged_export"), "staged_export_exists": staged.exists(), } @@ -366,7 +366,7 @@ def _raise_export_cleanup_failed( "cleanup_attempted": True, "cleanup_succeeded": False, "previous_destination_existed": had_previous, - "durable_backup_survives_staging_cleanup": backup.exists(), + "durable_backup_survives_staging_cleanup": _path_entry_exists(backup), } cleanup_audit_operation_id = None cleanup_audit_fault = None @@ -452,7 +452,7 @@ def _mark_export_failed_after_restore( "backup": _path_reference( backup, role="durable_backup", ), - "backup_exists": backup.exists(), + "backup_exists": _path_entry_exists(backup), "operation_id": operation_id, }, "deterministic_recovery_evidence": { @@ -750,21 +750,21 @@ def export_sav_dataset( "backup": _path_reference( backup, role="durable_backup", ), - "backup_exists": backup.exists(), + "backup_exists": _path_entry_exists(backup), }, "deterministic_recovery_evidence": { "procedure_id": "openstatspec.export-audit-reconciliation.v1", "operation_terminal_state_verified": False, "automatic_filesystem_recovery_performed": False, "published_file_preserved": destination_path.exists(), - "durable_backup_preserved": backup.exists(), + "durable_backup_preserved": _path_entry_exists(backup), "manual_reconciliation_required": True, "terminal_reporting": "out_of_band_exception", }, "success_forbidden": True, }, ) from finalization_error - if backup.exists(): + if _path_entry_exists(backup): try: backup.unlink() except Exception as cleanup_error: From c923f72f465854fb9d3c81409d46fb2fa076c7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:40:38 +0300 Subject: [PATCH 071/119] Cover emitted text and active write capability gates --- tests/test_sql_profiles.py | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 2406bd4..020f640 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -645,3 +645,43 @@ def test_dolt_operational_write_flag_requires_claimed_driver() -> None: assert capabilities.dolt_operational_write_enabled( {"driver_eligible": True}, declaration_matched=False, ) is False + +def test_mysql_preflight_matches_emitted_text_limit() -> None: + assert MYSQL.max_text_value_bytes == 65_535 + variables = [{ + "ordinal": 1, + "source_name": "value", + "physical_name": "value", + "storage_kind": "string", + "string_width": 65_536, + }] + + with pytest.raises(UnsupportedOperationError) as error: + preflight(MYSQL, variables) + + assert error.value.details["reason"] == "declared_string_width_limit" + assert error.value.details["maximum"] == 65_535 + + +@pytest.mark.parametrize( + ("claimed_supported", "driver_eligible"), + [(False, True), (True, False)], +) +def test_active_non_dolt_write_flag_requires_version_and_driver( + monkeypatch, claimed_supported, driver_eligible, +) -> None: + active = { + "profile": "postgresql", + "claimed_supported": claimed_supported, + "driver_eligible": driver_eligible, + "observed": {}, + } + monkeypatch.setattr( + capabilities, "active_connection", lambda _url, **_kwargs: active, + ) + + declaration = capabilities.profile_declarations( + "postgresql+psycopg://user@host/database", + )["postgresql"] + + assert declaration["operational_write_enabled"] is False From db94767d27222f2f33e55b7cd2b8f8c2620f7587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:40:44 +0300 Subject: [PATCH 072/119] Cover successful dangling backup disposal --- tests/test_sav_sqlite.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index c4dcbfd..fbf094d 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -241,3 +241,16 @@ def test_export_recovery_preserves_dangling_destination_symlink(tmp_path) -> Non assert destination.is_symlink() assert destination.readlink() == missing_target + +def test_successful_export_cleanup_removes_dangling_symlink_backup( + tmp_path, +) -> None: + missing_target = tmp_path / "missing-original.sav" + backup = tmp_path / ".destination.previous" + backup.symlink_to(missing_target) + + assert sav_module._path_entry_exists(backup) is True + if sav_module._path_entry_exists(backup): + backup.unlink() + + assert sav_module._path_entry_exists(backup) is False From 970486d09ad9b3d29d2be974bd5312a9bab2db0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:47:19 +0300 Subject: [PATCH 073/119] Track successful import table ownership before cleanup --- src/openstatspec/sql/wide.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index a8e942f..6db47ba 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1797,6 +1797,14 @@ def _delete_normative_import_state( )) +def _create_operation_owned_data_table( + connection: Any, data_table: Table, state: dict[str, Any], +) -> None: + """Mark ownership only after this operation successfully creates the table.""" + data_table.create(connection) + state["data_table_created"] = True + + def _cleanup_import_state( connection: Any, *, dataset_id: str, operation_id: str, data_table: Table, state: Mapping[str, Any], normative: Any, legacy: tuple[Table, ...], @@ -1824,7 +1832,7 @@ def _cleanup_import_state( fidelity_events.c.operation_id == operation_id )) connection.execute(delete(operations).where(operations.c.operation_id == operation_id)) - if state["data_table_creation_attempted"]: + if state["data_table_created"]: data_table.drop(connection, checkfirst=True) @@ -2028,7 +2036,7 @@ def _import_cleanup_guard( snapshot_connection: Any, pre_dolt_state: dict[str, Any] | None, ) -> Iterable[dict[str, Any]]: state: dict[str, Any] = { - "data_table_creation_attempted": False, + "data_table_created": False, "legacy_dataset_created": False, "normative_dataset_creation_attempted": False, "normative_dataset_id": None, @@ -2377,7 +2385,7 @@ def create_wide_dataset( normative.fidelity_event.name, normative.operation.name, } preflight_state = { - "data_table_creation_attempted": False, + "data_table_created": False, "legacy_dataset_created": False, "normative_dataset_creation_attempted": False, "normative_dataset_id": None, @@ -2554,8 +2562,9 @@ def create_wide_dataset( raise ValueError(f"Dataset {dataset_id!r} already exists; imports never overwrite a dataset.") if connection.execute(select(datasets.c.dataset_id).where(datasets.c.data_table == data_table.name)).first(): raise ValueError(f"Dataset ID {dataset_id!r} collides with an existing physical data-table name; import was not started.") - mutation["data_table_creation_attempted"] = True - data_table.create(connection) + _create_operation_owned_data_table( + connection, data_table, mutation, + ) materialized = [ {"__case_ordinal": ordinal, **row} for ordinal, row in enumerate(source_rows, start=1) From 192d7d7eb5cec1e496b26ed496b1e244039379b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:47:36 +0300 Subject: [PATCH 074/119] Cover concurrent import table ownership --- tests/test_atomic_import.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_atomic_import.py b/tests/test_atomic_import.py index 59fae18..68be04b 100755 --- a/tests/test_atomic_import.py +++ b/tests/test_atomic_import.py @@ -359,3 +359,17 @@ def fail_first_normative_finish(*args, **kwargs): assert connection.execute( "select code, dataset_id from fidelity_event_catalog" ).fetchall() == [("import_failed", None)] + +def test_failed_table_create_never_claims_cleanup_ownership() -> None: + class ConcurrentTable: + def create(self, _connection): + raise RuntimeError("table already created by concurrent import") + + state = {"data_table_created": False} + + with pytest.raises(RuntimeError, match="concurrent import"): + wide._create_operation_owned_data_table( + object(), ConcurrentTable(), state, + ) + + assert state["data_table_created"] is False From 148c988d54b02d043e1a249be65e5b383970dec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:54:39 +0300 Subject: [PATCH 075/119] Recognize supported MySQL-wire driver pairings --- src/openstatspec/sql/capabilities.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/openstatspec/sql/capabilities.py b/src/openstatspec/sql/capabilities.py index e541c39..83f2a66 100644 --- a/src/openstatspec/sql/capabilities.py +++ b/src/openstatspec/sql/capabilities.py @@ -61,16 +61,16 @@ def _bound_specification_commit() -> str: def _active_driver_eligible(database_url: str, profile_name: str) -> bool: driver = make_url(database_url).drivername.lower() - required = { - "postgresql": "postgresql+psycopg", - "mysql": "mysql+pymysql", - "mariadb": "mariadb+mariadbconnector", - "dolt": "mysql+pymysql", + eligible = { + "postgresql": {"postgresql+psycopg"}, + "mysql": {"mysql+pymysql", "mariadb+mariadbconnector"}, + "mariadb": {"mysql+pymysql", "mariadb+mariadbconnector"}, + "dolt": {"mysql+pymysql"}, } return ( make_url(database_url).get_backend_name() == "sqlite" if profile_name == "sqlite" - else driver == required[profile_name] + else driver in eligible[profile_name] ) From 98866edbdd66b99b581c1cd6166d01dbed47fe31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:54:44 +0300 Subject: [PATCH 076/119] Require LONGTEXT when validating Dolt datasets --- src/openstatspec/sql/wide.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 6db47ba..5d2cb8a 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -68,6 +68,15 @@ def _wide_column_type(profile: Any, storage_kind: str) -> Any: return binary64_type() if storage_kind == "numeric" else string_type(profile) +def _valid_wide_string_type(profile: Any, column_type: Any) -> bool: + """Require Dolt's declared LONGTEXT boundary during reflected validation.""" + return ( + isinstance(column_type, mysql.LONGTEXT) + if profile.name == "dolt" + else isinstance(column_type, Text) + ) + + def _canonical_sha256(value: Any) -> str: return hashlib.sha256( json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") @@ -3244,6 +3253,9 @@ def validate_wide_dataset( dataset_id: str, dolt_conformance_source: DoltConformanceSource | None = None, ) -> dict[str, Any]: + profile, _active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) dataset, variables, rows = read_wide_dataset( database_url=database_url, dataset_id=dataset_id, @@ -3274,7 +3286,7 @@ def validate_wide_dataset( if item["storage_kind"] == "numeric": if not isinstance(column.type, Float) or not column.nullable: raise ValueError(f"Numeric variable {item['source_name']!r} must be a nullable binary64 column.") - elif not isinstance(column.type, Text) or column.nullable: + elif not _valid_wide_string_type(profile, column.type) or column.nullable: raise ValueError(f"String variable {item['source_name']!r} must be a non-null text column.") if [row["__case_ordinal"] for row in rows] != list(range(1, len(rows) + 1)): raise ValueError("Case ordinals are not contiguous source order.") From 6b95895475bdd90e1f53e6e9150b80156e860b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:55:14 +0300 Subject: [PATCH 077/119] Cover MariaDB driver and Dolt LONGTEXT validation --- tests/test_sql_profiles.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index 020f640..d5549af 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -3,7 +3,7 @@ from types import SimpleNamespace import pytest -from sqlalchemy import Column, MetaData, Table +from sqlalchemy import Column, MetaData, Table, Text from sqlalchemy.dialects import mysql, postgresql, sqlite from sqlalchemy.schema import CreateTable @@ -685,3 +685,14 @@ def test_active_non_dolt_write_flag_requires_version_and_driver( )["postgresql"] assert declaration["operational_write_enabled"] is False + +def test_mariadb_accepts_tested_pymysql_driver_for_capability_flag() -> None: + assert capabilities._active_driver_eligible( + "mysql+pymysql://user@host/database", "mariadb", + ) is True + + +def test_dolt_validation_requires_longtext_not_generic_text() -> None: + assert wide._valid_wide_string_type(DOLT, mysql.LONGTEXT()) is True + assert wide._valid_wide_string_type(DOLT, Text()) is False + assert wide._valid_wide_string_type(MYSQL, Text()) is True From dff51d19ed3234df4dcf619af5b57d7446654e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 20:59:23 +0300 Subject: [PATCH 078/119] Keep Dolt validation propagation test independent of SQL extras --- tests/test_dolt_conformance.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_dolt_conformance.py b/tests/test_dolt_conformance.py index 0722101..701c987 100644 --- a/tests/test_dolt_conformance.py +++ b/tests/test_dolt_conformance.py @@ -162,11 +162,19 @@ def test_validate_wide_dataset_propagates_explicit_source( ) -> None: sentinel = object() calls: list[object] = [] + profile_calls: list[object] = [] + + def capture_effective_profile( + _database_url: str, *, dolt_conformance_source: object, + ) -> tuple[object, dict[str, object]]: + profile_calls.append(dolt_conformance_source) + return object(), {} def stop_after_read_preflight(**kwargs: object) -> tuple[object, object, object]: calls.append(kwargs["dolt_conformance_source"]) raise UnsupportedOperationError("stop after propagation check") + monkeypatch.setattr(wide, "effective_profile", capture_effective_profile) monkeypatch.setattr(wide, "read_wide_dataset", stop_after_read_preflight) with pytest.raises(UnsupportedOperationError, match="propagation check"): wide.validate_wide_dataset( @@ -174,6 +182,7 @@ def stop_after_read_preflight(**kwargs: object) -> tuple[object, object, object] dataset_id="synthetic", dolt_conformance_source=sentinel, ) + assert profile_calls == [sentinel] assert calls == [sentinel] From cc0597951d0b7a2a71ca2327b9e8e84c76dec4c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:06:59 +0300 Subject: [PATCH 079/119] Propagate Dolt conformance through in-place writes --- src/openstatspec/sql/inplace_transform.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/openstatspec/sql/inplace_transform.py b/src/openstatspec/sql/inplace_transform.py index 6e1b556..e19690f 100644 --- a/src/openstatspec/sql/inplace_transform.py +++ b/src/openstatspec/sql/inplace_transform.py @@ -21,6 +21,7 @@ transformation_plan_from_dict, ) from .capabilities import effective_profile +from .dolt_conformance import DoltConformanceSource from .normative import catalog as core_catalog from .wide import catalog as legacy_catalog from .wide import normalized_metadata_tables, physical_name @@ -554,11 +555,17 @@ def _dolt_state(connection: Any) -> tuple[str, str, int]: return branch, head, dirty -def install_in_place_transformation_schema(*, database_url: str) -> None: +def install_in_place_transformation_schema( + *, + database_url: str, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: """Install the compact operation audit separately from any data apply.""" # Resolve the effective profile first so Dolt conformance and explicit # driver checks fail closed before an engine transaction can execute DDL. - effective_profile(database_url) + effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) try: with engine.begin() as connection: @@ -595,13 +602,16 @@ def _run_in_place_submission( prepare: Callable[[Any, str], InPlacePlanSubmission], expected_branch: str | None = None, expected_head: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> dict[str, Any]: """Prepare and apply one canonical plan in the same controlled transaction.""" if not actor: raise TransformationError( "actor_required", "A non-empty actor identity is mandatory.", ) - profile, _active = effective_profile(database_url) + profile, _active = effective_profile( + database_url, dolt_conformance_source=dolt_conformance_source, + ) engine = create_engine(database_url) try: with engine.begin() as connection: @@ -662,6 +672,7 @@ def apply_transformation_plan_in_place( actor: str, expected_branch: str | None = None, expected_head: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> dict[str, Any]: """Apply a canonical plan without knowing which frontend produced it.""" normalized = ( @@ -682,4 +693,5 @@ def apply_transformation_plan_in_place( prepare=lambda _connection, _dataset_id: submission, expected_branch=expected_branch, expected_head=expected_head, + dolt_conformance_source=dolt_conformance_source, ) From 2da412e89ac699dfae44c219119b00bd250bbc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:07:04 +0300 Subject: [PATCH 080/119] Propagate Dolt conformance through SPSS apply --- src/openstatspec/frontends/spss/execution.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/openstatspec/frontends/spss/execution.py b/src/openstatspec/frontends/spss/execution.py index 30c672a..1d947da 100644 --- a/src/openstatspec/frontends/spss/execution.py +++ b/src/openstatspec/frontends/spss/execution.py @@ -4,6 +4,7 @@ from typing import Any +from ...sql.dolt_conformance import DoltConformanceSource from ...sql.inplace_transform import ( InPlacePlanSubmission, _run_in_place_submission, @@ -21,6 +22,7 @@ def apply_spss_in_place( actor: str, expected_branch: str | None = None, expected_head: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> dict[str, Any]: """Compile SPSS syntax and apply its canonical plan in one transaction.""" @@ -45,4 +47,5 @@ def prepare(connection: Any, live_dataset_id: str) -> InPlacePlanSubmission: prepare=prepare, expected_branch=expected_branch, expected_head=expected_head, + dolt_conformance_source=dolt_conformance_source, ) From 63bb245b8d50081ff5b438cfac33d2dc9894f4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:07:09 +0300 Subject: [PATCH 081/119] Expose Dolt conformance for in-place public APIs --- src/openstatspec/api.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/openstatspec/api.py b/src/openstatspec/api.py index ba0efc8..e8ee582 100644 --- a/src/openstatspec/api.py +++ b/src/openstatspec/api.py @@ -221,6 +221,7 @@ def apply_spss_in_place( *, database_url: Any, dataset_id: str, source_text: str, actor: str, expected_branch: str | None = None, expected_head: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> Mapping[str, Any]: """Apply supported SPSS-like syntax to the same SQL dataset/table.""" return result(_apply_spss_in_place( @@ -230,6 +231,7 @@ def apply_spss_in_place( actor=actor, expected_branch=expected_branch, expected_head=expected_head, + dolt_conformance_source=dolt_conformance_source, )) @@ -238,6 +240,7 @@ def apply_transformation_plan_in_place( plan: TransformationPlan | Mapping[str, Any], actor: str, expected_branch: str | None = None, expected_head: str | None = None, + dolt_conformance_source: DoltConformanceSource | None = None, ) -> Mapping[str, Any]: """Apply a canonical plan to the same logical dataset and physical table.""" return result(_apply_transformation_plan_in_place( @@ -247,12 +250,20 @@ def apply_transformation_plan_in_place( actor=actor, expected_branch=expected_branch, expected_head=expected_head, + dolt_conformance_source=dolt_conformance_source, )) -def install_in_place_transformation_schema(*, database_url: Any) -> None: +def install_in_place_transformation_schema( + *, + database_url: Any, + dolt_conformance_source: DoltConformanceSource | None = None, +) -> None: """Install the compact apply-audit relation before the first apply.""" - _install_in_place_schema(database_url=str(database_url)) + _install_in_place_schema( + database_url=str(database_url), + dolt_conformance_source=dolt_conformance_source, + ) def validate_derived(*, database_url: Any, derived_dataset_id: str) -> Mapping[str, Any]: From 730e122046691a0a12e956ebc0562f21935a6bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:07:28 +0300 Subject: [PATCH 082/119] Expose generated identifier preflight --- src/openstatspec/sql/profiles.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/openstatspec/sql/profiles.py b/src/openstatspec/sql/profiles.py index 235d527..7be63ab 100644 --- a/src/openstatspec/sql/profiles.py +++ b/src/openstatspec/sql/profiles.py @@ -102,6 +102,23 @@ def _exceeded(reason: str, message: str, **details: Any) -> TargetCapabilityExce ) +def preflight_identifier( + profile: SqlProfile, identifier: str, *, role: str, +) -> None: + """Validate one generated identifier against the effective profile.""" + identifier_bytes = len(identifier.encode("utf-8")) + if identifier_bytes > profile.identifier_limit: + raise _exceeded( + "identifier_limit", + f"{role} {identifier!r} is {identifier_bytes} bytes; " + f"{profile.name} permits {profile.identifier_limit}.", + identifier=identifier, + identifier_bytes=identifier_bytes, + maximum=profile.identifier_limit, + role=role, + ) + + def preflight( profile: SqlProfile, variables_or_count: int | Iterable[Mapping[str, Any]], @@ -143,15 +160,9 @@ def preflight( source_name=source_name, expected_physical_name=expected_name, actual_physical_name=actual_name, ) - identifier_bytes = len(expected_name.encode("utf-8")) - if identifier_bytes > profile.identifier_limit: - raise _exceeded( - "identifier_limit", - f"identifier {expected_name!r} is {identifier_bytes} bytes; " - f"{profile.name} permits {profile.identifier_limit}.", - identifier=expected_name, identifier_bytes=identifier_bytes, - maximum=profile.identifier_limit, - ) + preflight_identifier( + profile, expected_name, role="physical variable identifier", + ) if variable.get("storage_kind") == "string": declared_width = variable.get("string_width") if declared_width is not None and ( From 30858aea385a9d23e09fe2237a5d028219c3c99b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:07:33 +0300 Subject: [PATCH 083/119] Preflight generated table identifier before DDL --- src/openstatspec/sql/wide.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 5d2cb8a..4ba6b80 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -19,7 +19,10 @@ active_connection, dolt_operational_write_enabled, effective_profile, ) from .dolt_conformance import DoltConformanceSource -from .profiles import preflight, statement_payload_bytes, validate_connection_url +from .profiles import ( + preflight, preflight_identifier, statement_payload_bytes, + validate_connection_url, +) from .normative import ( binary64_type, CATALOG_CONTRACT_ID, @@ -2410,6 +2413,9 @@ def create_wide_dataset( ) preflight_connection.rollback() try: + preflight_identifier( + profile, data_table.name, role="physical data-table identifier", + ) preflight(profile, variables, rows=source_rows) validate_spss_catalog( variables, From 60a0eac11cb680f3ac662d1a4ef7fbbaf6f93408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:07:58 +0300 Subject: [PATCH 084/119] Cover in-place Dolt conformance propagation --- tests/test_inplace_transform.py | 46 ++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 38bdabd..1a20c63 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -512,7 +512,7 @@ def test_capability_declares_dolt_owned_versioning() -> None: def test_schema_install_fails_before_engine_or_ddl_when_profile_is_rejected( monkeypatch, ) -> None: - def reject_profile(_database_url): + def reject_profile(_database_url, **_kwargs): raise openstatspec.UnsupportedOperationError("Dolt declaration mismatch") def unexpected_engine(_database_url): @@ -528,3 +528,47 @@ def unexpected_engine(_database_url): inplace_transform.install_in_place_transformation_schema( database_url="mysql+pymysql://user@host/database", ) + +def test_schema_install_forwards_explicit_dolt_conformance_source( + monkeypatch, +) -> None: + sentinel = object() + captured = [] + + def capture_profile(_database_url, *, dolt_conformance_source): + captured.append(dolt_conformance_source) + raise openstatspec.UnsupportedOperationError("stop after gate") + + monkeypatch.setattr(inplace_transform, "effective_profile", capture_profile) + + with pytest.raises(openstatspec.UnsupportedOperationError, match="stop after gate"): + inplace_transform.install_in_place_transformation_schema( + database_url="mysql+pymysql://user@host/database", + dolt_conformance_source=sentinel, + ) + + assert captured == [sentinel] + + +def test_plan_apply_forwards_explicit_dolt_conformance_source(monkeypatch) -> None: + sentinel = object() + captured = [] + + def capture_submission(**kwargs): + captured.append(kwargs["dolt_conformance_source"]) + return {"ok": True} + + monkeypatch.setattr( + inplace_transform, "_run_in_place_submission", capture_submission, + ) + + result = inplace_transform.apply_transformation_plan_in_place( + database_url="sqlite://", + dataset_id="synthetic", + plan=_plan("RECODE score (1 = 2)."), + actor="test-agent", + dolt_conformance_source=sentinel, + ) + + assert result == {"ok": True} + assert captured == [sentinel] From 0af6c40232f8a0b5349825eaf373f872f6a6dd09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:08:03 +0300 Subject: [PATCH 085/119] Cover generated table identifier preflight --- tests/test_sql_profiles.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_sql_profiles.py b/tests/test_sql_profiles.py index d5549af..6b568fa 100755 --- a/tests/test_sql_profiles.py +++ b/tests/test_sql_profiles.py @@ -10,7 +10,10 @@ from openstatspec.sql.normative import catalog as normative_catalog from openstatspec.core import UnsupportedOperationError -from openstatspec.sql.profiles import DOLT, MYSQL, POSTGRESQL, SQLITE, preflight, profile_for_url +from openstatspec.sql.profiles import ( + DOLT, MYSQL, POSTGRESQL, SQLITE, preflight, preflight_identifier, + profile_for_url, +) import openstatspec.sql.capabilities as capabilities import openstatspec.sql.wide as wide import openstatspec.spss.sav as sav @@ -696,3 +699,15 @@ def test_dolt_validation_requires_longtext_not_generic_text() -> None: assert wide._valid_wide_string_type(DOLT, mysql.LONGTEXT()) is True assert wide._valid_wide_string_type(DOLT, Text()) is False assert wide._valid_wide_string_type(MYSQL, Text()) is True + +def test_generated_data_table_identifier_uses_effective_limit() -> None: + limited = replace(DOLT, identifier_limit=5) + + with pytest.raises(UnsupportedOperationError) as error: + preflight_identifier( + limited, "data_dataset", role="physical data-table identifier", + ) + + assert error.value.details["reason"] == "identifier_limit" + assert error.value.details["role"] == "physical data-table identifier" + assert error.value.details["maximum"] == 5 From fd429390c4a2f9ba8b65743d04644e776f4ff1b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:13:29 +0300 Subject: [PATCH 086/119] Update Dolt apply mocks for conformance keyword --- tests/test_inplace_transform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 1a20c63..aac0fa5 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -406,7 +406,7 @@ def test_public_apply_binds_expected_dolt_branch_and_head( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), ) states = iter([ ("feature/recode", "abc123", 0), @@ -436,7 +436,7 @@ def test_public_apply_rejects_dirty_dolt_working_set_before_mutation( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), ) monkeypatch.setattr( inplace_transform, @@ -472,7 +472,7 @@ def test_public_apply_rejects_dolt_context_mismatch_before_mutation( monkeypatch.setattr( inplace_transform, "effective_profile", - lambda _url: (SimpleNamespace(name="dolt"), {}), + lambda _url, **_kwargs: (SimpleNamespace(name="dolt"), {}), ) monkeypatch.setattr( inplace_transform, "_dolt_state", lambda _connection: state From db7477e37730970ba6dca560ac9575bb7cccd520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:16:28 +0300 Subject: [PATCH 087/119] Avoid stale compensation after transactional rollback --- src/openstatspec/sql/wide.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 4ba6b80..096a191 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -20,8 +20,8 @@ ) from .dolt_conformance import DoltConformanceSource from .profiles import ( - preflight, preflight_identifier, statement_payload_bytes, - validate_connection_url, + MYSQL_WIRE_PROFILES, preflight, preflight_identifier, + statement_payload_bytes, validate_connection_url, ) from .normative import ( binary64_type, @@ -2040,6 +2040,11 @@ def count_rows(table: Table, condition: Any) -> int | None: return {"inspection_error_type": type(inventory_error).__name__} +def _requires_compensating_import_cleanup(profile_name: str) -> bool: + """Transactional profiles rely on rollback, never stale cleanup markers.""" + return profile_name in MYSQL_WIRE_PROFILES + + @contextmanager def _import_cleanup_guard( *, engine: Any, dataset_id: str, operation_id: str, data_table: Table, @@ -2069,12 +2074,13 @@ def capture_boundary() -> dict[str, Any]: yield state except Exception as import_error: try: - with engine.begin() as cleanup_connection: - _cleanup_import_state( - cleanup_connection, dataset_id=dataset_id, - operation_id=operation_id, data_table=data_table, - state=state, normative=normative, legacy=legacy, - ) + if _requires_compensating_import_cleanup(profile_name): + with engine.begin() as cleanup_connection: + _cleanup_import_state( + cleanup_connection, dataset_id=dataset_id, + operation_id=operation_id, data_table=data_table, + state=state, normative=normative, legacy=legacy, + ) except Exception as cleanup_error: inventory = _import_residual_inventory( engine, dataset_id=dataset_id, operation_id=operation_id, From ffcfb5608e58e26955637e57a7cb95f280392da9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:16:41 +0300 Subject: [PATCH 088/119] Cover transactional rollback cleanup boundary --- tests/test_atomic_import.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_atomic_import.py b/tests/test_atomic_import.py index 68be04b..be54772 100755 --- a/tests/test_atomic_import.py +++ b/tests/test_atomic_import.py @@ -373,3 +373,10 @@ def create(self, _connection): ) assert state["data_table_created"] is False + +def test_transactional_profiles_never_run_stale_compensating_cleanup() -> None: + assert wide._requires_compensating_import_cleanup("sqlite") is False + assert wide._requires_compensating_import_cleanup("postgresql") is False + assert wide._requires_compensating_import_cleanup("mysql") is True + assert wide._requires_compensating_import_cleanup("mariadb") is True + assert wide._requires_compensating_import_cleanup("dolt") is True From 5ed2ce6a341cd8fcbeef0aebec976bc238d364a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:21:28 +0300 Subject: [PATCH 089/119] Test cleanup failure on nontransactional profile path --- tests/test_catalog_lifecycle.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 8e35664..7596278 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -184,6 +184,11 @@ def fail_cleanup(*_args, **_kwargs): def fail_mutation(*_args, **_kwargs): raise RuntimeError("injected mutation failure") + # Exercise the non-transactional MySQL-wire compensation path while + # retaining SQLite as the dependency-free test transport. + monkeypatch.setattr( + wide, "effective_profile", lambda _url, **_kwargs: (MYSQL, {}), + ) monkeypatch.setattr(wide, "_cleanup_import_state", fail_cleanup) monkeypatch.setattr(wide, "store_normative_dataset", fail_mutation) with pytest.raises(ImportRecoveryError) as error: From ee3dfd6481de5794b7d276a1ba18bdc82d8a725f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:28:30 +0300 Subject: [PATCH 090/119] Recognize in-place audit as optional catalog relation --- src/openstatspec/sql/wide.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 096a191..e77f82f 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1303,6 +1303,17 @@ def _registered_physical_relations( physical_tables.add(name) elif workflow_tables & existing_tables: return static_tables, declared_tables, physical_tables, physical_views, "foreign" + + # The compact in-place apply audit is optional but catalog-owned. Import + # locally because inplace_transform depends on this module at load time. + from .inplace_transform import ( # pylint: disable=import-outside-toplevel + apply_audit_catalog, + ) + apply_audit = apply_audit_catalog(MetaData()) + if apply_audit.name in existing_tables: + declared_tables += (apply_audit,) + static_tables.add(apply_audit.name) + return static_tables, declared_tables, physical_tables, physical_views, "valid" From 306b4a97da30d7da117fc7f6f2b717c7bbd95aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:28:53 +0300 Subject: [PATCH 091/119] Cover optional in-place audit catalog ownership --- tests/test_inplace_transform.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index aac0fa5..97af27a 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -572,3 +572,13 @@ def capture_submission(**kwargs): assert result == {"ok": True} assert captured == [sentinel] + +def test_in_place_audit_relation_remains_catalog_owned(catalog) -> None: + url, _path, dataset_id, _table_name = catalog + + dataset = openstatspec.get_dataset( + database_url=url, + dataset_id=dataset_id, + ) + + assert dataset["dataset_id"] == dataset_id From cd1c12862ce767179cc1a58c35ae79945a4281ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:32:42 +0300 Subject: [PATCH 092/119] Use core catalog kind in audit ownership regression --- tests/test_inplace_transform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 97af27a..c64f566 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -579,6 +579,7 @@ def test_in_place_audit_relation_remains_catalog_owned(catalog) -> None: dataset = openstatspec.get_dataset( database_url=url, dataset_id=dataset_id, + kind="core", ) assert dataset["dataset_id"] == dataset_id From 8ffa2a136913c5489608cb66dc5536788b5846df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:35:26 +0300 Subject: [PATCH 093/119] Assert nested core catalog result in audit ownership test --- tests/test_inplace_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index c64f566..65b9172 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -582,4 +582,4 @@ def test_in_place_audit_relation_remains_catalog_owned(catalog) -> None: kind="core", ) - assert dataset["dataset_id"] == dataset_id + assert dataset["dataset"]["dataset_id"] == dataset_id From 1e22ac02f121a242584aa5222285aafbae319762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:45:01 +0300 Subject: [PATCH 094/119] Expose full catalog verification for mutation gates --- src/openstatspec/sql/wide.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index e77f82f..0d0d607 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1429,6 +1429,13 @@ def _require_verified_catalog( ) +def require_verified_catalog(connection: Any) -> None: + """Require full OpenStatSpec catalog ownership, shape, and dataset bijection.""" + metadata = MetaData() + legacy, normative = _catalog_layout(metadata) + _require_verified_catalog(connection, normative, legacy) + + def _catalog_snapshot( connection: Any, ) -> tuple[set[str], dict[str, set[str]]]: From 073a659f2541a79ffb9a073133993a6bff219852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:45:20 +0300 Subject: [PATCH 095/119] Gate in-place schema and applies on verified catalog --- src/openstatspec/sql/inplace_transform.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/inplace_transform.py b/src/openstatspec/sql/inplace_transform.py index e19690f..0c29b0c 100644 --- a/src/openstatspec/sql/inplace_transform.py +++ b/src/openstatspec/sql/inplace_transform.py @@ -24,7 +24,9 @@ from .dolt_conformance import DoltConformanceSource from .normative import catalog as core_catalog from .wide import catalog as legacy_catalog -from .wide import normalized_metadata_tables, physical_name +from .wide import ( + normalized_metadata_tables, physical_name, require_verified_catalog, +) from .workflow import TransformationError @@ -569,6 +571,7 @@ def install_in_place_transformation_schema( engine = create_engine(database_url) try: with engine.begin() as connection: + require_verified_catalog(connection) apply_audit_catalog(MetaData()).create(connection, checkfirst=True) columns = { str(column["name"]) @@ -615,6 +618,7 @@ def _run_in_place_submission( engine = create_engine(database_url) try: with engine.begin() as connection: + require_verified_catalog(connection) branch: str | None = None head: str | None = None if profile.name == "dolt": From b078a6a69cd426405d9b23ad4a05c1fe2eead686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:45:59 +0300 Subject: [PATCH 096/119] Preserve newer concurrent export destinations --- src/openstatspec/spss/sav.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index d211aeb..4b8276e 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -310,6 +310,17 @@ def _path_entry_exists(path: Path) -> bool: return path.exists() or path.is_symlink() +_DESTINATION_IDENTITY_UNCHECKED = object() + + +def _destination_identity(path: Path) -> tuple[int, int] | None: + """Identify the current directory entry without following symlinks.""" + if not _path_entry_exists(path): + return None + status = path.lstat() + return status.st_dev, status.st_ino + + def _reserve_export_backup(destination: Path) -> Path: descriptor, name = mkstemp( dir=destination.parent, prefix=f".{destination.name}.", @@ -321,7 +332,17 @@ def _reserve_export_backup(destination: Path) -> Path: def _restore_export_destination( *, destination: Path, backup: Path, had_previous: bool, + expected_identity: tuple[int, int] | None | object = ( + _DESTINATION_IDENTITY_UNCHECKED + ), ) -> None: + if ( + expected_identity is not _DESTINATION_IDENTITY_UNCHECKED + and _destination_identity(destination) != expected_identity + ): + raise FileExistsError( + "The export destination is no longer owned by this operation." + ) if had_previous: os.replace(backup, destination) else: @@ -493,6 +514,7 @@ def _export_staging_directory( destination=destination, backup=backup, had_previous=had_previous, + expected_identity=publication_state["published_identity"], ) except Exception as restore_error: _raise_export_cleanup_failed( @@ -633,8 +655,14 @@ def export_sav_dataset( os.replace(destination_path, backup) backup_installed = True os.replace(staged_destination, destination_path) + published_identity = _destination_identity(destination_path) + if published_identity is None: + raise FileNotFoundError( + "The published export destination disappeared." + ) publication_state.update({ "published": True, + "published_identity": published_identity, "backup": backup, "staged": staged_destination, "had_previous": had_previous, @@ -645,7 +673,7 @@ def export_sav_dataset( try: _restore_export_destination( destination=destination_path, backup=backup, - had_previous=had_previous, + had_previous=had_previous, expected_identity=None, ) except Exception as cleanup_error: _raise_export_cleanup_failed( @@ -707,6 +735,7 @@ def export_sav_dataset( _restore_export_destination( destination=destination_path, backup=backup, had_previous=had_previous, + expected_identity=publication_state["published_identity"], ) except Exception as cleanup_error: _raise_export_cleanup_failed( From 6f81a18d799ee258e3f1cd73feee3e7db4541da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:46:18 +0300 Subject: [PATCH 097/119] Cover catalog gates for in-place lifecycle --- tests/test_inplace_transform.py | 46 +++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 65b9172..d2e8b76 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -583,3 +583,49 @@ def test_in_place_audit_relation_remains_catalog_owned(catalog) -> None: ) assert dataset["dataset"]["dataset_id"] == dataset_id + +def test_schema_install_requires_initialized_verified_catalog(tmp_path) -> None: + path = tmp_path / "empty.sqlite" + url = f"sqlite:///{path}" + + with pytest.raises( + openstatspec.UnsupportedOperationError, + match="explicit catalog initialization", + ): + openstatspec.install_in_place_transformation_schema(database_url=url) + + connection = sqlite3.connect(path) + assert connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ).fetchall() == [] + + +def test_public_apply_rejects_divergent_catalog_before_mutation(catalog) -> None: + url, path, dataset_id, table_name = catalog + connection = sqlite3.connect(path) + connection.execute( + "DELETE FROM dataset_catalog WHERE dataset_id = ?", (dataset_id,) + ) + connection.commit() + before = connection.execute( + f'SELECT score FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() + + with pytest.raises( + openstatspec.UnsupportedOperationError, + match="explicit catalog initialization", + ): + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text="RECODE score (1 = 9).", + actor="test-agent", + ) + + assert connection.execute( + f'SELECT score FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() == before + assert connection.execute( + "SELECT COUNT(*) FROM transformation_apply" + ).fetchone() == (0,) + From 70991e9f5e3d96c728301ae87ac040d1bc2e0e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:47:04 +0300 Subject: [PATCH 098/119] Cover concurrent export recovery ownership --- tests/test_sav_sqlite.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index fbf094d..6008946 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -237,6 +237,7 @@ def test_export_recovery_preserves_dangling_destination_symlink(tmp_path) -> Non destination=destination, backup=backup, had_previous=True, + expected_identity=sav_module._destination_identity(destination), ) assert destination.is_symlink() @@ -254,3 +255,31 @@ def test_successful_export_cleanup_removes_dangling_symlink_backup( backup.unlink() assert sav_module._path_entry_exists(backup) is False + +def test_export_recovery_does_not_replace_newer_concurrent_destination( + tmp_path, +) -> None: + destination = tmp_path / "destination.sav" + backup = tmp_path / "destination.previous" + later_export = tmp_path / "later-export.sav" + backup.write_bytes(b"original destination") + destination.write_bytes(b"earlier operation publication") + earlier_identity = sav_module._destination_identity(destination) + + later_export.write_bytes(b"later successful export") + later_export.replace(destination) + + with pytest.raises( + FileExistsError, + match="no longer owned by this operation", + ): + sav_module._restore_export_destination( + destination=destination, + backup=backup, + had_previous=True, + expected_identity=earlier_identity, + ) + + assert destination.read_bytes() == b"later successful export" + assert backup.read_bytes() == b"original destination" + From b45a4d0ab99846c3f56ee6e64819b028c7e73411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:52:08 +0300 Subject: [PATCH 099/119] Delete actual legacy row in divergence regression --- tests/test_inplace_transform.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index d2e8b76..d0c7b6b 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -603,9 +603,8 @@ def test_schema_install_requires_initialized_verified_catalog(tmp_path) -> None: def test_public_apply_rejects_divergent_catalog_before_mutation(catalog) -> None: url, path, dataset_id, table_name = catalog connection = sqlite3.connect(path) - connection.execute( - "DELETE FROM dataset_catalog WHERE dataset_id = ?", (dataset_id,) - ) + deleted = connection.execute("DELETE FROM dataset_catalog") + assert deleted.rowcount == 1 connection.commit() before = connection.execute( f'SELECT score FROM "{table_name}" ORDER BY __case_ordinal' From b68f7ceec0dc0e9e9e9c7de859ab58f09eb5f252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 21:59:53 +0300 Subject: [PATCH 100/119] Scope audit migration and reject ephemeral catalog URLs --- src/openstatspec/sql/wide.py | 45 +++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 0d0d607..b7d6559 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -14,6 +14,7 @@ from sqlalchemy import delete, BigInteger, Boolean, Column, DateTime, Float, Integer, MetaData, String, Table, Text, create_engine, insert, inspect, select, text, update from sqlalchemy.dialects import mysql +from sqlalchemy.engine import make_url from ..core import UnsupportedOperationError, safe_error_identity as _safe_error_identity from .capabilities import ( active_connection, dolt_operational_write_enabled, effective_profile, @@ -1226,9 +1227,11 @@ def _catalog_missing_columns( def _catalog_missing_columns_are_migratable( missing: Mapping[str, set[str]], + *, allowed: Mapping[str, set[str]] | None = None, ) -> bool: + allowed_columns = _MIGRATABLE_CATALOG_COLUMNS if allowed is None else allowed return bool(missing) and all( - columns <= _MIGRATABLE_CATALOG_COLUMNS.get(table_name, set()) + columns <= allowed_columns.get(table_name, set()) for table_name, columns in missing.items() ) @@ -1353,7 +1356,10 @@ def _catalog_dataset_bijection_state( return "valid" if set(legacy_rows) == set(normative_rows) else "unverified" -def _catalog_state(connection: Any, normative: Any, legacy: Iterable[Table]) -> str: +def _catalog_state( + connection: Any, normative: Any, legacy: Iterable[Table], + *, allowed_migrations: Mapping[str, set[str]] | None = None, +) -> str: inspector = inspect(connection) existing_tables = set(inspector.get_table_names()) existing_views = set(inspector.get_view_names()) @@ -1408,7 +1414,9 @@ def _catalog_state(connection: Any, normative: Any, legacy: Iterable[Table]) -> if missing_columns: return ( "migration_required" - if _catalog_missing_columns_are_migratable(missing_columns) + if _catalog_missing_columns_are_migratable( + missing_columns, allowed=allowed_migrations, + ) else "unverified" ) mapping_state = _catalog_dataset_bijection_state( @@ -1421,19 +1429,32 @@ def _catalog_state(connection: Any, normative: Any, legacy: Iterable[Table]) -> def _require_verified_catalog( connection: Any, normative: Any, legacy: Iterable[Table], + *, allowed_migrations: Mapping[str, set[str]] | None = None, ) -> None: - state = _catalog_state(connection, normative, legacy) - if state != "verified": + state = _catalog_state( + connection, normative, legacy, allowed_migrations=allowed_migrations, + ) + accepted_states = ( + {"verified", "migration_required"} + if allowed_migrations is not None + else {"verified"} + ) + if state not in accepted_states: raise UnsupportedOperationError( f"The selected OpenStatSpec catalog is {state}; run explicit catalog initialization first." ) -def require_verified_catalog(connection: Any) -> None: - """Require full OpenStatSpec catalog ownership, shape, and dataset bijection.""" +def require_verified_catalog( + connection: Any, + *, allowed_migrations: Mapping[str, set[str]] | None = None, +) -> None: + """Require catalog ownership, shape, bijection, and only explicit migrations.""" metadata = MetaData() legacy, normative = _catalog_layout(metadata) - _require_verified_catalog(connection, normative, legacy) + _require_verified_catalog( + connection, normative, legacy, allowed_migrations=allowed_migrations, + ) def _catalog_snapshot( @@ -1554,6 +1575,14 @@ def initialize_wide_catalog( ) -> dict[str, Any]: """Install or explicitly migrate a dedicated catalog after server preflight.""" validate_connection_url(database_url) + parsed_url = make_url(database_url) + if ( + parsed_url.get_backend_name() == "sqlite" + and parsed_url.database in {None, "", ":memory:"} + ): + raise UnsupportedOperationError( + "Catalog initialization requires a persistent SQLite database URL." + ) profile, active = effective_profile( database_url, dolt_conformance_source=dolt_conformance_source, ) From cbafaee72d74617cd3f6625673fdae74ac974020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:00:09 +0300 Subject: [PATCH 101/119] Verify audit migration before and after upgrade --- src/openstatspec/sql/inplace_transform.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/inplace_transform.py b/src/openstatspec/sql/inplace_transform.py index 0c29b0c..298d0a1 100644 --- a/src/openstatspec/sql/inplace_transform.py +++ b/src/openstatspec/sql/inplace_transform.py @@ -571,7 +571,14 @@ def install_in_place_transformation_schema( engine = create_engine(database_url) try: with engine.begin() as connection: - require_verified_catalog(connection) + require_verified_catalog( + connection, + allowed_migrations={ + "transformation_apply": { + "source_kind", "frontend_contract", + }, + }, + ) apply_audit_catalog(MetaData()).create(connection, checkfirst=True) columns = { str(column["name"]) @@ -588,6 +595,7 @@ def install_in_place_transformation_schema( f"ALTER TABLE {quote('transformation_apply')} " f"ADD COLUMN {quote(name)} {sql_type}" ) + require_verified_catalog(connection) finally: engine.dispose() From c8b4fadc83c5e5f77fa25e65299bae2e8f85dada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:00:31 +0300 Subject: [PATCH 102/119] Cover legacy in-place audit migration --- tests/test_inplace_transform.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index d0c7b6b..0a5f764 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -628,3 +628,29 @@ def test_public_apply_rejects_divergent_catalog_before_mutation(catalog) -> None "SELECT COUNT(*) FROM transformation_apply" ).fetchone() == (0,) +def test_schema_installer_upgrades_legacy_apply_audit_columns(catalog) -> None: + url, path, dataset_id, _table_name = catalog + connection = sqlite3.connect(path) + connection.execute( + "ALTER TABLE transformation_apply DROP COLUMN source_kind" + ) + connection.execute( + "ALTER TABLE transformation_apply DROP COLUMN frontend_contract" + ) + connection.commit() + connection.close() + + openstatspec.install_in_place_transformation_schema(database_url=url) + + connection = sqlite3.connect(path) + columns = { + row[1] for row in connection.execute( + "PRAGMA table_info(transformation_apply)" + ) + } + connection.close() + assert {"source_kind", "frontend_contract"} <= columns + assert openstatspec.get_dataset( + database_url=url, dataset_id=dataset_id, kind="core", + )["dataset"]["dataset_id"] == dataset_id + From 85d4a850c8ba2d1273f4321e5bc36c5e7521ee0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:00:51 +0300 Subject: [PATCH 103/119] Cover ephemeral SQLite initialization rejection --- tests/test_catalog_lifecycle.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 7596278..74430ae 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -416,3 +416,12 @@ def test_reflected_mysql_integer_display_width_is_not_semantic(): assert wide._normalized_sql_type( inspector, mysql.VARCHAR(length=255), ) == "VARCHAR(255)" + +@pytest.mark.parametrize("database_url", ["sqlite://", "sqlite:///:memory:"]) +def test_catalog_initialization_rejects_ephemeral_sqlite_url(database_url): + with pytest.raises( + UnsupportedOperationError, + match="requires a persistent SQLite database URL", + ): + openstatspec.initialize_catalog(database_url=database_url) + From 5c4bf8b8cd5b4e6765f2c26398d8de1c0a292f4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:11:28 +0300 Subject: [PATCH 104/119] Keep export audits out of import fidelity reads --- src/openstatspec/sql/wide.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index b7d6559..1abf34d 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -2877,7 +2877,10 @@ def read_fidelity_events( _require_verified_catalog(connection, normative, legacy) events = connection.execute( select(fidelity_event_catalog) - .where(fidelity_event_catalog.c.dataset_id == dataset_id) + .where( + fidelity_event_catalog.c.dataset_id == dataset_id, + fidelity_event_catalog.c.direction == "import", + ) .order_by(fidelity_event_catalog.c.code) ).mappings().all() return tuple({ From 3db6c19cd5a6ac092d0769ff778d2790e0df1e4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:12:01 +0300 Subject: [PATCH 105/119] Claim export destinations without clobbering races --- src/openstatspec/spss/sav.py | 53 ++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index 4b8276e..f43b894 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -330,6 +330,32 @@ def _reserve_export_backup(destination: Path) -> Path: return Path(name) +def _publish_staged_destination( + *, staged: Path, destination: Path, backup: Path, + state: dict[str, Any], +) -> None: + """Publish without overwriting an entry created during export preparation.""" + state.update({ + "had_previous": False, + "backup_installed": False, + "published_identity": None, + }) + if _path_entry_exists(destination): + os.replace(destination, backup) + state.update({"had_previous": True, "backup_installed": True}) + else: + backup.unlink(missing_ok=True) + + # The hard-link claim is atomic and fails if another process publishes the + # destination after the check above. Staging is on the same filesystem. + os.link(staged, destination) + published_identity = _destination_identity(destination) + if published_identity is None: + raise FileNotFoundError("The published export destination disappeared.") + state["published_identity"] = published_identity + staged.unlink() + + def _restore_export_destination( *, destination: Path, backup: Path, had_previous: bool, expected_identity: tuple[int, int] | None | object = ( @@ -651,29 +677,32 @@ def export_sav_dataset( raise backup_installed = False try: - if had_previous: - os.replace(destination_path, backup) - backup_installed = True - os.replace(staged_destination, destination_path) - published_identity = _destination_identity(destination_path) - if published_identity is None: - raise FileNotFoundError( - "The published export destination disappeared." - ) + _publish_staged_destination( + staged=staged_destination, + destination=destination_path, + backup=backup, + state=publication_state, + ) + had_previous = publication_state["had_previous"] + backup_installed = publication_state["backup_installed"] publication_state.update({ "published": True, - "published_identity": published_identity, "backup": backup, "staged": staged_destination, - "had_previous": had_previous, "operation_id": operation_id, }) except Exception as publish_error: + had_previous = publication_state.get("had_previous", had_previous) + backup_installed = publication_state.get( + "backup_installed", backup_installed, + ) + published_identity = publication_state.get("published_identity") if backup_installed or not had_previous: try: _restore_export_destination( destination=destination_path, backup=backup, - had_previous=had_previous, expected_identity=None, + had_previous=had_previous, + expected_identity=published_identity, ) except Exception as cleanup_error: _raise_export_cleanup_failed( From 6252f70d9c1f4ca7980be30c0b9d23c00d74d7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:12:21 +0300 Subject: [PATCH 106/119] Cover import-only fidelity reads --- tests/test_catalog_lifecycle.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 74430ae..e895756 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -425,3 +425,36 @@ def test_catalog_initialization_rejects_ephemeral_sqlite_url(database_url): ): openstatspec.initialize_catalog(database_url=database_url) +def test_import_fidelity_reader_excludes_export_operational_events(tmp_path): + path = tmp_path / "fidelity-directions.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": "ok"}], + variables=_variables(), + ) + connection = sqlite3.connect(path) + connection.executemany( + "INSERT INTO fidelity_event_catalog " + "(operation_id, ordinal, dataset_id, direction, severity, detail, " + "details, code) VALUES (?, 1, 'sample', ?, 'error', ?, '{}', ?)", + [ + ("import-event", "import", "source fidelity", "source_loss"), + ("export-event", "export", "transient export failure", "export_failed"), + ], + ) + connection.commit() + connection.close() + + assert wide.read_fidelity_events( + database_url=database, dataset_id="sample", + ) == ({ + "code": "source_loss", + "detail": "source fidelity", + "details": {}, + },) + From f18e02bd00b26dee77ccd2b91bbc4c01e28bfb9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:12:35 +0300 Subject: [PATCH 107/119] Cover export preparation publication race --- tests/test_sav_sqlite.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index 6008946..a517e6b 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -283,3 +283,31 @@ def test_export_recovery_does_not_replace_newer_concurrent_destination( assert destination.read_bytes() == b"later successful export" assert backup.read_bytes() == b"original destination" +def test_export_publication_does_not_clobber_intervening_destination( + tmp_path, monkeypatch, +) -> None: + staged = tmp_path / "staged.sav" + destination = tmp_path / "destination.sav" + backup = tmp_path / "destination.previous" + staged.write_bytes(b"earlier staged export") + real_link = sav_module.os.link + + def publish_concurrently(source, target): + destination.write_bytes(b"intervening successful export") + return real_link(source, target) + + monkeypatch.setattr(sav_module.os, "link", publish_concurrently) + state = {} + with pytest.raises(FileExistsError): + sav_module._publish_staged_destination( + staged=staged, + destination=destination, + backup=backup, + state=state, + ) + + assert destination.read_bytes() == b"intervening successful export" + assert staged.read_bytes() == b"earlier staged export" + assert sav_module._path_entry_exists(backup) is False + assert state["published_identity"] is None + From 1185c3bf32d2f7cd410234c2c95008a7e1192094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:21:26 +0300 Subject: [PATCH 108/119] Verify variable mappings and concurrent init ownership --- src/openstatspec/sql/wide.py | 61 ++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 1abf34d..f2f681c 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1356,6 +1356,52 @@ def _catalog_dataset_bijection_state( return "valid" if set(legacy_rows) == set(normative_rows) else "unverified" +def _catalog_variable_bijection_state( + connection: Any, *, normative: Any, legacy: Iterable[Table], +) -> str: + """Require exact legacy-to-normative variable identity mappings.""" + variables = tuple(legacy)[1] + legacy_rows = [ + ( + row["dataset_id"], row["ordinal"], row["source_name"], + row["physical_name"], + ) + for row in connection.execute(select( + variables.c.dataset_id, variables.c.ordinal, + variables.c.source_name, variables.c.physical_name, + )).mappings() + ] + normative_rows = [ + ( + row["dataset_name"], row["source_ordinal"], row["source_name"], + row["physical_name"], + ) + for row in connection.execute( + select( + normative.dataset.c.dataset_name, + normative.variable.c.source_ordinal, + normative.variable.c.source_name, + normative.variable.c.physical_name, + ).join( + normative.variable, + normative.variable.c.dataset_id == normative.dataset.c.dataset_id, + ) + ).mappings() + ] + for rows in (legacy_rows, normative_rows): + if any( + not isinstance(dataset_name, str) or not dataset_name.strip() + or not isinstance(ordinal, int) or ordinal < 1 + or not isinstance(source_name, str) or not source_name.strip() + or not isinstance(physical_name, str) or not physical_name.strip() + for dataset_name, ordinal, source_name, physical_name in rows + ): + return "unverified" + if len(set(rows)) != len(rows): + return "ambiguous" + return "valid" if set(legacy_rows) == set(normative_rows) else "unverified" + + def _catalog_state( connection: Any, normative: Any, legacy: Iterable[Table], *, allowed_migrations: Mapping[str, set[str]] | None = None, @@ -1424,6 +1470,11 @@ def _catalog_state( ) if mapping_state != "valid": return mapping_state + variable_mapping_state = _catalog_variable_bijection_state( + connection, normative=normative, legacy=legacy, + ) + if variable_mapping_state != "valid": + return variable_mapping_state return "verified" @@ -1472,9 +1523,12 @@ def _catalog_snapshot( def _compensate_catalog_initialization( connection: Any, *, metadata: MetaData, before_tables: set[str], - before_columns: Mapping[str, set[str]], + before_columns: Mapping[str, set[str]], normative: Any, + legacy: Iterable[Table], ) -> None: - """Restore the pre-initialization relation/column inventory after DDL failure.""" + """Restore only when no concurrent initializer completed the catalog.""" + if _catalog_state(connection, normative, legacy) == "verified": + return current_tables = set(inspect(connection).get_table_names()) for table in reversed(metadata.sorted_tables): if table.name in current_tables and table.name not in before_tables: @@ -1626,7 +1680,8 @@ def initialize_wide_catalog( with connection.begin(): _compensate_catalog_initialization( connection, metadata=metadata, before_tables=before_tables, - before_columns=before_columns, + before_columns=before_columns, normative=normative, + legacy=legacy, ) except Exception as cleanup_error: inventory = _catalog_residual_inventory( From 0f3c88535b4e07cccbda52817922c7d1f41793d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:21:46 +0300 Subject: [PATCH 109/119] Cover concurrent initializer compensation guard --- tests/test_catalog_lifecycle.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index e895756..0f3e677 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -2,7 +2,7 @@ import sqlite3 import pytest -from sqlalchemy import Table +from sqlalchemy import MetaData, Table import openstatspec import openstatspec.sql.wide as wide @@ -458,3 +458,27 @@ def test_import_fidelity_reader_excludes_export_operational_events(tmp_path): "details": {}, },) +def test_stale_initializer_compensation_preserves_concurrent_verified_catalog( + tmp_path, +): + path = tmp_path / "concurrent-init.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + before = _table_names(path) + metadata = MetaData() + legacy, normative = wide._catalog_layout(metadata) + engine = wide.create_engine(database) + with engine.begin() as connection: + wide._compensate_catalog_initialization( + connection, + metadata=metadata, + before_tables=set(), + before_columns={}, + normative=normative, + legacy=legacy, + ) + wide.require_verified_catalog(connection) + engine.dispose() + + assert _table_names(path) == before + From 04f86fa8be27f122f5e12cbe1ec8dd9318159e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:22:02 +0300 Subject: [PATCH 110/119] Cover variable catalog bijection gate --- tests/test_inplace_transform.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_inplace_transform.py b/tests/test_inplace_transform.py index 0a5f764..10346e4 100644 --- a/tests/test_inplace_transform.py +++ b/tests/test_inplace_transform.py @@ -654,3 +654,33 @@ def test_schema_installer_upgrades_legacy_apply_audit_columns(catalog) -> None: database_url=url, dataset_id=dataset_id, kind="core", )["dataset"]["dataset_id"] == dataset_id +def test_public_apply_rejects_divergent_variable_mapping_before_mutation( + catalog, +) -> None: + url, path, dataset_id, table_name = catalog + connection = sqlite3.connect(path) + deleted = connection.execute("DELETE FROM variable_catalog") + assert deleted.rowcount == 1 + connection.commit() + before = connection.execute( + f'SELECT score FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() + + with pytest.raises( + openstatspec.UnsupportedOperationError, + match="explicit catalog initialization", + ): + openstatspec.apply_spss_in_place( + database_url=url, + dataset_id=dataset_id, + source_text="VARIABLE LABELS score 'Changed'.", + actor="test-agent", + ) + + assert connection.execute( + f'SELECT score FROM "{table_name}" ORDER BY __case_ordinal' + ).fetchall() == before + assert connection.execute( + "SELECT COUNT(*) FROM transformation_apply" + ).fetchone() == (0,) + From a6276456af8d0ed7ba799612d294dcfff3186ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:30:46 +0300 Subject: [PATCH 111/119] Serialize MySQL-wire catalog initialization --- src/openstatspec/sql/wide.py | 333 +++++++++++++++++++---------------- 1 file changed, 183 insertions(+), 150 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index f2f681c..9570c66 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1622,6 +1622,36 @@ def dolt_state_snapshot( } +@contextmanager +def _catalog_initialization_serialization( + connection: Any, *, profile_name: str, +): + """Serialize non-transactional catalog DDL on one database server.""" + if profile_name not in MYSQL_WIRE_PROFILES: + yield + return + lock_name = "openstatspec.catalog-initialize.v1" + acquired = connection.execute( + text("SELECT GET_LOCK(:lock_name, 30)"), {"lock_name": lock_name}, + ).scalar_one() + connection.commit() + if acquired != 1: + raise UnsupportedOperationError( + "Could not acquire the catalog initialization lock." + ) + try: + yield + finally: + try: + connection.execute( + text("SELECT RELEASE_LOCK(:lock_name)"), + {"lock_name": lock_name}, + ) + connection.commit() + except Exception: + connection.invalidate() + + def initialize_wide_catalog( *, database_url: str, @@ -1645,48 +1675,99 @@ def initialize_wide_catalog( legacy, normative = _catalog_layout(metadata) datasets, variables, multiple_response = legacy[:3] with engine.connect() as connection: - state = _catalog_state(connection, normative, legacy) - if state not in {"absent", "verified", "migration_required"}: - raise UnsupportedOperationError( - f"The selected database catalog is {state}; initialization is not permitted." - ) - before_tables, before_columns = _catalog_snapshot(connection) - pre_dolt_state = _capture_dolt_state( - connection, profile_name=profile.name, audit_relations=set(), - ) - _require_dolt_working_set_binding( - pre_dolt_state, active, phase="catalog initialization preflight", - ) - connection.rollback() - try: - with connection.begin(): - create_normative_catalog(connection, normative) - metadata.create_all(connection, tables=list(legacy)) - _migrate_catalog_columns( - connection, datasets, variables, multiple_response, + with _catalog_initialization_serialization( + connection, profile_name=profile.name, + ): + state = _catalog_state(connection, normative, legacy) + if state not in {"absent", "verified", "migration_required"}: + raise UnsupportedOperationError( + f"The selected database catalog is {state}; initialization is not permitted." ) - _require_verified_catalog(connection, normative, legacy) - post_dolt_state = _capture_dolt_state( + before_tables, before_columns = _catalog_snapshot(connection) + pre_dolt_state = _capture_dolt_state( connection, profile_name=profile.name, audit_relations=set(), ) _require_dolt_working_set_binding( - post_dolt_state, active, phase="catalog initialization completion", + pre_dolt_state, active, phase="catalog initialization preflight", ) - _require_dolt_success_identity( - pre_dolt_state, post_dolt_state, phase="catalog initialization", - ) - except Exception as install_error: + connection.rollback() try: with connection.begin(): - _compensate_catalog_initialization( - connection, metadata=metadata, before_tables=before_tables, - before_columns=before_columns, normative=normative, - legacy=legacy, + create_normative_catalog(connection, normative) + metadata.create_all(connection, tables=list(legacy)) + _migrate_catalog_columns( + connection, datasets, variables, multiple_response, ) - except Exception as cleanup_error: - inventory = _catalog_residual_inventory( - engine, before_tables=before_tables, before_columns=before_columns, + _require_verified_catalog(connection, normative, legacy) + post_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + _require_dolt_working_set_binding( + post_dolt_state, active, phase="catalog initialization completion", + ) + _require_dolt_success_identity( + pre_dolt_state, post_dolt_state, phase="catalog initialization", ) + except Exception as install_error: + try: + with connection.begin(): + _compensate_catalog_initialization( + connection, metadata=metadata, before_tables=before_tables, + before_columns=before_columns, normative=normative, + legacy=legacy, + ) + except Exception as cleanup_error: + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + try: + after_dolt_state = _capture_dolt_state( + connection, profile_name=profile.name, audit_relations=set(), + ) + dolt_boundary = _dolt_failure_boundary_evidence( + pre_dolt_state, after_dolt_state, + ) + except Exception as snapshot_error: + dolt_boundary = { + "applicable": profile.name == "dolt", + "verified": False, + "snapshot_fault": _safe_error_identity( + snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", + ), + } + raise ImportRecoveryError( + "cleanup_failed", + "Catalog initialization failed and its DDL compensation also failed.", + details={ + "subcode": "catalog_install_cleanup_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _safe_error_identity( + cleanup_error, phase="catalog_compensation", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": { + "procedure_id": "openstatspec.catalog-init-compensation.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted( + table.name for table in metadata.tables.values() + ), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "cleanup_attempted": True, + "cleanup_succeeded": False, + "preexisting_unverified_catalog_mutation_forbidden": True, + "dolt_failure_boundary": dolt_boundary, + }, + "success_forbidden": True, + }, + ) from cleanup_error try: after_dolt_state = _capture_dolt_state( connection, profile_name=profile.name, audit_relations=set(), @@ -1695,125 +1776,77 @@ def initialize_wide_catalog( pre_dolt_state, after_dolt_state, ) except Exception as snapshot_error: - dolt_boundary = { - "applicable": profile.name == "dolt", - "verified": False, - "snapshot_fault": _safe_error_identity( - snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", - ), + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + recovery = { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted(metadata.tables), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": { + "applicable": profile.name == "dolt", + "verified": False, + }, } - raise ImportRecoveryError( - "cleanup_failed", - "Catalog initialization failed and its DDL compensation also failed.", - details={ - "subcode": "catalog_install_cleanup_failed", - "original_cause": _safe_error_identity( - install_error, phase="catalog_initialization", - ), - "cleanup_fault": _safe_error_identity( - cleanup_error, phase="catalog_compensation", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": { - "procedure_id": "openstatspec.catalog-init-compensation.v1", - "action_id": _canonical_sha256({ - "namespace": active["catalog_binding"]["namespace"], - "before_tables": sorted(before_tables), - }), - "targets": { - "namespace": active["catalog_binding"]["namespace"], - "catalog_relations": sorted( - table.name for table in metadata.tables.values() - ), - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "cleanup_attempted": True, - "cleanup_succeeded": False, - "preexisting_unverified_catalog_mutation_forbidden": True, - "dolt_failure_boundary": dolt_boundary, + raise ImportRecoveryError( + "cleanup_failed", + "Catalog compensation completed but Dolt state could not be verified.", + details={ + "subcode": "dolt_state_capture_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _safe_error_identity( + snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": recovery, + "success_forbidden": True, }, - "success_forbidden": True, - }, - ) from cleanup_error - try: - after_dolt_state = _capture_dolt_state( - connection, profile_name=profile.name, audit_relations=set(), - ) - dolt_boundary = _dolt_failure_boundary_evidence( - pre_dolt_state, after_dolt_state, - ) - except Exception as snapshot_error: - inventory = _catalog_residual_inventory( - engine, before_tables=before_tables, before_columns=before_columns, - ) - recovery = { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": _canonical_sha256({ - "namespace": active["catalog_binding"]["namespace"], - "before_tables": sorted(before_tables), - }), - "targets": { - "namespace": active["catalog_binding"]["namespace"], - "catalog_relations": sorted(metadata.tables), - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "dolt_failure_boundary": { - "applicable": profile.name == "dolt", - "verified": False, - }, - } - raise ImportRecoveryError( - "cleanup_failed", - "Catalog compensation completed but Dolt state could not be verified.", - details={ - "subcode": "dolt_state_capture_failed", - "original_cause": _safe_error_identity( - install_error, phase="catalog_initialization", - ), - "cleanup_fault": _safe_error_identity( - snapshot_error, phase="post_catalog_cleanup_dolt_state_capture", - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": recovery, - "success_forbidden": True, - }, - ) from snapshot_error - if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): - inventory = _catalog_residual_inventory( - engine, before_tables=before_tables, before_columns=before_columns, - ) - recovery = { - "procedure_id": "openstatspec.dolt-failure-boundary.v1", - "action_id": _canonical_sha256({ - "namespace": active["catalog_binding"]["namespace"], - "before_tables": sorted(before_tables), - }), - "targets": { - "namespace": active["catalog_binding"]["namespace"], - "catalog_relations": sorted(metadata.tables), - }, - "residual_inventory_sha256": _canonical_sha256(inventory), - "dolt_failure_boundary": dolt_boundary, - } - raise ImportRecoveryError( - "cleanup_failed", - "Catalog compensation did not preserve Dolt failure-boundary invariants.", - details={ - "subcode": "dolt_state_invariant_failed", - "original_cause": _safe_error_identity( - install_error, phase="catalog_initialization", - ), - "cleanup_fault": _verification_fault_identity( - "dolt_state_invariant_failed", - phase="post_catalog_cleanup_dolt_state_verification", - evidence=dolt_boundary, - ), - "residual_object_inventory": inventory, - "deterministic_recovery_evidence": recovery, - "success_forbidden": True, - }, - ) from install_error - raise + ) from snapshot_error + if dolt_boundary.get("applicable") and not dolt_boundary.get("verified"): + inventory = _catalog_residual_inventory( + engine, before_tables=before_tables, before_columns=before_columns, + ) + recovery = { + "procedure_id": "openstatspec.dolt-failure-boundary.v1", + "action_id": _canonical_sha256({ + "namespace": active["catalog_binding"]["namespace"], + "before_tables": sorted(before_tables), + }), + "targets": { + "namespace": active["catalog_binding"]["namespace"], + "catalog_relations": sorted(metadata.tables), + }, + "residual_inventory_sha256": _canonical_sha256(inventory), + "dolt_failure_boundary": dolt_boundary, + } + raise ImportRecoveryError( + "cleanup_failed", + "Catalog compensation did not preserve Dolt failure-boundary invariants.", + details={ + "subcode": "dolt_state_invariant_failed", + "original_cause": _safe_error_identity( + install_error, phase="catalog_initialization", + ), + "cleanup_fault": _verification_fault_identity( + "dolt_state_invariant_failed", + phase="post_catalog_cleanup_dolt_state_verification", + evidence=dolt_boundary, + ), + "residual_object_inventory": inventory, + "deterministic_recovery_evidence": recovery, + "success_forbidden": True, + }, + ) from install_error + raise return { "profile": profile.name, "server_version": active["server_version"], From 6d7e3e2f1d4df188b712340528df6eb7c05d594f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 22:31:28 +0300 Subject: [PATCH 112/119] Cover catalog initialization serialization boundary --- tests/test_catalog_lifecycle.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 0f3e677..231dab7 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -482,3 +482,35 @@ def test_stale_initializer_compensation_preserves_concurrent_verified_catalog( assert _table_names(path) == before +def test_mysql_catalog_initialization_lock_spans_mutation_boundary(): + events = [] + + class Result: + @staticmethod + def scalar_one(): + return 1 + + class Connection: + def execute(self, statement, parameters): + events.append((str(statement), parameters["lock_name"])) + return Result() + + def commit(self): + events.append(("commit", None)) + + def invalidate(self): + raise AssertionError("healthy lock connection must not be invalidated") + + with wide._catalog_initialization_serialization( + Connection(), profile_name="mysql", + ): + events.append(("mutation", None)) + + assert [event[0] for event in events] == [ + "SELECT GET_LOCK(:lock_name, 30)", + "commit", + "mutation", + "SELECT RELEASE_LOCK(:lock_name)", + "commit", + ] + From c566aaa50f323db9e42c4073aebb434bde28f22f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:06:36 +0300 Subject: [PATCH 113/119] fix: avoid stale PostgreSQL catalog cleanup --- src/openstatspec/sql/wide.py | 25 +++++++++++++++++-------- tests/test_catalog_lifecycle.py | 24 +++++++++++++++++++++++- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 9570c66..e3bf337 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1550,6 +1550,11 @@ def _compensate_catalog_initialization( )) +def _requires_compensating_catalog_cleanup(profile_name: str) -> bool: + """Only PostgreSQL rolls back catalog DDL without stale compensation.""" + return profile_name != "postgresql" + + def _catalog_residual_inventory( engine: Any, *, before_tables: set[str], before_columns: Mapping[str, set[str]], @@ -1709,14 +1714,18 @@ def initialize_wide_catalog( pre_dolt_state, post_dolt_state, phase="catalog initialization", ) except Exception as install_error: - try: - with connection.begin(): - _compensate_catalog_initialization( - connection, metadata=metadata, before_tables=before_tables, - before_columns=before_columns, normative=normative, - legacy=legacy, - ) - except Exception as cleanup_error: + cleanup_error = None + if _requires_compensating_catalog_cleanup(profile.name): + try: + with connection.begin(): + _compensate_catalog_initialization( + connection, metadata=metadata, before_tables=before_tables, + before_columns=before_columns, normative=normative, + legacy=legacy, + ) + except Exception as error: + cleanup_error = error + if cleanup_error is not None: inventory = _catalog_residual_inventory( engine, before_tables=before_tables, before_columns=before_columns, ) diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 231dab7..719a83a 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -7,7 +7,9 @@ import openstatspec import openstatspec.sql.wide as wide from openstatspec.core import UnsupportedOperationError -from openstatspec.sql.profiles import MYSQL, TargetCapabilityExceededError, preflight +from openstatspec.sql.profiles import ( + MYSQL, POSTGRESQL, TargetCapabilityExceededError, preflight, +) from openstatspec.sql.wide import ( ImportRecoveryError, _bounded_batches, @@ -514,3 +516,23 @@ def invalidate(self): "commit", ] +def test_postgresql_initializer_relies_on_transaction_rollback(tmp_path, monkeypatch): + path = tmp_path / "postgresql-rollback.sqlite" + + def fail_migration(*_args, **_kwargs): + raise RuntimeError("injected catalog migration failure") + + def compensation_must_not_run(*_args, **_kwargs): + raise AssertionError("PostgreSQL must not use stale DDL compensation") + + monkeypatch.setattr( + wide, "effective_profile", lambda _url, **_kwargs: (POSTGRESQL, {}), + ) + monkeypatch.setattr(wide, "_migrate_catalog_columns", fail_migration) + monkeypatch.setattr( + wide, "_compensate_catalog_initialization", compensation_must_not_run, + ) + + with pytest.raises(RuntimeError, match="injected catalog migration failure"): + openstatspec.initialize_catalog(database_url=f"sqlite:///{path}") + From f17b3b277b0a11272ae5dbc63b03eb9fac966898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:13:01 +0300 Subject: [PATCH 114/119] Serialize SAV export publication --- src/openstatspec/spss/sav.py | 49 +++++++++++++++++++++++++++++++++++- tests/test_sav_sqlite.py | 34 +++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index f43b894..866e21f 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -12,9 +12,15 @@ import math from contextlib import ExitStack, contextmanager from datetime import UTC, datetime +from functools import wraps from pathlib import Path from tempfile import mkstemp, TemporaryDirectory -from typing import Any +from typing import Any, Callable + +try: # POSIX locks let independent export processes share one publication gate. + import fcntl +except ImportError: # pragma: no cover - the supported CI/runtime is POSIX. + fcntl = None import pandas as pd import pyspssio @@ -356,6 +362,46 @@ def _publish_staged_destination( staged.unlink() + + +@contextmanager +def _export_destination_lock(destination: Path): + """Serialize export publication and recovery for one destination directory. + + The lock covers the existence observation, backup move, publication, and + compensating restore. Without it, two exporters could each observe the + old destination, and the later exporter could move or delete the newer + export while attempting to restore its own failure. + """ + if fcntl is None: # pragma: no cover - see import guard above. + raise UnsupportedOperationError( + "SAV export publication requires POSIX advisory file locking." + ) + descriptor = os.open(destination.parent, os.O_RDONLY) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + yield + finally: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + + +def _serialize_export_publication( + export: Callable[..., dict[str, Any]], +) -> Callable[..., dict[str, Any]]: + """Guard a complete export, including any post-publication recovery.""" + @wraps(export) + def guarded(*args: Any, **kwargs: Any) -> dict[str, Any]: + destination = kwargs.get("destination") + if destination is None: + raise TypeError("export requires a destination keyword argument") + with _export_destination_lock(Path(destination)): + return export(*args, **kwargs) + return guarded + + def _restore_export_destination( *, destination: Path, backup: Path, had_previous: bool, expected_identity: tuple[int, int] | None | object = ( @@ -568,6 +614,7 @@ def _export_staging_directory( raise +@_serialize_export_publication def export_sav_dataset( *, database_url: str, dataset_id: str, destination: str | Path, allow_loss: tuple[str, ...] = (), legacy_locale: str | None = None, diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index a517e6b..c64071f 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -1,6 +1,7 @@ import hashlib import json import sqlite3 +import threading import pandas as pd import pyspssio @@ -15,6 +16,39 @@ _COMPAT_NAME_LOSS = _REQUIRED_ENGINE_LOSS + + +def test_export_destination_lock_serializes_observation_and_publication(tmp_path) -> None: + """A second exporter cannot replace a destination seen by the first.""" + destination = tmp_path / "shared-output.sav" + first_entered = threading.Event() + release_first = threading.Event() + second_entered = threading.Event() + + def first_export() -> None: + with sav_module._export_destination_lock(destination): + first_entered.set() + release_first.wait(timeout=2) + + def second_export() -> None: + first_entered.wait(timeout=2) + with sav_module._export_destination_lock(destination): + second_entered.set() + + first = threading.Thread(target=first_export) + second = threading.Thread(target=second_export) + first.start() + assert first_entered.wait(timeout=2) + second.start() + assert not second_entered.wait(timeout=0.1) + release_first.set() + first.join(timeout=2) + second.join(timeout=2) + assert not first.is_alive() + assert not second.is_alive() + assert second_entered.is_set() + + def test_pyspssio_round_trip_uses_one_wide_table_and_catalog(tmp_path) -> None: source = tmp_path / "tiny.sav" database_path = tmp_path / "dataset.sqlite" From 0b1e891891153b0ac7903959acda5d28c046a775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:27:26 +0300 Subject: [PATCH 115/119] Support Windows SAV publication locks --- src/openstatspec/spss/sav.py | 67 +++++++++++++++++++++++++++--------- tests/test_sav_sqlite.py | 39 +++++++++++++++++++++ 2 files changed, 89 insertions(+), 17 deletions(-) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index 866e21f..9056dc0 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -10,6 +10,7 @@ import os import json import math +import ctypes from contextlib import ExitStack, contextmanager from datetime import UTC, datetime from functools import wraps @@ -362,30 +363,62 @@ def _publish_staged_destination( staged.unlink() - - @contextmanager def _export_destination_lock(destination: Path): - """Serialize export publication and recovery for one destination directory. - - The lock covers the existence observation, backup move, publication, and - compensating restore. Without it, two exporters could each observe the - old destination, and the later exporter could move or delete the newer - export while attempting to restore its own failure. - """ - if fcntl is None: # pragma: no cover - see import guard above. - raise UnsupportedOperationError( - "SAV export publication requires POSIX advisory file locking." - ) - descriptor = os.open(destination.parent, os.O_RDONLY) + """Serialize export publication and recovery across cooperating processes.""" + if fcntl is not None: + descriptor = os.open(destination.parent, os.O_RDONLY) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + yield + finally: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + finally: + os.close(descriptor) + return + + kernel32 = _windows_kernel32() + path_identity = os.path.normcase(os.path.abspath(os.fspath(destination))) + mutex_name = "Global\\OpenStatSpec.SAV." + hashlib.sha256( + path_identity.encode("utf-8") + ).hexdigest() + handle = kernel32.CreateMutexW(None, False, mutex_name) + if not handle: + raise ctypes.WinError() + acquired = False try: - fcntl.flock(descriptor, fcntl.LOCK_EX) + wait_result = kernel32.WaitForSingleObject(handle, 0xFFFFFFFF) + if wait_result not in {0x00000000, 0x00000080}: + raise OSError( + f"Windows export mutex wait failed with status {wait_result}." + ) + acquired = True yield finally: try: - fcntl.flock(descriptor, fcntl.LOCK_UN) + if acquired and not kernel32.ReleaseMutex(handle): + raise ctypes.WinError() finally: - os.close(descriptor) + kernel32.CloseHandle(handle) + + +def _windows_kernel32(): + """Return a typed Win32 mutex API for cross-process export locking.""" + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.CreateMutexW.argtypes = ( + ctypes.c_void_p, ctypes.c_bool, ctypes.c_wchar_p, + ) + kernel32.CreateMutexW.restype = ctypes.c_void_p + kernel32.WaitForSingleObject.argtypes = ( + ctypes.c_void_p, ctypes.c_uint32, + ) + kernel32.WaitForSingleObject.restype = ctypes.c_uint32 + kernel32.ReleaseMutex.argtypes = (ctypes.c_void_p,) + kernel32.ReleaseMutex.restype = ctypes.c_bool + kernel32.CloseHandle.argtypes = (ctypes.c_void_p,) + kernel32.CloseHandle.restype = ctypes.c_bool + return kernel32 def _serialize_export_publication( diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index c64071f..4b70a8a 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -49,6 +49,45 @@ def second_export() -> None: assert second_entered.is_set() +def test_export_destination_lock_uses_windows_mutex_without_fcntl( + tmp_path, monkeypatch, +) -> None: + calls = [] + + class Kernel32: + def CreateMutexW(self, security, initially_owned, name): + calls.append(("create", security, initially_owned, name)) + return 123 + + def WaitForSingleObject(self, handle, timeout): + calls.append(("wait", handle, timeout)) + return 0x00000080 + + def ReleaseMutex(self, handle): + calls.append(("release", handle)) + return True + + def CloseHandle(self, handle): + calls.append(("close", handle)) + return True + + monkeypatch.setattr(sav_module, "fcntl", None) + monkeypatch.setattr(sav_module, "_windows_kernel32", Kernel32) + destination = tmp_path / "shared-output.sav" + + with sav_module._export_destination_lock(destination): + calls.append(("critical",)) + + assert calls[0][:3] == ("create", None, False) + assert calls[0][3].startswith("Global\\OpenStatSpec.SAV.") + assert calls[1:] == [ + ("wait", 123, 0xFFFFFFFF), + ("critical",), + ("release", 123), + ("close", 123), + ] + + def test_pyspssio_round_trip_uses_one_wide_table_and_catalog(tmp_path) -> None: source = tmp_path / "tiny.sav" database_path = tmp_path / "dataset.sqlite" From b3a225ec172c217a216e1649891d59efcab6b96b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:39:35 +0300 Subject: [PATCH 116/119] Budget numeric MySQL wire parameters conservatively --- src/openstatspec/sql/profiles.py | 4 +++- tests/test_catalog_lifecycle.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/openstatspec/sql/profiles.py b/src/openstatspec/sql/profiles.py index 7be63ab..4ddcd57 100644 --- a/src/openstatspec/sql/profiles.py +++ b/src/openstatspec/sql/profiles.py @@ -307,7 +307,9 @@ def statement_payload_bytes( size = 32 for variable in variables: if variable.get("storage_kind") == "numeric": - size += 8 + # PyMySQL serializes binary64 parameters as decimal wire text; + # the longest finite literal is 24 bytes (including its sign). + size += 24 else: value = row.get(str(variable["physical_name"]), "") size += len(str(value).encode("utf-8")) + 8 diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index 719a83a..dc0ec0c 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -277,6 +277,19 @@ def test_bounded_batches_never_exceed_statement_payload_limit(): assert batches == [rows[:2], rows[2:]] +def test_bounded_batches_budget_numeric_pymysql_wire_literals(): + variables = [{ + "source_name": "value", + "physical_name": "value", + "storage_kind": "numeric", + }] + rows = [{"value": -1.7976931348623157e+308}] * 2 + + assert wide.statement_payload_bytes(rows[0], variables) == 56 + assert list(_bounded_batches(rows, variables, 111)) == [[rows[0]], [rows[1]]] + assert list(_bounded_batches(rows, variables, 112)) == [rows] + + def test_duplicate_import_failure_preserves_existing_dataset(tmp_path): path = tmp_path / "duplicate.sqlite" database = f"sqlite:///{path}" From cceff0605ca7690c7f498d4a6342cf8b88b07c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:41:42 +0300 Subject: [PATCH 117/119] Support SAV publication without hard links --- src/openstatspec/spss/sav.py | 22 ++++++++++++++++++---- tests/test_sav_sqlite.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/openstatspec/spss/sav.py b/src/openstatspec/spss/sav.py index 9056dc0..9128782 100644 --- a/src/openstatspec/spss/sav.py +++ b/src/openstatspec/spss/sav.py @@ -9,6 +9,7 @@ import hashlib import os import json +import errno import math import ctypes from contextlib import ExitStack, contextmanager @@ -352,15 +353,28 @@ def _publish_staged_destination( state.update({"had_previous": True, "backup_installed": True}) else: backup.unlink(missing_ok=True) - # The hard-link claim is atomic and fails if another process publishes the - # destination after the check above. Staging is on the same filesystem. - os.link(staged, destination) + # destination after the check above. Some portable filesystems do not + # implement hard links; the surrounding export lock serializes those + # fallback replacements with every other OpenStatSpec publisher. + try: + os.link(staged, destination) + except OSError as link_error: + if link_error.errno not in { + errno.EOPNOTSUPP, errno.ENOTSUP, errno.EPERM, errno.EXDEV, + }: + raise + if _path_entry_exists(destination): + raise FileExistsError( + "The export destination was published by another operation." + ) from link_error + os.replace(staged, destination) + else: + staged.unlink() published_identity = _destination_identity(destination) if published_identity is None: raise FileNotFoundError("The published export destination disappeared.") state["published_identity"] = published_identity - staged.unlink() @contextmanager diff --git a/tests/test_sav_sqlite.py b/tests/test_sav_sqlite.py index 4b70a8a..37485e6 100644 --- a/tests/test_sav_sqlite.py +++ b/tests/test_sav_sqlite.py @@ -1,4 +1,5 @@ import hashlib +import errno import json import sqlite3 import threading @@ -384,3 +385,30 @@ def publish_concurrently(source, target): assert sav_module._path_entry_exists(backup) is False assert state["published_identity"] is None + +def test_export_publication_falls_back_when_hard_links_are_unsupported( + tmp_path, monkeypatch, +) -> None: + staged = tmp_path / "staged.sav" + destination = tmp_path / "destination.sav" + backup = tmp_path / "destination.previous" + staged.write_bytes(b"portable staged export") + + def unsupported_hard_link(*_args, **_kwargs): + raise OSError(errno.EOPNOTSUPP, "hard links are unsupported") + + monkeypatch.setattr(sav_module.os, "link", unsupported_hard_link) + state = {} + sav_module._publish_staged_destination( + staged=staged, + destination=destination, + backup=backup, + state=state, + ) + + assert destination.read_bytes() == b"portable staged export" + assert not staged.exists() + assert state["published_identity"] == sav_module._destination_identity( + destination, + ) + From 0713ca79e32069c819e8acfc6b9413e48315d32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 3 Aug 2026 23:59:05 +0300 Subject: [PATCH 118/119] Serialize SQLite catalog initialization --- src/openstatspec/sql/wide.py | 26 ++++++++++++--- tests/test_catalog_lifecycle.py | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index e3bf337..656197a 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1551,8 +1551,8 @@ def _compensate_catalog_initialization( def _requires_compensating_catalog_cleanup(profile_name: str) -> bool: - """Only PostgreSQL rolls back catalog DDL without stale compensation.""" - return profile_name != "postgresql" + """Only non-transactional MySQL-wire DDL needs stale compensation.""" + return profile_name in MYSQL_WIRE_PROFILES def _catalog_residual_inventory( @@ -1631,7 +1631,17 @@ def dolt_state_snapshot( def _catalog_initialization_serialization( connection: Any, *, profile_name: str, ): - """Serialize non-transactional catalog DDL on one database server.""" + """Serialize catalog DDL across connections on one database server.""" + if profile_name == "sqlite": + connection.exec_driver_sql("BEGIN IMMEDIATE") + try: + yield + except Exception: + connection.rollback() + raise + else: + connection.commit() + return if profile_name not in MYSQL_WIRE_PROFILES: yield return @@ -1695,9 +1705,15 @@ def initialize_wide_catalog( _require_dolt_working_set_binding( pre_dolt_state, active, phase="catalog initialization preflight", ) - connection.rollback() + if profile.name != "sqlite": + connection.rollback() try: - with connection.begin(): + mutation = ( + connection.begin_nested() + if profile.name == "sqlite" + else connection.begin() + ) + with mutation: create_normative_catalog(connection, normative) metadata.create_all(connection, tables=list(legacy)) _migrate_catalog_columns( diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index dc0ec0c..bc286ef 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -1,5 +1,6 @@ from decimal import Decimal import sqlite3 +import threading import pytest from sqlalchemy import MetaData, Table @@ -497,6 +498,63 @@ def test_stale_initializer_compensation_preserves_concurrent_verified_catalog( assert _table_names(path) == before +def test_sqlite_initializer_serializes_failure_before_concurrent_winner( + tmp_path, monkeypatch, +): + path = tmp_path / "serialized-concurrent-init.sqlite" + database = f"sqlite:///{path}" + loser_inside_migration = threading.Event() + winner_started = threading.Event() + release_loser = threading.Event() + original_migrate = wide._migrate_catalog_columns + failures = {} + + def migrate(connection, datasets, variables, multiple_response): + if threading.current_thread().name == "losing-initializer": + loser_inside_migration.set() + assert winner_started.wait(5) + assert release_loser.wait(5) + raise RuntimeError("injected catalog migration failure") + return original_migrate(connection, datasets, variables, multiple_response) + + def compensation_must_not_run(*_args, **_kwargs): + raise AssertionError("SQLite must roll back DDL without stale compensation") + + def initialize(name): + if name == "winner": + winner_started.set() + try: + openstatspec.initialize_catalog(database_url=database) + except Exception as error: + failures[name] = error + + monkeypatch.setattr(wide, "_migrate_catalog_columns", migrate) + monkeypatch.setattr( + wide, "_compensate_catalog_initialization", compensation_must_not_run, + ) + + loser = threading.Thread( + target=initialize, args=("loser",), name="losing-initializer", + ) + winner = threading.Thread( + target=initialize, args=("winner",), name="winning-initializer", + ) + loser.start() + assert loser_inside_migration.wait(5) + winner.start() + assert winner_started.wait(5) + release_loser.set() + loser.join(10) + winner.join(10) + + assert not loser.is_alive() + assert not winner.is_alive() + assert type(failures.get("loser")) is RuntimeError + assert str(failures["loser"]) == "injected catalog migration failure" + assert "winner" not in failures + assert openstatspec.initialize_catalog(database_url=database)["catalog"] == "verified" + + def test_mysql_catalog_initialization_lock_spans_mutation_boundary(): events = [] From e98ef83eb93f0246076ae3613edda909e5a90782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Tue, 4 Aug 2026 00:16:29 +0300 Subject: [PATCH 119/119] Validate catalog storage and SQLite URI persistence --- src/openstatspec/sql/wide.py | 17 ++++++++++--- tests/test_catalog_lifecycle.py | 43 ++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/openstatspec/sql/wide.py b/src/openstatspec/sql/wide.py index 656197a..e7715df 100644 --- a/src/openstatspec/sql/wide.py +++ b/src/openstatspec/sql/wide.py @@ -1364,17 +1364,18 @@ def _catalog_variable_bijection_state( legacy_rows = [ ( row["dataset_id"], row["ordinal"], row["source_name"], - row["physical_name"], + row["physical_name"], row["storage_kind"], ) for row in connection.execute(select( variables.c.dataset_id, variables.c.ordinal, variables.c.source_name, variables.c.physical_name, + variables.c.storage_kind, )).mappings() ] normative_rows = [ ( row["dataset_name"], row["source_ordinal"], row["source_name"], - row["physical_name"], + row["physical_name"], row["storage_kind"], ) for row in connection.execute( select( @@ -1382,6 +1383,7 @@ def _catalog_variable_bijection_state( normative.variable.c.source_ordinal, normative.variable.c.source_name, normative.variable.c.physical_name, + normative.variable.c.storage_kind, ).join( normative.variable, normative.variable.c.dataset_id == normative.dataset.c.dataset_id, @@ -1394,7 +1396,8 @@ def _catalog_variable_bijection_state( or not isinstance(ordinal, int) or ordinal < 1 or not isinstance(source_name, str) or not source_name.strip() or not isinstance(physical_name, str) or not physical_name.strip() - for dataset_name, ordinal, source_name, physical_name in rows + or storage_kind not in {"numeric", "string"} + for dataset_name, ordinal, source_name, physical_name, storage_kind in rows ): return "unverified" if len(set(rows)) != len(rows): @@ -1675,9 +1678,15 @@ def initialize_wide_catalog( """Install or explicitly migrate a dedicated catalog after server preflight.""" validate_connection_url(database_url) parsed_url = make_url(database_url) + sqlite_database = parsed_url.database or "" + sqlite_mode = str(parsed_url.query.get("mode", "")).lower() if ( parsed_url.get_backend_name() == "sqlite" - and parsed_url.database in {None, "", ":memory:"} + and ( + sqlite_database in {"", ":memory:"} + or sqlite_database.lower() == "file::memory:" + or sqlite_mode == "memory" + ) ): raise UnsupportedOperationError( "Catalog initialization requires a persistent SQLite database URL." diff --git a/tests/test_catalog_lifecycle.py b/tests/test_catalog_lifecycle.py index bc286ef..07fb41e 100644 --- a/tests/test_catalog_lifecycle.py +++ b/tests/test_catalog_lifecycle.py @@ -328,6 +328,34 @@ def test_duplicate_import_failure_preserves_existing_dataset(tmp_path): +def test_variable_bijection_rejects_storage_kind_mismatch(tmp_path): + path = tmp_path / "variable-storage-mismatch.sqlite" + database = f"sqlite:///{path}" + openstatspec.initialize_catalog(database_url=database) + variables = _variables() + variables[0].update({ + "storage_kind": "numeric", + "string_width": None, + }) + create_wide_dataset( + database_url=database, + dataset_id="sample", + source_name="fixture.sav", + source_format="SAV", + rows=[{"name": 1.0}], + variables=variables, + ) + connection = sqlite3.connect(path) + connection.execute( + "update variable set storage_kind = 'string' where source_name = 'name'" + ) + connection.commit() + connection.close() + + with pytest.raises(UnsupportedOperationError, match="catalog is unverified"): + read_wide_dataset(database_url=database, dataset_id="sample") + + def test_missing_additive_column_requires_explicit_migration_without_audit_mutation( tmp_path, ): @@ -433,7 +461,12 @@ def test_reflected_mysql_integer_display_width_is_not_semantic(): inspector, mysql.VARCHAR(length=255), ) == "VARCHAR(255)" -@pytest.mark.parametrize("database_url", ["sqlite://", "sqlite:///:memory:"]) +@pytest.mark.parametrize("database_url", [ + "sqlite://", + "sqlite:///:memory:", + "sqlite:///file:memdb1?mode=memory&cache=shared&uri=true", + "sqlite:///file::memory:?cache=shared&uri=true", +]) def test_catalog_initialization_rejects_ephemeral_sqlite_url(database_url): with pytest.raises( UnsupportedOperationError, @@ -441,6 +474,14 @@ def test_catalog_initialization_rejects_ephemeral_sqlite_url(database_url): ): openstatspec.initialize_catalog(database_url=database_url) +def test_catalog_initialization_allows_file_backed_sqlite_uri(tmp_path): + path = tmp_path / "uri-catalog.sqlite" + database = f"sqlite:///file:{path}?mode=rwc&uri=true" + + assert openstatspec.initialize_catalog(database_url=database)["catalog"] == "verified" + assert path.is_file() + + def test_import_fidelity_reader_excludes_export_operational_events(tmp_path): path = tmp_path / "fidelity-directions.sqlite" database = f"sqlite:///{path}"