diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c69956..e5d92ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,10 +9,15 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: quality: name: Python ${{ matrix.python-version }} runs-on: ubuntu-latest + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -20,6 +25,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -36,9 +43,12 @@ jobs: package: name: Package and example runs-on: ubuntu-latest + timeout-minutes: 15 steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 02ca5ef..e3e83ff 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -11,16 +11,23 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: analyze: name: Analyze Python runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read security-events: write steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Initialize CodeQL uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 with: diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index 84d8d10..b9c5cd9 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -11,15 +11,21 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: scan: name: Full-history Gitleaks scan runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Check out complete history uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 + persist-credentials: false - name: Download checksum-pinned Gitleaks env: GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index 8d513ce..9445e04 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -1,33 +1,297 @@ name: Release assets on: - release: - types: [published] + workflow_dispatch: + inputs: + release_tag: + description: Existing annotated or signed v-prefixed release tag + required: true + type: string permissions: contents: read +concurrency: + group: release-assets-${{ inputs.release_tag }} + cancel-in-progress: false + jobs: build: - name: Build and attach Python distributions + name: Build, publish, and verify release runs-on: ubuntu-latest + timeout-minutes: 15 permissions: + artifact-metadata: write + attestations: write contents: write + id-token: write steps: + - name: Verify release tag targets protected main + id: verify-tag + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "release tag must be an exact v-prefixed semantic version" >&2 + exit 1 + fi + object_type="$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$RELEASE_TAG" --jq '.object.type')" + object_sha="$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$RELEASE_TAG" --jq '.object.sha')" + if [ "$object_type" != "tag" ]; then + echo "release tag must be an annotated or signed tag object" >&2 + exit 1 + fi + for attempt in 1 2 3 4 5; do + if [ "$object_type" = "commit" ]; then + break + fi + if [ "$object_type" != "tag" ]; then + echo "release tag does not resolve to a commit" >&2 + exit 1 + fi + tag_object="$(gh api "repos/$GITHUB_REPOSITORY/git/tags/$object_sha")" + tagger_name="$(jq -r '.tagger.name // ""' <<<"$tag_object")" + tagger_email="$(jq -r '.tagger.email // ""' <<<"$tag_object")" + if [ "$tagger_name" != "Tovellan Maintainers" ] || [ "$tagger_email" != "noreply@github.com" ]; then + echo "release tag must use the generic maintainer identity" >&2 + exit 1 + fi + tag_message="$(jq -r '.message // ""' <<<"$tag_object")" + case "$tag_message" in + "SplitSeal $RELEASE_TAG"|"SplitSeal $RELEASE_TAG"$'\n-----BEGIN PGP SIGNATURE-----'*) ;; + *) echo "release tag must use the exact public annotation message" >&2; exit 1 ;; + esac + object_type="$(jq -r '.object.type' <<<"$tag_object")" + object_sha="$(jq -r '.object.sha' <<<"$tag_object")" + done + if [ "$object_type" != "commit" ]; then + echo "release tag indirection is too deep" >&2 + exit 1 + fi + if [ "$GITHUB_REF" != "refs/tags/$RELEASE_TAG" ] || [ "$GITHUB_SHA" != "$object_sha" ]; then + echo "workflow must be dispatched from the exact protected release tag" >&2 + exit 1 + fi + release_record="$( + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/releases?per_page=100" | + jq -c --arg tag "$RELEASE_TAG" \ + '[.[][] | select(.tag_name == $tag)] | if length > 1 then error("duplicate release tag") elif length == 1 then .[0] else null end' + )" + if [ "$release_record" = "null" ]; then + release_state="absent" + release_id="" + elif [ "$(jq -r '.draft' <<<"$release_record")" = "true" ]; then + release_state="draft" + release_id="$(jq -r '.id' <<<"$release_record")" + else + release_state="published" + release_id="$(jq -r '.id' <<<"$release_record")" + fi + main_sha="$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/main" --jq '.object.sha')" + if [ "$release_state" = "absent" ]; then + if [ "$object_sha" != "$main_sha" ]; then + echo "new release tag must target the current protected main commit" >&2 + exit 1 + fi + else + comparison_status="$( + gh api "repos/$GITHUB_REPOSITORY/compare/$object_sha...$main_sha" --jq '.status' + )" + if [ "$comparison_status" != "identical" ] && [ "$comparison_status" != "ahead" ]; then + echo "existing release tag must remain in protected main history" >&2 + exit 1 + fi + fi + printf 'target_sha=%s\n' "$object_sha" >> "$GITHUB_OUTPUT" + printf 'release_state=%s\n' "$release_state" >> "$GITHUB_OUTPUT" + printf 'release_id=%s\n' "$release_id" >> "$GITHUB_OUTPUT" - name: Check out release tag uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.release.tag_name }} + persist-credentials: false + ref: ${{ steps.verify-tag.outputs.target_sha }} - name: Install Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.13" + - name: Verify tag matches package version + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + package_version="$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')" + if [ "$RELEASE_TAG" != "v$package_version" ]; then + echo "release tag does not match package version" >&2 + exit 1 + fi - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - name: Build distributions - run: uv build - - name: Attach distributions to the GitHub release + with: + version: "0.12.5" + - name: Build tag-matched distributions and checksums + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: python scripts/release_assets.py --tag "$RELEASE_TAG" --output-dir dist + - name: Attest wheel and source archive provenance + if: steps.verify-tag.outputs.release_state != 'published' + uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1 + with: + subject-checksums: dist/SHA256SUMS + - name: Generate and validate public release notes + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag }} + RELEASE_ID: ${{ steps.verify-tag.outputs.release_id }} + RELEASE_STATE: ${{ steps.verify-tag.outputs.release_state }} + TARGET_SHA: ${{ steps.verify-tag.outputs.target_sha }} + run: | + if [ "$RELEASE_STATE" = "published" ]; then + gh api "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" \ + > "$RUNNER_TEMP/release-notes.json" + else + gh api --method POST "repos/$GITHUB_REPOSITORY/releases/generate-notes" \ + -f tag_name="$RELEASE_TAG" -f target_commitish="$TARGET_SHA" \ + > "$RUNNER_TEMP/generated-release-notes.json" + python scripts/validate_release_metadata.py \ + --tag "$RELEASE_TAG" \ + --input "$RUNNER_TEMP/generated-release-notes.json" \ + --output "$RUNNER_TEMP/release-notes.json" \ + --sanitize-generated + fi + if [ "$RELEASE_STATE" = "published" ]; then + python scripts/validate_release_metadata.py \ + --tag "$RELEASE_TAG" --input "$RUNNER_TEMP/release-notes.json" + fi + - name: Create or resume draft release + id: release + if: steps.verify-tag.outputs.release_state != 'published' + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag }} + RELEASE_ID: ${{ steps.verify-tag.outputs.release_id }} + RELEASE_STATE: ${{ steps.verify-tag.outputs.release_state }} + run: | + if [ "$RELEASE_STATE" = "absent" ]; then + release_notes="$(jq -r '.body' "$RUNNER_TEMP/release-notes.json")" + release_record="$( + gh api --method POST "repos/$GITHUB_REPOSITORY/releases" \ + -f tag_name="$RELEASE_TAG" -f name="SplitSeal $RELEASE_TAG" \ + -f body="$release_notes" -F draft=true + )" + RELEASE_ID="$(jq -r '.id' <<<"$release_record")" + else + gh api "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" \ + > "$RUNNER_TEMP/existing-draft.json" + python scripts/validate_release_metadata.py \ + --tag "$RELEASE_TAG" --input "$RUNNER_TEMP/existing-draft.json" + fi + printf 'release_id=%s\n' "$RELEASE_ID" >> "$GITHUB_OUTPUT" + - name: Attach exact draft assets + if: steps.verify-tag.outputs.release_state != 'published' + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag }} + RELEASE_ID: ${{ steps.release.outputs.release_id }} + run: | + for artifact in dist/*; do + asset_name="${artifact##*/}" + asset_record="$( + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?per_page=100" | + jq -c --arg name "$asset_name" \ + '[.[][] | select(.name == $name)] | if length > 1 then error("duplicate release asset") elif length == 1 then .[0] else null end' + )" + if [ "$asset_record" = "null" ]; then + gh release upload "$RELEASE_TAG" "$artifact" + continue + fi + asset_id="$(jq -r '.id' <<<"$asset_record")" + if [ "$(jq -r '.state' <<<"$asset_record")" != "uploaded" ]; then + gh api --method DELETE "repos/$GITHUB_REPOSITORY/releases/assets/$asset_id" + gh release upload "$RELEASE_TAG" "$artifact" + continue + fi + downloaded="$RUNNER_TEMP/existing-$asset_name" + gh api -H 'Accept: application/octet-stream' \ + "repos/$GITHUB_REPOSITORY/releases/assets/$asset_id" > "$downloaded" + if ! cmp -s "$artifact" "$downloaded"; then + echo "existing draft asset does not match the verified build: $asset_name" >&2 + exit 1 + fi + done + gh api --paginate --slurp \ + "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?per_page=100" | + jq '[.[][]]' > "$RUNNER_TEMP/draft-assets.json" + python scripts/validate_release_assets.py \ + --local-dir dist --inventory "$RUNNER_TEMP/draft-assets.json" + - name: Verify distribution provenance + env: + GH_TOKEN: ${{ github.token }} + run: | + signer_workflow="github.com/$GITHUB_REPOSITORY/.github/workflows/release-assets.yml" + for artifact in dist/*.whl dist/*.tar.gz; do + verified=false + for attempt in {1..20}; do + if gh attestation verify "$artifact" \ + --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$signer_workflow" \ + --signer-digest "$GITHUB_SHA" \ + --source-ref "$GITHUB_REF" \ + --source-digest "$GITHUB_SHA" >/dev/null 2>&1; then + verified=true + break + fi + sleep 3 + done + if [ "$verified" != "true" ]; then + echo "distribution provenance verification failed: ${artifact##*/}" >&2 + exit 1 + fi + done + - name: Publish complete draft release + if: steps.verify-tag.outputs.release_state != 'published' + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ steps.release.outputs.release_id }} + run: | + gh api --method PATCH "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" \ + -F draft=false >/dev/null + - name: Verify exact published assets + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ steps.release.outputs.release_id || steps.verify-tag.outputs.release_id }} + run: | + remote_assets="$( + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/assets?per_page=100" | + jq -c '[.[][]]' + )" + printf '%s\n' "$remote_assets" > "$RUNNER_TEMP/published-assets.json" + python scripts/validate_release_assets.py \ + --local-dir dist --inventory "$RUNNER_TEMP/published-assets.json" + for artifact in dist/*; do + asset_name="${artifact##*/}" + asset_id="$(jq -r --arg name "$asset_name" '.[] | select(.name == $name) | .id' <<<"$remote_assets")" + downloaded="$RUNNER_TEMP/published-$asset_name" + gh api -H 'Accept: application/octet-stream' \ + "repos/$GITHUB_REPOSITORY/releases/assets/$asset_id" > "$downloaded" + if ! cmp -s "$artifact" "$downloaded"; then + echo "published release asset does not match the verified build: $asset_name" >&2 + exit 1 + fi + done + - name: Verify immutable release and automatic attestation env: GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - run: gh release upload "$RELEASE_TAG" dist/* --clobber + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + for attempt in {1..40}; do + state="" + if state="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$RELEASE_TAG" --jq '.immutable' 2>/dev/null)" && \ + [ "$state" = "true" ] && gh release verify "$RELEASE_TAG" --format json >/dev/null; then + printf 'GitHub release immutable: true\n' + printf 'GitHub automatic release attestation: verified\n' + exit 0 + fi + sleep 15 + done + echo "GitHub automatic release attestation verification failed" >&2 + exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d4d14fa..a3dea92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,86 @@ All notable changes are recorded here. The format follows Keep a Changelog and the project uses Semantic Versioning. +## [Unreleased] + +### Added + +- Enforce full immutable commit pins for every external action reference in GitHub + workflows and local composite-action definitions as part of the release gate. +- Release-gate package, runtime, lockfile, install, changelog, support, API, manifest, and + roadmap version consistency. + +### Changed + +- Remove persisted checkout credentials from every workflow job, bound job runtimes, and + cancel superseded per-ref validation runs without cancelling published-release jobs. +- Parse both workflow filename extensions and validate workflow security policy from YAML + structure rather than formatting-dependent text patterns. +- Require an exact release-tag and package-version match before building release assets. +- Publish a sorted `SHA256SUMS` file with the wheel and source archive, and refuse stale + output directories or release-asset overwrites. +- Record Sigstore-signed GitHub build-provenance attestations for the checksummed wheel + and source archive. +- Scope checksum instructions to releases produced after the new workflow takes effect. +- Scope provenance instructions to releases produced after attestation takes effect. + +### Fixed + +- Parse YAML action definitions semantically and discover composite actions in any + tracked directory so formatting and placement cannot bypass immutable-pin checks. +- Reject unknown private-seal envelope fields plus padded or noncanonical base64url while + preserving the v1 schema and all generated seal bytes. +- Require exact JSON integer types for every private-seal scrypt parameter. +- Avoid exposing resolved absolute configuration paths in machine-readable read errors. +- Return stable machine-readable errors for excessively nested structured inputs. +- Normalize invalid digest input types and count ranges to stable `SS012` errors. +- Reject whitespace and every other non-exact spelling of 64-character SHA-256 hex. +- Preserve one-pass digest iterables required by bounded-memory release processing. +- Normalize non-byte key material to stable `SS041` errors across Python APIs. +- Normalize plugin discovery, loading, execution, and evidence failures to SS060 or SS061. +- Attach distributions while a release is still a draft, then require GitHub release + immutability and automatic release-attestation verification after publication. +- Resolve the release tag before checkout and require its commit to equal protected + `main`, then build from the verified commit SHA. +- Require the release ref to begin with an annotated or signed tag object rather than a + lightweight tag. +- Depend on organization-enforced release immutability and require generic tagger metadata. +- Make release-attestation verification retryable without attempting to recreate an + already published release. +- Resume partial draft uploads without overwriting conflicting assets, and allow + verification-only reruns after protected `main` advances. +- Bind manual release dispatches to the exact protected version-tag revision and require + server-side immutable action SHA pins. +- Validate tag annotations and generated release notes as public metadata. +- Require exact draft and published remote asset names, SHA-256 digests, and bytes against + a protected-tag rebuild. +- Verify each distribution's exact provenance identity before irreversible publication + and on published-release reruns. +- Remove the build tool's hidden output helper before exact release-asset validation. +- Remove generated contributor credits and reject every private-workflow, local-path, + attribution, personal-account, positioning, and prohibited-punctuation class before + release-note publication. + +## [0.4.0] - 2026-08-24 + +### Added + +- Stream exact-only JSONL, CSV, and optional Parquet records without retaining complete + splits in Python memory. +- Use a disk-backed exact-digest index and ordered digest spools while preserving canonical + private-manifest and public-attestation bytes. +- Commit to and encrypt canonical manifests incrementally with the existing v1 + cryptographic format and rollback-safe artifact pair installation. +- Add generated equivalence properties, a bounded-peak integration test, and a local + synthetic throughput and peak-memory benchmark. + +### Fixed + +- Preserve the pre-streaming universal-newline normalization for multiline CSV fields so + unchanged CRLF inputs retain their canonical record digests. +- Preserve SS011 and SS021 domain errors from valid Parquet containers instead of wrapping + them as malformed-container SS020 errors. + ## [0.3.0] - 2026-08-24 ### Added @@ -68,6 +148,8 @@ project uses Semantic Versioning. - JSON and SARIF 2.1.0 reports. - Repository-root containment, Unicode normalization, and symlink checks. +[Unreleased]: https://github.com/tovellan/splitseal/compare/v0.4.0...HEAD +[0.4.0]: https://github.com/tovellan/splitseal/compare/v0.3.0...v0.4.0 [0.3.0]: https://github.com/tovellan/splitseal/compare/v0.2.3...v0.3.0 [0.2.3]: https://github.com/tovellan/splitseal/compare/v0.2.2...v0.2.3 [0.2.2]: https://github.com/tovellan/splitseal/compare/v0.2.1...v0.2.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 30cf7ae..5b28634 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,10 +17,22 @@ or redaction rules require tests for backwards compatibility and adversarial inp Schema changes must use a new schema identifier when an old reader could misinterpret the result. +Release version changes must update `pyproject.toml`, runtime `__version__`, `uv.lock`, the +README install tag, changelog, support policy, API and manifest compatibility text, and +the delivered roadmap line together. `make check` rejects partial updates. + Pull requests should explain the problem, compatibility effect, security effect, and validation performed. By contributing, you agree that your contribution is licensed under Apache License 2.0. +External GitHub Actions must use a full 40-character commit SHA. Version tags, branches, +short SHAs, dynamic action expressions, and mutable container tags fail the repository +audit. Keep the human-readable upstream version in a trailing comment. + Public commits and merge commits must use the generic organization identity `Tovellan Maintainers `. Do not publish personal names or email addresses in commit metadata, and do not add authorship or generator-attribution trailers. + +Protected releases must use an existing annotated or signed version tag and the +`Release assets` workflow. Do not publish a release manually before its assets are +attached: publication activates release immutability and prevents later asset changes. diff --git a/Makefile b/Makefile index 19db944..8e909c0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: format lint test check build clean-install audit example release-gate +.PHONY: format lint test check version-audit build clean-install audit example release-gate format: uv run ruff format . @@ -12,10 +12,13 @@ lint: test: uv run pytest -check: lint test +check: lint test version-audit uv run python scripts/check_text_policy.py uv run python scripts/repository_audit.py +version-audit: + uv run python scripts/version_audit.py + build: uv build diff --git a/README.md b/README.md index 9a5ba93..7bd23f1 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,30 @@ SplitSeal requires Python 3.11 or newer. The project is not published to a packa registry. Install a tagged source release from GitHub: ```console -python -m pip install "splitseal @ git+https://github.com/tovellan/splitseal.git@v0.3.0" +python -m pip install "splitseal @ git+https://github.com/tovellan/splitseal.git@v0.4.0" +``` + +Starting with the first release produced after this workflow change, tagged GitHub +releases include a wheel, a source archive, and `SHA256SUMS`. Release v0.2.3 predates the +workflow and does not include a checksum manifest. For a release that includes all three +files, verify the distributions before installation: + +```console +shasum -a 256 -c SHA256SUMS +``` + +Starting with the first release produced after the provenance workflow change, GitHub +also records signed build provenance for both distributions. Release v0.2.3 predates +that workflow and has no distribution attestations. For a later release, substitute its +actual version and verify a downloaded wheel against this public repository: + +```console +gh attestation verify splitseal-X.Y.Z-py3-none-any.whl \ + --repo tovellan/splitseal \ + --signer-workflow github.com/tovellan/splitseal/.github/workflows/release-assets.yml \ + --signer-digest TAG_COMMIT_SHA \ + --source-ref refs/tags/vX.Y.Z \ + --source-digest TAG_COMMIT_SHA ``` For Parquet input, add the optional dependency after cloning: @@ -136,6 +159,9 @@ The complete synthetic example is in [`examples/synthetic`](examples/synthetic). unsafe. - Every input and output path is relative to a caller-selected repository root. Absolute paths, traversal, non-NFC path spellings, and escaping symlinks are rejected. +- Exact-only freezes stream records through disk-backed duplicate and digest spools. + Similarity-plugin configurations retain the in-memory plugin path. See + [`docs/streaming-freezes.md`](docs/streaming-freezes.md). ## Similarity plugins diff --git a/ROADMAP.md b/ROADMAP.md index 701dd62..1828c68 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -11,9 +11,13 @@ The roadmap is ordered by dependency and user evidence, not by a promised date. - Optional local Ed25519 detached signatures with derived key identity, explicit trust stores, rotation, all-history revocation, and public verification. +## Delivered in 0.4 + +- Bounded-memory exact-only freezes with disk-backed duplicate control, canonical + manifest spooling, incremental commitment, and streaming encryption. + ## Next candidates -- Streaming manifest construction for datasets that do not use similarity plugins. - Additional independently maintained similarity plugin examples. ## Later investigations diff --git a/SECURITY.md b/SECURITY.md index 22fbbc8..9b8545a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -16,5 +16,46 @@ days and will coordinate validation, remediation, and disclosure. Security-sensitive areas include canonicalization ambiguity, path escape, symlink races, attestation disclosure, cross-split duplicate bypass, seal authentication, key handling, -and unsafe plugin behavior. The documented trust boundaries in +unsafe plugin behavior, and CI dependency substitution. External GitHub Action references +are release-gated to full commit SHAs. The documented trust boundaries in [`docs/threat-model.md`](docs/threat-model.md) are part of the security contract. + +CI jobs receive read-only repository contents by default and checkout removes persisted +credentials after initial fetch. Jobs have explicit timeouts, and superseded validation +runs are cancelled per ref. Published-release jobs are serialized per tag but never +cancelled by a later run. + +## Release publication + +The release workflow accepts an existing version tag, requires it to match the package +version exactly, and builds into an empty directory. It produces a sorted `SHA256SUMS` +alongside the wheel and source archive, then attaches every asset while the release is +still a draft before publication. Checksums establish download integrity against the +GitHub release. GitHub records Sigstore-signed build-provenance attestations for the +checksummed wheel and source archive. Each attestation binds the artifact digest to the +repository's release workflow identity. It does not replace source review, establish +dataset quality, or act as an independent transparency log. Before publication and on a +published-release rerun, the workflow verifies each distribution against the exact signer +workflow, signer commit, protected tag ref, and source commit. +Closure requires the GitHub Releases API to report `immutable: true` and GitHub's +automatic release attestation to verify. The GitHub Releases API reports +`immutable: false` for release v0.2.3. + +Organization policy enforces immutable releases for this repository. The release job does +not receive an organization Administration token and cannot weaken that prerequisite. +Before checkout, the workflow requires an annotated or signed version-tag object, resolves +it through the GitHub API, and requires a new release target to equal the current protected +`main` commit. Every tag object must use the generic maintainer name and email documented +in the release process plus its exact public annotation. Generated release notes remove +contributor credits and pass the complete public-text policy before draft creation. The +workflow does not print removed account metadata and checks out only the verified commit +SHA. A partial draft rerun +accepts a tag that remains in protected `main` history, verifies existing asset bytes, and +resumes missing uploads without overwriting conflicts. A published-release rerun performs +a protected-tag rebuild and requires exact remote names, SHA-256 digests, and bytes without +uploading or republishing. The build tool is version-pinned, but the hosted runner and +Python minor runtime are not hermetic. A later byte mismatch therefore fails closed as an +integrity error; this check is not a guarantee of indefinite byte-for-byte reproducibility. +Active repository rules block updates and deletions of `v*` tags without bypass actors. +The workflow must itself run from that exact protected tag revision, and the repository's +server-side Actions policy requires immutable action SHA pins. diff --git a/SUPPORT.md b/SUPPORT.md index 45e3da0..5828729 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -7,7 +7,7 @@ SplitSeal supports maintained CPython versions from 3.11 through 3.14 on Linux, and Windows. CI exercises those Python versions on Linux and performs an additional Windows path test. Optional Parquet behavior follows the supported platforms of PyArrow. -Within the 0.3 release line, documented Python APIs, command names, JSON error codes, and +Within the 0.4 release line, documented Python APIs, command names, JSON error codes, and schema identifiers are compatibility commitments. New optional fields may be added to reports. Private or public artifact schemas change identifiers when interpretation would otherwise be ambiguous. diff --git a/benchmarks/bench_streaming.py b/benchmarks/bench_streaming.py new file mode 100644 index 0000000..06f8093 --- /dev/null +++ b/benchmarks/bench_streaming.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Measure local synthetic exact-only freeze throughput and peak Python memory.""" + +from __future__ import annotations + +import argparse +import json +import tempfile +import time +import tracemalloc +from pathlib import Path + +from splitseal import freeze_release + +_MINIMUM_RECORDS = 2 + + +def _write_jsonl(path: Path, prefix: str, records: int, payload_bytes: int) -> None: + payload = "x" * payload_bytes + with path.open("w", encoding="utf-8", newline="\n") as stream: + for index in range(records): + stream.write( + json.dumps( + {"id": f"{prefix}-{index:08d}", "payload": payload}, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--records", type=int, default=10_000) + parser.add_argument("--payload-bytes", type=int, default=128) + args = parser.parse_args() + if args.records < _MINIMUM_RECORDS: + parser.error("--records must be at least 2") + if args.payload_bytes < 0: + parser.error("--payload-bytes cannot be negative") + + with tempfile.TemporaryDirectory(prefix="splitseal-stream-benchmark-") as directory: + root = Path(directory) + (root / "data").mkdir() + (root / "artifacts").mkdir() + first_count = args.records // 2 + second_count = args.records - first_count + _write_jsonl(root / "data" / "first.jsonl", "first", first_count, args.payload_bytes) + _write_jsonl(root / "data" / "second.jsonl", "second", second_count, args.payload_bytes) + (root / "splitseal.toml").write_text( + """schema_version = "splitseal.config.v1" +[release] +name = "synthetic-stream-benchmark" +version = "1.0.0" +[[splits]] +name = "first" +path = "data/first.jsonl" +format = "jsonl" +[[splits]] +name = "second" +path = "data/second.jsonl" +format = "jsonl" +""", + encoding="utf-8", + ) + input_bytes = sum(path.stat().st_size for path in (root / "data").iterdir()) + tracemalloc.start() + started = time.perf_counter() + freeze_release( + root=root, + config_path="splitseal.toml", + seal_path="artifacts/release.sseal", + attestation_path="artifacts/release.attestation.json", + secret=b"synthetic-benchmark-key-material", + ) + elapsed = time.perf_counter() - started + _current, peak_python_bytes = tracemalloc.get_traced_memory() + tracemalloc.stop() + print( + json.dumps( + { + "input_bytes": input_bytes, + "payload_bytes": args.payload_bytes, + "peak_python_bytes": peak_python_bytes, + "records": args.records, + "records_per_second": args.records / elapsed, + "seconds": elapsed, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/docs/api.md b/docs/api.md index c7da325..e0fe37a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,6 +1,6 @@ # Python API -The stable 0.3 API is exported from `splitseal`. +The stable 0.4 API is exported from `splitseal`. ## Canonicalization @@ -14,7 +14,16 @@ digest = record_digest(record) `canonicalize` accepts JSON-compatible values in the RFC 8785 interoperable domain. It rejects non-string object keys, non-finite floats, unsupported objects, and integers -outside the exactly interoperable range. +outside the exactly interoperable range. Structured values may contain at most 100 nested +array or object levels; deeper values fail with `SS011` instead of depending on the +Python interpreter recursion limit. + +`dataset_digest` accepts string split names paired with a non-negative 64-bit record +count and exactly 64 lowercase or uppercase hexadecimal SHA-256 characters. +`sequence_digest` applies the same exact grammar to each record digest from a one-pass +iterable. Text and byte containers are not treated as iterables of digests. Whitespace is +not accepted. Invalid runtime types, encodings, lengths, and count ranges fail with +`SS012`. ## Release operations @@ -44,6 +53,10 @@ validate_public_attestation( ) ``` +Release keys passed to Python APIs must be `bytes` containing at least 16 bytes. Wrong +runtime types and shorter values fail with `SS041`; `splitseal keygen` remains the +recommended way to create a 32-byte key. + `validate_public_attestation` accepts only a public attestation path. It validates the schema, RFC 8785 encoding, field types, aggregate consistency, and redaction constraints. It does not authenticate the commitment or establish provenance, and its report always @@ -83,5 +96,10 @@ local rotation workflows that need to assemble multiple sorted active or revoked Paths are always relative to `root`. Expected failures raise `SplitSealError` with stable `code`, `message`, and `details` fields. Details are intended for local diagnostics and -may include a caller-supplied file name. Do not publish raw local error logs without -review. +may include a caller-supplied relative path or an artifact basename. Resolved absolute +dataset, key, configuration, and artifact paths are not included. Do not publish raw +local error logs without review. + +Exact-only `freeze_release` calls use bounded-memory disk spooling automatically. +Configurations containing a similarity plugin retain the in-memory plugin path. Both +paths produce identical canonical private-manifest bytes for the same inputs. diff --git a/docs/architecture.md b/docs/architecture.md index dd59517..85a9265 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,6 +14,11 @@ SplitSeal has four layers. derived HMAC-SHA256 key commits to the canonical manifest in an aggregate-only public attestation. +Exact-only configurations stream strict records through a disk-backed SQLite digest +index and ordered digest spools. Canonical manifest construction, HMAC commitment, and +AES-GCM encryption are incremental. Similarity-plugin configurations retain the original +in-memory path required by the plugin protocol. + ## Data flow ```text diff --git a/docs/clean-room.md b/docs/clean-room.md index 999c2dd..a64d297 100644 --- a/docs/clean-room.md +++ b/docs/clean-room.md @@ -50,5 +50,7 @@ not a replacement for them. Runtime dependencies are RFC8785 under Apache-2.0 and cryptography under dual Apache-2.0 or BSD-3-Clause terms. The optional PyArrow dependency is Apache-2.0. +The PyYAML development dependency is MIT licensed and parses action metadata during the +repository audit and workflow policy tests. Development dependencies are not bundled in the wheel. The release gate records an installed dependency audit and wheel-content review. diff --git a/docs/manifest-format.md b/docs/manifest-format.md index 574eac3..e5b9050 100644 --- a/docs/manifest-format.md +++ b/docs/manifest-format.md @@ -8,8 +8,10 @@ Readers reject unsupported schema identifiers. Schema: `splitseal.seal.v1` The outer object contains fixed scrypt parameters, a random salt, an AES-256-GCM nonce, -and ciphertext. Binary fields use unpadded base64url. The schema identifier is authenticated -as additional data. +and ciphertext. Binary fields use canonical unpadded base64url. The outer object contains +exactly `schema_version`, `kdf`, and `cipher`; the nested parameter objects also reject +unknown or missing fields with `SS040`. The schema identifier is authenticated as +additional data. The encrypted `splitseal.private-manifest.v1` object contains: @@ -54,7 +56,7 @@ and it does not prove authenticity or provenance. ## Compatibility -New optional report fields may appear within a 0.3 release. Artifact interpretation does +New optional report fields may appear within a 0.4 release. Artifact interpretation does not change without a new schema identifier. Attestation validation fails closed on unknown fields as well as an unknown schema so disclosure constraints remain explicit. diff --git a/docs/plugin-api.md b/docs/plugin-api.md index f46e6a8..bada797 100644 --- a/docs/plugin-api.md +++ b/docs/plugin-api.md @@ -50,6 +50,12 @@ Any returned finding blocks the freeze. Public attestations report only `pass` o `not_run`; plugin names, versions, scores, settings, record indexes, and split names stay inside local execution or the encrypted private manifest. +The declared plugin `name` must be a non-empty string equal to the configured entry-point +name, and `version` must be a non-empty string. Entry-point discovery, loading, identity, +and interface failures return `SS060`. Failures from a custom loader, identity or +interface validation, analysis iteration, or version evidence return `SS061`. No release +artifacts are written after either failure. + Plugins are trusted code with direct access to private records. Process isolation, network restriction, deterministic execution, and dependency review are the operator's responsibility. diff --git a/docs/release-process.md b/docs/release-process.md index 10d3f6c..0c4bab4 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -1,16 +1,32 @@ # Release process -Maintainers release from a clean `main` checkout. +Maintainers release from a clean `main` checkout. Organization-enforced release +immutability and the no-bypass `v*` update and deletion rules must remain enabled. 1. Update `CHANGELOG.md` and confirm package, manifest, and tool versions agree. 2. Run `make release-gate`. 3. Review `git status`, the complete diff, tracked file types and sizes, and commit history. -4. Create and push a signed or annotated `vX.Y.Z` tag. +4. Create a signed or annotated `vX.Y.Z` tag with tagger name `Tovellan Maintainers` and + tagger email `noreply@github.com` and the exact annotation `SplitSeal vX.Y.Z`, then push + it. The tag must target the current protected `main` commit. 5. Inspect every GitHub Actions job. -6. Create the GitHub release from the tag and attach the locally verified wheel and sdist. +6. Dispatch the `Release assets` workflow from the exact protected `vX.Y.Z` tag ref and + supply that same tag as its input. Never select a branch or another tag as the workflow + revision. Do not create the GitHub release or attach assets manually: the workflow + exclusively builds the wheel, source archive, and `SHA256SUMS`, records and verifies + exact distribution provenance, attaches all three, and publishes the complete draft + before verifying immutability and the automatic release attestation. Before draft + creation, generated notes remove contributor credits and validate the complete + public-text policy without printing the removed account metadata. -The release workflow builds artifacts and can attach them to a GitHub release. It does not -publish to PyPI, another package registry, or a container registry. +The workflow is safe to rerun after a partial draft upload: it resumes the existing draft, +keeps byte-identical assets, replaces only incomplete uploads, and refuses conflicting +or unexpected assets. After publication it rebuilds from the protected tag, requires exact +remote asset names and SHA-256 digests plus byte equality, skips upload and publication, +and repeats exact distribution-provenance, immutable-release, and automatic-attestation +verification. These recovery paths require the tag target to remain in protected `main` +history, although it need not remain the branch tip. The workflow does not publish to +PyPI, another package registry, or a container registry. `make release-gate` performs tests, formatting checks, lint, static typing, package build, wheel installation, example execution, dependency audit, text policy checks, tracked-file diff --git a/docs/streaming-freezes.md b/docs/streaming-freezes.md new file mode 100644 index 0000000..21f82de --- /dev/null +++ b/docs/streaming-freezes.md @@ -0,0 +1,40 @@ +# Streaming exact-only freezes + +SplitSeal 0.4 uses a disk-spooled streaming path whenever a configuration has no +similarity plugin. Configurations with a trusted similarity plugin retain the original +in-memory path because the plugin protocol receives complete split record sequences. + +## Compatibility + +The streaming path preserves record order, strict JSONL, CSV, and Parquet decoding, +record digests, exact cross-split duplicate blocking, split roots, dataset roots, private +manifest schema, canonical manifest bytes, public attestation bytes, and error codes. +Tests compare streaming output with the in-memory builder over generated inputs. Private +seal bytes remain intentionally nondeterministic because every seal uses a new salt and +nonce. + +## Memory and disk bound + +The exact-only path retains one decoded JSONL or CSV record at a time. Parquet retains at +most one 1,024-row decoded batch. Python-side working memory is bounded by the largest +decoded record or Parquet batch, 64 KiB I/O buffers, split metadata, and a 2 MiB SQLite +page cache. The fixed scrypt parameters also require native memory independently of input +size. Verification remains an in-memory operation in 0.4. + +Temporary disk use is linear in record count. It includes a SQLite exact-digest index, +ordered hexadecimal digest spools, the canonical private manifest, and encrypted +ciphertext staging. Digest spools and the private manifest live in an owner-only temporary +directory; ciphertext and output staging files are created with mode 0600. Successful and +failed operations remove their temporary files. + +## Local benchmark + +Run the synthetic benchmark with caller-selected input size: + +```console +uv run python benchmarks/bench_streaming.py --records 10000 --payload-bytes 128 +``` + +The command reports measured input bytes, elapsed seconds, records per second, and peak +Python bytes for that invocation. These local measurements are not published performance +claims and vary by host, filesystem, Python version, and cryptographic backend. diff --git a/pyproject.toml b/pyproject.toml index 78868c7..2a0c671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "splitseal" -version = "0.3.0" +version = "0.4.0" description = "Deterministic integrity and release controls for evaluation datasets" readme = "README.md" requires-python = ">=3.11" @@ -36,6 +36,7 @@ dev = [ "hypothesis==6.165.10", "mypy==2.3.1", "pip-audit==2.10.1", + "pyyaml==6.0.2", "pytest==9.1.1", "pytest-cov==7.1.0", "ruff==0.16.4", diff --git a/scripts/release_assets.py b/scripts/release_assets.py new file mode 100644 index 0000000..4d23b67 --- /dev/null +++ b/scripts/release_assets.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Build a tag-matched release and emit a portable checksum manifest.""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import shutil +import subprocess +import sys +import tomllib +from collections.abc import Iterable +from pathlib import Path + +_CHECKSUM_FILE = "SHA256SUMS" +_READ_SIZE = 64 * 1024 + + +def project_identity(root: Path) -> tuple[str, str]: + """Read and validate the package name and version from pyproject.toml.""" + + with (root / "pyproject.toml").open("rb") as stream: + document = tomllib.load(stream) + project = document.get("project") + if not isinstance(project, dict): + raise ValueError("pyproject.toml is missing [project]") + name = project.get("name") + version = project.get("version") + if not isinstance(name, str) or not name: + raise ValueError("project.name must be a non-empty string") + if not isinstance(version, str) or not version: + raise ValueError("project.version must be a non-empty string") + return name, version + + +def validate_release_tag(tag: str, version: str) -> None: + """Require an exact v-prefixed match between the release tag and package version.""" + + expected = f"v{version}" + if tag != expected: + raise ValueError(f"release tag {tag!r} does not match package version {expected!r}") + + +def expected_artifact_names(name: str, version: str) -> frozenset[str]: + """Return the exact pure-Python wheel and source archive names.""" + + wheel_name = re.sub(r"[-_.]+", "_", name).lower() + source_name = re.sub(r"[-_.]+", "-", name).lower() + return frozenset( + { + f"{wheel_name}-{version}-py3-none-any.whl", + f"{source_name}-{version}.tar.gz", + } + ) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(_READ_SIZE): + digest.update(chunk) + return digest.hexdigest() + + +def write_checksum_manifest(output_dir: Path, expected_names: Iterable[str]) -> Path: + """Validate the distribution set and write sorted SHA-256 checksums.""" + + expected = frozenset(expected_names) + actual = frozenset(path.name for path in output_dir.iterdir()) + if actual != expected: + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + raise ValueError( + f"release artifact set is invalid; missing={missing!r}, unexpected={unexpected!r}" + ) + for name in expected: + artifact = output_dir / name + if artifact.is_symlink() or not artifact.is_file(): + raise ValueError(f"release artifact must be a regular non-symlink file: {name}") + checksum_path = output_dir / _CHECKSUM_FILE + content = "".join(f"{_sha256(output_dir / name)} {name}\n" for name in sorted(expected)) + with checksum_path.open("x", encoding="ascii", newline="\n") as stream: + stream.write(content) + return checksum_path + + +def build_release_assets(*, root: Path, tag: str, output_dir: Path) -> Path: + """Build a clean, version-matched distribution set and its checksums.""" + + name, version = project_identity(root) + validate_release_tag(tag, version) + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError("release output directory must be empty") + output_dir.mkdir(parents=True, exist_ok=True) + uv = shutil.which("uv") + if uv is None: + raise ValueError("uv executable was not found") + subprocess.run( # noqa: S603 + [uv, "build", "--out-dir", str(output_dir)], + cwd=root, + check=True, + ) + (output_dir / ".gitignore").unlink(missing_ok=True) + return write_checksum_manifest(output_dir, expected_artifact_names(name, version)) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--tag", required=True) + parser.add_argument("--output-dir", type=Path, default=Path("dist")) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + root = Path(__file__).resolve().parents[1] + try: + checksum_path = build_release_assets( + root=root, + tag=args.tag, + output_dir=args.output_dir.resolve(), + ) + except (OSError, subprocess.CalledProcessError, ValueError) as exc: + sys.stderr.write(f"release asset build failed: {exc}\n") + return 1 + sys.stdout.write(f"release assets: pass ({checksum_path})\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/repository_audit.py b/scripts/repository_audit.py index 7b69c12..e49f455 100644 --- a/scripts/repository_audit.py +++ b/scripts/repository_audit.py @@ -5,8 +5,12 @@ import re import subprocess +from collections.abc import Iterator from pathlib import Path +import yaml +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode + MAX_TRACKED_BYTES = 1_000_000 BINARY_SUFFIXES = { ".7z", @@ -28,6 +32,7 @@ ".webm", ".zip", } +_PINNED_ACTION = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_./-]+)?@[0-9a-f]{40}$") def _forbidden_patterns() -> list[tuple[str, re.Pattern[str]]]: @@ -54,6 +59,59 @@ def tracked_files(root: Path) -> list[Path]: return [root / item.decode("utf-8") for item in result.stdout.split(b"\x00") if item] +def _uses_nodes(node: Node, seen: set[int] | None = None) -> Iterator[Node]: + visited = seen if seen is not None else set() + identity = id(node) + if identity in visited: + return + visited.add(identity) + + if isinstance(node, MappingNode): + for key, value in node.value: + if isinstance(key, ScalarNode) and key.value == "uses": + yield value + yield from _uses_nodes(value, visited) + elif isinstance(node, SequenceNode): + for value in node.value: + yield from _uses_nodes(value, visited) + + +def action_reference_violations(relative: Path, text: str) -> list[str]: + """Return invalid YAML and mutable or unsupported external action references.""" + + violations: list[str] = [] + try: + document = yaml.compose(text, Loader=yaml.SafeLoader) + except yaml.YAMLError as exc: + mark = getattr(exc, "problem_mark", None) + line = mark.line + 1 if mark is not None else 1 + return [f"{relative}:{line}: action definition is not valid YAML"] + if document is None: + return violations + + for node in _uses_nodes(document): + line = node.start_mark.line + 1 + if not isinstance(node, ScalarNode): + violations.append(f"{relative}:{line}: action reference must be a scalar string") + continue + reference = node.value + if reference.startswith("./"): + continue + if not _PINNED_ACTION.fullmatch(reference): + violations.append( + f"{relative}:{line}: external action is not pinned to a full commit SHA" + ) + return violations + + +def contains_action_references(relative: Path) -> bool: + if relative.suffix not in {".yml", ".yaml"}: + return False + if relative.parts[:2] == (".github", "workflows"): + return True + return relative.name in {"action.yml", "action.yaml"} + + def main() -> int: root = Path(__file__).resolve().parents[1] violations: list[str] = [] @@ -76,6 +134,8 @@ def main() -> int: for label, pattern in _forbidden_patterns(): if pattern.search(text): violations.append(f"{relative}: contains {label}") + if contains_action_references(relative): + violations.extend(action_reference_violations(relative, text)) if violations: print("\n".join(sorted(violations))) return 1 diff --git a/scripts/validate_release_assets.py b/scripts/validate_release_assets.py new file mode 100644 index 0000000..110e801 --- /dev/null +++ b/scripts/validate_release_assets.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Require an exact release-asset name and SHA-256 digest set.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(64 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def validate_release_assets(local_dir: Path, inventory: object) -> list[str]: + violations: list[str] = [] + if not isinstance(inventory, list) or not all(isinstance(item, dict) for item in inventory): + return ["invalid remote asset inventory"] + local = { + path.name: f"sha256:{_sha256(path)}" + for path in local_dir.iterdir() + if path.is_file() and not path.is_symlink() + } + remote: dict[str, str] = {} + for item in inventory: + name = item.get("name") + digest = item.get("digest") + state = item.get("state") + if not isinstance(name, str) or not name or name in remote: + violations.append("invalid or duplicate remote asset name") + continue + if state != "uploaded": + violations.append(f"remote asset is incomplete: {name}") + if not isinstance(digest, str) or not digest.startswith("sha256:"): + violations.append(f"remote asset is missing a SHA-256 digest: {name}") + continue + remote[name] = digest + if set(local) != set(remote): + violations.append("remote asset names do not match the verified build") + violations.extend( + f"remote asset digest does not match the verified build: {name}" + for name in sorted(set(local) & set(remote)) + if local[name] != remote[name] + ) + return violations + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--local-dir", type=Path, required=True) + parser.add_argument("--inventory", type=Path, required=True) + arguments = parser.parse_args() + inventory = json.loads(arguments.inventory.read_text(encoding="utf-8")) + violations = validate_release_assets(arguments.local_dir, inventory) + if violations: + print("release asset validation failed: " + "; ".join(violations)) + return 1 + print("release asset validation: pass") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_release_metadata.py b/scripts/validate_release_metadata.py new file mode 100644 index 0000000..72c03c2 --- /dev/null +++ b/scripts/validate_release_metadata.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Validate public release names and generated notes before publication.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +TAG_PATTERN = re.compile(r"v[0-9]+\.[0-9]+\.[0-9]+") +EMAIL_PATTERN = re.compile(r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b") +HANDLE_PATTERN = re.compile(r"(?https://github\.com/[^/\s]+/[^/\s]+/pull/(?P[0-9]+))\s*$" +) +PRIVATE_REFERENCE_PATTERNS = ( + re.compile(r"(?i)\b(?:codex|dept)/[a-z0-9._/-]+"), + re.compile(r"(?i)\binternal\s+workflow\b"), + re.compile(r"(?i)\bindependent\s+review\b"), + re.compile(re.escape("Mission" + " Control"), re.IGNORECASE), + re.compile(re.escape("startup" + "-idea"), re.IGNORECASE), + re.compile( + r"(?i)tovellan-(?:platform|bench|codex|design|trust|web|infra|handbook|" + r"research|brand|sdk)\b" + ), + re.compile(re.escape("GST" + "-Bench"), re.IGNORECASE), + re.compile("61f214" + "e7272095", re.IGNORECASE), + re.compile("d57d6f" + "04a22e1e", re.IGNORECASE), + re.compile(r"(?i)cosine\s*(?:>=|≥)\s*0\.88"), +) +PRODUCT_POSITIONING_PATTERNS = ( + re.compile(r"(?i)\b" + "found" + r"ers?\b"), + re.compile(r"(?i)(? object: + if not isinstance(body, str): + return body + sanitized: list[str] = [] + skipping_contributors = False + for line in body.splitlines(): + if line.strip().casefold() == "## new contributors": + skipping_contributors = True + continue + if skipping_contributors: + if line.startswith(("## ", "**Full Changelog**")): + skipping_contributors = False + else: + continue + processed_line = AUTHOR_CREDIT_PATTERN.sub(r" ([#\g](\g))", line) + sanitized.append(processed_line) + return "\n".join(sanitized).strip() + "\n" + + +def validate_release_metadata(tag: object, name: object, body: object) -> list[str]: + violations: list[str] = [] + if not isinstance(tag, str) or TAG_PATTERN.fullmatch(tag) is None: + violations.append("invalid release tag") + return violations + if name != f"SplitSeal {tag}": + violations.append("invalid release name") + if not isinstance(body, str) or not body.strip(): + violations.append("missing release notes") + return violations + if any(character in body for character in PROHIBITED_CHARACTERS): + violations.append("prohibited Unicode punctuation") + if TRAILER_PATTERN.search(body): + violations.append("prohibited attribution trailer") + if any(pattern.search(body) for pattern in PRIVATE_REFERENCE_PATTERNS): + violations.append("prohibited private workflow reference") + if any(pattern.search(body) for pattern in PRODUCT_POSITIONING_PATTERNS): + violations.append("prohibited product-positioning term") + if any(pattern.search(body) for pattern in LOCAL_PATH_PATTERNS): + violations.append("prohibited local path") + if HANDLE_PATTERN.search(body): + violations.append("prohibited personal account handle") + unexpected_emails = { + address.lower() + for address in EMAIL_PATTERN.findall(body) + if address.lower() != "noreply@github.com" + } + if unexpected_emails: + violations.append("prohibited personal email address") + return violations + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--tag", required=True) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--sanitize-generated", action="store_true") + arguments = parser.parse_args() + document = json.loads(arguments.input.read_text(encoding="utf-8")) + if not isinstance(document, dict): + print("release metadata validation failed: invalid response") + return 1 + if arguments.sanitize_generated: + document = { + "name": f"SplitSeal {arguments.tag}", + "body": sanitize_generated_notes(document.get("body")), + } + violations = validate_release_metadata( + arguments.tag, document.get("name"), document.get("body") + ) + if violations: + print("release metadata validation failed: " + "; ".join(violations)) + return 1 + if arguments.output is not None: + arguments.output.write_text( + json.dumps(document, ensure_ascii=False, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + print("release metadata validation: pass") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/version_audit.py b/scripts/version_audit.py new file mode 100644 index 0000000..1418142 --- /dev/null +++ b/scripts/version_audit.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Fail when release and compatibility versions drift across tracked surfaces.""" + +from __future__ import annotations + +import ast +import re +import tomllib +from pathlib import Path + + +def _runtime_version(path: Path) -> str | None: + module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + versions = [ + node.value.value + for node in module.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "__version__" for target in node.targets + ) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ] + return versions[0] if len(versions) == 1 else None + + +def _locked_project_version(path: Path) -> str | None: + with path.open("rb") as stream: + document = tomllib.load(stream) + packages = document.get("package") + if not isinstance(packages, list): + return None + versions = [ + package.get("version") + for package in packages + if isinstance(package, dict) and package.get("name") == "splitseal" + ] + return versions[0] if len(versions) == 1 and isinstance(versions[0], str) else None + + +def version_violations(root: Path) -> list[str]: + """Return every version surface that disagrees with project.version.""" + + with (root / "pyproject.toml").open("rb") as stream: + project = tomllib.load(stream).get("project") + if not isinstance(project, dict) or not isinstance(project.get("version"), str): + return ["pyproject.toml: project.version is missing or invalid"] + version = project["version"] + release_match = re.fullmatch(r"(\d+)\.(\d+)(?:\.\d+.*)?", version) + if release_match is None: + return ["pyproject.toml: project.version does not identify a release line"] + release_line = f"{release_match.group(1)}.{release_match.group(2)}" + violations: list[str] = [] + + runtime_version = _runtime_version(root / "src" / "splitseal" / "__init__.py") + if runtime_version != version: + violations.append( + f"src/splitseal/__init__.py: __version__ is {runtime_version!r}, expected {version!r}" + ) + + locked_version = _locked_project_version(root / "uv.lock") + if locked_version != version: + violations.append(f"uv.lock: splitseal is {locked_version!r}, expected {version!r}") + + exact_checks = { + "README.md": rf"splitseal\.git@v{re.escape(version)}(?:\"|\s)", + "CHANGELOG.md": rf"^## \[{re.escape(version)}\] - \d{{4}}-\d{{2}}-\d{{2}}$", + "SUPPORT.md": rf"Within the {re.escape(release_line)} release line", + "docs/api.md": rf"The stable {re.escape(release_line)} API", + "docs/manifest-format.md": rf"within a {re.escape(release_line)} release", + "ROADMAP.md": rf"^## Delivered in {re.escape(release_line)}$", + } + for relative, pattern in exact_checks.items(): + text = (root / relative).read_text(encoding="utf-8") + if re.search(pattern, text, re.MULTILINE) is None: + violations.append(f"{relative}: does not declare release {version}") + return violations + + +def main() -> int: + root = Path(__file__).resolve().parents[1] + violations = version_violations(root) + if violations: + print("\n".join(violations)) + return 1 + print("version audit: pass") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/splitseal/__init__.py b/src/splitseal/__init__.py index b291aa2..1ade8c3 100644 --- a/src/splitseal/__init__.py +++ b/src/splitseal/__init__.py @@ -1,6 +1,6 @@ """SplitSeal public API.""" -__version__ = "0.3.0" +__version__ = "0.4.0" from splitseal.canonical import canonicalize, dataset_digest, record_digest from splitseal.errors import SplitSealError diff --git a/src/splitseal/canonical.py b/src/splitseal/canonical.py index bf33e69..1061704 100644 --- a/src/splitseal/canonical.py +++ b/src/splitseal/canonical.py @@ -4,7 +4,8 @@ import hashlib import math -from collections.abc import Mapping, Sequence +import re +from collections.abc import Iterable, Mapping from typing import TypeAlias, cast import rfc8785 @@ -19,9 +20,13 @@ _SEQUENCE_DOMAIN = b"splitseal-sequence-v1\x00" _DATASET_DOMAIN = b"splitseal-dataset-v1\x00" _MAX_INTEROPERABLE_INTEGER = 9_007_199_254_740_991 +_MAX_NESTING_DEPTH = 100 +_MAX_RECORD_COUNT = 2**64 - 1 +_SPLIT_DIGEST_ENTRY_SIZE = 2 +_SHA256_HEX = re.compile(r"[0-9A-Fa-f]{64}") -def _validate_json(value: object, location: str = "$") -> None: +def _validate_json(value: object, location: str = "$", depth: int = 0) -> None: if value is None or isinstance(value, (str, bool)): return if isinstance(value, int): @@ -37,14 +42,28 @@ def _validate_json(value: object, location: str = "$") -> None: raise fail("SS011", "non-finite numbers are not canonical JSON", location=location) return if isinstance(value, list): + if depth >= _MAX_NESTING_DEPTH: + raise fail( + "SS011", + "structured value exceeds the maximum nesting depth", + location=location, + maximum_depth=_MAX_NESTING_DEPTH, + ) for index, item in enumerate(value): - _validate_json(item, f"{location}[{index}]") + _validate_json(item, f"{location}[{index}]", depth + 1) return if isinstance(value, Mapping): + if depth >= _MAX_NESTING_DEPTH: + raise fail( + "SS011", + "structured value exceeds the maximum nesting depth", + location=location, + maximum_depth=_MAX_NESTING_DEPTH, + ) for key, item in value.items(): if not isinstance(key, str): raise fail("SS011", "JSON object keys must be strings", location=location) - _validate_json(item, f"{location}.{key}") + _validate_json(item, f"{location}.{key}", depth + 1) return raise fail("SS011", "unsupported value in structured record", location=location) @@ -52,10 +71,13 @@ def _validate_json(value: object, location: str = "$") -> None: def canonicalize(value: JSONValue) -> bytes: """Return RFC 8785 canonical JSON bytes after strict input validation.""" - _validate_json(value) + try: + _validate_json(value) + except RecursionError as exc: + raise fail("SS011", "structured value exceeds the maximum nesting depth") from exc try: return rfc8785.dumps(value) - except (rfc8785.CanonicalizationError, UnicodeError) as exc: + except (RecursionError, rfc8785.CanonicalizationError, UnicodeError) as exc: raise fail("SS011", "value cannot be encoded as canonical JSON") from exc @@ -70,17 +92,20 @@ def record_digest(record: Record) -> str: return hashlib.sha256(_RECORD_DOMAIN + _framed(payload)).hexdigest() -def sequence_digest(record_digests: Sequence[str]) -> str: +def sequence_digest(record_digests: Iterable[str]) -> str: """Hash an ordered sequence of hexadecimal record digests.""" + if isinstance(record_digests, (str, bytes, bytearray)) or not isinstance( + record_digests, Iterable + ): + raise fail("SS012", "record digests must be an iterable of strings") digest = hashlib.sha256(_SEQUENCE_DOMAIN) for item in record_digests: - try: - raw = bytes.fromhex(item) - except ValueError as exc: - raise fail("SS012", "record digest is not hexadecimal") from exc - if len(raw) != hashlib.sha256().digest_size: - raise fail("SS012", "record digest has an invalid length") + if not isinstance(item, str): + raise fail("SS012", "record digest must be a string") + if not _SHA256_HEX.fullmatch(item): + raise fail("SS012", "record digest must contain exactly 64 hexadecimal characters") + raw = bytes.fromhex(item) digest.update(_framed(raw)) return digest.hexdigest() @@ -88,18 +113,38 @@ def sequence_digest(record_digests: Sequence[str]) -> str: def dataset_digest(splits: Mapping[str, tuple[int, str]]) -> str: """Hash named split roots and counts in split-name order.""" + if not isinstance(splits, Mapping): + raise fail("SS012", "dataset splits must be a mapping") + validated: list[tuple[str, int, str]] = [] + for name, value in splits.items(): + if not isinstance(name, str): + raise fail("SS012", "split names must be strings") + if not isinstance(value, tuple) or len(value) != _SPLIT_DIGEST_ENTRY_SIZE: + raise fail("SS012", "split digest entry must be a count and digest pair", split=name) + count, split_digest = value + if type(count) is not int or count < 0 or count > _MAX_RECORD_COUNT: + raise fail( + "SS012", + "record count must be an unsigned 64-bit integer", + split=name, + ) + if not isinstance(split_digest, str): + raise fail("SS012", "split digest must be a string", split=name) + validated.append((name, count, split_digest)) + digest = hashlib.sha256(_DATASET_DOMAIN) - for name in sorted(splits): - count, split_digest = splits[name] - if count < 0: - raise fail("SS012", "record count cannot be negative", split=name) + for name, count, split_digest in sorted(validated, key=lambda item: item[0]): + if not _SHA256_HEX.fullmatch(split_digest): + raise fail( + "SS012", + "split digest must contain exactly 64 hexadecimal characters", + split=name, + ) + raw_digest = bytes.fromhex(split_digest) try: - raw_digest = bytes.fromhex(split_digest) - except ValueError as exc: - raise fail("SS012", "split digest is not hexadecimal", split=name) from exc - if len(raw_digest) != hashlib.sha256().digest_size: - raise fail("SS012", "split digest has an invalid length", split=name) - encoded_name = name.encode("utf-8") + encoded_name = name.encode("utf-8") + except UnicodeEncodeError as exc: + raise fail("SS012", "split name is not valid UTF-8") from exc digest.update(_framed(encoded_name)) digest.update(count.to_bytes(8, "big")) digest.update(_framed(raw_digest)) diff --git a/src/splitseal/crypto.py b/src/splitseal/crypto.py index 35fe204..2f8aec7 100644 --- a/src/splitseal/crypto.py +++ b/src/splitseal/crypto.py @@ -7,16 +7,20 @@ import hmac import json import os +import tempfile +from pathlib import Path +from typing import BinaryIO from cryptography.exceptions import InvalidTag from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives.ciphers.aead import AESGCM from cryptography.hazmat.primitives.hashes import HashAlgorithm from cryptography.hazmat.primitives.kdf.hkdf import HKDF from cryptography.hazmat.primitives.kdf.scrypt import Scrypt from splitseal.canonical import JSONValue, canonicalize -from splitseal.errors import fail +from splitseal.errors import SplitSealError, fail SEAL_SCHEMA = "splitseal.seal.v1" _AAD = SEAL_SCHEMA.encode("ascii") @@ -33,15 +37,33 @@ def _b64encode(value: bytes) -> str: def _b64decode(value: object, field: str) -> bytes: - if not isinstance(value, str): - raise fail("SS040", "sealed manifest field must be a string", field=field) + if not isinstance(value, str) or "=" in value: + raise fail( + "SS040", + "sealed manifest field must be unpadded base64url", + field=field, + ) try: - return base64.b64decode(value + "=" * (-len(value) % 4), altchars=b"-_", validate=True) + decoded = base64.b64decode( + value + "=" * (-len(value) % 4), + altchars=b"-_", + validate=True, + ) except (ValueError, TypeError) as exc: raise fail("SS040", "sealed manifest contains invalid base64url", field=field) from exc + if _b64encode(decoded) != value: + raise fail("SS040", "sealed manifest contains noncanonical base64url", field=field) + return decoded + + +def _require_fields(value: dict[object, object], expected: set[str], context: str) -> None: + if set(value) != expected: + raise fail("SS040", "sealed manifest fields do not match the schema", context=context) def validate_secret(secret: bytes) -> None: + if not isinstance(secret, bytes): + raise fail("SS041", "key material must be bytes") if len(secret) < _MINIMUM_SECRET_BYTES: raise fail("SS041", "key material must contain at least 16 bytes") @@ -73,6 +95,68 @@ def commitment(manifest_bytes: bytes, secret: bytes) -> str: return hmac.new(_commitment_key(secret), manifest_bytes, hashlib.sha256).hexdigest() +def commitment_file(manifest_path: Path, secret: bytes) -> str: + """Commit to manifest bytes from disk without loading the manifest into memory.""" + + digest = hmac.new(_commitment_key(secret), digestmod=hashlib.sha256) + with manifest_path.open("rb") as stream: + while chunk := stream.read(64 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _write_base64url(source: BinaryIO, destination: BinaryIO) -> None: + carry = b"" + while chunk := source.read(64 * 1024): + data = carry + chunk + complete = len(data) // 3 * 3 + if complete: + destination.write(base64.urlsafe_b64encode(data[:complete])) + carry = data[complete:] + if carry: + destination.write(base64.urlsafe_b64encode(carry).rstrip(b"=")) + + +def seal_manifest_file(manifest_path: Path, seal_path: Path, secret: bytes) -> None: + """Encrypt canonical manifest bytes from disk into a canonical v1 seal.""" + + salt = os.urandom(_SALT_BYTES) + nonce = os.urandom(_NONCE_BYTES) + descriptor, raw_cipher_path = tempfile.mkstemp(prefix=".splitseal-ciphertext-") + os.close(descriptor) + ciphertext_path = Path(raw_cipher_path) + try: + encryptor = Cipher( + algorithms.AES(_encryption_key(secret, salt)), + modes.GCM(nonce), + ).encryptor() + encryptor.authenticate_additional_data(_AAD) + with manifest_path.open("rb") as manifest, ciphertext_path.open("wb") as ciphertext: + while chunk := manifest.read(64 * 1024): + ciphertext.write(encryptor.update(chunk)) + ciphertext.write(encryptor.finalize()) + ciphertext.write(encryptor.tag) + kdf: JSONValue = { + "name": "scrypt", + "n": _KDF_N, + "r": _KDF_R, + "p": _KDF_P, + "salt": _b64encode(salt), + } + with seal_path.open("wb") as seal, ciphertext_path.open("rb") as ciphertext: + seal.write(b'{"cipher":{"ciphertext":"') + _write_base64url(ciphertext, seal) + seal.write(b'","name":"aes-256-gcm","nonce":') + seal.write(canonicalize(_b64encode(nonce))) + seal.write(b'},"kdf":') + seal.write(canonicalize(kdf)) + seal.write(b',"schema_version":"splitseal.seal.v1"}\n') + seal.flush() + os.fsync(seal.fileno()) + finally: + ciphertext_path.unlink(missing_ok=True) + + def seal_manifest(manifest: JSONValue, secret: bytes) -> bytes: plaintext = canonicalize(manifest) salt = os.urandom(_SALT_BYTES) @@ -97,14 +181,22 @@ def seal_manifest(manifest: JSONValue, secret: bytes) -> bytes: def open_seal(container: object, secret: bytes) -> dict[str, JSONValue]: - if not isinstance(container, dict) or container.get("schema_version") != SEAL_SCHEMA: + if not isinstance(container, dict): + raise fail("SS040", "sealed manifest must be an object") + _require_fields(container, {"schema_version", "kdf", "cipher"}, "seal") + if container.get("schema_version") != SEAL_SCHEMA: raise fail("SS040", "sealed manifest has an unsupported schema") kdf = container.get("kdf") cipher = container.get("cipher") if not isinstance(kdf, dict) or not isinstance(cipher, dict): raise fail("SS040", "sealed manifest is missing cryptographic parameters") - expected_kdf = {"name": "scrypt", "n": _KDF_N, "r": _KDF_R, "p": _KDF_P} - if any(kdf.get(key) != value for key, value in expected_kdf.items()): + _require_fields(kdf, {"name", "n", "r", "p", "salt"}, "kdf") + _require_fields(cipher, {"name", "nonce", "ciphertext"}, "cipher") + expected_kdf_integers = {"n": _KDF_N, "r": _KDF_R, "p": _KDF_P} + if kdf.get("name") != "scrypt" or any( + type(kdf.get(key)) is not int or kdf.get(key) != value + for key, value in expected_kdf_integers.items() + ): raise fail("SS040", "sealed manifest uses unsupported KDF parameters") if cipher.get("name") != "aes-256-gcm": raise fail("SS040", "sealed manifest uses an unsupported cipher") @@ -119,10 +211,14 @@ def open_seal(container: object, secret: bytes) -> dict[str, JSONValue]: raise fail("SS042", "sealed manifest authentication failed") from exc try: manifest = json.loads(plaintext) - except (UnicodeDecodeError, ValueError) as exc: + except (RecursionError, UnicodeDecodeError, ValueError) as exc: raise fail("SS040", "decrypted manifest is not valid JSON") from exc if not isinstance(manifest, dict): raise fail("SS040", "decrypted manifest must be an object") - if canonicalize(manifest) != plaintext: + try: + canonical = canonicalize(manifest) + except SplitSealError as exc: + raise fail("SS040", "decrypted manifest contains invalid JSON values") from exc + if canonical != plaintext: raise fail("SS040", "decrypted manifest is not canonically encoded") return manifest diff --git a/src/splitseal/loaders.py b/src/splitseal/loaders.py index e31ca9d..0aad298 100644 --- a/src/splitseal/loaders.py +++ b/src/splitseal/loaders.py @@ -2,11 +2,11 @@ from __future__ import annotations +import codecs import csv import importlib -import io import json -from collections.abc import Callable +from collections.abc import Callable, Iterator from pathlib import Path from typing import Any, cast @@ -27,13 +27,60 @@ def _reject_constant(value: str) -> None: raise fail("SS020", "JSON contains a non-finite number", value=value) -def _load_jsonl(path: Path) -> list[Record]: +_SPLITLINE_SEPARATORS = frozenset( + {"\n", "\v", "\f", "\x1c", "\x1d", "\x1e", "\x85", "\u2028", "\u2029"} +) +_PARQUET_BATCH_SIZE = 1024 + + +def _iter_text_lines(path: Path, encoding: str) -> Iterator[str]: # noqa: PLR0912 + decoder = codecs.getincrementaldecoder(encoding)(errors="strict") + line: list[str] = [] + pending_cr = False try: - text = path.read_text(encoding="utf-8") + with path.open("rb") as stream: + while chunk := stream.read(64 * 1024): + text = decoder.decode(chunk) + for character in text: + if pending_cr: + yield "".join(line) + line = [] + pending_cr = False + if character == "\n": + continue + if character == "\r": + pending_cr = True + elif character in _SPLITLINE_SEPARATORS: + yield "".join(line) + line = [] + else: + line.append(character) + for character in decoder.decode(b"", final=True): + if pending_cr: + yield "".join(line) + line = [] + pending_cr = False + if character == "\n": + continue + if character == "\r": + pending_cr = True + elif character in _SPLITLINE_SEPARATORS: + yield "".join(line) + line = [] + else: + line.append(character) except (OSError, UnicodeDecodeError) as exc: - raise fail("SS020", "JSONL input must be readable UTF-8", path=path.name) from exc - records: list[Record] = [] - for line_number, line in enumerate(text.splitlines(), start=1): + raise fail( + "SS020", "structured input must be readable Unicode text", path=path.name + ) from exc + if pending_cr or line: + yield "".join(line) + + +def _iter_jsonl(path: Path) -> Iterator[Record]: + found = False + for line_number, line in enumerate(_iter_text_lines(path, "utf-8"), start=1): + found = True if not line.strip(): raise fail("SS020", "JSONL input cannot contain blank lines", line=line_number) try: @@ -44,41 +91,47 @@ def _load_jsonl(path: Path) -> list[Record]: ) except SplitSealError: raise - except (json.JSONDecodeError, UnicodeError) as exc: + except (RecursionError, json.JSONDecodeError, UnicodeError) as exc: raise fail("SS020", "malformed JSONL record", line=line_number) from exc - records.append(ensure_record(value, location=f"line {line_number}")) - if not records: + yield ensure_record(value, location=f"line {line_number}") + if not found: raise fail("SS020", "dataset split cannot be empty", path=path.name) - return records -def _load_csv(path: Path) -> list[Record]: +def _load_jsonl(path: Path) -> list[Record]: + return list(_iter_jsonl(path)) + + +def _iter_csv(path: Path) -> Iterator[Record]: try: - text = path.read_text(encoding="utf-8-sig") + with path.open("r", encoding="utf-8-sig") as stream: + reader = csv.DictReader(stream, strict=True) + headers = reader.fieldnames + if not headers or any(not header for header in headers): + raise fail("SS020", "CSV input must have non-empty headers") + if len(set(headers)) != len(headers): + raise fail("SS020", "CSV input contains duplicate headers") + found = False + for row_number, row in enumerate(reader, start=2): + found = True + if None in row: + raise fail("SS020", "CSV row has more fields than its header", row=row_number) + if any(value is None for value in row.values()): + raise fail("SS020", "CSV row has fewer fields than its header", row=row_number) + yield {key: value for key, value in row.items() if value is not None} except (OSError, UnicodeDecodeError) as exc: raise fail("SS020", "CSV input must be readable UTF-8", path=path.name) from exc - try: - reader = csv.DictReader(io.StringIO(text, newline=""), strict=True) - headers = reader.fieldnames - if not headers or any(not header for header in headers): - raise fail("SS020", "CSV input must have non-empty headers") - if len(set(headers)) != len(headers): - raise fail("SS020", "CSV input contains duplicate headers") - records: list[Record] = [] - for row_number, row in enumerate(reader, start=2): - if None in row: - raise fail("SS020", "CSV row has more fields than its header", row=row_number) - if any(value is None for value in row.values()): - raise fail("SS020", "CSV row has fewer fields than its header", row=row_number) - records.append({key: value for key, value in row.items() if value is not None}) except csv.Error as exc: raise fail("SS020", "malformed CSV input") from exc - if not records: + if not found: raise fail("SS020", "dataset split cannot be empty", path=path.name) - return records -def _load_parquet(path: Path) -> list[Record]: +def _load_csv(path: Path) -> list[Record]: + return list(_iter_csv(path)) + + +def _iter_parquet(path: Path) -> Iterator[Record]: try: parquet = importlib.import_module("pyarrow.parquet") except ImportError as exc: @@ -87,12 +140,25 @@ def _load_parquet(path: Path) -> list[Record]: "Parquet support is optional; install splitseal[parquet]", ) from exc try: - rows = cast("list[dict[str, Any]]", parquet.read_table(path).to_pylist()) + parquet_file = parquet.ParquetFile(path) + found = False + row_index = 0 + for batch in parquet_file.iter_batches(batch_size=_PARQUET_BATCH_SIZE): + rows = cast("list[dict[str, Any]]", batch.to_pylist()) + for row in rows: + found = True + yield ensure_record(row, location=f"row {row_index}") + row_index += 1 + except SplitSealError: + raise except Exception as exc: raise fail("SS020", "malformed Parquet input", path=path.name) from exc - if not rows: + if not found: raise fail("SS020", "dataset split cannot be empty", path=path.name) - return [ensure_record(row, location=f"row {index}") for index, row in enumerate(rows)] + + +def _load_parquet(path: Path) -> list[Record]: + return list(_iter_parquet(path)) _LOADERS: dict[str, Callable[[Path], list[Record]]] = { @@ -101,6 +167,12 @@ def _load_parquet(path: Path) -> list[Record]: "parquet": _load_parquet, } +_ITERATORS: dict[str, Callable[[Path], Iterator[Record]]] = { + "jsonl": _iter_jsonl, + "csv": _iter_csv, + "parquet": _iter_parquet, +} + def load_records(path: Path, format_name: str) -> list[Record]: try: @@ -108,3 +180,13 @@ def load_records(path: Path, format_name: str) -> list[Record]: except KeyError as exc: raise fail("SS010", "unsupported split format", format=format_name) from exc return loader(path) + + +def iter_records(path: Path, format_name: str) -> Iterator[Record]: + """Yield strict structured records without retaining the full split in memory.""" + + try: + iterator = _ITERATORS[format_name] + except KeyError as exc: + raise fail("SS010", "unsupported split format", format=format_name) from exc + yield from iterator(path) diff --git a/src/splitseal/models.py b/src/splitseal/models.py index cf1596e..4b7619a 100644 --- a/src/splitseal/models.py +++ b/src/splitseal/models.py @@ -65,7 +65,7 @@ def _validate_name(value: str, context: str) -> None: def parse_config_bytes(data: bytes) -> DatasetConfig: try: raw = tomllib.loads(data.decode("utf-8")) - except (UnicodeDecodeError, tomllib.TOMLDecodeError) as exc: + except (RecursionError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc: raise fail("SS010", "configuration is not valid UTF-8 TOML") from exc if raw.get("schema_version") != "splitseal.config.v1": raise fail("SS010", "configuration schema_version must be splitseal.config.v1") @@ -124,4 +124,4 @@ def load_config(path: Path) -> DatasetConfig: try: return parse_config_bytes(path.read_bytes()) except OSError as exc: - raise fail("SS001", "configuration could not be read", path=str(path)) from exc + raise fail("SS001", "configuration could not be read", path=path.name) from exc diff --git a/src/splitseal/plugins.py b/src/splitseal/plugins.py index b29e7a5..755d24d 100644 --- a/src/splitseal/plugins.py +++ b/src/splitseal/plugins.py @@ -34,14 +34,29 @@ def analyze( def load_similarity_plugin(name: str) -> SimilarityPlugin: - matches = [entry for entry in entry_points(group="splitseal.similarity") if entry.name == name] + try: + matches = [ + entry for entry in entry_points(group="splitseal.similarity") if entry.name == name + ] + except Exception as exc: + raise fail("SS060", "similarity plugins could not be discovered", plugin=name) from exc if len(matches) != 1: raise fail("SS060", "similarity plugin is not installed exactly once", plugin=name) try: plugin = matches[0].load()() + analyze = plugin.analyze + plugin_name = plugin.name + version = plugin.version except Exception as exc: raise fail("SS060", "similarity plugin could not be loaded", plugin=name) from exc - if not hasattr(plugin, "analyze") or not hasattr(plugin, "version"): + if ( + not callable(analyze) + or not isinstance(plugin_name, str) + or not plugin_name + or plugin_name != name + or not isinstance(version, str) + or not version + ): raise fail( "SS060", "similarity plugin does not implement the required interface", diff --git a/src/splitseal/service.py b/src/splitseal/service.py index dc27b97..fde1a5d 100644 --- a/src/splitseal/service.py +++ b/src/splitseal/service.py @@ -16,12 +16,19 @@ from splitseal import __version__ from splitseal.canonical import JSONValue, Record, canonicalize, dataset_digest, record_digest from splitseal.canonical import sequence_digest as ordered_digest -from splitseal.crypto import commitment, open_seal, seal_manifest +from splitseal.crypto import ( + commitment, + commitment_file, + open_seal, + seal_manifest, + seal_manifest_file, +) from splitseal.errors import SplitSealError, fail from splitseal.loaders import load_records from splitseal.models import DatasetConfig, load_config from splitseal.paths import safe_input_path, safe_output_path from splitseal.plugins import SimilarityPlugin, load_similarity_plugin +from splitseal.streaming import StreamingManifest, build_streaming_manifest PRIVATE_SCHEMA = "splitseal.private-manifest.v1" ATTESTATION_SCHEMA = "splitseal.public-attestation.v1" @@ -77,11 +84,47 @@ def _build_manifest( plugin_evidence: list[dict[str, Any]] = [] similarity_finding_count = 0 for plugin_config in config.similarity: - plugin = plugin_loader(plugin_config.plugin) try: - findings = list(plugin.analyze(split_records, plugin_config.settings)) - except SplitSealError: - raise + plugin = plugin_loader(plugin_config.plugin) + except SplitSealError as exc: + if plugin_loader is load_similarity_plugin and exc.code == "SS060": + raise + raise fail( + "SS061", + "similarity plugin loading failed", + plugin=plugin_config.plugin, + ) from exc + except Exception as exc: + raise fail( + "SS061", + "similarity plugin loading failed", + plugin=plugin_config.plugin, + ) from exc + try: + analyze = plugin.analyze + plugin_name = plugin.name + plugin_version = plugin.version + except Exception as exc: + raise fail( + "SS061", + "similarity plugin interface inspection failed", + plugin=plugin_config.plugin, + ) from exc + if ( + not callable(analyze) + or not isinstance(plugin_name, str) + or not plugin_name + or plugin_name != plugin_config.plugin + or not isinstance(plugin_version, str) + or not plugin_version + ): + raise fail( + "SS061", + "similarity plugin does not implement the required interface", + plugin=plugin_config.plugin, + ) + try: + findings = list(analyze(split_records, plugin_config.settings)) except Exception as exc: raise fail( "SS061", @@ -92,7 +135,7 @@ def _build_manifest( plugin_evidence.append( { "name": plugin_config.plugin, - "version": str(plugin.version), + "version": plugin_version, "status": "pass" if not findings else "fail", } ) @@ -205,15 +248,9 @@ def _reserve_backup(target: Path) -> Path: return Path(raw_path) -def _atomic_write_pair( # noqa: PLR0912 - first: tuple[Path, bytes, int], - second: tuple[Path, bytes, int], - *, - force: bool, -) -> None: - outputs = (first, second) +def _output_state(targets: tuple[Path, Path], *, force: bool) -> dict[Path, bool]: existing: dict[Path, bool] = {} - for target, _content, _mode in outputs: + for target in targets: target_exists = target.exists() existing[target] = target_exists if target_exists and not force: @@ -222,17 +259,21 @@ def _atomic_write_pair( # noqa: PLR0912 "output already exists; pass --force to replace it", path=target.name, ) - first_temp = _write_temp(*first) - try: - second_temp = _write_temp(*second) - except BaseException: - first_temp.unlink(missing_ok=True) - raise + return existing + + +def _install_staged_pair( # noqa: PLR0912 + first: tuple[Path, Path], + second: tuple[Path, Path], + *, + existing: Mapping[Path, bool], +) -> None: + outputs = (first, second) backups: dict[Path, Path] = {} installed: set[Path] = set() preserve_backups = False try: - for target, _content, _mode in outputs: + for target, _temporary in outputs: if existing[target]: backup = _reserve_backup(target) try: @@ -242,11 +283,7 @@ def _atomic_write_pair( # noqa: PLR0912 raise backups[target] = backup - for temporary, (target, _content, _mode) in zip( - (first_temp, second_temp), - outputs, - strict=True, - ): + for target, temporary in outputs: os.replace(temporary, target) installed.add(target) except BaseException: @@ -271,13 +308,96 @@ def _atomic_write_pair( # noqa: PLR0912 ) from rollback_errors[0] raise finally: - first_temp.unlink(missing_ok=True) - second_temp.unlink(missing_ok=True) + for _target, temporary in outputs: + temporary.unlink(missing_ok=True) if not preserve_backups: for backup in backups.values(): backup.unlink(missing_ok=True) +def _atomic_write_pair( + first: tuple[Path, bytes, int], + second: tuple[Path, bytes, int], + *, + force: bool, +) -> None: + existing = _output_state((first[0], second[0]), force=force) + first_temp = _write_temp(*first) + try: + second_temp = _write_temp(*second) + except BaseException: + first_temp.unlink(missing_ok=True) + raise + _install_staged_pair( + (first[0], first_temp), + (second[0], second_temp), + existing=existing, + ) + + +def _streaming_attestation(manifest: StreamingManifest, secret: bytes) -> dict[str, Any]: + return { + "schema_version": ATTESTATION_SCHEMA, + "tool": {"name": "splitseal", "version": __version__}, + "release": {"name": manifest.release_name, "version": manifest.release_version}, + "commitment": { + "algorithm": "hmac-sha256", + "value": commitment_file(manifest.path, secret), + }, + "aggregates": { + "record_count": manifest.record_count, + "split_count": manifest.split_count, + "split_counts": list(manifest.split_counts), + }, + "checks": { + "exact_cross_split_duplicates": "pass", + "similarity": "not_run", + }, + } + + +def _freeze_streaming( # noqa: PLR0913 + *, + config: DatasetConfig, + root: Path, + seal_file: Path, + attestation_file: Path, + secret: bytes, + force: bool, +) -> dict[str, Any]: + existing = _output_state((seal_file, attestation_file), force=force) + with build_streaming_manifest(config, root) as manifest: + attestation = _streaming_attestation(manifest, secret) + descriptor, raw_seal_temp = tempfile.mkstemp( + prefix=f".{seal_file.name}.", + dir=seal_file.parent, + ) + os.fchmod(descriptor, 0o600) + os.close(descriptor) + seal_temp = Path(raw_seal_temp) + try: + seal_manifest_file(manifest.path, seal_temp, secret) + attestation_temp = _write_temp( + attestation_file, + canonicalize(_json_value(attestation)) + b"\n", + 0o644, + ) + except BaseException: + seal_temp.unlink(missing_ok=True) + raise + _install_staged_pair( + (seal_file, seal_temp), + (attestation_file, attestation_temp), + existing=existing, + ) + return { + "status": "created", + "release": attestation["release"], + "record_count": manifest.record_count, + "split_count": manifest.split_count, + } + + def freeze_release( # noqa: PLR0913 *, root: Path, @@ -296,6 +416,15 @@ def freeze_release( # noqa: PLR0913 if seal_file == attestation_file: raise fail("SS004", "seal and attestation paths must differ") config = load_config(config_file) + if not config.similarity: + return _freeze_streaming( + config=config, + root=root, + seal_file=seal_file, + attestation_file=attestation_file, + secret=secret, + force=force, + ) manifest = _build_manifest(config, root, plugin_loader=plugin_loader) attestation = _public_attestation(manifest, secret) seal_bytes = seal_manifest(_json_value(manifest), secret) @@ -330,11 +459,15 @@ def _load_json_file(path: Path, description: str) -> dict[str, Any]: value = json.loads(raw, object_pairs_hook=_reject_duplicate_keys) except SplitSealError: raise - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + except (OSError, RecursionError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise fail("SS044", f"{description} is not valid UTF-8 JSON", path=path.name) from exc if not isinstance(value, dict): raise fail("SS044", f"{description} must be a JSON object", path=path.name) - if raw != canonicalize(_json_value(value)) + b"\n": + try: + canonical = canonicalize(_json_value(value)) + b"\n" + except SplitSealError as exc: + raise fail("SS044", f"{description} contains invalid JSON values", path=path.name) from exc + if raw != canonical: raise fail("SS044", f"{description} is not canonically encoded", path=path.name) return value diff --git a/src/splitseal/streaming.py b/src/splitseal/streaming.py new file mode 100644 index 0000000..981b507 --- /dev/null +++ b/src/splitseal/streaming.py @@ -0,0 +1,213 @@ +"""Disk-spooled exact-only manifest construction.""" + +from __future__ import annotations + +import os +import sqlite3 +import tempfile +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, cast + +from splitseal import __version__ +from splitseal.canonical import JSONValue, canonicalize, dataset_digest, record_digest +from splitseal.canonical import sequence_digest as ordered_digest +from splitseal.errors import fail +from splitseal.loaders import iter_records +from splitseal.models import DatasetConfig +from splitseal.paths import safe_input_path + +_SQLITE_CACHE_KIB = 2048 + + +@dataclass(frozen=True) +class StreamingManifest: + path: Path + release_name: str + release_version: str + record_count: int + split_count: int + split_counts: tuple[int, ...] + + +@dataclass(frozen=True) +class _SplitSpool: + name: str + format: str + record_count: int + content_digest: str + digests_path: Path + + +def _configure_digest_database(connection: sqlite3.Connection) -> None: + connection.execute("PRAGMA journal_mode=OFF") + connection.execute("PRAGMA synchronous=OFF") + connection.execute("PRAGMA temp_store=FILE") + connection.execute(f"PRAGMA cache_size=-{_SQLITE_CACHE_KIB}") + connection.execute( + "CREATE TABLE digests (digest TEXT PRIMARY KEY, owner TEXT NOT NULL) WITHOUT ROWID" + ) + + +def _digest_lines(path: Path) -> Iterator[str]: + with path.open("r", encoding="ascii") as stream: + for line in stream: + yield line.removesuffix("\n") + + +def _spool_splits(config: DatasetConfig, root: Path, directory: Path) -> list[_SplitSpool]: + database_path = directory / "digest-owners.sqlite3" + connection = sqlite3.connect(database_path) + duplicate_count = 0 + spools: list[_SplitSpool] = [] + try: + _configure_digest_database(connection) + for index, split in enumerate(sorted(config.splits, key=lambda item: item.name)): + source = safe_input_path(root, split.path) + digests_path = directory / f"split-{index}.digests" + record_count = 0 + with digests_path.open("w", encoding="ascii", newline="\n") as digests: + for record in iter_records(source, split.format): + digest = record_digest(record) + cursor = connection.execute( + "INSERT OR IGNORE INTO digests (digest, owner) VALUES (?, ?)", + (digest, split.name), + ) + if cursor.rowcount == 0: + owner_row = connection.execute( + "SELECT owner FROM digests WHERE digest = ?", + (digest,), + ).fetchone() + if owner_row is not None and owner_row[0] != split.name: + duplicate_count += 1 + digests.write(digest + "\n") + record_count += 1 + spools.append( + _SplitSpool( + name=split.name, + format=split.format, + record_count=record_count, + content_digest=ordered_digest(_digest_lines(digests_path)), + digests_path=digests_path, + ) + ) + finally: + connection.close() + if duplicate_count: + raise fail( + "SS030", + "exact duplicate records were found across dataset splits", + duplicate_count=duplicate_count, + ) + return spools + + +def _write_json_value(stream: BinaryIO, value: JSONValue) -> None: + stream.write(canonicalize(value)) + + +def _write_split(stream: BinaryIO, split: _SplitSpool) -> None: + stream.write(b'{"content_digest":') + _write_json_value(stream, split.content_digest) + stream.write(b',"format":') + _write_json_value(stream, split.format) + stream.write(b',"name":') + _write_json_value(stream, split.name) + stream.write(b',"record_count":') + _write_json_value(stream, split.record_count) + stream.write(b',"record_digests":[') + first = True + for digest in _digest_lines(split.digests_path): + if not first: + stream.write(b",") + _write_json_value(stream, digest) + first = False + stream.write(b"]}") + + +def _write_manifest( + path: Path, + config: DatasetConfig, + spools: list[_SplitSpool], +) -> None: + split_roots = {split.name: (split.record_count, split.content_digest) for split in spools} + record_count = sum(split.record_count for split in spools) + canonicalization = cast( + "JSONValue", + { + "profile": "RFC8785", + "record_hash": "sha256", + "sequence_hash": "splitseal-sequence-v1", + "dataset_hash": "splitseal-dataset-v1", + }, + ) + checks = cast( + "JSONValue", + { + "exact_cross_split_duplicates": "pass", + "similarity": "not_run", + "similarity_plugins": [], + }, + ) + with path.open("wb") as stream: + stream.write(b'{"canonicalization":') + _write_json_value(stream, canonicalization) + stream.write(b',"checks":') + _write_json_value(stream, checks) + stream.write(b',"dataset":{"content_digest":') + _write_json_value(stream, dataset_digest(split_roots)) + stream.write(b',"record_count":') + _write_json_value(stream, record_count) + stream.write(b',"split_count":') + _write_json_value(stream, len(spools)) + stream.write(b',"splits":[') + for index, split in enumerate(spools): + if index: + stream.write(b",") + _write_split(stream, split) + stream.write(b']},"release":') + _write_json_value( + stream, + cast( + "JSONValue", + {"name": config.release.name, "version": config.release.version}, + ), + ) + stream.write(b',"schema_version":"splitseal.private-manifest.v1","tool":') + _write_json_value( + stream, + cast("JSONValue", {"name": "splitseal", "version": __version__}), + ) + stream.write(b"}") + stream.flush() + os.fsync(stream.fileno()) + path.chmod(0o600) + + +@contextmanager +def build_streaming_manifest( + config: DatasetConfig, + root: Path, +) -> Iterator[StreamingManifest]: + """Build an exact-only canonical manifest using bounded Python memory and temp disk.""" + + if config.similarity: + raise ValueError("streaming manifests do not accept similarity plugins") + with tempfile.TemporaryDirectory(prefix="splitseal-streaming-") as raw_directory: + directory = Path(raw_directory) + spools = _spool_splits(config, root, directory) + manifest_path = directory / "private-manifest.json" + _write_manifest(manifest_path, config, spools) + yield StreamingManifest( + path=manifest_path, + release_name=config.release.name, + release_version=config.release.version, + record_count=sum(split.record_count for split in spools), + split_count=len(spools), + split_counts=tuple(sorted(split.record_count for split in spools)), + ) + + +__all__ = ["StreamingManifest", "build_streaming_manifest"] diff --git a/tests/test_canonical.py b/tests/test_canonical.py index 0a958fa..7022772 100644 --- a/tests/test_canonical.py +++ b/tests/test_canonical.py @@ -56,12 +56,44 @@ def test_canonicalization_rejects_non_string_keys_and_objects() -> None: assert object_error.value.code == "SS011" +def test_canonicalization_rejects_excessive_nesting_with_stable_error() -> None: + value: object = 0 + for _ in range(101): + value = [value] + + with pytest.raises(SplitSealError) as caught: + canonicalize(value) # type: ignore[arg-type] + + assert caught.value.code == "SS011" + assert caught.value.details["maximum_depth"] == 100 + + empty_container: object = [] + for _ in range(100): + empty_container = [empty_container] + with pytest.raises(SplitSealError) as empty_caught: + canonicalize(empty_container) # type: ignore[arg-type] + assert empty_caught.value.code == "SS011" + assert empty_caught.value.details["maximum_depth"] == 100 + + +def test_canonicalization_accepts_exact_nesting_limit() -> None: + value: object = 0 + for _ in range(100): + value = [value] + canonicalize(value) # type: ignore[arg-type] + + def test_sequence_digest_is_order_sensitive() -> None: first = record_digest({"id": "one"}) second = record_digest({"id": "two"}) assert sequence_digest([first, second]) != sequence_digest([second, first]) +def test_sequence_digest_accepts_one_pass_iterables() -> None: + items = [record_digest({"id": "one"}), record_digest({"id": "two"})] + assert sequence_digest(item for item in items) == sequence_digest(items) + + @pytest.mark.parametrize("digest", ["not-hex", "ab"]) def test_sequence_digest_rejects_malformed_digest(digest: str) -> None: with pytest.raises(SplitSealError) as caught: @@ -69,6 +101,34 @@ def test_sequence_digest_rejects_malformed_digest(digest: str) -> None: assert caught.value.code == "SS012" +@pytest.mark.parametrize( + "digests", + [None, "00" * 32, b"00", [None], [1]], +) +def test_sequence_digest_rejects_wrong_runtime_types(digests: object) -> None: + with pytest.raises(SplitSealError) as caught: + sequence_digest(digests) # type: ignore[arg-type] + assert caught.value.code == "SS012" + + +@pytest.mark.parametrize( + "digest", + [ + "00 " * 32, + "00" * 16 + "\n" + "00" * 16, + "00" * 16 + "\t" + "00" * 16, + ], +) +def test_digest_functions_reject_ascii_whitespace(digest: str) -> None: + with pytest.raises(SplitSealError) as sequence_error: + sequence_digest([digest]) + assert sequence_error.value.code == "SS012" + + with pytest.raises(SplitSealError) as dataset_error: + dataset_digest({"split": (1, digest)}) + assert dataset_error.value.code == "SS012" + + def test_dataset_digest_sorts_split_names_but_includes_counts() -> None: root = record_digest({"id": "one"}) left = dataset_digest({"b": (1, root), "a": (1, root)}) @@ -86,6 +146,25 @@ def test_dataset_digest_rejects_invalid_inputs() -> None: dataset_digest({"a": (1, "00")}) +@pytest.mark.parametrize( + "splits", + [ + None, + {1: (1, "00" * 32)}, + {"a": [1, "00" * 32]}, + {"a": ("1", "00" * 32)}, + {"a": (True, "00" * 32)}, + {"a": (2**64, "00" * 32)}, + {"a": (1, None)}, + {"\ud800": (1, "00" * 32)}, + ], +) +def test_dataset_digest_rejects_wrong_runtime_types(splits: object) -> None: + with pytest.raises(SplitSealError) as caught: + dataset_digest(splits) # type: ignore[arg-type] + assert caught.value.code == "SS012" + + def test_ensure_record_rejects_non_object() -> None: with pytest.raises(SplitSealError) as caught: ensure_record(["not", "an", "object"], location="test") diff --git a/tests/test_cli_reporting_plugins.py b/tests/test_cli_reporting_plugins.py index 66abf67..0d4cd84 100644 --- a/tests/test_cli_reporting_plugins.py +++ b/tests/test_cli_reporting_plugins.py @@ -97,6 +97,29 @@ def test_cli_errors_are_machine_readable(project: Path, capsys: pytest.CaptureFi assert json.loads(captured.err)["error"]["code"] == "SS001" +def test_cli_deeply_nested_artifact_failure_is_machine_readable( + project: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + path = project / "artifacts" / "deep.json" + path.write_text('{"value":' + "[" * 101 + "0" + "]" * 101 + "}\n", encoding="utf-8") + + exit_code = main( + [ + "validate-public", + "--root", + str(project), + "--attestation", + "artifacts/deep.json", + ] + ) + + captured = capsys.readouterr() + assert exit_code == 2 + assert captured.out == "" + assert json.loads(captured.err)["error"]["code"] == "SS044" + + def test_cli_rollback_failure_is_machine_readable( project: Path, capsys: pytest.CaptureFixture[str], @@ -223,3 +246,105 @@ def test_plugin_loader_rejects_missing_entry_point() -> None: with pytest.raises(SplitSealError) as caught: load_similarity_plugin("not-installed") assert caught.value.code == "SS060" + + +def test_plugin_loader_wraps_discovery_failure(monkeypatch: pytest.MonkeyPatch) -> None: + def fail_discovery(*, group: str) -> object: + assert group == "splitseal.similarity" + raise RuntimeError("synthetic discovery failure") + + monkeypatch.setattr("splitseal.plugins.entry_points", fail_discovery) + with pytest.raises(SplitSealError) as caught: + load_similarity_plugin("synthetic") + assert caught.value.code == "SS060" + + +def test_plugin_loader_wraps_interface_property_failure(monkeypatch: pytest.MonkeyPatch) -> None: + class BrokenPlugin: + name = "synthetic" + + def analyze(self) -> None: + return None + + @property + def version(self) -> str: + raise RuntimeError("synthetic version failure") + + class EntryPoint: + name = "synthetic" + + def load(self) -> type[BrokenPlugin]: + return BrokenPlugin + + monkeypatch.setattr("splitseal.plugins.entry_points", lambda **_kwargs: [EntryPoint()]) + with pytest.raises(SplitSealError) as caught: + load_similarity_plugin("synthetic") + assert caught.value.code == "SS060" + + +def test_plugin_loader_rejects_invalid_declared_identity(monkeypatch: pytest.MonkeyPatch) -> None: + def analyze(*_args: object) -> list[object]: + return [] + + class EntryPoint: + name = "synthetic" + + def __init__(self, attributes: dict[str, object]) -> None: + self.attributes = attributes + + def load(self) -> object: + attributes = self.attributes + + class Plugin: + pass + + for key, value in attributes.items(): + setattr(Plugin, key, value) + return Plugin + + invalid_interfaces = [ + {"version": "1.0.0", "analyze": analyze}, + {"name": 1, "version": "1.0.0", "analyze": analyze}, + {"name": "", "version": "1.0.0", "analyze": analyze}, + {"name": "other", "version": "1.0.0", "analyze": analyze}, + {"name": "synthetic", "analyze": analyze}, + {"name": "synthetic", "version": 1, "analyze": analyze}, + {"name": "synthetic", "version": "", "analyze": analyze}, + ] + for attributes in invalid_interfaces: + monkeypatch.setattr( + "splitseal.plugins.entry_points", + lambda attributes=attributes, **_kwargs: [EntryPoint(attributes)], + ) + with pytest.raises(SplitSealError) as caught: + load_similarity_plugin("synthetic") + assert caught.value.code == "SS060" + + +@pytest.mark.parametrize("attribute", ["name", "version"]) +def test_plugin_loader_wraps_identity_property_failure( + monkeypatch: pytest.MonkeyPatch, + attribute: str, +) -> None: + class BrokenPlugin: + name = "synthetic" + version = "1.0.0" + + def analyze(self) -> None: + return None + + def __getattribute__(self, key: str) -> object: + if key == attribute: + raise RuntimeError(f"synthetic {key} failure") + return super().__getattribute__(key) + + class EntryPoint: + name = "synthetic" + + def load(self) -> type[BrokenPlugin]: + return BrokenPlugin + + monkeypatch.setattr("splitseal.plugins.entry_points", lambda **_kwargs: [EntryPoint()]) + with pytest.raises(SplitSealError) as caught: + load_similarity_plugin("synthetic") + assert caught.value.code == "SS060" diff --git a/tests/test_crypto.py b/tests/test_crypto.py index 9f8e862..83805fa 100644 --- a/tests/test_crypto.py +++ b/tests/test_crypto.py @@ -43,8 +43,13 @@ def test_wrong_key_and_ciphertext_tampering_fail_authentication() -> None: "mutator", [ lambda value: value.update(schema_version="wrong"), + lambda value: value.update(unknown=True), lambda value: value.pop("kdf"), + lambda value: value["kdf"].update(unknown=True), + lambda value: value["kdf"].pop("salt"), lambda value: value["kdf"].update(n=1), + lambda value: value["cipher"].update(unknown=True), + lambda value: value["cipher"].pop("nonce"), lambda value: value["cipher"].update(name="unknown"), lambda value: value["kdf"].update(salt="!"), lambda value: value["cipher"].update(nonce="AA"), @@ -58,6 +63,49 @@ def test_malformed_seal_parameters_are_rejected(mutator: object) -> None: assert caught.value.code == "SS040" +@pytest.mark.parametrize( + ("field", "value"), + [ + ("n", float(2**15)), + ("r", 8.0), + ("p", True), + ("p", 1.0), + ], +) +def test_seal_rejects_non_integer_kdf_parameters(field: str, value: object) -> None: + seal = json.loads(seal_manifest({"value": "synthetic"}, SECRET)) + seal["kdf"][field] = value + + with pytest.raises(SplitSealError) as caught: + open_seal(seal, SECRET) + + assert caught.value.code == "SS040" + + +@pytest.mark.parametrize( + ("section", "field"), + [ + ("kdf", "salt"), + ("cipher", "nonce"), + ("cipher", "ciphertext"), + ], +) +def test_seal_rejects_padded_base64url(section: str, field: str) -> None: + seal = json.loads(seal_manifest({"value": "synthetic"}, SECRET)) + seal[section][field] += "=" + with pytest.raises(SplitSealError) as caught: + open_seal(seal, SECRET) + assert caught.value.code == "SS040" + + +def test_seal_rejects_standard_base64_alphabet() -> None: + seal = json.loads(seal_manifest({"value": "synthetic"}, SECRET)) + seal["kdf"]["salt"] = "/////////////////////w" + with pytest.raises(SplitSealError) as caught: + open_seal(seal, SECRET) + assert caught.value.code == "SS040" + + def test_noncanonical_plaintext_is_rejected_after_authentication( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -75,8 +123,36 @@ def decrypt(self, _nonce: bytes, _ciphertext: bytes, _aad: bytes) -> bytes: open_seal(seal, SECRET) +@pytest.mark.parametrize("depth", [101, 2_000]) +def test_nested_decrypted_manifest_is_rejected_with_stable_error( + monkeypatch: pytest.MonkeyPatch, + depth: int, +) -> None: + plaintext = b'{"value":' + b"[" * depth + b"0" + b"]" * depth + b"}" + + class FakeAES: + def __init__(self, _key: bytes) -> None: + pass + + def decrypt(self, _nonce: bytes, _ciphertext: bytes, _aad: bytes) -> bytes: + return plaintext + + seal = json.loads(seal_manifest({"value": "synthetic"}, SECRET)) + monkeypatch.setattr("splitseal.crypto.AESGCM", FakeAES) + with pytest.raises(SplitSealError) as caught: + open_seal(seal, SECRET) + assert caught.value.code == "SS040" + + def test_key_material_has_a_minimum_length() -> None: with pytest.raises(SplitSealError) as caught: validate_secret(b"short") assert caught.value.code == "SS041" assert len(generate_secret()) == 32 + + +@pytest.mark.parametrize("secret", [None, "x" * 32, bytearray(32), 32]) +def test_key_material_rejects_wrong_runtime_types(secret: object) -> None: + with pytest.raises(SplitSealError) as caught: + validate_secret(secret) # type: ignore[arg-type] + assert caught.value.code == "SS041" diff --git a/tests/test_models_loaders.py b/tests/test_models_loaders.py index c980f18..95d93da 100644 --- a/tests/test_models_loaders.py +++ b/tests/test_models_loaders.py @@ -5,8 +5,9 @@ import pytest +from splitseal.canonical import record_digest from splitseal.errors import SplitSealError -from splitseal.loaders import load_records +from splitseal.loaders import iter_records, load_records from splitseal.models import load_config, parse_config_bytes @@ -16,6 +17,17 @@ def test_load_config_accepts_minimal_valid_document(project: Path) -> None: assert [split.name for split in config.splits] == ["development", "private-evaluation"] +def test_load_config_read_failure_does_not_expose_absolute_path(tmp_path: Path) -> None: + path = tmp_path / "sensitive-project-name" / "splitseal.toml" + + with pytest.raises(SplitSealError) as caught: + load_config(path) + + assert caught.value.code == "SS001" + assert caught.value.details == {"path": "splitseal.toml"} + assert str(tmp_path) not in str(caught.value.to_dict()) + + @pytest.mark.parametrize( ("content", "message"), [ @@ -89,12 +101,33 @@ def test_jsonl_loader_rejects_empty_malformed_and_non_finite(tmp_path: Path) -> load_records(path, "jsonl") +def test_jsonl_loader_rejects_excessive_nesting_with_stable_error(tmp_path: Path) -> None: + path = tmp_path / "input.jsonl" + path.write_text('{"value":' + "[" * 101 + "0" + "]" * 101 + "}\n", encoding="utf-8") + + with pytest.raises(SplitSealError) as caught: + load_records(path, "jsonl") + + assert caught.value.code == "SS011" + + def test_csv_loader_maps_all_values_to_strings(tmp_path: Path) -> None: path = tmp_path / "input.csv" path.write_text("id,value\none,42\n", encoding="utf-8") assert load_records(path, "csv") == [{"id": "one", "value": "42"}] +def test_csv_loader_preserves_normalized_multiline_crlf_compatibility(tmp_path: Path) -> None: + path = tmp_path / "input.csv" + path.write_bytes(b'id,text\r\n1,"first\r\nline"\r\n') + expected = {"id": "1", "text": "first\nline"} + assert load_records(path, "csv") == [expected] + assert list(iter_records(path, "csv")) == [expected] + assert record_digest(load_records(path, "csv")[0]) == ( + "411170b180d64efcff74f35d2e94fd600af8502e35c71636cfa28208a8b8379a" + ) + + @pytest.mark.parametrize( "content", [ diff --git a/tests/test_parquet.py b/tests/test_parquet.py index 3805944..bbb137e 100644 --- a/tests/test_parquet.py +++ b/tests/test_parquet.py @@ -6,6 +6,7 @@ import pytest +from splitseal.errors import SplitSealError from splitseal.loaders import load_records pytestmark = pytest.mark.parquet @@ -19,3 +20,15 @@ def test_parquet_rows_are_loaded_as_structured_records(tmp_path: Path) -> None: path = tmp_path / "input.parquet" pq.write_table(pa.table({"id": ["synthetic-1"], "score": [3]}), path) assert load_records(path, "parquet") == [{"id": "synthetic-1", "score": 3}] + + +@pytest.mark.skipif(importlib.util.find_spec("pyarrow") is None, reason="PyArrow is optional") +def test_parquet_preserves_domain_error_for_non_json_values(tmp_path: Path) -> None: + pa = importlib.import_module("pyarrow") + pq = importlib.import_module("pyarrow.parquet") + + path = tmp_path / "binary.parquet" + pq.write_table(pa.table({"id": ["synthetic-1"], "payload": [b"not-json"]}), path) + with pytest.raises(SplitSealError) as caught: + load_records(path, "parquet") + assert caught.value.code == "SS011" diff --git a/tests/test_release_asset_contract.py b/tests/test_release_asset_contract.py new file mode 100644 index 0000000..4aa8696 --- /dev/null +++ b/tests/test_release_asset_contract.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import hashlib +import runpy +from collections.abc import Callable +from pathlib import Path +from typing import cast + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "validate_release_assets.py" +VALIDATE = cast( + Callable[[Path, object], list[str]], + runpy.run_path(str(SCRIPT_PATH))["validate_release_assets"], +) + + +def _asset(name: str, content: bytes) -> dict[str, object]: + return { + "name": name, + "digest": f"sha256:{hashlib.sha256(content).hexdigest()}", + "state": "uploaded", + } + + +def test_release_asset_inventory_requires_exact_names_and_digests(tmp_path: Path) -> None: + wheel = b"wheel" + source = b"source" + (tmp_path / "package.whl").write_bytes(wheel) + (tmp_path / "package.tar.gz").write_bytes(source) + assert ( + VALIDATE( + tmp_path, + [_asset("package.whl", wheel), _asset("package.tar.gz", source)], + ) + == [] + ) + + +def test_release_asset_inventory_rejects_extra_missing_or_changed_assets(tmp_path: Path) -> None: + wheel = b"wheel" + (tmp_path / "package.whl").write_bytes(wheel) + assert VALIDATE(tmp_path, [_asset("package.whl", wheel), _asset("extra.zip", b"extra")]) + assert VALIDATE(tmp_path, []) + assert VALIDATE(tmp_path, [_asset("package.whl", b"changed")]) + + +def test_release_asset_inventory_rejects_incomplete_or_ambiguous_entries(tmp_path: Path) -> None: + wheel = b"wheel" + (tmp_path / "package.whl").write_bytes(wheel) + valid = _asset("package.whl", wheel) + incomplete = {**valid, "state": "starter"} + missing_digest = {"name": "package.whl", "state": "uploaded"} + assert VALIDATE(tmp_path, [incomplete]) + assert VALIDATE(tmp_path, [missing_digest]) + assert VALIDATE(tmp_path, [valid, valid]) diff --git a/tests/test_release_assets.py b/tests/test_release_assets.py new file mode 100644 index 0000000..4ba5a03 --- /dev/null +++ b/tests/test_release_assets.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from scripts.release_assets import ( + build_release_assets, + expected_artifact_names, + project_identity, + validate_release_tag, + write_checksum_manifest, +) + + +def test_repository_release_identity_matches_artifact_names() -> None: + root = Path(__file__).resolve().parents[1] + name, version = project_identity(root) + validate_release_tag(f"v{version}", version) + assert expected_artifact_names(name, version) == { + f"splitseal-{version}-py3-none-any.whl", + f"splitseal-{version}.tar.gz", + } + + +def test_release_tag_must_match_exact_package_version() -> None: + for tag in ("0.2.3", "v0.2.2", "v0.2.3-rc1", "refs/tags/v0.2.3"): + with pytest.raises(ValueError, match="does not match package version"): + validate_release_tag(tag, "0.2.3") + + +def test_checksum_manifest_is_complete_sorted_and_reproducible(tmp_path: Path) -> None: + contents = { + "splitseal-1.2.3-py3-none-any.whl": b"wheel bytes", + "splitseal-1.2.3.tar.gz": b"source bytes", + } + for name, content in contents.items(): + (tmp_path / name).write_bytes(content) + checksum_path = write_checksum_manifest(tmp_path, contents) + expected = "".join( + f"{hashlib.sha256(contents[name]).hexdigest()} {name}\n" for name in sorted(contents) + ) + assert checksum_path.read_text(encoding="ascii") == expected + with pytest.raises(ValueError, match=r"unexpected=.*SHA256SUMS"): + write_checksum_manifest(tmp_path, contents) + + +@pytest.mark.parametrize("extra_name", [None, "unexpected.zip"]) +def test_checksum_manifest_rejects_incomplete_or_unexpected_sets( + tmp_path: Path, + extra_name: str | None, +) -> None: + expected = expected_artifact_names("splitseal", "1.2.3") + wheel = "splitseal-1.2.3-py3-none-any.whl" + (tmp_path / wheel).write_bytes(b"wheel") + if extra_name is not None: + (tmp_path / extra_name).write_bytes(b"unexpected") + with pytest.raises(ValueError, match="release artifact set is invalid"): + write_checksum_manifest(tmp_path, expected) + + +def test_release_build_refuses_nonempty_output_before_invoking_builder(tmp_path: Path) -> None: + root = Path(__file__).resolve().parents[1] + _name, version = project_identity(root) + output = tmp_path / "dist" + output.mkdir() + sentinel = output / "do-not-overwrite" + sentinel.write_bytes(b"existing") + with pytest.raises(ValueError, match="output directory must be empty"): + build_release_assets(root=root, tag=f"v{version}", output_dir=output) + assert sentinel.read_bytes() == b"existing" diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py new file mode 100644 index 0000000..7392ef2 --- /dev/null +++ b/tests/test_release_metadata.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import runpy +from collections.abc import Callable +from pathlib import Path +from typing import cast + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "validate_release_metadata.py" +SCRIPT = runpy.run_path(str(SCRIPT_PATH)) +VALIDATE = cast( + Callable[[object, object, object], list[str]], + SCRIPT["validate_release_metadata"], +) +SANITIZE = cast(Callable[[object], object], SCRIPT["sanitize_generated_notes"]) + + +def test_release_metadata_accepts_generic_generated_notes() -> None: + assert ( + VALIDATE( + "v0.3.0", + "SplitSeal v0.3.0", + "## Changes\n\n* Harden release publication ([#35](https://github.com/example/project/pull/35))", + ) + == [] + ) + + +def test_release_metadata_rejects_private_or_attributed_content() -> None: + invalid_bodies = [ + "Internal " + "workflow handoff", + "Independent " + "review handoff", + "See codex/private-release-branch", + "Assisted-by: release tool", + "Co-authored-by: Contributor ", + "Generated-by: release tool", + "Reviewed-by: release reviewer", + "Built for a " + "found" + "er audience", + "Presented to " + "Y" + "C", + "Apply " + "humani" + "zation wording", + "Humani" + "ze the release notes", + "Humani" + "zed release notes", + "Humani" + "zing release notes", + "Handled by " + "Mission" + " Control", + "Opened from /" + "Users/example/project", + "Opened from C:\\" + "Users\\example\\project", + "Opened from /home/example/project", + "Opened from /private/var/example", + "Opened from file://local/project", + "See " + "startup" + "-idea notes", + "See tovellan-" + "codex worktree", + "See tovellan-" + "infra checkout", + "See tovellan-" + "handbook worktree", + "Contains personal account @example-user", + "Contains a prohibited \u2014 character", + "Contact person@example.com", + ] + for body in invalid_bodies: + assert VALIDATE("v0.3.0", "SplitSeal v0.3.0", body) + + +def test_release_metadata_requires_exact_tag_name_and_nonempty_notes() -> None: + assert VALIDATE("0.3.0", "SplitSeal 0.3.0", "notes") == ["invalid release tag"] + assert VALIDATE("v0.3.0", "v0.3.0", "notes") == ["invalid release name"] + assert VALIDATE("v0.3.0", "SplitSeal v0.3.0", "") == ["missing release notes"] + + +def test_generated_notes_remove_contributor_credits_and_sections() -> None: + generated = """## Changes + +* Harden release publication by @example-user in https://github.com/example/project/pull/35 + +## New Contributors +* @example-user made their first contribution in https://github.com/example/project/pull/35 + +**Full Changelog**: https://github.com/example/project/compare/v0.2.3...v0.3.0 +""" + sanitized = SANITIZE(generated) + assert isinstance(sanitized, str) + assert "@" not in sanitized + assert "New Contributors" not in sanitized + assert "[#35](https://github.com/example/project/pull/35)" in sanitized + assert "Full Changelog" in sanitized + assert VALIDATE("v0.3.0", "SplitSeal v0.3.0", sanitized) == [] diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py new file mode 100644 index 0000000..c2999c7 --- /dev/null +++ b/tests/test_release_workflow.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import yaml + +WORKFLOW_PATH = Path(__file__).resolve().parents[1] / ".github" / "workflows" / "release-assets.yml" + + +def _release_job() -> Mapping[str, Any]: + document = yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) + assert isinstance(document, Mapping) + jobs = document.get("jobs") + assert isinstance(jobs, Mapping) + build = jobs.get("build") + assert isinstance(build, Mapping) + return build + + +def _release_steps() -> list[Mapping[str, Any]]: + steps = _release_job().get("steps") + assert isinstance(steps, list) + assert all(isinstance(step, Mapping) for step in steps) + return steps + + +def _step(name: str) -> Mapping[str, Any]: + return next(step for step in _release_steps() if step.get("name") == name) + + +def _run(name: str) -> str: + command = _step(name).get("run") + assert isinstance(command, str) + return command + + +def _step_index(name: str) -> int: + return next(index for index, step in enumerate(_release_steps()) if step.get("name") == name) + + +def test_release_preflight_precedes_verified_checkout() -> None: + preflight = _run("Verify release tag targets protected main") + assert "repos/$GITHUB_REPOSITORY/immutable-releases" not in WORKFLOW_PATH.read_text( + encoding="utf-8" + ) + assert "/git/ref/tags/$RELEASE_TAG" in preflight + assert "/git/tags/$object_sha" in preflight + assert ".tagger.name" in preflight + assert ".tagger.email" in preflight + assert ".message" in preflight + assert '"SplitSeal $RELEASE_TAG"' in preflight + assert "Tovellan Maintainers" in preflight + assert "noreply@github.com" in preflight + assert "/git/ref/heads/main" in preflight + assert '"$GITHUB_REF" != "refs/tags/$RELEASE_TAG"' in preflight + assert '"$GITHUB_SHA" != "$object_sha"' in preflight + assert "/compare/$object_sha...$main_sha" in preflight + assert '"$release_state" = "absent"' in preflight + assert '"$comparison_status" != "ahead"' in preflight + assert "target_sha=" in preflight + assert "/releases?per_page=100" in preflight + assert 'release_state="draft"' in preflight + assert _step_index("Verify release tag targets protected main") < _step_index( + "Check out release tag" + ) + + checkout = _step("Check out release tag") + assert "if" not in checkout + settings = checkout.get("with") + assert isinstance(settings, Mapping) + assert settings.get("persist-credentials") is False + assert settings.get("ref") == "${{ steps.verify-tag.outputs.target_sha }}" + + uv_settings = _step("Install uv").get("with") + assert isinstance(uv_settings, Mapping) + assert uv_settings.get("version") == "0.12.5" + + +def test_release_publication_is_draft_first_and_resumable() -> None: + create = _step("Create or resume draft release") + attach = _step("Attach exact draft assets") + publish = _step("Publish complete draft release") + for step in (create, attach, publish): + assert step.get("if") == "steps.verify-tag.outputs.release_state != 'published'" + + create_run = _run("Create or resume draft release") + assert "--method POST" in create_run + assert '"repos/$GITHUB_REPOSITORY/releases"' in create_run + assert "-F draft=true" in create_run + assert '-f name="SplitSeal $RELEASE_TAG"' in create_run + assert '-f body="$release_notes"' in create_run + assert '"repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID"' in create_run + assert "existing-draft.json" in create_run + assert "scripts/validate_release_metadata.py" in create_run + + metadata_run = _run("Generate and validate public release notes") + assert "/releases/generate-notes" in metadata_run + assert "generated-release-notes.json" in metadata_run + assert "scripts/validate_release_metadata.py" in metadata_run + assert '--output "$RUNNER_TEMP/release-notes.json"' in metadata_run + assert "--sanitize-generated" in metadata_run + + build_run = _run("Build tag-matched distributions and checksums") + assert "scripts/release_assets.py" in build_run + assert '--tag "$RELEASE_TAG"' in build_run + assert "--output-dir dist" in build_run + + attach_run = _run("Attach exact draft assets") + assert "/assets?per_page=100" in attach_run + assert "'.state'" in attach_run + assert "--method DELETE" in attach_run + assert "gh release upload" in attach_run + assert "cmp -s" in attach_run + assert "--clobber" not in attach_run + assert "scripts/validate_release_assets.py" in attach_run + assert "draft-assets.json" in attach_run + + publish_run = _run("Publish complete draft release") + assert "--method PATCH" in publish_run + assert "-F draft=false" in publish_run + assert "gh release create" not in WORKFLOW_PATH.read_text(encoding="utf-8") + + published_run = _run("Verify exact published assets") + assert "scripts/validate_release_assets.py" in published_run + assert "published-assets.json" in published_run + assert "cmp -s" in published_run + + assert _step_index("Build tag-matched distributions and checksums") < _step_index( + "Generate and validate public release notes" + ) + assert _step_index("Generate and validate public release notes") < _step_index( + "Create or resume draft release" + ) + assert _step_index("Create or resume draft release") < _step_index("Attach exact draft assets") + assert _step_index("Attach exact draft assets") < _step_index("Publish complete draft release") + assert _step_index("Publish complete draft release") < _step_index( + "Verify immutable release and automatic attestation" + ) + + +def test_release_closure_is_retryable_after_publication() -> None: + verify = _run("Verify immutable release and automatic attestation") + assert "for attempt in {1..40}" in verify + assert "'.immutable'" in verify + assert '"$state" = "true"' in verify + assert "2>/dev/null" in verify + assert 'gh release verify "$RELEASE_TAG" --format json' in verify + assert "sleep 15" in verify + + +def test_distribution_provenance_precedes_irreversible_publication() -> None: + assert _release_job().get("permissions") == { + "artifact-metadata": "write", + "attestations": "write", + "contents": "write", + "id-token": "write", + } + + attest = _step("Attest wheel and source archive provenance") + assert attest.get("if") == "steps.verify-tag.outputs.release_state != 'published'" + assert attest.get("uses") == ("actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d") + settings = attest.get("with") + assert isinstance(settings, Mapping) + assert settings.get("subject-checksums") == "dist/SHA256SUMS" + + verify_step = _step("Verify distribution provenance") + assert "if" not in verify_step + verify = _run("Verify distribution provenance") + assert "dist/*.whl dist/*.tar.gz" in verify + assert "for attempt in {1..20}" in verify + assert '--repo "$GITHUB_REPOSITORY"' in verify + assert '--signer-workflow "$signer_workflow"' in verify + assert '--signer-digest "$GITHUB_SHA"' in verify + assert '--source-ref "$GITHUB_REF"' in verify + assert '--source-digest "$GITHUB_SHA"' in verify + + assert _step_index("Build tag-matched distributions and checksums") < _step_index( + "Attest wheel and source archive provenance" + ) + assert _step_index("Attest wheel and source archive provenance") < _step_index( + "Attach exact draft assets" + ) + assert _step_index("Attach exact draft assets") < _step_index("Verify distribution provenance") + assert _step_index("Verify distribution provenance") < _step_index( + "Publish complete draft release" + ) diff --git a/tests/test_repository_audit.py b/tests/test_repository_audit.py new file mode 100644 index 0000000..b35e7ee --- /dev/null +++ b/tests/test_repository_audit.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.repository_audit import action_reference_violations, contains_action_references + +_FULL_SHA = "0123456789abcdef0123456789abcdef01234567" + + +@pytest.mark.parametrize( + "reference", + [ + f"actions/checkout@{_FULL_SHA}", + f"github/codeql-action/analyze@{_FULL_SHA}", + "./.github/actions/local-check", + ], +) +def test_workflow_audit_accepts_commit_pins_and_local_actions(reference: str) -> None: + workflow = f"steps:\n - uses: {reference}\n" + assert action_reference_violations(Path(".github/workflows/test.yml"), workflow) == [] + + +@pytest.mark.parametrize( + "reference", + [ + "actions/checkout@v7", + "actions/checkout@main", + "actions/checkout@0123456", + "docker://python:3.14", + "${{ inputs.action }}", + ], +) +def test_workflow_audit_rejects_mutable_or_dynamic_external_actions(reference: str) -> None: + workflow = f"name: test\nsteps:\n - uses: {reference}\n" + assert action_reference_violations(Path(".github/workflows/test.yml"), workflow) == [ + ".github/workflows/test.yml:3: external action is not pinned to a full commit SHA" + ] + + +@pytest.mark.parametrize( + ("workflow", "line"), + [ + ("steps:\n - uses : actions/checkout@main\n", 2), + ('steps:\n - "uses": actions/checkout@main\n', 2), + ('steps: [{"uses": actions/checkout@main}]\n', 1), + ("steps:\n - {uses: actions/checkout@main}\n", 2), + ], +) +def test_workflow_audit_rejects_yaml_formatting_bypasses(workflow: str, line: int) -> None: + assert action_reference_violations(Path(".github/workflows/test.yaml"), workflow) == [ + f".github/workflows/test.yaml:{line}: external action is not pinned to a full commit SHA" + ] + + +def test_workflow_audit_rejects_non_scalar_and_invalid_yaml_references() -> None: + assert action_reference_violations( + Path("nested/action.yml"), + "runs:\n steps:\n - uses: [actions/checkout@main]\n", + ) == ["nested/action.yml:3: action reference must be a scalar string"] + assert action_reference_violations( + Path(".github/workflows/test.yml"), + "steps: [\n", + ) == [".github/workflows/test.yml:2: action definition is not valid YAML"] + + +@pytest.mark.parametrize( + "relative", + [ + Path("action.yml"), + Path("tools/private/action.yaml"), + Path("vendor/deep/local/action.yml"), + Path(".github/workflows/check.yaml"), + ], +) +def test_action_audit_discovers_workflows_and_composite_actions_anywhere(relative: Path) -> None: + assert contains_action_references(relative) + + +def test_action_audit_ignores_unrelated_yaml() -> None: + assert not contains_action_references(Path("docs/example.yaml")) diff --git a/tests/test_service.py b/tests/test_service.py index 09e1aa8..c7c29b7 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -10,7 +10,7 @@ import splitseal.service as service_module from splitseal.canonical import Record, canonicalize -from splitseal.errors import SplitSealError +from splitseal.errors import SplitSealError, fail from splitseal.plugins import SimilarityFinding from splitseal.service import ( diff_releases, @@ -53,6 +53,22 @@ def test_freeze_and_verify_current_sources(project: Path) -> None: assert mode == 0o600 +def test_freeze_rejects_non_byte_key_without_outputs(project: Path) -> None: + invalid_key = "not-byte-key-material" + with pytest.raises(SplitSealError) as caught: + freeze_release( + root=project, + config_path="splitseal.toml", + seal_path="artifacts/release.sseal", + attestation_path="artifacts/release.attestation.json", + secret=invalid_key, # type: ignore[arg-type] + ) + + assert caught.value.code == "SS041" + assert not (project / "artifacts" / "release.sseal").exists() + assert not (project / "artifacts" / "release.attestation.json").exists() + + def test_public_and_private_outer_outputs_do_not_contain_records_or_membership( project: Path, ) -> None: @@ -420,6 +436,8 @@ def test_similarity_plugin_exception_is_wrapped(project: Path) -> None: write_config(project, similarity='\n[[similarity]]\nplugin="broken"\n') class BrokenPlugin(PassingPlugin): + name = "broken" + def analyze( self, splits: Mapping[str, Sequence[Record]], @@ -439,6 +457,118 @@ def analyze( assert caught.value.code == "SS061" +def test_similarity_plugin_loader_and_evidence_exceptions_are_wrapped(project: Path) -> None: + write_config(project, similarity='\n[[similarity]]\nplugin="broken"\n') + + class BrokenVersionPlugin(PassingPlugin): + name = "broken" + + @property + def version(self) -> str: + raise RuntimeError("synthetic version failure") + + def broken_loader(_name: str) -> PassingPlugin: + raise RuntimeError("synthetic loader failure") + + for loader in (broken_loader, lambda _name: BrokenVersionPlugin()): + with pytest.raises(SplitSealError) as caught: + freeze_release( + root=project, + config_path="splitseal.toml", + seal_path="artifacts/fail.sseal", + attestation_path="artifacts/fail.json", + secret=SECRET, + plugin_loader=loader, + ) + assert caught.value.code == "SS061" + + +def test_plugin_splitseal_errors_are_normalized(project: Path) -> None: + write_config(project, similarity='\n[[similarity]]\nplugin="broken"\n') + + class BrokenVersionPlugin(PassingPlugin): + name = "broken" + + @property + def version(self) -> str: + raise fail("SS999", "synthetic version failure", private="detail") + + class BrokenGeneratorPlugin(PassingPlugin): + name = "broken" + + def analyze( + self, + splits: Mapping[str, Sequence[Record]], + settings: Mapping[str, Any], + ) -> Iterable[SimilarityFinding]: + yield from () + raise fail("SS999", "synthetic iteration failure", private="detail") + + def broken_loader(_name: str) -> PassingPlugin: + raise fail("SS999", "synthetic loader failure", private="detail") + + loaders = ( + broken_loader, + lambda _name: BrokenVersionPlugin(), + lambda _name: BrokenGeneratorPlugin(), + ) + for loader in loaders: + with pytest.raises(SplitSealError) as caught: + freeze_release( + root=project, + config_path="splitseal.toml", + seal_path="artifacts/fail.sseal", + attestation_path="artifacts/fail.json", + secret=SECRET, + plugin_loader=loader, + ) + assert caught.value.code == "SS061" + assert caught.value.details == {"plugin": "broken"} + + +def test_official_plugin_loader_preserves_ss060(project: Path) -> None: + write_config(project, similarity='\n[[similarity]]\nplugin="not-installed"\n') + + with pytest.raises(SplitSealError) as caught: + freeze_release( + root=project, + config_path="splitseal.toml", + seal_path="artifacts/fail.sseal", + attestation_path="artifacts/fail.json", + secret=SECRET, + ) + assert caught.value.code == "SS060" + + +def test_custom_similarity_loader_rejects_invalid_declared_identity(project: Path) -> None: + write_config(project, similarity='\n[[similarity]]\nplugin="synthetic-plugin"\n') + + def analyze(*_args: object) -> list[object]: + return [] + + invalid_interfaces = [ + {"version": "1.0.0", "analyze": analyze}, + {"name": 1, "version": "1.0.0", "analyze": analyze}, + {"name": "", "version": "1.0.0", "analyze": analyze}, + {"name": "other", "version": "1.0.0", "analyze": analyze}, + {"name": "synthetic-plugin", "analyze": analyze}, + {"name": "synthetic-plugin", "version": 1, "analyze": analyze}, + {"name": "synthetic-plugin", "version": "", "analyze": analyze}, + ] + for attributes in invalid_interfaces: + plugin = type("Plugin", (), attributes)() + with pytest.raises(SplitSealError) as caught: + freeze_release( + root=project, + config_path="splitseal.toml", + seal_path="artifacts/fail.sseal", + attestation_path="artifacts/fail.json", + secret=SECRET, + plugin_loader=lambda _name, value=plugin: value, + ) + assert caught.value.code == "SS061" + + def test_diff_reports_aggregate_changes_without_identifiers(project: Path) -> None: freeze(project, prefix="old", secret=SECRET) write_jsonl( diff --git a/tests/test_streaming.py b/tests/test_streaming.py new file mode 100644 index 0000000..9a66271 --- /dev/null +++ b/tests/test_streaming.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import json +import tempfile +import tracemalloc +from pathlib import Path + +from hypothesis import given, settings +from hypothesis import strategies as st + +import splitseal.service as service_module +from splitseal.canonical import canonicalize +from splitseal.loaders import iter_records, load_records +from splitseal.models import load_config +from splitseal.service import freeze_release, verify_release +from splitseal.streaming import build_streaming_manifest + +from .conftest import SECRET, write_config, write_jsonl + + +def test_streaming_manifest_and_attestation_match_in_memory(project: Path) -> None: + config = load_config(project / "splitseal.toml") + expected_manifest = service_module._build_manifest(config, project) + expected_attestation = service_module._public_attestation(expected_manifest, SECRET) + with build_streaming_manifest(config, project) as streamed: + assert streamed.path.read_bytes() == canonicalize(expected_manifest) + assert streamed.record_count == 4 + assert streamed.split_counts == (2, 2) + + freeze_release( + root=project, + config_path="splitseal.toml", + seal_path="artifacts/streamed.sseal", + attestation_path="artifacts/streamed.json", + secret=SECRET, + ) + assert (project / "artifacts" / "streamed.json").read_bytes() == ( + canonicalize(expected_attestation) + b"\n" + ) + assert ( + verify_release( + root=project, + seal_path="artifacts/streamed.sseal", + attestation_path="artifacts/streamed.json", + config_path="splitseal.toml", + secret=SECRET, + )["status"] + == "pass" + ) + + +@settings(max_examples=20, deadline=None) +@given( + development=st.lists(st.integers(min_value=0, max_value=10_000), min_size=1, max_size=12), + private=st.lists(st.integers(min_value=0, max_value=10_000), min_size=1, max_size=12), +) +def test_streaming_manifest_property_matches_in_memory( + development: list[int], + private: list[int], +) -> None: + with tempfile.TemporaryDirectory(prefix="splitseal-stream-property-") as directory: + root = Path(directory) + (root / "data").mkdir() + write_jsonl( + root / "data" / "development.jsonl", + [ + json.dumps({"id": f"development-{index}", "value": value}) + for index, value in enumerate(development) + ], + ) + write_jsonl( + root / "data" / "private.jsonl", + [ + json.dumps({"id": f"private-{index}", "value": value}) + for index, value in enumerate(private) + ], + ) + write_config(root) + config = load_config(root / "splitseal.toml") + expected = canonicalize(service_module._build_manifest(config, root)) + with build_streaming_manifest(config, root) as streamed: + assert streamed.path.read_bytes() == expected + + +def test_streaming_jsonl_preserves_unicode_splitlines(tmp_path: Path) -> None: + path = tmp_path / "records.jsonl" + path.write_text( + '{"id":1}\r\n{"id":2}\u2028{"id":3}\x85{"id":4}\r{"id":5}\n', + encoding="utf-8", + ) + assert list(iter_records(path, "jsonl")) == [ + {"id": 1}, + {"id": 2}, + {"id": 3}, + {"id": 4}, + {"id": 5}, + ] + assert list(iter_records(path, "jsonl")) == load_records(path, "jsonl") + + +def test_streaming_csv_matches_list_loader(tmp_path: Path) -> None: + path = tmp_path / "records.csv" + path.write_text('id,text\n1,"first\nline"\n2,second\n', encoding="utf-8") + assert list(iter_records(path, "csv")) == load_records(path, "csv") + assert next(iter_records(path, "csv"))["text"] == "first\nline" + + +def test_exact_only_freeze_has_bounded_python_peak(project: Path) -> None: + payload = "x" * 4096 + records = [ + json.dumps({"id": f"large-{index:05d}", "payload": payload}) for index in range(1000) + ] + write_jsonl(project / "data" / "development.jsonl", records) + write_jsonl( + project / "data" / "private.jsonl", + [json.dumps({"id": "private-only", "payload": payload})], + ) + input_bytes = (project / "data" / "development.jsonl").stat().st_size + tracemalloc.start() + try: + freeze_release( + root=project, + config_path="splitseal.toml", + seal_path="artifacts/bounded.sseal", + attestation_path="artifacts/bounded.json", + secret=SECRET, + ) + _current, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert peak < input_bytes // 2 diff --git a/tests/test_version_audit.py b/tests/test_version_audit.py new file mode 100644 index 0000000..8c065f0 --- /dev/null +++ b/tests/test_version_audit.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.version_audit import version_violations + + +def _write_fixture(root: Path) -> None: + files = { + "pyproject.toml": '[project]\nname = "splitseal"\nversion = "1.2.3"\n', + "uv.lock": ( + 'version = 1\nrevision = 1\n[[package]]\nname = "splitseal"\nversion = "1.2.3"\n' + ), + "src/splitseal/__init__.py": '__version__ = "1.2.3"\n', + "README.md": 'install "https://example.test/splitseal.git@v1.2.3"\n', + "CHANGELOG.md": "## [1.2.3] - 2026-08-24\n", + "SUPPORT.md": "Within the 1.2 release line, contracts are stable.\n", + "docs/api.md": "The stable 1.2 API is exported.\n", + "docs/manifest-format.md": "Fields may appear within a 1.2 release.\n", + "ROADMAP.md": "## Delivered in 1.2\n", + } + for relative, content in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def test_current_repository_versions_are_consistent() -> None: + root = Path(__file__).resolve().parents[1] + assert version_violations(root) == [] + + +@pytest.mark.parametrize( + ("relative", "old", "new", "label"), + [ + ("src/splitseal/__init__.py", "1.2.3", "1.2.2", "__version__"), + ("uv.lock", "1.2.3", "1.2.2", "uv.lock"), + ("README.md", "v1.2.3", "v1.2.2", "README.md"), + ("CHANGELOG.md", "[1.2.3]", "[1.2.2]", "CHANGELOG.md"), + ("SUPPORT.md", "1.2 release", "1.1 release", "SUPPORT.md"), + ("docs/api.md", "1.2 API", "1.1 API", "docs/api.md"), + ("docs/manifest-format.md", "1.2 release", "1.1 release", "docs/manifest-format.md"), + ("ROADMAP.md", "Delivered in 1.2", "Delivered in 1.1", "ROADMAP.md"), + ], +) +def test_version_audit_reports_each_drifted_surface( + tmp_path: Path, + relative: str, + old: str, + new: str, + label: str, +) -> None: + _write_fixture(tmp_path) + path = tmp_path / relative + path.write_text(path.read_text(encoding="utf-8").replace(old, new), encoding="utf-8") + violations = version_violations(tmp_path) + assert len(violations) == 1 + assert label in violations[0] diff --git a/tests/test_workflow_security.py b/tests/test_workflow_security.py new file mode 100644 index 0000000..dc58eb9 --- /dev/null +++ b/tests/test_workflow_security.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import yaml + + +def _workflow_paths(workflow_root: Path) -> list[Path]: + return sorted((*workflow_root.glob("*.yml"), *workflow_root.glob("*.yaml"))) + + +def _workflows() -> list[tuple[Path, Mapping[str, Any]]]: + workflow_root = Path(__file__).resolve().parents[1] / ".github" / "workflows" + workflows: list[tuple[Path, Mapping[str, Any]]] = [] + for path in _workflow_paths(workflow_root): + document = yaml.safe_load(path.read_text(encoding="utf-8")) + assert isinstance(document, Mapping), path + workflows.append((path, document)) + assert workflows + return workflows + + +def _mapping(value: object, path: Path, context: str) -> Mapping[str, Any]: + assert isinstance(value, Mapping), f"{path}: {context} must be a mapping" + return value + + +def test_every_checkout_drops_persisted_credentials() -> None: + checkout_steps = 0 + for path, workflow in _workflows(): + jobs = _mapping(workflow.get("jobs"), path, "jobs") + for job_name, job_value in jobs.items(): + job = _mapping(job_value, path, f"job {job_name}") + steps = job.get("steps") + assert isinstance(steps, list), f"{path}: job {job_name} must define steps" + for step_value in steps: + step = _mapping(step_value, path, f"step in job {job_name}") + reference = step.get("uses") + if not isinstance(reference, str) or not reference.startswith("actions/checkout@"): + continue + checkout_steps += 1 + settings = _mapping(step.get("with"), path, "checkout with") + assert settings.get("persist-credentials") is False, path + assert checkout_steps > 0 + + +def test_every_workflow_has_concurrency_and_every_job_has_a_timeout() -> None: + for path, workflow in _workflows(): + concurrency = _mapping(workflow.get("concurrency"), path, "concurrency") + assert isinstance(concurrency.get("group"), str), path + assert type(concurrency.get("cancel-in-progress")) is bool, path + jobs = _mapping(workflow.get("jobs"), path, "jobs") + assert jobs, path + for job_name, job_value in jobs.items(): + job = _mapping(job_value, path, f"job {job_name}") + timeout = job.get("timeout-minutes") + assert type(timeout) is int and timeout > 0, f"{path}: job {job_name} needs a timeout" + + +def test_release_jobs_are_not_cancelled_after_publication() -> None: + workflows = dict(_workflows()) + release_path = next(path for path in workflows if path.stem == "release-assets") + concurrency = _mapping(workflows[release_path].get("concurrency"), release_path, "concurrency") + assert concurrency.get("cancel-in-progress") is False + group = concurrency.get("group") + assert isinstance(group, str) + assert "${{ inputs.release_tag }}" in group + + +def test_validation_workflows_use_per_ref_cancellation() -> None: + for path, workflow in _workflows(): + if path.stem == "release-assets": + continue + concurrency = _mapping(workflow.get("concurrency"), path, "concurrency") + assert concurrency.get("cancel-in-progress") is True, path + group = concurrency.get("group") + assert isinstance(group, str), path + assert "${{ github.workflow }}" in group, path + assert "${{ github.ref }}" in group, path + + +def test_workflow_discovery_includes_yml_and_yaml(tmp_path: Path) -> None: + (tmp_path / "one.yml").write_text("name: one\n", encoding="utf-8") + (tmp_path / "two.yaml").write_text("name: two\n", encoding="utf-8") + (tmp_path / "ignored.txt").write_text("name: ignored\n", encoding="utf-8") + assert [path.name for path in _workflow_paths(tmp_path)] == ["one.yml", "two.yaml"] diff --git a/uv.lock b/uv.lock index 4035652..6abd0f5 100644 --- a/uv.lock +++ b/uv.lock @@ -1187,6 +1187,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1260,7 +1295,7 @@ wheels = [ [[package]] name = "splitseal" -version = "0.3.0" +version = "0.4.0" source = { editable = "." } dependencies = [ { name = "cryptography" }, @@ -1275,6 +1310,7 @@ dev = [ { name = "pip-audit" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "pyyaml" }, { name = "ruff" }, ] parquet = [ @@ -1291,6 +1327,7 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'parquet'", specifier = "==25.0.1" }, { name = "pytest", marker = "extra == 'dev'", specifier = "==9.1.1" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = "==7.1.0" }, + { name = "pyyaml", marker = "extra == 'dev'", specifier = "==6.0.2" }, { name = "rfc8785", specifier = "==0.1.4" }, { name = "ruff", marker = "extra == 'dev'", specifier = "==0.16.4" }, ]