From a0d6ef0938b976c316c7f56cfd9ae1981abc158d Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:14:58 +0200 Subject: [PATCH 1/3] 0.6.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 67aac70..1ed7361 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "scrollcase", - "version": "0.6.0", + "version": "0.6.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "scrollcase", - "version": "0.6.0", + "version": "0.6.1", "license": "Apache-2.0", "dependencies": { "tar": "7.5.22", diff --git a/package.json b/package.json index 322a121..057936e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scrollcase", - "version": "0.6.0", + "version": "0.6.1", "description": "Build signed, verifiable, installable Python environment boxes for scientific models.", "license": "Apache-2.0", "author": "Lorenzo S. (https://github.com/suffro)", From dcf781f334faefeb4d431fb9fa8314c9a83c9c8e Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:16:49 +0200 Subject: [PATCH 2/3] Keep the release notes and the guide telling the same story MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The demo box page moved to a box/ and keys/ layout and grew per-platform commands, but the notes attached to the GitHub release still taught the flat one — and still carried the glob that PowerShell does not expand, on the page a Windows reader lands on first. They now match, and point at the guide for the consumer libraries. "What just happened" explains the CLI step, so it belongs before the section on driving a box from an application rather than after it. --- .github/demo-box-release-notes.md | 25 +- .../src/build/archive.mjs | 356 ++++++++++++ .../src/build/assets.mjs | 123 +++++ .../src/build/audit.mjs | 69 +++ .../src/build/authoring.mjs | 403 ++++++++++++++ .../src/build/box.mjs | 339 ++++++++++++ .../src/build/consumer-setup.mjs | 110 ++++ .../src/build/execution.mjs | 60 ++ .../src/build/filesystem.mjs | 198 +++++++ .../src/build/identity.mjs | 39 ++ .../src/build/index.mjs | 38 ++ .../src/build/launchers.mjs | 60 ++ .../src/build/licenses.mjs | 138 +++++ .../src/build/parity.mjs | 111 ++++ .../src/build/pixi.mjs | 437 +++++++++++++++ .../src/build/process.mjs | 66 +++ .../src/build/project.mjs | 276 ++++++++++ .../src/build/schema-validation.mjs | 194 +++++++ .../src/build/scroll.mjs | 200 +++++++ .../src/build/toolchain.mjs | 246 +++++++++ .../src/build/verify.mjs | 204 +++++++ .../src/build/workspace.mjs | 245 +++++++++ .../src/cli-args.mjs | 31 ++ .../src/cli-authoring.mjs | 187 +++++++ .../src/cli-init.mjs | 46 ++ .../src/cli-menu.mjs | 102 ++++ .../src/cli-output.mjs | 34 ++ .../src/cli-run.mjs | 56 ++ .../src/cli-signing.mjs | 33 ++ .../src/cli-targets.mjs | 192 +++++++ .scrollcase-runtime-types-RzUfxr/src/cli.mjs | 501 +++++++++++++++++ .../src/consumer/index.mjs | 17 + .../src/consumer/run-box.mjs | 46 ++ .../src/consumer/run-extracted.mjs | 161 ++++++ .../src/consumer/verify-and-extract.mjs | 175 ++++++ .../src/contract/browser.mjs | 28 + .../src/contract/document-shape.mjs | 92 ++++ .../src/contract/documents.mjs | 48 ++ .../fixtures/consumer-conformance.json | 516 ++++++++++++++++++ .../examples/box-manifest.example.json | 32 ++ .../examples/channel-manifest.example.json | 20 + .../examples/release-manifest.example.json | 45 ++ .../examples/scroll-pixi.example.json | 72 +++ .../fixtures/examples/scroll.example.json | 44 ++ .../examples/signed-release.example.json | 13 + .../examples/signed-release.public-key.json | 5 + .../contract/fixtures/target-id-contract.json | 64 +++ .../src/contract/index.mjs | 55 ++ .../src/contract/links.mjs | 162 ++++++ .../contract/schema/box-manifest.schema.json | 124 +++++ .../schema/channel-manifest.schema.json | 80 +++ .../src/contract/schema/execution.schema.json | 94 ++++ .../schema/release-manifest.schema.json | 272 +++++++++ .../schema/revocations-manifest.schema.json | 64 +++ .../src/contract/schema/scroll.schema.json | 339 ++++++++++++ .../schema/signed-document.schema.json | 43 ++ .../src/contract/schema/target.schema.json | 57 ++ .../src/contract/targets.mjs | 254 +++++++++ .../src/contract/types/index.d.ts | 467 ++++++++++++++++ .../src/sign/index.mjs | 132 +++++ .../src/sign/keys.mjs | 160 ++++++ docs/guides/demo-box.md | 24 +- 62 files changed, 8805 insertions(+), 19 deletions(-) create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/archive.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/assets.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/audit.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/authoring.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/box.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/consumer-setup.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/execution.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/filesystem.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/identity.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/index.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/launchers.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/licenses.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/parity.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/pixi.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/process.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/project.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/schema-validation.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/scroll.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/toolchain.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/verify.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/build/workspace.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-args.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-authoring.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-init.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-menu.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-output.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-run.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-signing.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/cli-targets.mjs create mode 100755 .scrollcase-runtime-types-RzUfxr/src/cli.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/consumer/index.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/consumer/run-box.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/consumer/run-extracted.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/consumer/verify-and-extract.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/browser.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/document-shape.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/documents.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/consumer-conformance.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/box-manifest.example.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/channel-manifest.example.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/release-manifest.example.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll-pixi.example.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll.example.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.example.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.public-key.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/fixtures/target-id-contract.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/index.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/links.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/box-manifest.schema.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/channel-manifest.schema.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/execution.schema.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/release-manifest.schema.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/revocations-manifest.schema.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/scroll.schema.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/signed-document.schema.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/schema/target.schema.json create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/targets.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/contract/types/index.d.ts create mode 100644 .scrollcase-runtime-types-RzUfxr/src/sign/index.mjs create mode 100644 .scrollcase-runtime-types-RzUfxr/src/sign/keys.mjs diff --git a/.github/demo-box-release-notes.md b/.github/demo-box-release-notes.md index 3bdc2f4..8eee68a 100644 --- a/.github/demo-box-release-notes.md +++ b/.github/demo-box-release-notes.md @@ -13,23 +13,34 @@ Download the one archive matching your machine: | macOS, Apple silicon | `hello-box-1.0.0-macos-aarch64-metal.zip` | | Windows, Intel or AMD | `hello-box-1.0.0-windows-x86_64-cpu.zip` | -Unpack it and you get two files: the box and its signed release document. Run the commands from the -directory holding them, with -[`examples/keys/example-signing-public.json`](../blob/main/examples/keys/example-signing-public.json) -from the repository: +Unpack it and you get two files: the box and its signed release document. Keep them in a directory of +their own, and take the trust key from the repository rather than from this page — a signature only +proves where something came from if the key does not travel with it: ```sh npm install -g scrollcase -unzip hello-box-1.0.0-.zip -d hello-box && cd hello-box -scrollcase verify *.release.json --public-key example-signing-public.json -scrollcase run *.release.json --public-key example-signing-public.json +mkdir scrollcase-demo && cd scrollcase-demo +unzip ../hello-box-1.0.0-.zip -d box +mkdir keys +curl -o keys/example-signing-public.json \ + https://raw.githubusercontent.com/suffro/scrollcase/main/examples/keys/example-signing-public.json + +scrollcase verify box/*.release.json --public-key keys/example-signing-public.json +scrollcase run box/*.release.json --public-key keys/example-signing-public.json ``` +On PowerShell that glob is not expanded for a command like this — use +`(Get-ChildItem box\*.release.json).FullName`, or simply type the file name you see after unpacking. + `verify` checks the signature, the archive's size and hash, the entry names and the manifest. Adding `--self-test` extracts the box and imports with the interpreter inside it, and `run` executes its entry point — both need the machine to match the box's target. `verify` on its own works anywhere. +An application would not shell out to the CLI: the same box runs through the Node and Python +consumer libraries, which is covered in the +[demo box guide](https://scrollcase.dev/guides/demo-box). + The two unpacked names are SHA-256 digests of their own contents: two builds of the same commit produce the same names, which is what makes the archive verifiable in the first place. Keep them as they are and side by side — `verify` finds the box by the hash its release document commits to, and diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/archive.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/archive.mjs new file mode 100644 index 0000000..fa2cc33 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/archive.mjs @@ -0,0 +1,356 @@ +/** + * Deterministic archive creation and defensive extraction. + * + * Writing: every box ships as a ZIP whose bytes depend only on its contents — fixed timestamps, + * stable file ordering, and modes derived from the target adapter — so rebuilding the same commit + * reproduces the archive bit for bit. + * + * Reading: nothing from inside an archive is trusted before it is validated. Entry names are + * checked against path traversal, links and special entries are rejected outright, and both ZIP + * and TAR are handled by pinned Node implementations rather than whatever tools the host happens + * to have — an archive behaves the same on every machine that opens it. + */ +import { constants, createWriteStream } from 'node:fs'; +import { copyFile, mkdir, mkdtemp, rm, stat, symlink } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { pipeline } from 'node:stream/promises'; +import * as tar from 'tar'; +import yauzl from 'yauzl'; +import yazl from 'yazl'; +import { findEntryThroughLink, findUnresolvableLink } from '../contract/links.mjs'; +import { + FIXED_ARCHIVE_TIME, + collectEntries, + collectFiles, + fileExists, + safeRelativePath, + validateExtractedTree, +} from './filesystem.mjs'; +import { fail } from './process.mjs'; + +const ZIP_FILE_TYPE_MASK = 0o170000; +const ZIP_REGULAR_FILE = 0o100000; +const ZIP_DIRECTORY = 0o040000; +const ZIP_SYMBOLIC_LINK = 0o120000; + +/** Returns the stable archive mode for a box payload file. */ +function archiveFileMode(adapter, relativePath) { + if (adapter.host.platform === 'win32') return 0o100644; + const scriptsDirectory = adapter.python.scriptsDirectory; + return relativePath === adapter.python.entryPoint || relativePath.startsWith(`${scriptsDirectory}/`) + ? 0o100755 + : 0o100644; +} + +/** + * Streams a deterministic, Zip64-capable box archive using the pinned Node backend. + * + * @param {string} payloadDir + * @param {string} archivePath + * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter + * @returns {Promise} + */ +export async function createDeterministicZip(payloadDir, archivePath, adapter) { + const entries = await collectEntries(payloadDir); + assertPayloadLinksAreCarryable(entries); + await rm(archivePath, { force: true }); + await mkdir(dirname(archivePath), { recursive: true }); + const zip = new yazl.ZipFile(); + const output = pipeline(zip.outputStream, createWriteStream(archivePath, { flags: 'wx' })); + for (const entry of entries) { + if (entry.kind === 'link') { + // A link is its target string, stored under a mode whose type bits say what it is — the same + // two facts every ZIP implementation reads it back from. + zip.addBuffer(Buffer.from(entry.linkTarget, 'utf8'), entry.path, { + compress: false, + mtime: FIXED_ARCHIVE_TIME, + mode: ZIP_SYMBOLIC_LINK | 0o777, + forceDosTimestamp: true, + }); + continue; + } + zip.addFile(join(payloadDir, ...entry.path.split('/')), entry.path, { + compress: true, + compressionLevel: 6, + mtime: FIXED_ARCHIVE_TIME, + mode: archiveFileMode(adapter, entry.path), + forceDosTimestamp: true, + }); + } + zip.end({ forceZip64Format: false }); + await output; +} + +/** + * Refuses to archive a payload whose links do not satisfy the contract rule. + * + * The builder settles links against the real filesystem; this asks the same question of the entry + * set that will actually be written, which is what a consumer will later be handed. A failure here + * is a bug in this repository rather than bad input — but shipping a box a consumer must reject is + * worse than not building one. + * + * @param {Array<{ path: string, kind: string, linkTarget?: string }>} entries + */ +function assertPayloadLinksAreCarryable(entries) { + const unresolvable = findUnresolvableLink(entries); + if (unresolvable) fail(`Box link does not resolve to a file inside the payload: ${unresolvable}`); + const throughLink = findEntryThroughLink(entries); + if (throughLink) fail(`Box entry would be written through a link: ${throughLink}`); +} + +/** + * The longest link target a payload may carry. A real one is a file name; anything approaching a + * path limit is either corrupt or an attempt to make reading the archive expensive. + */ +const MAX_LINK_TARGET_BYTES = 1024; + +/** Classifies a ZIP entry and rejects special entries and encrypted files. */ +function classifyZipEntry(entry) { + if ((entry.generalPurposeBitFlag & 0x1) !== 0) fail(`Encrypted ZIP entries are not allowed: ${entry.fileName}`); + const path = safeRelativePath(entry.fileName.endsWith('/') ? entry.fileName.slice(0, -1) : entry.fileName); + const unixType = (entry.externalFileAttributes >>> 16) & ZIP_FILE_TYPE_MASK; + if (unixType === ZIP_SYMBOLIC_LINK) { + if (entry.uncompressedSize > MAX_LINK_TARGET_BYTES) fail(`Archive link target is too long: ${path}`); + // The target itself is the entry's content, so it is not known yet; listZipEntries reads it + // before anything is validated, and nothing may be extracted until it has. + return { path, kind: 'link', size: entry.uncompressedSize, mode: 0o777, linkTarget: null }; + } + const directory = entry.fileName.endsWith('/') || unixType === ZIP_DIRECTORY; + if (!directory && unixType !== 0 && unixType !== ZIP_REGULAR_FILE) { + fail(`Archive special entries are not allowed: ${path}`); + } + return { + path, + kind: directory ? 'directory' : 'file', + size: entry.uncompressedSize, + mode: (entry.externalFileAttributes >>> 16) & 0o777, + }; +} + +/** Rejects duplicate paths and file/directory collisions before extraction begins. */ +function assertNoZipEntryCollisions(entries) { + const seen = new Map(); + const parentsWithChildren = new Set(); + for (const entry of entries) { + if (seen.has(entry.path)) fail(`Archive entry collides with another entry: ${entry.path}`); + const parts = entry.path.split('/'); + for (let index = 1; index < parts.length; index += 1) { + const parent = parts.slice(0, index).join('/'); + if (seen.get(parent) === 'file') { + fail(`Archive entry collides with another entry: ${entry.path}`); + } + parentsWithChildren.add(parent); + } + if (entry.kind === 'file' && parentsWithChildren.has(entry.path)) { + fail(`Archive entry collides with another entry: ${entry.path}`); + } + seen.set(entry.path, entry.kind); + } +} + +/** Opens a ZIP with strict names, path validation, and uncompressed-size checks enabled. */ +async function openZip(archivePath) { + return yauzl.openPromise(archivePath, { + autoClose: false, + decodeStrings: true, + lazyEntries: true, + strictFileNames: true, + validateEntrySizes: true, + }); +} + +/** + * Lists and validates all entries before any ZIP data is trusted or extracted. + * + * @param {string} archivePath + * @returns {Promise>} + */ +export async function listZipEntries(archivePath) { + const zip = await openZip(archivePath); + const entries = []; + try { + for await (const entry of zip.eachEntry()) { + const classified = classifyZipEntry(entry); + if (classified.kind === 'link') { + const chunks = []; + const stream = await zip.openReadStreamPromise(entry); + for await (const chunk of stream) chunks.push(chunk); + classified.linkTarget = Buffer.concat(chunks).toString('utf8'); + } + entries.push(classified); + } + } finally { + await zip.close(); + } + assertNoZipEntryCollisions(entries); + // Every link is judged by the same rule the builder applied, against the archive as received + // rather than as intended. A box assembled by hand gets no benefit of the doubt here. + const unresolvable = findUnresolvableLink(entries); + if (unresolvable) fail(`Archive link does not resolve to a file inside the payload: ${unresolvable}`); + const throughLink = findEntryThroughLink(entries); + if (throughLink) fail(`Archive entry would be written through a link: ${throughLink}`); + return entries; +} + +/** + * Reads one small ZIP metadata entry without extracting the surrounding archive. + * + * @param {string} archivePath + * @param {string} wantedPath + * @param {number} [maximumBytes] + * @returns {Promise} + */ +export async function readZipEntry(archivePath, wantedPath, maximumBytes = 1024 * 1024) { + const safePath = safeRelativePath(wantedPath); + const zip = await openZip(archivePath); + try { + for await (const entry of zip.eachEntry()) { + const classified = classifyZipEntry(entry); + if (classified.path !== safePath || classified.kind !== 'file') continue; + if (classified.size > maximumBytes) fail(`ZIP entry is too large to read as metadata: ${safePath}`); + const stream = await zip.openReadStreamPromise(entry); + const chunks = []; + let length = 0; + for await (const chunk of stream) { + length += chunk.length; + if (length > maximumBytes) fail(`ZIP entry is too large to read as metadata: ${safePath}`); + chunks.push(chunk); + } + return Buffer.concat(chunks).toString('utf8'); + } + } finally { + await zip.close(); + } + fail(`ZIP archive does not contain ${safePath}`); +} + +/** + * Extracts a prevalidated ZIP without shelling out to whatever unzip the host provides. + * + * @param {string} archivePath + * @param {string} destination + * @returns {Promise} + */ +export async function extractZipArchive(archivePath, destination) { + // Validated in full first, and the targets it returns are the only ones written below: reading + // the link target twice would let a concurrently rewritten archive pass the check with one value + // and extract with another. + const validated = await listZipEntries(archivePath); + const linkTargets = new Map(validated + .filter((entry) => entry.kind === 'link') + .map((entry) => [entry.path, entry.linkTarget])); + await mkdir(destination, { recursive: true }); + const zip = await openZip(archivePath); + try { + for await (const entry of zip.eachEntry()) { + const classified = classifyZipEntry(entry); + const outputPath = join(destination, ...classified.path.split('/')); + if (classified.kind === 'directory') { + await mkdir(outputPath, { recursive: true }); + continue; + } + await mkdir(dirname(outputPath), { recursive: true }); + if (classified.kind === 'link') { + // Written as the relative string it was validated as, never as a resolved absolute path: + // the link must mean the same thing wherever the box is extracted. + await symlink(linkTargets.get(classified.path), outputPath); + continue; + } + const stream = await zip.openReadStreamPromise(entry); + await pipeline(stream, createWriteStream(outputPath, { + flags: 'wx', + mode: classified.mode || 0o644, + })); + } + } finally { + await zip.close(); + } + await validateExtractedTree(destination, { allowLinks: true }); +} + +/** Lists TAR assets and rejects paths, links, and special entries before extraction. */ +async function validateTarArchive(archivePath) { + let violation; + await tar.t({ + file: archivePath, + gzip: true, + strict: true, + onentry(entry) { + if (violation) return; + try { + safeRelativePath(entry.path.endsWith('/') ? entry.path.slice(0, -1) : entry.path); + if (!['File', 'OldFile', 'Directory'].includes(entry.type)) { + violation = `Archive links and special entries are not allowed: ${entry.path}`; + } + } catch (error) { + violation = error instanceof Error ? error.message : String(error); + } + }, + }); + if (violation) fail(violation); +} + +/** + * Extracts scroll assets using only pinned Node archive implementations. + * + * @param {string} archivePath + * @param {'zip' | 'tar.gz'} format + * @param {string} destination + * @param {number} [stripComponents] + * @returns {Promise} + */ +export async function extractScrollArchive(archivePath, format, destination, stripComponents = 0) { + const tempRoot = await mkdtemp(join(tmpdir(), 'scrollcase-extract-')); + try { + if (format === 'zip') { + await extractZipArchive(archivePath, tempRoot); + } else if (format === 'tar.gz') { + await validateTarArchive(archivePath); + await tar.x({ + file: archivePath, + cwd: tempRoot, + gzip: true, + preservePaths: false, + strict: true, + }); + await validateExtractedTree(tempRoot); + } else { + fail(`Unsupported archive format: ${format}`); + } + + let source = tempRoot; + for (let index = 0; index < stripComponents; index += 1) { + const entries = await collectFiles(source); + const topLevels = [...new Set(entries + .map((entry) => entry.split('/')[0]) + .filter((entry) => entry !== '__MACOSX'))]; + if (topLevels.length !== 1) fail(`Cannot strip archive component ${index + 1}: expected one root directory`); + const nextSource = join(source, topLevels[0]); + if (!(await stat(nextSource)).isDirectory()) { + fail(`Cannot strip archive component ${index + 1}: expected one root directory`); + } + source = nextSource; + } + const files = await collectFiles(source); + // Archives may add a subtree beside verified assets, but must never replace those assets. + for (const file of files) { + if (await fileExists(join(destination, ...file.split('/')))) { + fail(`Scroll archive entry already exists in destination: ${file}`); + } + } + await mkdir(destination, { recursive: true }); + for (const file of files) { + const outputPath = join(destination, ...file.split('/')); + await mkdir(dirname(outputPath), { recursive: true }); + await copyFile(join(source, ...file.split('/')), outputPath, constants.COPYFILE_EXCL); + } + } finally { + await rm(tempRoot, { recursive: true, force: true }); + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/assets.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/assets.mjs new file mode 100644 index 0000000..247b8e0 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/assets.mjs @@ -0,0 +1,123 @@ +/** + * Fetching and staging the files a box carries. + * + * Every asset is declared with a size and a SHA-256 in the scroll, and nothing enters the payload + * before both match. That is what makes a box reproducible even though its inputs live on servers + * outside anyone's control: if an upstream file is moved, replaced, or silently re-uploaded, the + * build fails instead of quietly producing a different box under the same version. + */ + +import { createWriteStream } from 'node:fs'; +import { copyFile, mkdir, rename, rm, stat } from 'node:fs/promises'; +import { dirname, join, sep } from 'node:path'; +import { pipeline } from 'node:stream/promises'; +import { extractScrollArchive as extractArchive } from './archive.mjs'; +import { fileExists, safeRelativePath, sha256File } from './filesystem.mjs'; +import { fail } from './process.mjs'; + +const MAX_DOWNLOAD_ATTEMPTS = 5; + +/** + * Downloads an asset and enforces the scroll's declared size and hash. + * + * Model files are large, so retries inside one download operation resume from a `.part` file with a + * Range request. Two safeguards matter — a complete destination is reused only after size and hash + * verification, and the `.part` file is renamed into place only *after* the hash matches, so an + * interrupted or corrupted transfer can never masquerade as a finished asset. The build scratch + * tree is recreated at process start, so this is intentionally not a cross-process cache. + */ +export async function downloadVerified(asset, destination, options = {}) { + const { + fetchImpl = fetch, + log = console.error, + wait = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), + } = options; + const expectedPath = safeRelativePath(asset.relativePath).split('/').join(sep); + if (!destination.endsWith(expectedPath)) fail(`Unexpected asset destination: ${destination}`); + await mkdir(dirname(destination), { recursive: true }); + if (await fileExists(destination)) { + const current = await stat(destination); + if (current.size === asset.sizeBytes && await sha256File(destination) === asset.sha256) return; + } + const partPath = `${destination}.part`; + // Large assets come from hosts that occasionally drop a connection mid-stream. Retry with backoff, + // resuming from the partial via Range, so a reset near the end is not paid for in full. + for (let attempt = 1; ; attempt += 1) { + const resumeAt = await fileExists(partPath) ? (await stat(partPath)).size : 0; + try { + const response = await fetchImpl(asset.url, { + headers: resumeAt > 0 ? { Range: `bytes=${resumeAt}-` } : undefined, + redirect: 'follow', + }); + if (!response.ok) fail(`Asset download failed (${response.status}): ${asset.url}`); + // Only append when the server actually honoured the range (206). A server that ignores Range + // replies 200 with the whole body, which must overwrite rather than be appended to a partial. + const append = resumeAt > 0 && response.status === 206; + await pipeline(response.body, createWriteStream(partPath, { flags: append ? 'a' : 'w' })); + break; + } catch (error) { + // A failed status is a hard error, not a transient network drop — do not retry it. + const message = error instanceof Error ? error.message : String(error); + if (message.startsWith('Asset download failed') || attempt >= MAX_DOWNLOAD_ATTEMPTS) throw error; + log(`scrollcase: asset ${asset.relativePath} attempt ${attempt} failed (${message}); retrying.`); + await wait(2000 * attempt); + } + } + const downloaded = await stat(partPath); + if (downloaded.size !== asset.sizeBytes) fail(`Asset size mismatch for ${asset.relativePath}.`); + if (await sha256File(partPath) !== asset.sha256) { + // A full-size partial with the wrong digest cannot be resumed: asking for bytes after its end + // would either fail forever or append unrelated data. Remove it so the next build starts from + // byte zero and has a chance to recover from a corrupt mirror response. + await rm(partPath, { force: true }); + fail(`Asset SHA-256 mismatch for ${asset.relativePath}.`); + } + await rename(partPath, destination); +} + +/** Copies a file from the project into the payload after verifying its declared hash. */ +export async function copyVerifiedLocalFile(file, payloadDir, projectRoot) { + const source = join(projectRoot, safeRelativePath(file.sourcePath)); + if (!await fileExists(source) || !(await stat(source)).isFile()) { + fail(`Local box file is missing: ${file.sourcePath}`); + } + if (await sha256File(source) !== file.sha256) { + fail(`Local box file SHA-256 mismatch: ${file.sourcePath}`); + } + const destination = join(payloadDir, safeRelativePath(file.relativePath)); + await mkdir(dirname(destination), { recursive: true }); + await copyFile(source, destination); +} + +/** + * Moves a built file to the name it is published under, without ever leaving two copies behind. + * + * A box archive is measured in gigabytes, so this renames rather than copies: on one filesystem the + * bytes never move at all. The copy-and-remove fallback is for the case a project points its build + * and dist directories at different volumes, where rename cannot work. + */ +export async function moveIntoPlace(source, destination) { + await rm(destination, { force: true }); + try { + await rename(source, destination); + } catch { + await copyFile(source, destination); + await rm(source, { force: true }); + } +} + +/** + * Unpacks a downloaded archive into the payload tree. + * + * Entries are listed and validated *before* extraction (archive-slip defence). `stripComponents` + * drops the redundant top-level wrapper directory many published archives carry; it insists on + * finding exactly one directory to strip, so a surprising layout fails loudly rather than producing + * a wrong tree. + */ +export async function expandAssetArchive(payloadDir, archive) { + const archivePath = join(payloadDir, safeRelativePath(archive.relativePath)); + const destination = join(payloadDir, safeRelativePath(archive.destination)); + await extractArchive(archivePath, archive.format, destination, Number(archive.stripComponents ?? 0)); + // The compressed original is dead weight inside the payload once unpacked. + if (archive.removeAfterExtract !== false) await rm(archivePath, { force: true }); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/audit.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/audit.mjs new file mode 100644 index 0000000..c284042 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/audit.mjs @@ -0,0 +1,69 @@ +/** + * `audit` — the dependency licence inventory, without building anything. + * + * The inventory is a pure function of the committed lock, so it can be produced, reviewed and + * checked into a repository long before any box exists. That matters because licence review is a + * human step: it should happen when dependencies change, not in the middle of a multi-gigabyte build + * that then fails at the end. + * + * The same function the build runs is used here, so a reviewed audit and the one a build produces + * cannot disagree by construction. + */ + +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { boxTargetId } from '../contract/targets.mjs'; +import { compareStableStrings, fileExists, safeRelativePath } from './filesystem.mjs'; +import { createCondaDependencyLicenseAudit, validateCondaDependencyLicenseAudit } from './licenses.mjs'; +import { fail } from './process.mjs'; +import { readScroll } from './scroll.mjs'; +import { getWorkspace } from './workspace.mjs'; + +/** + * Produces the inventory for a scroll, and either checks it against the reviewed copy or writes it. + * + * Writing is explicit (`write: true`) because overwriting the reviewed file is how an unreviewed + * licence change would slip through: the default is to compare and fail on any difference. + */ +export async function auditScroll(name, { write = false, namespace } = {}) { + const workspace = getWorkspace(); + const { dir, scroll } = await readScroll(name); + const lockPath = join(dir, 'pixi.lock'); + if (!await fileExists(lockPath)) fail(`Missing dependency lock: ${lockPath}`); + const inventory = createCondaDependencyLicenseAudit({ + lockBytes: await readFile(lockPath), + targetId: boxTargetId(scroll.target), + ...(namespace ? { namespace } : {}), + }); + + // A package with no declared licence never reaches here: parsing the lock rejects it outright, + // which is the point — an unlicensed dependency is a legal problem, not a reporting gap. + const licences = new Map(); + for (const entry of inventory.packages) { + licences.set(entry.declaredLicense, (licences.get(entry.declaredLicense) ?? 0) + 1); + } + const summary = { + scrollId: scroll.scrollId, + targetId: inventory.targetId, + packageCount: inventory.packages.length, + licenses: [...licences] + .sort((left, right) => right[1] - left[1] || compareStableStrings(left[0], right[0])) + .map(([license, count]) => ({ license, count })), + }; + + if (!scroll.condaDependencyLicenseAudit) { + if (write) fail('The scroll declares no condaDependencyLicenseAudit path to write to.'); + return { inventory, summary, reviewed: null }; + } + const reviewedPath = join(workspace.root, safeRelativePath(scroll.condaDependencyLicenseAudit)); + if (write) { + await mkdir(dirname(reviewedPath), { recursive: true }); + await writeFile(reviewedPath, `${JSON.stringify(inventory, null, 2)}\n`); + return { inventory, summary, reviewed: reviewedPath, written: true }; + } + if (!await fileExists(reviewedPath)) { + fail(`Reviewed licence audit is missing: ${reviewedPath}. Run audit --write and review the result.`); + } + validateCondaDependencyLicenseAudit(JSON.parse(await readFile(reviewedPath, 'utf8')), inventory); + return { inventory, summary, reviewed: reviewedPath, written: false }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/authoring.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/authoring.mjs new file mode 100644 index 0000000..bb3adf4 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/authoring.mjs @@ -0,0 +1,403 @@ +/** + * Authoring one scroll inside an initialized workspace. + * + * `init` owns workspace structure; this module owns the atomic creation of one target-specific + * scroll. All material input is validated before the first write, existing paths are never + * overwritten, and a generated starter script is hashed from the exact bytes written to disk. + * Execution metadata is authored here and later copied unchanged into both signed manifests by the + * builder; keeping creation separate prevents this module from acquiring build or execution policy. + */ + +import { createHash } from 'node:crypto'; +import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { boxTargetAdapter, boxTargetId, condaSubdir } from '../contract/targets.mjs'; +import { fileExists, safeRelativePath, sha256File } from './filesystem.mjs'; +import { fail } from './process.mjs'; +import { schemaValidationError } from './schema-validation.mjs'; + +const scrollSchemaUrl = new URL('../contract/schema/scroll.schema.json', import.meta.url); +const targetSchemaUrl = new URL('../contract/schema/target.schema.json', import.meta.url); +const executionSchemaUrl = new URL('../contract/schema/execution.schema.json', import.meta.url); +const EXECUTION_KINDS = Object.freeze(['python-script', 'python-module', 'library-only']); +const WEIGHTS_MODES = Object.freeze(['embed', 'on-demand']); +export const EXAMPLE_PIXI_VERSION = '0.73.0'; + +const STARTER_SCRIPT = `"""Minimal application entry point generated by Scrollcase.""" + +import sys + + +def main() -> int: + print("Scrollcase box is ready.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +`; + +const TYPESCRIPT_CONSUMER_TEMPLATE = `/** + * Runs a local box through the typed Node consumer. + * + * SETUP (once): + * npm install scrollcase + * npm install --save-dev tsx typescript + * + * RUN: + * npx tsx consumer-templates/run-box.ts + * + * Replace and below with the values printed by scrollcase build. + */ +import { runBox } from 'scrollcase/consumer'; + +const releaseToRun = + '.scrollcase/dist/boxes/example-box/1.0.0//.release.json'; + +runBox(releaseToRun, { + publicPath: '.scrollcase/keys/signing-public.json', + args: [], + stdin: 'inherit', + stdout: 'inherit', + stderr: 'inherit', + onPrepared: ({ boxId, version, targetId }) => { + console.log(\`Running \${boxId} \${version} (\${targetId})\`); + }, +}).then((result) => { + if (result.signal) console.error(\`Box exited after \${result.signal}.\`); + process.exitCode = result.exitCode ?? 1; +}); +`; + +const CONSUMER_PACKAGE_JSON = `${JSON.stringify({ + private: true, + type: 'module', +}, null, 2)}\n`; + +const PYTHON_CONSUMER_TEMPLATE = `""" +Runs a local box through the typed Python consumer. + +The Python consumer is published separately on PyPI. +npm install scrollcase does not install this Python package. + +SETUP (once): + + python -m pip install scrollcase-consumer + +RUN (from the project root): + + python consumer-templates/run_box.py + +Replace and below with the values printed by scrollcase build. +""" + +from __future__ import annotations + +import sys + +from scrollcase_consumer import PreparedBox, run_box + + +RELEASE_TO_RUN = ( + ".scrollcase/dist/boxes/example-box/1.0.0//.release.json" +) + + +def _report(prepared: PreparedBox) -> None: + print( + f"Running {prepared.box_id} {prepared.version} ({prepared.target_id})" + ) + + +def main() -> int: + result = run_box( + RELEASE_TO_RUN, + public_key_path=".scrollcase/keys/signing-public.json", + args=[], + on_prepared=_report, + ) + + if result.signal is not None: + print(f"Box exited after {result.signal}.", file=sys.stderr) + return result.exit_code if result.exit_code is not None else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) +`; + +const textHash = (value) => createHash('sha256').update(value, 'utf8').digest('hex'); + +async function ensureTextFile(path, contents) { + await mkdir(dirname(path), { recursive: true }); + try { + await writeFile(path, contents, { flag: 'wx' }); + return true; + } catch (error) { + if (error?.code === 'EEXIST') return false; + throw error; + } +} + +function requiredText(value, name) { + if (typeof value !== 'string' || value.trim() === '') fail(`${name} is required.`); + return value.trim(); +} + +function pixiManifest(environmentName, target, pythonVersion) { + const pythonConstraint = /^\d+\.\d+$/.test(pythonVersion) + ? `${pythonVersion}.*` + : pythonVersion; + return `# Solved by \`scrollcase lock\` into pixi.lock, which is committed and reviewed. +# \`platforms\` must equal the target's conda subdirectory, or the solve produces an environment +# that cannot run on the machine the box is for. +[workspace] +name = "${environmentName}" +channels = ["conda-forge"] +platforms = ["${condaSubdir(target)}"] + +[dependencies] +python = "${pythonConstraint}" +`; +} + +async function validateScroll(scroll) { + const [scrollSchema, targetSchema, executionSchema] = await Promise.all( + [scrollSchemaUrl, targetSchemaUrl, executionSchemaUrl] + .map(async (url) => JSON.parse(await readFile(url, 'utf8'))), + ); + const error = schemaValidationError(scroll, scrollSchema, [targetSchema, executionSchema]); + if (error) fail(`Generated scroll is invalid: ${error}.`); +} + +/** + * Creates one nested `/` scroll without overwriting any authored file. + * + * @param {object} options + * @returns {Promise<{ written: string[], scroll: object, scrollDir: string, scrollRef: string, + * targetId: string, generatedScriptPath: string | null }>} + */ +export async function createScroll({ + workspace, + boxId, + target, + modelId, + runtimeId, + version, + scrollVersion, + sourceRevision, + pythonVersion, + pixiVersion, + compatibility, + assetBaseUrl, + weights, + executionKind, + scriptSourcePath = null, + generateScript = false, + generatedScriptSourcePath = null, + scriptRelativePath = 'entrypoint.py', + module = null, + defaultArgs = [], +}) { + if (!workspace?.configPath || !await fileExists(workspace.configPath) + || !await fileExists(workspace.scrollsDir)) { + fail('No initialized Scrollcase workspace; run scrollcase init first.'); + } + + const identity = { + boxId: requiredText(boxId, 'boxId'), + modelId: requiredText(modelId, 'modelId'), + runtimeId: requiredText(runtimeId, 'runtimeId'), + version: requiredText(version, 'version'), + scrollVersion: requiredText(scrollVersion, 'scrollVersion'), + sourceRevision: requiredText(sourceRevision, 'sourceRevision'), + pythonVersion: requiredText(pythonVersion, 'pythonVersion'), + pixiVersion: requiredText(pixiVersion, 'pixiVersion'), + assetBaseUrl: requiredText(assetBaseUrl, 'assetBaseUrl'), + }; + if (!compatibility || typeof compatibility !== 'object' || Array.isArray(compatibility)) { + fail('compatibility must be an object.'); + } + if (!WEIGHTS_MODES.includes(weights)) { + fail(`Unsupported weights mode: ${weights}. Use ${WEIGHTS_MODES.join(' or ')}.`); + } + if (!EXECUTION_KINDS.includes(executionKind)) { + fail(`Unsupported execution kind: ${executionKind}. Use ${EXECUTION_KINDS.join(', ')}.`); + } + if (!Array.isArray(defaultArgs) || defaultArgs.some((value) => typeof value !== 'string')) { + fail('defaultArgs must be an array of strings.'); + } + + const adapter = boxTargetAdapter(target); + const targetId = boxTargetId(target); + const scrollRef = `${identity.boxId}/${targetId}`; + const scrollDir = join(workspace.scrollsDir, identity.boxId, targetId); + if (await fileExists(scrollDir)) fail(`Scroll already exists: ${scrollRef}.`); + + let localFile = null; + let execution; + let generatedScriptPath = null; + let generatedSource = null; + if (executionKind === 'python-script') { + if (generateScript && scriptSourcePath) { + fail('Choose either an existing script or --generate-script, not both.'); + } + if (!generateScript && !scriptSourcePath) { + fail('python-script execution requires an existing script or --generate-script.'); + } + const relativePath = safeRelativePath(scriptRelativePath); + let sourcePath; + let sha256; + if (generateScript) { + sourcePath = safeRelativePath(generatedScriptSourcePath + ?? `box-entrypoints/${identity.boxId}/${targetId}/entrypoint.py`); + generatedScriptPath = join(workspace.root, ...sourcePath.split('/')); + if (await fileExists(generatedScriptPath)) { + fail(`Generated script already exists: ${sourcePath}.`); + } + generatedSource = STARTER_SCRIPT; + sha256 = textHash(generatedSource); + } else { + sourcePath = safeRelativePath(scriptSourcePath); + const source = join(workspace.root, ...sourcePath.split('/')); + let details; + try { + details = await lstat(source); + } catch { + fail(`Project script is missing: ${sourcePath}.`); + } + if (!details.isFile() || details.isSymbolicLink()) { + fail(`Project script must be a regular file: ${sourcePath}.`); + } + sha256 = await sha256File(source); + } + localFile = { sourcePath, relativePath, sha256 }; + execution = { kind: 'python-script', script: relativePath, defaultArgs: [...defaultArgs] }; + } else if (executionKind === 'python-module') { + execution = { + kind: 'python-module', + module: requiredText(module, 'module'), + defaultArgs: [...defaultArgs], + }; + } else if (module || scriptSourcePath || generateScript || defaultArgs.length > 0) { + fail('library-only execution cannot declare a script, module, or default arguments.'); + } + + const scroll = { + $schema: 'https://scrollcase.dev/schema/v2/scroll.schema.json', + schemaVersion: 2, + scrollVersion: identity.scrollVersion, + boxId: identity.boxId, + modelId: identity.modelId, + runtimeId: identity.runtimeId, + version: identity.version, + sourceRevision: identity.sourceRevision, + target, + compatibility: { ...compatibility }, + pythonVersion: identity.pythonVersion, + pixiVersion: identity.pixiVersion, + pythonEntryPoint: adapter.python.entryPoint, + modelCacheSubdir: `model-cache/${identity.boxId}`, + assetBaseUrl: identity.assetBaseUrl, + assets: [], + selfTest: { + imports: ['json'], + files: localFile ? [localFile.relativePath] : [], + }, + weights, + ...(localFile ? { localFiles: [localFile] } : {}), + ...(execution ? { execution } : {}), + }; + await validateScroll(scroll); + + const boxDir = dirname(scrollDir); + await mkdir(boxDir, { recursive: true }); + const staging = await mkdtemp(join(boxDir, '.scrollcase-new-')); + let generatedWritten = false; + try { + await writeFile(join(staging, 'scroll.json'), `${JSON.stringify(scroll, null, 2)}\n`); + await writeFile( + join(staging, 'pixi.toml'), + pixiManifest(`${identity.boxId}-${targetId}`, target, identity.pythonVersion), + ); + if (generatedScriptPath) { + await mkdir(dirname(generatedScriptPath), { recursive: true }); + await writeFile(generatedScriptPath, generatedSource, { flag: 'wx' }); + generatedWritten = true; + } + await rename(staging, scrollDir); + } catch (error) { + await rm(staging, { recursive: true, force: true }); + if (generatedWritten) await rm(generatedScriptPath, { force: true }); + throw error; + } + + const written = [ + join(scrollDir, 'scroll.json'), + join(scrollDir, 'pixi.toml'), + ...(generatedScriptPath ? [generatedScriptPath] : []), + ]; + return { written, scroll, scrollDir, scrollRef, targetId, generatedScriptPath }; +} + +/** + * Ensures the disposable example created by `init` exists for one native target. + * + * The example uses the same authoring path as every real scroll. An existing target directory is + * treated as authored input and left untouched, including when a user has edited the starter. + * + * @param {{ workspace: object, target: object, pixiVersion?: string }} options + * @returns {Promise} + */ +export async function ensureExampleScroll({ + workspace, + target, + pixiVersion = EXAMPLE_PIXI_VERSION, +}) { + const targetId = boxTargetId(target); + const scrollRef = `example-box/${targetId}`; + const scrollDir = join(workspace.scrollsDir, 'example-box', targetId); + let result; + if (await fileExists(scrollDir)) { + result = { + created: false, + written: [], + scrollDir, + scrollRef, + targetId, + generatedScriptPath: null, + }; + } else { + result = { + created: true, + ...await createScroll({ + workspace, + boxId: 'example-box', + target, + modelId: 'example-org-example-box', + runtimeId: 'example-box-runtime', + version: '1.0.0', + scrollVersion: '1.0.0', + sourceRevision: 'example-source-1.0.0', + pythonVersion: '3.11', + pixiVersion, + compatibility: { minHostAppVersion: '1.0.0' }, + assetBaseUrl: 'https://example.org/boxes', + weights: 'embed', + executionKind: 'python-script', + generateScript: true, + }), + }; + } + + const consumerFiles = [ + [join(workspace.root, 'package.json'), CONSUMER_PACKAGE_JSON], + [join(workspace.root, 'consumer-templates', 'run-box.ts'), TYPESCRIPT_CONSUMER_TEMPLATE], + [join(workspace.root, 'consumer-templates', 'run_box.py'), PYTHON_CONSUMER_TEMPLATE], + ]; + const consumerFilesWritten = []; + for (const [path, contents] of consumerFiles) { + if (await ensureTextFile(path, contents)) consumerFilesWritten.push(path); + } + return { ...result, written: [...result.written, ...consumerFilesWritten] }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/box.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/box.mjs new file mode 100644 index 0000000..41b7ac0 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/box.mjs @@ -0,0 +1,339 @@ +/** + * `build` — assemble the environment, prove it works, archive it, sign it. + * + * In order: solve and pack the conda-forge environment from the committed lock, fetch and unpack the + * declared assets, prune what is not needed at run time, audit dependency licences, self-test with + * the payload's *own* interpreter, normalise timestamps, zip deterministically, and emit a signed + * release plus a signed channel pointer. + * + * The self-test is the step that earns the box its name. The builder runs target, import, optional + * Python-code, and file assertions; the release signs the import subset that a consumer can repeat + * after extraction. The distinction is deliberate rather than pretending the narrower consumer + * check reproduces scroll-only assertions it cannot see. + * + * The archive is content-addressed by its own hash, so the release document can commit to it and any + * consumer can verify it byte for byte. + */ + +import { createHash } from 'node:crypto'; +import { mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { assertNativeHost, boxTargetId } from '../contract/targets.mjs'; +import { CHANNELS, documentKinds } from '../contract/documents.mjs'; +import { signDocument } from '../sign/index.mjs'; +import { copyVerifiedLocalFile, downloadVerified, expandAssetArchive, moveIntoPlace } from './assets.mjs'; +import { createDeterministicZip } from './archive.mjs'; +import { + collectFiles, + fileExists, + normalizeTree, + payloadSize, + safeRelativePath, + sha256File, +} from './filesystem.mjs'; +import { assertExecutionFiles } from './execution.mjs'; +import { boxReleaseObjectPrefix, boxReleaseStem, builderVersionFields } from './identity.mjs'; +import { createCondaDependencyLicenseAudit, validateCondaDependencyLicenseAudit } from './licenses.mjs'; +import { checkParity } from './parity.mjs'; +import { findCondaPack, findPixi, installAndPackPixiEnvironment } from './pixi.mjs'; +import { fail, run as runProcess } from './process.mjs'; +import { readScroll, sourceBuildState, sourceBuildTime } from './scroll.mjs'; +import { getWorkspace } from './workspace.mjs'; + +const SELF_TEST_TIMEOUT_SECONDS = 180; +const sha256Hex = (bytes) => createHash('sha256').update(bytes).digest('hex'); + +/** Runs the scroll's self-test with the payload's own interpreter, under the target's environment. */ +function runSelfTest({ interpreter, adapter, scroll, payloadDir, run }) { + const imports = `import ${scroll.selfTest.imports.join(', ')}`; + const code = scroll.selfTest.pythonCode + ? `${adapter.selfTestPython}\n${imports}\n${scroll.selfTest.pythonCode}` + : `${adapter.selfTestPython}\n${imports}`; + run(interpreter, ['-c', code], { + cwd: payloadDir, + env: adapter.validationEnvironments[scroll.target.accelerator], + }); +} + +/** Writes the licence inventory the box ships, after proving it still matches the reviewed one. */ +async function writeLicenceAudit({ scroll, lockPath, payloadDir, projectRoot }) { + if (!scroll.condaDependencyLicenseAudit) return; + const actual = createCondaDependencyLicenseAudit({ + lockBytes: await readFile(lockPath), + targetId: boxTargetId(scroll.target), + }); + const reviewedPath = join(projectRoot, safeRelativePath(scroll.condaDependencyLicenseAudit)); + const reviewed = JSON.parse(await readFile(reviewedPath, 'utf8')); + validateCondaDependencyLicenseAudit(reviewed, actual); + const auditPath = join(payloadDir, 'THIRD_PARTY_NOTICES', 'conda-distributions.json'); + await mkdir(dirname(auditPath), { recursive: true }); + await writeFile(auditPath, `${JSON.stringify(actual, null, 2)}\n`); +} + +/** + * Builds, self-tests, archives, and signs the box a scroll describes — the whole pipeline the + * module header narrates. `name` is an exact scroll reference, or an unambiguous box shorthand; + * options override signing, channel, weights mode, namespace, and toolchain paths. `run`, + * `runResult`, and `fetchImpl` are the injection seams the tests use to substitute the toolchain + * and asset transport. + */ +export async function buildBox(name, options = {}) { + const { + allowDirty = false, + channel = 'beta', + weights = null, + assetBaseUrl: assetBaseUrlOverride = null, + namespace, + signerCommand = null, + privatePath, + publicPath, + pixiPath = null, + condaPackPath = null, + run = runProcess, + runResult = null, + fetchImpl = fetch, + log = console.log, + } = options; + // Tool discovery probes with its own runner; a caller may substitute one to drive a build without + // the real toolchain on PATH. + const probe = runResult ? { runResult } : {}; + const workspace = getWorkspace(); + const { adapter, dir, scroll } = await readScroll(name); + const weightsMode = weights || scroll.weights || 'embed'; + if (!CHANNELS.includes(channel)) { + fail(`Unsupported channel: ${channel}. Use ${CHANNELS.join(' or ')}.`); + } + if (weightsMode !== 'embed' && weightsMode !== 'on-demand') { + fail(`Unsupported weights mode: ${weightsMode}. Use embed or on-demand.`); + } + if (weightsMode === 'on-demand' && (scroll.assetArchives ?? []).length > 0) { + fail('on-demand weights cannot be combined with assetArchives, which are expanded at build time.'); + } + // Wheels, native libraries, and the interpreter are proven on the exact OS/architecture they ship for. + assertNativeHost(adapter); + const pixi = findPixi({ requiredVersion: scroll.pixiVersion, path: pixiPath, ...probe }); + const condaPack = findCondaPack({ path: condaPackPath, ...probe }); + const lockPath = join(dir, 'pixi.lock'); + // A build installs from the lock and never resolves, so a missing lock is a hard error rather than + // an invitation to resolve dependencies on the fly. + if (!await fileExists(lockPath)) fail(`Missing dependency lock: ${lockPath}`); + const lockSha = await sha256File(lockPath); + + const source = sourceBuildState(workspace.root); + if (!source) fail('A box records the commit it was built from; run inside a git checkout.'); + // If the tree is dirty that record is a lie — the artefact would not be reproducible from that + // revision — so refuse unless the caller explicitly accepts it for local development. + if (source.dirty && !allowDirty) { + fail('Refusing to build from a dirty source tree. Commit first, or pass --allow-dirty for local development.'); + } + + const buildDir = join(workspace.buildDir, scroll.scrollId); + const payloadDir = join(buildDir, 'payload'); + // `dist` is laid out as the two things a publisher does with it, and nothing else. `boxes/` is + // the tree that goes under the asset base URL verbatim — the same prefix the signed documents + // write into their own URLs, so uploading it is a copy rather than a mapping. `channels/` is + // separate because a channel is not part of any one version: it is a pointer that moves to the + // next one, and filing it under 1.0.0 would leave a stale copy claiming to be current the moment + // 1.0.1 ships. Nothing is written twice: what is on disk here is what gets published. + const objectPrefix = boxReleaseObjectPrefix(scroll); + const objectDir = join(workspace.distDir, ...objectPrefix.split('/')); + const archivePath = join(buildDir, `${boxReleaseStem(scroll)}.zip`); + // Always start from an empty tree: leftovers from a previous build would end up in the archive. + await rm(buildDir, { recursive: true, force: true }); + await rm(objectDir, { recursive: true, force: true }); + await mkdir(payloadDir, { recursive: true }); + + const { interpreter } = await installAndPackPixiEnvironment({ + pixi, + condaPack, + manifestPath: join(dir, 'pixi.toml'), + lockPath, + buildDir, + payloadDir, + adapter, + run, + }); + + // `embed` packs the assets into the archive, so an installed box needs no network and works + // air-gapped. `on-demand` leaves them out for the caller's distribution layer to materialize from + // descriptors carried in the signed release. Consumers verify those bytes before execution; the + // declared hash is what keeps that safe. The choice trades archive size against an install-time + // dependency on the asset host, so it is the project's to make, per build. + const embedded = weightsMode === 'embed'; + for (const asset of embedded ? scroll.assets : []) { + log(`Downloading ${asset.relativePath}`); + await downloadVerified(asset, join(payloadDir, safeRelativePath(asset.relativePath)), { + fetchImpl, + log, + }); + } + const deferredAssets = new Set(embedded ? [] : scroll.assets.map((asset) => asset.relativePath)); + for (const file of scroll.localFiles ?? []) { + await copyVerifiedLocalFile(file, payloadDir, workspace.root); + } + for (const archive of embedded ? scroll.assetArchives ?? [] : []) { + await expandAssetArchive(payloadDir, archive); + } + // Drops what is only needed to build (tests, docs, bundled sample data). A box is a multi-gigabyte + // download for an end user, so pruning is a user-facing concern rather than tidiness. + for (const prunePath of scroll.prunePaths ?? []) { + await rm(join(payloadDir, safeRelativePath(prunePath)), { recursive: true, force: true }); + } + await writeLicenceAudit({ scroll, lockPath, payloadDir, projectRoot: workspace.root }); + // Guards against over-pruning: the files the box needs at run time must still be there. + for (const requiredFile of scroll.selfTest.files ?? []) { + // A deferred asset is legitimately absent from the payload; anything else missing means pruning + // removed something the box needs at run time. + if (deferredAssets.has(requiredFile)) continue; + if (!await fileExists(join(payloadDir, safeRelativePath(requiredFile)))) { + fail(`Missing self-test file: ${requiredFile}`); + } + } + assertExecutionFiles({ + execution: scroll.execution, + adapter, + pythonVersion: scroll.pythonVersion, + files: new Set(await collectFiles(payloadDir)), + }); + runSelfTest({ interpreter, adapter, scroll, payloadDir, run }); + // Parity runs after the self-test, on the same payload: there is no point comparing accelerators + // in a box that cannot import its dependencies in the first place. + const parity = await checkParity({ + parity: scroll.parity, + adapter, + interpreter, + payloadDir, + run, + }); + if (parity) { + log(`Parity passed on ${parity.comparisons.map((c) => c.accelerator).join(', ')} against ${parity.comparisons[0].reference}`); + } + + // Everything needed to answer "where did this box come from, and could I rebuild it?". + const provenance = { + scrollId: scroll.scrollId, + scrollVersion: scroll.scrollVersion, + builderRevision: source.revision, + sourceTreeDirty: source.dirty, + sourceRevision: scroll.sourceRevision, + pythonVersion: scroll.pythonVersion, + ...builderVersionFields(scroll), + dependencyLockSha256: lockSha, + builtAt: sourceBuildTime(workspace.root), + }; + const selfTest = { + pythonImports: scroll.selfTest.imports, + timeoutSeconds: SELF_TEST_TIMEOUT_SECONDS, + }; + // Descriptors travel with the box only when the consumer has to fetch the assets itself. + const deferred = embedded ? {} : { + weights: 'on-demand', + assets: scroll.assets.map(({ url, relativePath, sizeBytes, sha256 }) => ({ + url, relativePath, sizeBytes, sha256, + })), + }; + const identity = { + boxId: scroll.boxId, + modelId: scroll.modelId, + runtimeId: scroll.runtimeId, + version: scroll.version, + }; + const execution = scroll.execution ? { execution: scroll.execution } : {}; + // box.json travels *inside* the archive. A consumer compares it field by field against the signed + // release, which is what binds the archive's contents to its signed metadata. + await writeFile(join(payloadDir, 'box.json'), `${JSON.stringify({ + schemaVersion: 2, + ...identity, + target: scroll.target, + pythonEntryPoint: scroll.pythonEntryPoint, + modelCacheSubdir: scroll.modelCacheSubdir, + selfTest, + ...execution, + ...deferred, + provenance, + }, null, 2)}\n`); + await normalizeTree(payloadDir); + const installedSizeBytes = await payloadSize(payloadDir); + await mkdir(workspace.distDir, { recursive: true }); + await createDeterministicZip(payloadDir, archivePath, adapter); + + const archiveSha = await sha256File(archivePath); + const archiveSize = (await stat(archivePath)).size; + // Content-addressed: the object is named after its own hash, so publishing is idempotent and an + // object can never be replaced with different bytes under the same URL. + const archiveObject = `${objectPrefix}/${archiveSha}.zip`; + const assetBaseUrl = String(assetBaseUrlOverride || scroll.assetBaseUrl || '').replace(/\/$/, ''); + if (!assetBaseUrl) fail('No asset base URL: declare assetBaseUrl in the scroll or pass --asset-base-url.'); + const kinds = documentKinds(namespace); + const signing = { signerCommand, privatePath, publicPath }; + + const release = { + schemaVersion: 2, + kind: kinds.release, + ...identity, + target: scroll.target, + compatibility: scroll.compatibility, + archive: { format: 'zip', url: `${assetBaseUrl}/${archiveObject}`, sha256: archiveSha, sizeBytes: archiveSize }, + installedSizeBytes, + pythonEntryPoint: scroll.pythonEntryPoint, + modelCacheSubdir: scroll.modelCacheSubdir, + selfTest, + ...execution, + ...deferred, + provenance, + }; + // Written beside the archive, under the same scratch rule: named for its own hash once it has one. + const stagedReleasePath = join(buildDir, 'release.json'); + await writeFile(stagedReleasePath, `${JSON.stringify(await signDocument(release, signing), null, 2)}\n`); + + // The channel points at the release document by *its* hash too, so the whole chain is + // content-addressed: channel -> release document -> archive. + const releaseDocumentSha = await sha256File(stagedReleasePath); + const channelDocument = { + schemaVersion: 2, + kind: kinds.channel, + channel, + boxId: scroll.boxId, + target: scroll.target, + updatedAt: provenance.builtAt, + // Derived from box and version rather than random, so rebuilding the same release reproduces the + // same cohort assignment instead of reshuffling which users receive it. + cohortSalt: sha256Hex(Buffer.from(`${scroll.boxId}:${scroll.version}`)).slice(0, 32), + // A freshly built channel goes out at 100%; a staged rollout is arranged by editing this document + // rather than by the builder. + releases: [{ + version: scroll.version, + releaseManifestUrl: `${assetBaseUrl}/${objectPrefix}/${releaseDocumentSha}.release.json`, + rolloutPercentage: 100, + }], + }; + // One file per channel per target, filed by channel rather than by version: it is a pointer, and + // the next release moves it rather than adding a second one. + const channelDir = join(workspace.distDir, 'channels', scroll.boxId, channel); + const channelPath = join(channelDir, `${boxTargetId(scroll.target)}.json`); + await mkdir(channelDir, { recursive: true }); + await writeFile(channelPath, `${JSON.stringify(await signDocument(channelDocument, signing), null, 2)}\n`); + + // Both documents move — not copy — into the tree a publisher uploads, so the only copy that + // exists is the one that gets published and there is no second name for the same bytes. + await mkdir(objectDir, { recursive: true }); + const publishedArchive = join(objectDir, `${archiveSha}.zip`); + const publishedRelease = join(objectDir, `${releaseDocumentSha}.release.json`); + await moveIntoPlace(archivePath, publishedArchive); + await moveIntoPlace(stagedReleasePath, publishedRelease); + log(`Box: ${publishedArchive}`); + log(`Release: ${publishedRelease}`); + log(`Channel: ${channelPath}`); + log(''); + log(`Publish: upload ${join(workspace.distDir, 'boxes')} under ${assetBaseUrl}, keeping its paths,`); + log(' then publish the channel document where your clients look for it.'); + return { + archivePath: publishedArchive, + releasePath: publishedRelease, + channelPath, + archiveSha256: archiveSha, + installedSizeBytes, + weights: weightsMode, + parity, + }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/consumer-setup.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/consumer-setup.mjs new file mode 100644 index 0000000..358c0c3 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/consumer-setup.mjs @@ -0,0 +1,110 @@ +/** + * Optional dependencies for the generated consumer templates. + * + * These installations belong to the initialized project, not Scrollcase's managed build + * toolchain. Every command therefore runs from the workspace root, beside + * `scrollcase.config.json`. Node uses the root package and `node_modules`; Python uses the + * interpreter selected from the caller's environment. Consent and the Python package source are + * chosen at the CLI edge and passed in explicitly. + */ + +import { readFileSync } from 'node:fs'; +import { fail, run as defaultRun, runResult as defaultRunResult } from './process.mjs'; + +const packageJson = JSON.parse(readFileSync( + new URL('../../package.json', import.meta.url), + 'utf8', +)); + +export const SCROLLCASE_NPM_VERSION = packageJson.version; + +export function installTypeScriptConsumerDependencies({ + root, + scrollcaseVersion = SCROLLCASE_NPM_VERSION, + platform = process.platform, + comspec = process.env.ComSpec || 'cmd.exe', + run = defaultRun, +}) { + const runNpm = (args) => { + if (platform === 'win32') { + // npm is a .cmd shim on Windows, which spawnSync cannot execute directly. + run(comspec, ['/d', '/s', '/c', 'npm', ...args], { cwd: root }); + return; + } + run('npm', args, { cwd: root }); + }; + runNpm(['install', `scrollcase@${scrollcaseVersion}`]); + runNpm(['install', '--save-dev', 'tsx', 'typescript']); + return { scrollcaseVersion }; +} + +function findPython({ root, runResult }) { + for (const command of ['python', 'python3', 'py']) { + const result = runResult(command, ['--version'], { capture: true, cwd: root }); + if (!result.error && result.status === 0) return command; + } + fail('Python was not found. Install Python 3.10 or newer, then re-run scrollcase init.'); +} + +export function isCondaAvailable({ + root, + runResult = defaultRunResult, +}) { + const result = runResult('conda', ['--version'], { capture: true, cwd: root }); + return !result.error && result.status === 0; +} + +export function installPythonConsumerDependency({ + root, + source, + run = defaultRun, + runResult = defaultRunResult, +}) { + if (!['pypi', 'conda-forge'].includes(source)) { + fail(`Unsupported Python consumer source ${source}.`); + } + + if (source === 'pypi') { + const command = findPython({ root, runResult }); + const args = ['-m', 'pip', 'install', 'scrollcase-consumer']; + const result = runResult(command, args, { capture: true, cwd: root }); + if (result.error) fail(`${command} failed to start: ${result.error.message}`); + if (result.status === 0) return { source, command }; + + const detail = `${result.stderr || ''}\n${result.stdout || ''}`; + if (/externally-managed-environment/i.test(detail)) { + // PEP 668 blocks even user installs unless pip receives the override. Pairing it with + // --user keeps package files out of Homebrew's or the distribution's managed prefix. + run( + command, + [ + '-m', + 'pip', + 'install', + '--user', + '--break-system-packages', + 'scrollcase-consumer', + ], + { cwd: root }, + ); + return { source, command }; + } + fail(`${command} exited with status ${result.status}\n${detail.trim()}`); + } + + if (!isCondaAvailable({ root, runResult })) { + fail('Conda is not installed. Re-run scrollcase init and choose PyPI with pip.'); + } + run( + 'conda', + [ + 'install', + '--yes', + '--channel', + 'conda-forge', + 'scrollcase-consumer', + ], + { cwd: root }, + ); + return { source, command: 'python' }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/execution.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/execution.mjs new file mode 100644 index 0000000..965097c --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/execution.mjs @@ -0,0 +1,60 @@ +/** + * Static execution prerequisites shared by the builder and verifier. + * + * Execution metadata is not a command string: it names either one regular payload file or one + * dotted Python module. Checking the archive file set proves those names can resolve without + * importing a package, running an `__init__.py`, or starting the application. The later consumer + * may therefore launch only after the complete trust chain has passed. + */ + +import { safeRelativePath } from './filesystem.mjs'; +import { fail } from './process.mjs'; + +function pythonMajorMinor(version) { + const match = /^(\d+)\.(\d+)(?:\.|$)/.exec(version); + if (!match) fail(`Invalid Python version for execution discovery: ${version}.`); + return `${match[1]}.${match[2]}`; +} + +function moduleEntryPoints({ adapter, module, pythonVersion }) { + const modulePath = module.split('.').join('/'); + const relativeCandidates = [`${modulePath}.py`, `${modulePath}/__main__.py`]; + const standardLibrary = adapter.platform === 'windows' + ? 'venv/Lib' + : `venv/lib/python${pythonMajorMinor(pythonVersion)}`; + const roots = ['', standardLibrary, `${standardLibrary}/site-packages`]; + return roots.flatMap((root) => + relativeCandidates.map((path) => (root ? `${root}/${path}` : path))); +} + +/** + * Confirms that optional execution metadata names runnable regular files in a payload/archive. + * + * `files` must contain only regular archive entries. Both collectFiles() during build and the ZIP + * entry classifier during verify provide exactly that representation. + */ +export function assertExecutionFiles({ + execution, + adapter, + pythonVersion, + files, +}) { + if (!execution) return; + if (execution.kind === 'python-script') { + const script = safeRelativePath(execution.script); + if (!files.has(script)) fail(`Execution script is missing from the box: ${script}.`); + return; + } + if (execution.kind === 'python-module') { + const candidates = moduleEntryPoints({ + adapter, + module: execution.module, + pythonVersion, + }); + if (!candidates.some((path) => files.has(path))) { + fail(`Execution module is not discoverable in the box: ${execution.module}.`); + } + return; + } + fail(`Unsupported execution kind: ${String(execution.kind)}.`); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/filesystem.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/filesystem.mjs new file mode 100644 index 0000000..0da0cd0 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/filesystem.mjs @@ -0,0 +1,198 @@ +/** + * Filesystem primitives shared by the build, archive, and verify layers. + * + * Two invariants live here. Determinism: payload files are always enumerated in one stable order + * and stamped with one fixed timestamp, so hashing and archiving the same tree twice produces the + * same bytes. Safety: every relative path that will be joined to a directory is screened against + * traversal, and trees that will enter a box are refused if they contain links or special nodes. + */ +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { access, lstat, lutimes, readdir, readlink } from 'node:fs/promises'; +import { join, relative, sep } from 'node:path'; +import { fail } from './process.mjs'; + +/** + * The single mtime every archived file carries. Any fixed instant works — what matters is that it + * never varies between builds; this one is simply a recognisable round date safely past the 1980 + * floor of DOS/ZIP timestamps. + */ +export const FIXED_ARCHIVE_TIME = new Date('2000-01-01T00:00:00.000Z'); + +/** + * Returns whether a filesystem entry exists without exposing platform-specific error codes. + * + * @param {string} path + * @returns {Promise} + */ +export async function fileExists(path) { + try { + await access(path); + return true; + } catch { + return false; + } +} + +/** + * Compares machine-facing identifiers by code unit, independent of host locale and ICU data. + * + * @param {string} left + * @param {string} right + * @returns {-1 | 0 | 1} + */ +export function compareStableStrings(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +/** + * Rejects paths that could escape a box staging directory. + * + * @param {unknown} value + * @returns {string} the path, normalised to forward slashes + * @throws {Error} when the path is absolute, empty, contains `..`, a drive letter or a NUL + */ +export function safeRelativePath(value) { + const normalized = String(value).replaceAll('\\', '/'); + if (!normalized || normalized.startsWith('/') || normalized.includes('\0')) { + fail(`Unsafe relative path: ${value}`); + } + if (/^[A-Za-z]:\//.test(normalized) + || normalized.split('/').some((part) => part === '..' || part === '')) { + fail(`Unsafe relative path: ${value}`); + } + return normalized; +} + +/** + * Lists payload entries in the stable order used by hashing and archive creation. + * + * A payload may hold regular files and the narrow class of symbolic links `src/contract/links.mjs` + * permits; anything else — a socket, a device, a fifo — is still refused, because nothing that is + * not one of those two things can be archived, hashed or relocated meaningfully. + * + * @param {string} root + * @param {string} [current] + * @returns {Promise>} + */ +export async function collectEntries(root, current = root) { + const entries = await readdir(current, { withFileTypes: true }); + const collected = []; + for (const entry of entries.sort((a, b) => compareStableStrings(a.name, b.name))) { + if (entry.name === '__pycache__' || entry.name === '.DS_Store' || entry.name.endsWith('.pyc')) continue; + const fullPath = join(current, entry.name); + const path = relative(root, fullPath).split(sep).join('/'); + // Order matters: a symlink to a directory reports isDirectory() as false but would be walked + // into by isDirectory() checks that stat rather than lstat, so links are classified first. + if (entry.isSymbolicLink()) { + collected.push({ path, kind: 'link', linkTarget: (await readlink(fullPath)).split(sep).join('/') }); + } else if (entry.isDirectory()) { + collected.push(...await collectEntries(root, fullPath)); + } else if (entry.isFile()) { + collected.push({ path, kind: 'file' }); + } else { + fail(`box special entries are not allowed: ${path}`); + } + } + return collected; +} + +/** + * Lists every payload path — files and links alike — in the stable archive order. + * + * Callers asking "is this path in the box" want a link to count, because a linked path is a path + * that resolves. Callers that must read or rewrite bytes want `collectRegularFiles` instead. + * + * @param {string} root + * @returns {Promise} + */ +export async function collectFiles(root) { + return (await collectEntries(root)).map((entry) => entry.path); +} + +/** + * Lists only the payload paths backed by their own bytes. + * + * Anything that rewrites file contents belongs here rather than on `collectFiles`: writing through + * a link would edit the target a second time, once under its own name and once under the link's. + * + * @param {string} root + * @returns {Promise} + */ +export async function collectRegularFiles(root) { + return (await collectEntries(root)).filter((entry) => entry.kind === 'file').map((entry) => entry.path); +} + +/** + * Sums what a box actually occupies once extracted. + * + * `lstat`, not `stat`: a link costs its own few bytes, not the size of what it points at. Counting + * the target would restore on paper exactly the duplication that preserving links removes from + * disk, and this number is what a consumer checks free space against. + * + * @param {string} root + * @returns {Promise} + */ +export async function payloadSize(root) { + let total = 0; + for (const file of await collectFiles(root)) { + total += (await lstat(join(root, ...file.split('/')))).size; + if (!Number.isSafeInteger(total)) fail('box installed size exceeds the safe integer range.'); + } + return total; +} + +/** + * Rejects links and special nodes before an extracted tree is copied. + * + * This guards a *scroll-declared asset archive* — a third-party tar or zip a project points at — + * whose contents are then copied into the payload. A link here is not the narrow, checked kind a + * box may carry: it arrives from outside, and the copy that follows would write through it. The + * links a payload does carry come from the packed conda prefix, which is a different path with its + * own contract check, so this stays as strict as it has always been. + * + * A box archive is the one caller that passes `allowLinks`, because its links were each checked + * against the contract rule before extraction wrote them. Every other caller keeps the default. + * + * @param {string} root + * @param {{ allowLinks?: boolean, current?: string }} [options] + * @returns {Promise} + */ +export async function validateExtractedTree(root, { allowLinks = false, current = root } = {}) { + for (const entry of await readdir(current, { withFileTypes: true })) { + const fullPath = join(current, entry.name); + if (entry.isSymbolicLink()) { + if (allowLinks) continue; + fail(`Archive links and special entries are not allowed: ${relative(root, fullPath)}`); + } + if (entry.isDirectory()) await validateExtractedTree(root, { allowLinks, current: fullPath }); + else if (!entry.isFile()) fail(`Archive special entries are not allowed: ${relative(root, fullPath)}`); + } +} + +/** + * Applies the archive timestamp to every payload entry. + * + * `lutimes` stamps the link itself rather than following it to its target, which would otherwise be + * stamped once under its own name and again through every link that points at it. + * + * @param {string} root + * @returns {Promise} + */ +export async function normalizeTree(root) { + for (const file of await collectFiles(root)) { + await lutimes(join(root, ...file.split('/')), FIXED_ARCHIVE_TIME, FIXED_ARCHIVE_TIME); + } +} + +/** + * Streams a file into SHA-256 without buffering large boxes in memory. + * + * @param {string} path + * @returns {Promise} + */ +export async function sha256File(path) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest('hex'); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/identity.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/identity.mjs new file mode 100644 index 0000000..026be4b --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/identity.mjs @@ -0,0 +1,39 @@ +/** + * Naming: where a release's artefacts live relative to everything else. + * + * The stem and object prefix are derived from the release's identity fields alone, so the archive, + * its release document, and the staged objects always agree on their names without any of them + * recording the others' paths. Whatever a consumer uses to serve boxes, laying storage out under + * this prefix means the URLs inside the signed documents already point at the right objects. + */ +import { boxTargetId } from '../contract/targets.mjs'; + +/** + * Returns the single filename stem shared by an archive and its release document. + * + * @param {Pick} release + * @returns {string} `--` + */ +export function boxReleaseStem(release) { + return `${release.boxId}-${release.version}-${boxTargetId(release.target)}`; +} + +/** + * Returns the immutable object prefix for one box release target. + * + * @param {Pick} release + * @returns {string} `boxes///` + */ +export function boxReleaseObjectPrefix(release) { + return `boxes/${release.boxId}/${release.version}/${boxTargetId(release.target)}`; +} + +/** + * Returns the builder-identity field recorded in provenance: the pixi release that solved the box. + * + * @param {{ pixiVersion?: string } | null | undefined} source + * @returns {{ pixiVersion: string | undefined }} + */ +export function builderVersionFields(source) { + return { pixiVersion: source?.pixiVersion }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/index.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/index.mjs new file mode 100644 index 0000000..dbd079d --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/index.mjs @@ -0,0 +1,38 @@ +/** + * The build layer: everything needed to turn a scroll into a packed, relocatable box. + * + * One substrate only — pixi solves a conda-forge environment from a committed `pixi.lock`, conda-pack + * relocates it, and the result is extracted into the box's `venv/`. There is deliberately no second + * dependency backend: a packaging tool with two substrates has to prove every guarantee twice. + */ + +export { createDeterministicZip, extractZipArchive, listZipEntries } from './archive.mjs'; +export { collectFiles, fileExists, sha256File } from './filesystem.mjs'; +export { boxReleaseObjectPrefix, boxReleaseStem, builderVersionFields } from './identity.mjs'; +export { repairPosixLaunchers } from './launchers.mjs'; +export { + createCondaDependencyLicenseAudit, + lockedCondaDistributions, + parseCondaPackageReference, + validateCondaDependencyLicenseAudit, +} from './licenses.mjs'; +export { + condaPackArguments, + findCondaPack, + findPixi, + installAndPackPixiEnvironment, + pixiInstallArguments, + pixiLockArguments, +} from './pixi.mjs'; +export { fail, run, runResult } from './process.mjs'; +export { CONDA_PACK_VERSION } from './toolchain.mjs'; +export { + DEFAULT_WORKSPACE_PATHS, + SCROLLCASE_CONFIG_FILENAME, + configureWorkspace, + findWorkspaceConfig, + getWorkspace, + resolveWorkspace, + workspaceOverridesFromArgv, + workspaceOverridesFromFlags, +} from './workspace.mjs'; diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/launchers.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/launchers.mjs new file mode 100644 index 0000000..a153592 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/launchers.mjs @@ -0,0 +1,60 @@ +/** + * Repairs the console scripts a conda environment generates. + * + * Console scripts (tqdm, isympy, f2py, …) are written at solve time with the *build machine's* + * absolute interpreter path in their shebang. That path means nothing on a user's machine, and + * shipping it also leaks a developer's directory layout. Rewriting them to resolve Python next to + * themselves is what makes the packed environment genuinely relocatable. + */ + +import { chmod, readFile, writeFile } from 'node:fs/promises'; +import { basename, join } from 'node:path'; +import { collectFiles, fileExists } from './filesystem.mjs'; + +/** + * Removes either a direct shebang or a shell trampoline header from a launcher, leaving the Python + * body. The trampoline appears when an absolute shebang would exceed the POSIX length limit; it + * closes its quote either on its own `' '''` line or at the end of the same line (`… "$@" #'''`), + * so both are handled by scanning forward to the line that closes the quote. + */ +function posixLauncherBody(text) { + const lines = text.split('\n'); + if (lines.length === 0 || !lines[0].startsWith('#!')) return text; + if (lines[1]?.startsWith("'''exec'")) { + for (let index = 1; index < lines.length; index += 1) { + if (lines[index].trimEnd().endsWith("'''")) return lines.slice(index + 1).join('\n'); + } + } + return lines.slice(1).join('\n'); +} + +/** + * Makes generated POSIX console scripts resolve Python relative to their own installed path. + * + * @param {import('../contract/targets.mjs').BoxTargetAdapter} adapter + * @param {string} payloadDir + * @param {readonly string[]} forbiddenPaths + * @returns {Promise} + */ +export async function repairPosixLaunchers(adapter, payloadDir, forbiddenPaths) { + const scriptsRoot = join(payloadDir, ...adapter.python.scriptsDirectory.split('/')); + if (!await fileExists(scriptsRoot)) return; + const pythonName = basename(adapter.python.entryPoint); + for (const file of await collectFiles(scriptsRoot)) { + const path = join(scriptsRoot, ...file.split('/')); + const bytes = await readFile(path); + if (!bytes.subarray(0, 2).equals(Buffer.from('#!'))) continue; + const text = bytes.toString('utf8'); + // Search the complete generated launcher, since a trampoline hides the path below line one. + if (!forbiddenPaths.some((value) => text.includes(value))) continue; + const body = posixLauncherBody(text); + const launcher = [ + '#!/bin/sh', + `'''exec' "$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)/${pythonName}" "$0" "$@"`, + "' '''", + body, + ].join('\n'); + await writeFile(path, launcher); + await chmod(path, 0o755); + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/licenses.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/licenses.mjs new file mode 100644 index 0000000..ae7a58b --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/licenses.mjs @@ -0,0 +1,138 @@ +/** + * Builds the dependency licence inventory shipped inside every box. + * + * The inventory is derived from the committed lock file rather than from the installed tree: the + * lock already carries an SPDX licence per package, and `pixi install --frozen` guarantees the + * installed set equals it. That makes the audit a pure function of a file the user reviews, so it + * can be computed without a built prefix and cannot drift from what was approved. + */ + +import { createHash } from 'node:crypto'; +import { DEFAULT_DOCUMENT_NAMESPACE } from '../contract/documents.mjs'; +import { compareStableStrings } from './filesystem.mjs'; + +/** + * One package as the lock declares it. + * + * @typedef {object} LockedDistribution + * @property {string} name + * @property {string} version + * @property {string} declaredLicense the SPDX expression carried by the lock + * @property {'conda' | 'pypi'} source + */ + +function fail(message) { + throw new Error(`box licence audit: ${message}`); +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +/** conda ships packages as `.conda` or the older `.tar.bz2`; both encode name-version-build. */ +const CONDA_PACKAGE_FILE = /\.(?:conda|tar\.bz2)$/; + +/** + * Derives (name, version) from a conda package filename: `name-version-build.conda`. + * + * @param {string} url a conda package URL or filename + * @returns {{ name: string, version: string }} + * @throws {Error} when the filename is not `name-version-build.conda` + */ +export function parseCondaPackageReference(url) { + const file = String(url).split('/').pop() ?? ''; + const stem = file.replace(CONDA_PACKAGE_FILE, ''); + const parts = stem.split('-'); + // conda names may contain '-', but version and build never do, so they are the last two segments. + if (parts.length < 3 || stem === file) fail(`unparseable conda package filename: ${file}`); + parts.pop(); // build string + const version = parts.pop(); + return { name: parts.join('-'), version }; +} + +/** + * Parses the exact conda + pypi distributions and their declared licenses from a pixi.lock. + * + * The `packages:` section is a YAML list of `- conda: ` / `- pypi: ` items, each followed + * by indented `key: value` fields. This scans that regular, machine-generated structure directly + * rather than taking a transitive YAML dependency. + * + * @param {Buffer} lockBytes the committed `pixi.lock` + * @returns {LockedDistribution[]} sorted by name then version + * @throws {Error} when the lock is unparseable or a package lacks a licence + */ +export function lockedCondaDistributions(lockBytes) { + const lines = lockBytes.toString('utf8').split(/\r?\n/); + const start = lines.findIndex((line) => line === 'packages:'); + if (start === -1) fail('pixi.lock has no packages section'); + const distributions = []; + let current = null; + const flush = () => { + if (!current) return; + let { name, version } = current; + if (current.source === 'conda') ({ name, version } = parseCondaPackageReference(current.url)); + if (!name || !version) fail(`pixi.lock package lacks a name or version: ${current.url}`); + if (!current.license || current.license.toUpperCase() === 'UNKNOWN') { + fail(`${name}==${version} lacks a declared license in pixi.lock`); + } + // conda/pypi filenames already carry the canonical name, so keep raw names — normalizing + // would mangle legitimate leading-underscore conda names like `_openmp_mutex`. + distributions.push({ name, version, declaredLicense: current.license, source: current.source }); + current = null; + }; + for (let index = start + 1; index < lines.length; index += 1) { + const line = lines[index]; + const entry = /^- (conda|pypi): (.+)$/.exec(line); + if (entry) { + flush(); + current = { source: entry[1], url: entry[2], name: null, version: null, license: null }; + continue; + } + if (!current) continue; + // A non-indented, non-empty line ends the packages section (defensive; it is normally last). + if (line !== '' && !line.startsWith(' ')) { flush(); break; } + const field = /^ {2}(\w[\w-]*): (.*)$/.exec(line); + if (!field) continue; + const [, key, value] = field; + if (key === 'license' && current.license === null) current.license = value.trim(); + else if (key === 'name' && current.name === null) current.name = value.trim(); + else if (key === 'version' && current.version === null) current.version = value.trim(); + } + flush(); + return distributions.sort((left, right) => + compareStableStrings(left.name, right.name) || compareStableStrings(left.version, right.version)); +} + +/** + * Builds the deterministic conda license audit bound to one pixi.lock and target. + * + * @param {{ lockBytes: Buffer, targetId: string, namespace?: string }} options + * @returns {{ schemaVersion: 2, kind: string, targetId: string, dependencyLockSha256: string, + * packages: LockedDistribution[] }} + * @throws {Error} when a locked package declares no licence + */ +export function createCondaDependencyLicenseAudit({ lockBytes, targetId, namespace = DEFAULT_DOCUMENT_NAMESPACE }) { + return { + schemaVersion: 2, + kind: `${namespace}.dependency-license-audit`, + targetId, + dependencyLockSha256: sha256(lockBytes), + packages: lockedCondaDistributions(lockBytes), + }; +} + +/** + * Ensures a reviewed conda audit still matches the current pixi.lock exactly. + * + * @param {unknown} reviewed the audit committed to the repository + * @param {ReturnType} actual + * @returns {ReturnType} `actual`, when they agree + * @throws {Error} when the lock no longer matches what was reviewed + */ +export function validateCondaDependencyLicenseAudit(reviewed, actual) { + if (reviewed?.schemaVersion !== 2 || reviewed.kind !== actual.kind) fail('reviewed conda audit contract is invalid'); + if (JSON.stringify(reviewed) !== JSON.stringify(actual)) { + fail('locked conda dependency licenses differ from the reviewed audit'); + } + return actual; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/parity.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/parity.mjs new file mode 100644 index 0000000..6a67d90 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/parity.mjs @@ -0,0 +1,111 @@ +/** + * Accelerator parity: does this box compute the same thing on the GPU as on the CPU? + * + * That question sounds scientific but is a packaging question. It catches the failures this tool is + * responsible for — the wrong wheels solved in, a CPU-only build shipped as CUDA, a broken BLAS — + * and it catches them on the build machine rather than on a user's. + * + * The division of labour matters. Scrollcase owns the mechanism: run the declared check inside the + * box once per accelerator, compare the numbers, enforce the declared tolerances, report what was + * measured. The project owns the meaning: which input to feed the model, which tensor to read, and + * what closeness is acceptable for it. The tool never decides what is scientifically correct — it + * enforces a threshold its user wrote down. + */ + +import { fail } from './process.mjs'; + +/** Reads the array of numbers a parity check prints, rejecting anything else. */ +function readValues(output, accelerator) { + let parsed; + try { + parsed = JSON.parse(output); + } catch { + fail(`Parity check on ${accelerator} did not print JSON: ${String(output).trim().slice(0, 200)}`); + } + const values = Array.isArray(parsed) ? parsed : parsed?.values; + if (!Array.isArray(values) || values.length === 0) { + fail(`Parity check on ${accelerator} printed no "values" array.`); + } + if (!values.every((value) => typeof value === 'number' && Number.isFinite(value))) { + // A NaN or an infinity is the classic symptom of a broken accelerator build, so it is reported + // as such rather than being allowed to poison the comparison arithmetic below. + fail(`Parity check on ${accelerator} produced non-finite values.`); + } + return values; +} + +/** Compares two runs: largest absolute and relative difference, and cosine similarity. */ +export function compareValues(reference, candidate) { + if (reference.length !== candidate.length) { + fail(`Parity outputs differ in length: ${reference.length} vs ${candidate.length}.`); + } + let maximumAbsolute = 0; + let maximumRelative = 0; + let dot = 0; + let referenceNorm = 0; + let candidateNorm = 0; + for (const [index, expected] of reference.entries()) { + const actual = candidate[index]; + const absolute = Math.abs(actual - expected); + maximumAbsolute = Math.max(maximumAbsolute, absolute); + // Relative error is meaningless around zero, so it is only counted where the reference has + // magnitude; the absolute bound is what guards the near-zero entries. + if (Math.abs(expected) > 0) maximumRelative = Math.max(maximumRelative, absolute / Math.abs(expected)); + dot += expected * actual; + referenceNorm += expected * expected; + candidateNorm += actual * actual; + } + const norms = Math.sqrt(referenceNorm) * Math.sqrt(candidateNorm); + return { + maximumAbsoluteError: maximumAbsolute, + maximumRelativeError: maximumRelative, + cosineSimilarity: norms > 0 ? dot / norms : 1, + }; +} + +/** Reports which declared tolerance a measurement breaches, or null when it satisfies them all. */ +export function breachedTolerance(measured, tolerances) { + const { absolute = null, relative = null, minimumCosine = null } = tolerances ?? {}; + if (absolute !== null && measured.maximumAbsoluteError > absolute) { + return `maximum absolute error ${measured.maximumAbsoluteError} exceeds ${absolute}`; + } + if (relative !== null && measured.maximumRelativeError > relative) { + return `maximum relative error ${measured.maximumRelativeError} exceeds ${relative}`; + } + if (minimumCosine !== null && measured.cosineSimilarity < minimumCosine) { + return `cosine similarity ${measured.cosineSimilarity} is below ${minimumCosine}`; + } + return null; +} + +/** + * Runs the scroll's parity check across the declared accelerators and enforces its tolerances. + * + * The first accelerator listed is the reference every other run is compared against — conventionally + * `cpu`, because it is the one available everywhere and the least likely to be wrong. Returns the + * measurements so they can be recorded as evidence even when nothing failed. + */ +export async function checkParity({ parity, adapter, interpreter, payloadDir, run }) { + if (!parity) return null; + const { script, accelerators, tolerances } = parity; + if (!Array.isArray(accelerators) || accelerators.length < 2) { + fail('A parity check needs at least two accelerators to compare.'); + } + const runs = []; + for (const accelerator of accelerators) { + const environment = adapter.validationEnvironments[accelerator]; + if (!environment) { + fail(`Target ${adapter.id} defines no validation environment for accelerator ${accelerator}.`); + } + const output = run(interpreter, [script], { cwd: payloadDir, env: environment, capture: true }); + runs.push({ accelerator, values: readValues(output, accelerator) }); + } + const [reference, ...others] = runs; + const comparisons = others.map(({ accelerator, values }) => { + const measured = compareValues(reference.values, values); + const breach = breachedTolerance(measured, tolerances); + if (breach) fail(`Parity check ${reference.accelerator} vs ${accelerator}: ${breach}.`); + return { accelerator, reference: reference.accelerator, ...measured }; + }); + return { script, tolerances, valueCount: reference.values.length, comparisons }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/pixi.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/pixi.mjs new file mode 100644 index 0000000..f7f5b85 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/pixi.mjs @@ -0,0 +1,437 @@ +/** + * pixi + conda-forge builder helpers. + * + * This module owns the deterministic, side-effect-free pieces (tool discovery and exact argument + * vectors) so they can be unit-tested; the orchestration that actually installs and packs a + * prefix lives below in installAndPackPixiEnvironment, composed with an injected runner. + * + * Relocation model: build the env with pixi from the committed pixi.lock, pack it with + * conda-pack, and extract it into the box as `venv/`. conda-pack already rewrites the build + * prefix to a neutral placeholder, and a conda-forge prefix imports and runs from any location + * with **no activation environment and no relocation fixer** (proven cold on macOS and Windows, + * CPU + GPU). So conda-unpack is deliberately never run: doing so would bake the build machine's + * path into the shipped box. A box needs no relocation step at install time. + */ + +import { existsSync } from 'node:fs'; +import { chmod, copyFile, cp, mkdir, readFile, readdir, readlink, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import * as tar from 'tar'; +import { resolvePayloadLinkTarget, targetCarriesLinks } from '../contract/links.mjs'; +import { compareStableStrings, safeRelativePath } from './filesystem.mjs'; +import { fail, runResult as defaultRunResult } from './process.mjs'; +import { repairPosixLaunchers } from './launchers.mjs'; +import { CONDA_PACK_VERSION, toolchainPaths } from './toolchain.mjs'; +import { getWorkspace } from './workspace.mjs'; + +/** + * Resolves a tool, highest precedence first: an explicit path, the environment override, a + * toolchain the project installed for itself, and finally the bare name on PATH. The project-local + * toolchain is looked up rather than configured so that `init --install-toolchain` is enough on its + * own: nothing has to be added to PATH for the next command to find what was just installed. + */ +function toolCandidate({ path, environmentVariable, toolchainKey, name }) { + if (path) return String(path); + const fromEnvironment = process.env[environmentVariable]; + if (fromEnvironment) return String(fromEnvironment); + let installed = null; + try { + installed = toolchainPaths(getWorkspace().toolchainDir)[toolchainKey]; + } catch { + // No resolvable workspace (an unusual cwd, a test): fall through to PATH. + } + return installed && existsSync(installed) ? installed : name; +} + +/** + * Verifies the pinned pixi is installed. `build` and `lock` must use the same pixi the scroll was + * pinned against, never whatever happens to be on PATH: a different resolver version can select + * different packages and silently change the box. + * `runResult` is injectable so a caller can drive discovery without a real pixi on PATH. + * + * @param {{ requiredVersion: string, path?: string | null, runResult?: typeof defaultRunResult }} options + * @returns {string} the executable to invoke + * @throws {Error} when pixi is absent or is not the pinned version + */ +export function findPixi({ requiredVersion, path = null, runResult = defaultRunResult }) { + const found = probePixi({ path, runResult }); + if (!found) { + fail(`pixi ${requiredVersion} is required. Install it from https://pixi.sh/, run \`scrollcase init --install-toolchain\`, or pass --pixi .`); + } + if (found.version !== requiredVersion) fail(`Scroll requires pixi ${requiredVersion}, found ${found.version}.`); + return found.path; +} + +/** + * Reports which pixi is available and at what version, without requiring a particular one. + * + * `findPixi` answers "is the pinned pixi here?"; this answers "is there a pixi at all?", which is + * what `init` needs before it can offer to install one. Returns null when nothing runs. + * + * @param {{ path?: string | null, runResult?: typeof defaultRunResult }} [options] + * @returns {{ path: string, version: string | null } | null} null when nothing runs + */ +export function probePixi({ path = null, runResult = defaultRunResult } = {}) { + const candidate = toolCandidate({ + path, + environmentVariable: 'SCROLLCASE_PIXI', + toolchainKey: 'pixi', + name: 'pixi', + }); + const result = runResult(candidate, ['--version'], { capture: true }); + if (result.error || result.status !== 0) return null; + // `pixi --version` prints "pixi 0.x.y"; the version is the second token. + return { path: candidate, version: String(result.stdout ?? '').trim().split(/\s+/)[1] ?? null }; +} + +/** + * Reports whether conda-pack is available, and where. Returns null when nothing runs. + * + * @param {{ path?: string | null, runResult?: typeof defaultRunResult }} [options] + * @returns {{ path: string } | null} null when nothing runs + */ +export function probeCondaPack({ path = null, runResult = defaultRunResult } = {}) { + const candidate = toolCandidate({ + path, + environmentVariable: 'SCROLLCASE_CONDA_PACK', + toolchainKey: 'condaPack', + name: 'conda-pack', + }); + const result = runResult(candidate, ['--help'], { capture: true }); + return result.error || result.status !== 0 ? null : { path: candidate }; +} + +/** + * `lock` — resolves a scroll's pixi.toml into its committed pixi.lock without installing anything. + * Run by a human when dependencies change; the lock is committed and reviewed, and `build` then + * only installs from it. The manifest itself pins the channels and the single target platform, so + * resolution is host-independent without any per-invocation platform flag. + * + * @param {string} manifestPath + * @returns {string[]} + */ +export function pixiLockArguments(manifestPath) { + return ['lock', '--manifest-path', manifestPath]; +} + +/** + * `build` install — materializes the env from the committed lock, never re-resolving. `--frozen` + * installs exactly the locked packages without touching or re-checking the lock, so what ships is + * byte-for-byte what was reviewed: install-from-lock, never-resolve. + * Lock freshness against the manifest is a separate CI `check` concern, not a build-time resolve. + * + * @param {string} manifestPath + * @returns {string[]} + */ +export function pixiInstallArguments(manifestPath) { + return ['install', '--manifest-path', manifestPath, '--frozen']; +} + +/** + * conda-pack arguments to pack an installed conda prefix into a relocatable tarball. The tarball + * is extracted into the box as `venv/`; the embedded conda-unpack fixer is deliberately removed + * rather than run (see installAndPackPixiEnvironment). + * + * @param {string} prefix + * @param {string} outputPath + * @returns {string[]} + */ +export function condaPackArguments(prefix, outputPath) { + return ['-p', prefix, '-o', outputPath, '--format', 'tar.gz']; +} + +/** + * Verifies conda-pack is available. Its `--version` is unreliable (prints 0.0.0), so we only + * confirm it runs; the exact version pin is recorded elsewhere (via the pixi global manifest). + * + * @param {{ path?: string | null, runResult?: typeof defaultRunResult }} [options] + * @returns {string} the executable to invoke + * @throws {Error} when conda-pack is absent + */ +export function findCondaPack({ path = null, runResult = defaultRunResult } = {}) { + const found = probeCondaPack({ path, runResult }); + if (!found) { + fail(`conda-pack ${CONDA_PACK_VERSION} is required. Install it with \`scrollcase init --install-toolchain\` or \`pixi global install "conda-pack==${CONDA_PACK_VERSION}"\`, or pass --conda-pack .`); + } + return found.path; +} + +/** + * The only fields a box keeps from conda's per-package records: which exact binary this is, and + * what it is licensed under. Everything else is dropped. + * + * `build` earns its place because name and version do not identify a conda binary — one version is + * published in many builds, and a CPU and a CUDA build of the same library can differ in nothing + * else. All four are properties of the package as published rather than of the install that placed + * it, which is what makes them stable across rebuilds: `license` here was measured equal to the + * lock's declared licence for every package, and the lock is already the source the shipped licence + * inventory is derived from. A package declaring no licence simply has no such field, which is + * equally a property of the package and not of the run. + * + * The dependency graph is deliberately not here: nothing resolves dependencies inside a box, and + * the lock records them where they can actually be acted on. + */ +const CONDA_RECORD_FIELDS = Object.freeze([ + 'name', + 'version', + 'build', + 'license', +]); + +/** + * Rewrites `conda-meta/` into a canonical, build-independent form. + * + * These records are written by the installer, not by the package, and two installs of the identical + * lock do not produce identical ones: a per-file `sha256_in_prefix` is recorded on one run and not + * the next. They also carry absolute paths into the build machine's package cache. The first breaks + * the promise the whole trust chain rests on — rebuild a commit, get the same bytes — and the second + * ships a developer's directory layout to users, which is the very leak conda-unpack is refused + * over. Anything that is not a record (conda's `history` log) goes entirely. + * + * Nothing in a box reads any of this: conda is never shipped inside one, and package versions stay + * readable from `site-packages` where a Python tool actually looks. So the kept fields are copied + * verbatim and the rule is an allowlist rather than a list of known-volatile fields — deliberately, + * because a field pixi starts writing in a later release then cannot reintroduce the drift. It was + * never eligible to be written in the first place. + */ +async function canonicalizeCondaRecords(venvDir) { + const metaDir = join(venvDir, 'conda-meta'); + if (!existsSync(metaDir)) return; + for (const entry of (await readdir(metaDir)).sort(compareStableStrings)) { + const entryPath = join(metaDir, entry); + if (!entry.endsWith('.json')) { + await rm(entryPath, { recursive: true, force: true }); + continue; + } + let record; + try { + record = JSON.parse(await readFile(entryPath, 'utf8')); + } catch (error) { + return fail(`Unreadable conda package record ${entry}: ${error instanceof Error ? error.message : String(error)}`); + } + const canonical = {}; + for (const field of CONDA_RECORD_FIELDS) { + if (record[field] !== undefined) canonical[field] = record[field]; + } + await writeFile(entryPath, `${JSON.stringify(canonical, null, 2)}\n`); + } +} + +/** + * Settles every symbolic link under `root`: kept when the payload may carry it, materialized into + * real content when it may not, dropped when it points nowhere useful. + * + * A conda prefix is dense with links, and materializing all of them was expensive in a way nobody + * had measured: on Linux the soname convention alone (`libfoo.so` → `.so.N` → `.so.N.M`) meant + * roughly 60% of an extracted box was duplicates of its own bytes. What may be kept is decided by + * `src/contract/links.mjs`, not here — this function only supplies the filesystem facts that rule + * needs and applies its answer. + * + * Two conditions are checked against the disk rather than the path string, because only the disk + * knows them: that the link resolves inside the prefix even after passing through other links, and + * that what it ends at is a regular file. A link to a directory is materialized, which is what + * keeps anything from ever being written *through* a link. + * + * @param {string} root + * @param {boolean} keepLinks whether this target can extract links at all + * @param {string} [current] + * @returns {Promise} + */ +async function settleSymlinksInPlace(root, keepLinks, current = root) { + const canonicalRoot = await realpath(root); + // Sorted, because whether a link is kept can depend on what an earlier entry became, and + // readdir order is the filesystem's business. Two builds must settle the tree identically. + const children = (await readdir(current, { withFileTypes: true })) + .sort((left, right) => compareStableStrings(left.name, right.name)); + for (const entry of children) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) { + let target; + try { + target = await realpath(path); + } catch { + await rm(path, { force: true }); // dangling link + continue; + } + const insideTree = target === canonicalRoot + || target.startsWith(`${canonicalRoot}${sep}`); + let info; + try { + info = await stat(target); + } catch { + await rm(path, { force: true }); + continue; + } + if (!insideTree) { + // A link escaping the prefix would drag a host file into the box; drop it instead. + await rm(path, { force: true }); + continue; + } + if (keepLinks && !info.isDirectory() && await keepsAsLink(root, path, canonicalRoot)) continue; + await rm(path, { force: true }); + if (info.isDirectory()) { + await cp(target, path, { recursive: true, dereference: true }); + await settleSymlinksInPlace(root, keepLinks, path); + } else { + await copyFile(target, path); + await chmod(path, info.mode & 0o777); + } + } else if (entry.isDirectory()) { + await settleSymlinksInPlace(root, keepLinks, path); + } + } +} + +/** + * Whether one link satisfies the payload rule, judged against the tree it actually sits in. + * + * The raw target is what gets archived, so it is what must be checked: an absolute target names the + * build machine and is exactly what relocation exists to erase, and a relative one has to land + * inside the prefix both lexically and after the filesystem has followed it. + * + * @param {string} root + * @param {string} linkPath + * @param {string} canonicalRoot + * @returns {Promise} + */ +async function keepsAsLink(root, linkPath, canonicalRoot) { + const rawTarget = (await readlink(linkPath)).split(sep).join('/'); + const relativeLink = relative(root, linkPath).split(sep).join('/'); + const resolved = resolvePayloadLinkTarget(relativeLink, rawTarget); + if (resolved === null) return false; + // The lexical answer and the filesystem's answer must agree. They can differ when the target is + // reached through another link, which is precisely the case a purely lexical check cannot see. + const lexicalPath = join(root, ...resolved.split('/')); + let lexicalReal; + try { + lexicalReal = await realpath(lexicalPath); + } catch { + return false; + } + if (lexicalReal !== canonicalRoot && !lexicalReal.startsWith(`${canonicalRoot}${sep}`)) return false; + return (await stat(lexicalPath)).isFile(); +} + +/** + * Builds the box's `venv/` prefix from a scroll's committed pixi.lock and packs it for relocation. + * + * Flow: install the exact locked env into an isolated workspace so pixi's `.pixi/envs` never + * lands in the tracked scroll dir; conda-pack the prefix into a relocatable tarball; extract it + * into `payloadDir/venv`; remove the service files that carry the build prefix (conda-unpack is + * never run — see below); then dereference every symlink so the payload is link-free for the + * archive layer. The multi-gigabyte workspace and tarball are removed before the payload is + * archived. + * + * `run` is injected so this composes with the orchestrator's logging and error model. + * + * @param {{ + * pixi: string, + * condaPack: string, + * manifestPath: string, + * lockPath: string, + * buildDir: string, + * payloadDir: string, + * adapter: import('../contract/targets.mjs').BoxTargetAdapter, + * run: typeof import('./process.mjs').run, + * }} options + * @returns {Promise<{ interpreter: string, prefix: string }>} + */ +export async function installAndPackPixiEnvironment({ + pixi, + condaPack, + manifestPath, + lockPath, + buildDir, + payloadDir, + adapter, + run, +}) { + const workspace = join(buildDir, 'pixi-workspace'); + await rm(workspace, { recursive: true, force: true }); + await mkdir(workspace, { recursive: true }); + // pixi installs from the manifest+lock sitting next to each other; stage both into the workspace + // so the resulting `.pixi/envs/default` prefix is build-local, never inside the scroll. + await copyFile(manifestPath, join(workspace, 'pixi.toml')); + await copyFile(lockPath, join(workspace, 'pixi.lock')); + run(pixi, pixiInstallArguments(join(workspace, 'pixi.toml'))); + const prefix = join(workspace, '.pixi', 'envs', 'default'); + + const packPath = join(buildDir, 'pixi-env.tar.gz'); + await rm(packPath, { force: true }); + run(condaPack, condaPackArguments(prefix, packPath)); + + const venvDir = join(payloadDir, 'venv'); + await rm(venvDir, { recursive: true, force: true }); + await mkdir(venvDir, { recursive: true }); + // conda-pack emits the prefix contents at the tar root, so extracting into `venv` yields the + // conda layout (bin/, lib/, conda-meta/) directly under it. Use the pinned Node implementation + // rather than a host `tar`: builds then have exactly the dependencies `doctor` reports, and the + // archive behaves the same on macOS, Linux and Windows. Symlinks are expected in a conda prefix + // and are deliberately handled by dereferenceSymlinksInPlace immediately below. + // + // They cannot, however, be created *during* extraction. The extractor refuses a link whose target + // leaves the tree, and refuses one whose target passes through another link — both are defences + // against writing file content through a link, and neither is negotiable. A conda prefix trips + // the second routinely: icu ships `current -> ` and then `pkgdata.inc -> + // current/pkgdata.inc`, which arrives in a plain `python` environment that never asked for icu, + // and made the whole box unbuildable. + // + // So links are extracted in a second pass, once every regular entry is already on disk and there + // is nothing left that could be written through one. Creating a link is not traversing it: the + // targets are resolved and checked immediately below, where anything leaving the tree is dropped. + const deferredLinks = []; + await tar.x({ + file: packPath, + cwd: venvDir, + gzip: true, + preservePaths: false, + strict: true, + filter: (entryPath, entry) => { + if (entry.type !== 'SymbolicLink') return true; + deferredLinks.push({ path: safeRelativePath(entryPath), target: String(entry.linkpath) }); + return false; + }, + }); + // Sorted so the tree is built the same way whatever order the tar happened to list them in. + for (const link of deferredLinks.sort((left, right) => compareStableStrings(left.path, right.path))) { + const linkPath = join(venvDir, ...link.path.split('/')); + // A regular entry already holding the path wins: content beats an alias to it. + if (existsSync(linkPath)) continue; + await mkdir(dirname(linkPath), { recursive: true }); + // The type argument is inert on POSIX and decides junction-vs-file on Windows, where a conda + // prefix carries no links at all — so a target that is not yet a directory is simply a file. + const resolved = resolve(dirname(linkPath), link.target); + const type = existsSync(resolved) && (await stat(resolved)).isDirectory() ? 'dir' : 'file'; + await symlink(link.target, linkPath, type); + } + + const interpreter = join(payloadDir, ...adapter.python.entryPoint.split('/')); + // Deliberately do NOT run conda-unpack. conda-pack already replaces the build prefix with a + // neutral placeholder, and the box imports and runs fine that way (a cold import from a moved + // prefix was proven before any fixer). Running the fixer here would stamp the *build machine's* + // absolute path into dozens of files that then ship to users — measured on a probe env: 0 files + // carry the prefix before, 36 after — leaking a developer path while still being wrong at the + // user's install location. Instead drop the few service files that do carry the build prefix. + for (const servicePath of [ + ['conda-meta', 'pixi_env_prefix'], + ['conda-meta', 'pixi'], + ['bin', 'conda-unpack'], + ['Scripts', 'conda-unpack.exe'], + ['Scripts', 'conda-unpack-script.py'], + ]) { + await rm(join(venvDir, ...servicePath), { force: true }); + } + // The rest of conda-meta carries the same two problems in a less obvious form; see above. + await canonicalizeCondaRecords(venvDir); + // Order matters: settle the links first, so the launcher repair that follows walks a tree whose + // shape is final and rewrites each script's bytes exactly once, under its own name. + await settleSymlinksInPlace(venvDir, targetCarriesLinks(adapter.platform)); + // conda console scripts (tqdm, isympy, …) embed the absolute build interpreter in a shell + // trampoline shebang. Rewrite them to resolve Python next to themselves, so no build path + // ships inside the box. + await repairPosixLaunchers(adapter, payloadDir, [prefix, workspace, payloadDir]); + + await rm(workspace, { recursive: true, force: true }); + await rm(packPath, { force: true }); + return { interpreter, venvDir, sitePackagesRelative: relative(payloadDir, venvDir) }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/process.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/process.mjs new file mode 100644 index 0000000..0c9e560 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/process.mjs @@ -0,0 +1,66 @@ +/** + * Error and subprocess primitives for the whole tool. + * + * Every validation failure funnels through `fail`, and every external command through `run` / + * `runResult` — which is also the seam the tests use: injecting a fake runner is how the pipeline + * suite builds boxes without pixi or conda-pack installed. + */ +import { spawnSync } from 'node:child_process'; + +/** + * Subprocess options shared by the library surface and its injected test seams. + * + * @typedef {object} RunOptions + * @property {string} [cwd] + * @property {NodeJS.ProcessEnv} [env] + * @property {string | Uint8Array} [input] + * @property {number} [maxBuffer] + * @property {boolean} [capture] + */ + +/** + * Throws a consistent CLI error from validation helpers. + * + * @param {unknown} message + * @returns {never} + */ +export function fail(message) { + throw new Error(message); +} + +/** + * Runs a subprocess without interpreting its result. + * + * @param {string} command + * @param {readonly string[]} args + * @param {RunOptions} [options] + * @returns {import('node:child_process').SpawnSyncReturns} + */ +export function runResult(command, args, options = {}) { + return spawnSync(command, args, { + cwd: options.cwd, + env: { ...process.env, ...options.env }, + encoding: 'utf8', + input: options.input, + maxBuffer: options.maxBuffer ?? 64 * 1024 * 1024, + stdio: options.capture ? 'pipe' : ['pipe', 'inherit', 'inherit'], + }); +} + +/** + * Runs a subprocess and throws when it cannot start or exits unsuccessfully. + * + * @param {string} command + * @param {readonly string[]} args + * @param {RunOptions} [options] + * @returns {string} + */ +export function run(command, args, options = {}) { + const result = runResult(command, args, options); + if (result.error) fail(`${command} failed to start: ${result.error.message}`); + if (result.status !== 0) { + const detail = options.capture ? `\n${result.stderr || result.stdout}` : ''; + fail(`${command} exited with status ${result.status}${detail}`); + } + return (result.stdout ?? '').trim(); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/project.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/project.mjs new file mode 100644 index 0000000..5770678 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/project.mjs @@ -0,0 +1,276 @@ +/** + * Setting a project up, and telling it what is wrong. + * + * `initProject` scaffolds only the workspace; the CLI may compose it with the explicitly + * disposable example used for onboarding. `doctor` inspects and never writes. Real scroll + * authoring remains a separate operation because a workspace may carry many boxes and targets. + * + * `init` may also install the build toolchain, but only after asking: scaffolding never reaches for + * the network on its own, and the download is verified against a pinned checksum. See + * `ensureToolchain` below and `toolchain.mjs` for why the consent and the pin are the design rather + * than a nicety. + */ + +import { readFile, mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fileExists } from './filesystem.mjs'; +import { findCondaPack, findPixi, probeCondaPack, probePixi } from './pixi.mjs'; +import { fail, run as defaultRun, runResult as defaultRunResult } from './process.mjs'; +import { + CONDA_PACK_VERSION, + installCondaPack, + installPixi, + latestPixiVersion, + pixiReleaseAsset, +} from './toolchain.mjs'; +import { DEFAULT_WORKSPACE_PATHS, SCROLLCASE_CONFIG_FILENAME } from './workspace.mjs'; + +// Written into a project's .gitignore and matched on re-run to stay idempotent. Changing the text +// makes an already-scaffolded project look unmarked and append the rules a second time. +const GITIGNORE_MARKER = '# scrollcase build state'; + +const PROJECT_GUIDE = `[Scrollcase documentation](https://scrollcase.dev/) + +# Scrollcase in this project + +Scrollcase turns a declarative [scroll](https://scrollcase.dev/reference/scroll) into a signed, +portable [box](https://scrollcase.dev/reference/box-format) for one [target](https://scrollcase.dev/reference/box-format#targets). + +## Usual workflow + +Run \`npm install scrollcase\` to install Scrollcase CLI. Then: + +1. \`scrollcase init\` +2. \`scrollcase lock /\` +3. \`scrollcase keygen\` +4. \`scrollcase build /\` +5. \`scrollcase verify --self-test\` or \`scrollcase run \` + +See the [CLI reference](https://scrollcase.dev/reference/cli) and +[signing guidance](https://scrollcase.dev/guides/signing-and-custody). The \`consumer-templates/\` +files demonstrate the [consumer APIs](https://scrollcase.dev/reference/api) against local releases. + +## Node consumer + +\`\`\`sh +npm install scrollcase +npm install --save-dev tsx typescript +npx tsx consumer-templates/run-box.ts +\`\`\` + +## Python consumer + +npm does not install the Python consumer. A Python-only application does not need the Node CLI: + +\`\`\`sh +python -m pip install scrollcase-consumer +python consumer-templates/run_box.py +\`\`\` + +[Scrollcase documentation](https://scrollcase.dev/) +`; + +/** + * Scaffolds a workspace config, concise project guide, scroll root, and generated-state ignores. + * + * This low-level primitive deliberately creates no scroll. Existing files are never overwritten, + * so a half-configured workspace can be completed by running the command again without changing + * authored inputs. + */ +export async function initProject({ + root, + scrollsDir = join(root, DEFAULT_WORKSPACE_PATHS.scrolls), +}) { + const written = []; + const skipped = []; + const write = async (path, contents) => { + if (await fileExists(path)) return skipped.push(path); + await mkdir(join(path, '..'), { recursive: true }); + await writeFile(path, contents); + return written.push(path); + }; + + await write(join(root, SCROLLCASE_CONFIG_FILENAME), `${JSON.stringify({ + version: 1, + paths: { ...DEFAULT_WORKSPACE_PATHS }, + }, null, 2)}\n`); + await write(join(root, 'SCROLLCASE.md'), PROJECT_GUIDE); + + if (await fileExists(scrollsDir)) skipped.push(scrollsDir); + else { + await mkdir(scrollsDir, { recursive: true }); + written.push(scrollsDir); + } + + // Build state is regenerated on every build and must never be committed; the lock and the scroll + // must be. Appending rather than rewriting leaves an existing .gitignore alone. + const gitignorePath = join(root, '.gitignore'); + const existing = await fileExists(gitignorePath) ? await readFile(gitignorePath, 'utf8') : ''; + if (!existing.includes(GITIGNORE_MARKER)) { + const rules = `${existing.endsWith('\n') || existing === '' ? '' : '\n'}${GITIGNORE_MARKER}\n.scrollcase/\n`; + await writeFile(gitignorePath, `${existing}${rules}`); + written.push(gitignorePath); + } else { + skipped.push(gitignorePath); + } + return { + written, + skipped, + root, + scrollsDir, + }; +} + +/** Reads the project config back, so a toolchain pin is added to it rather than replacing it. */ +async function readConfig(configPath) { + if (!await fileExists(configPath)) return { version: 1, paths: { ...DEFAULT_WORKSPACE_PATHS } }; + return JSON.parse(await readFile(configPath, 'utf8')); +} + +/** + * Installs the build toolchain into the project, if it is missing and only if allowed. + * + * `confirm` is the consent, injected rather than assumed: the CLI asks a human, a scripted setup + * passes a flag, and CI without a terminal answers no. Nothing is downloaded before it returns + * true, which is what keeps `init` a command that is always safe to run. + * + * The pixi version is the caller's requested pin, the installed pixi's version when one is already + * present, and otherwise the newest release. Managed installs record that choice and the verified + * archive digest in the workspace config; each scroll separately declares which resolver it uses. + * + * The archive's verified digest and managed conda-pack version are recorded under `toolchain` in + * the project config. The first pixi install trusts the checksum published beside the release; + * every later one is checked against the value the project committed, so a teammate or a CI runner + * cannot silently receive different bytes. + */ +export async function ensureToolchain({ + workspace, + pixiVersion = null, + confirm, + host = process, + fetchImpl = fetch, + run = defaultRun, + runResult = defaultRunResult, + log = console.log, +}) { + const discoveredPixi = probePixi({ runResult }); + // A present but different pixi is still missing for this project: resolver versions are part of + // the scroll's reproducibility contract, so `init --pixi-version` must install what it promises. + const pixi = discoveredPixi && (!pixiVersion || discoveredPixi.version === pixiVersion) + ? discoveredPixi + : null; + const condaPack = probeCondaPack({ runResult }); + const missing = [!pixi && 'pixi', !condaPack && 'conda-pack'].filter(Boolean); + if (missing.length === 0) { + return { + installed: [], + missing: [], + pixiVersion: pixi.version, + condaPackVersion: CONDA_PACK_VERSION, + declined: false, + }; + } + if (!pixiReleaseAsset(host)) { + return { installed: [], missing, declined: false, unsupportedHost: `${host.platform}/${host.arch}` }; + } + if (!await confirm(missing)) return { installed: [], missing, declined: true }; + + const configPath = join(workspace.root, SCROLLCASE_CONFIG_FILENAME); + const config = await readConfig(configPath); + const installed = []; + let pixiPath = pixi?.path ?? null; + let version = pixiVersion ?? pixi?.version ?? null; + + if (!pixi) { + if (!version) { + version = await latestPixiVersion({ fetchImpl }); + log(`Newest pixi release is ${version}; recording it for the workspace toolchain.`); + } + const pinned = config.toolchain?.pixi?.version === version + ? config.toolchain?.pixi?.assets?.[pixiReleaseAsset(host).asset] ?? null + : null; + const result = await installPixi({ + version, + toolchainDir: workspace.toolchainDir, + expectedSha256: pinned, + host, + fetchImpl, + log, + }); + pixiPath = result.path; + installed.push(`pixi ${version}`); + // Record the digest that was actually verified, so the next machine checks against it. + config.toolchain = { + ...config.toolchain, + pixi: { + version, + assets: { ...config.toolchain?.pixi?.assets, [result.asset]: result.sha256 }, + }, + }; + await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`); + } + + if (!condaPack) { + if (!pixiPath) fail('conda-pack is installed with pixi, but no pixi is available.'); + await installCondaPack({ pixi: pixiPath, toolchainDir: workspace.toolchainDir, run, log }); + installed.push(`conda-pack ${CONDA_PACK_VERSION}`); + config.toolchain = { + ...config.toolchain, + condaPack: { version: CONDA_PACK_VERSION }, + }; + await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`); + } + + return { + installed, + missing: [], + declined: false, + pixiVersion: version, + condaPackVersion: CONDA_PACK_VERSION, + configPath, + }; +} + +/** + * Diagnoses whether this machine can build a box, and says what to do when it cannot. + * + * Every check reports rather than throws, so one missing tool does not hide the next problem: a + * user with neither pixi nor conda-pack should learn both in one run, not one per attempt. + */ +export async function diagnose({ workspace, pixiVersion = null, pixiPath = null, condaPackPath = null, runResult = defaultRunResult }) { + const checks = []; + const record = (name, ok, detail, remedy = null) => checks.push({ name, ok, detail, remedy }); + + record('workspace', true, workspace.configPath + ? `config ${workspace.configPath}` + : `no ${SCROLLCASE_CONFIG_FILENAME} found; using defaults under ${workspace.root}`); + record('scrolls', await fileExists(workspace.scrollsDir), workspace.scrollsDir, + `Create it, or point "paths.scrolls" at where your scrolls live.`); + + const git = runResult('git', ['rev-parse', 'HEAD'], { capture: true, cwd: workspace.root }); + record('git', git.status === 0, + git.status === 0 ? `HEAD ${git.stdout.trim().slice(0, 12)}` : 'not a git checkout', + 'A box records the commit it was built from. Initialise a repository and commit your scrolls.'); + + if (pixiVersion) { + try { + const pixi = findPixi({ requiredVersion: pixiVersion, path: pixiPath, runResult }); + record('pixi', true, `${pixi} at ${pixiVersion}`); + } catch (error) { + record('pixi', false, error.message, + `Install pixi ${pixiVersion} from https://pixi.sh/, or pass --pixi .`); + } + } else { + record('pixi', true, 'not checked: pass --pixi-version, or run doctor with a scroll'); + } + + try { + const condaPack = findCondaPack({ path: condaPackPath, runResult }); + record('conda-pack', true, condaPack); + } catch (error) { + record('conda-pack', false, error.message, + `Install conda-pack ${CONDA_PACK_VERSION} with \`scrollcase init --install-toolchain\`, or pass --conda-pack .`); + } + + return { checks, ok: checks.every((check) => check.ok) }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/schema-validation.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/schema-validation.mjs new file mode 100644 index 0000000..5821cd3 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/schema-validation.mjs @@ -0,0 +1,194 @@ +/** + * Runtime validation for the JSON Schemas Scrollcase ships. + * + * Ajv remains a development dependency because adding a fourth runtime package would widen the + * installed surface for one narrow job. This validator implements the 2020-12 keywords used by the + * scroll and target schemas, including local and absolute references and target conditionals. The + * schemas remain the source of truth; this module deliberately contains no scroll field list. + */ + +import { isDeepStrictEqual } from 'node:util'; + +const own = (value, key) => Object.prototype.hasOwnProperty.call(value, key); +const objectValue = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +function valueType(value) { + if (Array.isArray(value)) return 'array'; + if (value === null) return 'null'; + if (Number.isInteger(value)) return 'integer'; + return typeof value; +} + +function matchesType(value, expected) { + if (expected === 'object') return objectValue(value); + if (expected === 'array') return Array.isArray(value); + if (expected === 'integer') return Number.isInteger(value); + if (expected === 'number') return typeof value === 'number' && Number.isFinite(value); + return typeof value === expected; +} + +function decodePointerPart(part) { + return decodeURIComponent(part).replaceAll('~1', '/').replaceAll('~0', '~'); +} + +function resolveReference(reference, rootSchema, registry) { + const [id, fragment = ''] = reference.split('#', 2); + const document = id ? registry.get(id) : rootSchema; + if (!document) throw new Error(`Schema reference is not registered: ${reference}`); + if (!fragment) return { schema: document, rootSchema: document }; + if (!fragment.startsWith('/')) throw new Error(`Unsupported schema fragment: #${fragment}`); + const schema = fragment.slice(1).split('/').map(decodePointerPart) + .reduce((current, part) => current?.[part], document); + if (!schema) throw new Error(`Schema reference does not resolve: ${reference}`); + return { schema, rootSchema: document }; +} + +function validate(value, schema, context, path, errors) { + if (schema.$ref) { + const resolved = resolveReference(schema.$ref, context.rootSchema, context.registry); + validate(value, resolved.schema, { ...context, rootSchema: resolved.rootSchema }, path, errors); + if (errors.length > 0) return; + } + + if (schema.const !== undefined && !isDeepStrictEqual(value, schema.const)) { + errors.push(`${path} must equal ${JSON.stringify(schema.const)}`); + return; + } + if (schema.enum && !schema.enum.some((candidate) => isDeepStrictEqual(value, candidate))) { + errors.push(`${path} must be one of ${schema.enum.map((item) => JSON.stringify(item)).join(', ')}`); + return; + } + if (schema.type && !matchesType(value, schema.type)) { + errors.push(`${path} must be ${schema.type}, received ${valueType(value)}`); + return; + } + + if (typeof value === 'string') { + if (schema.minLength !== undefined && value.length < schema.minLength) { + errors.push(`${path} must contain at least ${schema.minLength} character${schema.minLength === 1 ? '' : 's'}`); + return; + } + if (schema.pattern && !new RegExp(schema.pattern, 'u').test(value)) { + errors.push(`${path} does not match the required pattern`); + return; + } + } + + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + errors.push(`${path} must be finite`); + return; + } + if (schema.minimum !== undefined && value < schema.minimum) { + errors.push(`${path} must be at least ${schema.minimum}`); + return; + } + if (schema.exclusiveMinimum !== undefined && value <= schema.exclusiveMinimum) { + errors.push(`${path} must be greater than ${schema.exclusiveMinimum}`); + return; + } + if (schema.maximum !== undefined && value > schema.maximum) { + errors.push(`${path} must be at most ${schema.maximum}`); + return; + } + } + + if (Array.isArray(value)) { + if (schema.minItems !== undefined && value.length < schema.minItems) { + errors.push(`${path} must contain at least ${schema.minItems} item${schema.minItems === 1 ? '' : 's'}`); + return; + } + if (schema.items) { + for (let index = 0; index < value.length && errors.length === 0; index += 1) { + validate(value[index], schema.items, context, `${path}[${index}]`, errors); + } + } + } + + if (objectValue(value)) { + if (schema.minProperties !== undefined && Object.keys(value).length < schema.minProperties) { + errors.push(`${path} must contain at least ${schema.minProperties} property`); + return; + } + for (const required of schema.required ?? []) { + if (!own(value, required)) { + errors.push(`${path}.${required} is required`); + return; + } + } + for (const [key, propertySchema] of Object.entries(schema.properties ?? {})) { + if (own(value, key)) validate(value[key], propertySchema, context, `${path}.${key}`, errors); + if (errors.length > 0) return; + } + if (schema.additionalProperties === false) { + const allowed = new Set(Object.keys(schema.properties ?? {})); + const unexpected = Object.keys(value).find((key) => !allowed.has(key)); + if (unexpected) { + errors.push(`${path}.${unexpected} is not allowed`); + return; + } + } + for (const [key, dependencies] of Object.entries(schema.dependentRequired ?? {})) { + if (!own(value, key)) continue; + const missing = dependencies.find((dependency) => !own(value, dependency)); + if (missing) { + errors.push(`${path}.${missing} is required when ${key} is present`); + return; + } + } + } + + for (const branch of schema.allOf ?? []) { + validate(value, branch, context, path, errors); + if (errors.length > 0) return; + } + + if (schema.oneOf) { + const results = schema.oneOf.map((branch) => { + const branchErrors = []; + validate(value, branch, context, path, branchErrors); + return branchErrors; + }); + const matches = results.filter((branchErrors) => branchErrors.length === 0); + if (matches.length === 0) { + const closest = results.reduce((best, candidate) => + candidate.length < best.length ? candidate : best); + errors.push(closest[0] ?? `${path} does not match an allowed shape`); + return; + } + if (matches.length > 1) { + errors.push(`${path} must match exactly one allowed shape`); + return; + } + } + + if (schema.if) { + const conditionErrors = []; + validate(value, schema.if, context, path, conditionErrors); + const branch = conditionErrors.length === 0 ? schema.then : schema.else; + if (branch) validate(value, branch, context, path, errors); + } + + if (schema.not) { + const forbiddenErrors = []; + validate(value, schema.not, context, path, forbiddenErrors); + if (forbiddenErrors.length === 0) errors.push(`${path} matches a forbidden shape`); + } +} + +/** + * Returns the first structural disagreement with a schema, or null when the value matches. + * + * @param {unknown} value + * @param {object} schema + * @param {object[]} [relatedSchemas] + * @returns {string | null} + */ +export function schemaValidationError(value, schema, relatedSchemas = []) { + const registry = new Map([schema, ...relatedSchemas] + .filter((candidate) => candidate.$id) + .map((candidate) => [candidate.$id, candidate])); + const errors = []; + validate(value, schema, { registry, rootSchema: schema }, '$', errors); + return errors[0] ?? null; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/scroll.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/scroll.mjs new file mode 100644 index 0000000..4a1ece7 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/scroll.mjs @@ -0,0 +1,200 @@ +/** + * Reading a scroll, and the provenance of the build that reads it. + * + * A scroll is the only input a build accepts, so it is validated before anything is installed. In + * the nested layout, the meaningful declarations police the path: `boxId` names the parent and the + * canonical target names the child. Python layout is checked against the target before the scroll + * reaches any tool discovery or build mutation. + */ + +import { readFile, readdir } from 'node:fs/promises'; +import { join, resolve, sep } from 'node:path'; +import { assertPythonEntryPoint, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; +import { compareStableStrings, fileExists, safeRelativePath } from './filesystem.mjs'; +import { fail, runResult } from './process.mjs'; +import { schemaValidationError } from './schema-validation.mjs'; +import { getWorkspace } from './workspace.mjs'; + +const scrollSchemaUrl = new URL('../contract/schema/scroll.schema.json', import.meta.url); +const targetSchemaUrl = new URL('../contract/schema/target.schema.json', import.meta.url); +const executionSchemaUrl = new URL('../contract/schema/execution.schema.json', import.meta.url); +let scrollSchemas; + +async function loadScrollSchemas() { + scrollSchemas ??= Promise.all([scrollSchemaUrl, targetSchemaUrl, executionSchemaUrl] + .map(async (url) => JSON.parse(await readFile(url, 'utf8')))); + return scrollSchemas; +} + +/** Resolves an exact scroll reference to its directory, refusing anything outside the scrolls root. */ +export function scrollDirectory(reference) { + const root = getWorkspace().scrollsDir; + const normalized = safeRelativePath(reference); + const path = resolve(root, ...normalized.split('/')); + if (path === root || !path.startsWith(`${root}${sep}`)) fail(`Invalid scroll: ${reference}`); + return path; +} + +/** Loads one exact nested scroll reference and normalises its provenance identity. */ +async function readExactScroll(reference) { + const normalized = safeRelativePath(reference); + const parts = normalized.split('/'); + if (parts.length !== 2) fail(`Invalid scroll reference ${reference}; use /.`); + const dir = scrollDirectory(normalized); + const scroll = JSON.parse(await readFile(resolve(dir, 'scroll.json'), 'utf8')); + const [scrollSchema, targetSchema, executionSchema] = await loadScrollSchemas(); + const validationError = schemaValidationError(scroll, scrollSchema, [targetSchema, executionSchema]); + if (validationError) fail(`Invalid scroll ${normalized}: ${validationError}.`); + if (scroll.weights === 'on-demand' && (scroll.assetArchives ?? []).length > 0) { + fail('on-demand weights cannot be combined with assetArchives, which are expanded at build time.'); + } + const payloadPaths = [ + scroll.modelCacheSubdir, + ...scroll.assets.map((asset) => asset.relativePath), + ...(scroll.assetArchives ?? []).flatMap((archive) => [archive.relativePath, archive.destination]), + ...(scroll.localFiles ?? []).flatMap((file) => [file.sourcePath, file.relativePath]), + ...(scroll.prunePaths ?? []), + ...scroll.selfTest.files, + ...(scroll.execution?.kind === 'python-script' ? [scroll.execution.script] : []), + ...(scroll.parity ? [scroll.parity.script] : []), + ...(scroll.condaDependencyLicenseAudit ? [scroll.condaDependencyLicenseAudit] : []), + ]; + for (const path of payloadPaths) safeRelativePath(path); + const adapter = boxTargetAdapter(scroll.target); + const targetId = boxTargetId(scroll.target); + if (parts.length === 2) { + const [boxDirectory, targetDirectory] = parts; + if (boxDirectory !== scroll.boxId) { + fail(`Nested scroll box directory ${boxDirectory} does not match scroll boxId ${scroll.boxId}.`); + } + if (targetDirectory !== targetId) { + fail(`Nested scroll target directory ${targetDirectory} does not match declared target ${targetId}.`); + } + } + assertPythonEntryPoint(adapter, scroll.pythonEntryPoint); + return { + adapter, + dir, + scroll: { + ...scroll, + // Provenance needs a stable source identity. It is derived when the scroll does not name one, + // so the directory layout remains checked context rather than a second wire identity. + scrollId: scroll.scrollId ?? `${scroll.boxId}-${targetId}`, + }, + reference: normalized, + targetId, + }; +} + +/** + * Lists the scrolls named by a CLI/library reference. + * + * An exact `/` reference loads one scroll. A single box name expands to its + * `scrolls///` children. Omitting the name discovers every nested scroll in the + * workspace for CLI selection. Every child is validated before it is offered, so a misleading + * directory never becomes a selectable target. + */ +export async function scrollCandidates(name = null) { + if (name === null || name === undefined) { + let boxes; + try { + boxes = await readdir(getWorkspace().scrollsDir, { withFileTypes: true }); + } catch { + return fail('No scrolls found; run scrollcase init or scrollcase new scroll.'); + } + const candidates = []; + for (const box of boxes.sort((left, right) => compareStableStrings(left.name, right.name))) { + if (!box.isDirectory()) continue; + let targets; + try { + targets = await readdir(scrollDirectory(box.name), { withFileTypes: true }); + } catch { + continue; + } + for (const target of targets.sort((left, right) => + compareStableStrings(left.name, right.name))) { + if (!target.isDirectory()) continue; + const nestedReference = `${box.name}/${target.name}`; + if (await fileExists(join(scrollDirectory(nestedReference), 'scroll.json'))) { + candidates.push(await readExactScroll(nestedReference)); + } + } + } + if (candidates.length === 0) { + fail('No scrolls found; run scrollcase init or scrollcase new scroll.'); + } + return candidates; + } + + const reference = safeRelativePath(name); + if (reference.includes('/')) { + if (reference.split('/').length !== 2 + || !await fileExists(join(scrollDirectory(reference), 'scroll.json'))) { + fail(`Scroll not found: ${reference}.`); + } + return [await readExactScroll(reference)]; + } + + let entries; + try { + entries = await readdir(scrollDirectory(reference), { withFileTypes: true }); + } catch { + return fail(`Scroll or box not found: ${reference}.`); + } + const candidates = []; + for (const entry of entries.sort((left, right) => compareStableStrings(left.name, right.name))) { + if (!entry.isDirectory()) continue; + const nestedReference = `${reference}/${entry.name}`; + if (await fileExists(join(scrollDirectory(nestedReference), 'scroll.json'))) { + candidates.push(await readExactScroll(nestedReference)); + } + } + if (candidates.length === 0) fail(`Box ${reference} contains no target scrolls.`); + return candidates; +} + +/** + * Loads a scroll without prompting. + * + * Library callers may select a target explicitly. An unambiguous box shorthand is also accepted; + * ambiguity is a hard error here because only the CLI edge is allowed to ask a person. + */ +export async function readScroll(name, { targetId = null } = {}) { + let candidates = await scrollCandidates(name); + if (targetId) { + candidates = candidates.filter((candidate) => candidate.targetId === targetId); + if (candidates.length === 0) { + fail(`Target ${targetId} is not available for ${name}.`); + } + } + if (candidates.length > 1) { + fail( + `Box ${name} has multiple scroll targets (${candidates.map((candidate) => candidate.targetId).join(', ')}); ` + + 'use / or select a target explicitly.', + ); + } + return candidates[0]; +} + +/** + * Build timestamp taken from the HEAD commit rather than the clock, so rebuilding the same commit + * produces the same provenance. Falls back to the epoch outside a git checkout — deliberately a + * constant, since a wall-clock fallback would reintroduce the nondeterminism this avoids. + */ +export function sourceBuildTime(cwd) { + const result = runResult('git', ['show', '-s', '--format=%cI', 'HEAD'], { capture: true, cwd }); + return result.status === 0 ? result.stdout.trim() : new Date(0).toISOString(); +} + +/** + * The commit a box was built from, and whether the tree had uncommitted changes at the time. + * + * Outside a git checkout there is no revision to record, which callers must handle explicitly rather + * than inventing one: an unversioned build is reproducible by nobody. + */ +export function sourceBuildState(cwd) { + const revision = runResult('git', ['rev-parse', 'HEAD'], { capture: true, cwd }); + if (revision.status !== 0) return null; + const status = runResult('git', ['status', '--porcelain', '--untracked-files=all'], { capture: true, cwd }); + return { revision: revision.stdout.trim(), dirty: status.stdout.trim().length > 0 }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/toolchain.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/toolchain.mjs new file mode 100644 index 0000000..72cd7a1 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/toolchain.mjs @@ -0,0 +1,246 @@ +/** + * Installing the build toolchain, only when a human says so. + * + * `init` prepares a workspace without touching the network. When pixi or conda-pack is missing it + * *offers* to install them and downloads nothing until an explicit yes. That consent is the design, + * not a courtesy: a command that quietly fetched and ran a binary would be one nobody dares re-run, + * and the whole point of `init` is that it is always safe to run again. + * + * What is downloaded is verified before it is used. The archive's SHA-256 is checked against the + * checksum pixi publishes beside it, and the verified digest is then recorded in the project's + * config, so every later install — a teammate's machine, CI — is checked against a value the + * project reviewed rather than against whatever the server serves that day. A mismatch is a hard + * failure: an unverified toolchain would undermine every guarantee built on top of it. + * + * The toolchain is installed inside the project, under the workspace's toolchain directory. Nothing + * is placed on PATH, nothing is installed system-wide, and removing the directory undoes it. + */ + +import { createWriteStream } from 'node:fs'; +import { chmod, copyFile, mkdir, mkdtemp, rm, rename } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pipeline } from 'node:stream/promises'; +import { extractScrollArchive } from './archive.mjs'; +import { collectFiles, fileExists, sha256File } from './filesystem.mjs'; +import { fail, run as defaultRun } from './process.mjs'; + +const PIXI_RELEASES = 'https://github.com/prefix-dev/pixi/releases'; +const PIXI_LATEST_API = 'https://api.github.com/repos/prefix-dev/pixi/releases/latest'; +const SHA256_TOKEN = /\b[a-f0-9]{64}\b/; + +/** + * One host-specific archive published by pixi. + * + * @typedef {object} PixiReleaseAsset + * @property {string} asset + * @property {'zip' | 'tar.gz'} format + * @property {string} binary + */ + +// conda-pack changes the bytes staged into a box, so letting the resolver select a newer release +// would make the same Scrollcase version produce a different payload over time. Keep the pin with +// the implementation that relies on its output; changing it is a reviewed Scrollcase release. +export const CONDA_PACK_VERSION = '0.9.2'; + +/** + * The release asset for each host pixi publishes a build for, keyed by `platform/arch` as Node + * reports them. A host outside this table is not a failure of the project — it just means the + * toolchain has to be installed by hand. + */ +/** @type {Readonly>>} */ +export const PIXI_RELEASE_ASSETS = Object.freeze({ + 'darwin/arm64': Object.freeze({ asset: 'pixi-aarch64-apple-darwin.tar.gz', format: 'tar.gz', binary: 'pixi' }), + 'darwin/x64': Object.freeze({ asset: 'pixi-x86_64-apple-darwin.tar.gz', format: 'tar.gz', binary: 'pixi' }), + 'linux/x64': Object.freeze({ asset: 'pixi-x86_64-unknown-linux-musl.tar.gz', format: 'tar.gz', binary: 'pixi' }), + 'linux/arm64': Object.freeze({ asset: 'pixi-aarch64-unknown-linux-musl.tar.gz', format: 'tar.gz', binary: 'pixi' }), + 'win32/x64': Object.freeze({ asset: 'pixi-x86_64-pc-windows-msvc.zip', format: 'zip', binary: 'pixi.exe' }), + 'win32/arm64': Object.freeze({ asset: 'pixi-aarch64-pc-windows-msvc.zip', format: 'zip', binary: 'pixi.exe' }), +}); + +/** + * Where the project keeps the tools it installed for itself. + * + * @param {string} toolchainDir + * @returns {{ binDir: string, pixi: string, condaPack: string }} + */ +export function toolchainPaths(toolchainDir) { + const binDir = join(toolchainDir, 'bin'); + const suffix = process.platform === 'win32' ? '.exe' : ''; + return { + binDir, + pixi: join(binDir, `pixi${suffix}`), + condaPack: join(binDir, `conda-pack${suffix}`), + }; +} + +/** + * Returns the release asset for this host, or null when pixi publishes no build for it. + * + * @param {{ platform: string, arch: string }} [host] + * @returns {Readonly | null} + */ +export function pixiReleaseAsset(host = process) { + return PIXI_RELEASE_ASSETS[`${host.platform}/${host.arch}`] ?? null; +} + +/** + * The archive and checksum URLs for one pixi release. + * + * @param {string} version + * @param {string} asset + * @returns {{ archiveUrl: string, checksumUrl: string }} + */ +export function pixiAssetUrls(version, asset) { + const base = `${PIXI_RELEASES}/download/v${version}/${asset}`; + return { archiveUrl: base, checksumUrl: `${base}.sha256` }; +} + +/** + * Reads the digest out of a published checksum file, which may or may not name the file beside it. + * + * @param {unknown} text + * @returns {string} + */ +export function parseChecksumFile(text) { + const match = SHA256_TOKEN.exec(String(text).toLowerCase()); + if (!match) fail('Published pixi checksum file does not contain a SHA-256 digest.'); + return match[0]; +} + +/** + * Resolves the newest pixi release, for a project that has not pinned a version yet. + * + * @param {{ fetchImpl?: typeof fetch }} [options] + * @returns {Promise} + */ +export async function latestPixiVersion({ fetchImpl = fetch } = {}) { + const response = await fetchImpl(PIXI_LATEST_API, { headers: { accept: 'application/vnd.github+json' } }); + if (!response.ok) fail(`Could not look up the latest pixi release (${response.status}).`); + const tag = (await response.json())?.tag_name; + if (typeof tag !== 'string' || !tag) fail('The pixi release feed returned no version.'); + return tag.replace(/^v/, ''); +} + +async function fetchText(url, fetchImpl) { + const response = await fetchImpl(url); + if (!response.ok) fail(`Download failed (${response.status}): ${url}`); + return response.text(); +} + +async function fetchToFile(url, destination, fetchImpl) { + const response = await fetchImpl(url); + if (!response.ok) fail(`Download failed (${response.status}): ${url}`); + await pipeline(response.body, createWriteStream(destination)); +} + +/** + * Moves a staged file onto its final path, falling back to a copy across filesystems. + * + * Staging happens in the OS temp directory while the toolchain lives inside the project, and those + * are routinely on different volumes: on Windows temp sits on `C:` while a checkout commonly sits + * on another drive, which is the default on a GitHub runner and ordinary on a developer's machine. + * `rename` cannot cross a volume boundary and fails with `EXDEV`, so a plain rename left the + * toolchain uninstalled for those users. Copying is slower and only needed on that path, which is + * why it is the fallback rather than the rule. + * + * @param {string} source + * @param {string} destination + * @returns {Promise} + */ +async function moveInto(source, destination) { + try { + await rename(source, destination); + } catch (error) { + if (error?.code !== 'EXDEV') throw error; + // The staging directory is removed by the caller's `finally`, so the copy needs no cleanup. + await copyFile(source, destination); + } +} + +/** + * Downloads one pixi release and installs its binary into the project's toolchain directory. + * + * `expectedSha256` is the digest the project has already reviewed, when it has one; without it the + * checksum published beside the archive is used and returned, so the caller can pin it. Either way + * the bytes on disk are hashed and compared before anything is installed. + * + * @param {{ + * version: string, + * toolchainDir: string, + * expectedSha256?: string | null, + * host?: { platform: string, arch: string }, + * fetchImpl?: typeof fetch, + * log?: (message: string) => void, + * }} options + * @returns {Promise<{ path: string, version: string, sha256: string, asset: string }>} + */ +export async function installPixi({ + version, + toolchainDir, + expectedSha256 = null, + host = process, + fetchImpl = fetch, + log = console.log, +}) { + const release = pixiReleaseAsset(host); + if (!release) { + fail(`pixi publishes no build for ${host.platform}/${host.arch}; install it manually from https://pixi.sh/.`); + } + const { archiveUrl, checksumUrl } = pixiAssetUrls(version, release.asset); + const staging = await mkdtemp(join(tmpdir(), 'scrollcase-toolchain-')); + try { + const expected = expectedSha256 ?? parseChecksumFile(await fetchText(checksumUrl, fetchImpl)); + log(`Downloading pixi ${version} (${release.asset})`); + const archivePath = join(staging, release.asset); + await fetchToFile(archiveUrl, archivePath, fetchImpl); + + const actual = await sha256File(archivePath); + if (actual !== expected) { + fail(`pixi ${version} failed its checksum: expected ${expected}, got ${actual}. Nothing was installed.`); + } + + // Unpacked through the same guarded extractor the payload uses, so a hostile archive cannot + // write outside the staging directory even though this one came from a known publisher. + const unpacked = join(staging, 'unpacked'); + await extractScrollArchive(archivePath, release.format, unpacked); + const entry = (await collectFiles(unpacked)).find((file) => file.split('/').pop() === release.binary); + if (!entry) fail(`The pixi archive did not contain ${release.binary}.`); + + const { binDir, pixi } = toolchainPaths(toolchainDir); + await mkdir(binDir, { recursive: true }); + await rm(pixi, { force: true }); + await moveInto(join(unpacked, ...entry.split('/')), pixi); + if (process.platform !== 'win32') await chmod(pixi, 0o755); + return { path: pixi, version, sha256: expected, asset: release.asset }; + } finally { + await rm(staging, { recursive: true, force: true }); + } +} + +/** + * Installs conda-pack with the project's own pixi, into the project's own toolchain directory. + * + * `PIXI_HOME` points pixi at the toolchain directory, so the result lands beside pixi instead of in + * the user's home. Integrity here is conda-forge's to provide: the package is resolved and verified + * by pixi exactly as any other dependency is. + * + * @param {{ + * pixi: string, + * toolchainDir: string, + * run?: typeof defaultRun, + * log?: (message: string) => void, + * }} options + * @returns {Promise<{ path: string, version: typeof CONDA_PACK_VERSION }>} + */ +export async function installCondaPack({ pixi, toolchainDir, run = defaultRun, log = console.log }) { + log(`Installing conda-pack ${CONDA_PACK_VERSION} with pixi`); + run(pixi, ['global', 'install', `conda-pack==${CONDA_PACK_VERSION}`], { + env: { PIXI_HOME: toolchainDir }, + }); + const { condaPack } = toolchainPaths(toolchainDir); + if (!await fileExists(condaPack)) { + fail(`pixi reported success but ${condaPack} is missing.`); + } + return { path: condaPack, version: CONDA_PACK_VERSION }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/verify.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/verify.mjs new file mode 100644 index 0000000..52e737c --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/verify.mjs @@ -0,0 +1,204 @@ +/** + * `verify` — re-run a consumer's install-time checks locally, before anything is published. + * + * This deliberately mirrors what an installing client does on a user's machine: check the signature, + * the archive's size and hash, that entry names are safe, that `box.json` agrees with the signed + * release, and that the declared interpreter is actually present. `selfTest` goes one step further + * and imports the modules from a real extraction, which is the closest thing to a dry-run install. + * + * The point is that a box which would fail on a user's machine fails here instead. + */ + +import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { assertNativeHost, assertPythonEntryPoint, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; +import { BOX_SCHEMA_VERSION, parseDocumentKind } from '../contract/documents.mjs'; +import { verifySignedDocument } from '../sign/index.mjs'; + +const AGREEMENT_FIELDS = [ + 'schemaVersion', + 'boxId', + 'modelId', + 'runtimeId', + 'version', + 'target', + 'pythonEntryPoint', + 'modelCacheSubdir', + 'selfTest', + 'execution', + 'weights', + 'assets', + 'provenance', +]; + +/** + * Binds the self-description inside the archive to the signed release outside it. + * + * Only fields present in both schema-version-2 documents belong here. Release-only transport data + * has no counterpart in box.json; every shared identity, target, layout, consumer self-test, + * asset-policy, and provenance field must agree recursively. + */ +export function assertBoxManifestAgreement(box, release) { + for (const field of AGREEMENT_FIELDS) { + if (!isDeepStrictEqual(box[field], release[field])) fail(`box.json mismatch: ${field}`); + } +} +import { extractZipArchive, listZipEntries, readZipEntry } from './archive.mjs'; +import { assertExecutionFiles } from './execution.mjs'; +import { fileExists, payloadSize, safeRelativePath, sha256File } from './filesystem.mjs'; +import { fail, run as runProcess } from './process.mjs'; +import { schemaValidationError } from './schema-validation.mjs'; + +const schemaUrls = [ + new URL('../contract/schema/release-manifest.schema.json', import.meta.url), + new URL('../contract/schema/box-manifest.schema.json', import.meta.url), + new URL('../contract/schema/target.schema.json', import.meta.url), + new URL('../contract/schema/execution.schema.json', import.meta.url), + new URL('../contract/schema/signed-document.schema.json', import.meta.url), +]; +let manifestSchemas; + +async function loadManifestSchemas() { + manifestSchemas ??= Promise.all(schemaUrls.map(async (url) => JSON.parse(await readFile(url, 'utf8')))); + return manifestSchemas; +} + +/** + * Performs the complete read-only trust chain shared by `verify` and the local consumer. + * + * Keeping this as one operation matters: adding an execution API must not create a second, + * subtly different interpretation of a signed release. The caller receives the validated + * in-memory objects and exact archive path, but extraction and execution remain separate steps. + */ +export async function inspectBoxArchive(releaseDocumentPath, options = {}) { + const { publicPath, archive: archiveOverride = null } = options; + const releasePath = resolve(releaseDocumentPath); + const signed = JSON.parse(await readFile(releasePath, 'utf8')); + if (signed?.schemaVersion === 1) { + fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + } + const [releaseSchema, boxSchema, targetSchema, executionSchema, signedSchema] = + await loadManifestSchemas(); + const signedError = schemaValidationError(signed, signedSchema); + if (signedError) fail(`Invalid signed document: ${signedError}.`); + const release = await verifySignedDocument(signed, publicPath); + if (release?.schemaVersion === 1) { + fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + } + if (release?.schemaVersion !== BOX_SCHEMA_VERSION) { + fail(`Unsupported schemaVersion ${String(release?.schemaVersion)}; expected ${BOX_SCHEMA_VERSION}.`); + } + const releaseError = schemaValidationError( + release, + releaseSchema, + [boxSchema, targetSchema, executionSchema], + ); + if (releaseError) fail(`Invalid release manifest: ${releaseError}.`); + if (parseDocumentKind(release.kind)?.type !== 'release') fail('Document is not a box release.'); + const adapter = boxTargetAdapter(release.target); + assertPythonEntryPoint(adapter, release.pythonEntryPoint); + + // The archive sits next to its release document under the hash that document commits to — the + // same name it is published under, so this resolves identically against a local dist tree and a + // directory downloaded from a mirror. + const archivePath = archiveOverride + ? resolve(archiveOverride) + : join(dirname(releasePath), `${release.archive.sha256}.zip`); + if (!await fileExists(archivePath)) fail(`Archive not found: ${archivePath}`); + if ((await stat(archivePath)).size !== release.archive.sizeBytes) fail('Archive size mismatch.'); + if (await sha256File(archivePath) !== release.archive.sha256) fail('Archive SHA-256 mismatch.'); + if (release.installedSizeBytes !== undefined + && (!Number.isSafeInteger(release.installedSizeBytes) || release.installedSizeBytes <= 0)) { + fail('Invalid installed size.'); + } + + const entries = await listZipEntries(archivePath); + // Two questions, deliberately not the same set. `box.json` is read out of the archive, so it must + // be an entry with its own bytes. Everything else asks only whether a path resolves — and a link + // does resolve, to a file inside this same payload, because nothing else was allowed in. + const files = new Set(entries.filter((entry) => entry.kind === 'file').map((entry) => entry.path)); + const resolvablePaths = new Set(entries + .filter((entry) => entry.kind === 'file' || entry.kind === 'link') + .map((entry) => entry.path)); + if (!files.has('box.json')) fail('Archive is missing box.json.'); + const box = JSON.parse(await readZipEntry(archivePath, 'box.json')); + const boxError = schemaValidationError( + box, + boxSchema, + [releaseSchema, targetSchema, executionSchema], + ); + if (boxError) fail(`Invalid box.json: ${boxError}.`); + assertBoxManifestAgreement(box, release); + if (!resolvablePaths.has(release.pythonEntryPoint)) fail(`Archive is missing ${release.pythonEntryPoint}.`); + assertExecutionFiles({ + execution: release.execution, + adapter, + pythonVersion: release.provenance.pythonVersion, + files: resolvablePaths, + }); + + return { + releasePath, + archivePath, + signed, + release, + box, + adapter, + entries, + files, + }; +} + +/** + * Verifies a signed release document and the archive it commits to. + * + * `publicPath` names the trusted key file; `archive` overrides the convention of the archive + * sitting next to its release document; `selfTest` additionally extracts the box and runs its own + * interpreter, which only works on a matching native host. Returns a summary of what was checked. + */ +export async function verifyBox(releaseDocumentPath, options = {}) { + const { + selfTest = false, + run = runProcess, + log = console.log, + } = options; + const inspected = await inspectBoxArchive(releaseDocumentPath, options); + const { + archivePath, + signed, + release, + adapter, + } = inspected; + + if (selfTest) { + assertNativeHost(adapter); + const extracted = await mkdtemp(join(tmpdir(), 'scrollcase-verify-')); + try { + await extractZipArchive(archivePath, extracted); + if (release.installedSizeBytes !== undefined + && await payloadSize(extracted) !== release.installedSizeBytes) { + fail('Extracted payload size does not match the signed release.'); + } + const python = join(extracted, safeRelativePath(release.pythonEntryPoint)); + run(python, ['-c', `${adapter.selfTestPython}\nimport ${release.selfTest.pythonImports.join(', ')}`], { + cwd: extracted, + env: adapter.validationEnvironments[release.target.accelerator], + }); + } finally { + await rm(extracted, { recursive: true, force: true }); + } + } + + log(`Verified ${release.boxId} ${release.version} (${boxTargetId(release.target)})`); + return { + status: 'passed', + localSignatureVerified: true, + signingKeyIds: signed.signatures.map((signature) => signature.keyId), + releasePayloadSha256: signed.payloadSha256, + archiveSha256: release.archive.sha256, + archiveSizeBytes: release.archive.sizeBytes, + selfTest: selfTest ? 'passed' : 'not-requested', + }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/build/workspace.mjs b/.scrollcase-runtime-types-RzUfxr/src/build/workspace.mjs new file mode 100644 index 0000000..0823c5b --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/build/workspace.mjs @@ -0,0 +1,245 @@ +/** + * Scrollcase workspace resolution. + * + * Where a project keeps its scrolls, and where the tool writes what it builds, is the project's + * decision, not the tool's. A workspace is declared by a `scrollcase.config.json` at the project + * root, discovered by walking up from the working directory and overridable per invocation by CLI + * flags. A project that declares nothing gets the defaults below. + * + * Precedence, highest first: CLI flag, then `scrollcase.config.json`, then the built-in default. + * Flag values resolve against the current working directory (what a shell user expects); config + * values resolve against the project root (so a config file is portable). + */ +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, isAbsolute, parse as parsePath, resolve } from 'node:path'; +import { fail } from './process.mjs'; + +/** + * The absolute layout a command works against. Every path is resolved and the object is frozen. + * + * @typedef {object} Workspace + * @property {string} root the project root the config was found in, and the git checkout provenance + * is recorded from + * @property {string | null} configPath the config that produced it, or null when defaults applied + * @property {string} scrollsDir + * @property {string} buildDir + * @property {string} distDir + * @property {string} keysDir + * @property {string} toolchainDir + */ + +/** + * Per-invocation overrides, highest precedence in workspace resolution. + * + * @typedef {object} WorkspaceOverrides + * @property {string} [projectRoot] + * @property {string} [config] + * @property {string} [scrolls] + * @property {string} [build] + * @property {string} [dist] + * @property {string} [keys] + * @property {string} [toolchain] + */ + +export const SCROLLCASE_CONFIG_FILENAME = 'scrollcase.config.json'; + +/** + * The layout a project gets when it declares nothing. A project that already keeps its scrolls + * elsewhere — or that adopted the tool after building its own convention — overrides these in its + * config rather than moving its files. + */ +export const DEFAULT_WORKSPACE_PATHS = Object.freeze({ + scrolls: 'scrolls', + build: '.scrollcase/build', + dist: '.scrollcase/dist', + keys: '.scrollcase/keys', + toolchain: '.scrollcase/toolchain', +}); + +/** Config path key -> resolved workspace field. */ +const PATH_FIELDS = Object.freeze({ + scrolls: 'scrollsDir', + build: 'buildDir', + dist: 'distDir', + keys: 'keysDir', + toolchain: 'toolchainDir', +}); + +/** CLI flag -> config path key. */ +const PATH_FLAGS = Object.freeze({ + 'scrolls-dir': 'scrolls', + 'build-dir': 'build', + 'out-dir': 'dist', + 'keys-dir': 'keys', + 'toolchain-dir': 'toolchain', +}); + +/** + * Walks up from `startDir` to the filesystem root looking for a workspace config. + * + * @param {string} startDir + * @returns {string | null} the nearest config path, or null at the filesystem root + */ +export function findWorkspaceConfig(startDir) { + let current = resolve(startDir); + const { root } = parsePath(current); + for (;;) { + const candidate = resolve(current, SCROLLCASE_CONFIG_FILENAME); + if (existsSync(candidate)) return candidate; + if (current === root) return null; + const parent = dirname(current); + if (parent === current) return null; + current = parent; + } +} + +/** Reads and shape-checks a config file; an unreadable or malformed config is a hard error. */ +function readWorkspaceConfig(configPath) { + let config; + try { + config = JSON.parse(readFileSync(configPath, 'utf8')); + } catch (error) { + return fail(`Invalid ${SCROLLCASE_CONFIG_FILENAME} at ${configPath}: ${error instanceof Error ? error.message : String(error)}`); + } + if (!config || typeof config !== 'object' || Array.isArray(config)) { + return fail(`Invalid ${SCROLLCASE_CONFIG_FILENAME} at ${configPath}: expected a JSON object`); + } + if (config.version !== undefined && config.version !== 1) { + return fail(`Unsupported ${SCROLLCASE_CONFIG_FILENAME} version ${config.version} at ${configPath}; this builder understands version 1`); + } + const paths = config.paths ?? {}; + if (typeof paths !== 'object' || paths === null || Array.isArray(paths)) { + return fail(`Invalid ${SCROLLCASE_CONFIG_FILENAME} at ${configPath}: "paths" must be an object`); + } + for (const [key, value] of Object.entries(paths)) { + if (!(key in PATH_FIELDS)) { + fail(`Unknown "paths" entry "${key}" in ${configPath}; expected one of ${Object.keys(PATH_FIELDS).join(', ')}`); + } + if (typeof value !== 'string' || value.trim() === '') { + fail(`Invalid "paths.${key}" in ${configPath}: expected a non-empty path string`); + } + } + return { ...config, paths }; +} + +/** + * Collects workspace overrides from an already-parsed CLI flag map. + * + * @param {ReadonlyMap | null | undefined} flags + * @returns {WorkspaceOverrides} + */ +export function workspaceOverridesFromFlags(flags) { + const overrides = {}; + const stringFlag = (name) => { + const value = flags?.get(name); + if (value === undefined) return undefined; + if (typeof value !== 'string' || value.trim() === '') fail(`--${name} requires a path value`); + return value; + }; + const projectRoot = stringFlag('project-root'); + if (projectRoot !== undefined) overrides.projectRoot = projectRoot; + const config = stringFlag('config'); + if (config !== undefined) overrides.config = config; + for (const [flag, key] of Object.entries(PATH_FLAGS)) { + const value = stringFlag(flag); + if (value !== undefined) overrides[key] = value; + } + return overrides; +} + +/** + * Collects workspace overrides directly from raw arguments, for entry points that parse the rest of + * their command line themselves. Only the workspace flags are read, in `--name value` or + * `--name=value` form; anything else is left untouched for the caller's own parser. + * + * @param {readonly string[]} values + * @returns {WorkspaceOverrides} + */ +export function workspaceOverridesFromArgv(values) { + const flags = new Map(); + const known = new Set(['project-root', 'config', ...Object.keys(PATH_FLAGS)]); + for (let index = 0; index < values.length; index += 1) { + const value = values[index]; + if (typeof value !== 'string' || !value.startsWith('--')) continue; + const [name, inline] = value.slice(2).split('=', 2); + if (!known.has(name)) continue; + if (inline !== undefined) flags.set(name, inline); + else if (values[index + 1] !== undefined) flags.set(name, values[index + 1]); + else fail(`--${name} requires a path value`); + } + return workspaceOverridesFromFlags(flags); +} + +/** + * Resolves the absolute workspace layout. + * + * Root selection, highest precedence first: `--project-root`, the directory of an explicit + * `--config`, the nearest `scrollcase.config.json` above the working directory, and finally the + * working directory itself. + * + * @param {{ cwd?: string, overrides?: WorkspaceOverrides }} [options] + * @returns {Workspace} frozen, with every path absolute + * @throws {Error} when a named config is missing or malformed + */ +export function resolveWorkspace({ cwd = process.cwd(), overrides = {} } = {}) { + const base = resolve(cwd); + let configPath = null; + let root; + if (overrides.config !== undefined) { + // An explicitly named config must exist: silently ignoring it would hide a typo behind defaults. + configPath = resolve(base, overrides.config); + if (!existsSync(configPath)) fail(`Workspace config not found: ${configPath}`); + root = overrides.projectRoot !== undefined ? resolve(base, overrides.projectRoot) : dirname(configPath); + } else if (overrides.projectRoot !== undefined) { + root = resolve(base, overrides.projectRoot); + const candidate = resolve(root, SCROLLCASE_CONFIG_FILENAME); + configPath = existsSync(candidate) ? candidate : null; + } else { + configPath = findWorkspaceConfig(base); + root = configPath ? dirname(configPath) : base; + } + const config = configPath ? readWorkspaceConfig(configPath) : { paths: {} }; + const workspace = { root, configPath }; + for (const [key, field] of Object.entries(PATH_FIELDS)) { + const override = overrides[key]; + if (override !== undefined) { + // A flag is typed by a user standing in some directory, so it resolves from there. + workspace[field] = isAbsolute(override) ? override : resolve(base, override); + continue; + } + const declared = config.paths[key]; + // Config and default values belong to the project, so they resolve from its root. + workspace[field] = resolve(root, declared ?? DEFAULT_WORKSPACE_PATHS[key]); + } + return Object.freeze(workspace); +} + +let current = null; + +/** + * Installs the workspace for this process. Entry points call this once, before any other work, so + * every module downstream observes the same resolved layout. + * + * @param {{ cwd?: string, overrides?: WorkspaceOverrides }} [options] + * @returns {Workspace} + */ +export function configureWorkspace(options = {}) { + current = resolveWorkspace(options); + return current; +} + +/** + * Returns the process workspace, resolving a flag-free default on first use. Modules read paths + * through this rather than at import time, so an entry point can still configure them from flags. + * + * @returns {Workspace} resolving a flag-free default on first use + */ +export function getWorkspace() { + if (!current) current = resolveWorkspace(); + return current; +} + +/** Test seam: forgets the resolved workspace so the next read re-resolves it. */ +export function resetWorkspace() { + current = null; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-args.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-args.mjs new file mode 100644 index 0000000..1c93473 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli-args.mjs @@ -0,0 +1,31 @@ +/** + * Argument parsing at the CLI edge. + * + * The `--` separator is a hard boundary: every string after it belongs unchanged to the box + * application, even when it looks like a Scrollcase flag or contains shell syntax. + */ + +/** Parses `--name=value`, `--name value`, bare flags, and an application argument tail. */ +export function parseArgs(values) { + const positional = []; + const flags = new Map(); + let passthrough = []; + for (let index = 0; index < values.length; index += 1) { + const value = values[index]; + if (value === '--') { + passthrough = values.slice(index + 1); + break; + } + if (!value.startsWith('--')) { + positional.push(value); + continue; + } + const [name, inline] = value.slice(2).split('=', 2); + if (inline !== undefined) flags.set(name, inline); + else if (values[index + 1] && !values[index + 1].startsWith('--')) { + flags.set(name, values[index + 1]); + index += 1; + } else flags.set(name, true); + } + return { positional, flags, passthrough }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-authoring.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-authoring.mjs new file mode 100644 index 0000000..2f5316d --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli-authoring.mjs @@ -0,0 +1,187 @@ +/** + * Interactive and scripted input collection for `scrollcase new scroll`. + * + * This is a CLI-edge module: finite decisions use the shared navigable menu, free-form values use + * explicit text prompts, and a non-terminal process must provide every material value as a flag. + * The build layer receives one complete object and never reads a terminal. + */ + +import { createInterface } from 'node:readline/promises'; +import { fail } from './build/process.mjs'; +import { chooseCliValue } from './cli-menu.mjs'; +import { chooseTarget, cliTargetFamilies, parseCliTarget } from './cli-targets.mjs'; + +const flagText = (flags, name) => { + if (!flags.has(name)) return null; + const value = flags.get(name); + if (typeof value !== 'string' || value.trim() === '') fail(`--${name} requires a value.`); + return value.trim(); +}; + +async function promptText(question, { + defaultValue = null, + optional = false, + input = process.stdin, + output = process.stdout, +} = {}) { + const readline = createInterface({ input, output }); + try { + const suffix = defaultValue === null ? '' : ` [${defaultValue}]`; + const value = (await readline.question(`${question}${suffix}: `)).trim(); + if (value) return value; + if (defaultValue !== null) return defaultValue; + if (optional) return null; + fail(`${question} is required.`); + } finally { + readline.close(); + } +} + +function parseDefaultArgs(value) { + if (value === null) return []; + let parsed; + try { + parsed = JSON.parse(value); + } catch { + fail('--default-args must be a JSON array of strings.'); + } + if (!Array.isArray(parsed) || parsed.some((item) => typeof item !== 'string')) { + fail('--default-args must be a JSON array of strings.'); + } + return parsed; +} + +async function collectTarget(flags, { terminal, ask, chooseTargetValue }) { + const requested = flagText(flags, 'target'); + if (requested) return parseCliTarget(requested); + if (!terminal) fail('new scroll requires --target without a terminal.'); + + const selected = await chooseTargetValue(cliTargetFamilies(), { terminal: true }); + if (selected.target.accelerator !== 'cuda') return parseCliTarget(selected.targetId); + const cudaVersion = await ask('CUDA version (major.minor)'); + return parseCliTarget(`${selected.targetId}${cudaVersion}`); +} + +/** + * Collects a complete `createScroll` argument object from flags or interactive prompts. + * + * @param {ReadonlyMap} flags + * @param {object} [options] + * @returns {Promise} + */ +export async function collectNewScrollOptions(flags, { + terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), + ask = promptText, + choose = chooseCliValue, + chooseTargetValue = chooseTarget, +} = {}) { + const required = async (flag, question, defaultValue = null) => { + const supplied = flagText(flags, flag); + if (supplied !== null) return supplied; + if (!terminal) fail(`new scroll requires --${flag} without a terminal.`); + return ask(question, { defaultValue }); + }; + const optional = async (flag, question) => { + const supplied = flagText(flags, flag); + if (supplied !== null) return supplied; + if (!terminal) return null; + return ask(question, { optional: true }); + }; + const finite = async (flag, question, choices) => { + const supplied = flagText(flags, flag); + if (!supplied && !terminal) { + fail(`new scroll requires --${flag} <${choices.join('|')}> without a terminal.`); + } + return choose(question, choices, { flag: supplied, terminal }); + }; + + const target = await collectTarget(flags, { terminal, ask, chooseTargetValue }); + const boxId = await required('box-id', 'Box ID'); + const modelId = await required('model-id', 'Model ID'); + const runtimeId = await required('runtime-id', 'Runtime ID'); + const version = await required('version', 'Box version', '1.0.0'); + const scrollVersion = await required('scroll-version', 'Scroll version', '1.0.0'); + const sourceRevision = await required('source-revision', 'Upstream source revision'); + const pythonVersion = await required('python-version', 'Python version', '3.11'); + const pixiVersion = await required('pixi-version', 'pixi version'); + const minHostAppVersion = await required( + 'min-host-app-version', + 'Minimum host application version', + '1.0.0', + ); + const compatibility = { minHostAppVersion }; + const maxHostAppVersionExclusive = await optional( + 'max-host-app-version-exclusive', + 'Maximum host application version (exclusive, optional)', + ); + if (maxHostAppVersionExclusive) compatibility.maxHostAppVersionExclusive = maxHostAppVersionExclusive; + if (target.platform === 'macos') { + const minMacosVersion = await optional('min-macos-version', 'Minimum macOS version (optional)'); + if (minMacosVersion) compatibility.minMacosVersion = minMacosVersion; + } + const minRam = await optional('min-ram-gb', 'Minimum RAM in GB (optional)'); + if (minRam !== null) { + const minRamGb = Number(minRam); + if (!Number.isFinite(minRamGb) || minRamGb <= 0) fail('--min-ram-gb must be a positive number.'); + compatibility.minRamGb = minRamGb; + } + if (target.accelerator === 'cuda') { + const minNvidiaDriverVersion = await optional( + 'min-nvidia-driver-version', + 'Minimum NVIDIA driver version (optional)', + ); + if (minNvidiaDriverVersion) compatibility.minNvidiaDriverVersion = minNvidiaDriverVersion; + } + const assetBaseUrl = await required('asset-base-url', 'Asset base URL'); + const weights = await finite('weights', 'weights mode', ['embed', 'on-demand']); + const executionKind = await finite( + 'execution', + 'execution kind', + ['python-script', 'python-module', 'library-only'], + ); + const defaultArgs = parseDefaultArgs(flagText(flags, 'default-args')); + + const result = { + boxId, + target, + modelId, + runtimeId, + version, + scrollVersion, + sourceRevision, + pythonVersion, + pixiVersion, + compatibility, + assetBaseUrl, + weights, + executionKind, + defaultArgs, + }; + if (executionKind === 'python-module') { + result.module = await required('module', 'Python module'); + } else if (executionKind === 'python-script') { + const existing = flagText(flags, 'script'); + const generateScript = Boolean(flags.get('generate-script')); + if (existing && generateScript) { + fail('Choose either --script or --generate-script, not both.'); + } + if (existing) result.scriptSourcePath = existing; + else if (generateScript) result.generateScript = true; + else if (!terminal) { + fail('python-script execution requires --script or --generate-script without a terminal.'); + } else { + const source = await choose( + 'script source', + ['existing project script', 'generate starter script'], + { terminal: true }, + ); + if (source === 'existing project script') { + result.scriptSourcePath = await ask('Project-relative script path'); + } else result.generateScript = true; + } + result.scriptRelativePath = flagText(flags, 'script-destination') ?? 'entrypoint.py'; + const generatedScriptSourcePath = flagText(flags, 'generated-script-path'); + if (generatedScriptSourcePath) result.generatedScriptSourcePath = generatedScriptSourcePath; + } + return result; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-init.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-init.mjs new file mode 100644 index 0000000..4ad5ea8 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli-init.mjs @@ -0,0 +1,46 @@ +/** + * Orders the optional work performed by `scrollcase init`. + * + * Every answer is collected before the first installer runs. Besides making the interaction easier + * to review, this prevents an early download or package install from interrupting the remaining + * questions and leaving the user's choices only half collected. + */ + +export async function resolvePythonConsumerSource({ + selectedSource, + condaAvailable, + confirmPyPIFallback, +}) { + if (selectedSource !== 'conda-forge' || condaAvailable) return selectedSource; + return await confirmPyPIFallback() ? 'pypi' : null; +} + +export async function runInitDependencySetup({ + hasExample, + confirmTypeScript, + confirmPython, + choosePythonSource, + installToolchain, + installTypeScript, + installPython, +}) { + let shouldInstallTypeScript = false; + let pythonSource = null; + + if (hasExample) { + shouldInstallTypeScript = await confirmTypeScript(); + if (await confirmPython()) pythonSource = await choosePythonSource(); + } + + const toolchain = await installToolchain(); + const typescript = shouldInstallTypeScript ? installTypeScript() : null; + const python = pythonSource ? installPython(pythonSource) : null; + + return { + installTypeScript: shouldInstallTypeScript, + pythonSource, + toolchain, + typescript, + python, + }; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-menu.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-menu.mjs new file mode 100644 index 0000000..3abfb74 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli-menu.mjs @@ -0,0 +1,102 @@ +/** + * Navigable choices at the CLI edge. + * + * Closed choices use one raw-key menu instead of several subtly different text prompts. Free-form + * values and safety consent remain explicit flags or text input: a menu must not pretend they are + * finite choices. + */ + +import { emitKeypressEvents } from 'node:readline'; +import { fail } from './build/process.mjs'; + +/** Shows a raw-key menu and resolves to the selected index. */ +export function selectCliMenu(question, choices, { + initialIndex = null, + input = process.stdin, + output = process.stdout, +} = {}) { + if (!input.isTTY || typeof input.setRawMode !== 'function') { + fail(`${question} selection requires an interactive terminal.`); + } + + return new Promise((resolve, reject) => { + let selectedIndex = initialIndex; + const previousRawMode = Boolean(input.isRaw); + const frameLines = choices.length + 1; + let firstFrame = true; + + const render = () => { + if (!firstFrame) output.write(`\x1b[${frameLines}A`); + for (let index = 0; index < choices.length; index += 1) { + const marker = index === selectedIndex ? '❯' : ' '; + output.write(`\x1b[2K\r${marker} ${choices[index]}\n`); + } + output.write('\x1b[2K\rUse ↑/↓ to move, Enter to select.\n'); + firstFrame = false; + }; + + const cleanup = () => { + input.removeListener('keypress', onKeypress); + input.setRawMode(previousRawMode); + input.pause(); + output.write('\x1b[?25h'); + }; + + const onKeypress = (_character, key = {}) => { + if (key.ctrl && key.name === 'c') { + cleanup(); + reject(new Error(`${question} selection cancelled.`)); + return; + } + if (key.name === 'up') { + selectedIndex = selectedIndex === null + ? choices.length - 1 + : (selectedIndex - 1 + choices.length) % choices.length; + render(); + } else if (key.name === 'down') { + selectedIndex = selectedIndex === null ? 0 : (selectedIndex + 1) % choices.length; + render(); + } else if ((key.name === 'return' || key.name === 'enter') && selectedIndex !== null) { + cleanup(); + resolve(selectedIndex); + } + }; + + emitKeypressEvents(input); + input.on('keypress', onKeypress); + input.setRawMode(true); + input.resume(); + output.write(`Which ${question}?\n\x1b[?25l`); + render(); + }); +} + +/** + * Resolves a CLI choice from a flag, a menu, or the reported non-terminal default. + * + * `open` applies only to explicit flags; custom values cannot be represented by a finite menu. + */ +export async function chooseCliValue(question, choices, { + flag = null, + open = false, + terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), + menu = selectCliMenu, + log = console.log, +} = {}) { + const [fallback] = choices; + if (flag) { + if (!open && !choices.includes(flag)) { + fail(`Unsupported ${question}: ${flag}. Use ${choices.join(' or ')}.`); + } + return flag; + } + if (!terminal) { + log(`scrollcase: no terminal to ask which ${question}; using ${fallback}.`); + return fallback; + } + const selectedIndex = await menu(question, choices, { initialIndex: 0 }); + if (!Number.isInteger(selectedIndex) || selectedIndex < 0 || selectedIndex >= choices.length) { + fail(`${question} menu returned an invalid selection.`); + } + return choices[selectedIndex]; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-output.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-output.mjs new file mode 100644 index 0000000..4aaecb2 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli-output.mjs @@ -0,0 +1,34 @@ +/** + * Restrained terminal presentation for the human CLI. + * + * Symbols keep redirected logs readable on their own; ANSI colour is an optional enhancement only + * for a real terminal, and `NO_COLOR` always wins. The library modules remain presentation-free. + */ + +import { dirname, relative, sep } from 'node:path'; + +const styles = Object.freeze({ + success: { symbol: '✓', ansi: 32 }, + step: { symbol: '→', ansi: 36 }, + info: { symbol: '·', ansi: 90 }, + warning: { symbol: '⚠', ansi: 33 }, + error: { symbol: '✗', ansi: 31 }, +}); + +/** Formats one CLI status line, colouring only its symbol when the terminal supports it. */ +export function statusLine(kind, message, { + stream = process.stdout, + env = process.env, +} = {}) { + const style = styles[kind]; + const colour = Boolean(stream.isTTY && !Object.hasOwn(env, 'NO_COLOR') && env.TERM !== 'dumb'); + const symbol = colour ? `\x1b[${style.ansi}m${style.symbol}\x1b[0m` : style.symbol; + return `${symbol} ${message}`; +} + +/** Builds the concise, relative distribution instruction printed after a successful build. */ +export function buildDistributionSummary({ archivePath, channelPath }, distDir) { + const displayPath = (path) => relative(distDir, path).split(sep).join('/'); + return `Build complete — you can distribute the 2 files under ${displayPath(dirname(archivePath))}/ ` + + `and ${displayPath(channelPath)}`; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-run.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-run.mjs new file mode 100644 index 0000000..5c2f44e --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli-run.mjs @@ -0,0 +1,56 @@ +/** + * The `run` command's deliberately thin edge over the Node consumer. + * + * Verification, extraction, execution, signals, and cleanup remain owned by `runBox`. This module + * adds only terminal presentation and translates the child's terminal result into CLI process + * semantics. + */ + +import { runBox } from './consumer/index.mjs'; + +/** + * Runs one local release through the consumer and applies its terminal result to this process. + * + * @param {string} releaseDocumentPath + * @param {{ + * publicPath: string, + * archive?: string | null, + * args?: readonly string[], + * run?: typeof runBox, + * log?: (message: string) => void, + * setExitCode?: (code: number) => void, + * terminate?: (signal: NodeJS.Signals) => void, + * }} options + * @returns {Promise} + */ +export async function runCliBox(releaseDocumentPath, { + publicPath, + archive = null, + args = [], + run = runBox, + log = console.log, + setExitCode = (code) => { + process.exitCode = code; + }, + terminate = (signal) => { + process.kill(process.pid, signal); + }, +}) { + const result = await run(releaseDocumentPath, { + publicPath, + archive, + args, + stdin: 'inherit', + stdout: 'inherit', + stderr: 'inherit', + onPrepared: (prepared) => { + log( + `Running ${prepared.boxId} ${prepared.version} ` + + `(${prepared.targetId}, ${prepared.execution?.kind ?? 'library-only'})`, + ); + }, + }); + if (result.signal) terminate(result.signal); + else setExitCode(result.exitCode ?? 1); + return result; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-signing.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-signing.mjs new file mode 100644 index 0000000..4df0d3c --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli-signing.mjs @@ -0,0 +1,33 @@ +/** + * Signing readiness at the CLI edge. + * + * Signing readiness is a read-only preflight. `build` never creates or repairs identity material: + * doing so would mutate the project before provenance checks and could silently rotate the + * identity used by already-published documents. + */ + +import { fileExists } from './build/filesystem.mjs'; +import { fail } from './build/process.mjs'; + +/** Ensures the selected signing path is ready before any expensive build work begins. */ +export async function ensureBuildSigningKeys({ + privatePath, + publicPath, + signerCommand = null, +}) { + const publicExists = await fileExists(publicPath); + if (signerCommand) { + if (!publicExists) { + fail(`Trusted public key not found: ${publicPath}. Supply the key used to verify the external signer.`); + } + return; + } + + const privateExists = await fileExists(privatePath); + if (privateExists && publicExists) return; + if (privateExists || publicExists) { + const missing = privateExists ? publicPath : privatePath; + fail(`Signing key pair is incomplete; missing ${missing}. Refusing to replace the existing key.`); + } + fail('Signing keys not found. Run scrollcase keygen before building.'); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli-targets.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli-targets.mjs new file mode 100644 index 0000000..3c9267d --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli-targets.mjs @@ -0,0 +1,192 @@ +/** + * Target choices at the CLI edge. + * + * Modules beneath the CLI receive a resolved scroll or target and never read a terminal. This file + * owns the one interactive policy: choices are made through a keyboard menu, with a sole native + * target as the default and Metal preferred on macOS. Non-interactive callers get the same + * decision without ever blocking. + */ + +import { boxTargetAdapters, boxTargetId } from './contract/targets.mjs'; +import { compareStableStrings } from './build/filesystem.mjs'; +import { fail } from './build/process.mjs'; +import { selectCliMenu } from './cli-menu.mjs'; + +/** Parses a complete canonical target ID back into the target it names. */ +export function parseCliTarget(value) { + const targetId = String(value); + for (const adapter of boxTargetAdapters()) { + for (const accelerator of Object.keys(adapter.validationEnvironments)) { + const target = { platform: adapter.platform, arch: adapter.arch, accelerator }; + if (accelerator === 'cuda') { + const prefix = `${adapter.platform}-${adapter.arch}-cuda`; + if (!targetId.startsWith(prefix)) continue; + target.cudaVersion = targetId.slice(prefix.length); + } + try { + if (boxTargetId(target) === targetId) return target; + } catch { + // Keep looking. A partial CUDA ID reaches here, then receives the one canonical error below. + } + } + } + return fail( + `Invalid target ${targetId}; specify a complete target such as ` + + 'macos-aarch64-metal or linux-x86_64-cuda12.4.', + ); +} + +/** + * Lists target families for `new scroll`. CUDA is shown without an ABI version; selecting it is + * followed by the separate version question that turns it into a complete canonical target. + */ +export function cliTargetFamilies(platform) { + const families = []; + for (const adapter of boxTargetAdapters()) { + if (platform && adapter.platform !== platform) continue; + for (const accelerator of Object.keys(adapter.validationEnvironments)) { + families.push({ + adapter, + target: { platform: adapter.platform, arch: adapter.arch, accelerator }, + targetId: `${adapter.platform}-${adapter.arch}-${accelerator}`, + }); + } + } + return families.sort((left, right) => compareStableStrings(left.targetId, right.targetId)); +} + +/** + * Chooses the deterministic native target for the example created by `init`. + * + * The demo prefers Metal on Apple Silicon and CPU elsewhere, so it never guesses a CUDA ABI and + * remains usable from a non-interactive setup. + */ +export function nativeExampleTarget( + host = { platform: process.platform, arch: process.arch }, +) { + const adapter = boxTargetAdapters().find((candidate) => + candidate.host.platform === host.platform && candidate.host.arch === host.arch); + if (!adapter) { + return fail( + `No example target is available for ${host.platform}/${host.arch}; ` + + 'use scrollcase init --no-example.', + ); + } + return { + platform: adapter.platform, + arch: adapter.arch, + accelerator: adapter.platform === 'macos' ? 'metal' : 'cpu', + }; +} + +/** + * Selects one complete scroll reference when a CLI caller omitted the positional argument. + * + * Unlike target selection, this has no non-terminal default: locking or building an arbitrary + * first scroll would mutate or package the wrong input without consent. + * + * @template {{ reference: string }} T + * @param {T[]} candidates + * @returns {Promise} + */ +export async function chooseScroll(candidates, { + terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), + menu = selectCliMenu, +} = {}) { + if (candidates.length === 0) fail('No scrolls are available.'); + const choices = [...candidates] + .sort((left, right) => compareStableStrings(left.reference, right.reference)); + if (new Set(choices.map(({ reference }) => reference)).size !== choices.length) { + fail('Scroll choices must have unique references.'); + } + if (!terminal) { + fail( + 'scroll selection requires an interactive terminal; ' + + 'pass / explicitly.', + ); + } + const selectedIndex = await menu( + 'scroll', + choices.map(({ reference }) => reference), + { initialIndex: 0 }, + ); + if (!Number.isInteger(selectedIndex) || selectedIndex < 0 || selectedIndex >= choices.length) { + fail('scroll menu returned an invalid selection.'); + } + return choices[selectedIndex]; +} + +/** Shows a raw-key target menu and resolves to the selected index. */ +export function selectTargetMenu(targetIds, { + initialIndex = null, + input = process.stdin, + output = process.stdout, +} = {}) { + return selectCliMenu('target', targetIds, { initialIndex, input, output }); +} + +/** + * Chooses one target candidate under the CLI's terminal policy. + * + * @template {{ targetId: string, adapter: { host: { platform: string, arch: string } } }} T + * @param {T[]} candidates + * @param {{ requested?: string | null, terminal?: boolean, + * host?: { platform: string, arch: string }, + * menu?: (targetIds: string[], options: { initialIndex: number | null }) => Promise, + * log?: (message: string) => void }} [options] + * @returns {Promise} + */ +export async function chooseTarget(candidates, { + requested = null, + terminal = Boolean(process.stdin.isTTY && process.stdout.isTTY), + host = { platform: process.platform, arch: process.arch }, + menu = selectTargetMenu, + log = console.log, +} = {}) { + if (candidates.length === 0) fail('No supported targets are available.'); + const choices = [...candidates] + .sort((left, right) => compareStableStrings(left.targetId, right.targetId)); + if (new Set(choices.map(({ targetId }) => targetId)).size !== choices.length) { + fail('Target choices must have unique canonical IDs.'); + } + + if (requested) { + const selected = choices.find((candidate) => candidate.targetId === requested); + if (!selected) { + fail(`Target ${requested} is not available; choose one of ${choices.map(({ targetId }) => targetId).join(', ')}.`); + } + return selected; + } + if (choices.length === 1) return choices[0]; + + const native = choices.filter(({ adapter }) => + adapter.host.platform === host.platform && adapter.host.arch === host.arch); + const macMetal = host.platform === 'darwin' + ? native.find(({ targetId }) => targetId.endsWith('-metal')) + : null; + const fallback = native.length === 1 ? native[0] : macMetal; + if (!terminal) { + if (fallback) { + log(`scrollcase: no terminal to ask which target; using host target ${fallback.targetId}.`); + return fallback; + } + if (native.length > 1) { + fail( + `This host can build more than one available target (${native.map(({ targetId }) => targetId).join(', ')}); ` + + 'specify --target .', + ); + } + fail( + `No available target is an unambiguous match for this host; specify --target ` + + `from ${choices.map(({ targetId }) => targetId).join(', ')}.`, + ); + } + + const selectedIndex = await menu(choices.map(({ targetId }) => targetId), { + initialIndex: fallback ? choices.indexOf(fallback) : null, + }); + if (!Number.isInteger(selectedIndex) || selectedIndex < 0 || selectedIndex >= choices.length) { + fail('Target menu returned an invalid selection.'); + } + return choices[selectedIndex]; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/cli.mjs b/.scrollcase-runtime-types-RzUfxr/src/cli.mjs new file mode 100755 index 0000000..82cdd38 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/cli.mjs @@ -0,0 +1,501 @@ +#!/usr/bin/env node + +/** + * The Scrollcase command line. + * + * One job: turn a scroll into a portable, locked, self-contained box and prove it works. `init` + * prepares the workspace, `new scroll` authors one input, `doctor` checks the machine, `lock` + * resolves dependencies once so a human can review and commit the result, `audit` reports what + * licences that pulls in, `build` installs only from the lock, `verify` re-runs a consumer's + * install-time checks, `run` executes one caller-supplied local release through that consumer, and + * `keygen` produces the signing key that makes any of it trustworthy. + * + * Every command resolves its paths through the workspace, so the tool runs from anywhere against any + * project that declares a scrollcase.config.json. + */ + +import { createInterface } from 'node:readline/promises'; +import { join, resolve } from 'node:path'; +import { auditScroll } from './build/audit.mjs'; +import { + createScroll, + ensureExampleScroll, + EXAMPLE_PIXI_VERSION, +} from './build/authoring.mjs'; +import { buildBox } from './build/box.mjs'; +import { + isCondaAvailable, + installPythonConsumerDependency, + installTypeScriptConsumerDependencies, + SCROLLCASE_NPM_VERSION, +} from './build/consumer-setup.mjs'; +import { findPixi, pixiLockArguments } from './build/pixi.mjs'; +import { fail, run } from './build/process.mjs'; +import { diagnose, ensureToolchain, initProject } from './build/project.mjs'; +import { scrollCandidates, readScroll } from './build/scroll.mjs'; +import { verifyBox } from './build/verify.mjs'; +import { + configureWorkspace, + getWorkspace, + SCROLLCASE_CONFIG_FILENAME, + workspaceOverridesFromFlags, +} from './build/workspace.mjs'; +import { collectNewScrollOptions } from './cli-authoring.mjs'; +import { parseArgs } from './cli-args.mjs'; +import { + resolvePythonConsumerSource, + runInitDependencySetup, +} from './cli-init.mjs'; +import { chooseCliValue } from './cli-menu.mjs'; +import { buildDistributionSummary, statusLine } from './cli-output.mjs'; +import { runCliBox } from './cli-run.mjs'; +import { ensureBuildSigningKeys } from './cli-signing.mjs'; +import { chooseScroll, chooseTarget, nativeExampleTarget } from './cli-targets.mjs'; +import { CHANNELS } from './contract/index.mjs'; +import { generateSigningKey } from './sign/index.mjs'; + +const success = (message) => console.log(statusLine('success', message)); +const step = (message) => console.log(statusLine('step', message)); +const info = (message) => console.log(statusLine('info', message)); +const warning = (message) => console.log(statusLine('warning', message)); + +const text = (flags, name) => (flags.has(name) ? String(flags.get(name)) : null); + +/** Signing key locations, defaulting into the workspace's key directory. */ +function keyPaths(flags) { + const keysDir = getWorkspace().keysDir; + return { + privatePath: resolve(text(flags, 'private-key') || join(keysDir, 'signing-private.pem')), + publicPath: resolve(text(flags, 'public-key') || join(keysDir, 'signing-public.json')), + }; +} + +async function keygen(flags) { + const { privatePath, publicPath } = keyPaths(flags); + const created = await generateSigningKey({ + privatePath, + publicPath, + keyId: text(flags, 'key-id'), + force: Boolean(flags.get('force')), + }); + success(`Created signing key ${created.keyId}`); + info(`Private: ${created.privatePath}`); + info(`Public: ${created.publicPath}`); +} + +/** + * `lock` — resolve the scroll's pixi manifest into a fully pinned lock file. + * + * Run by a human when dependencies change; the result is committed and reviewed. Builds then only + * *install* from it, so what ships is exactly what was reviewed. The manifest pins the channels and + * the single target platform, which is what makes resolution independent of the machine doing it. + */ +async function lock(name, flags) { + const reference = await selectScrollReference(name, flags); + const { dir, scroll } = await readScroll(reference); + const pixi = findPixi({ requiredVersion: scroll.pixiVersion, path: text(flags, 'pixi') }); + run(pixi, pixiLockArguments(join(dir, 'pixi.toml'))); + success(`Updated ${join(dir, 'pixi.lock')}`); +} + +/** + * Asks a yes/no question, defaulting to no. + * + * Only ever asks when both ends are a terminal. Without one — CI, a pipe — there is nobody to + * answer, and silence must not be read as consent, so the answer is no. + */ +async function confirm(question) { + if (!process.stdin.isTTY || !process.stdout.isTTY) return false; + console.log(); + const readline = createInterface({ input: process.stdin, output: process.stdout }); + try { + return /^y(es)?$/i.test((await readline.question(`${question} [y/N] `)).trim()); + } finally { + readline.close(); + } +} + +/** Resolves a box shorthand at the CLI edge, where an ambiguous target can be asked about. */ +async function selectScrollReference(name, flags) { + const candidates = await scrollCandidates(name); + if (!name) return (await chooseScroll(candidates)).reference; + return (await chooseTarget(candidates, { requested: text(flags, 'target') })).reference; +} + +/** + * `init` — scaffold the workspace and its disposable runnable example, then offer its dependencies. + * + * Real scroll creation remains separate: the fixed `example-box` is onboarding material, never a + * guess at the project's identity. Toolchain and consumer installs each require explicit consent. + */ +async function init(flags) { + const workspace = getWorkspace(); + const authoringFlags = [ + 'target', + 'platform', + 'accelerator', + 'cuda-version', + 'box-id', + 'model-id', + 'runtime-id', + ].filter((name) => flags.has(name)); + if (authoringFlags.length > 0) { + fail(`init accepts only the fixed example; pass ${authoringFlags.map((name) => `--${name}`).join(', ')} to scrollcase new scroll.`); + } + const exampleTarget = flags.get('no-example') ? null : nativeExampleTarget(); + const result = await initProject({ root: workspace.root, scrollsDir: workspace.scrollsDir }); + for (const path of result.written) success(`Created ${path}`); + for (const path of result.skipped) info(`Kept ${path} (already present)`); + + let example = null; + const pixiVersion = text(flags, 'pixi-version') + ?? (exampleTarget ? EXAMPLE_PIXI_VERSION : null); + if (exampleTarget) { + const initializedWorkspace = workspace.configPath + ? workspace + : { ...workspace, configPath: join(workspace.root, SCROLLCASE_CONFIG_FILENAME) }; + example = await ensureExampleScroll({ + workspace: initializedWorkspace, + target: exampleTarget, + pixiVersion, + }); + if (example.created) { + success(`Created example scroll ${example.scrollRef}`); + } else { + info(`Kept example scroll ${example.scrollRef} (already present)`); + } + for (const path of example.written) info(path); + } + + const always = Boolean(flags.get('install-toolchain')); + const never = Boolean(flags.get('no-install-toolchain')); + const setup = await runInitDependencySetup({ + hasExample: Boolean(example), + confirmTypeScript: () => confirm( + `Install scrollcase, TypeScript, and tsx in ${workspace.root}?`, + ), + confirmPython: () => confirm( + 'Install scrollcase-consumer for Python?', + ), + choosePythonSource: async () => { + console.log(); + const selectedSource = await chooseCliValue( + 'Python consumer package source', + ['PyPI with pip', 'conda-forge with conda'], + ); + const source = selectedSource.startsWith('PyPI') ? 'pypi' : 'conda-forge'; + return resolvePythonConsumerSource({ + selectedSource: source, + condaAvailable: source === 'pypi' || isCondaAvailable({ root: workspace.root }), + confirmPyPIFallback: () => confirm( + 'Conda is not installed. Install scrollcase-consumer from PyPI with pip instead?', + ), + }); + }, + installToolchain: () => ensureToolchain({ + workspace, + pixiVersion, + confirm: async (missing) => { + if (never) return false; + if (always) return true; + console.log(); + return confirm(`This project needs ${missing.join(' and ')} to build a box.\nInstall ${missing.length > 1 ? 'them' : 'it'} into ${workspace.toolchainDir}?`); + }, + }), + installTypeScript: () => installTypeScriptConsumerDependencies({ root: workspace.root }), + installPython: (source) => installPythonConsumerDependency({ + root: workspace.root, + source, + }), + }); + const { toolchain } = setup; + + if (toolchain.installed.length > 0) { + success(`Installed ${toolchain.installed.join(' and ')} into ${workspace.toolchainDir}`); + info('Nothing was added to PATH; scrollcase finds them there on its own.'); + if (toolchain.configPath) success(`Recorded the toolchain pins in ${toolchain.configPath}`); + } else if (toolchain.unsupportedHost) { + warning(`pixi publishes no build for ${toolchain.unsupportedHost}; install ${toolchain.missing.join(' and ')} manually.`); + } else if (toolchain.missing.length > 0) { + warning(`Skipped installing ${toolchain.missing.join(' and ')}.`); + info('Install them yourself, or re-run with --install-toolchain. `scrollcase doctor` reports what is missing.'); + } + + if (setup.typescript) { + const installed = setup.typescript; + success( + `Installed scrollcase ${installed.scrollcaseVersion}, TypeScript, and tsx in ${workspace.root}`, + ); + } + + if (setup.python) { + const installed = setup.python; + success( + `Installed scrollcase-consumer from ${installed.source} using ${installed.command}`, + ); + } + + success('Workspace initialized'); + if (example) step(`Example: scrollcase lock ${example.scrollRef}`); + step(example ? 'Create your own: scrollcase new scroll' : 'Next: scrollcase new scroll'); +} + +/** `new scroll` — collect one complete authoring decision and create it atomically. */ +async function newScroll(flags) { + const workspace = getWorkspace(); + const options = await collectNewScrollOptions(flags); + const result = await createScroll({ workspace, ...options }); + success(`Created scroll ${result.scrollRef}`); + for (const path of result.written) info(path); + step(`Next: scrollcase lock ${result.scrollRef}`); +} + +/** `doctor` — report whether this machine can build a box. Reads only; never writes. */ +async function doctor(flags) { + let pixiVersion = text(flags, 'pixi-version'); + const scrollName = text(flags, 'scroll'); + if (!pixiVersion && scrollName) { + const reference = await selectScrollReference(scrollName, flags); + pixiVersion = (await readScroll(reference)).scroll.pixiVersion; + } + const { checks, ok } = await diagnose({ + workspace: getWorkspace(), + pixiVersion, + pixiPath: text(flags, 'pixi'), + condaPackPath: text(flags, 'conda-pack'), + }); + for (const check of checks) { + console.log(statusLine(check.ok ? 'success' : 'error', `${check.name.padEnd(11)} ${check.detail}`)); + if (!check.ok && check.remedy) console.log(` ${statusLine('step', check.remedy)}`); + } + if (!ok) fail('Some checks failed; see the remedies above.'); +} + +/** `audit` — the dependency licence inventory, derived from the lock without building. */ +async function audit(name, flags) { + const reference = await selectScrollReference(name, flags); + const write = Boolean(flags.get('write')); + const { summary, reviewed, written } = await auditScroll(reference, { + write, + namespace: text(flags, 'namespace') || undefined, + }); + info(`${summary.packageCount} packages for ${summary.scrollId} (${summary.targetId})`); + for (const entry of summary.licenses) console.log(` ${String(entry.count).padStart(4)} ${entry.license}`); + if (written) success(`Wrote reviewed audit: ${reviewed}`); + else if (reviewed) success(`Matches the reviewed audit: ${reviewed}`); +} + +async function build(name, flags) { + const reference = await selectScrollReference(name, flags); + const signing = { + ...keyPaths(flags), + signerCommand: text(flags, 'signer-command'), + }; + await ensureBuildSigningKeys(signing); + // Asked at the CLI edge and passed down: buildBox never reads a terminal itself. + const channel = await chooseCliValue( + 'channel', + ['beta', ...CHANNELS.filter((value) => value !== 'beta')], + { flag: text(flags, 'channel') }, + ); + const weights = await chooseCliValue( + 'weights mode', + ['embed', 'on-demand'], + { flag: text(flags, 'weights') }, + ); + step(`Building ${reference} (${channel}, ${weights})`); + const built = await buildBox(reference, { + ...signing, + allowDirty: Boolean(flags.get('allow-dirty')), + channel, + weights, + assetBaseUrl: text(flags, 'asset-base-url'), + namespace: text(flags, 'namespace') || undefined, + pixiPath: text(flags, 'pixi'), + condaPackPath: text(flags, 'conda-pack'), + log: (message) => { + if (!message || /^(Box:|Release:|Channel:|Publish:| {9}then )/.test(message)) return; + step(message); + }, + }); + const workspace = getWorkspace(); + success(buildDistributionSummary(built, workspace.distDir)); +} + +async function verify(path, flags) { + await verifyBox(path, { + publicPath: keyPaths(flags).publicPath, + archive: text(flags, 'archive'), + selfTest: Boolean(flags.get('self-test')), + }); +} + +async function runRelease(path, flags, args) { + return runCliBox(path, { + publicPath: keyPaths(flags).publicPath, + archive: text(flags, 'archive'), + args, + log: step, + }); +} + +function usage() { + console.log(`Usage: scrollcase [options] + scrollcase -v | --version + +Commands: + init Initialize a workspace with a runnable example + new scroll Create one guided target-specific scroll + doctor Report whether this machine can build a box + keygen Create a local ed25519 signing key + lock [] Resolve the scroll's pixi manifest into pixi.lock + audit Dependency licence inventory, derived from the lock + build [] Build, self-test, archive, and sign a box + verify Verify signature, archive hash, and layout + run Verify, temporarily extract, and run a local box + +Init options: + --pixi-version Install this pixi release when setup is approved + --no-example Initialize an empty workspace without example-box + --install-toolchain Install missing pixi/conda-pack without asking + --no-install-toolchain Never install them; just report what is missing + With neither flag, init asks before downloading anything, and + installs into after a verified checksum check. + When the example is present, init separately offers to install + its TypeScript and Python consumer dependencies in the project + root. Missing Conda offers a PyPI fallback. + +New scroll options: + --target Complete target, including the CUDA ABI when applicable + --box-id Box identity + --model-id Packaged model identity + --runtime-id Runtime identity + --version Box version + --scroll-version Scroll authoring version + --source-revision Upstream source revision recorded in provenance + --python-version Python dependency version + --pixi-version pixi resolver version + --min-host-app-version Minimum compatible host application version + --asset-base-url Base URL used in built release documents + --weights embed or on-demand + --execution python-script, python-module, or library-only + --script Existing project script for python-script + --generate-script Generate a minimal project script instead + --script-destination Payload path for the script (default entrypoint.py) + --generated-script-path Project path for a generated starter + --module Dotted module name for python-module + --default-args JSON array of default application arguments + --max-host-app-version-exclusive + --min-macos-version + --min-ram-gb + --min-nvidia-driver-version + Without a terminal, every material value must be supplied. + +Doctor options: + --scroll Take the required pixi version from this scroll + --target Select a target when is a box with several scrolls + --pixi-version Check for this pixi release + +Keygen options: + --key-id Identifier recorded in signatures (default derived from key) + --force Overwrite both named key files; unsafe for rotation + +Audit options: + --target Select a target when names a box + --write Write the inventory to the scroll's reviewed audit path + --namespace Document kind namespace (default scrollcase.box) + +Build options: + --target Select a target when names a box + --channel Channel the signed pointer names (nightly, beta, or stable; + default beta) + --weights embed (default: assets packed in, works air-gapped) or + on-demand (caller-materialized; verified before execution) + Without either flag, build shows an arrow-key menu. With no + terminal to ask, it says which default it took and carries on. + --asset-base-url Override the scroll's published base URL + --namespace Document kind namespace (default scrollcase.box) + --allow-dirty Permit a build from an uncommitted source tree + --pixi Use this pixi executable + --conda-pack Use this conda-pack executable (managed installs pin 0.9.2) + +Scroll targets: + lock, audit and build accept either / or a box ID plus + --target . With only a box ID, a terminal shows an arrow-key menu. + lock and build also let an interactive terminal choose from every workspace + scroll when the argument is omitted; non-interactive callers must name one. + A sole target for this host is the default; Metal is preferred on macOS. + Without a terminal, any other ambiguous target is an error. + +Verify options: + --archive Archive to check, if not beside the release document + --self-test Extract and import with the box's own interpreter + +Run: + scrollcase run [--archive ] -- [application args] + --archive Local archive, if not beside the release document + Uses --public-key from Signing below, attaches terminal stdio, + forwards signals, and exits with the application result. + +Signing: + --private-key Local signing key (default /signing-private.pem) + --public-key Trusted key set (default /signing-public.json) + --signer-command Sign through an external command instead of a local key. + It receives the payload on stdin and returns the signed + document as JSON on stdout; the result is verified locally. + Before build work starts, missing local keys fail with an + explicit instruction to run scrollcase keygen. + +Workspace: + Paths come from scrollcase.config.json at the project root, discovered by walking + up from the working directory, and can be overridden per invocation: + --config Use this workspace config explicitly + --project-root Treat this directory as the project root + --scrolls-dir Where scrolls live (default scrolls) + --build-dir Payload scratch space (default .scrollcase/build) + --out-dir Built artefacts (default .scrollcase/dist) + --keys-dir Local signing keys (default .scrollcase/keys) + --toolchain-dir Project-local pixi/conda-pack (default .scrollcase/toolchain) +`); +} + +async function main() { + const [command, ...rest] = process.argv.slice(2); + if (command === '-v' || command === '--version') { + console.log(SCROLLCASE_NPM_VERSION); + return; + } + const { positional, flags, passthrough } = parseArgs(rest); + if (!command || command === 'help' || command === '--help') return usage(); + // Resolve the workspace before any command touches a path, so flags win over the project config. + configureWorkspace({ overrides: workspaceOverridesFromFlags(flags) }); + if (command === 'init') return init(flags); + if (command === 'new') { + if (positional[0] !== 'scroll' || positional.length !== 1) { + fail('Usage: scrollcase new scroll [options]'); + } + return newScroll(flags); + } + if (command === 'doctor') return doctor(flags); + if (command === 'keygen') return keygen(flags); + if (command === 'audit') return audit(positional[0] || fail('audit requires a scroll name.'), flags); + if (command === 'lock') return lock(positional[0], flags); + if (command === 'build') return build(positional[0], flags); + if (command === 'verify') return verify(positional[0] || fail('verify requires a signed release document.'), flags); + if (command === 'run') { + if (positional.length !== 1) fail('Usage: scrollcase run [--archive ] -- [application args]'); + return runRelease(positional[0], flags, passthrough); + } + fail(`Unknown command: ${command}`); +} + +// Single failure path: every `fail()` anywhere lands here as a one-line message and a non-zero exit +// code, so CI and shell callers can rely on the status. +main().catch((error) => { + console.error(statusLine( + 'error', + `scrollcase: ${error instanceof Error ? error.message : String(error)}`, + { stream: process.stderr }, + )); + process.exitCode = 1; +}); diff --git a/.scrollcase-runtime-types-RzUfxr/src/consumer/index.mjs b/.scrollcase-runtime-types-RzUfxr/src/consumer/index.mjs new file mode 100644 index 0000000..22ef358 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/consumer/index.mjs @@ -0,0 +1,17 @@ +/** + * Local box preparation and execution. + * + * This surface composes with a caller's distribution policy; it does not become that policy. Every + * path, trust anchor, archive and destination comes from the caller, and no box code runs until the + * complete signed release and archive trust chain has passed. + */ + +/** @typedef {import('./verify-and-extract.mjs').PreparedBox} PreparedBox */ +/** @typedef {import('./verify-and-extract.mjs').RequiredAsset} RequiredAsset */ +/** @typedef {import('./run-extracted.mjs').BoxRunResult} BoxRunResult */ +/** @typedef {import('./run-extracted.mjs').RunExtractedBoxOptions} RunExtractedBoxOptions */ +/** @typedef {import('./run-box.mjs').RunBoxOptions} RunBoxOptions */ + +export { verifyAndExtractBox } from './verify-and-extract.mjs'; +export { runExtractedBox } from './run-extracted.mjs'; +export { runBox } from './run-box.mjs'; diff --git a/.scrollcase-runtime-types-RzUfxr/src/consumer/run-box.mjs b/.scrollcase-runtime-types-RzUfxr/src/consumer/run-box.mjs new file mode 100644 index 0000000..e4af208 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/consumer/run-box.mjs @@ -0,0 +1,46 @@ +/** + * One-shot local execution: prepare into a private temporary root, run, then remove every byte. + * + * The `finally` owns cleanup for every terminal path — normal exit, non-zero exit, spawn failure, + * or a forwarded signal. The child result is returned unchanged so callers retain application exit + * semantics instead of having them translated into a Scrollcase success/failure convention. + */ + +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { runExtractedBox } from './run-extracted.mjs'; +import { verifyAndExtractBox } from './verify-and-extract.mjs'; + +/** + * @typedef {import('./run-extracted.mjs').RunExtractedBoxOptions & { + * publicPath: string, + * archive?: string | null, + * temporaryDirectory?: string, + * onPrepared?: (prepared: Readonly) => + * void | Promise, + * }} RunBoxOptions + */ + +/** + * Verifies, temporarily extracts, and executes one caller-supplied local box. + * + * @param {string} releaseDocumentPath + * @param {RunBoxOptions} options + * @returns {Promise} + */ +export async function runBox(releaseDocumentPath, options) { + const temporaryParent = resolve(options.temporaryDirectory ?? tmpdir()); + const temporaryRoot = await mkdtemp(join(temporaryParent, 'scrollcase-run-')); + try { + const prepared = await verifyAndExtractBox(releaseDocumentPath, { + publicPath: options.publicPath, + archive: options.archive, + destination: join(temporaryRoot, 'box'), + }); + await options.onPrepared?.(prepared); + return await runExtractedBox(prepared, options); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/consumer/run-extracted.mjs b/.scrollcase-runtime-types-RzUfxr/src/consumer/run-extracted.mjs new file mode 100644 index 0000000..4fe2f29 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/consumer/run-extracted.mjs @@ -0,0 +1,161 @@ +/** + * Shell-free execution of a box that this process has already prepared. + * + * The verified manifest supplies the interpreter and script/module identity; the caller supplies + * only additional argument strings, streams, and environment values. Signals are forwarded while + * the child is alive and listeners are removed at the same point the result settles. + */ + +import { spawn as spawnProcess } from 'node:child_process'; +import { lstat } from 'node:fs/promises'; +import { join } from 'node:path'; +import { collectFiles, safeRelativePath, sha256File } from '../build/filesystem.mjs'; +import { assertExecutionFiles } from '../build/execution.mjs'; +import { fail } from '../build/process.mjs'; +import { assertNativeHost, boxTargetAdapter, boxTargetId } from '../contract/targets.mjs'; +import { preparedBoxState } from './verify-and-extract.mjs'; + +/** + * @typedef {'pipe' | 'overlapped' | 'ignore' | 'inherit' | number | + * import('node:stream').Stream | null | undefined} BoxStdio + */ + +/** + * @typedef {object} RunExtractedBoxOptions + * @property {readonly string[]} [args] + * @property {NodeJS.ProcessEnv} [env] values merged over the current process environment + * @property {BoxStdio} [stdin] + * @property {BoxStdio} [stdout] + * @property {BoxStdio} [stderr] + * @property {typeof spawnProcess} [spawn] injectable process seam + * @property {Pick} [signalSource] injectable signal seam + */ + +/** + * @typedef {object} BoxRunResult + * @property {number | null} exitCode + * @property {NodeJS.Signals | null} signal + */ + +const FORWARDED_SIGNALS = /** @type {const} */ (['SIGINT', 'SIGTERM', 'SIGHUP']); + +function stringArguments(values) { + if (!Array.isArray(values) || !values.every((value) => typeof value === 'string')) { + fail('Box execution arguments must be an array of strings.'); + } + return values; +} + +async function verifyRequiredAssets(root, assets) { + for (const asset of assets) { + const path = join(root, ...safeRelativePath(asset.relativePath).split('/')); + let metadata; + try { + metadata = await lstat(path); + } catch (error) { + if (error?.code === 'ENOENT') { + fail(`Required on-demand asset is missing: ${asset.relativePath}.`); + } + throw error; + } + if (!metadata.isFile()) fail(`Required on-demand asset is not a regular file: ${asset.relativePath}.`); + if (metadata.size !== asset.sizeBytes) { + fail(`Required on-demand asset size mismatch: ${asset.relativePath}.`); + } + if (await sha256File(path) !== asset.sha256) { + fail(`Required on-demand asset SHA-256 mismatch: ${asset.relativePath}.`); + } + } +} + +function waitForChild(child, signalSource) { + return new Promise((resolve, reject) => { + const handlers = new Map(); + const cleanup = () => { + for (const [signal, handler] of handlers) signalSource.removeListener(signal, handler); + handlers.clear(); + }; + const onError = (error) => { + cleanup(); + reject(error); + }; + const onClose = (exitCode, signal) => { + cleanup(); + resolve({ exitCode, signal }); + }; + child.once('error', onError); + child.once('close', onClose); + for (const signal of FORWARDED_SIGNALS) { + const handler = () => child.kill(signal); + handlers.set(signal, handler); + signalSource.on(signal, handler); + } + }); +} + +/** + * Executes a prepared box with its own interpreter and returns its terminal result. + * + * @param {import('./verify-and-extract.mjs').PreparedBox} prepared + * @param {RunExtractedBoxOptions} [options] + * @returns {Promise} + */ +export async function runExtractedBox(prepared, options = {}) { + const { release, rootIdentity } = preparedBoxState(prepared); + if (!release.execution) fail('Box does not declare an execution entry point.'); + const callerArgs = stringArguments(options.args ?? []); + const adapter = boxTargetAdapter(release.target); + try { + assertNativeHost(adapter); + } catch { + fail( + `Box target ${boxTargetId(release.target)} cannot run on ${process.platform}/${process.arch}; ` + + `requires ${adapter.host.platform}/${adapter.host.arch}.`, + ); + } + + let rootMetadata; + try { + rootMetadata = await lstat(prepared.root); + } catch (error) { + if (error?.code === 'ENOENT') { + fail('Prepared box root no longer matches the prepared box.'); + } + throw error; + } + if (!rootMetadata.isDirectory() + || rootMetadata.dev !== rootIdentity.device + || rootMetadata.ino !== rootIdentity.inode) { + fail('Prepared box root no longer matches the prepared box.'); + } + const files = new Set(await collectFiles(prepared.root)); + if (!files.has(release.pythonEntryPoint)) { + fail(`Prepared box is missing ${release.pythonEntryPoint}.`); + } + assertExecutionFiles({ + execution: release.execution, + adapter, + pythonVersion: release.provenance.pythonVersion, + files, + }); + await verifyRequiredAssets(prepared.root, prepared.requiredAssets); + + const python = join(prepared.root, ...safeRelativePath(release.pythonEntryPoint).split('/')); + const executionArgs = release.execution.kind === 'python-script' + ? [join(prepared.root, ...safeRelativePath(release.execution.script).split('/'))] + : ['-m', release.execution.module]; + executionArgs.push(...release.execution.defaultArgs, ...callerArgs); + + const spawn = options.spawn ?? spawnProcess; + const child = spawn(python, executionArgs, { + cwd: prepared.root, + env: { ...process.env, ...options.env }, + stdio: [ + options.stdin ?? 'inherit', + options.stdout ?? 'inherit', + options.stderr ?? 'inherit', + ], + shell: false, + }); + return waitForChild(child, options.signalSource ?? process); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/consumer/verify-and-extract.mjs b/.scrollcase-runtime-types-RzUfxr/src/consumer/verify-and-extract.mjs new file mode 100644 index 0000000..32e14f5 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/consumer/verify-and-extract.mjs @@ -0,0 +1,175 @@ +/** + * Verification and durable preparation of a caller-supplied local box. + * + * A prepared box is deliberately opaque. Its public receipt contains useful signed identity and + * audit data, while private state binds that exact object to the release Scrollcase verified. A + * caller therefore cannot construct an object that looks prepared and use it to bypass the trust + * chain before execution. + */ + +import { + lstat, + mkdir, + mkdtemp, + rename, + rm, +} from 'node:fs/promises'; +import { basename, dirname, join, resolve } from 'node:path'; +import { extractZipArchive } from '../build/archive.mjs'; +import { payloadSize, safeRelativePath, sha256File } from '../build/filesystem.mjs'; +import { fail } from '../build/process.mjs'; +import { inspectBoxArchive } from '../build/verify.mjs'; +import { boxTargetId } from '../contract/targets.mjs'; + +/** + * An on-demand asset whose signed bytes the caller must place under `root` before execution. + * + * @typedef {object} RequiredAsset + * @property {string} url + * @property {string} relativePath + * @property {number} sizeBytes + * @property {string} sha256 + */ + +/** + * The immutable result of a successfully verified and atomically prepared local box. + * + * @typedef {object} PreparedBox + * @property {'prepared'} status + * @property {string} root absolute extracted box root + * @property {string} boxId + * @property {string} modelId + * @property {string} runtimeId + * @property {string} version + * @property {import('../contract/types/index.d.ts').BoxTarget} target + * @property {string} targetId + * @property {string} pythonEntryPoint + * @property {import('../contract/types/index.d.ts').BoxExecution | null} execution + * @property {readonly RequiredAsset[]} requiredAssets assets the caller must materialize, never + * downloaded by Scrollcase + * @property {readonly string[]} signingKeyIds + * @property {string} releasePayloadSha256 + * @property {string} archiveSha256 + * @property {number} archiveSizeBytes + * @property {number} installedSizeBytes logical size of the verified extracted payload + */ + +/** @type {WeakMap} */ +const preparedBoxes = new WeakMap(); + +function freezeValue(value) { + if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value; + for (const nested of Object.values(value)) freezeValue(nested); + return Object.freeze(value); +} + +async function pathExists(path) { + try { + await lstat(path); + return true; + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +/** + * Returns the private, verified release bound to a prepared receipt. + * + * This is internal to the consumer module graph; it is not re-exported from the package surface. + */ +export function preparedBoxState(/** @type {unknown} */ prepared) { + const state = preparedBoxes.get(prepared); + if (!state) fail('Expected a PreparedBox returned by verifyAndExtractBox().'); + return state; +} + +/** + * Verifies and extracts one local box without executing any code from it. + * + * The destination must not exist. Extraction happens in a fresh sibling directory so the final + * rename stays on one filesystem and exposes either the complete verified tree or nothing. + * + * @param {string} releaseDocumentPath + * @param {{ publicPath: string, archive?: string | null, destination: string }} options + * @returns {Promise>} + */ +export async function verifyAndExtractBox(releaseDocumentPath, { + publicPath, + archive = null, + destination, +}) { + if (!destination) fail('A destination is required to prepare a box.'); + const finalRoot = resolve(destination); + if (await pathExists(finalRoot)) fail(`Destination already exists: ${finalRoot}`); + + const inspected = await inspectBoxArchive(releaseDocumentPath, { publicPath, archive }); + const { + archivePath, + signed, + release, + } = inspected; + const requiredAssets = release.weights === 'on-demand' ? release.assets : []; + for (const asset of requiredAssets) safeRelativePath(asset.relativePath); + + const parent = dirname(finalRoot); + await mkdir(parent, { recursive: true }); + if (await pathExists(finalRoot)) fail(`Destination already exists: ${finalRoot}`); + + const stageRoot = await mkdtemp(join(parent, `.scrollcase-prepare-${basename(finalRoot)}-`)); + const extractedRoot = join(stageRoot, 'payload'); + try { + await extractZipArchive(archivePath, extractedRoot); + const extractedSize = await payloadSize(extractedRoot); + if (release.installedSizeBytes !== undefined + && extractedSize !== release.installedSizeBytes) { + fail('Extracted payload size does not match the signed release.'); + } + + // Re-check the source after extraction. This catches a local archive being replaced between + // the initial trust decision and the move into the caller's durable destination. + if (await sha256File(archivePath) !== release.archive.sha256) { + fail('Archive SHA-256 changed during extraction.'); + } + const stagedMetadata = await lstat(extractedRoot); + if (await pathExists(finalRoot)) fail(`Destination already exists: ${finalRoot}`); + await rename(extractedRoot, finalRoot); + const installedMetadata = await lstat(finalRoot); + if (installedMetadata.dev !== stagedMetadata.dev || installedMetadata.ino !== stagedMetadata.ino) { + fail('Prepared destination identity changed during installation.'); + } + + const frozenRelease = freezeValue(release); + const receipt = freezeValue({ + status: 'prepared', + root: finalRoot, + boxId: release.boxId, + modelId: release.modelId, + runtimeId: release.runtimeId, + version: release.version, + target: release.target, + targetId: boxTargetId(release.target), + pythonEntryPoint: release.pythonEntryPoint, + execution: release.execution ?? null, + requiredAssets, + signingKeyIds: signed.signatures.map((signature) => signature.keyId), + releasePayloadSha256: signed.payloadSha256, + archiveSha256: release.archive.sha256, + archiveSizeBytes: release.archive.sizeBytes, + installedSizeBytes: extractedSize, + }); + preparedBoxes.set(receipt, { + release: frozenRelease, + rootIdentity: { + device: installedMetadata.dev, + inode: installedMetadata.ino, + }, + }); + return receipt; + } finally { + await rm(stageRoot, { recursive: true, force: true }); + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/browser.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/browser.mjs new file mode 100644 index 0000000..1141613 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/browser.mjs @@ -0,0 +1,28 @@ +/** + * Browser-safe reference helpers for the Scrollcase contract. + * + * The full `scrollcase/contract` entry point also decodes and hashes signed payloads through Node's + * crypto implementation. Consumers that only need target identity, document names, constants, or + * the structural envelope guard can use this entry point in browsers, Workers, and Node alike. + */ + +export { + assertNativeHost, + assertPythonEntryPoint, + condaSubdir, + pixiAccelerator, + boxTargetAdapter, + boxTargetAdapters, + boxTargetId, +} from './targets.mjs'; + +export { + CHANNELS, + DEFAULT_DOCUMENT_NAMESPACE, + PAYLOAD_ENCODING, + BOX_SCHEMA_VERSION, + SIGNATURE_ALGORITHM, + documentKinds, + isSignedBoxDocument, + parseDocumentKind, +} from './document-shape.mjs'; diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/document-shape.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/document-shape.mjs new file mode 100644 index 0000000..2aa9fbc --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/document-shape.mjs @@ -0,0 +1,92 @@ +/** + * Platform-neutral parts of the signed-document contract. + * + * Shape checks and namespacing have no reason to depend on Node. Keeping them in this module lets + * browser and Worker consumers share the reference implementation without pulling in the + * cryptographic decoder, while the main contract entry point continues to expose the complete API. + */ + +/** Format version carried by every document this contract describes. */ +export const BOX_SCHEMA_VERSION = 2; + +/** The only payload encoding the format defines. */ +export const PAYLOAD_ENCODING = 'base64-json-utf8'; + +/** The only signature algorithm the format defines. */ +export const SIGNATURE_ALGORITHM = 'ed25519'; + +/** + * Namespace prefixing every document's `kind` discriminator. + * + * A project that already publishes boxes owns its own namespace and must keep emitting it, or its + * installed clients stop recognizing the documents they are handed. So the namespace is the + * consumer's to declare, not the tool's to impose: this is only the default used by a project that + * has no published history to preserve. + */ +export const DEFAULT_DOCUMENT_NAMESPACE = 'scrollcase.box'; + +const DOCUMENT_TYPES = Object.freeze(['release', 'channel', 'revocations']); +const NAMESPACE_PATTERN = /^[a-z0-9]+(?:[.-][a-z0-9]+)*$/; + +/** + * Returns the `kind` discriminator for each document type under a namespace. + * + * Pass the namespace a project has already published under to keep its documents byte-compatible; + * omit it for a new project. + * + * @param {string} [namespace] defaults to `scrollcase.box` + * @returns {Readonly<{ release: string, channel: string, revocations: string }>} + * @throws {TypeError} when the namespace is not a dotted lowercase identifier + */ +export function documentKinds(namespace = DEFAULT_DOCUMENT_NAMESPACE) { + if (typeof namespace !== 'string' || !NAMESPACE_PATTERN.test(namespace)) { + throw new TypeError(`Invalid document namespace: ${namespace}`); + } + return Object.freeze(Object.fromEntries( + DOCUMENT_TYPES.map((type) => [type, `${namespace}.${type}`]), + )); +} + +/** + * Splits a `kind` back into its namespace and document type, or returns null if it is not one. + * + * @param {unknown} kind + * @returns {{ namespace: string, type: 'release' | 'channel' | 'revocations' } | null} null when + * the value is not a document kind at all + */ +export function parseDocumentKind(kind) { + if (typeof kind !== 'string') return null; + const separator = kind.lastIndexOf('.'); + if (separator <= 0) return null; + const namespace = kind.slice(0, separator); + const type = kind.slice(separator + 1); + if (!DOCUMENT_TYPES.includes(type) || !NAMESPACE_PATTERN.test(namespace)) return null; + return { namespace, type }; +} + +/** Channels a box may be published to, ordered from least to most stable. */ +export const CHANNELS = Object.freeze(['nightly', 'beta', 'stable']); + +/** + * Reports whether a value is a structurally valid signed envelope. + * + * This is a shape check, not a verification: it says the document is worth attempting to verify, + * never that its signature is good. Callers must still verify the payload hash and at least one + * signature against a trusted key before acting on the contents. + * + * @param {unknown} value + * @returns {value is import('./types/index.d.ts').SignedBoxDocument} true when the envelope is + * well formed and therefore worth verifying — never that its signature is valid + */ +export function isSignedBoxDocument(value) { + if (!value || typeof value !== 'object') return false; + return value.schemaVersion === BOX_SCHEMA_VERSION + && value.payloadEncoding === PAYLOAD_ENCODING + && typeof value.payloadBase64 === 'string' + && typeof value.payloadSha256 === 'string' + && Array.isArray(value.signatures) + && value.signatures.length > 0 + && value.signatures.every((signature) => signature?.algorithm === SIGNATURE_ALGORITHM + && typeof signature.keyId === 'string' + && typeof signature.signatureBase64 === 'string'); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/documents.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/documents.mjs new file mode 100644 index 0000000..78f12a4 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/documents.mjs @@ -0,0 +1,48 @@ +/** + * Reference implementation of the Scrollcase signed-document envelope. + * + * Signed documents carry their payload as exact base64-encoded JSON rather than canonicalized JSON. + * That choice is deliberate: verifying a signature then means hashing bytes that were transmitted + * verbatim, so Node, Rust, a Worker, and any future client agree without each maintaining a + * canonical-JSON implementation — historically the richest source of cross-language signature bugs. + */ + +import { createHash } from 'node:crypto'; +import { isSignedBoxDocument } from './document-shape.mjs'; + +export { + BOX_SCHEMA_VERSION, + CHANNELS, + DEFAULT_DOCUMENT_NAMESPACE, + PAYLOAD_ENCODING, + SIGNATURE_ALGORITHM, + documentKinds, + isSignedBoxDocument, + parseDocumentKind, +} from './document-shape.mjs'; + +/** + * Decodes an envelope's payload without verifying any signature. + * + * Throws when the envelope is malformed or when the embedded payload hash does not match the bytes, + * which catches a truncated or edited document before its contents are ever read. + * + * @param {import('./types/index.d.ts').SignedBoxDocument} document + * @returns {unknown} the decoded payload, still unverified + * @throws {TypeError} when the envelope is malformed + * @throws {Error} when the embedded payload hash does not match the bytes + */ +export function decodeDocumentPayload(document) { + if (document?.schemaVersion === 1) { + throw new TypeError('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + } + if (!isSignedBoxDocument(document)) { + throw new TypeError('Not a signed box document'); + } + const bytes = Buffer.from(document.payloadBase64, 'base64'); + const digest = createHash('sha256').update(bytes).digest('hex'); + if (digest !== document.payloadSha256) { + throw new Error('Signed box payload hash does not match its bytes'); + } + return JSON.parse(bytes.toString('utf8')); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/consumer-conformance.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/consumer-conformance.json new file mode 100644 index 0000000..6b6f236 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/consumer-conformance.json @@ -0,0 +1,516 @@ +{ + "schemaVersion": 1, + "description": "Language-neutral semantic cases shared by the Node and Python Scrollcase consumers.", + "errorPatterns": { + "invalid-signature": "no valid signature", + "altered-payload": "Signed payload SHA-256 mismatch", + "archive-hash": "Archive SHA-256 mismatch", + "archive-size": "Archive size mismatch", + "manifest-disagreement": "box.json mismatch: modelId", + "execution-disagreement": "box.json mismatch: execution", + "missing-interpreter": "Archive is missing venv/", + "missing-script": "Execution script is missing", + "missing-module": "Execution module is not discoverable", + "unsafe-path": "Unsafe relative path|invalid relative path|absolute path", + "link-entry": "link does not resolve to a file inside the payload|would be written through a link|link target is too long", + "special-entry": "special entries", + "encrypted-entry": "Encrypted ZIP entries", + "entry-collision": "Archive entry collides with another entry", + "existing-destination": "Destination already exists", + "asset-missing": "asset is missing", + "asset-size": "asset size mismatch", + "asset-hash": "asset SHA-256 mismatch", + "spawn-failure": "failed to start|fixture spawn failed" + }, + "cases": [ + { + "id": "valid-local-signer", + "action": "prepare", + "fixture": { + "signer": "local" + }, + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "pythonEntryPoint": "$NATIVE_PYTHON", + "targetId": "$NATIVE_TARGET" + } + } + }, + { + "id": "valid-external-signer", + "action": "prepare", + "fixture": { + "signer": "external" + }, + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "pythonEntryPoint": "$NATIVE_PYTHON", + "targetId": "$NATIVE_TARGET" + } + } + }, + { + "id": "altered-signature", + "action": "prepare", + "mutation": "alter-signature", + "expected": { + "outcome": "rejected", + "error": "invalid-signature", + "destinationExists": false + } + }, + { + "id": "altered-payload", + "action": "prepare", + "mutation": "alter-payload", + "expected": { + "outcome": "rejected", + "error": "altered-payload", + "destinationExists": false + } + }, + { + "id": "altered-archive-hash", + "action": "prepare", + "mutation": "alter-archive-bytes", + "expected": { + "outcome": "rejected", + "error": "archive-hash", + "destinationExists": false + } + }, + { + "id": "altered-archive-size", + "action": "prepare", + "mutation": "alter-archive-size", + "expected": { + "outcome": "rejected", + "error": "archive-size", + "destinationExists": false + } + }, + { + "id": "release-box-disagreement", + "action": "prepare", + "mutation": "alter-release-model", + "expected": { + "outcome": "rejected", + "error": "manifest-disagreement", + "destinationExists": false + } + }, + { + "id": "altered-execution-metadata", + "action": "prepare", + "mutation": "alter-release-execution", + "expected": { + "outcome": "rejected", + "error": "execution-disagreement", + "destinationExists": false + } + }, + { + "id": "missing-interpreter", + "action": "prepare", + "mutation": "remove-interpreter", + "expected": { + "outcome": "rejected", + "error": "missing-interpreter", + "destinationExists": false + } + }, + { + "id": "missing-script", + "action": "prepare", + "mutation": "remove-script", + "expected": { + "outcome": "rejected", + "error": "missing-script", + "destinationExists": false + } + }, + { + "id": "missing-module", + "action": "prepare", + "fixture": { + "execution": "module" + }, + "mutation": "remove-module", + "expected": { + "outcome": "rejected", + "error": "missing-module", + "destinationExists": false + } + }, + { + "id": "traversal-entry", + "action": "prepare", + "mutation": "add-traversal-entry", + "expected": { + "outcome": "rejected", + "error": "unsafe-path", + "destinationExists": false + } + }, + { + "id": "absolute-entry", + "action": "prepare", + "mutation": "add-absolute-entry", + "expected": { + "outcome": "rejected", + "error": "unsafe-path", + "destinationExists": false + } + }, + { + "id": "link-entry", + "action": "prepare", + "mutation": "add-link-entry", + "expected": { + "outcome": "rejected", + "error": "link-entry", + "destinationExists": false + } + }, + { + "id": "linked-interpreter", + "action": "prepare", + "mutation": "link-interpreter", + "requiresSymlinks": true, + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "pythonEntryPoint": "$NATIVE_PYTHON", + "targetId": "$NATIVE_TARGET" + } + } + }, + { + "id": "special-entry", + "action": "prepare", + "mutation": "add-special-entry", + "expected": { + "outcome": "rejected", + "error": "special-entry", + "destinationExists": false + } + }, + { + "id": "encrypted-entry", + "action": "prepare", + "mutation": "encrypt-entry", + "expected": { + "outcome": "rejected", + "error": "encrypted-entry", + "destinationExists": false + } + }, + { + "id": "extraction-collision", + "action": "prepare", + "mutation": "duplicate-entry", + "expected": { + "outcome": "rejected", + "error": "entry-collision", + "destinationExists": false + } + }, + { + "id": "file-directory-collision", + "action": "prepare", + "mutation": "file-directory-collision", + "expected": { + "outcome": "rejected", + "error": "entry-collision", + "destinationExists": false + } + }, + { + "id": "existing-destination", + "action": "prepare", + "mutation": "create-destination", + "expected": { + "outcome": "rejected", + "error": "existing-destination", + "destinationExists": true + } + }, + { + "id": "macos-entry-point", + "action": "prepare", + "fixture": { + "target": "macos-aarch64-cpu" + }, + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "pythonEntryPoint": "venv/bin/python", + "targetId": "macos-aarch64-cpu" + } + } + }, + { + "id": "linux-entry-point", + "action": "prepare", + "fixture": { + "target": "linux-x86_64-cpu" + }, + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "pythonEntryPoint": "venv/bin/python", + "targetId": "linux-x86_64-cpu" + } + } + }, + { + "id": "windows-entry-point", + "action": "prepare", + "fixture": { + "target": "windows-x86_64-cpu" + }, + "expected": { + "outcome": "prepared", + "receipt": { + "status": "prepared", + "boxId": "consumer-fixture", + "executionKind": "python-script", + "requiredAssetCount": 0, + "pythonEntryPoint": "venv/python.exe", + "targetId": "windows-x86_64-cpu" + } + } + }, + { + "id": "persistent-prepared-execution", + "action": "run-prepared", + "runtime": { + "exitCode": 0 + }, + "expected": { + "outcome": "completed", + "result": { + "exitCode": 0, + "signal": null + }, + "persistentRootExists": true, + "spawned": true + } + }, + { + "id": "default-user-argument-ordering", + "action": "run-prepared", + "runtime": { + "args": [ + "--caller", + "caller value" + ], + "exitCode": 0, + "inspectInvocation": true + }, + "expected": { + "outcome": "completed", + "result": { + "exitCode": 0, + "signal": null + }, + "argv": [ + "$BOX/$NATIVE_PYTHON", + "$BOX/app/main.py", + "--default", + "value with spaces", + "--caller", + "caller value" + ], + "cwd": "$BOX", + "shell": false + } + }, + { + "id": "shell-metacharacter-preservation", + "action": "run-prepared", + "runtime": { + "args": [ + "$(touch never)", + "semi;colon", + "quote'\"value" + ], + "exitCode": 0, + "inspectInvocation": true + }, + "expected": { + "outcome": "completed", + "result": { + "exitCode": 0, + "signal": null + }, + "argv": [ + "$BOX/$NATIVE_PYTHON", + "$BOX/app/main.py", + "--default", + "value with spaces", + "$(touch never)", + "semi;colon", + "quote'\"value" + ], + "cwd": "$BOX", + "shell": false + } + }, + { + "id": "standard-stream-forwarding", + "action": "run-prepared", + "runtime": { + "exitCode": 0, + "streams": true + }, + "expected": { + "outcome": "completed", + "result": { + "exitCode": 0, + "signal": null + }, + "streamsPreserved": true + } + }, + { + "id": "child-non-zero-exit", + "action": "run-prepared", + "runtime": { + "exitCode": 23 + }, + "expected": { + "outcome": "completed", + "result": { + "exitCode": 23, + "signal": null + }, + "spawned": true + } + }, + { + "id": "signal-forwarding", + "action": "run-prepared", + "runtime": { + "signal": "SIGTERM" + }, + "expected": { + "outcome": "completed", + "result": { + "exitCode": null, + "signal": "SIGTERM" + }, + "forwardedSignal": "SIGTERM" + } + }, + { + "id": "temporary-cleanup-success", + "action": "run-box", + "runtime": { + "exitCode": 0 + }, + "expected": { + "outcome": "completed", + "result": { + "exitCode": 0, + "signal": null + }, + "temporaryDirectoryEmpty": true + } + }, + { + "id": "temporary-cleanup-failure", + "action": "run-box", + "runtime": { + "spawnError": true + }, + "expected": { + "outcome": "rejected", + "error": "spawn-failure", + "temporaryDirectoryEmpty": true + } + }, + { + "id": "temporary-cleanup-signal", + "action": "run-box", + "runtime": { + "signal": "SIGTERM" + }, + "expected": { + "outcome": "completed", + "result": { + "exitCode": null, + "signal": "SIGTERM" + }, + "temporaryDirectoryEmpty": true + } + }, + { + "id": "on-demand-asset-missing", + "action": "run-prepared", + "fixture": { + "requiredAsset": true + }, + "runtime": { + "assetState": "missing" + }, + "expected": { + "outcome": "rejected", + "error": "asset-missing", + "spawned": false + } + }, + { + "id": "on-demand-asset-size-mismatch", + "action": "run-prepared", + "fixture": { + "requiredAsset": true + }, + "runtime": { + "assetState": "wrong-size" + }, + "expected": { + "outcome": "rejected", + "error": "asset-size", + "spawned": false + } + }, + { + "id": "on-demand-asset-hash-mismatch", + "action": "run-prepared", + "fixture": { + "requiredAsset": true + }, + "runtime": { + "assetState": "wrong-hash" + }, + "expected": { + "outcome": "rejected", + "error": "asset-hash", + "spawned": false + } + } + ] +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/box-manifest.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/box-manifest.example.json new file mode 100644 index 0000000..3613501 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/box-manifest.example.json @@ -0,0 +1,32 @@ +{ + "schemaVersion": 2, + "boxId": "example-model", + "modelId": "example-org-example-model", + "runtimeId": "example-model-runtime", + "version": "1.0.0", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "pythonEntryPoint": "venv/bin/python", + "modelCacheSubdir": "model-cache/example-model", + "selfTest": { + "pythonImports": [ + "torch", + "numpy" + ], + "timeoutSeconds": 180 + }, + "provenance": { + "scrollId": "example-model-macos-arm64-metal", + "scrollVersion": "1.0.0", + "builderRevision": "4c1d9b7e2a0f5836d419e7c05b3a8f61d2704e93", + "sourceTreeDirty": false, + "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", + "pythonVersion": "3.11.15", + "pixiVersion": "0.73.0", + "dependencyLockSha256": "3b1f8c47a2d9e05b6c7418af23d5e69017b4c8ad91e2f350768bd4ca19e0f5b7", + "builtAt": "2026-07-25T12:00:00+00:00" + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/channel-manifest.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/channel-manifest.example.json new file mode 100644 index 0000000..7682e91 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/channel-manifest.example.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 2, + "kind": "scrollcase.box.channel", + "channel": "beta", + "boxId": "example-model", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "updatedAt": "2026-07-25T12:05:00+00:00", + "cohortSalt": "41547ba146d88df877b97a331854567e", + "releases": [ + { + "version": "1.0.0", + "releaseManifestUrl": "https://assets.example.org/boxes/boxes/example-model/1.0.0/macos-aarch64-metal/c1a0f67d97543c9f0deccffdd00fbad45662ceed828f84e866b997ab4b019d1f.release.json", + "rolloutPercentage": 100 + } + ] +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/release-manifest.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/release-manifest.example.json new file mode 100644 index 0000000..9a04e5b --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/release-manifest.example.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 2, + "kind": "scrollcase.box.release", + "boxId": "example-model", + "modelId": "example-org-example-model", + "runtimeId": "example-model-runtime", + "version": "1.0.0", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minMacosVersion": "13.0", + "minRamGb": 8 + }, + "archive": { + "format": "zip", + "url": "https://assets.example.org/boxes/boxes/example-model/1.0.0/macos-aarch64-metal/7d2c9a41e8b350f6c174a9de20358bf41c6e97d05a8b3f2619e4c7081da5b3f2.zip", + "sha256": "7d2c9a41e8b350f6c174a9de20358bf41c6e97d05a8b3f2619e4c7081da5b3f2", + "sizeBytes": 655752216 + }, + "installedSizeBytes": 1892340112, + "pythonEntryPoint": "venv/bin/python", + "modelCacheSubdir": "model-cache/example-model", + "selfTest": { + "pythonImports": [ + "torch", + "numpy" + ], + "timeoutSeconds": 180 + }, + "provenance": { + "scrollId": "example-model-macos-arm64-metal", + "scrollVersion": "1.0.0", + "builderRevision": "4c1d9b7e2a0f5836d419e7c05b3a8f61d2704e93", + "sourceTreeDirty": false, + "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", + "pythonVersion": "3.11.15", + "pixiVersion": "0.73.0", + "dependencyLockSha256": "3b1f8c47a2d9e05b6c7418af23d5e69017b4c8ad91e2f350768bd4ca19e0f5b7", + "builtAt": "2026-07-25T12:00:00+00:00" + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll-pixi.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll-pixi.example.json new file mode 100644 index 0000000..b758030 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll-pixi.example.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "schemaVersion": 2, + "scrollId": "example-model-linux-x86_64-cuda12.9", + "scrollVersion": "1.0.0", + "boxId": "example-model", + "modelId": "example-org-example-model", + "runtimeId": "example-model-runtime", + "version": "1.0.0", + "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", + "target": { + "platform": "linux", + "arch": "x86_64", + "accelerator": "cuda", + "cudaVersion": "12.9" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minRamGb": 16, + "minNvidiaDriverVersion": "525.60.13" + }, + "pythonVersion": "3.11.15", + "pixiVersion": "0.73.0", + "condaDependencyLicenseAudit": "legal/audits/example-model-linux-x86_64-cuda12.9.json", + "pythonEntryPoint": "venv/bin/python", + "modelCacheSubdir": "model-cache/example-model", + "assetBaseUrl": "https://assets.example.org/boxes", + "assets": [ + { + "url": "https://assets.example.org/example-model/weights.safetensors", + "relativePath": "model-cache/example-model/weights.safetensors", + "sizeBytes": 205385258, + "sha256": "6cb5d451ab5c4b33eb673adbe4fddc61d2389df1b89b7651a9fe2e557572b922" + }, + { + "url": "https://codeload.example.org/example-org/example-model/zip/9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", + "relativePath": ".sources/example-model-source.zip", + "sizeBytes": 12117557, + "sha256": "1b825a687b4855dbb8c7b530f17cc246ef677d93d0e2df3fbc82330ff9d189db" + } + ], + "assetArchives": [ + { + "relativePath": ".sources/example-model-source.zip", + "format": "zip", + "destination": "source/example-model", + "stripComponents": 1, + "removeAfterExtract": true + } + ], + "localFiles": [ + { + "sourcePath": "legal/notices/example-model-THIRD-PARTY.txt", + "relativePath": "THIRD_PARTY_NOTICES/example-model.txt", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + ], + "prunePaths": [ + "source/example-model/docs", + "source/example-model/tests", + "venv/lib/python3.11/site-packages/pip", + "venv/lib/python3.11/tkinter" + ], + "selfTest": { + "imports": ["torch", "numpy"], + "files": [ + "model-cache/example-model/weights.safetensors", + "source/example-model/LICENSE" + ], + "pythonCode": "assert torch.cuda.is_available()" + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll.example.json new file mode 100644 index 0000000..4790c73 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/scroll.example.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "schemaVersion": 2, + "scrollId": "example-model-macos-arm64-metal", + "scrollVersion": "1.0.0", + "boxId": "example-model", + "modelId": "example-org-example-model", + "runtimeId": "example-model-runtime", + "version": "1.0.0", + "sourceRevision": "9f0d3a1c4b5e6f7081920a3b4c5d6e7f80910a2b", + "target": { + "platform": "macos", + "arch": "aarch64", + "accelerator": "metal" + }, + "compatibility": { + "minHostAppVersion": "1.0.0", + "minMacosVersion": "13.0", + "minRamGb": 8 + }, + "pythonVersion": "3.11.15", + "pythonEntryPoint": "venv/bin/python", + "modelCacheSubdir": "model-cache/example-model", + "assetBaseUrl": "https://assets.example.org/boxes", + "assets": [ + { + "url": "https://assets.example.org/example-model/weights.safetensors", + "relativePath": "model-cache/example-model/weights.safetensors", + "sizeBytes": 205385258, + "sha256": "6cb5d451ab5c4b33eb673adbe4fddc61d2389df1b89b7651a9fe2e557572b922" + } + ], + "selfTest": { + "imports": [ + "torch", + "numpy" + ], + "files": [ + "model-cache/example-model/weights.safetensors" + ] + }, + "pixiVersion": "0.73.0", + "condaDependencyLicenseAudit": "legal/audits/example-model-macos-arm64-metal.json" +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.example.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.example.json new file mode 100644 index 0000000..64b1a63 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.example.json @@ -0,0 +1,13 @@ +{ + "schemaVersion": 2, + "payloadEncoding": "base64-json-utf8", + "payloadBase64": "ewogICJzY2hlbWFWZXJzaW9uIjogMiwKICAia2luZCI6ICJzY3JvbGxjYXNlLmJveC5yZWxlYXNlIiwKICAiYm94SWQiOiAiZXhhbXBsZS1tb2RlbCIsCiAgIm1vZGVsSWQiOiAiZXhhbXBsZS1vcmctZXhhbXBsZS1tb2RlbCIsCiAgInJ1bnRpbWVJZCI6ICJleGFtcGxlLW1vZGVsLXJ1bnRpbWUiLAogICJ2ZXJzaW9uIjogIjEuMC4wIiwKICAidGFyZ2V0IjogewogICAgInBsYXRmb3JtIjogIm1hY29zIiwKICAgICJhcmNoIjogImFhcmNoNjQiLAogICAgImFjY2VsZXJhdG9yIjogIm1ldGFsIgogIH0sCiAgImNvbXBhdGliaWxpdHkiOiB7CiAgICAibWluSG9zdEFwcFZlcnNpb24iOiAiMS4wLjAiLAogICAgIm1pbk1hY29zVmVyc2lvbiI6ICIxMy4wIiwKICAgICJtaW5SYW1HYiI6IDgKICB9LAogICJhcmNoaXZlIjogewogICAgImZvcm1hdCI6ICJ6aXAiLAogICAgInVybCI6ICJodHRwczovL2Fzc2V0cy5leGFtcGxlLm9yZy9ib3hlcy9ib3hlcy9leGFtcGxlLW1vZGVsLzEuMC4wL21hY29zLWFhcmNoNjQtbWV0YWwvN2QyYzlhNDFlOGIzNTBmNmMxNzRhOWRlMjAzNThiZjQxYzZlOTdkMDVhOGIzZjI2MTllNGM3MDgxZGE1YjNmMi56aXAiLAogICAgInNoYTI1NiI6ICI3ZDJjOWE0MWU4YjM1MGY2YzE3NGE5ZGUyMDM1OGJmNDFjNmU5N2QwNWE4YjNmMjYxOWU0YzcwODFkYTViM2YyIiwKICAgICJzaXplQnl0ZXMiOiA2NTU3NTIyMTYKICB9LAogICJpbnN0YWxsZWRTaXplQnl0ZXMiOiAxODkyMzQwMTEyLAogICJweXRob25FbnRyeVBvaW50IjogInZlbnYvYmluL3B5dGhvbiIsCiAgIm1vZGVsQ2FjaGVTdWJkaXIiOiAibW9kZWwtY2FjaGUvZXhhbXBsZS1tb2RlbCIsCiAgInNlbGZUZXN0IjogewogICAgInB5dGhvbkltcG9ydHMiOiBbCiAgICAgICJ0b3JjaCIsCiAgICAgICJudW1weSIKICAgIF0sCiAgICAidGltZW91dFNlY29uZHMiOiAxODAKICB9LAogICJwcm92ZW5hbmNlIjogewogICAgInNjcm9sbElkIjogImV4YW1wbGUtbW9kZWwtbWFjb3MtYXJtNjQtbWV0YWwiLAogICAgInNjcm9sbFZlcnNpb24iOiAiMS4wLjAiLAogICAgImJ1aWxkZXJSZXZpc2lvbiI6ICI0YzFkOWI3ZTJhMGY1ODM2ZDQxOWU3YzA1YjNhOGY2MWQyNzA0ZTkzIiwKICAgICJzb3VyY2VUcmVlRGlydHkiOiBmYWxzZSwKICAgICJzb3VyY2VSZXZpc2lvbiI6ICI5ZjBkM2ExYzRiNWU2ZjcwODE5MjBhM2I0YzVkNmU3ZjgwOTEwYTJiIiwKICAgICJweXRob25WZXJzaW9uIjogIjMuMTEuMTUiLAogICAgInBpeGlWZXJzaW9uIjogIjAuNzMuMCIsCiAgICAiZGVwZW5kZW5jeUxvY2tTaGEyNTYiOiAiM2IxZjhjNDdhMmQ5ZTA1YjZjNzQxOGFmMjNkNWU2OTAxN2I0YzhhZDkxZTJmMzUwNzY4YmQ0Y2ExOWUwZjViNyIsCiAgICAiYnVpbHRBdCI6ICIyMDI2LTA3LTI1VDEyOjAwOjAwKzAwOjAwIgogIH0KfQo=", + "payloadSha256": "bbf60de7d31035b2bfcb98c6c57624220f3bf59900391189f5e55da955055bc6", + "signatures": [ + { + "algorithm": "ed25519", + "keyId": "scrollcase-example-v2", + "signatureBase64": "c4ftYhvfGwJicd3kK9DzyTj+HNvJeiCfwk049BKk8hxSgAMxxU8BE51Q51ZkSK0mPolXRJ9pz52HO3KoKD1mAA==" + } + ] +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.public-key.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.public-key.json new file mode 100644 index 0000000..f506853 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/examples/signed-release.public-key.json @@ -0,0 +1,5 @@ +{ + "algorithm": "ed25519", + "keyId": "scrollcase-example-v2", + "publicKeyBase64": "frGNF6Fa2cw9m5HvWYBacydXujD4+ldqo6xGSCnCFyc=" +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/target-id-contract.json b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/target-id-contract.json new file mode 100644 index 0000000..a18a9ed --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/fixtures/target-id-contract.json @@ -0,0 +1,64 @@ +{ + "valid": [ + { + "name": "macOS arm64 Metal", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal" }, + "targetId": "macos-aarch64-metal" + }, + { + "name": "macOS arm64 CPU", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "cpu" }, + "targetId": "macos-aarch64-cpu" + }, + { + "name": "Linux x86_64 CPU", + "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cpu" }, + "targetId": "linux-x86_64-cpu" + }, + { + "name": "Linux x86_64 CUDA 12.4", + "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "12.4" }, + "targetId": "linux-x86_64-cuda12.4" + }, + { + "name": "Windows x86_64 CPU", + "target": { "platform": "windows", "arch": "x86_64", "accelerator": "cpu" }, + "targetId": "windows-x86_64-cpu" + }, + { + "name": "Windows x86_64 CUDA 12.4", + "target": { "platform": "windows", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "12.4" }, + "targetId": "windows-x86_64-cuda12.4" + } + ], + "invalid": [ + { + "name": "CUDA without a version", + "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cuda" } + }, + { + "name": "CPU with a CUDA version", + "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cpu", "cudaVersion": "12.4" } + }, + { + "name": "Metal with a CUDA version", + "target": { "platform": "macos", "arch": "aarch64", "accelerator": "metal", "cudaVersion": "12.4" } + }, + { + "name": "CUDA version with a prefix", + "target": { "platform": "windows", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "cuda12.4" } + }, + { + "name": "CUDA version without a minor component", + "target": { "platform": "linux", "arch": "x86_64", "accelerator": "cuda", "cudaVersion": "12" } + }, + { + "name": "unsupported macOS Intel target", + "target": { "platform": "macos", "arch": "x86_64", "accelerator": "cpu" } + }, + { + "name": "unsupported Linux arm64 target", + "target": { "platform": "linux", "arch": "aarch64", "accelerator": "cpu" } + } + ] +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/index.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/index.mjs new file mode 100644 index 0000000..f3ec625 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/index.mjs @@ -0,0 +1,55 @@ +/** + * The Scrollcase box-format contract. + * + * This module is the single source of truth for what a box *is*: which targets exist, how a + * target is named, what layout the payload has, and the shape of every document a build emits. It + * ships three things that must never disagree — a reference implementation (this code), a + * machine-readable spec (`schema/*.json`), and golden fixtures (`fixtures/*.json`) that any other + * implementation can validate itself against. + * + * A consumer written in another language does not import this code; it mirrors the rules and proves + * the mirror against the fixtures. That is how clients in other languages stay honest. + */ + +export { + assertNativeHost, + assertPythonEntryPoint, + condaSubdir, + pixiAccelerator, + boxTargetAdapter, + boxTargetAdapters, + boxTargetId, +} from './targets.mjs'; + +export { + CHANNELS, + DEFAULT_DOCUMENT_NAMESPACE, + PAYLOAD_ENCODING, + BOX_SCHEMA_VERSION, + SIGNATURE_ALGORITHM, + decodeDocumentPayload, + documentKinds, + isSignedBoxDocument, + parseDocumentKind, +} from './documents.mjs'; + +/** + * Absolute URL of a shipped JSON Schema, for consumers that validate documents themselves. + * + * @param {'target' | 'scroll' | 'box-manifest' | 'release-manifest' | 'channel-manifest' + * | 'revocations-manifest' | 'signed-document'} name + * @returns {URL} + */ +export function schemaUrl(name) { + return new URL(`./schema/${name}.schema.json`, import.meta.url); +} + +/** + * Absolute URL of a shipped fixture file, for consumers proving a mirror implementation. + * + * @param {string} name fixture file name without its extension + * @returns {URL} + */ +export function fixtureUrl(name) { + return new URL(`./fixtures/${name}.json`, import.meta.url); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/links.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/links.mjs new file mode 100644 index 0000000..6619706 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/links.mjs @@ -0,0 +1,162 @@ +/** + * The rule deciding which symbolic links a box payload may carry. + * + * A conda prefix is dense with links: the shared-library soname convention alone stores every large + * library two or three times (`libfoo.so` → `libfoo.so.N` → `libfoo.so.N.M`), and `bin` carries + * interpreter aliases. Materialising all of them produced a Linux box where roughly 60% of the + * bytes were duplicates of other bytes in the same box. Preserving them costs nothing to store and + * everything to get wrong, because a link is the classic way an archive writes outside the + * directory it was extracted into. + * + * So the rule is deliberately narrow and purely lexical, which is what makes it provable: + * + * 1. a target is relative — never absolute, never a drive letter, never a backslash; + * 2. resolved against the link's own directory it stays inside the payload, so `..` is allowed + * exactly as far as it cannot escape; + * 3. a link resolves to a *regular file*, never to a directory; + * 4. no entry may have a link as a path prefix, so nothing is ever written *through* a link; + * 5. chains terminate, within a small bound, without a cycle. + * + * Rule 3 is what keeps the rest small. A directory link is legitimate in a conda prefix — + * `lib/python3.1` → `python3.11` is real — but it is also the only reason an entry could ever be + * written *through* a link and land somewhere its own name does not describe. Refusing directory + * links costs one duplicated standard library and removes an entire class of escape, so rule 4 + * survives only as a second lock on a door that rule 3 already welded shut. + * + * Nothing here consults the filesystem: the same inputs give the same answer on every host, which + * is what lets the builder, the Node consumer and the Python consumer apply one rule rather than + * three approximations of it. The builder additionally confirms its own links with `realpath`, + * because it can — but no consumer trusts that, and every rule here is re-checked before extraction + * writes anything. + * + * Targets that fail this rule are not an error at build time; they are simply materialised into + * real files, which is what every link used to become. + */ + +/** + * How many links a single resolution may traverse before it is treated as hostile. Real prefixes + * use one or two hops (`python` → `python3.11`, `libfoo.so` → `.so.N` → `.so.N.M`); a longer chain + * has no legitimate source and is the cheap way to make resolution expensive. + */ +export const MAX_PAYLOAD_LINK_DEPTH = 8; + +/** + * Whether a raw link target is shaped like one a payload may carry, before resolving it. + * + * @param {unknown} target + * @returns {boolean} + */ +export function isRelativeLinkTarget(target) { + if (typeof target !== 'string' || target === '') return false; + if (target.includes('\0') || target.includes('\\')) return false; + if (target.startsWith('/')) return false; + return !/^[A-Za-z]:/.test(target); +} + +/** + * Resolves a link target against the link's own location, staying inside the payload. + * + * @param {string} linkPath forward-slash path of the link itself, relative to the payload root + * @param {string} target the raw link body + * @returns {string | null} the resolved payload-relative path, or null when the link may not be + * carried — an absolute target, an escape through `..`, or a link onto itself + */ +export function resolvePayloadLinkTarget(linkPath, target) { + if (!isRelativeLinkTarget(target)) return null; + const segments = String(linkPath).split('/'); + // The link's own name is not part of the directory its target resolves against. + const stack = segments.slice(0, -1); + if (segments.length === 0 || segments.at(-1) === '') return null; + for (const part of target.split('/')) { + if (part === '' || part === '.') continue; + if (part === '..') { + // Underflow means the target climbed past the payload root: exactly the escape being + // guarded against, and the reason this is checked per segment rather than on the result. + if (stack.length === 0) return null; + stack.pop(); + continue; + } + stack.push(part); + } + if (stack.length === 0) return null; + const resolved = stack.join('/'); + return resolved === linkPath ? null : resolved; +} + +/** + * Rejects an entry set in which anything could be written through a link. + * + * A directory link is legitimate — conda ships `lib/python3.1` → `python3.11` — but it means an + * entry named under that link lands wherever the link points. Forbidding a link as any entry's path + * prefix removes the question entirely, and is why resolution never has to model what earlier + * entries did to the filesystem. + * + * @param {Array<{ path: string, kind: string }>} entries + * @returns {string | null} the offending entry path, or null when the set is safe + */ +export function findEntryThroughLink(entries) { + const links = new Set(entries.filter((entry) => entry.kind === 'link').map((entry) => entry.path)); + if (links.size === 0) return null; + for (const entry of entries) { + const parts = entry.path.split('/'); + for (let index = 1; index < parts.length; index += 1) { + if (links.has(parts.slice(0, index).join('/'))) return entry.path; + } + } + return null; +} + +/** + * Follows every link in an entry set until it reaches a regular file. + * + * A chain that ends anywhere else is refused: at a directory (rule 3), at nothing at all, at + * itself, or at more hops than a real prefix ever needs. The terminal entry must exist in the same + * archive, which is what makes a link a statement about this payload rather than about the host. + * + * @param {Array<{ path: string, kind: string, linkTarget?: string }>} entries + * @returns {string | null} the offending link path, or null when every chain ends at a file + */ +export function findUnresolvableLink(entries) { + const byPath = new Map(entries.map((entry) => [entry.path, entry])); + const directories = new Set(); + for (const entry of entries) { + if (entry.kind === 'directory') directories.add(entry.path); + const parts = entry.path.split('/'); + for (let index = 1; index < parts.length; index += 1) directories.add(parts.slice(0, index).join('/')); + } + for (const entry of entries) { + if (entry.kind !== 'link') continue; + const seen = new Set([entry.path]); + let current = entry; + for (let depth = 0; ; depth += 1) { + if (depth >= MAX_PAYLOAD_LINK_DEPTH) return entry.path; + const resolved = resolvePayloadLinkTarget(current.path, current.linkTarget ?? ''); + if (resolved === null) return entry.path; + // A directory may exist implicitly, through its children, without an entry of its own — so + // this has to be asked before looking the path up as an entry. + if (directories.has(resolved)) return entry.path; + const next = byPath.get(resolved); + if (!next) return entry.path; + if (next.kind === 'file') break; + if (next.kind !== 'link') return entry.path; + if (seen.has(next.path)) return entry.path; + seen.add(next.path); + current = next; + } + } + return null; +} + +/** + * Whether a target platform can extract a payload containing links. + * + * Creating a symbolic link on Windows needs Developer Mode or elevation, so a Windows box keeps + * materialising every link rather than producing an archive that fails to extract on an ordinary + * machine. + * + * @param {string} platform the target platform, as a scroll declares it + * @returns {boolean} + */ +export function targetCarriesLinks(platform) { + return platform !== 'windows'; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/box-manifest.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/box-manifest.schema.json new file mode 100644 index 0000000..61b0288 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/box-manifest.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/box-manifest.schema.json", + "title": "Box manifest (box.json)", + "description": "The manifest packed inside the archive at box.json. It restates the box's identity, layout and provenance so an extracted box is self-describing: a consumer that has the directory but not the release document can still tell what it is holding and how it was built.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "boxId", + "modelId", + "runtimeId", + "version", + "target", + "pythonEntryPoint", + "modelCacheSubdir", + "selfTest", + "provenance" + ], + "properties": { + "schemaVersion": { + "const": 2 + }, + "boxId": { + "type": "string", + "minLength": 1 + }, + "modelId": { + "type": "string", + "minLength": 1 + }, + "runtimeId": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + }, + "pythonEntryPoint": { + "type": "string", + "minLength": 1 + }, + "modelCacheSubdir": { + "type": "string", + "minLength": 1 + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "pythonImports", + "timeoutSeconds" + ], + "properties": { + "pythonImports": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "timeoutSeconds": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + }, + "provenance": { + "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/provenance" + }, + "weights": { + "const": "on-demand", + "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." + }, + "assets": { + "type": "array", + "minItems": 1, + "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "url", + "relativePath", + "sizeBytes", + "sha256" + ], + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "relativePath": { + "type": "string", + "minLength": 1 + }, + "sizeBytes": { + "type": "integer", + "exclusiveMinimum": 0 + }, + "sha256": { + "$ref": "https://scrollcase.dev/schema/v2/release-manifest.schema.json#/$defs/sha256" + } + } + } + } + }, + "dependentRequired": { + "assets": [ + "weights" + ], + "weights": [ + "assets" + ] + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/channel-manifest.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/channel-manifest.schema.json new file mode 100644 index 0000000..70102ed --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/channel-manifest.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/channel-manifest.schema.json", + "title": "Box channel manifest", + "description": "A small mutable pointer from a channel to the releases it currently serves. Signed independently from releases, so promoting a build never requires re-signing it.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "kind", + "channel", + "boxId", + "target", + "updatedAt", + "cohortSalt", + "releases" + ], + "properties": { + "schemaVersion": { + "const": 2 + }, + "kind": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.channel$", + "description": "Wire discriminator, \".channel\", carrying the same namespace as the releases it refers to." + }, + "channel": { + "enum": [ + "nightly", + "beta", + "stable" + ] + }, + "boxId": { + "type": "string", + "minLength": 1 + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + }, + "updatedAt": { + "type": "string", + "minLength": 1 + }, + "cohortSalt": { + "type": "string", + "minLength": 1, + "description": "Salt mixed into a client's rollout hash. It makes cohort assignment stable per client and unpredictable across channels, so a staged rollout cannot be gamed by reinstalling." + }, + "releases": { + "type": "array", + "minItems": 1, + "description": "Candidate releases in evaluation order. A client takes the first entry whose rollout cohort it falls into.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "releaseManifestUrl", + "rolloutPercentage" + ], + "properties": { + "version": { + "type": "string", + "minLength": 1 + }, + "releaseManifestUrl": { + "type": "string", + "minLength": 1 + }, + "rolloutPercentage": { + "type": "integer", + "minimum": 1, + "maximum": 100 + } + } + } + } + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/execution.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/execution.schema.json new file mode 100644 index 0000000..a37ebfb --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/execution.schema.json @@ -0,0 +1,94 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/execution.schema.json", + "title": "Box execution", + "description": "The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only.", + "oneOf": [ + { + "title": "Python script", + "description": "Run one regular payload file with the box's own Python interpreter.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "script", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "python-script", + "description": "Selects direct script execution." + }, + "script": { + "$ref": "#/$defs/payloadPath", + "description": "Safe path to a regular Python file inside the box." + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } + }, + { + "title": "Python module", + "description": "Run an importable dotted module with Python's -m option.", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "module", + "defaultArgs" + ], + "properties": { + "kind": { + "const": "python-module", + "description": "Selects dotted-module execution." + }, + "module": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*)*$", + "description": "Strict Python dotted-module name, without command-line syntax or shell fragments.", + "examples": [ + "example_model.main" + ] + }, + "defaultArgs": { + "$ref": "#/$defs/defaultArgs" + } + } + } + ], + "examples": [ + { + "kind": "python-script", + "script": "entrypoint.py", + "defaultArgs": [] + }, + { + "kind": "python-module", + "module": "example_model.main", + "defaultArgs": [ + "--serve" + ] + } + ], + "$defs": { + "defaultArgs": { + "type": "array", + "description": "Arguments placed before caller-supplied arguments. Every item is passed directly without a shell.", + "default": [], + "items": { + "type": "string" + } + }, + "payloadPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", + "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root.", + "examples": [ + "entrypoint.py", + "app/main.py" + ] + } + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/release-manifest.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/release-manifest.schema.json new file mode 100644 index 0000000..2437bcf --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/release-manifest.schema.json @@ -0,0 +1,272 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/release-manifest.schema.json", + "title": "Box release manifest", + "description": "The immutable description of one built box: what it is, which target it runs on, where its archive lives, and how it was produced. Published as the payload of a signed document and never edited after signing; a correction ships as a new version.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "kind", + "boxId", + "modelId", + "runtimeId", + "version", + "target", + "compatibility", + "archive", + "pythonEntryPoint", + "modelCacheSubdir", + "selfTest", + "provenance" + ], + "properties": { + "schemaVersion": { + "const": 2 + }, + "kind": { + "$ref": "#/$defs/kind", + "description": "Wire discriminator, \".release\". The namespace belongs to the publishing project \u2014 a project with boxes already in the field must keep emitting the one its clients recognise \u2014 and defaults to scrollcase.box for a new one." + }, + "boxId": { + "$ref": "#/$defs/identifier" + }, + "modelId": { + "$ref": "#/$defs/identifier" + }, + "runtimeId": { + "$ref": "#/$defs/identifier" + }, + "version": { + "type": "string", + "minLength": 1 + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + }, + "compatibility": { + "type": "object", + "additionalProperties": true, + "description": "What the host must satisfy before this box may be installed. The builder copies these constraints through verbatim and never interprets them, so a project may add its own alongside the ones defined here. A consumer that cannot evaluate a constraint must refuse the box rather than assume it passes.", + "properties": { + "minHostAppVersion": { + "type": "string", + "minLength": 1, + "description": "Lowest version of the installing application this box supports." + }, + "maxHostAppVersionExclusive": { + "type": "string", + "minLength": 1 + }, + "minMacosVersion": { + "type": "string", + "minLength": 1 + }, + "minRamGb": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Installed memory in decimal gigabytes (1 GB = 1,000,000,000 bytes)." + }, + "minNvidiaDriverVersion": { + "type": "string", + "minLength": 1 + }, + "hostEnvironments": { + "type": "array", + "minItems": 1, + "items": { + "enum": [ + "native", + "windows-wsl2" + ] + }, + "description": "Host environments this payload was validated on." + } + } + }, + "archive": { + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "url", + "sha256", + "sizeBytes" + ], + "properties": { + "format": { + "const": "zip" + }, + "url": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "sizeBytes": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "installedSizeBytes": { + "type": "integer", + "exclusiveMinimum": 0, + "description": "Sum of extracted payload file sizes before activation metadata is written, so a consumer can check free space before downloading." + }, + "pythonEntryPoint": { + "type": "string", + "minLength": 1, + "description": "Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter." + }, + "modelCacheSubdir": { + "type": "string", + "minLength": 1, + "description": "Directory relative to the extracted box root holding model assets." + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "pythonImports", + "timeoutSeconds" + ], + "description": "The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks.", + "properties": { + "pythonImports": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "timeoutSeconds": { + "type": "integer", + "exclusiveMinimum": 0 + } + } + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + }, + "provenance": { + "$ref": "#/$defs/provenance" + }, + "weights": { + "const": "on-demand", + "description": "Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it." + }, + "assets": { + "type": "array", + "minItems": 1, + "description": "Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "url", + "relativePath", + "sizeBytes", + "sha256" + ], + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "relativePath": { + "type": "string", + "minLength": 1 + }, + "sizeBytes": { + "type": "integer", + "exclusiveMinimum": 0 + }, + "sha256": { + "$ref": "#/$defs/sha256" + } + } + } + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" + }, + "kind": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.release$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "description": "How this box was produced. Every field is recorded by the builder from observed state, never accepted from caller input, so the record cannot be dressed up after the fact.", + "required": [ + "scrollId", + "scrollVersion", + "builderRevision", + "sourceTreeDirty", + "sourceRevision", + "pythonVersion", + "dependencyLockSha256", + "builtAt", + "pixiVersion" + ], + "properties": { + "scrollId": { + "type": "string", + "minLength": 1 + }, + "scrollVersion": { + "type": "string", + "minLength": 1 + }, + "builderRevision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$", + "description": "Exact commit of the builder source that produced the box." + }, + "sourceTreeDirty": { + "type": "boolean", + "description": "Whether the builder's working tree carried uncommitted changes. True means the build is not reproducible from the recorded revision alone." + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "description": "Upstream revision of the packaged model source, as declared by the scroll." + }, + "pythonVersion": { + "type": "string", + "minLength": 1 + }, + "pixiVersion": { + "type": "string", + "minLength": 1 + }, + "dependencyLockSha256": { + "$ref": "#/$defs/sha256", + "description": "Hash of the pixi.lock the environment was solved from." + }, + "builtAt": { + "type": "string", + "minLength": 1 + } + } + } + }, + "dependentRequired": { + "assets": [ + "weights" + ], + "weights": [ + "assets" + ] + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/revocations-manifest.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/revocations-manifest.schema.json new file mode 100644 index 0000000..67470c7 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/revocations-manifest.schema.json @@ -0,0 +1,64 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/revocations-manifest.schema.json", + "title": "Box revocations manifest", + "description": "The signed list of releases that must no longer be installed or activated. A published release is immutable, so withdrawing one is an explicit statement rather than a deletion: clients keep honouring this list even when the archive is still reachable.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "kind", + "updatedAt", + "revocations" + ], + "properties": { + "schemaVersion": { + "const": 2 + }, + "kind": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[.-][a-z0-9]+)*\\.revocations$", + "description": "Wire discriminator, \".revocations\", carrying the same namespace as the releases it refers to." + }, + "updatedAt": { + "type": "string", + "minLength": 1 + }, + "revocations": { + "type": "array", + "description": "May be empty: an empty signed list is a positive statement that nothing is revoked, which a client can distinguish from a missing or withheld document.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "boxId", + "version", + "reason", + "revokedAt" + ], + "properties": { + "boxId": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json", + "description": "Omitted when every target of that version is revoked." + }, + "reason": { + "type": "string", + "minLength": 1 + }, + "revokedAt": { + "type": "string", + "minLength": 1 + } + } + } + } + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/scroll.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/scroll.schema.json new file mode 100644 index 0000000..9cb4ee6 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/scroll.schema.json @@ -0,0 +1,339 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "title": "Box scroll", + "description": "The declarative input to a build: an identity, a target, a pinned dependency environment, the assets to fetch, and the self-test the result must pass. A scroll is checked into the consumer's repository next to its lock file; everything a build produces is derived from it.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "scrollVersion", + "boxId", + "modelId", + "runtimeId", + "version", + "sourceRevision", + "target", + "compatibility", + "pythonVersion", + "pythonEntryPoint", + "modelCacheSubdir", + "assets", + "selfTest", + "pixiVersion" + ], + "properties": { + "$schema": { + "const": "https://scrollcase.dev/schema/v2/scroll.schema.json", + "description": "Associates this file with the published Scrollcase v2 schema for editor validation, completion, and hover help." + }, + "schemaVersion": { + "const": 2, + "description": "Scrollcase wire version. Version 2 is the only active format.", + "examples": [ + 2 + ] + }, + "scrollId": { + "type": "string", + "minLength": 1, + "description": "Optional provenance identity. When omitted, Scrollcase derives it deterministically from boxId and the canonical target." + }, + "scrollVersion": { + "type": "string", + "minLength": 1, + "description": "Version of this declarative build input, recorded in provenance.", + "examples": [ + "1.0.0" + ] + }, + "boxId": { + "$ref": "#/$defs/identifier" + }, + "modelId": { + "$ref": "#/$defs/identifier" + }, + "runtimeId": { + "$ref": "#/$defs/identifier" + }, + "version": { + "type": "string", + "minLength": 1, + "description": "Version of the box this scroll produces, as it will appear in the release manifest." + }, + "sourceRevision": { + "type": "string", + "minLength": 1, + "description": "Upstream revision of the packaged source, recorded verbatim into provenance." + }, + "target": { + "$ref": "https://scrollcase.dev/schema/v2/target.schema.json" + }, + "compatibility": { + "type": "object", + "additionalProperties": true, + "properties": { + "minHostAppVersion": { + "type": "string", + "minLength": 1, + "description": "Lowest version of the installing application this box supports." + }, + "maxHostAppVersionExclusive": { + "type": "string", + "minLength": 1 + }, + "minMacosVersion": { + "type": "string", + "minLength": 1 + }, + "minRamGb": { + "type": "number", + "exclusiveMinimum": 0 + }, + "minNvidiaDriverVersion": { + "type": "string", + "minLength": 1 + } + }, + "description": "Constraints the installing host must satisfy. Copied through into the release manifest verbatim and never interpreted by the builder, so a project may declare its own alongside these." + }, + "pythonVersion": { + "type": "string", + "minLength": 1, + "description": "Python version solved into the box.", + "examples": [ + "3.11.15" + ] + }, + "pixiVersion": { + "type": "string", + "minLength": 1, + "description": "Pins the pixi release used to solve and install the conda-forge environment from the committed pixi.lock." + }, + "condaDependencyLicenseAudit": { + "type": "string", + "minLength": 1, + "description": "Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed." + }, + "pythonEntryPoint": { + "type": "string", + "minLength": 1, + "description": "Interpreter path relative to the box root. Must match the target adapter's layout." + }, + "modelCacheSubdir": { + "type": "string", + "minLength": 1 + }, + "assetBaseUrl": { + "type": "string", + "minLength": 1, + "description": "Base URL of the mirror the built archive and its objects are published under." + }, + "assets": { + "type": "array", + "description": "Files fetched during the build. Every entry is size- and hash-checked before use, so a moved or replaced upstream file fails the build instead of silently changing the box. May be empty.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "url", + "relativePath", + "sizeBytes", + "sha256" + ], + "properties": { + "url": { + "type": "string", + "minLength": 1 + }, + "relativePath": { + "$ref": "#/$defs/payloadPath" + }, + "sizeBytes": { + "type": "integer", + "exclusiveMinimum": 0 + }, + "sha256": { + "$ref": "#/$defs/sha256" + } + } + } + }, + "assetArchives": { + "type": "array", + "description": "Downloaded archives to expand into the payload. Extraction preserves files already present in the destination and refuses to overwrite them.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "relativePath", + "format", + "destination" + ], + "properties": { + "relativePath": { + "$ref": "#/$defs/payloadPath" + }, + "format": { + "enum": [ + "zip", + "tar.gz" + ] + }, + "destination": { + "$ref": "#/$defs/payloadPath" + }, + "stripComponents": { + "type": "integer", + "minimum": 0 + }, + "removeAfterExtract": { + "type": "boolean" + } + } + } + }, + "localFiles": { + "type": "array", + "description": "Files copied from the consumer's own repository into the payload, each verified against a declared hash so a licence notice or runtime shim cannot drift from what was reviewed.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "sourcePath", + "relativePath", + "sha256" + ], + "properties": { + "sourcePath": { + "type": "string", + "minLength": 1 + }, + "relativePath": { + "$ref": "#/$defs/payloadPath" + }, + "sha256": { + "$ref": "#/$defs/sha256" + } + } + } + }, + "prunePaths": { + "type": "array", + "description": "Payload paths deleted before packing, to keep the box to what it actually needs at run time. Pruning a distribution the lock requires is rejected.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "selfTest": { + "type": "object", + "additionalProperties": false, + "required": [ + "imports", + "files" + ], + "description": "Builder checks run with the payload's own interpreter before archiving. Schema version 2 signs only the import subset for a consumer to repeat; file and optional Python-code assertions remain builder-only.", + "properties": { + "imports": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "files": { + "type": "array", + "description": "Files that must still exist after pruning, which is what stops an over-aggressive prune from shipping a broken box.", + "items": { + "$ref": "#/$defs/payloadPath" + } + }, + "pythonCode": { + "type": "string", + "minLength": 1, + "description": "Extra Python executed after the imports succeed, for checks a bare import cannot make." + } + } + }, + "weights": { + "enum": [ + "embed", + "on-demand" + ], + "default": "embed", + "description": "Whether assets are packed into the archive (embed, the default: the box installs with no network and works air-gapped) or left out for the caller to materialize from descriptors in the signed release (on-demand). Consumers verify materialized assets before execution and do not download them. A build may override this." + }, + "execution": { + "$ref": "https://scrollcase.dev/schema/v2/execution.schema.json" + }, + "parity": { + "type": "object", + "additionalProperties": false, + "required": [ + "script", + "accelerators", + "tolerances" + ], + "description": "An optional numerical gate: run a check inside the box on more than one accelerator and require the results to agree. This catches a mis-solved environment \u2014 CPU-only wheels shipped as CUDA, a broken BLAS \u2014 on the build machine rather than on a user's. The tool runs the check and enforces the thresholds; what the check computes, and what closeness is acceptable, belong to the project.", + "properties": { + "script": { + "type": "string", + "minLength": 1, + "description": "Path inside the box, run with the box's own interpreter. It must print a JSON array of numbers, or an object with a \"values\" array." + }, + "accelerators": { + "type": "array", + "minItems": 2, + "items": { + "enum": [ + "cpu", + "metal", + "cuda" + ] + }, + "description": "Accelerators to run under, each with its target's validation environment. The first is the reference the others are compared against \u2014 conventionally cpu, being the one available everywhere." + }, + "tolerances": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "description": "At least one bound. Absolute guards entries near zero, where relative error is meaningless; cosine similarity catches a result that drifted in direction rather than magnitude.", + "properties": { + "absolute": { + "type": "number", + "exclusiveMinimum": 0 + }, + "relative": { + "type": "number", + "exclusiveMinimum": 0 + }, + "minimumCosine": { + "type": "number", + "maximum": 1 + } + } + } + } + } + }, + "$defs": { + "identifier": { + "type": "string", + "pattern": "^[a-z0-9]+(?:[-.][a-z0-9]+)*$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "payloadPath": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*\\\\)(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*//).+$", + "description": "A forward-slash path inside the box payload: relative, non-empty, and unable to escape the payload root.", + "examples": [ + "model-cache/example-model/weights.safetensors" + ] + } + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/signed-document.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/signed-document.schema.json new file mode 100644 index 0000000..602af0c --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/signed-document.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/signed-document.schema.json", + "title": "Signed box document", + "description": "The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid.", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "payloadEncoding", "payloadBase64", "payloadSha256", "signatures"], + "properties": { + "schemaVersion": { "const": 2 }, + "payloadEncoding": { "const": "base64-json-utf8" }, + "payloadBase64": { + "type": "string", + "minLength": 1, + "description": "The document payload: UTF-8 JSON, base64-encoded, signed and hashed exactly as it appears here." + }, + "payloadSha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the decoded payload bytes." + }, + "signatures": { + "type": "array", + "minItems": 1, + "description": "Detached signatures over the decoded payload bytes. A verifier accepts the document when any one signature verifies against a trusted key, which is what allows a key to be rotated without reissuing every document.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["algorithm", "keyId", "signatureBase64"], + "properties": { + "algorithm": { "const": "ed25519" }, + "keyId": { "type": "string", "minLength": 1 }, + "signatureBase64": { "type": "string", "minLength": 1 } + } + } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + } +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/schema/target.schema.json b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/target.schema.json new file mode 100644 index 0000000..6894c12 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/schema/target.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://scrollcase.dev/schema/v2/target.schema.json", + "title": "Box target", + "description": "The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed.", + "type": "object", + "additionalProperties": false, + "required": ["platform", "arch", "accelerator"], + "properties": { + "platform": { + "enum": ["macos", "linux", "windows"], + "description": "Operating system the box runs on.", + "examples": ["linux"] + }, + "arch": { + "enum": ["aarch64", "x86_64"], + "description": "CPU architecture the box runs on; supported combinations are constrained below.", + "examples": ["x86_64"] + }, + "accelerator": { + "enum": ["cpu", "metal", "cuda"], + "description": "Acceleration backend built into the environment.", + "default": "cpu", + "examples": ["cpu"] + }, + "cudaVersion": { + "type": "string", + "pattern": "^[1-9][0-9]*\\.[0-9]+$", + "description": "CUDA ABI as major.minor, for example \"12.8\". Required for a CUDA target and forbidden for any other, so an identifier can never be ambiguous." + } + }, + "allOf": [ + { + "if": { "properties": { "accelerator": { "const": "cuda" } }, "required": ["accelerator"] }, + "then": { "required": ["cudaVersion"] }, + "else": { "not": { "required": ["cudaVersion"] } } + }, + { + "if": { "properties": { "platform": { "const": "macos" } }, "required": ["platform"] }, + "then": { + "properties": { + "arch": { "const": "aarch64" }, + "accelerator": { "enum": ["metal", "cpu"] } + } + } + }, + { + "if": { "properties": { "platform": { "enum": ["linux", "windows"] } }, "required": ["platform"] }, + "then": { + "properties": { + "arch": { "const": "x86_64" }, + "accelerator": { "enum": ["cpu", "cuda"] } + } + } + } + ] +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/targets.mjs b/.scrollcase-runtime-types-RzUfxr/src/contract/targets.mjs new file mode 100644 index 0000000..a4fa2b3 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/targets.mjs @@ -0,0 +1,254 @@ +/** + * Reference implementation of the Scrollcase box-format target model. + * + * A target is the (platform, arch, accelerator) triple a box is built for, plus a CUDA ABI version + * when the accelerator is CUDA. `boxTargetId()` turns it into the canonical slug that appears + * in archive names, object keys, and registry routes, so every implementation of the format — this + * one, a consumer's own client, a signer — must agree character for character. The golden cases in + * `fixtures/target-id-contract.json` are what "agree" means, and are the fixtures other languages + * validate their mirrors against. + * + * The adapters below describe what a target implies for the built payload: the Python layout inside + * the box, the archive backend, how native libraries are inspected, and the environment a validation + * run gets. They are part of the format because a consumer unpacking a box relies on that layout. + */ +/** + * What a target implies for the built payload. Part of the format rather than an implementation + * detail: a consumer unpacking a box relies on this layout. + * + * @typedef {object} BoxTargetAdapter + * @property {string} id canonical adapter id, e.g. `macos-aarch64` + * @property {'macos' | 'linux' | 'windows'} platform + * @property {'aarch64' | 'x86_64'} arch + * @property {{ platform: string, arch: string }} host the Node platform/arch a build must run on + * @property {'osx-arm64' | 'linux-64' | 'win-64'} condaSubdir the scroll's pixi `platforms` value + * @property {{ payloadRoot: string, entryPoint: string, scriptsDirectory: string, + * executableSuffix: string, launcherKind: string }} python layout of the interpreter in the box + * @property {{ format: 'zip', writer: string, reader: string, assetTarReader: string, + * zip64: boolean }} archive the pinned archive backend + * @property {{ command: string, argsPrefix: readonly string[], + * extensions: readonly string[] }} nativeLibraryInspection + * @property {Readonly>>>} validationEnvironments + * the environment that forces a run onto one accelerator, keyed by accelerator + * @property {string} selfTestPython the platform assertion prepended to every self-test + */ + +const TARGET_ACCELERATORS = { + macos: { aarch64: ['metal', 'cpu'] }, + linux: { x86_64: ['cpu', 'cuda'] }, + windows: { x86_64: ['cpu', 'cuda'] }, +}; +const CUDA_VERSION = /^[1-9][0-9]*\.[0-9]+$/; + +// The exact libraries that wrote and read a box, so a consumer knows what produced the bytes it +// holds rather than inferring it. Each version is the one this package installs: they are pinned in +// `package.json` and `tests/unit/contract-targets.test.mjs` fails when the two drift, because a +// descriptor naming a release that never touched the archive is worse than no descriptor at all. +const ARCHIVE_BACKEND = Object.freeze({ + format: 'zip', + writer: 'yazl@3.3.1', + reader: 'yauzl@3.4.0', + assetTarReader: 'tar@7.5.22', + zip64: true, +}); + +const TARGET_ADAPTERS = Object.freeze([ + Object.freeze({ + id: 'macos-aarch64', + platform: 'macos', + arch: 'aarch64', + host: Object.freeze({ platform: 'darwin', arch: 'arm64' }), + // conda platform subdir: the `platforms` value in the scroll's pixi.toml. + condaSubdir: 'osx-arm64', + python: Object.freeze({ + payloadRoot: 'venv', + entryPoint: 'venv/bin/python', + scriptsDirectory: 'venv/bin', + executableSuffix: '', + launcherKind: 'posix-polyglot', + }), + archive: ARCHIVE_BACKEND, + nativeLibraryInspection: Object.freeze({ + command: 'otool', + argsPrefix: Object.freeze(['-L']), + extensions: Object.freeze(['.dylib', '.so']), + }), + validationEnvironments: Object.freeze({ + cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), + metal: Object.freeze({ PYTORCH_ENABLE_MPS_FALLBACK: '0' }), + }), + selfTestPython: "import sys; assert sys.platform == 'darwin'", + }), + Object.freeze({ + id: 'linux-x86_64', + platform: 'linux', + arch: 'x86_64', + host: Object.freeze({ platform: 'linux', arch: 'x64' }), + condaSubdir: 'linux-64', + python: Object.freeze({ + payloadRoot: 'venv', + entryPoint: 'venv/bin/python', + scriptsDirectory: 'venv/bin', + executableSuffix: '', + launcherKind: 'posix-polyglot', + }), + archive: ARCHIVE_BACKEND, + nativeLibraryInspection: Object.freeze({ + command: 'ldd', + argsPrefix: Object.freeze([]), + extensions: Object.freeze(['.so']), + }), + validationEnvironments: Object.freeze({ + cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), + cuda: Object.freeze({ CUDA_VISIBLE_DEVICES: '0' }), + }), + selfTestPython: "import sys; assert sys.platform.startswith('linux')", + }), + Object.freeze({ + id: 'windows-x86_64', + platform: 'windows', + arch: 'x86_64', + host: Object.freeze({ platform: 'win32', arch: 'x64' }), + condaSubdir: 'win-64', + python: Object.freeze({ + payloadRoot: 'venv', + entryPoint: 'venv/python.exe', + scriptsDirectory: 'venv/Scripts', + executableSuffix: '.exe', + launcherKind: 'uv-windows-pe', + }), + archive: ARCHIVE_BACKEND, + nativeLibraryInspection: Object.freeze({ + command: 'dumpbin', + argsPrefix: Object.freeze(['/DEPENDENTS']), + extensions: Object.freeze(['.dll', '.pyd']), + }), + validationEnvironments: Object.freeze({ + cpu: Object.freeze({ CUDA_VISIBLE_DEVICES: '' }), + cuda: Object.freeze({ CUDA_VISIBLE_DEVICES: '0' }), + }), + selfTestPython: "import sys; assert sys.platform == 'win32'", + }), +]); + +/** + * Returns the canonical target slug used in box filenames, object keys, and routes. + * + * @param {import('./types/index.d.ts').BoxTarget} target + * @returns {string} the canonical slug, e.g. `linux-x86_64-cuda12.4` + * @throws {TypeError} when the target is outside the supported matrix, or its CUDA version is + * missing on a CUDA target or present on any other + */ +export function boxTargetId(target) { + if (!target || typeof target !== 'object') { + throw new TypeError('Box target must be an object'); + } + const accelerators = TARGET_ACCELERATORS[target?.platform]?.[target?.arch]; + if (!accelerators?.includes(target?.accelerator)) { + throw new TypeError( + `Unsupported box target: ${target?.platform}/${target?.arch}/${target?.accelerator}`, + ); + } + if (target.accelerator === 'cuda') { + if (typeof target.cudaVersion !== 'string' || !CUDA_VERSION.test(target.cudaVersion)) { + throw new TypeError('A CUDA box target requires a numeric major.minor CUDA version'); + } + return `${target.platform}-${target.arch}-cuda${target.cudaVersion}`; + } + if (target.cudaVersion !== undefined) { + throw new TypeError('Only CUDA box targets may declare a CUDA version'); + } + return `${target.platform}-${target.arch}-${target.accelerator}`; +} + +/** + * Returns the native builder adapter for a validated box target. + * + * @param {import('./types/index.d.ts').BoxTarget} target + * @returns {BoxTargetAdapter} + * @throws {TypeError} when the target is unsupported + */ +export function boxTargetAdapter(target) { + boxTargetId(target); + const adapter = TARGET_ADAPTERS.find((candidate) => + candidate.platform === target.platform && candidate.arch === target.arch); + if (!adapter) throw new TypeError(`No box target adapter exists for ${target.platform}/${target.arch}`); + return adapter; +} + +/** + * Ensures a build or target lock runs on the OS and architecture it will ship for. + * + * @param {BoxTargetAdapter} adapter + * @param {{ platform: string, arch: string }} [host] defaults to the current process + * @returns {void} + * @throws {TypeError} when the host is not the OS and architecture the box ships for + */ +export function assertNativeHost(adapter, host = process) { + if (host.platform !== adapter.host.platform || host.arch !== adapter.host.arch) { + throw new TypeError( + `${adapter.id} boxes must be built natively on ${adapter.host.platform}/${adapter.host.arch}; ` + + `current host is ${host.platform}/${host.arch}`, + ); + } +} + +/** + * Ensures the scroll entry point agrees with the adapter's standalone Python layout. + * + * @param {BoxTargetAdapter} adapter + * @param {string} entryPoint + * @returns {void} + * @throws {TypeError} when the entry point does not match the adapter's layout + */ +export function assertPythonEntryPoint(adapter, entryPoint) { + if (entryPoint !== adapter.python.entryPoint) { + throw new TypeError( + `${adapter.id} scrolls must use Python entry point ${adapter.python.entryPoint}`, + ); + } +} + +/** + * Lists every adapter, for contract tests and for consumers enumerating supported targets. + * + * @returns {BoxTargetAdapter[]} every supported adapter, as a fresh array + */ +export function boxTargetAdapters() { + return [...TARGET_ADAPTERS]; +} + +/** + * Maps a validated box target to its conda platform subdir (the pixi `platforms` value). + * + * @param {import('./types/index.d.ts').BoxTarget} target + * @returns {'osx-arm64' | 'linux-64' | 'win-64'} the pixi `platforms` value for the target + */ +export function condaSubdir(target) { + const adapter = boxTargetAdapter(target); + return adapter.condaSubdir; +} + +/** + * Returns the conda accelerator descriptor a scroll selects, rejecting target drift. `metal` and + * `cpu` need no extra conda knobs (osx-arm64 ships MPS in the pytorch build; cpu is the default build); `cuda` pins a + * `cuda-version` and declares a CUDA system requirement so the solver picks the GPU pytorch build. + * + * @param {Pick} scroll + * @returns {{ accelerator: 'cpu' | 'metal' | 'cuda', cudaVersion: string | null }} + * @throws {TypeError} when the accelerator is unsupported, or a CUDA target lacks a version + */ +export function pixiAccelerator(scroll) { + const accelerator = scroll?.target?.accelerator; + if (accelerator === 'metal' || accelerator === 'cpu') { + return Object.freeze({ accelerator, cudaVersion: null }); + } + if (accelerator === 'cuda') { + const cudaVersion = scroll?.target?.cudaVersion; + if (typeof cudaVersion !== 'string' || !CUDA_VERSION.test(cudaVersion)) { + throw new TypeError('A CUDA box target requires a numeric major.minor CUDA version'); + } + return Object.freeze({ accelerator, cudaVersion }); + } + throw new TypeError(`Unsupported box accelerator: ${accelerator}`); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/contract/types/index.d.ts b/.scrollcase-runtime-types-RzUfxr/src/contract/types/index.d.ts new file mode 100644 index 0000000..da0a0cd --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/contract/types/index.d.ts @@ -0,0 +1,467 @@ +/** + * Types for the scrollcase box format, generated from the JSON Schemas in + * src/contract/schema/. Do not edit by hand: run `npm run types` instead. + * + * The schemas are the source of truth. These types are a projection of them, and the test suite + * fails if the two disagree. + */ + +export type BoxTarget = { + [k: string]: unknown; +} & { + /** + * Operating system the box runs on. + */ + platform: 'macos' | 'linux' | 'windows'; + /** + * CPU architecture the box runs on; supported combinations are constrained below. + */ + arch: 'aarch64' | 'x86_64'; + /** + * Acceleration backend built into the environment. + */ + accelerator: 'cpu' | 'metal' | 'cuda'; + /** + * CUDA ABI as major.minor, for example "12.8". Required for a CUDA target and forbidden for any other, so an identifier can never be ambiguous. + */ + cudaVersion?: string; +}; + +/** + * The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only. + */ +export type BoxExecution = PythonScript | PythonModule; +/** + * Arguments placed before caller-supplied arguments. Every item is passed directly without a shell. + */ +export type DefaultArgs = string[]; + +/** + * Run one regular payload file with the box's own Python interpreter. + */ +export interface PythonScript { + /** + * Selects direct script execution. + */ + kind: 'python-script'; + /** + * Safe path to a regular Python file inside the box. + */ + script: string; + defaultArgs: DefaultArgs; +} +/** + * Run an importable dotted module with Python's -m option. + */ +export interface PythonModule { + /** + * Selects dotted-module execution. + */ + kind: 'python-module'; + /** + * Strict Python dotted-module name, without command-line syntax or shell fragments. + */ + module: string; + defaultArgs: DefaultArgs; +} + +export type Identifier = string; +/** + * The platform, architecture and accelerator a box is built for. The supported combinations are closed: a target outside this matrix has no defined identifier and cannot be built, signed, or routed. + */ +export type PayloadPath = string; +export type Sha256 = string; +/** + * The optional, shell-free application entry point shared by a scroll, the signed release, and box.json. Its absence means the box is intentionally library-only. + */ +export interface BoxScroll { + /** + * Associates this file with the published Scrollcase v2 schema for editor validation, completion, and hover help. + */ + $schema?: 'https://scrollcase.dev/schema/v2/scroll.schema.json'; + /** + * Scrollcase wire version. Version 2 is the only active format. + */ + schemaVersion: 2; + /** + * Optional provenance identity. When omitted, Scrollcase derives it deterministically from boxId and the canonical target. + */ + scrollId?: string; + /** + * Version of this declarative build input, recorded in provenance. + */ + scrollVersion: string; + boxId: Identifier; + modelId: Identifier; + runtimeId: Identifier; + /** + * Version of the box this scroll produces, as it will appear in the release manifest. + */ + version: string; + /** + * Upstream revision of the packaged source, recorded verbatim into provenance. + */ + sourceRevision: string; + target: BoxTarget; + /** + * Constraints the installing host must satisfy. Copied through into the release manifest verbatim and never interpreted by the builder, so a project may declare its own alongside these. + */ + compatibility: { + /** + * Lowest version of the installing application this box supports. + */ + minHostAppVersion?: string; + maxHostAppVersionExclusive?: string; + minMacosVersion?: string; + minRamGb?: number; + minNvidiaDriverVersion?: string; + [k: string]: unknown; + }; + /** + * Python version solved into the box. + */ + pythonVersion: string; + /** + * Pins the pixi release used to solve and install the conda-forge environment from the committed pixi.lock. + */ + pixiVersion: string; + /** + * Path to the reviewed licence inventory derived from pixi.lock, which carries an SPDX licence per package. The build fails if the lock no longer matches what was reviewed. + */ + condaDependencyLicenseAudit?: string; + /** + * Interpreter path relative to the box root. Must match the target adapter's layout. + */ + pythonEntryPoint: string; + modelCacheSubdir: string; + /** + * Base URL of the mirror the built archive and its objects are published under. + */ + assetBaseUrl?: string; + /** + * Files fetched during the build. Every entry is size- and hash-checked before use, so a moved or replaced upstream file fails the build instead of silently changing the box. May be empty. + */ + assets: { + url: string; + relativePath: PayloadPath; + sizeBytes: number; + sha256: Sha256; + }[]; + /** + * Downloaded archives to expand into the payload. Extraction preserves files already present in the destination and refuses to overwrite them. + */ + assetArchives?: { + relativePath: PayloadPath; + format: 'zip' | 'tar.gz'; + destination: PayloadPath; + stripComponents?: number; + removeAfterExtract?: boolean; + }[]; + /** + * Files copied from the consumer's own repository into the payload, each verified against a declared hash so a licence notice or runtime shim cannot drift from what was reviewed. + */ + localFiles?: { + sourcePath: string; + relativePath: PayloadPath; + sha256: Sha256; + }[]; + /** + * Payload paths deleted before packing, to keep the box to what it actually needs at run time. Pruning a distribution the lock requires is rejected. + */ + prunePaths?: PayloadPath[]; + /** + * Builder checks run with the payload's own interpreter before archiving. Schema version 2 signs only the import subset for a consumer to repeat; file and optional Python-code assertions remain builder-only. + */ + selfTest: { + /** + * @minItems 1 + */ + imports: [string, ...string[]]; + /** + * Files that must still exist after pruning, which is what stops an over-aggressive prune from shipping a broken box. + */ + files: PayloadPath[]; + /** + * Extra Python executed after the imports succeed, for checks a bare import cannot make. + */ + pythonCode?: string; + }; + /** + * Whether assets are packed into the archive (embed, the default: the box installs with no network and works air-gapped) or left out for the caller to materialize from descriptors in the signed release (on-demand). Consumers verify materialized assets before execution and do not download them. A build may override this. + */ + weights?: 'embed' | 'on-demand'; + execution?: BoxExecution; + /** + * An optional numerical gate: run a check inside the box on more than one accelerator and require the results to agree. This catches a mis-solved environment — CPU-only wheels shipped as CUDA, a broken BLAS — on the build machine rather than on a user's. The tool runs the check and enforces the thresholds; what the check computes, and what closeness is acceptable, belong to the project. + */ + parity?: { + /** + * Path inside the box, run with the box's own interpreter. It must print a JSON array of numbers, or an object with a "values" array. + */ + script: string; + /** + * Accelerators to run under, each with its target's validation environment. The first is the reference the others are compared against — conventionally cpu, being the one available everywhere. + * + * @minItems 2 + */ + accelerators: ['cpu' | 'metal' | 'cuda', 'cpu' | 'metal' | 'cuda', ...('cpu' | 'metal' | 'cuda')[]]; + /** + * At least one bound. Absolute guards entries near zero, where relative error is meaningless; cosine similarity catches a result that drifted in direction rather than magnitude. + */ + tolerances: { + absolute?: number; + relative?: number; + minimumCosine?: number; + }; + }; +} +/** + * Run one regular payload file with the box's own Python interpreter. + */ +export interface BoxManifest { + schemaVersion: 2; + boxId: string; + modelId: string; + runtimeId: string; + version: string; + target: BoxTarget; + pythonEntryPoint: string; + modelCacheSubdir: string; + selfTest: { + /** + * @minItems 1 + */ + pythonImports: [string, ...string[]]; + timeoutSeconds: number; + }; + execution?: BoxExecution; + provenance: Provenance; + /** + * Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it. + */ + weights?: 'on-demand'; + /** + * Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe. + * + * @minItems 1 + */ + assets?: [ + { + url: string; + relativePath: string; + sizeBytes: number; + sha256: string; + }, + ...{ + url: string; + relativePath: string; + sizeBytes: number; + sha256: string; + }[] + ]; +} +/** + * Run one regular payload file with the box's own Python interpreter. + */ +export interface Provenance { + scrollId: string; + scrollVersion: string; + /** + * Exact commit of the builder source that produced the box. + */ + builderRevision: string; + /** + * Whether the builder's working tree carried uncommitted changes. True means the build is not reproducible from the recorded revision alone. + */ + sourceTreeDirty: boolean; + /** + * Upstream revision of the packaged model source, as declared by the scroll. + */ + sourceRevision: string; + pythonVersion: string; + pixiVersion: string; + /** + * Hash of the pixi.lock the environment was solved from. + */ + dependencyLockSha256: string; + builtAt: string; +} + +export interface BoxReleaseManifest { + schemaVersion: 2; + /** + * Wire discriminator, ".release". The namespace belongs to the publishing project — a project with boxes already in the field must keep emitting the one its clients recognise — and defaults to scrollcase.box for a new one. + */ + kind: string; + boxId: Identifier; + modelId: Identifier; + runtimeId: Identifier; + version: string; + target: BoxTarget; + /** + * What the host must satisfy before this box may be installed. The builder copies these constraints through verbatim and never interprets them, so a project may add its own alongside the ones defined here. A consumer that cannot evaluate a constraint must refuse the box rather than assume it passes. + */ + compatibility: { + /** + * Lowest version of the installing application this box supports. + */ + minHostAppVersion?: string; + maxHostAppVersionExclusive?: string; + minMacosVersion?: string; + /** + * Installed memory in decimal gigabytes (1 GB = 1,000,000,000 bytes). + */ + minRamGb?: number; + minNvidiaDriverVersion?: string; + /** + * Host environments this payload was validated on. + * + * @minItems 1 + */ + hostEnvironments?: ['native' | 'windows-wsl2', ...('native' | 'windows-wsl2')[]]; + [k: string]: unknown; + }; + archive: { + format: 'zip'; + url: string; + sha256: Sha256; + sizeBytes: number; + }; + /** + * Sum of extracted payload file sizes before activation metadata is written, so a consumer can check free space before downloading. + */ + installedSizeBytes?: number; + /** + * Interpreter path relative to the extracted box root, for example venv/bin/python. Fixed per target by the adapter. + */ + pythonEntryPoint: string; + /** + * Directory relative to the extracted box root holding model assets. + */ + modelCacheSubdir: string; + /** + * The import check a consumer can repeat after extraction with the box's own interpreter. The builder also ran the scroll's Python-code and file assertions, which are builder-only checks. + */ + selfTest: { + /** + * @minItems 1 + */ + pythonImports: [string, ...string[]]; + timeoutSeconds: number; + }; + execution?: BoxExecution; + provenance: Provenance; + /** + * Present only when the assets were deliberately left out of the archive. Absent means the box is self-contained: everything it needs is inside it. + */ + weights?: 'on-demand'; + /** + * Assets the consumer must fetch and place under the box root before first use. Present only with on-demand weights. The declared hash is what makes fetching them safe. + * + * @minItems 1 + */ + assets?: [ + { + url: string; + relativePath: string; + sizeBytes: number; + sha256: Sha256; + }, + ...{ + url: string; + relativePath: string; + sizeBytes: number; + sha256: Sha256; + }[] + ]; +} +/** + * Run one regular payload file with the box's own Python interpreter. + */ +export interface BoxChannelManifest { + schemaVersion: 2; + /** + * Wire discriminator, ".channel", carrying the same namespace as the releases it refers to. + */ + kind: string; + channel: 'nightly' | 'beta' | 'stable'; + boxId: string; + target: BoxTarget; + updatedAt: string; + /** + * Salt mixed into a client's rollout hash. It makes cohort assignment stable per client and unpredictable across channels, so a staged rollout cannot be gamed by reinstalling. + */ + cohortSalt: string; + /** + * Candidate releases in evaluation order. A client takes the first entry whose rollout cohort it falls into. + * + * @minItems 1 + */ + releases: [ + { + version: string; + releaseManifestUrl: string; + rolloutPercentage: number; + }, + ...{ + version: string; + releaseManifestUrl: string; + rolloutPercentage: number; + }[] + ]; +} + +/** + * Omitted when every target of that version is revoked. + */ +export interface BoxRevocationsManifest { + schemaVersion: 2; + /** + * Wire discriminator, ".revocations", carrying the same namespace as the releases it refers to. + */ + kind: string; + updatedAt: string; + /** + * May be empty: an empty signed list is a positive statement that nothing is revoked, which a client can distinguish from a missing or withheld document. + */ + revocations: { + boxId: string; + version: string; + target?: BoxTarget; + reason: string; + revokedAt: string; + }[]; +} + +/** + * The envelope wrapping every signed document. The payload travels as exact base64-encoded JSON so that verifying a signature means hashing the bytes as transmitted, with no canonical-JSON implementation to keep in sync across languages. Passing this schema means the envelope is well-formed, never that its signature is valid. + */ +export interface SignedBoxDocument { + schemaVersion: 2; + payloadEncoding: 'base64-json-utf8'; + /** + * The document payload: UTF-8 JSON, base64-encoded, signed and hashed exactly as it appears here. + */ + payloadBase64: string; + /** + * SHA-256 of the decoded payload bytes. + */ + payloadSha256: string; + /** + * Detached signatures over the decoded payload bytes. A verifier accepts the document when any one signature verifies against a trusted key, which is what allows a key to be rotated without reissuing every document. + * + * @minItems 1 + */ + signatures: [ + { + algorithm: 'ed25519'; + keyId: string; + signatureBase64: string; + }, + ...{ + algorithm: 'ed25519'; + keyId: string; + signatureBase64: string; + }[] + ]; +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/sign/index.mjs b/.scrollcase-runtime-types-RzUfxr/src/sign/index.mjs new file mode 100644 index 0000000..29a0b11 --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/sign/index.mjs @@ -0,0 +1,132 @@ +/** + * Signing, with key custody left to the operator. + * + * Two paths, one envelope. The built-in path signs with a local ed25519 key, which is enough for + * development and for anyone happy to hold their own key. The external path hands the payload to a + * command the operator configures — a KMS, an HSM, a signing service — so the private key never + * touches the build machine and Scrollcase never learns anything about the custody model. + * + * What the external path does *not* do is take the result on faith. The returned document must echo + * back exactly the payload that was sent, and its signature is verified locally before the build + * continues. A signer that substitutes a payload, or returns a signature that does not verify, fails + * the build rather than producing a box nobody can install. + */ + +import { fail, runResult as defaultRunResult } from '../build/process.mjs'; +import { readSigningKey, signWithLocalKey, verifySignedDocument } from './keys.mjs'; + +export { + decodeSignedDocument, + generateSigningKey, + readSigningKey, + verifySignedDocument, +} from './keys.mjs'; + +/** + * Runs an external signer command. + * + * The contract is deliberately the simplest thing that composes with anything: the command receives + * the payload bytes on stdin and writes the complete signed document as JSON on stdout. Any language, + * any credential mechanism, no plugin API to keep compatible. + */ +function commandArguments(command) { + if (Array.isArray(command)) { + if (!command.every((part) => typeof part === 'string')) { + fail('External signer command array must contain only strings.'); + } + return command; + } + const source = String(command); + const args = []; + let current = ''; + let quote = null; + let tokenStarted = false; + for (let index = 0; index < source.length; index += 1) { + const character = source[index]; + if (quote === "'") { + if (character === quote) quote = null; + else current += character; + } else if (quote === '"') { + if (character === quote) { + quote = null; + } else if (character === '\\' && ['\\', '"'].includes(source[index + 1])) { + current += source[index + 1]; + index += 1; + } else { + current += character; + } + } else if (character === '"' || character === "'") { + quote = character; + tokenStarted = true; + } else if (/\s/.test(character)) { + if (tokenStarted) { + args.push(current); + current = ''; + tokenStarted = false; + } + } else if (character === '\\' && source[index + 1] + && (/[\s'"\\]/).test(source[index + 1])) { + current += source[index + 1]; + tokenStarted = true; + index += 1; + } else { + current += character; + tokenStarted = true; + } + } + if (quote) fail('External signer command has an unmatched quote.'); + if (tokenStarted) args.push(current); + return args; +} + +function signWithCommand(payloadBytes, command, runResult) { + const [executable, ...args] = commandArguments(command); + if (!executable) fail('External signer command is empty.'); + const result = runResult(executable, args, { + input: payloadBytes, + capture: true, + maxBuffer: 16 * 1024 * 1024, + }); + if (result.error) fail(`External signer failed to start: ${result.error.message}`); + if (result.status !== 0) { + const stderr = (result.stderr?.toString() || '').trim(); + fail(`External signer exited with ${result.status}${stderr ? `: ${stderr}` : ''}`); + } + try { + return JSON.parse(result.stdout.toString('utf8')); + } catch (error) { + fail(`External signer did not return a JSON document: ${error instanceof Error ? error.message : String(error)}`); + } +} + +/** + * Wraps a manifest in the signed envelope, through whichever signer is configured. + * + * The payload is serialised once and both hashed and signed as-is, so what gets signed is + * byte-for-byte what gets published. + * + * @param {unknown} payload the manifest to wrap; serialised once and signed exactly as serialised + * @param {{ signerCommand?: string | string[] | null, privatePath?: string, publicPath: string, + * runResult?: typeof defaultRunResult }} signing + * @returns {Promise} + * @throws {Error} when an external signer fails, alters the payload, or returns an unverifiable + * signature + */ +export async function signDocument(payload, { + signerCommand = null, + privatePath, + publicPath, + runResult = defaultRunResult, +}) { + const payloadBytes = Buffer.from(`${JSON.stringify(payload, null, 2)}\n`, 'utf8'); + if (signerCommand) { + const document = signWithCommand(payloadBytes, signerCommand, runResult); + if (document?.payloadBase64 !== payloadBytes.toString('base64')) { + fail('External signer returned a different payload than the one it was given.'); + } + // Verified against the trust anchor the operator points at, not against the signer's word. + await verifySignedDocument(document, publicPath); + return document; + } + return signWithLocalKey(payloadBytes, await readSigningKey({ privatePath, publicPath })); +} diff --git a/.scrollcase-runtime-types-RzUfxr/src/sign/keys.mjs b/.scrollcase-runtime-types-RzUfxr/src/sign/keys.mjs new file mode 100644 index 0000000..e58d23f --- /dev/null +++ b/.scrollcase-runtime-types-RzUfxr/src/sign/keys.mjs @@ -0,0 +1,160 @@ +/** + * Local signing keys and signature verification. + * + * A box is only worth as much as the signature over its release document, so the tool ships a + * working signer out of the box: `keygen` produces an ed25519 pair, and every document it emits can + * be verified with the matching public key. Production key custody is a separate concern — see the + * external signer in `index.mjs` — but verification always lives here, because a signature nobody + * checks is theatre. + */ + +import { + createPrivateKey, + createPublicKey, + generateKeyPairSync, + createHash, + sign as edSign, + verify as edVerify, +} from 'node:crypto'; +import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { fail } from '../build/process.mjs'; +import { fileExists } from '../build/filesystem.mjs'; +import { BOX_SCHEMA_VERSION, PAYLOAD_ENCODING } from '../contract/document-shape.mjs'; + +/** + * A published public key, as written by `keygen` and read back when verifying. + * + * @typedef {object} TrustedKey + * @property {'ed25519'} algorithm + * @property {string} keyId stable identifier derived from the key itself + * @property {string} publicKeyBase64 the raw 32-byte key, for non-Node verifiers + * @property {string} publicKeyPem + */ + +const sha256Hex = (bytes) => createHash('sha256').update(bytes).digest('hex'); + +/** + * Creates an ed25519 signing pair. + * + * Overwriting an existing key is gated behind `force` because doing so silently would invalidate + * every document previously signed with it, with no way to tell which. + * + * @param {{ privatePath: string, publicPath: string, keyId?: string | null, force?: boolean }} options + * @returns {Promise<{ keyId: string, privatePath: string, publicPath: string }>} + * @throws {Error} when a key already exists and `force` was not passed + */ +export async function generateSigningKey({ privatePath, publicPath, keyId, force = false }) { + if (await fileExists(privatePath) && !force) { + fail(`Signing key already exists: ${privatePath}. Pass --force to rotate it explicitly.`); + } + const { privateKey, publicKey } = generateKeyPairSync('ed25519'); + const privatePem = privateKey.export({ type: 'pkcs8', format: 'pem' }); + const publicPem = publicKey.export({ type: 'spki', format: 'pem' }); + const publicDer = publicKey.export({ type: 'spki', format: 'der' }); + // An ed25519 SPKI DER is a fixed 12-byte header followed by the 32-byte key, so the raw key is + // simply the tail. That raw form is what non-Node verifiers expect in base64. + const rawPublicKey = publicDer.subarray(publicDer.length - 32); + // Deriving the ID from the key itself makes it stable and collision-resistant without a registry. + const resolvedKeyId = keyId || `scrollcase-${sha256Hex(rawPublicKey).slice(0, 16)}`; + await mkdir(dirname(privatePath), { recursive: true }); + await mkdir(dirname(publicPath), { recursive: true }); + // Owner-only, and chmod again afterwards in case a permissive umask widened the mode on create. + await writeFile(privatePath, privatePem, { mode: 0o600 }); + await chmod(privatePath, 0o600); + await writeFile(publicPath, `${JSON.stringify({ + algorithm: 'ed25519', + keyId: resolvedKeyId, + publicKeyBase64: rawPublicKey.toString('base64'), + publicKeyPem: publicPem, + }, null, 2)}\n`); + return { keyId: resolvedKeyId, privatePath, publicPath }; +} + +/** + * Loads the private key and cross-checks it against the published public key file, so a mismatched + * pair is caught here rather than producing documents nobody can verify. + * + * @param {{ privatePath: string, publicPath: string }} options + * @returns {Promise<{ privateKey: import('node:crypto').KeyObject, metadata: TrustedKey }>} + * @throws {Error} when the key is missing, or the pair does not match + */ +export async function readSigningKey({ privatePath, publicPath }) { + if (!await fileExists(privatePath)) fail(`Signing key not found: ${privatePath}. Run keygen first.`); + const privateKey = createPrivateKey(await readFile(privatePath, 'utf8')); + const publicKey = createPublicKey(privateKey); + const rawPublicKey = publicKey.export({ type: 'spki', format: 'der' }).subarray(-32); + const metadata = JSON.parse(await readFile(publicPath, 'utf8')); + if (metadata.publicKeyBase64 !== rawPublicKey.toString('base64')) { + fail('Private and public signing keys do not match.'); + } + return { privateKey, metadata }; +} + +/** Accepts both trust-file shapes: a bundle of keys, or a single bare key. */ +function trustedKeyEntries(value) { + return Array.isArray(value?.keys) ? value.keys : [value]; +} + +/** + * Signs payload bytes with a local key, producing the envelope the format defines. + * + * @param {Buffer} payloadBytes the exact bytes to sign, which are also the bytes published + * @param {{ privateKey: import('node:crypto').KeyObject, metadata: TrustedKey }} key + * @returns {import('../contract/types/index.d.ts').SignedBoxDocument} + */ +export function signWithLocalKey(payloadBytes, { privateKey, metadata }) { + return { + schemaVersion: BOX_SCHEMA_VERSION, + payloadEncoding: PAYLOAD_ENCODING, + payloadBase64: payloadBytes.toString('base64'), + payloadSha256: sha256Hex(payloadBytes), + signatures: [{ + algorithm: 'ed25519', + keyId: metadata.keyId, + signatureBase64: edSign(null, payloadBytes, privateKey).toString('base64'), + }], + }; +} + +/** + * Unwraps an envelope and checks its checksum. Does *not* check the signature. + * + * @param {import('../contract/types/index.d.ts').SignedBoxDocument} document + * @returns {{ bytes: Buffer, payload: unknown }} + * @throws {Error} when the envelope is unsupported or its checksum does not match + */ +export function decodeSignedDocument(document) { + if (document?.schemaVersion === 1) { + fail('Unsupported schemaVersion 1; rebuild this box with Scrollcase v2.'); + } + if (document?.schemaVersion !== BOX_SCHEMA_VERSION || document?.payloadEncoding !== PAYLOAD_ENCODING) { + fail('Unsupported signed document.'); + } + const bytes = Buffer.from(document.payloadBase64, 'base64'); + if (sha256Hex(bytes) !== document.payloadSha256) fail('Signed payload SHA-256 mismatch.'); + return { bytes, payload: JSON.parse(bytes.toString('utf8')) }; +} + +/** + * Verifies a signed document against a trusted key file and returns its payload. + * + * The document is accepted when *any one* signature verifies against a trusted key, which is what + * allows a document signed by both an outgoing and an incoming key to stay valid across a rotation. + * + * @param {import('../contract/types/index.d.ts').SignedBoxDocument} document + * @param {string} publicKeyPath a single trusted key, or a `{ keys: [...] }` bundle + * @returns {Promise} the payload, once a signature has verified against a trusted key + * @throws {Error} when no signature verifies + */ +export async function verifySignedDocument(document, publicKeyPath) { + const trusted = trustedKeyEntries(JSON.parse(await readFile(publicKeyPath, 'utf8'))); + const { bytes, payload } = decodeSignedDocument(document); + const valid = document.signatures?.some((signature) => { + const key = trusted.find((candidate) => candidate.keyId === signature.keyId); + return key?.publicKeyPem + && edVerify(null, bytes, createPublicKey(key.publicKeyPem), Buffer.from(signature.signatureBase64, 'base64')); + }); + if (!valid) fail('Document has no valid signature from a trusted ed25519 key.'); + return payload; +} diff --git a/docs/guides/demo-box.md b/docs/guides/demo-box.md index c62f4b7..4c03e57 100644 --- a/docs/guides/demo-box.md +++ b/docs/guides/demo-box.md @@ -107,6 +107,18 @@ scrollcase-demo/ └── example-signing-public.json ``` +## What just happened + +`verify` checks the signature, the archive's size and hash, the entry names and manifest agreement, +and works on any machine. `run` extracts the box to a temporary directory and executes its entry +point with the interpreter *inside* it — so it needs a machine matching the box's target. What it +prints is `sys.prefix`, which is the point: the interpreter answering is the one from the box. + +::: warning The demo key is a demo key +Those boxes are signed with a key that exists only for the example. It signs nothing else and no +trust chain depends on it. A signature from it means the example is intact — nothing more. +::: + ## Run it from your own app The CLI is the quickest way to see a box work, but an application does not shell out to it: both @@ -189,15 +201,3 @@ trust chain itself. The Python package is published separately: `npm install scrollcase` does not install it, and `pip install scrollcase-consumer` needs no Node at all. Full surface in the [Library APIs reference](/reference/api). - -## What just happened - -`verify` checks the signature, the archive's size and hash, the entry names and manifest agreement, -and works on any machine. `run` extracts the box to a temporary directory and executes its entry -point with the interpreter *inside* it — so it needs a machine matching the box's target. What it -prints is `sys.prefix`, which is the point: the interpreter answering is the one from the box. - -::: warning The demo key is a demo key -Those boxes are signed with a key that exists only for the example. It signs nothing else and no -trust chain depends on it. A signature from it means the example is intact — nothing more. -::: From bf3f7007f3e3650519c2f9dded8cb5fb9068bc5a Mon Sep 17 00:00:00 2001 From: Lorenzo S <46894435+Suffro@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:39:36 +0200 Subject: [PATCH 3/3] Ship the demo box as a folder that already runs The page asked a reader to copy twenty lines of TypeScript and eighteen of Python into files with exact names under exact paths, which is the kind of step that fails silently: one wrong name and the example does not run, with nothing saying why. Each published archive now unpacks to `box/` holding the content-addressed pair, beside `run-box.ts`, `run_box.py` and a `package.json`. The three ways to run a box are two commands each, and the guide embeds those files from `examples/demo-consumers/` rather than restating them, so the page cannot document something other than what ships. The trust key is still absent from the archive on purpose, and the guide, the release notes and the bundled README each say why: a signature proves nothing if the key arrives in the same package as what it signs. --- .github/demo-box-release-notes.md | 49 +++++--- .github/workflows/demo-box.yml | 24 +++- CHANGELOG.md | 18 ++- docs/guides/demo-box.md | 168 ++++++++++++--------------- examples/README.md | 7 ++ examples/demo-consumers/README.md | 62 ++++++++++ examples/demo-consumers/package.json | 16 +++ examples/demo-consumers/run-box.ts | 36 ++++++ examples/demo-consumers/run_box.py | 51 ++++++++ 9 files changed, 318 insertions(+), 113 deletions(-) create mode 100644 examples/demo-consumers/README.md create mode 100644 examples/demo-consumers/package.json create mode 100644 examples/demo-consumers/run-box.ts create mode 100644 examples/demo-consumers/run_box.py diff --git a/.github/demo-box-release-notes.md b/.github/demo-box-release-notes.md index 8eee68a..c11c079 100644 --- a/.github/demo-box-release-notes.md +++ b/.github/demo-box-release-notes.md @@ -13,39 +13,60 @@ Download the one archive matching your machine: | macOS, Apple silicon | `hello-box-1.0.0-macos-aarch64-metal.zip` | | Windows, Intel or AMD | `hello-box-1.0.0-windows-x86_64-cpu.zip` | -Unpack it and you get two files: the box and its signed release document. Keep them in a directory of -their own, and take the trust key from the repository rather than from this page — a signature only -proves where something came from if the key does not travel with it: +Unpacking it gives a folder that already runs: the box under `box/`, plus `run-box.ts` and +`run_box.py` for driving it from an application instead of the terminal. + +```text +scrollcase-demo/ +├── box/ +│ ├── .zip the box — leave it zipped and named as it is +│ └── .release.json +├── run-box.ts +├── run_box.py +└── package.json +``` + +The trust key is deliberately **not** in that archive. A signature only proves where something came +from if the key does not arrive in the same package, so it is fetched from the repository: ```sh -npm install -g scrollcase -mkdir scrollcase-demo && cd scrollcase-demo -unzip ../hello-box-1.0.0-.zip -d box +unzip hello-box-1.0.0-.zip -d scrollcase-demo +cd scrollcase-demo mkdir keys curl -o keys/example-signing-public.json \ https://raw.githubusercontent.com/suffro/scrollcase/main/examples/keys/example-signing-public.json +``` + +Then any one of these three — they perform the same checks in the same order: +```sh +# Terminal +npm install -g scrollcase scrollcase verify box/*.release.json --public-key keys/example-signing-public.json scrollcase run box/*.release.json --public-key keys/example-signing-public.json + +# Node +npm install && npx tsx run-box.ts + +# Python +python -m pip install scrollcase-consumer && python run_box.py ``` On PowerShell that glob is not expanded for a command like this — use -`(Get-ChildItem box\*.release.json).FullName`, or simply type the file name you see after unpacking. +`(Get-ChildItem box\*.release.json).FullName`, or simply type the file name you see under `box/`. `verify` checks the signature, the archive's size and hash, the entry names and the manifest. Adding `--self-test` extracts the box and imports with the interpreter inside it, and `run` executes its entry point — both need the machine to match the box's target. `verify` on its own works anywhere. -An application would not shell out to the CLI: the same box runs through the Node and Python -consumer libraries, which is covered in the -[demo box guide](https://scrollcase.dev/guides/demo-box). - -The two unpacked names are SHA-256 digests of their own contents: two builds of the same commit +The two names under `box/` are SHA-256 digests of their own contents: two builds of the same commit produce the same names, which is what makes the archive verifiable in the first place. Keep them as they are and side by side — `verify` finds the box by the hash its release document commits to, and -renaming or separating them breaks that. The enclosing zip exists only so the download says which -machine it is for. +renaming or separating them breaks that. The enclosing zip carries no guarantee of its own; it holds +the pair and the examples, and its name says which machine they are for. + +Full walkthrough: [the demo box guide](https://scrollcase.dev/guides/demo-box). ## About the signing key diff --git a/.github/workflows/demo-box.yml b/.github/workflows/demo-box.yml index fa5c52b..bee5504 100644 --- a/.github/workflows/demo-box.yml +++ b/.github/workflows/demo-box.yml @@ -139,10 +139,16 @@ jobs: # that is how `verify` finds it. Published flat, that gave six hex names on the release page and # no way to tell which three were yours before downloading them. # - # So each target ships as one plainly named container holding that pair. The names inside stay - # content-addressed and adjacent, which is what `verify` needs; the name outside says which + # So each target ships as one plainly named container. The pair keeps content-addressed names + # and stays adjacent under `box/`, which is what `verify` needs; the name outside says which # machine it is for, which is what a person needs. Stored rather than compressed: the inner # archive is already deflated, and recompressing it would cost minutes to save nothing. + # + # The consumer examples travel with it, so unpacking gives a folder that already runs three + # ways instead of source to retype. They are copied from `examples/demo-consumers/`, the same + # files the guide embeds, so the page cannot document something other than what ships. The + # trust key is deliberately absent: a signature proves nothing if the key arrives in the same + # package as what it signs. - name: Wrap each target as one plainly named archive shell: bash run: | @@ -159,7 +165,19 @@ jobs: ls -l "$dir" >&2 exit 1 fi - zip -0 -X -j "wrapped/$BOX_ID-$BOX_VERSION-$target.zip" "${zips[0]}" "${releases[0]}" + stage="staging/$target" + mkdir -p "$stage/box" + cp "${zips[0]}" "${releases[0]}" "$stage/box/" + cp examples/demo-consumers/run-box.ts \ + examples/demo-consumers/run_box.py \ + examples/demo-consumers/package.json \ + examples/demo-consumers/README.md "$stage/" + # Entries named explicitly rather than with `.`, so the archive carries no `./` prefix + # and no stray dotfile the runner happens to leave in the staging directory. + ( cd "$stage" && zip -0 -X -r -q \ + "$GITHUB_WORKSPACE/wrapped/$BOX_ID-$BOX_VERSION-$target.zip" \ + box run-box.ts run_box.py package.json README.md ) + unzip -l "wrapped/$BOX_ID-$BOX_VERSION-$target.zip" done ls -l wrapped diff --git a/CHANGELOG.md b/CHANGELOG.md index 2406aa8..322aeaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,22 @@ All notable changes to Scrollcase are documented here. The format follows ### Changed - Publish the demo box as one plainly named archive per operating system — - `hello-box-1.0.0-macos-aarch64-metal.zip` and its two siblings — each holding the box and its - signed release document. A box archive has to be named for its own SHA-256 and sit beside that + `hello-box-1.0.0-macos-aarch64-metal.zip` and its two siblings — that unpacks to a folder which + already runs. A box archive has to be named for its own SHA-256 and sit beside its release document, because that is how `verify` finds it, so publishing the files flat gave the release page six hex names and no way to tell which three belonged to your machine before downloading - them. The names inside the wrapper stay content-addressed and adjacent; only the name outside is - for people. Stored rather than compressed, it costs 2.7 KB on a 37 MB box. + them. The pair now keeps those names under `box/`, where `verify` still resolves one from the + other, while the name outside says which machine it is for. + + Beside it travel `run-box.ts`, `run_box.py` and a `package.json`, so each of the three ways to run + a box — CLI, Node consumer, Python consumer — is two commands rather than source to copy out of a + page and save under the right name. Those files come from `examples/demo-consumers/` and the guide + embeds them from there, so what is documented and what is shipped cannot drift apart. The trust + key is deliberately not among them and still comes from the repository: a signature proves nothing + if the key arrives in the same package as what it signs. + + Stored rather than compressed, since the box archive is already deflated: the container costs + about 9 KB on a 37 MB box. ### Fixed diff --git a/docs/guides/demo-box.md b/docs/guides/demo-box.md index 4c03e57..185f4f5 100644 --- a/docs/guides/demo-box.md +++ b/docs/guides/demo-box.md @@ -22,9 +22,9 @@ Download the demo for your system: ::: tip NOTE -The file you download (eg. hello-box-1.0.0-macos-aarch64-metal.zip) is **NOT** the demo box — it is just a container, named so you can tell which machine it is for. +The file you download (eg. hello-box-1.0.0-macos-aarch64-metal.zip) is **NOT** the demo box — it is a container, named so you can tell which machine it is for, holding the box together with two ready-to-run examples. -The demo box is the .zip you find inside it, next to the .release.json. Do not unzip that one: **it's ready to run**. Leave both files named as they are and side by side, because that is how `verify` finds the box. +The demo box is the .zip inside it under box/, next to its .release.json. Do not unzip that one: **it's ready to run**. Leave both named as they are and side by side, because that is how `verify` finds the box. ::: @@ -32,15 +32,15 @@ The demo box is the .zip you find inside it, next to the .rel Once you have downloaded the demo, follow these steps: -1. **Install scrollcase and unpack the demo into a folder of its own:** +1. **Unpack the demo into a folder of its own:** ```sh -npm install -g scrollcase # this will install it globally -mkdir scrollcase-demo && cd scrollcase-demo -unzip ../hello-box-1.0.0-.zip -d box +unzip hello-box-1.0.0-.zip -d scrollcase-demo +cd scrollcase-demo ``` -> **box/** now holds 2 files: the **demo box** `.zip` to run, and its matching `.release.json`.
+> **box/** holds 2 files: the **demo box** `.zip` to run, and its matching `.release.json`. Beside it +> you already have `run-box.ts` and `run_box.py` — nothing to retype.
--- @@ -64,16 +64,25 @@ habit to carry into a real project, where the key will not be a demo key. 3. **Verify and run the box:** + + + +This path needs the CLI, and nothing else — no pixi, no conda-pack, no build: + +```sh +npm install -g scrollcase +``` + - + ```sh scrollcase verify box/*.release.json --public-key keys/example-signing-public.json scrollcase run box/*.release.json --public-key keys/example-signing-public.json ``` - - + + ```powershell scrollcase verify (Get-ChildItem box\*.release.json).FullName --public-key keys\example-signing-public.json @@ -89,115 +98,90 @@ scrollcase run (Get-ChildItem box\*.release.json).FullName --public-key keys\ > after unzipping. You never name the box archive: `verify` finds it beside the release document, > under the hash that document commits to. - - ---- - -4. **Check out the results, that's it.** - -At this point the folder looks like this — the box untouched in its own directory, the key beside -it, nothing loose: - -```text -scrollcase-demo/ -├── box/ -│ ├── .zip # the demo box, left exactly as downloaded -│ └── .release.json -└── keys/ - └── example-signing-public.json -``` - -## What just happened +### What just happened `verify` checks the signature, the archive's size and hash, the entry names and manifest agreement, and works on any machine. `run` extracts the box to a temporary directory and executes its entry point with the interpreter *inside* it — so it needs a machine matching the box's target. What it prints is `sys.prefix`, which is the point: the interpreter answering is the one from the box. -::: warning The demo key is a demo key -Those boxes are signed with a key that exists only for the example. It signs nothing else and no -trust chain depends on it. A signature from it means the example is intact — nothing more. -::: + + -## Run it from your own app -The CLI is the quickest way to see a box work, but an application does not shell out to it: both -consumers expose the same verify-then-run semantics as a library, and they take the very files you -just downloaded. Same folder, same key, nothing rebuilt. +### Run it from your own app - - +The CLI is the quickest way to see a box work, but an application does not shell out to it: both +consumers expose the same verify-then-run semantics as a library. `run-box.ts` and `run_box.py` are +already in the folder you unpacked, so this is two commands, not a copy-paste. -```sh -npm install scrollcase -npm install --save-dev tsx typescript -``` - -```ts -// run-box.ts -import { readdirSync } from 'node:fs'; -import { join } from 'node:path'; -import { runBox } from 'scrollcase/consumer'; - -const release = readdirSync('box').find((name) => name.endsWith('.release.json'))!; - -runBox(join('box', release), { - publicPath: 'keys/example-signing-public.json', - stdout: 'inherit', - stderr: 'inherit', - onPrepared: ({ boxId, version, targetId }) => { - console.log(`Running ${boxId} ${version} (${targetId})`); - }, -}).then((result) => { - process.exitCode = result.exitCode ?? 1; -}); -``` + + ```sh +npm install npx tsx run-box.ts ``` - - +::: details run-box.ts — the file you just ran +<<< @/../examples/demo-consumers/run-box.ts +::: + + + ```sh python -m pip install scrollcase-consumer +python run_box.py ``` -```python -# run_box.py -from pathlib import Path +::: details run_box.py — the file you just ran +<<< @/../examples/demo-consumers/run_box.py +::: -from scrollcase_consumer import PreparedBox, run_box + + -release = next(Path("box").glob("*.release.json")) +`runBox` verifies the signature, extracts to a private temporary directory, executes, and cleans up +after itself — the same chain `scrollcase run` performs, minus the terminal. `onPrepared` fires +after verification and before execution, which is how an application shows what it is about to run +without repeating the trust chain itself. +Neither file names the box archive. Both find the release document by its suffix and let the +consumer resolve the archive beside it, under the hash that document commits to. -def report(prepared: PreparedBox) -> None: - print(f"Running {prepared.box_id} {prepared.version} ({prepared.target_id})", flush=True) +The Python package is published separately: `npm install scrollcase` does not install it, and +`pip install scrollcase-consumer` needs no Node at all. Full surface in the +[Library APIs reference](/reference/api). + + -result = run_box( - release, - public_key_path="keys/example-signing-public.json", - on_prepared=report, -) -raise SystemExit(result.exit_code or 0) -``` +::: warning The demo key is a demo key +Those boxes are signed with a key that exists only for the example. It signs nothing else and no +trust chain depends on it. A signature from it means the example is intact — nothing more. +::: -```sh -python run_box.py -``` - - +--- -Drop either file at the top of `scrollcase-demo/` and run it from there. `runBox` verifies the -signature, extracts to a private temporary directory, executes, and cleans up after itself — the -same chain `scrollcase run` performs, minus the terminal. `onPrepared` fires after verification and -before execution, which is how an application shows what it is about to run without repeating the -trust chain itself. +4. **Check out the results, that's it.** -The Python package is published separately: `npm install scrollcase` does not install it, and -`pip install scrollcase-consumer` needs no Node at all. Full surface in the -[Library APIs reference](/reference/api). +At this point the folder looks like this — the box untouched in its own directory, the key in +another, the runnable examples above both: + +```text +scrollcase-demo/ +├── box/ # from the download, left exactly as it arrived +│ ├── .zip +│ └── .release.json +├── keys/ # step 2, from the repository — never from the release +│ └── example-signing-public.json +├── run-box.ts # from the download +├── run_box.py # from the download +├── package.json # from the download +└── README.md # from the download +``` + +Everything except `keys/` came out of the one file you downloaded. The key is the deliberate +exception, and the reason is above. diff --git a/examples/README.md b/examples/README.md index 1500fc5..bc36887 100644 --- a/examples/README.md +++ b/examples/README.md @@ -12,6 +12,13 @@ not the key for any Scrollcase release. Its private half lives in a repository s `.github/workflows/demo-box.yml` alone — a Linux or Windows box cannot be built on a maintainer's machine anyway, since conda-pack packs the host's own environment. +`demo-consumers/` holds what travels inside each published archive beside the box: `run-box.ts`, +`run_box.py`, a `package.json`, and a `README.md`, so unpacking a download gives a folder that +already runs three ways. The same files are embedded in +[the demo box guide](https://scrollcase.dev/guides/demo-box), which is why they live here rather +than in the page — documentation and shipped bytes cannot drift apart. The public key is never +copied in: a signature proves nothing if the key arrives in the same package as what it signs. + ## `hello-box` The smallest thing Scrollcase can build: a stdlib-only Python 3.11 environment from conda-forge, diff --git a/examples/demo-consumers/README.md b/examples/demo-consumers/README.md new file mode 100644 index 0000000..3d39894 --- /dev/null +++ b/examples/demo-consumers/README.md @@ -0,0 +1,62 @@ +# Scrollcase demo box + +You unpacked a signed box built by CI from `examples/hello-box/` in the Scrollcase repository. It is +a stdlib-only Python 3.11 environment, and it runs on this machine without pixi, conda-pack, or a +build. + +```text +. +├── box/ +│ ├── .zip the box — leave it zipped and named as it is +│ └── .release.json the signed release document +├── run-box.ts run it from Node +├── run_box.py run it from Python +└── package.json +``` + +Both names under `box/` are SHA-256 digests of their own contents. `verify` finds the archive beside +the release document, under the hash that document commits to, so renaming or separating the two +breaks it. + +## 1. Get the trust key + +It is deliberately not in this archive. A signature only proves where something came from if the key +does not arrive in the same package. + +```sh +mkdir keys +curl -o keys/example-signing-public.json \ + https://raw.githubusercontent.com/suffro/scrollcase/main/examples/keys/example-signing-public.json +``` + +## 2. Run it + +Any one of these three. They perform the same checks in the same order. + +```sh +# Terminal +npm install -g scrollcase +scrollcase verify box/*.release.json --public-key keys/example-signing-public.json +scrollcase run box/*.release.json --public-key keys/example-signing-public.json + +# Node +npm install && npx tsx run-box.ts + +# Python +python -m pip install scrollcase-consumer && python run_box.py +``` + +On PowerShell the `box/*.release.json` glob is not expanded for a command like this — use +`(Get-ChildItem box\*.release.json).FullName`, or type the file name you see under `box/`. + +`verify` checks the signature, the archive's size and hash, the entry names and the manifest, and +works on any machine. Running the box needs a machine matching its target, because the interpreter +inside it is executed. What it prints is `sys.prefix` — the interpreter answering is the box's own. + +## About the signing key + +This box is signed with a key that exists **only for this demo**. It signs nothing else, no trust +chain depends on it, and it is not the key for any Scrollcase release. Treat a signature from it as +evidence that the example is intact — never as evidence that anything else is. + +Full walkthrough: diff --git a/examples/demo-consumers/package.json b/examples/demo-consumers/package.json new file mode 100644 index 0000000..526fe9a --- /dev/null +++ b/examples/demo-consumers/package.json @@ -0,0 +1,16 @@ +{ + "name": "scrollcase-demo", + "private": true, + "type": "module", + "description": "Runs the published Scrollcase demo box through the Node consumer.", + "scripts": { + "start": "tsx run-box.ts" + }, + "dependencies": { + "scrollcase": "^0.6.1" + }, + "devDependencies": { + "tsx": "^4.19.2", + "typescript": "^5.7.2" + } +} diff --git a/examples/demo-consumers/run-box.ts b/examples/demo-consumers/run-box.ts new file mode 100644 index 0000000..a1812a0 --- /dev/null +++ b/examples/demo-consumers/run-box.ts @@ -0,0 +1,36 @@ +/** + * Runs the demo box through the typed Node consumer. + * + * SETUP (once, from this folder): + * npm install + * + * RUN: + * npx tsx run-box.ts + * + * The public key is not shipped with the box: download it first, as the guide describes. A + * signature only proves where something came from if the key does not travel with it. + */ +import { readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { runBox } from 'scrollcase/consumer'; + +// The release document is named for its own SHA-256, so it is found rather than hard-coded. The box +// archive is never named here: the consumer resolves it beside this document, under the hash the +// document commits to. +const release = readdirSync('box').find((name) => name.endsWith('.release.json')); +if (!release) throw new Error('No .release.json in box/ — unpack the downloaded archive first.'); + +const result = await runBox(join('box', release), { + publicPath: 'keys/example-signing-public.json', + stdout: 'inherit', + stderr: 'inherit', + // Fires after the signature, the archive hash and the manifest have been checked, and before the + // box interpreter starts, so an application can show what it is about to run without repeating + // the trust chain itself. + onPrepared: ({ boxId, version, targetId }) => { + console.log(`Running ${boxId} ${version} (${targetId})`); + }, +}); + +if (result.signal) console.error(`Box exited after ${result.signal}.`); +process.exitCode = result.exitCode ?? 1; diff --git a/examples/demo-consumers/run_box.py b/examples/demo-consumers/run_box.py new file mode 100644 index 0000000..0afa9b5 --- /dev/null +++ b/examples/demo-consumers/run_box.py @@ -0,0 +1,51 @@ +"""Runs the demo box through the typed Python consumer. + +The Python consumer is published separately on PyPI, and needs no Node at all. + +SETUP (once): + + python -m pip install scrollcase-consumer + +RUN (from this folder): + + python run_box.py + +The public key is not shipped with the box: download it first, as the guide describes. A signature +only proves where something came from if the key does not travel with it. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from scrollcase_consumer import PreparedBox, run_box + +# The release document is named for its own SHA-256, so it is found rather than hard-coded. The box +# archive is never named here: the consumer resolves it beside this document, under the hash the +# document commits to. +releases = sorted(Path("box").glob("*.release.json")) +if not releases: + raise SystemExit("No .release.json in box/ — unpack the downloaded archive first.") + + +def report(prepared: PreparedBox) -> None: + """Runs after verification and before the box interpreter starts.""" + + # Flushed because the box writes straight to this process's stdout, and an unflushed line would + # appear after the output it introduces. + print( + f"Running {prepared.box_id} {prepared.version} ({prepared.target_id})", + flush=True, + ) + + +result = run_box( + releases[0], + public_key_path="keys/example-signing-public.json", + on_prepared=report, +) + +if result.signal: + print(f"Box exited after {result.signal}.", file=sys.stderr) +raise SystemExit(result.exit_code if result.exit_code is not None else 1)