From 0ed2792bb8a91711115e3d04368f4f815aa894b1 Mon Sep 17 00:00:00 2001 From: Andrew Peris Date: Fri, 11 Sep 2026 09:34:08 -0400 Subject: [PATCH] Add reusable SDK conformance suite and reference goldens --- .github/workflows/prompt-sdk-tests.yml | 23 ++ CONSTITUTION_CONFORMANCE.md | 19 ++ README.md | 3 + examples/reference-prompts/README.md | 47 ++++ .../context-authorization.json | 25 ++ .../reference-prompts/context-package.json | 72 +++++ .../reference-prompts/context.prompt.json | 160 +++++++++++ examples/reference-prompts/run.mjs | 54 ++++ .../reference-prompts/structured.prompt.json | 144 ++++++++++ examples/reference-prompts/text.prompt.json | 143 ++++++++++ tests/README.md | 52 ++++ tests/fixtures/golden/context.json | 71 +++++ tests/fixtures/golden/structured.json | 258 ++++++++++++++++++ tests/fixtures/golden/text.json | 132 +++++++++ tests/prompt-sdk/adapter-conformance.test.js | 31 +++ tests/prompt-sdk/reference-prompts.test.js | 27 ++ tests/support/adapter-conformance.js | 57 ++++ 17 files changed, 1318 insertions(+) create mode 100644 .github/workflows/prompt-sdk-tests.yml create mode 100644 examples/reference-prompts/README.md create mode 100644 examples/reference-prompts/context-authorization.json create mode 100644 examples/reference-prompts/context-package.json create mode 100644 examples/reference-prompts/context.prompt.json create mode 100644 examples/reference-prompts/run.mjs create mode 100644 examples/reference-prompts/structured.prompt.json create mode 100644 examples/reference-prompts/text.prompt.json create mode 100644 tests/fixtures/golden/context.json create mode 100644 tests/fixtures/golden/structured.json create mode 100644 tests/fixtures/golden/text.json create mode 100644 tests/prompt-sdk/adapter-conformance.test.js create mode 100644 tests/prompt-sdk/reference-prompts.test.js create mode 100644 tests/support/adapter-conformance.js diff --git a/.github/workflows/prompt-sdk-tests.yml b/.github/workflows/prompt-sdk-tests.yml new file mode 100644 index 0000000..e25cb76 --- /dev/null +++ b/.github/workflows/prompt-sdk-tests.yml @@ -0,0 +1,23 @@ +name: Prompt SDK conformance +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + node: [22, 24] + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: ${{ matrix.node }} + cache: npm + - run: npm ci + - run: npm test diff --git a/CONSTITUTION_CONFORMANCE.md b/CONSTITUTION_CONFORMANCE.md index b1375d2..0096341 100644 --- a/CONSTITUTION_CONFORMANCE.md +++ b/CONSTITUTION_CONFORMANCE.md @@ -1,5 +1,24 @@ # Constitution conformance record +## Issue #71 assessment — 2026-09-11 + +Scope: reusable offline adapter conformance suite; text, structured and context +reference prompts; deterministic integration goldens and failure cases; CI +matrix and coverage documentation. Builds on the merged issue #70 baseline. +Constitution v1.0.0 at `a9cc8a503aa30e17820edc62ac95f7cbe10e0564` +remains the authority. Accountable owner: @andrewperis; status: Conforming +candidate pending owner review and merge. + +Evidence: 100 passing local tests, including 12 common adapter cases across two +implementations and five end-to-end reference cases. Goldens use fixed clocks +and synthetic inputs; non-public context bodies and identities are excluded. +CI configuration uses immutable action pins, read-only permissions and no +provider secrets. No new contracts, dependencies or constitutional exceptions. +Optional live-provider mappings, unsupported image/multimodal execution and +immutable publication (#72) are outside scope. Existing dependency advisories +remain separate. Review before release or any fixture/privacy/adapter boundary +change. The CI matrix must also pass remotely before merge. + ## Issue #70 assessment — 2026-09-11 Scope: CLI inspection, explicit fixture rendering, mock/trusted-module execution, diff --git a/README.md b/README.md index 226539f..1d36af5 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ Production software for the Definitely Secure Studio creative toolchain. +The [reference prompt library](examples/reference-prompts/README.md) demonstrates +offline end-to-end v1 flows with reviewed golden fixtures. + Use the [Prompt CLI](docs/prompt-cli.md) to validate, inspect, render, execute synthetic tests and validate structured outputs. diff --git a/examples/reference-prompts/README.md b/examples/reference-prompts/README.md new file mode 100644 index 0000000..16379d1 --- /dev/null +++ b/examples/reference-prompts/README.md @@ -0,0 +1,47 @@ +# Prompt SDK v1 reference library + +All prompts, inputs, outputs and context are synthetic and publishable. The +internal classification in the context example exercises policy; it is not +private production material. No real credentials or remote providers are used. + +```sh +npm ci +node examples/reference-prompts/run.mjs text +node examples/reference-prompts/run.mjs structured +node examples/reference-prompts/run.mjs context +npm test +``` + +- `text.prompt.json`: typed inputs, explicit defaults, optional context, text + execution, validation evidence and public provenance. +- `structured.prompt.json`: independent json-schema validation against the + existing synthetic reference-facts schema, raw/normalized identities and + a second provenance revision for processing evidence. +- `context.prompt.json`: prepared package and explicit authorization, required + text/JSON context slots, classification propagation and metadata redaction. + +The runner uses shared synthetic inputs and request settings from +`../cli/`. The context clock is explicitly 2026-08-20; execution uses a fixed +epoch clock. This keeps historical fixtures reproducible without weakening +expiry enforcement (a separate negative test uses an expired timestamp). + +The reference-facts artifact URI, release tag and all-2 commit are deliberately +synthetic identity metadata, not a claim of an immutable published contract. +The runner supplies verified schema bytes locally; it never downloads that URI. +Actual immutable contract release remains gated on Studio #72. + +Prompt SDK v1 supports text messages and JSON output here. It has no supported +image/multimodal execution contract; these fixtures do not pretend otherwise. + +## Golden review + +`tests/fixtures/golden/` stores the complete public render result, structured +result when applicable, and observer records. Non-public context render/output +bodies are omitted. Golden comparisons do not mask hashes, ids, defaults or +timestamps. Every observer/structured document is independently validated too. + +To inspect a proposed change, run the corresponding runner and compare its JSON +with the committed golden. Update a golden only after reviewing the contract, +canonicalization, redaction and intended behavior change; do not regenerate +snapshots just to make a failing test pass. The test command never writes goldens. +Check fixture safety before committing any updated content. diff --git a/examples/reference-prompts/context-authorization.json b/examples/reference-prompts/context-authorization.json new file mode 100644 index 0000000..ac67baf --- /dev/null +++ b/examples/reference-prompts/context-authorization.json @@ -0,0 +1,25 @@ +{ + "spec_version": "1.0.0", + "kind": "context-authorization", + "decision_id": "context_auth_reference_0001", + "decision": "allow", + "package": { + "id": "context-package.reference.synthetic", + "version": "1.0.0", + "instance_id": "ctxpkg_reference_0001" + }, + "prompt": { + "id": "prompt.reference.context-package", + "version": "1.0.0" + }, + "sections": [ + "approved_notes", + "context_facts" + ], + "max_classification": "internal", + "purpose": "Render synthetic approved context for issue #67 integration testing.", + "decided_by": "@andrewperis", + "decided_at": "2026-08-20T00:00:00Z", + "expires_at": "2026-08-21T00:00:00Z", + "authority_reference": "https://github.com/DefinitelySecureStudio/studio/issues/67" +} diff --git a/examples/reference-prompts/context-package.json b/examples/reference-prompts/context-package.json new file mode 100644 index 0000000..6c13066 --- /dev/null +++ b/examples/reference-prompts/context-package.json @@ -0,0 +1,72 @@ +{ + "spec_version": "1.0.0", + "kind": "context-package", + "manifest": { + "package": { + "id": "context-package.reference.synthetic", + "version": "1.0.0", + "instance_id": "ctxpkg_reference_0001" + }, + "builder": { + "id": "studio.mock.context-builder", + "version": "1.0.0" + }, + "created_at": "2026-08-20T00:00:00Z", + "review_after": "2026-08-21T00:00:00Z", + "expires_at": "2026-08-22T00:00:00Z", + "purpose": "Render synthetic approved context for issue #67 integration testing.", + "authority_reference": "https://github.com/DefinitelySecureStudio/studio/issues/67", + "classification": "internal", + "total_content_bytes": 57, + "sources": [ + { + "source_id": "source.synthetic.public", + "kind": "synthetic", + "version": "1.0.0", + "classification": "public", + "evidence_reference": "https://github.com/DefinitelySecureStudio/studio/issues/67" + }, + { + "source_id": "source.synthetic.internal", + "kind": "synthetic", + "version": "1.0.0", + "classification": "internal", + "evidence_reference": "https://github.com/DefinitelySecureStudio/studio/issues/67" + } + ], + "sections": [ + { + "slot": "approved_notes", + "classification": "public", + "media_type": "text/plain", + "content": "Synthetic approved note.", + "byte_size": 24, + "sha256": "sha256:c847bf684b5aabe0ea2d290978d299a6060a8132a01bd03fd606d6bdbb465384", + "source_ids": [ + "source.synthetic.public" + ] + }, + { + "slot": "context_facts", + "classification": "internal", + "media_type": "application/json", + "content": { + "count": 1, + "facts": [ + "synthetic" + ] + }, + "byte_size": 33, + "sha256": "sha256:086a926808322e2c933c498209d220850d84c1366dc967ddd502417d1f33560f", + "source_ids": [ + "source.synthetic.internal" + ] + } + ] + }, + "manifest_identity": { + "canonicalization": "studio-json-v1", + "byte_size": 1427, + "sha256": "sha256:1c5b3e293a37898c045c4830c116545f52d38e87e40220b1733637b2ef734c2b" + } +} diff --git a/examples/reference-prompts/context.prompt.json b/examples/reference-prompts/context.prompt.json new file mode 100644 index 0000000..abce4a2 --- /dev/null +++ b/examples/reference-prompts/context.prompt.json @@ -0,0 +1,160 @@ +{ + "spec_version": "1.0.0", + "id": "prompt.reference.context-package", + "version": "1.0.0", + "name": "Describe a public item", + "description": "Synthetic renderer fixture for an explicitly supplied public item.", + "purpose": "Exercise provider-neutral rendering; it does not retrieve data, approve output, or establish Canon.", + "owners": [ + "@andrewperis" + ], + "lifecycle": { + "status": "draft" + }, + "inputs": [ + { + "name": "item", + "description": "Synthetic public item name.", + "type": "string", + "required": true, + "classification": "public", + "constraints": { + "min_length": 1 + } + }, + { + "name": "attributes", + "description": "Synthetic attributes.", + "type": "object", + "required": true, + "classification": "public" + }, + { + "name": "concise", + "description": "Request concise output.", + "type": "boolean", + "required": false, + "classification": "public", + "default": true + } + ], + "context_slots": [ + { + "name": "approved_notes", + "description": "Explicit synthetic notes.", + "required": true, + "accepted_classifications": [ + "public", + "internal" + ], + "accepted_media_types": [ + "text/plain" + ], + "max_bytes": 100 + }, + { + "name": "context_facts", + "description": "Synthetic facts", + "required": true, + "accepted_classifications": [ + "internal" + ], + "accepted_media_types": [ + "application/json" + ], + "max_bytes": 100 + } + ], + "template": { + "format": "studio-messages-v1", + "messages": [ + { + "role": "instruction", + "parts": [ + { + "type": "text", + "text": "Describe only the supplied synthetic item." + } + ] + }, + { + "role": "user", + "parts": [ + { + "type": "text", + "text": "Item: " + }, + { + "type": "input", + "name": "item" + }, + { + "type": "text", + "text": "\nAttributes: " + }, + { + "type": "input", + "name": "attributes", + "format": "json" + }, + { + "type": "text", + "text": "\nConcise: " + }, + { + "type": "input", + "name": "concise", + "format": "text" + }, + { + "type": "text", + "text": "\nNotes: " + }, + { + "type": "context", + "slot": "approved_notes" + }, + { + "type": "context", + "slot": "context_facts", + "format": "json" + } + ] + } + ] + }, + "capabilities": { + "required": [ + "text-generation" + ], + "optional": [] + }, + "output": { + "kind": "text", + "media_type": "text/plain", + "description": "A synthetic plain-text description for human review." + }, + "provenance": { + "origin": "studio-original", + "created_by": "@andrewperis", + "created_at": "2026-08-17T00:00:00Z", + "source_references": [], + "rights": { + "basis": "studio-original", + "reviewed_by": "@andrewperis", + "reviewed_at": "2026-08-17T00:00:00Z", + "notices": [] + } + }, + "governance": { + "constitution": { + "version": "1.0.0", + "tag": "constitution/v1.0.0", + "commit": "a9cc8a503aa30e17820edc62ac95f7cbe10e0564" + }, + "decision_owner": "@andrewperis", + "evidence": [ + "https://github.com/DefinitelySecureStudio/studio/issues/71" + ] + } +} diff --git a/examples/reference-prompts/run.mjs b/examples/reference-prompts/run.mjs new file mode 100644 index 0000000..fe2d603 --- /dev/null +++ b/examples/reference-prompts/run.mjs @@ -0,0 +1,54 @@ +import { readFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { pathToFileURL } from 'node:url'; +import { renderPrompt, renderPromptWithContextPackage, createExecutionRequest, executePrompt, + MockTextAdapter, LocalExecutionObserver, validatePromptDefinition, processStructuredOutput, + createExecutionProvenance } from '../../src/prompt-sdk/index.js'; +const read = async file => JSON.parse(await readFile(new URL(file, import.meta.url))); +export async function runReference(name, { invalidOutput = false, expiredContext = false } = {}) { + if (!['text', 'structured', 'context'].includes(name)) throw new Error('Unknown reference'); + const definition = await read(name + '.prompt.json'); + const validation = validatePromptDefinition(definition); + if (!validation.valid) throw new Error('Invalid reference definition'); + const inputs = await read('../cli/inputs.json'); + const rendered = name === 'context' + ? renderPromptWithContextPackage(definition, { ...inputs, packageDocument: await read('context-package.json'), authorization: await read('context-authorization.json'), at: expiredContext ? '2026-08-23T00:00:00Z' : '2026-08-20T12:00:00Z' }) + : renderPrompt(definition, inputs); + const options = await read('../cli/execution-options.json'); + options.execution_id = 'exec_reference_' + name; + options.idempotency_key = 'idem_reference_' + name; + options.correlation_id = 'build_reference'; + let schemaSource; + if (name === 'structured') { + schemaSource = await readFile(new URL('../../tests/fixtures/structured-output/reference-facts.schema.json', import.meta.url), 'utf8'); + options.capabilities.required.push('structured-output'); + options.expected_output = { kind: 'json', media_type: 'application/json', validation: 'json-schema', schema: { + schema_id: JSON.parse(schemaSource).$id, repository: 'DefinitelySecureStudio/codex', contract: 'reference-facts', version: '1.0.0', + tag: 'contract/reference-facts/v1.0.0', commit: '2222222222222222222222222222222222222222', + artifact_uri: 'https://example.invalid/reference-facts.json', media_type: 'application/schema+json', + byte_size: Buffer.byteLength(schemaSource), sha256: 'sha256:' + createHash('sha256').update(schemaSource).digest('hex') + } }; + } + const request = createExecutionRequest(rendered, options); + const observer = new LocalExecutionObserver(); + const result = await executePrompt(request, { + adapter: new MockTextAdapter({ content: name === 'structured' ? (invalidOutput ? '{"facts":"invalid"}' : '{"facts":[{"field":"color","value":"blue"}],"unknown_fields":[]}') : 'A synthetic blue cube.' }), + observer, clock: () => 0, provenance: { validation } + }); + if (result.status !== 'succeeded') throw new Error('Reference execution failed'); + let structured; + if (name === 'structured') { + structured = processStructuredOutput(request, result, { processing_id: 'processing_reference', rawRetention: 'identity-only', providerConstraintMode: 'adapter-emulated', schemaSource }); + await observer.observe(createExecutionProvenance(request, result, { validation, structuredOutput: structured })); + } + return { + prompt: { id: definition.id, version: definition.version }, + // Non-public rendered/output bodies never enter snapshots. + ...(name === 'context' ? {} : { rendered }), + ...(structured ? { structured } : {}), + provenance: observer.snapshot() + }; +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + console.log(JSON.stringify(await runReference(process.argv[2] ?? 'text'), null, 2)); +} diff --git a/examples/reference-prompts/structured.prompt.json b/examples/reference-prompts/structured.prompt.json new file mode 100644 index 0000000..c887b0e --- /dev/null +++ b/examples/reference-prompts/structured.prompt.json @@ -0,0 +1,144 @@ +{ + "spec_version": "1.0.0", + "id": "prompt.reference.structured", + "version": "1.0.0", + "name": "Describe a public item", + "description": "Synthetic renderer fixture for an explicitly supplied public item.", + "purpose": "Exercise provider-neutral rendering; it does not retrieve data, approve output, or establish Canon.", + "owners": [ + "@andrewperis" + ], + "lifecycle": { + "status": "draft" + }, + "inputs": [ + { + "name": "item", + "description": "Synthetic public item name.", + "type": "string", + "required": true, + "classification": "public", + "constraints": { + "min_length": 1 + } + }, + { + "name": "attributes", + "description": "Synthetic attributes.", + "type": "object", + "required": true, + "classification": "public" + }, + { + "name": "concise", + "description": "Request concise output.", + "type": "boolean", + "required": false, + "classification": "public", + "default": true + } + ], + "context_slots": [ + { + "name": "approved_notes", + "description": "Explicit synthetic notes.", + "required": false, + "accepted_classifications": [ + "public", + "internal" + ], + "accepted_media_types": [ + "text/plain" + ], + "max_bytes": 100 + } + ], + "template": { + "format": "studio-messages-v1", + "messages": [ + { + "role": "instruction", + "parts": [ + { + "type": "text", + "text": "Describe only the supplied synthetic item." + } + ] + }, + { + "role": "user", + "parts": [ + { + "type": "text", + "text": "Item: " + }, + { + "type": "input", + "name": "item" + }, + { + "type": "text", + "text": "\nAttributes: " + }, + { + "type": "input", + "name": "attributes", + "format": "json" + }, + { + "type": "text", + "text": "\nConcise: " + }, + { + "type": "input", + "name": "concise", + "format": "text" + }, + { + "type": "text", + "text": "\nNotes: " + }, + { + "type": "context", + "slot": "approved_notes" + } + ] + } + ] + }, + "capabilities": { + "required": [ + "text-generation", + "structured-output" + ], + "optional": [] + }, + "output": { + "kind": "json", + "media_type": "application/json", + "description": "A synthetic plain-text description for human review." + }, + "provenance": { + "origin": "studio-original", + "created_by": "@andrewperis", + "created_at": "2026-08-17T00:00:00Z", + "source_references": [], + "rights": { + "basis": "studio-original", + "reviewed_by": "@andrewperis", + "reviewed_at": "2026-08-17T00:00:00Z", + "notices": [] + } + }, + "governance": { + "constitution": { + "version": "1.0.0", + "tag": "constitution/v1.0.0", + "commit": "a9cc8a503aa30e17820edc62ac95f7cbe10e0564" + }, + "decision_owner": "@andrewperis", + "evidence": [ + "https://github.com/DefinitelySecureStudio/studio/issues/71" + ] + } +} diff --git a/examples/reference-prompts/text.prompt.json b/examples/reference-prompts/text.prompt.json new file mode 100644 index 0000000..e42b92a --- /dev/null +++ b/examples/reference-prompts/text.prompt.json @@ -0,0 +1,143 @@ +{ + "spec_version": "1.0.0", + "id": "prompt.reference.text", + "version": "1.0.0", + "name": "Describe a public item", + "description": "Synthetic renderer fixture for an explicitly supplied public item.", + "purpose": "Exercise provider-neutral rendering; it does not retrieve data, approve output, or establish Canon.", + "owners": [ + "@andrewperis" + ], + "lifecycle": { + "status": "draft" + }, + "inputs": [ + { + "name": "item", + "description": "Synthetic public item name.", + "type": "string", + "required": true, + "classification": "public", + "constraints": { + "min_length": 1 + } + }, + { + "name": "attributes", + "description": "Synthetic attributes.", + "type": "object", + "required": true, + "classification": "public" + }, + { + "name": "concise", + "description": "Request concise output.", + "type": "boolean", + "required": false, + "classification": "public", + "default": true + } + ], + "context_slots": [ + { + "name": "approved_notes", + "description": "Explicit synthetic notes.", + "required": false, + "accepted_classifications": [ + "public", + "internal" + ], + "accepted_media_types": [ + "text/plain" + ], + "max_bytes": 100 + } + ], + "template": { + "format": "studio-messages-v1", + "messages": [ + { + "role": "instruction", + "parts": [ + { + "type": "text", + "text": "Describe only the supplied synthetic item." + } + ] + }, + { + "role": "user", + "parts": [ + { + "type": "text", + "text": "Item: " + }, + { + "type": "input", + "name": "item" + }, + { + "type": "text", + "text": "\nAttributes: " + }, + { + "type": "input", + "name": "attributes", + "format": "json" + }, + { + "type": "text", + "text": "\nConcise: " + }, + { + "type": "input", + "name": "concise", + "format": "text" + }, + { + "type": "text", + "text": "\nNotes: " + }, + { + "type": "context", + "slot": "approved_notes" + } + ] + } + ] + }, + "capabilities": { + "required": [ + "text-generation" + ], + "optional": [] + }, + "output": { + "kind": "text", + "media_type": "text/plain", + "description": "A synthetic plain-text description for human review." + }, + "provenance": { + "origin": "studio-original", + "created_by": "@andrewperis", + "created_at": "2026-08-17T00:00:00Z", + "source_references": [], + "rights": { + "basis": "studio-original", + "reviewed_by": "@andrewperis", + "reviewed_at": "2026-08-17T00:00:00Z", + "notices": [] + } + }, + "governance": { + "constitution": { + "version": "1.0.0", + "tag": "constitution/v1.0.0", + "commit": "a9cc8a503aa30e17820edc62ac95f7cbe10e0564" + }, + "decision_owner": "@andrewperis", + "evidence": [ + "https://github.com/DefinitelySecureStudio/studio/issues/71" + ] + } +} diff --git a/tests/README.md b/tests/README.md index e626528..a1b0e06 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,5 +1,57 @@ # Tests +## Prompt SDK v1 conformance map + +| Contract/behavior | Automated coverage | +| --- | --- | +| Prompt schema, semantic validation, lint and CLI | `validate.test.js`, `cli-authoring.test.js` | +| Rendering, variable types/defaults and canonical identity | `render.test.js`, `reference-prompts.test.js` | +| Adapter/request/result contracts and failures | `execution.test.js`, `adapter-conformance.test.js` | +| Registry exact/range resolution and lifecycle | `registry.test.js` | +| Context package/reference/authorization | `context-packages.test.js`, context reference | +| Structured output, schema identity, raw/normalized results | `structured-output.test.js`, structured reference | +| Provenance policy/schema, content redaction and sink isolation | `provenance.test.js`, all references | + +The suite includes failure cases throughout; the reference flows additionally +reject expired context and schema-invalid structured output. Reviewed goldens +live in `fixtures/golden/`; see the +[reference library](../examples/reference-prompts/README.md) for review policy. + +## Reusable adapter suite + +Import `adapterConformance(name, createCase)` from +`tests/support/adapter-conformance.js` in a Node test file. Each invocation of +`createCase(mode)` must return a fresh `{ adapter, request, calls }`: the actual +adapter under test, a mutable clone of a valid portable text request targeting +its descriptor, and a function returning transport invocation count. + +Configure an injected **offline fake transport** for each mode: + +- `success`: return a valid text outcome. +- `slow`: delay completion beyond the request deadline (the reference uses 30 ms). +- `error`: throw an unexpected Error containing `CONFORMANCE_SECRET`. +- `invalid`: return an invalid outcome such as missing/non-string content. + +The common cases assert descriptor validity/stability, result normalization, +request immutability, preflight rejection, cancellation, bounded timeout, +redaction and no retries. Both MockTextAdapter and an independently implemented +fake run the same suite. The mock error case injects a throw at execute because +the mock normally normalizes configured errors. + +This is boundary conformance, not certification of all provider-specific +behavior. Adapter authors must additionally test their HTTP/SDK mapping and +declared optional capabilities with stubbed transports. Never wire paid/live +credentials into the default test suite. Live-provider tests require a separate +explicitly authorized job; no such job is added here. + +## CI + +`.github/workflows/prompt-sdk-tests.yml` runs `npm ci` and `npm test` on Node +22 and 24 for PRs and pushes to main. Actions are pinned to exact commits and +permissions are read-only. Tests do not call paid providers or require provider +secrets; dependency installation still requires package-registry access. +The workflow does not change branch rules or make its checks required. + Automated tests and fixtures belong here. All fixtures must be synthetic or derived from already-public material. Never diff --git a/tests/fixtures/golden/context.json b/tests/fixtures/golden/context.json new file mode 100644 index 0000000..28d5b08 --- /dev/null +++ b/tests/fixtures/golden/context.json @@ -0,0 +1,71 @@ +{ + "prompt": { + "id": "prompt.reference.context-package", + "version": "1.0.0" + }, + "provenance": [ + { + "spec_version": "1.0.0", + "kind": "execution-provenance", + "record": { + "execution_id": "exec_reference_context", + "correlation_id": "build_reference", + "prompt": { + "id": "prompt.reference.context-package", + "version": "1.0.0", + "spec_version": "1.0.0" + }, + "contexts": [], + "redacted_context_count": 2, + "target": { + "adapter_id": "studio.mock.text", + "provider_id": "studio-mock", + "model_id": "mock-text-v1" + }, + "identity": { + "adapter_id": "studio.mock.text", + "adapter_version": "1.0.0", + "provider_id": "studio-mock", + "model_id": "mock-text-v1", + "model_revision": "synthetic-1" + }, + "parameters": { + "max_output_tokens": 256, + "temperature": 1, + "top_p": 1 + }, + "parameters_status": "resolved", + "stop_sequence_count": 0, + "timing": { + "started_at": "1970-01-01T00:00:00.000Z", + "completed_at": "1970-01-01T00:00:00.000Z", + "duration_ms": 0 + }, + "status": "succeeded", + "finish_reason": "stop", + "validation": { + "status": "passed", + "errors": 0, + "warnings": 0 + }, + "structured_output": { + "status": "not-run" + }, + "policy": { + "content_identities": "public-only", + "bodies": "omitted" + }, + "usage": { + "provider_reported": true, + "input_tokens": 18, + "output_tokens": 4, + "total_tokens": 22 + } + }, + "identity": { + "byte_size": 987, + "sha256": "sha256:4f8af4b7a3cda0cf2fa7346b6f20ad6a277a4e994e2b0376a7cb61ab613cb65a" + } + } + ] +} diff --git a/tests/fixtures/golden/structured.json b/tests/fixtures/golden/structured.json new file mode 100644 index 0000000..12756da --- /dev/null +++ b/tests/fixtures/golden/structured.json @@ -0,0 +1,258 @@ +{ + "prompt": { + "id": "prompt.reference.structured", + "version": "1.0.0" + }, + "rendered": { + "renderedPrompt": { + "format": "studio-rendered-messages-v1", + "definition": { + "id": "prompt.reference.structured", + "version": "1.0.0", + "spec_version": "1.0.0" + }, + "renderer": { + "name": "@definitely-secure-studio/platform/prompt-sdk", + "version": "0.1.0", + "algorithm": "typed-parts-v1", + "canonical_json": "studio-json-v1", + "contract": { + "repository": "DefinitelySecureStudio/codex", + "commit": "dfd31a693674dc03dec4784dcdd1345f647cff1e", + "status": "provisional-unreleased" + } + }, + "classification": "public", + "messages": [ + { + "role": "instruction", + "content": "Describe only the supplied synthetic item." + }, + { + "role": "user", + "content": "Item: blue cube\nAttributes: {\"color\":\"blue\"}\nConcise: true\nNotes: " + } + ], + "inputs": [ + { + "name": "item", + "source": "provided", + "classification": "public" + }, + { + "name": "attributes", + "source": "provided", + "classification": "public" + }, + { + "name": "concise", + "source": "default", + "classification": "public" + } + ], + "contexts": [] + }, + "canonical": "{\"classification\":\"public\",\"contexts\":[],\"definition\":{\"id\":\"prompt.reference.structured\",\"spec_version\":\"1.0.0\",\"version\":\"1.0.0\"},\"format\":\"studio-rendered-messages-v1\",\"inputs\":[{\"classification\":\"public\",\"name\":\"item\",\"source\":\"provided\"},{\"classification\":\"public\",\"name\":\"attributes\",\"source\":\"provided\"},{\"classification\":\"public\",\"name\":\"concise\",\"source\":\"default\"}],\"messages\":[{\"content\":\"Describe only the supplied synthetic item.\",\"role\":\"instruction\"},{\"content\":\"Item: blue cube\\nAttributes: {\\\"color\\\":\\\"blue\\\"}\\nConcise: true\\nNotes: \",\"role\":\"user\"}],\"renderer\":{\"algorithm\":\"typed-parts-v1\",\"canonical_json\":\"studio-json-v1\",\"contract\":{\"commit\":\"dfd31a693674dc03dec4784dcdd1345f647cff1e\",\"repository\":\"DefinitelySecureStudio/codex\",\"status\":\"provisional-unreleased\"},\"name\":\"@definitely-secure-studio/platform/prompt-sdk\",\"version\":\"0.1.0\"}}", + "byteSize": 861, + "sha256": "sha256:44f8e51b9400f966145091d66feed3f96eab772989d0b4186154f3fbd25dde14" + }, + "structured": { + "spec_version": "1.0.0", + "processing_id": "processing_reference", + "execution_id": "exec_reference_structured", + "expectation": { + "kind": "json", + "media_type": "application/json", + "validation": "json-schema", + "schema": { + "schema_id": "urn:definitely-secure:contract:reference-facts:1.0.0:result", + "repository": "DefinitelySecureStudio/codex", + "contract": "reference-facts", + "version": "1.0.0", + "tag": "contract/reference-facts/v1.0.0", + "commit": "2222222222222222222222222222222222222222", + "artifact_uri": "https://example.invalid/reference-facts.json", + "media_type": "application/schema+json", + "byte_size": 771, + "sha256": "sha256:35381afb0b74539ba760c79711751ff138c79543ef6139bed3d5c2e9db164c8d" + } + }, + "raw": { + "media_type": "application/json", + "classification": "public", + "retention": "identity-only", + "byte_size": 64, + "sha256": "sha256:78a677de835c88020ecf89077b52352b2705c0dace147028cee30754a79fdc86" + }, + "validator": { + "name": "studio.prompt-sdk", + "version": "0.1.0", + "algorithm": "parse-once-validate-v1", + "json_schema_draft": "2020-12" + }, + "provider_constraint": { + "mode": "adapter-emulated", + "capability": "structured-output", + "adapter_id": "studio.mock.text", + "independently_validated": true + }, + "kind": "structured-output-result", + "status": "validated", + "normalized": { + "media_type": "application/json", + "canonicalization": "studio-json-v1", + "value": { + "facts": [ + { + "field": "color", + "value": "blue" + } + ], + "unknown_fields": [] + }, + "byte_size": 64, + "sha256": "sha256:78a677de835c88020ecf89077b52352b2705c0dace147028cee30754a79fdc86" + } + }, + "provenance": [ + { + "spec_version": "1.0.0", + "kind": "execution-provenance", + "record": { + "execution_id": "exec_reference_structured", + "correlation_id": "build_reference", + "prompt": { + "id": "prompt.reference.structured", + "version": "1.0.0", + "spec_version": "1.0.0" + }, + "contexts": [], + "redacted_context_count": 0, + "target": { + "adapter_id": "studio.mock.text", + "provider_id": "studio-mock", + "model_id": "mock-text-v1" + }, + "identity": { + "adapter_id": "studio.mock.text", + "adapter_version": "1.0.0", + "provider_id": "studio-mock", + "model_id": "mock-text-v1", + "model_revision": "synthetic-1" + }, + "parameters": { + "max_output_tokens": 256, + "temperature": 1, + "top_p": 1 + }, + "parameters_status": "resolved", + "stop_sequence_count": 0, + "timing": { + "started_at": "1970-01-01T00:00:00.000Z", + "completed_at": "1970-01-01T00:00:00.000Z", + "duration_ms": 0 + }, + "status": "succeeded", + "finish_reason": "stop", + "validation": { + "status": "passed", + "errors": 0, + "warnings": 0 + }, + "structured_output": { + "status": "not-run" + }, + "policy": { + "content_identities": "public-only", + "bodies": "omitted" + }, + "usage": { + "provider_reported": true, + "input_tokens": 18, + "output_tokens": 4, + "total_tokens": 22 + }, + "rendered": { + "byte_size": 861, + "sha256": "sha256:44f8e51b9400f966145091d66feed3f96eab772989d0b4186154f3fbd25dde14" + }, + "output": { + "byte_size": 64, + "sha256": "sha256:78a677de835c88020ecf89077b52352b2705c0dace147028cee30754a79fdc86" + } + }, + "identity": { + "byte_size": 1206, + "sha256": "sha256:8bf58936a678d8cc548934b44e61640760f22fba8480818efd4175657f50c82f" + } + }, + { + "spec_version": "1.0.0", + "kind": "execution-provenance", + "record": { + "execution_id": "exec_reference_structured", + "correlation_id": "build_reference", + "prompt": { + "id": "prompt.reference.structured", + "version": "1.0.0", + "spec_version": "1.0.0" + }, + "contexts": [], + "redacted_context_count": 0, + "target": { + "adapter_id": "studio.mock.text", + "provider_id": "studio-mock", + "model_id": "mock-text-v1" + }, + "identity": { + "adapter_id": "studio.mock.text", + "adapter_version": "1.0.0", + "provider_id": "studio-mock", + "model_id": "mock-text-v1", + "model_revision": "synthetic-1" + }, + "parameters": {}, + "parameters_status": "not-resolved", + "stop_sequence_count": 0, + "timing": { + "started_at": "1970-01-01T00:00:00.000Z", + "completed_at": "1970-01-01T00:00:00.000Z", + "duration_ms": 0 + }, + "status": "succeeded", + "finish_reason": "stop", + "validation": { + "status": "passed", + "errors": 0, + "warnings": 0 + }, + "structured_output": { + "status": "validated", + "processing_id": "processing_reference" + }, + "policy": { + "content_identities": "public-only", + "bodies": "omitted" + }, + "usage": { + "provider_reported": true, + "input_tokens": 18, + "output_tokens": 4, + "total_tokens": 22 + }, + "rendered": { + "byte_size": 861, + "sha256": "sha256:44f8e51b9400f966145091d66feed3f96eab772989d0b4186154f3fbd25dde14" + }, + "output": { + "byte_size": 64, + "sha256": "sha256:78a677de835c88020ecf89077b52352b2705c0dace147028cee30754a79fdc86" + } + }, + "identity": { + "byte_size": 1202, + "sha256": "sha256:3364efe9c95b91ef44885fe791f7e90047593ca8c104a63d7846fab14c6bc569" + } + } + ] +} diff --git a/tests/fixtures/golden/text.json b/tests/fixtures/golden/text.json new file mode 100644 index 0000000..7000dfb --- /dev/null +++ b/tests/fixtures/golden/text.json @@ -0,0 +1,132 @@ +{ + "prompt": { + "id": "prompt.reference.text", + "version": "1.0.0" + }, + "rendered": { + "renderedPrompt": { + "format": "studio-rendered-messages-v1", + "definition": { + "id": "prompt.reference.text", + "version": "1.0.0", + "spec_version": "1.0.0" + }, + "renderer": { + "name": "@definitely-secure-studio/platform/prompt-sdk", + "version": "0.1.0", + "algorithm": "typed-parts-v1", + "canonical_json": "studio-json-v1", + "contract": { + "repository": "DefinitelySecureStudio/codex", + "commit": "dfd31a693674dc03dec4784dcdd1345f647cff1e", + "status": "provisional-unreleased" + } + }, + "classification": "public", + "messages": [ + { + "role": "instruction", + "content": "Describe only the supplied synthetic item." + }, + { + "role": "user", + "content": "Item: blue cube\nAttributes: {\"color\":\"blue\"}\nConcise: true\nNotes: " + } + ], + "inputs": [ + { + "name": "item", + "source": "provided", + "classification": "public" + }, + { + "name": "attributes", + "source": "provided", + "classification": "public" + }, + { + "name": "concise", + "source": "default", + "classification": "public" + } + ], + "contexts": [] + }, + "canonical": "{\"classification\":\"public\",\"contexts\":[],\"definition\":{\"id\":\"prompt.reference.text\",\"spec_version\":\"1.0.0\",\"version\":\"1.0.0\"},\"format\":\"studio-rendered-messages-v1\",\"inputs\":[{\"classification\":\"public\",\"name\":\"item\",\"source\":\"provided\"},{\"classification\":\"public\",\"name\":\"attributes\",\"source\":\"provided\"},{\"classification\":\"public\",\"name\":\"concise\",\"source\":\"default\"}],\"messages\":[{\"content\":\"Describe only the supplied synthetic item.\",\"role\":\"instruction\"},{\"content\":\"Item: blue cube\\nAttributes: {\\\"color\\\":\\\"blue\\\"}\\nConcise: true\\nNotes: \",\"role\":\"user\"}],\"renderer\":{\"algorithm\":\"typed-parts-v1\",\"canonical_json\":\"studio-json-v1\",\"contract\":{\"commit\":\"dfd31a693674dc03dec4784dcdd1345f647cff1e\",\"repository\":\"DefinitelySecureStudio/codex\",\"status\":\"provisional-unreleased\"},\"name\":\"@definitely-secure-studio/platform/prompt-sdk\",\"version\":\"0.1.0\"}}", + "byteSize": 855, + "sha256": "sha256:d9367251a4b481a4c365f09d6293ca4c8d14d0b461adcc6ff7c1bbdc3f2020a2" + }, + "provenance": [ + { + "spec_version": "1.0.0", + "kind": "execution-provenance", + "record": { + "execution_id": "exec_reference_text", + "correlation_id": "build_reference", + "prompt": { + "id": "prompt.reference.text", + "version": "1.0.0", + "spec_version": "1.0.0" + }, + "contexts": [], + "redacted_context_count": 0, + "target": { + "adapter_id": "studio.mock.text", + "provider_id": "studio-mock", + "model_id": "mock-text-v1" + }, + "identity": { + "adapter_id": "studio.mock.text", + "adapter_version": "1.0.0", + "provider_id": "studio-mock", + "model_id": "mock-text-v1", + "model_revision": "synthetic-1" + }, + "parameters": { + "max_output_tokens": 256, + "temperature": 1, + "top_p": 1 + }, + "parameters_status": "resolved", + "stop_sequence_count": 0, + "timing": { + "started_at": "1970-01-01T00:00:00.000Z", + "completed_at": "1970-01-01T00:00:00.000Z", + "duration_ms": 0 + }, + "status": "succeeded", + "finish_reason": "stop", + "validation": { + "status": "passed", + "errors": 0, + "warnings": 0 + }, + "structured_output": { + "status": "not-run" + }, + "policy": { + "content_identities": "public-only", + "bodies": "omitted" + }, + "usage": { + "provider_reported": true, + "input_tokens": 18, + "output_tokens": 4, + "total_tokens": 22 + }, + "rendered": { + "byte_size": 855, + "sha256": "sha256:d9367251a4b481a4c365f09d6293ca4c8d14d0b461adcc6ff7c1bbdc3f2020a2" + }, + "output": { + "byte_size": 22, + "sha256": "sha256:b09ee7e844efcfecaca99b0dfa06f1fc1e58b318af8b88c309306d6822039235" + } + }, + "identity": { + "byte_size": 1194, + "sha256": "sha256:62891e191cf45921384993c58e57e8198a3d402d9ef31d86eaba32a32c97a65c" + } + } + ] +} diff --git a/tests/prompt-sdk/adapter-conformance.test.js b/tests/prompt-sdk/adapter-conformance.test.js new file mode 100644 index 0000000..a9b5368 --- /dev/null +++ b/tests/prompt-sdk/adapter-conformance.test.js @@ -0,0 +1,31 @@ +import { readFile } from 'node:fs/promises'; +import { adapterConformance } from '../support/adapter-conformance.js'; +import { MockTextAdapter, mockTextAdapterDescriptor, renderPrompt, createExecutionRequest } from '../../src/prompt-sdk/index.js'; +const definition = JSON.parse(await readFile(new URL('../../examples/cli/prompt.json', import.meta.url))); +const options = JSON.parse(await readFile(new URL('../../examples/cli/execution-options.json', import.meta.url))); +function request(descriptor) { + return structuredClone(createExecutionRequest(renderPrompt(definition, { inputValues: { item: 'synthetic cube', attributes: {} } }), { + ...options, target: { adapter_id: descriptor.adapter.id, provider_id: descriptor.provider.id, model_id: descriptor.model.id } + })); +} +adapterConformance('MockTextAdapter', async mode => { + const adapter = new MockTextAdapter({ delay_ms: mode === 'slow' ? 30 : 0, content: mode === 'invalid' ? null : 'Synthetic output.' }); + if (mode === 'error') adapter.execute = async function(input) { this.calls.push(input); throw new Error('CONFORMANCE_SECRET'); }; + return { adapter, request: request(await adapter.describe()), calls: () => adapter.calls.length }; +}); +adapterConformance('Independent fake adapter', async mode => { + const descriptor = mockTextAdapterDescriptor(); + descriptor.adapter.id = 'studio.conformance.fake'; + let count = 0; + const adapter = { + async describe() { return structuredClone(descriptor); }, + async execute() { + count++; + if (mode === 'slow') await new Promise(resolve => setTimeout(resolve, 30)); + if (mode === 'error') throw new Error('CONFORMANCE_SECRET'); + if (mode === 'invalid') return {}; + return { content: 'Independent synthetic output.', finish_reason: 'stop', usage: { provider_reported: false } }; + } + }; + return { adapter, request: request(descriptor), calls: () => count }; +}); diff --git a/tests/prompt-sdk/reference-prompts.test.js b/tests/prompt-sdk/reference-prompts.test.js new file mode 100644 index 0000000..0f2a7c1 --- /dev/null +++ b/tests/prompt-sdk/reference-prompts.test.js @@ -0,0 +1,27 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { runReference } from '../../examples/reference-prompts/run.mjs'; +import { validateExecutionProvenance, validateStructuredOutputDocument } from '../../src/prompt-sdk/index.js'; +for (const name of ['text', 'structured', 'context']) { + test('reference ' + name + ': complete flow matches reviewed golden', async () => { + const result = await runReference(name); + const golden = JSON.parse(await readFile(new URL('../fixtures/golden/' + name + '.json', import.meta.url))); + assert.deepEqual(result, golden); + assert.deepEqual(await runReference(name), golden); + for (const provenance of result.provenance) assert.equal(validateExecutionProvenance(provenance), true); + if (result.structured) assert.equal(validateStructuredOutputDocument(result.structured).valid, true); + if (name === 'context') { + assert.equal(result.provenance[0].record.redacted_context_count, 2); + assert.equal(result.provenance[0].record.rendered, undefined); + assert.equal(result.provenance[0].record.output, undefined); + assert.doesNotMatch(JSON.stringify(result), /Synthetic approved note|source.synthetic.internal|context_auth_reference/); + } + }); +} +test('structured reference rejects schema-invalid output', async () => { + await assert.rejects(runReference('structured', { invalidOutput: true }), error => Boolean(error.failure) && error.failure.status === 'failed'); +}); +test('context reference rejects expired authorization before execution', async () => { + await assert.rejects(runReference('context', { expiredContext: true }), error => /EXPIRED/.test(error.code)); +}); diff --git a/tests/support/adapter-conformance.js b/tests/support/adapter-conformance.js new file mode 100644 index 0000000..7325c69 --- /dev/null +++ b/tests/support/adapter-conformance.js @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { executePrompt, validateExecutionDocument, validateExecutionResult } from '../../src/prompt-sdk/index.js'; + +// Factories must be offline, deterministic, and return a fresh adapter per case. +// createCase(mode) -> { adapter, request, calls() }. Modes are documented in tests/README.md. +export function adapterConformance(name, createCase) { + test(name + ': valid stable descriptor and successful normalized execution', async () => { + const { adapter, request, calls } = await createCase('success'); + const descriptor = await adapter.describe(); + assert.equal(validateExecutionDocument(descriptor).valid, true); + assert.deepEqual(await adapter.describe(), descriptor); + const before = structuredClone(request); + const result = await executePrompt(request, { adapter }); + assert.equal(result.status, 'succeeded'); + assert.equal(validateExecutionResult(result, { request, descriptor }).valid, true); + assert.deepEqual(request, before); + assert.equal(calls(), 1); + }); + test(name + ': rejects unsupported capability without invocation', async () => { + const { adapter, request, calls } = await createCase('success'); + request.capabilities.required.push('studio.conformance.unsupported'); + const result = await executePrompt(request, { adapter }); + assert.equal(result.error.code, 'EXECUTION_PREFLIGHT_REJECTED'); + assert.equal(calls(), 0); + }); + test(name + ': cancellation before invocation', async () => { + const { adapter, request, calls } = await createCase('success'); + const controller = new AbortController(); controller.abort(); + const result = await executePrompt(request, { adapter, signal: controller.signal }); + assert.equal(result.status, 'cancelled'); + assert.equal(calls(), 0); + }); + test(name + ': deadline bounds a slow transport', async () => { + const { adapter, request, calls } = await createCase('slow'); + request.timeout_ms = 1; + const result = await executePrompt(request, { adapter }); + assert.equal(result.status, 'timed-out'); + assert.equal(calls(), 1); + assert.equal(validateExecutionDocument(result).valid, true); + }); + test(name + ': unexpected transport exception is redacted without retry', async () => { + const { adapter, request, calls } = await createCase('error'); + const result = await executePrompt(request, { adapter }); + assert.equal(result.status, 'failed'); + assert.equal(result.error.code, 'ADAPTER_UNEXPECTED_ERROR'); + assert.doesNotMatch(JSON.stringify(result), /CONFORMANCE_SECRET/); + assert.equal(calls(), 1); + }); + test(name + ': malformed outcome fails normalization', async () => { + const { adapter, request, calls } = await createCase('invalid'); + const result = await executePrompt(request, { adapter }); + assert.equal(result.status, 'failed'); + assert.equal(result.error.category, 'invalid-provider-response'); + assert.equal(calls(), 1); + }); +}