diff --git a/cli/commands/push/command.test.ts b/cli/commands/push/command.test.ts index 364c2e581e..35eac636f9 100644 --- a/cli/commands/push/command.test.ts +++ b/cli/commands/push/command.test.ts @@ -3789,11 +3789,12 @@ describe("push divergence guard", () => { assertEquals((error as Error & { slug?: string }).slug, "push-conflict"); assertStringIncludes(error.message, '"app.ts"'); assertEquals(fileListCalls, 3); - assertEquals(putPaths, [ + // Directory iteration order varies by filesystem; verify every attempted write. + assertEquals(putPaths.toSorted(), [ "/projects/my-project/files/app.ts", - "/projects/my-project/files/second.ts", "/projects/my-project/files/app.ts", "/projects/my-project/files/second.ts", + "/projects/my-project/files/second.ts", ]); assertEquals(await readPushReceipt(projectDir), null); assertEquals( diff --git a/deno.json b/deno.json index 743b0cf739..4fbfdc971f 100644 --- a/deno.json +++ b/deno.json @@ -559,7 +559,7 @@ "docs:coverage": "deno run --allow-read scripts/docs/docs-coverage.ts", "docs:copy": "rm -rf ../../docs/docs/code/api-reference && cp -r docs/api-reference/ ../../docs/docs/code/api-reference/", "docs:snippets:check": "deno run --allow-read --allow-write --allow-run --allow-env scripts/docs/check-guide-snippets.ts", - "docs:validate": "deno run --allow-read --allow-run=git scripts/docs/validate-tracked-docs.ts && deno run --allow-read scripts/docs/validate-api-reference.ts && deno run --allow-read scripts/docs/validate-guides.ts && deno run --config=scripts/test.deno.json --allow-read scripts/docs/validate-public-docs.ts && deno test --config=scripts/test.deno.json --no-check --allow-read scripts/docs/docs-coverage.test.ts && DENO_TESTING=1 deno test --no-check --allow-read --allow-env=DENO_TESTING tests/docs/guide-contracts.test.ts tests/docs/guide-content.test.ts && DENO_TESTING=1 deno test --no-check --allow-all tests/docs/guide-examples.test.ts tests/docs/guide-code-examples.test.ts && deno run -A scripts/lint/check-doc-links.ts && deno task docs:snippets:check", + "docs:validate": "deno run --allow-read --allow-run=git scripts/docs/validate-tracked-docs.ts && deno run --allow-read scripts/docs/validate-api-reference.ts && deno run --allow-read scripts/docs/validate-guides.ts && deno run --config=scripts/test.deno.json --allow-read scripts/docs/validate-public-docs.ts && deno test --config=scripts/test.deno.json --no-check --allow-read scripts/docs/docs-coverage.test.ts && deno task test:file tests/integration/docs/provider-client-examples.test.ts && DENO_TESTING=1 deno test --no-check --allow-read --allow-env=DENO_TESTING tests/docs/guide-contracts.test.ts tests/docs/guide-content.test.ts && DENO_TESTING=1 deno test --no-check --allow-all tests/docs/guide-examples.test.ts tests/docs/guide-code-examples.test.ts && deno run -A scripts/lint/check-doc-links.ts && deno task docs:snippets:check", "docs:verify-npm": "node scripts/docs/verify-npm-exports.mjs && node scripts/docs/verify-npm-node.mjs", "docs:check-links": "deno run -A scripts/lint/check-doc-links.ts", "lint:ban-zod": "deno run --allow-read scripts/lint/ban-zod-imports.ts", diff --git a/docs/guides/providers.md b/docs/guides/providers.md index fd8db8ea36..d652a55aff 100644 --- a/docs/guides/providers.md +++ b/docs/guides/providers.md @@ -61,6 +61,96 @@ export default agent({ `VERYFRONT_DEFAULT_MODEL` can select another gateway default for omitted models and `model: "auto"`. It is optional. +### Call the AI Gateway from other clients + +Code outside Veryfront can use the AI Gateway through the official OpenAI and +Anthropic clients. Create a project API key with the Write permission in Studio +under **Settings > API Keys**. Inference requests are writes, so a Read-only key +is refused. The key starts with `vf_` and is bound to its project. +The examples read it from `VERYFRONT_API_KEY`, a name for your own code only. +The Veryfront CLI and SDK read `VERYFRONT_API_TOKEN`, described above. + +Set the project API key in your shell before running the examples: + +```bash +export VERYFRONT_API_KEY="" +``` + +Name the model as `/`. List the models your key can use: + +```bash +curl https://api.veryfront.com/ai/models \ + -H "Authorization: Bearer $VERYFRONT_API_KEY" +``` + +Install the clients for the examples you want to run: + +```bash +npm install openai @anthropic-ai/sdk +``` + +Use a model that supports Chat Completions with the OpenAI client at +`https://api.veryfront.com/ai/v1`. Models supporting other operations use their +corresponding endpoints. Model identifiers can include vendors this SDK has no +built-in entry for: + +```ts +import OpenAI from "openai"; + +const projectApiKey = process.env.VERYFRONT_API_KEY; +if (!projectApiKey) { + throw new Error("Set VERYFRONT_API_KEY before running this example."); +} + +const client = new OpenAI({ + baseURL: "https://api.veryfront.com/ai/v1", + apiKey: projectApiKey, +}); + +const completion = await client.chat.completions.create({ + model: "mistral/mistral-small-2503", + messages: [{ role: "user", content: "Say hello." }], +}); +``` + +```bash +curl https://api.veryfront.com/ai/v1/chat/completions \ + -H "Authorization: Bearer $VERYFRONT_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"mistral/mistral-small-2503","messages":[{"role":"user","content":"Say hello."}]}' +``` + +Anthropic models also speak the Anthropic Messages protocol. The Anthropic +client adds `/v1/messages` to its base URL: + +```ts +import Anthropic from "@anthropic-ai/sdk"; + +const client = new Anthropic({ + baseURL: "https://api.veryfront.com/ai", + apiKey: null, + authToken: process.env.VERYFRONT_API_KEY, +}); + +const message = await client.messages.create({ + model: "anthropic/claude-sonnet-4-6", + max_tokens: 1024, + messages: [{ role: "user", content: "Say hello." }], +}); +``` + +```bash +curl https://api.veryfront.com/ai/v1/messages \ + -H "Authorization: Bearer $VERYFRONT_API_KEY" \ + -H "anthropic-version: 2023-06-01" \ + -H "Content-Type: application/json" \ + -d '{"model":"anthropic/claude-sonnet-4-6","max_tokens":1024,"messages":[{"role":"user","content":"Say hello."}]}' +``` + +Inside a Veryfront agent, use a `veryfront-cloud//` string +instead. The SDK routes through Veryfront's project runtime gateway for the +selected provider and model. + ## Runtime conventions (recommended) For most projects, omit `model` to use the default for your inference path. Set diff --git a/tests/docs/guide-code-examples.test.ts b/tests/docs/guide-code-examples.test.ts index 32b0eca07a..608afef7e1 100644 --- a/tests/docs/guide-code-examples.test.ts +++ b/tests/docs/guide-code-examples.test.ts @@ -78,6 +78,7 @@ import { parsePushArgs } from "../../cli/commands/push/command.ts"; import { parseCliArgs } from "../../cli/shared/args.ts"; import { AUTH_PRESETS } from "../../cli/scaffold/engine.ts"; import { getTemplate } from "../../templates/index.ts"; +import { getVeryfrontCloudGatewayBaseUrl } from "#veryfront/provider/veryfront-cloud/shared.ts"; const EXISTING_GUIDE_EXAMPLE_SUITE = [ "agents.md", @@ -206,6 +207,59 @@ describe("Guide code example coverage", () => { }); }); +describe("Guide: providers.md", () => { + const api = "https://api.veryfront.com"; + + async function gatewayClientSection(): Promise { + const guide = await readGuide("providers.md"); + const start = guide.indexOf("### Call the AI Gateway from other clients"); + assert(start !== -1, "providers.md documents calling the AI Gateway from other clients"); + const end = guide.indexOf("\n## ", start); + return guide.slice(start, end === -1 ? undefined : end); + } + + it("shows how to export a project key for the client examples", async () => { + const section = await gatewayClientSection(); + + assertStringIncludes(section, 'export VERYFRONT_API_KEY=""'); + }); + + // The guide documents the default routes, so a host that opts back into + // vendor routes must not change what this suite compares against. + function neutralBaseUrl(provider: string): Promise { + return withEnv( + { VERYFRONT_CLOUD_GATEWAY_ROUTES: "" }, + () => Promise.resolve(getVeryfrontCloudGatewayBaseUrl(api, provider)), + ); + } + + it("gives the OpenAI client the base URL the SDK sends OpenAI-protocol models to", async () => { + const section = await gatewayClientSection(); + const base = await neutralBaseUrl("openai"); + + assertStringIncludes(section, `baseURL: "${base}"`); + assertStringIncludes(section, `curl ${base}/chat/completions`); + }); + + it("gives the Anthropic client the canonical Messages base URL", async () => { + const section = await gatewayClientSection(); + const base = `${api}/ai/v1`; + + // The Anthropic client appends /v1/messages to its base URL. + assertEquals(base.endsWith("/v1"), true); + assertStringIncludes(section, `baseURL: "${base.slice(0, -"/v1".length)}"`); + assertStringIncludes(section, `curl ${base}/messages`); + }); + + it("maps unknown SDK vendors to the OpenAI protocol and documents open provider ids", async () => { + const section = await gatewayClientSection(); + + assertEquals(await neutralBaseUrl("acme-labs"), `${api}/ai/v1`); + assertStringIncludes(section, "`/`"); + assertStringIncludes(section, `curl ${api}/ai/models`); + }); +}); + describe("Guide: middleware.md", () => { it("uses the global CORS policy for preflight and the actual response", async () => { const config = defineConfig({ diff --git a/tests/integration/docs/provider-client-examples.deno.lock b/tests/integration/docs/provider-client-examples.deno.lock new file mode 100644 index 0000000000..f006384a39 --- /dev/null +++ b/tests/integration/docs/provider-client-examples.deno.lock @@ -0,0 +1,46 @@ +{ + "version": "5", + "specifiers": { + "npm:@anthropic-ai/sdk@0.128.0": "0.128.0", + "npm:openai@7.23.0": "7.23.0" + }, + "npm": { + "@anthropic-ai/sdk@0.128.0": { + "integrity": "sha512-tl5cBFZC1jVFrTuQyvQObA4WmuNgcYVeXfriqOumgtLAHUm4gPUj18YnszBW60v0PrjDB8nBDOC6wd9CmHQFlA==", + "dependencies": [ + "json-schema-to-ts", + "standardwebhooks" + ], + "bin": true + }, + "@babel/runtime@7.29.7": { + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==" + }, + "@stablelib/base64@1.0.1": { + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==" + }, + "fast-sha256@1.3.0": { + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==" + }, + "json-schema-to-ts@3.1.1": { + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "dependencies": [ + "@babel/runtime", + "ts-algebra" + ] + }, + "openai@7.23.0": { + "integrity": "sha512-0ecOXnFSMNWZq6cUBcTV6Lf93y+fm8BH/+QzFvpoP1UGNOE5pnytRa6HAPGCw+3IS9SIAW7rc1M8yyzN1PiBAQ==" + }, + "standardwebhooks@1.1.1": { + "integrity": "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==", + "dependencies": [ + "@stablelib/base64", + "fast-sha256" + ] + }, + "ts-algebra@2.0.0": { + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==" + } + } +} diff --git a/tests/integration/docs/provider-client-examples.test.ts b/tests/integration/docs/provider-client-examples.test.ts new file mode 100644 index 0000000000..bf0655ad68 --- /dev/null +++ b/tests/integration/docs/provider-client-examples.test.ts @@ -0,0 +1,203 @@ +import { assertEquals, assertRejects } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { makeTempDir } from "#veryfront/testing/deno-compat.ts"; + +type RecordedRequest = { + url: string; + method: string; + headers: Record; + body: unknown; +}; + +async function runSnippetsWithOfficialClients( + snippets: string[], + options: { includeVeryfrontApiKey?: boolean } = {}, +): Promise { + const tempDir = await makeTempDir({ prefix: "vf-provider-docs-" }); + try { + const modulePath = `${tempDir}/provider-snippets.ts`; + const lockPath = `${tempDir}/deno.lock`; + await Deno.copyFile( + new URL("./provider-client-examples.deno.lock", import.meta.url), + lockPath, + ); + const configPath = `${tempDir}/deno.json`; + await Deno.writeTextFile( + configPath, + JSON.stringify( + { + lock: lockPath, + nodeModulesDir: "none", + }, + null, + 2, + ), + ); + await Deno.writeTextFile(modulePath, buildSnippetModule(snippets)); + + const denoCacheDirectory = Deno.env.get("DENO_DIR"); + const command = new Deno.Command(Deno.execPath(), { + args: [ + "run", + "--quiet", + "--frozen", + "--config", + configPath, + "--allow-env", + modulePath, + ], + clearEnv: true, + env: { + ...(denoCacheDirectory === undefined ? {} : { DENO_DIR: denoCacheDirectory }), + ...(options.includeVeryfrontApiKey === false + ? {} + : { VERYFRONT_API_KEY: "example-project-key" }), + OPENAI_API_KEY: "openai-vendor-key-must-not-be-sent", + ANTHROPIC_API_KEY: "vendor-key-must-not-be-sent", + }, + stdout: "piped", + stderr: "piped", + }); + const output = await command.output(); + const decoder = new TextDecoder(); + const stdout = decoder.decode(output.stdout); + const stderr = decoder.decode(output.stderr); + if (!output.success) { + throw new Error( + `Provider snippet subprocess failed with ${output.code}:\n${stderr}\n${stdout}`, + ); + } + return JSON.parse(stdout) as RecordedRequest[]; + } finally { + await Deno.remove(tempDir, { recursive: true }); + } +} + +function buildSnippetModule(snippets: string[]): string { + const transformedSnippets = snippets.map((snippet) => + snippet + .replace( + 'import OpenAI from "openai";', + 'import OpenAI from "npm:openai@7.23.0";', + ) + .replace( + 'import Anthropic from "@anthropic-ai/sdk";', + 'import Anthropic from "npm:@anthropic-ai/sdk@0.128.0";', + ) + ); + const imports = transformedSnippets.map((snippet) => snippet.split("\n")[0]); + const bodies = transformedSnippets.map((snippet) => snippet.split("\n").slice(1).join("\n")); + + return ` +${imports.join("\n")} + +const requests = []; +const encoder = new TextEncoder(); +globalThis.fetch = async (input, init = {}) => { + const request = new Request(input, init); + requests.push({ + url: request.url, + method: request.method, + headers: Object.fromEntries(request.headers.entries()), + body: request.body ? JSON.parse(await request.text()) : null, + }); + const body = request.url.endsWith("/chat/completions") + ? { id: "chatcmpl_example", object: "chat.completion", choices: [] } + : { + id: "msg_example", + type: "message", + role: "assistant", + model: "anthropic/claude-sonnet-4-6", + content: [], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }; + return new Response(encoder.encode(JSON.stringify(body)), { + status: 200, + headers: { "content-type": "application/json" }, + }); +}; + +${bodies.map((body) => `{\n${body}\n}`).join("\n\n")} + +console.log(JSON.stringify(requests)); +`; +} + +describe("provider guide client snippets", () => { + async function getSnippets(): Promise { + const guide = await Deno.readTextFile( + new URL("../../../docs/guides/providers.md", import.meta.url), + ); + const section = guide.split("### Call the AI Gateway from other clients")[1]! + .split("\n## ")[0]!; + return [...section.matchAll(/```ts\n([\s\S]*?)```/g)].map((match) => match[1]!); + } + + it("sends the documented gateway requests", async () => { + const snippets = await getSnippets(); + assertEquals(snippets.length, 2); + assertEquals(snippets.map((snippet) => snippet.split("\n")[0]), [ + 'import OpenAI from "openai";', + 'import Anthropic from "@anthropic-ai/sdk";', + ]); + + const requests = await runSnippetsWithOfficialClients(snippets); + + assertEquals(requests.length, 2); + assertEquals(requests.map((request) => request.method), ["POST", "POST"]); + assertEquals(requests.map((request) => request.url), [ + "https://api.veryfront.com/ai/v1/chat/completions", + "https://api.veryfront.com/ai/v1/messages", + ]); + assertEquals( + requests[0]!.headers.authorization, + "Bearer example-project-key", + ); + assertEquals( + requests[1]!.headers.authorization, + "Bearer example-project-key", + ); + assertEquals(requests[1]!.headers["anthropic-version"], "2023-06-01"); + assertEquals(requests[1]!.headers["x-api-key"], undefined); + assertEquals(requests[0]!.body, { + model: "mistral/mistral-small-2503", + messages: [{ role: "user", content: "Say hello." }], + }); + assertEquals(requests[1]!.body, { + model: "anthropic/claude-sonnet-4-6", + max_tokens: 1024, + messages: [{ role: "user", content: "Say hello." }], + }); + }); + + it("passes an unknown vendor model id through the OpenAI client", async () => { + const [openAiSnippet] = await getSnippets(); + const unknownVendorSnippet = openAiSnippet!.replace( + "mistral/mistral-small-2503", + "acme-labs/model-not-in-the-built-in-catalog", + ); + + const [request] = await runSnippetsWithOfficialClients([unknownVendorSnippet]); + + assertEquals(request!.url, "https://api.veryfront.com/ai/v1/chat/completions"); + assertEquals( + (request!.body as { model: string }).model, + "acme-labs/model-not-in-the-built-in-catalog", + ); + }); + + it("requires a Veryfront key instead of falling back to a vendor key", async () => { + const error = await assertRejects( + async () => + runSnippetsWithOfficialClients(await getSnippets(), { includeVeryfrontApiKey: false }), + Error, + ); + + if (!(error instanceof Error)) { + throw new Error("Expected the missing-project-key guard to reject with an Error"); + } + assertEquals(error.message.includes("Set VERYFRONT_API_KEY"), true); + }); +});