From aba949367b5978fffa9f4ab080cf3b5f8650d1ef Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 13:43:51 -0600 Subject: [PATCH 01/13] Replace local SLM embeddings with scope.models.embed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes the bundled bge-small-en-v1.5 model and the harper-fabric-embeddings + node-llama-cpp dependency chain. Embeddings now run through Harper's new scope.models API (harper#510), which dispatches to whatever embedding backend the host has configured (Ollama on GPU hosts, Anthropic via OpenAI gateway, etc.) — zero model lifecycle to manage in the app. - lib/modelCapture.js: tiny Plugin API hook that stashes Scope on globalThis so Resource classes can reach scope.models. Wired up via `extensionModule:` in config.yaml. - lib/embeddings.js: now a one-line wrapper around scope.models.embed(). - package.json: bumps engines.harperdb to ^5.1, drops the SLM-only deps. - scripts/download-model.js and models/ removed. Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 1 - config.yaml | 4 + lib/embeddings.js | 22 ++-- lib/modelCapture.js | 11 ++ package-lock.json | 204 +------------------------------------- package.json | 11 +- resources/Chat.js | 4 +- scripts/download-model.js | 45 --------- 8 files changed, 30 insertions(+), 272 deletions(-) create mode 100644 lib/modelCapture.js delete mode 100644 scripts/download-model.js diff --git a/.gitignore b/.gitignore index ca8a006..cb565e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ node_modules/ -models/ .env CONFIG.env *.env diff --git a/config.yaml b/config.yaml index d6be5d1..861fdcc 100644 --- a/config.yaml +++ b/config.yaml @@ -6,6 +6,10 @@ loadEnv: rest: true +# Captures the Scope object so resources can call scope.models.embed(). +# See lib/modelCapture.js. +extensionModule: 'lib/modelCapture.js' + graphqlSchema: files: 'schemas/*.graphql' diff --git a/lib/embeddings.js b/lib/embeddings.js index 0b3102e..bddf169 100644 --- a/lib/embeddings.js +++ b/lib/embeddings.js @@ -1,14 +1,14 @@ -import { init, embed as llamaEmbed } from 'harper-fabric-embeddings' -import { resolve } from 'path' -import { fileURLToPath } from 'url' - -const __dirname = fileURLToPath(new URL('.', import.meta.url)) -const modelPath = resolve(__dirname, '../models/bge-small-en-v1.5-q4_k_m.gguf') - -// Model is pre-downloaded by the predev/prestart npm hook (scripts/download-model.js) -const initPromise = init({ modelPath }) +// Thin wrapper around `scope.models.embed()` (harper#510). The host's +// configured backend (Ollama on Fabric GPU hosts, or any backend configured +// via the `models:` block in harperdb-config.yaml / env vars) handles the +// actual inference. Returns a plain Array for compatibility with +// Harper's HNSW vector index storage. export async function embed(text) { - await initPromise - return llamaEmbed(text) + const scope = globalThis.harperScope; + if (!scope) { + throw new Error('Harper scope not yet captured — modelCapture plugin must run before first embed call'); + } + const [vector] = await scope.models.embed(text); + return Array.from(vector); } diff --git a/lib/modelCapture.js b/lib/modelCapture.js new file mode 100644 index 0000000..018348e --- /dev/null +++ b/lib/modelCapture.js @@ -0,0 +1,11 @@ +// Plugin entry — captures the Scope object so `resources/*.js` can call +// `scope.models.embed()` against the host's configured embedding backend. +// +// Harper's `scope` is passed to plugins via `handleApplication(scope)` but +// isn't exposed as a global to Resource classes. This tiny plugin stashes the +// Scope on `globalThis.harperScope` at app boot, then `lib/embeddings.js` +// reads it from there. + +export function handleApplication(scope) { + globalThis.harperScope = scope; +} diff --git a/package-lock.json b/package-lock.json index 1142a3b..48a64ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,8 +11,7 @@ "@anthropic-ai/sdk": "^0.39.0", "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", - "harper": "^5.2.1", - "harper-fabric-embeddings": "^0.2.2" + "harper": "^5.2.1" }, "devDependencies": { "@harperfast/integration-testing": "^0.3.1", @@ -20,11 +19,6 @@ }, "engines": { "harper": "^5.0" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-x64": "3.17.1", - "@node-llama-cpp/mac-arm64-metal": "3.17.1", - "@node-llama-cpp/mac-x64": "3.17.1" } }, "node_modules/@agoric/babel-generator": { @@ -1987,131 +1981,6 @@ "url": "https://paulmillr.com/funding/" } }, - "node_modules/@node-llama-cpp/linux-arm64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-arm64/-/linux-arm64-3.18.1.tgz", - "integrity": "sha512-rXMgZxUay78FOJV/fJ67apYP9eElH5jd4df5YRKPlLhLHHchuOSyDn+qtyW/L/EnPzpogoLkmULqCkdXU39XsQ==", - "cpu": [ - "arm64", - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/linux-armv7l": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-armv7l/-/linux-armv7l-3.18.1.tgz", - "integrity": "sha512-BrJL2cGo0pN5xd5nw+CzTn2rFMpz9MJyZZPUY81ptGkF2uIuXT2hdCVh56i9ImQrTwBfq1YcZL/l/Qe/1+HR/Q==", - "cpu": [ - "arm", - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/linux-x64": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-x64/-/linux-x64-3.17.1.tgz", - "integrity": "sha512-/o/UoqAdslg4ExdKYyYPqbw+21Dr4cQ2JgouXg8Ji3opRKoTMrlUNfrMwIsYZfbDDJ8l7xFnwfGIwdlQ5RPwJg==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/mac-arm64-metal": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-arm64-metal/-/mac-arm64-metal-3.17.1.tgz", - "integrity": "sha512-oRq6/7qCMsazO2Cw0oCyiILZmMvejKJgLAIG60E00WOZWhpJGjh71JGnOybRycKA015mFPNDHzT3SDdUZtZBew==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/mac-x64": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-x64/-/mac-x64-3.17.1.tgz", - "integrity": "sha512-3L0nFVi70j+Qk7Xb8p/RQVMU0E28G0xXX0YL6Vzkirq3DazPYhWOLWUUs9MtGW2FrBg/6PLqyddmxwBfCpjm3w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/win-arm64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/win-arm64/-/win-arm64-3.18.1.tgz", - "integrity": "sha512-S05YUzBMVSRS5KNbOS26cDYugeQHqogI3uewtTUBVC0tPbTHRSKjsdicmgWru1eNAry399LWWhzOf/3St/qsAw==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/win-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/win-x64/-/win-x64-3.18.1.tgz", - "integrity": "sha512-QLDVphPl+YDI+x/VYYgIV1N9g0GMXk3PqcoopOUG3cBRUtce7FO+YX903YdRJezs4oKbIp8YaO+xYBgeUSqhpA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -5558,77 +5427,6 @@ } } }, - "node_modules/harper-fabric-embeddings": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/harper-fabric-embeddings/-/harper-fabric-embeddings-0.2.3.tgz", - "integrity": "sha512-25F1xzRTJ+19NlDiMI0RLF47u4Fwd5Ve/V03q06kJjCioqvEc2yrAASQ3NY4O+LJDU9rNq9WQivFeU+9JKt4IA==", - "hasInstallScript": true, - "license": "MIT", - "engines": { - "node": ">=22" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-arm64": "3.18.1", - "@node-llama-cpp/linux-armv7l": "3.18.1", - "@node-llama-cpp/linux-x64": "3.18.1", - "@node-llama-cpp/mac-arm64-metal": "3.18.1", - "@node-llama-cpp/mac-x64": "3.18.1", - "@node-llama-cpp/win-arm64": "3.18.1", - "@node-llama-cpp/win-x64": "3.18.1" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/linux-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-x64/-/linux-x64-3.18.1.tgz", - "integrity": "sha512-tRmWcsyvAcqJHQHXHsaOkx6muGbcirA9nRdNgH6n7bjGUw4VuoBD3dChyNF3/Ktt7ohB9kz+XhhyZjbDHpXyMA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/mac-arm64-metal": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-arm64-metal/-/mac-arm64-metal-3.18.1.tgz", - "integrity": "sha512-cyZTdsUMlvuRlGmkkoBbN3v/DT6NuruEqoQYd9CqIrPyLa1xLNBTSKIZ9SgRnw23iCOj4URfITvRP+2pu63LuQ==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/mac-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-x64/-/mac-x64-3.18.1.tgz", - "integrity": "sha512-GfCPgdltaIpBhEnQ7WfsrRXrZO9r9pBtDUAQMXRuJwOPP5q7xKrQZUXI6J6mpc8tAG0//CTIuGn4hTKoD/8V8w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/harper/node_modules/uuid": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", diff --git a/package.json b/package.json index 2ea6531..102229b 100644 --- a/package.json +++ b/package.json @@ -7,9 +7,6 @@ "harper": "^5.0" }, "scripts": { - "setup": "node scripts/download-model.js", - "predev": "node scripts/download-model.js", - "prestart": "node scripts/download-model.js", "start": "npx -y dotenv-cli -e .env -o -- harper run .", "dev": "npx -y dotenv-cli -e .env -o -- harper dev .", "login": "node login.js", @@ -20,16 +17,10 @@ "@anthropic-ai/sdk": "^0.39.0", "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", - "harper": "^5.2.1", - "harper-fabric-embeddings": "^0.2.2" + "harper": "^5.2.1" }, "devDependencies": { "@harperfast/integration-testing": "^0.3.1", "@types/node": "^22.19.19" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-x64": "3.17.1", - "@node-llama-cpp/mac-arm64-metal": "3.17.1", - "@node-llama-cpp/mac-x64": "3.17.1" } } diff --git a/resources/Chat.js b/resources/Chat.js index 41ce403..f5f08eb 100644 --- a/resources/Chat.js +++ b/resources/Chat.js @@ -350,8 +350,8 @@ const HTML = /* html */ ` - Local SLM · bge-small-en-v1.5 - embeddings run in Harper · no API cost + Shared model · nomic-embed-text + scope.models.embed() · GPU on Fabric · no API cost diff --git a/scripts/download-model.js b/scripts/download-model.js deleted file mode 100644 index a8aeb63..0000000 --- a/scripts/download-model.js +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node -// Downloads the bge-small-en-v1.5 embedding model if not already present. -// Run automatically via the predev / prestart npm hooks. - -import { createWriteStream, existsSync, mkdirSync } from 'fs' -import { pipeline } from 'stream/promises' -import { resolve } from 'path' -import { fileURLToPath } from 'url' - -const __dirname = fileURLToPath(new URL('.', import.meta.url)) -const modelsDir = resolve(__dirname, '../models') -const modelPath = resolve(modelsDir, 'bge-small-en-v1.5-q4_k_m.gguf') -const MODEL_URL = - 'https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-q4_k_m.gguf' - -if (existsSync(modelPath)) { - console.log('✓ Embedding model already downloaded.') - process.exit(0) -} - -console.log('Downloading bge-small-en-v1.5 embedding model (~24 MB)...') -mkdirSync(modelsDir, { recursive: true }) - -const response = await fetch(MODEL_URL) -if (!response.ok) { - console.error(`Download failed: ${response.status} ${response.statusText}`) - process.exit(1) -} - -const total = Number(response.headers.get('content-length') || 0) -let downloaded = 0 - -const progress = new TransformStream({ - transform(chunk, controller) { - downloaded += chunk.byteLength - if (total) { - const pct = Math.round((downloaded / total) * 100) - process.stdout.write(`\r ${pct}% (${(downloaded / 1024 / 1024).toFixed(1)} MB)`) - } - controller.enqueue(chunk) - }, -}) - -await pipeline(response.body.pipeThrough(progress), createWriteStream(modelPath)) -console.log('\n✓ Model ready.') From a3f944ee1886ff5bc10f9edd9c3b70bc9a9c189b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 20:07:08 -0600 Subject: [PATCH 02/13] Switch Agent from Anthropic SDK to scope.models.generate() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the @anthropic-ai/sdk + @anthropic-ai/vertex-sdk direct calls with a single scope.models.generate() call. Routes to whatever backend the host has configured for models.generative.default — vLLM on Fabric GPU hosts, Anthropic/OpenAI/Ollama elsewhere. Removes: - @anthropic-ai/sdk + @anthropic-ai/vertex-sdk deps - lib/config.js (LLM_PROVIDER / ANTHROPIC_API_KEY / VERTEX_*) - Anthropic web search tool + pause_turn handling - Per-token cost calculation (local model = no $) - Multi-block response stitching (scope.models returns plain content) Embeddings already used scope.models.embed() (prior commit on this branch). The Stats table now tracks cacheHits only; totalSaved is held at zero. Co-Authored-By: Claude Sonnet 4.6 --- lib/config.js | 21 --- package-lock.json | 376 +-------------------------------------------- package.json | 4 +- resources/Agent.js | 121 ++++----------- 4 files changed, 31 insertions(+), 491 deletions(-) delete mode 100644 lib/config.js diff --git a/lib/config.js b/lib/config.js deleted file mode 100644 index d1e1885..0000000 --- a/lib/config.js +++ /dev/null @@ -1,21 +0,0 @@ -const required = (name) => { - const value = process.env[name] - if (!value) throw new Error(`Missing required env var: ${name}`) - return value -} - -const optional = (name, fallback) => process.env[name] ?? fallback - -export const config = { - // "anthropic" (direct API) or "vertex" (Google Cloud Vertex AI) - provider: () => optional('LLM_PROVIDER', 'anthropic'), - anthropic: { - apiKey: () => required('ANTHROPIC_API_KEY'), - model: () => optional('CLAUDE_MODEL', 'claude-sonnet-4-5-20250929'), - }, - vertex: { - projectId: () => required('VERTEX_PROJECT_ID'), - region: () => optional('VERTEX_REGION', 'global'), - model: () => optional('VERTEX_MODEL', 'claude-sonnet-4-6'), - }, -} diff --git a/package-lock.json b/package-lock.json index 48a64ee..6deb47b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,8 +8,6 @@ "name": "agent-example-harper", "version": "1.0.0", "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", - "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", "harper": "^5.2.1" }, @@ -35,67 +33,6 @@ "node": ">=6.9.0" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.39.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.39.0.tgz", - "integrity": "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg==", - "license": "MIT", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/@anthropic-ai/vertex-sdk": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/vertex-sdk/-/vertex-sdk-0.15.0.tgz", - "integrity": "sha512-i2LDdu6VB8Lqqip+kbNSXRxQgFsCg6GPBO/X2zRJwLl99dNzf28nb6Rdi0EodONXsyJfY2TKdGR+y5l1/AKFEg==", - "license": "MIT", - "dependencies": { - "@anthropic-ai/sdk": ">=0.50.3 <1", - "google-auth-library": "^9.4.2" - } - }, - "node_modules/@anthropic-ai/vertex-sdk/node_modules/@anthropic-ai/sdk": { - "version": "0.116.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.116.0.tgz", - "integrity": "sha512-4UEapYQ+epLEMsAuLZDvW8ExVSOtHD8a7zTyLzhw0H9RXJ1eilPgmqhjwgcdg22diwx13spw6fJ4rONZ+bS7Ww==", - "license": "MIT", - "dependencies": { - "json-schema-to-ts": "^3.1.1", - "standardwebhooks": "^1.0.0" - }, - "bin": { - "anthropic-ai-sdk": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } - } - }, "node_modules/@aws-sdk/checksums": { "version": "3.1000.26", "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.26.tgz", @@ -2425,12 +2362,6 @@ "node": ">=18.0.0" } }, - "node_modules/@stablelib/base64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", - "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", - "license": "MIT" - }, "node_modules/@turf/area": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@turf/area/-/area-6.5.0.tgz", @@ -2791,16 +2722,6 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, "node_modules/@types/readable-stream": { "version": "4.0.24", "resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.24.tgz", @@ -2907,22 +2828,12 @@ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -3117,12 +3028,6 @@ "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==", "license": "MIT" }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -3944,18 +3849,6 @@ "node": ">=0.1.90" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/commander": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", @@ -4210,15 +4103,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4429,21 +4313,6 @@ "node": ">= 0.4" } }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -4533,12 +4402,6 @@ "optional": true, "peer": true }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, "node_modules/eyes": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", @@ -4631,12 +4494,6 @@ "node": ">=6" } }, - "node_modules/fast-sha256": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", - "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", - "license": "Unlicense" - }, "node_modules/fast-uri": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.2.tgz", @@ -4921,41 +4778,6 @@ "optional": true, "peer": true }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/fraction.js": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.4.tgz", @@ -5033,36 +4855,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gcp-metadata": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", - "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^6.1.1", - "google-logging-utils": "^0.0.2", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -5194,32 +4986,6 @@ "node": ">= 6" } }, - "node_modules/google-auth-library": { - "version": "9.15.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", - "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/google-logging-utils": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", - "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -5262,19 +5028,6 @@ "graphql": ">=0.11 <=17" } }, - "node_modules/gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "license": "MIT", - "dependencies": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/gunzip-maybe": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", @@ -5552,6 +5305,8 @@ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "4" @@ -5569,15 +5324,6 @@ "knuth-shuffle": "^1.0.0" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -5925,18 +5671,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-unicode-supported": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", @@ -6273,15 +6007,6 @@ "node": ">=4" } }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, "node_modules/json-bigint-fixes": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/json-bigint-fixes/-/json-bigint-fixes-1.1.0.tgz", @@ -6310,19 +6035,6 @@ "dequal": "^2.0.3" } }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -7246,27 +6958,6 @@ "node": ">= 0.6" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types/node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -7466,26 +7157,6 @@ "node": "^18 || ^20 || >= 21" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -9454,16 +9125,6 @@ "node": ">=8" } }, - "node_modules/standardwebhooks": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", - "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", - "license": "MIT", - "dependencies": { - "@stablelib/base64": "^1.0.0", - "fast-sha256": "^1.3.0" - } - }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -9945,12 +9606,6 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "license": "MIT" }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "license": "MIT" - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -10092,20 +9747,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/validate.js": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/validate.js/-/validate.js-0.13.1.tgz", @@ -10146,15 +9787,6 @@ "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==", "license": "MIT" }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/package.json b/package.json index 102229b..15c4535 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "agent-example-harper", "version": "1.0.0", - "description": "A conversational AI agent with persistent memory, built on Harper and Claude", + "description": "A conversational AI agent with persistent memory, built on Harper with scope.models", "type": "module", "engines": { "harper": "^5.0" @@ -14,8 +14,6 @@ "test:integration": "harper-integration-test-run 'integrationTests/**/*.test.ts'" }, "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", - "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", "harper": "^5.2.1" }, diff --git a/resources/Agent.js b/resources/Agent.js index 02e9310..3851fa7 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -1,40 +1,10 @@ import { Resource, tables } from 'harper' -import Anthropic from '@anthropic-ai/sdk' -import { AnthropicVertex } from '@anthropic-ai/vertex-sdk' -import { config } from '../lib/config.js' import { embed } from '../lib/embeddings.js' -let _client -const getClient = () => { - if (_client) return _client - if (config.provider() === 'vertex') { - _client = new AnthropicVertex({ - projectId: config.vertex.projectId(), - region: config.vertex.region(), - }) - } else { - _client = new Anthropic({ apiKey: config.anthropic.apiKey() }) - } - return _client -} - -const getModel = () => - config.provider() === 'vertex' ? config.vertex.model() : config.anthropic.model() - const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the user's current question. \ Do NOT summarize, repeat, or reference prior conversation context in your response — use it silently \ as background knowledge only if it is directly relevant. Never recite or recap previous answers.` -// Approximate pricing for Claude Sonnet 4.5 (per token) -const COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens -const COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens -const COST_PER_WEB_SEARCH = 10 / 1_000 // $10 / 1K searches - -// Anthropic web search tool — executed server-side, no external API key needed. -// Not available on Vertex AI without an org policy change. -const WEB_SEARCH_TOOL = { type: 'web_search_20250305', name: 'web_search', max_uses: 5 } -const isVertex = () => config.provider() === 'vertex' - // Normalize text for embedding cache key — lowercase, strip punctuation, collapse whitespace const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() @@ -44,8 +14,7 @@ const normalize = (s) => // equivalent to cosine similarity >= 0.88 (distance = 1 - similarity = 0.12). const CACHE_DISTANCE_THRESHOLD = 0.12 -// Get or compute an embedding, using Harper as a cache to skip the SLM on repeated text. -// On Fabric, the SLM takes ~2.3s per embedding — this cache makes repeat queries instant. +// Get or compute an embedding, using Harper as a cache to skip the model on repeated text. async function cachedEmbed(text) { const key = normalize(text) const cached = await tables.EmbeddingCache.get(key) @@ -133,23 +102,17 @@ export class Agent extends Resource { const timing = { embedMs: tEmbed, convMs: tConv, storeMs: tStore, cacheSearchMs: tCache } console.log('[Agent] timing:', JSON.stringify(timing)) - // Return the cached answer — zero LLM cost + // Return the cached answer — zero LLM call if (cachedReply) { - const t5 = Date.now() - let savedCost = 0 try { - const origMsg = await tables.Message.get(cachedReply.id) - savedCost = origMsg?.cost ?? 0 const stats = await tables.Stats.get('global') await tables.Stats.put({ id: 'global', - totalSaved: ((stats?.totalSaved) ?? 0) + savedCost, - cacheHits: ((stats?.cacheHits) ?? 0) + 1, - updatedAt: new Date().toISOString(), + totalSaved: stats?.totalSaved ?? 0, + cacheHits: ((stats?.cacheHits) ?? 0) + 1, + updatedAt: new Date().toISOString(), }) } catch {} - const tStats = Date.now() - t5 - console.log('[Agent] cache hit stats update:', tStats + 'ms') return { conversationId, message: { role: 'assistant', content: cachedReply.content }, @@ -157,66 +120,40 @@ export class Agent extends Resource { latencyMs: Date.now() - startTime, timing, tokens: { input: 0, output: 0, total: 0 }, - cost: { input: 0, output: 0, total: 0, saved: savedCost }, vectorContext: { hit: true, count: 1, cached: true }, }, } } - // 5. Call Claude with web search enabled — standalone question, no conversation history. - // Anthropic executes searches server-side, no external search API or key required. - const messages = [{ role: 'user', content: message }] - - const tools = isVertex() ? [] : [WEB_SEARCH_TOOL] - - let apiResponse = await getClient().messages.create({ - model: getModel(), - max_tokens: 1024, - ...(tools.length && { tools }), - system: SYSTEM_PROMPT, - messages, - }) + // 5. Generate via scope.models.generate() — routes to whatever backend the host + // has configured for `models.generative.default` (vLLM on Fabric GPU hosts, + // Ollama / OpenAI / Anthropic on other deployments). + const scope = globalThis.harperScope + if (!scope) { + throw new Error('Harper scope not yet captured — modelCapture plugin must run before first generate call') + } - // Handle pause_turn — server hit the max_uses limit mid-response; continue once - if (apiResponse.stop_reason === 'pause_turn') { - apiResponse = await getClient().messages.create({ - model: getModel(), - max_tokens: 1024, - ...(tools.length && { tools }), + const result = await scope.models.generate( + { + messages: [{ role: 'user', content: message }], system: SYSTEM_PROMPT, - messages: [...messages, { role: 'assistant', content: apiResponse.content }], - }) - } + }, + { maxTokens: 1024 }, + ) const latencyMs = Date.now() - startTime - - // The API can split the answer across multiple text blocks (sentence fragments joined - // without separators) and may emit a text block BEFORE the web search tool call. - // Strategy: find the last non-text block (tool use / search result) and take only the - // text blocks that follow it — these form the actual answer. Join with '' since the - // fragments are already continuous prose. Falls back to all text blocks if no tools used. - const lastToolIdx = apiResponse.content.reduce((acc, b, i) => b.type !== 'text' ? i : acc, -1) - const assistantContent = apiResponse.content - .slice(lastToolIdx + 1) - .filter((b) => b.type === 'text') - .map((b) => b.text) - .join('') - .trim() - - const { input_tokens, output_tokens } = apiResponse.usage - const webSearches = apiResponse.usage?.server_tool_use?.web_search_requests ?? 0 + const assistantContent = result.content?.trim() ?? '' + const promptTokens = result.usage?.promptTokens ?? 0 + const completionTokens = result.usage?.completionTokens ?? 0 // 9. Store the assistant's response with its embedding const assistantMsgId = crypto.randomUUID() const assistantEmbedding = await cachedEmbed(assistantContent) - const searchCost = webSearches * COST_PER_WEB_SEARCH - const totalCost = (input_tokens * COST_INPUT_PER_TOKEN) + (output_tokens * COST_OUTPUT_PER_TOKEN) + searchCost await tables.Message.put({ id: assistantMsgId, conversationId, role: 'assistant', content: assistantContent, - cost: totalCost, embedding: assistantEmbedding, createdAt: new Date().toISOString(), }) @@ -232,18 +169,12 @@ export class Agent extends Resource { message: { role: 'assistant', content: assistantContent }, meta: { latencyMs, + timing, tokens: { - input: input_tokens, - output: output_tokens, - total: input_tokens + output_tokens, - }, - cost: { - input: +(input_tokens * COST_INPUT_PER_TOKEN).toFixed(6), - output: +(output_tokens * COST_OUTPUT_PER_TOKEN).toFixed(6), - search: +searchCost.toFixed(6), - total: +totalCost.toFixed(6), + input: promptTokens, + output: completionTokens, + total: promptTokens + completionTokens, }, - webSearches, vectorContext: { hit: false, count: 0, cached: false }, }, } @@ -253,6 +184,6 @@ export class Agent extends Resource { export class PublicStats extends Resource { static async get(target) { target.checkPermission = false - return await tables.Stats.get('global') ?? { id: 'global', totalSaved: 0, cacheHits: 0 } + return await tables.Stats.get('global') ?? { id: 'global', cacheHits: 0 } } } From 99c8f69b9ff7a39bc150121493ed885fe0e46251 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 20:13:35 -0600 Subject: [PATCH 03/13] Restore estimated-cost reporting (Claude pricing as comparator) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UI was breaking on `meta.cost.saved` undefined. Bring back the cost block but interpret it as a hypothetical — what each generation WOULD have cost on Claude Sonnet 4.5 ($3/1M input, $15/1M output). Local GPU compute is effectively $0; the dashboard tracks what we're saving by self-hosting + the semantic cache. - Estimated cost computed from token counts vLLM returns - Stored on each assistant Message (same Float field as before) - Cache hits credit the original message's cost to Stats.totalSaved - meta.cost { input, output, total, saved } shape matches what Chat.js's buildMeta() reads, so no UI changes needed Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 3851fa7..5633792 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -5,6 +5,16 @@ const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the use Do NOT summarize, repeat, or reference prior conversation context in your response — use it silently \ as background knowledge only if it is directly relevant. Never recite or recap previous answers.` +// Hypothetical Claude Sonnet 4.5 pricing used to estimate what each generation +// WOULD have cost if we'd called Anthropic instead of the local GPU. Real local +// compute cost is roughly $0 (sunk-cost GPU); the dashboard shows what we're +// saving by self-hosting + caching. +const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens +const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens + +const estimateClaudeCost = (promptTokens, completionTokens) => + promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN + // Normalize text for embedding cache key — lowercase, strip punctuation, collapse whitespace const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() @@ -102,13 +112,18 @@ export class Agent extends Resource { const timing = { embedMs: tEmbed, convMs: tConv, storeMs: tStore, cacheSearchMs: tCache } console.log('[Agent] timing:', JSON.stringify(timing)) - // Return the cached answer — zero LLM call + // Return the cached answer — zero LLM call. We credit the original message's + // estimated cost to `totalSaved` so the dashboard shows the running benefit + // of the semantic cache (and self-hosting more broadly). if (cachedReply) { + let savedCost = 0 try { + const origMsg = await tables.Message.get(cachedReply.id) + savedCost = origMsg?.cost ?? 0 const stats = await tables.Stats.get('global') await tables.Stats.put({ id: 'global', - totalSaved: stats?.totalSaved ?? 0, + totalSaved: (stats?.totalSaved ?? 0) + savedCost, cacheHits: ((stats?.cacheHits) ?? 0) + 1, updatedAt: new Date().toISOString(), }) @@ -120,6 +135,7 @@ export class Agent extends Resource { latencyMs: Date.now() - startTime, timing, tokens: { input: 0, output: 0, total: 0 }, + cost: { input: 0, output: 0, total: 0, saved: savedCost }, vectorContext: { hit: true, count: 1, cached: true }, }, } @@ -145,8 +161,11 @@ export class Agent extends Resource { const assistantContent = result.content?.trim() ?? '' const promptTokens = result.usage?.promptTokens ?? 0 const completionTokens = result.usage?.completionTokens ?? 0 + const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) - // 9. Store the assistant's response with its embedding + // 9. Store the assistant's response with its embedding. We persist the + // *hypothetical* Claude cost so a future cache-hit on this same message + // can credit that amount to `totalSaved`. const assistantMsgId = crypto.randomUUID() const assistantEmbedding = await cachedEmbed(assistantContent) await tables.Message.put({ @@ -154,6 +173,7 @@ export class Agent extends Resource { conversationId, role: 'assistant', content: assistantContent, + cost: estimatedCost, embedding: assistantEmbedding, createdAt: new Date().toISOString(), }) @@ -175,6 +195,13 @@ export class Agent extends Resource { output: completionTokens, total: promptTokens + completionTokens, }, + cost: { + input: +(promptTokens * CLAUDE_COST_INPUT_PER_TOKEN).toFixed(6), + output: +(completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN).toFixed(6), + total: +estimatedCost.toFixed(6), + // `saved` is what cache hits credit; on a real generation it stays 0. + saved: 0, + }, vectorContext: { hit: false, count: 0, cached: false }, }, } @@ -184,6 +211,6 @@ export class Agent extends Resource { export class PublicStats extends Resource { static async get(target) { target.checkPermission = false - return await tables.Stats.get('global') ?? { id: 'global', cacheHits: 0 } + return await tables.Stats.get('global') ?? { id: 'global', totalSaved: 0, cacheHits: 0 } } } From 7d0521171bb27642926c2a26298c7a0d92036413 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 20:14:49 -0600 Subject: [PATCH 04/13] Estimate token counts from text length (~4 chars/token heuristic) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scope.models.generate() only returns { content, finishReason } today — the backend's usage info isn't surfaced to the caller. Approximate input/output tokens from character counts so the estimated-Claude-cost comparator on the Chat dashboard shows non-zero values. Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 5633792..844b23d 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -12,6 +12,11 @@ as background knowledge only if it is directly relevant. Never recite or recap p const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens +// scope.models.generate() returns only { content, finishReason } today — the +// backend's token usage isn't surfaced to callers. Approximate with the +// ~4-chars-per-token rule of thumb for English; close enough for a comparator. +const estimateTokens = (text) => Math.max(1, Math.ceil((text?.length ?? 0) / 4)) + const estimateClaudeCost = (promptTokens, completionTokens) => promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN @@ -159,8 +164,8 @@ export class Agent extends Resource { const latencyMs = Date.now() - startTime const assistantContent = result.content?.trim() ?? '' - const promptTokens = result.usage?.promptTokens ?? 0 - const completionTokens = result.usage?.completionTokens ?? 0 + const promptTokens = estimateTokens(SYSTEM_PROMPT + message) + const completionTokens = estimateTokens(assistantContent) const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) // 9. Store the assistant's response with its embedding. We persist the From 61a9b3802b5a5185131ab93c409ce81d24bedf67 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 21:14:56 -0600 Subject: [PATCH 05/13] Fix cache-match selection: explicit distance sort + tighter threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs were stacking up: 1. The HNSW search returned matches that satisfy the threshold but didn't guarantee distance-ascending iteration order, so the loop's first match wasn't necessarily the closest. Collect candidates, compute cosine distance explicitly, sort, then pick the closest valid one. 2. The threshold (0.12 cosine distance ≈ 0.88 similarity) was loose enough that distinct queries about the same topic would collide ("describe the moon landing" vs "tell me about apollo 11"). Tighten to 0.05 (≈ 0.95 similarity), which still catches near-paraphrases and re-wordings while filtering out merely-related questions. Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 844b23d..5c96654 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -25,9 +25,23 @@ const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() // Cosine distance threshold for Harper's native HNSW vector search. -// Harper uses cosine *distance* (0 = identical, 2 = opposite), so this is -// equivalent to cosine similarity >= 0.88 (distance = 1 - similarity = 0.12). -const CACHE_DISTANCE_THRESHOLD = 0.12 +// Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.05 ≈ cosine +// similarity 0.95 — strict enough that semantically different queries +// ("describe the moon landing" vs "tell me about apollo 11") don't collide. +const CACHE_DISTANCE_THRESHOLD = 0.05 + +// HNSW search returns matches that satisfy the threshold but doesn't guarantee +// distance-ordered iteration. We compute distance ourselves and pick the closest. +function cosineDistance(a, b) { + let dot = 0, na = 0, nb = 0 + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i] + na += a[i] * a[i] + nb += b[i] * b[i] + } + const denom = Math.sqrt(na) * Math.sqrt(nb) + return denom === 0 ? 1 : 1 - dot / denom +} // Get or compute an embedding, using Harper as a cache to skip the model on repeated text. async function cachedEmbed(text) { @@ -84,6 +98,9 @@ export class Agent extends Resource { const tStore = Date.now() - t3 // 4. Semantic cache — Harper-native HNSW vector search with distance threshold. + // HNSW search returns matches under the threshold but iteration order isn't + // guaranteed to be distance-ascending, so we collect candidates, compute + // cosine distance ourselves, and pick the closest valid one. const t4 = Date.now() let cachedReply = null const nearbyMsgs = tables.Message.search({ @@ -93,11 +110,17 @@ export class Agent extends Resource { value: CACHE_DISTANCE_THRESHOLD, target: userEmbedding, }, - limit: 10, + limit: 20, }) + const candidates = [] for await (const match of nearbyMsgs) { - if (match.id === userMsgId || match.role !== 'user') continue + if (match.id === userMsgId || match.role !== 'user' || !match.embedding) continue + candidates.push({ match, distance: cosineDistance(userEmbedding, match.embedding) }) + } + candidates.sort((a, b) => a.distance - b.distance) + + for (const { match } of candidates) { const matchConvMsgs = [] const matchHistory = tables.Message.search({ conditions: [{ attribute: 'conversationId', value: match.conversationId }], From 6e8910075c6e99fd4c2e5451354dd342428f18fc Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 21:19:20 -0600 Subject: [PATCH 06/13] Debug: log cache candidates with computed distance + hard threshold filter --- resources/Agent.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/resources/Agent.js b/resources/Agent.js index 5c96654..6b2185f 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -120,7 +120,20 @@ export class Agent extends Resource { } candidates.sort((a, b) => a.distance - b.distance) - for (const { match } of candidates) { + // Debug: print what the search returned along with the actual computed distance. + // Harper's HNSW `lt` filter doesn't always cull things outside the threshold, + // so we apply a hard check using the distance we computed ourselves. + if (candidates.length > 0) { + console.log('[Agent] cache candidates:', candidates.slice(0, 5).map((c) => ({ + id: c.match.id, + role: c.match.role, + dist: +c.distance.toFixed(4), + content: c.match.content?.slice(0, 60), + }))) + } + const filtered = candidates.filter((c) => c.distance <= CACHE_DISTANCE_THRESHOLD) + + for (const { match } of filtered) { const matchConvMsgs = [] const matchHistory = tables.Message.search({ conditions: [{ attribute: 'conversationId', value: match.conversationId }], From 14b07d569d4ba9e6608e593b05aa44d60606dcf8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 21:23:46 -0600 Subject: [PATCH 07/13] Fix cache: assistant reply must be the IMMEDIATE next message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous .find(role==='assistant') walked past any subsequent user messages and grabbed the next assistant reply even if it was many turns later in the conversation. With a long conversation like: user: Is soccer fun? assistant: ...soccer is fun... user: Is soccer fun (cache hit, no new assistant stored) user: Is soccer fun?? (cache hit, no new assistant stored) user: what is 2 plus 3 assistant: 2 plus 3 is 5 matching the second "Is soccer fun" returned "2 plus 3 is 5" — the next assistant message in chronological order, but completely unrelated. Only accept the IMMEDIATELY-following message as the reply. If the next message is another user message, skip this candidate and try the next. Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 6b2185f..5b25d25 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -120,17 +120,8 @@ export class Agent extends Resource { } candidates.sort((a, b) => a.distance - b.distance) - // Debug: print what the search returned along with the actual computed distance. - // Harper's HNSW `lt` filter doesn't always cull things outside the threshold, + // Harper's HNSW `lt` filter doesn't always cull matches outside the threshold, // so we apply a hard check using the distance we computed ourselves. - if (candidates.length > 0) { - console.log('[Agent] cache candidates:', candidates.slice(0, 5).map((c) => ({ - id: c.match.id, - role: c.match.role, - dist: +c.distance.toFixed(4), - content: c.match.content?.slice(0, 60), - }))) - } const filtered = candidates.filter((c) => c.distance <= CACHE_DISTANCE_THRESHOLD) for (const { match } of filtered) { @@ -142,9 +133,14 @@ export class Agent extends Resource { for await (const m of matchHistory) matchConvMsgs.push(m) matchConvMsgs.sort((a, b) => a.createdAt.localeCompare(b.createdAt)) const midx = matchConvMsgs.findIndex((m) => m.id === match.id) - const reply = matchConvMsgs.slice(midx + 1).find((m) => m.role === 'assistant') - if (reply) { - cachedReply = reply + // The matched message's reply must be the IMMEDIATELY following message. + // `.find()` would walk past any subsequent user-msgs (cache hits that didn't + // generate a reply) and pull an unrelated answer from much later in the + // conversation — e.g. matching "is soccer fun" but returning the assistant + // reply to a later "what is 2 plus 3" question in the same conversation. + const next = matchConvMsgs[midx + 1] + if (next?.role === 'assistant') { + cachedReply = next break } } From e2447e773f2e7ea6502e1d6c1d77fe7b97bee9f3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 23 May 2026 21:28:47 -0600 Subject: [PATCH 08/13] Loosen cache threshold to 0.15 (~0.85 similarity) Now that the immediate-next-message fix is in, the threshold can be relaxed to catch more rewordings without serving wrong answers. The 0.05 setting was paranoia about false positives that turned out to come from the slice/find bug. Co-Authored-By: Claude Sonnet 4.6 --- resources/Agent.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 5b25d25..db61e01 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -25,10 +25,11 @@ const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() // Cosine distance threshold for Harper's native HNSW vector search. -// Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.05 ≈ cosine -// similarity 0.95 — strict enough that semantically different queries -// ("describe the moon landing" vs "tell me about apollo 11") don't collide. -const CACHE_DISTANCE_THRESHOLD = 0.05 +// Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.15 ≈ cosine +// similarity 0.85 — loose enough to catch rewordings and related phrasings +// ("describe the moon landing" / "tell me about apollo 11"), tight enough +// that the matched reply is reasonably on-topic. +const CACHE_DISTANCE_THRESHOLD = 0.15 // HNSW search returns matches that satisfy the threshold but doesn't guarantee // distance-ordered iteration. We compute distance ourselves and pick the closest. From 713348e8c3196bf9da9d77cb6a19f2db0e52a817 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 25 Aug 2026 08:54:28 -0600 Subject: [PATCH 09/13] Use the harper `models` export instead of a globalThis Scope shim `models` is a process-wide singleton that harper exports as a named package export (and an ambient global); `scope.models` is the same object. The `handleApplication(scope)` plugin that stashed the Scope on `globalThis` was therefore unnecessary, so `lib/modelCapture.js` and the `extensionModule` entry it needed are gone. Docs that this branch had already invalidated are corrected alongside: the README, CLAUDE.md and dot-env.example still told readers to obtain an Anthropic or Vertex key and described the removed bge-small-en-v1.5 and web-search paths. Co-Authored-By: Claude Opus --- CLAUDE.md | 51 ++++-------- README.md | 146 ++++++++++++++--------------------- config.yaml | 4 - dot-env.example | 15 +--- integrationTests/app.test.ts | 19 ++++- lib/embeddings.js | 17 ++-- lib/modelCapture.js | 11 --- package.json | 4 +- resources/Agent.js | 17 ++-- resources/Chat.js | 2 +- 10 files changed, 113 insertions(+), 173 deletions(-) delete mode 100644 lib/modelCapture.js diff --git a/CLAUDE.md b/CLAUDE.md index f0c8227..d93d080 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,16 +12,15 @@ These skills provide LLM-consumable guidelines at https://github.com/HarperFast/ ## Project Overview -Harper Demo Agent — a conversational AI agent running entirely on Harper with Claude as the LLM. Harper provides the database, vector index, semantic cache, API server, and deployment runtime in a single process. +Harper Demo Agent — a conversational AI agent running entirely on Harper. Harper provides the database, vector index, semantic cache, API server, model gateway, and deployment runtime in a single process; generation and embedding both go through Harper's `models` API, so the app carries no LLM SDK and no API key. Live demo: https://agent-example.stephen-demo-org.harperfabric.com/Chat ## Tech Stack - **Runtime:** Harper (harperdb) — unified DB/cache/vector/API -- **LLM:** Claude Sonnet via Anthropic SDK (`@anthropic-ai/sdk`) or Google Cloud Vertex AI (`@anthropic-ai/vertex-sdk`) -- **Embeddings:** `bge-small-en-v1.5` running locally via `harper-fabric-embeddings` (llama.cpp) — no embedding API -- **Web Search:** Anthropic's built-in server-side `web_search_20250305` tool +- **LLM:** `models.generate()` — routes to the host's configured `models.generative.default` backend (the shared inference process on Fabric GPU hosts; Ollama / OpenAI / Anthropic / Bedrock elsewhere) +- **Embeddings:** `models.embed()` — routes to the host's configured `models.embedding.default` backend - **Language:** JavaScript (ES modules, `"type": "module"`) - **License:** Apache 2.0 @@ -32,10 +31,8 @@ config.yaml # Harper app config (rest, schema, resources) schemas/schema.graphql # Database schema — 3 tables, HNSW vector index, TTL resources/Agent.js # Agent endpoint (POST /Agent) + PublicStats (GET /PublicStats/global) resources/Chat.js # Chat UI (GET /Chat) — full HTML/CSS/JS served from a Resource -lib/config.js # Environment variable helpers -lib/embeddings.js # Local SLM embeddings (bge-small-en-v1.5 via llama.cpp) -models/ # Auto-downloaded GGUF model (gitignored) -.env # ANTHROPIC_API_KEY (not committed) +lib/embeddings.js # Thin wrapper over `models.embed()` +.env # Deploy credentials only (not committed) ``` ## Key Architecture Decisions @@ -55,17 +52,16 @@ models/ # Auto-downloaded GGUF model (gitignored) ### Semantic Cache (two layers) 1. **Layer 1 — Exact match:** Normalize text (lowercase, strip punctuation, collapse whitespace) and compare against conversation history. No DB query needed. -2. **Layer 2 — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.12` (cosine distance). **Never do manual cosine similarity in JS** — always use Harper's native HNSW index for distance filtering. +2. **Layer 2 — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.15` (cosine distance). **Never do manual cosine similarity in JS** — always use Harper's native HNSW index for distance filtering. ### Vector Context (for LLM prompt) - Uses `sort: { attribute: 'embedding', target: userEmbedding }` with `limit: 10` — returns top 10 most similar messages - Top 5 injected into system prompt as silent background context - System prompt explicitly tells Claude NOT to repeat/summarize context in responses -### Web Search Response Handling -- Anthropic API returns multiple `text` blocks (sentence fragments) mixed with `server_tool_use` and `web_search_tool_result` blocks -- **Always join text blocks that appear AFTER the last non-text block** — text before tool calls is narration ("Let me search for that..."), not the answer -- Handle `pause_turn` stop reason by continuing with partial response as assistant message +### Models API access +- `import { models } from 'harper'` — `models` is a process-wide singleton, exported by the `harper` package and also available as a bare global. It is the same object as `scope.models`, so a `handleApplication(scope)` plugin that stashes the Scope on `globalThis` is not needed. +- `models.generate()` currently returns `{ content, finishReason }` only — no token usage — so `resources/Agent.js` estimates token counts from text length for the cost comparator. ### Chat UI (resources/Chat.js) - Full HTML/CSS/JS served from a single template literal via `new Response(HTML, ...)` @@ -82,28 +78,15 @@ npm run start # Start production server npm run deploy # Deploy to Harper Fabric ``` -## Environment Variables +## Model Configuration -``` -LLM_PROVIDER # Optional — "anthropic" (default) or "vertex" (Google Cloud Vertex AI) -ANTHROPIC_API_KEY # Required when LLM_PROVIDER=anthropic — Anthropic API key -CLAUDE_MODEL # Optional — defaults to claude-sonnet-4-5-20250929 (anthropic) or claude-sonnet-4-5@20250929 (vertex) -VERTEX_PROJECT_ID # Required when LLM_PROVIDER=vertex — Google Cloud project ID -VERTEX_REGION # Optional — Vertex AI region, defaults to "global" -``` - -### Vertex AI Setup - -To use Claude through Google Cloud Vertex AI instead of the direct Anthropic API: +The app names no model and holds no provider credentials. Backends are configured on the +Harper *host* — the `models:` block of `harperdb-config.yaml`, or the equivalent env vars — +under `models.embedding.default` and `models.generative.default`. With neither configured, +`POST /Agent` returns a `ModelBackendNotFoundError`. -1. Set up GCP credentials: `gcloud auth application-default login` -2. Configure env vars: - ``` - LLM_PROVIDER=vertex - VERTEX_PROJECT_ID=my-gcp-project - VERTEX_REGION=global - ``` -3. Note: Vertex model IDs use `@` version suffixes (e.g. `claude-sonnet-4-5@20250929`) while direct API uses `-` (e.g. `claude-sonnet-4-5-20250929`) +`.env` is still read (see `loadEnv` in `config.yaml`) but only carries the Fabric deploy +credentials: `CLI_TARGET`, `CLI_TARGET_USERNAME`, `CLI_TARGET_PASSWORD`. ## Common Tasks @@ -131,5 +114,5 @@ curl http://localhost:9926/PublicStats/global 1. **Template literal backslashes** — `\n` inside a JS template literal becomes a real newline. Use `\\n` in Chat.js script sections. Same for `\d`, `\s`, `\*` in regex patterns. 2. **Resource class naming** — naming a class `Stats` when there's a `Stats` table shadows `tables.Stats`. Always use a different name (e.g. `PublicStats`). 3. **`tables.Stats.get()` on empty DB** — returns `null`, not `{}`. Always provide a fallback: `?? { id: 'global', totalSaved: 0, cacheHits: 0 }`. -4. **Web search text blocks** — join only text blocks after the last tool block. Joining all text blocks concatenates narration with the answer. +4. **No token usage from `models.generate()`** — it returns `{ content, finishReason }` only. Token counts in `meta` are length-based estimates, not billed counts. 5. **V2 auth** — `target.checkPermission = false` is the only way to allow unauthenticated access when `loadAsInstance = false`. V1 methods (`allowRead`) are silently ignored. diff --git a/README.md b/README.md index 0b0f782..531f90d 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,16 @@ # Harper Demo Agent -A conversational AI agent with persistent semantic memory, a two-layer semantic cache, web search, cost tracking, and a browser chat UI — all running on [Harper](https://harper.fast) with Claude (via the Anthropic API or Google Cloud Vertex AI). +A conversational AI agent with persistent semantic memory, a two-layer semantic cache, cost tracking, and a browser chat UI — all running on [Harper](https://harper.fast). Generation and embedding go through Harper's `models` API, so the app ships no LLM SDK and holds no provider credentials. Live demo: **[agent-example.stephen-demo-org.harperfabric.com/Chat](https://agent-example.stephen-demo-org.harperfabric.com/Chat)** ## What It Does -- **Chat with Claude** via a REST endpoint (`POST /Agent`) or the built-in browser chat UI (`GET /Chat`) -- **Semantic cache** — two-layer cache catches repeated and rephrased questions before they reach Claude, returning answers instantly at zero LLM cost -- **Web search** — Anthropic's built-in server-side web search (`web_search_20250305`, up to 5 uses per turn); no external API key required +- **Chat** via a REST endpoint (`POST /Agent`) or the built-in browser chat UI (`GET /Chat`) +- **Semantic cache** — two-layer cache catches repeated and rephrased questions before they reach the model, returning answers instantly at zero LLM cost - **Persistent memory** — every message is embedded and stored in Harper; semantic recall surfaces relevant context from past conversations automatically -- **Local embeddings** — `bge-small-en-v1.5` runs via `harper-fabric-embeddings` (llama.cpp wrapper), entirely in-process; no embedding API key or billing -- **Per-response metadata** — every API response includes latency, token counts, cost breakdown, web searches used, and vector context stats +- **Host-provided models** — `models.generate()` and `models.embed()` resolve to whatever backend the Harper host has configured (the shared inference process on Fabric GPU hosts; Ollama / OpenAI / Anthropic / Bedrock elsewhere). No SDK dependency, no API key in the app +- **Per-response metadata** — every API response includes latency, token estimates, cost breakdown, and vector context stats - **Global savings tracker** — cache hits accumulate a running total of USD saved and hit count in a `Stats` table, displayed live in the chat sidebar - **Auto-generated REST APIs** — full CRUD on `Conversation`, `Message`, and `Stats` tables, generated from the GraphQL schema with zero route code @@ -27,55 +26,52 @@ User Query │ 1. Embed user message │ │ ┌─────────────────────┐ │ │ │ EmbeddingCache │ ← normalized text → vector │ -│ │ hit: ~1ms lookup │ miss: SLM generates it, │ -│ │ (skip SLM) │ then stores for next time │ +│ │ hit: ~1ms lookup │ miss: models.embed(), │ +│ │ (skip the model) │ then stores for next time │ │ └─────────────────────┘ │ -│ Local SLM: bge-small-en-v1.5 (llama.cpp, in-process)│ │ │ │ 2. Store user message + embedding │ -│ 3. HNSW semantic cache check (cosine distance < 0.12) │ +│ 3. HNSW semantic cache check (cosine distance < 0.15) │ │ │ │ │ │ Cache HIT Cache MISS │ │ │ │ │ -│ Return $0.00 Call Claude ──────────────────────┼──► Anthropic API -│ + saved $X │ │ + Web Search -│ Embed response (via cache/SLM) │◄──────────┘ -│ Store in Harper │ +│ Return $0.00 models.generate() ───────────────────┼──► host-configured +│ + saved $X │ │ model backend +│ Embed response (cache/embed) │◄──────────┘ +│ Store in Harper │ │ │ └──────────────────────────────────────────────────────────┘ ``` -Every request is standalone. Ask once, pay for Claude. Ask again — or rephrase the same question — and Harper serves the cached answer instantly at $0. The embedding cache eliminates the SLM cost on repeated text (~2.3s on Fabric → ~1ms). +Every request is standalone. Ask once, pay for the generation. Ask again — or rephrase the same question — and Harper serves the cached answer instantly at $0. The embedding cache eliminates the embedding round-trip on repeated text. ## How the Semantic Cache Works -Before calling Claude, the agent searches Harper's HNSW vector index for semantically similar past questions: +Before calling the model, the agent searches Harper's HNSW vector index for semantically similar past questions: ```javascript tables.Message.search({ conditions: { attribute: 'embedding', comparator: 'lt', - value: 0.12, // cosine distance < 0.12 ≡ cosine similarity ≥ 0.88 + value: 0.15, // cosine distance < 0.15 ≡ cosine similarity ≥ 0.85 target: userEmbedding, }, - limit: 10, + limit: 20, }) ``` -Harper's HNSW index evaluates the distance threshold internally — no full table scan, no in-memory cosine math. When a match is found, the agent looks up the assistant reply that followed it and returns that directly. No Claude call, no tokens, no cost. +Harper's HNSW index evaluates the distance threshold internally — no full table scan, no vector DB round-trip. The agent then ranks the returned candidates by cosine distance and takes the closest whose *immediately* following message is an assistant reply, and returns that directly. No generation call, no tokens, no cost. Cache hits return `cost.total: 0` and include a `cost.saved` field showing what the call would have cost. The saved amount is added to the global `Stats` record (`totalSaved`, `cacheHits`). ## Prerequisites - [Node.js](https://nodejs.org/) 22+ -- [Harper CLI](https://www.npmjs.com/package/harper): `npm install -g harper` -- **One of:** - - [Anthropic API key](https://console.anthropic.com/) (direct API — default) - - [Google Cloud project](https://console.cloud.google.com/) with Vertex AI enabled (GCP Vertex AI) +- [Harper](https://www.npmjs.com/package/harper) 5.2+: `npm install -g harper` +- A Harper host with an embedding backend and a generative backend configured (see [Model Configuration](#model-configuration)) -No embedding API key needed — embeddings run in-process. +No API key lives in this app — credentials, if the chosen backend needs any, belong to the host's model configuration. ## Quick Start @@ -87,69 +83,48 @@ cd agent-example-harper # Install dependencies npm install -# Configure environment +# Configure environment (deploy credentials only) cp dot-env.example .env -# Edit .env — see "LLM Provider Setup" below # Start the dev server npm run dev ``` -## LLM Provider Setup - -This agent supports two LLM backends — the direct Anthropic API and Google Cloud Vertex AI. Set `LLM_PROVIDER` in your `.env` to choose which one to use. - -### Option A: Anthropic API (default) - -The simplest path. You just need an API key from [console.anthropic.com](https://console.anthropic.com/). - -```env -LLM_PROVIDER=anthropic -ANTHROPIC_API_KEY=sk-ant-... -``` - -Web search is included automatically via Anthropic's server-side `web_search_20250305` tool — no additional API keys required. - -### Option B: Google Cloud Vertex AI - -Run Claude through your own GCP project. Useful for enterprise environments, org-level billing, data residency, and keeping everything inside Google Cloud. - -**1. Enable the Vertex AI API** in your GCP project: - -``` -https://console.developers.google.com/apis/api/aiplatform.googleapis.com/overview?project=YOUR_PROJECT_ID -``` - -**2. Enable a Claude model** in the [Vertex AI Model Garden](https://console.cloud.google.com/vertex-ai/model-garden) — search for "Claude" and enable the model you want. - -**3. Request quota** — new projects start with 0 tokens/min. Go to [IAM & Admin → Quotas](https://console.cloud.google.com/iam-admin/quotas), filter for your Claude model, and request an increase. - -**4. Create a service account** with the **Vertex AI User** role, download the JSON key, and place it in the project root. - -**5. Configure `.env`:** - -```env -LLM_PROVIDER=vertex -VERTEX_PROJECT_ID=my-gcp-project -VERTEX_REGION=us-east5 -GOOGLE_APPLICATION_CREDENTIALS=./your-service-account-key.json +## Model Configuration + +This app never names a model or holds a credential. `models.generate()` and `models.embed()` +resolve the logical names `models.generative.default` and `models.embedding.default` from the +**host's** configuration — the top-level `models:` block of `harperdb-config.yaml` at the +instance root. Change the backend there and the app is unchanged. + +```yaml +models: + embedding: + default: + backend: ollama + host: http://localhost:11434 + model: nomic-embed-text + generative: + default: + backend: anthropic + model: claude-sonnet-4-5 + apiKey: ${ANTHROPIC_API_KEY} ``` -> **Note:** Web search is not available on Vertex AI by default (requires an org policy change). The agent automatically disables it when running on Vertex. +Built-in backends: `ollama`, `openai`, `anthropic`, `bedrock`. Any other `backend` value is +resolved as a module specifier and imported, so a custom backend needs no core change. -### Environment Variable Reference +Two things worth knowing: -| Variable | Required | Default | Description | -|---|---|---|---| -| `LLM_PROVIDER` | No | `anthropic` | `anthropic` or `vertex` | -| `ANTHROPIC_API_KEY` | When `anthropic` | — | Anthropic API key | -| `VERTEX_PROJECT_ID` | When `vertex` | — | GCP project ID | -| `VERTEX_REGION` | No | `global` | Vertex AI region (e.g. `us-east5`, `global`) | -| `VERTEX_MODEL` | No | `claude-sonnet-4-6` | Vertex model ID | -| `GOOGLE_APPLICATION_CREDENTIALS` | When `vertex` | — | Path to GCP service account JSON key | -| `CLAUDE_MODEL` | No | `claude-sonnet-4-5-20250929` | Anthropic direct API model ID | +- **Keep credentials out of the YAML.** String leaves are env-expanded before they reach the + backend, so write `apiKey: ${ANTHROPIC_API_KEY}` rather than the literal key — Harper warns + at boot when it sees a literal in a credential field. +- **A misconfigured entry is logged and skipped, not fatal.** Harper still boots; the failure + surfaces on first use as `ModelBackendNotFoundError: No backend registered for + 'embedding.default'` from `POST /Agent`. -> **First run:** On startup, `bge-small-en-v1.5` (~24 MB) is auto-downloaded into `./models/`. This only happens once. +On Fabric GPU hosts the host-manager configures these entries for you against the shared +inference process, and no local setup is needed. The server starts at `http://localhost:9926`. Open `http://localhost:9926/Chat` in your browser. @@ -169,7 +144,7 @@ curl -X POST http://localhost:9926/Agent \ -d '{"message": "What is Harper?"}' ``` -Response: +Response (`models.generate()` reports no token usage, so `tokens` and `cost` are length-based estimates of what the same call would have cost on Claude Sonnet — the comparator behind the savings tracker, not a bill): ```json { @@ -178,8 +153,7 @@ Response: "meta": { "latencyMs": 1842, "tokens": { "input": 312, "output": 148, "total": 460 }, - "cost": { "input": 0.000936, "output": 0.00222, "search": 0, "total": 0.003156 }, - "webSearches": 0, + "cost": { "input": 0.000936, "output": 0.00222, "total": 0.003156, "saved": 0 }, "vectorContext": { "hit": false, "count": 0, "cached": false } } } @@ -226,18 +200,16 @@ curl "http://localhost:9926/Message?conversationId=abc-123" ## Project Structure ``` -├── config.yaml # Harper app configuration (6 lines) +├── config.yaml # Harper app configuration ├── schemas/ │ └── schema.graphql # Database schema (Conversation, Message, Stats + HNSW index) ├── resources/ -│ ├── Agent.js # POST /Agent (agent loop + semantic cache + web search) +│ ├── Agent.js # POST /Agent (agent loop + semantic cache) │ │ # GET /PublicStats/:id (public stats endpoint) │ └── Chat.js # GET /Chat (full browser chat UI served as HTML) ├── lib/ -│ ├── config.js # Environment variable helpers -│ └── embeddings.js # Local llama.cpp embeddings via harper-fabric-embeddings -├── models/ # Auto-downloaded GGUF model (gitignored) -├── .env.example # Environment template +│ └── embeddings.js # Thin wrapper over `models.embed()` +├── dot-env.example # Environment template (deploy credentials) └── package.json ``` @@ -297,7 +269,7 @@ Rolling restarts and replication are handled automatically. | Semantic cache | Redis + custom logic | Built in (native HNSW threshold filter) | | API server | Express / Fastify | Auto-generated from schema | | Chat UI server | Vite / Next.js | Resource returning `Response(html)` | -| Embeddings | Voyage / OpenAI API | Local via `harper-fabric-embeddings` (24 MB, in-process) | +| Model access | Per-provider SDK + key per app | `models.embed()` / `models.generate()`, backend configured on the host | | Deployment | Docker + K8s + cloud | `harper deploy .` | **Key insights from building this:** @@ -306,7 +278,7 @@ Rolling restarts and replication are handled automatically. - **Everything in one process means no network hops.** Database, vector index, cache, API, and agent code share the same runtime. No Redis round-trip, no vector DB round-trip. - **The schema is the only config you need.** One `@indexed(type: "HNSW", distance: "cosine")` directive creates the vector index. One `@export` generates the CRUD API. One `@indexed` on `conversationId` creates the secondary index. - **Resources can return anything.** A `Resource` subclass can return a `Response` with any content type — JSON, HTML, plain text. The chat UI lives in the same project and deploy as the agent logic. -- **Local embeddings eliminate a cost center.** `bge-small-en-v1.5` runs in-process via llama.cpp. No per-embedding billing, no embedding service SLA to worry about. +- **The model backend is host configuration, not app code.** `import { models } from 'harper'` gives a resource the process-wide models singleton — the same object as `scope.models`, so no `handleApplication(scope)` shim is needed. Swapping Ollama for Bedrock is a YAML edit on the host; this app does not change. ## License diff --git a/config.yaml b/config.yaml index 861fdcc..d6be5d1 100644 --- a/config.yaml +++ b/config.yaml @@ -6,10 +6,6 @@ loadEnv: rest: true -# Captures the Scope object so resources can call scope.models.embed(). -# See lib/modelCapture.js. -extensionModule: 'lib/modelCapture.js' - graphqlSchema: files: 'schemas/*.graphql' diff --git a/dot-env.example b/dot-env.example index 47133be..6591575 100644 --- a/dot-env.example +++ b/dot-env.example @@ -1,14 +1,7 @@ -#MUST USE ANTHROPIC OR GOOGLE VERTEX AI - -# Anthropic (Optional) -ANTHROPIC_API_KEY=sk-ant-your-key-here -CLAUDE_MODEL=claude-sonnet-4-6 - -#Google Vertex AI (optional) -#LLM_PROVIDER=vertex -#VERTEX_PROJECT_ID=your-gcp-project -#VERTEX_REGION=global - +# The agent holds no model credentials. Generation and embedding go through +# Harper's `models` API, which resolves `models.generative.default` and +# `models.embedding.default` from the HOST's configuration (the `models:` block +# of harperdb-config.yaml, or the equivalent env vars) — not from this file. # Harper Fabric deployment (optional, only needed for deploy) # CLI_TARGET=https://your-instance.your-org.harperfabric.com:9925/ diff --git a/integrationTests/app.test.ts b/integrationTests/app.test.ts index 9e4e1e4..9d386a6 100644 --- a/integrationTests/app.test.ts +++ b/integrationTests/app.test.ts @@ -1,8 +1,8 @@ /** * Integration tests for agent-example-harper. * Tests that the app starts and key endpoints respond correctly. - * Note: Full agent functionality requires Anthropic/Vertex API keys - * which are not available in CI, so we only test structural correctness. + * Note: Full agent functionality requires a configured `models` backend, which is not + * available in CI, so we only test structural correctness. */ import { suite, test, before, after } from 'node:test'; import { strictEqual, ok } from 'node:assert/strict'; @@ -66,6 +66,21 @@ void suite('agent-example-harper loads', (ctx: ContextWithHarper) => { ok([200, 404].includes(res.status), `unexpected status ${res.status}`); }); + // Guards the `import { models } from 'harper'` path: the resource must reach the + // models layer rather than fail on a missing global. CI configures no backend, so + // the expected outcome is the models layer's own "no backend" error, not a + // ReferenceError or a read of `undefined`. + void test('POST /Agent reaches the models layer', async () => { + const res = await authFetch(ctx, '/Agent', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message: 'hello there' }), + }); + if (res.status === 200) return; + const body = await res.json(); + strictEqual(body.code, 'ModelBackendNotFoundError', `unexpected error body ${JSON.stringify(body)}`); + }); + void test('POST /Conversation/ creates a conversation record', async () => { const res = await authFetch(ctx, '/Conversation/', { method: 'POST', diff --git a/lib/embeddings.js b/lib/embeddings.js index bddf169..69010ab 100644 --- a/lib/embeddings.js +++ b/lib/embeddings.js @@ -1,14 +1,11 @@ -// Thin wrapper around `scope.models.embed()` (harper#510). The host's -// configured backend (Ollama on Fabric GPU hosts, or any backend configured -// via the `models:` block in harperdb-config.yaml / env vars) handles the -// actual inference. Returns a plain Array for compatibility with -// Harper's HNSW vector index storage. +import { models } from 'harper'; + +// `models.embed()` (harper#510) dispatches to whatever embedding backend the host +// has configured — Ollama on Fabric GPU hosts, or any backend named in the +// `models:` block of harperdb-config.yaml. Returns a plain Array because +// Harper's HNSW index stores arrays, not the Float32Array the API returns. export async function embed(text) { - const scope = globalThis.harperScope; - if (!scope) { - throw new Error('Harper scope not yet captured — modelCapture plugin must run before first embed call'); - } - const [vector] = await scope.models.embed(text); + const [vector] = await models.embed(text); return Array.from(vector); } diff --git a/lib/modelCapture.js b/lib/modelCapture.js deleted file mode 100644 index 018348e..0000000 --- a/lib/modelCapture.js +++ /dev/null @@ -1,11 +0,0 @@ -// Plugin entry — captures the Scope object so `resources/*.js` can call -// `scope.models.embed()` against the host's configured embedding backend. -// -// Harper's `scope` is passed to plugins via `handleApplication(scope)` but -// isn't exposed as a global to Resource classes. This tiny plugin stashes the -// Scope on `globalThis.harperScope` at app boot, then `lib/embeddings.js` -// reads it from there. - -export function handleApplication(scope) { - globalThis.harperScope = scope; -} diff --git a/package.json b/package.json index 15c4535..d1a35b6 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "agent-example-harper", "version": "1.0.0", - "description": "A conversational AI agent with persistent memory, built on Harper with scope.models", + "description": "A conversational AI agent with persistent memory, built on Harper with the harper models API", "type": "module", "engines": { - "harper": "^5.0" + "harper": "^5.2" }, "scripts": { "start": "npx -y dotenv-cli -e .env -o -- harper run .", diff --git a/resources/Agent.js b/resources/Agent.js index db61e01..9c4ef31 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -1,4 +1,4 @@ -import { Resource, tables } from 'harper' +import { models, Resource, tables } from 'harper' import { embed } from '../lib/embeddings.js' const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the user's current question. \ @@ -12,7 +12,7 @@ as background knowledge only if it is directly relevant. Never recite or recap p const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens -// scope.models.generate() returns only { content, finishReason } today — the +// models.generate() returns only { content, finishReason } today — the // backend's token usage isn't surfaced to callers. Approximate with the // ~4-chars-per-token rule of thumb for English; close enough for a comparator. const estimateTokens = (text) => Math.max(1, Math.ceil((text?.length ?? 0) / 4)) @@ -179,15 +179,10 @@ export class Agent extends Resource { } } - // 5. Generate via scope.models.generate() — routes to whatever backend the host - // has configured for `models.generative.default` (vLLM on Fabric GPU hosts, - // Ollama / OpenAI / Anthropic on other deployments). - const scope = globalThis.harperScope - if (!scope) { - throw new Error('Harper scope not yet captured — modelCapture plugin must run before first generate call') - } - - const result = await scope.models.generate( + // 5. Generate via models.generate() — routes to whatever backend the host + // has configured for `models.generative.default` (the shared inference process + // on Fabric GPU hosts, Ollama / OpenAI / Anthropic / Bedrock elsewhere). + const result = await models.generate( { messages: [{ role: 'user', content: message }], system: SYSTEM_PROMPT, diff --git a/resources/Chat.js b/resources/Chat.js index f5f08eb..c75cadb 100644 --- a/resources/Chat.js +++ b/resources/Chat.js @@ -351,7 +351,7 @@ const HTML = /* html */ ` Shared model · nomic-embed-text - scope.models.embed() · GPU on Fabric · no API cost + models.embed() · GPU on Fabric · no API cost From a276022059c7ceee011627dc590f56b7c729c2dc Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 25 Aug 2026 09:16:20 -0600 Subject: [PATCH 10/13] Address pre-push review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `resources/Chat.js`: the architecture diagram and the metadata pills still described the removed Anthropic/web-search/local-SLM path on the live demo. The `webSearches` pill was dead code (`meta.webSearches` is no longer returned); token and cost pills are now marked `est.`. - `resources/Agent.js`: cache keys are a digest of the normalized text. Harper rejects a primary key over ~1978 bytes, so an assistant reply longer than that threw *after* the generation had been paid for and before `Message.put`, losing the answer. Reply embeddings now skip the cache entirely — unique text is a guaranteed miss whose write nothing reads. - `resources/Agent.js`: `cosineDistance` returns maximum distance for vectors of unequal length instead of scoring a prefix and yielding NaN, which silently dropped candidates when the host's embedding backend changed under stored vectors. - `resources/Agent.js`: an empty or `length`-truncated generation is no longer persisted; it would be served to every near-miss question thereafter. - `resources/Agent.js`: a candidate missing from the conversation scan (`findIndex` → -1) is skipped rather than resolving to index 0. - `schemas/schema.graphql`: `EmbeddingCache` gets the same 1-hour expiration as the other tables. It was the only unbounded table behind a public unauthenticated endpoint. - `package-lock.json`: root `engines` still recorded `^5.0`. - `CLAUDE.md`: the "never do manual cosine similarity in JS" invariant now matches what the code does, and names the HNSW `lt` behaviour to confirm. Co-Authored-By: Claude Opus --- CLAUDE.md | 2 +- integrationTests/app.test.ts | 6 ++-- lib/embeddings.js | 6 ++-- package-lock.json | 2 +- resources/Agent.js | 68 +++++++++++++++++++++--------------- resources/Chat.js | 24 +++++-------- schemas/schema.graphql | 2 +- 7 files changed, 55 insertions(+), 55 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d93d080..16c2cd4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ lib/embeddings.js # Thin wrapper over `models.embed()` ### Semantic Cache (two layers) 1. **Layer 1 — Exact match:** Normalize text (lowercase, strip punctuation, collapse whitespace) and compare against conversation history. No DB query needed. -2. **Layer 2 — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.15` (cosine distance). **Never do manual cosine similarity in JS** — always use Harper's native HNSW index for distance filtering. +2. **Layer 2 — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.15` (cosine distance). **Never scan a table and score it in JS** — the index does the filtering. `resources/Agent.js` does recompute cosine distance, but only over the ≤20 rows the index already returned, to rank them (HNSW iteration is not distance-ordered) and to re-check the bound; matches outside `lt` have been observed to survive it, which is worth confirming against harper core rather than leaving as app-side compensation. ### Vector Context (for LLM prompt) - Uses `sort: { attribute: 'embedding', target: userEmbedding }` with `limit: 10` — returns top 10 most similar messages diff --git a/integrationTests/app.test.ts b/integrationTests/app.test.ts index 9d386a6..b1fa2a4 100644 --- a/integrationTests/app.test.ts +++ b/integrationTests/app.test.ts @@ -66,10 +66,8 @@ void suite('agent-example-harper loads', (ctx: ContextWithHarper) => { ok([200, 404].includes(res.status), `unexpected status ${res.status}`); }); - // Guards the `import { models } from 'harper'` path: the resource must reach the - // models layer rather than fail on a missing global. CI configures no backend, so - // the expected outcome is the models layer's own "no backend" error, not a - // ReferenceError or a read of `undefined`. + // Guards `import { models } from 'harper'`: with no backend configured the request must + // fail in the models layer, not on an undefined import. void test('POST /Agent reaches the models layer', async () => { const res = await authFetch(ctx, '/Agent', { method: 'POST', diff --git a/lib/embeddings.js b/lib/embeddings.js index 69010ab..75ca762 100644 --- a/lib/embeddings.js +++ b/lib/embeddings.js @@ -1,9 +1,7 @@ import { models } from 'harper'; -// `models.embed()` (harper#510) dispatches to whatever embedding backend the host -// has configured — Ollama on Fabric GPU hosts, or any backend named in the -// `models:` block of harperdb-config.yaml. Returns a plain Array because -// Harper's HNSW index stores arrays, not the Float32Array the API returns. +// `models.embed()` (harper#510) dispatches to the host's configured embedding backend. +// Harper's HNSW index stores arrays, so the returned Float32Array is converted. export async function embed(text) { const [vector] = await models.embed(text); diff --git a/package-lock.json b/package-lock.json index 6deb47b..cf0c565 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@types/node": "^22.19.19" }, "engines": { - "harper": "^5.0" + "harper": "^5.2" } }, "node_modules/@agoric/babel-generator": { diff --git a/resources/Agent.js b/resources/Agent.js index 9c4ef31..2cd6ecb 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { models, Resource, tables } from 'harper' import { embed } from '../lib/embeddings.js' @@ -5,25 +6,24 @@ const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the use Do NOT summarize, repeat, or reference prior conversation context in your response — use it silently \ as background knowledge only if it is directly relevant. Never recite or recap previous answers.` -// Hypothetical Claude Sonnet 4.5 pricing used to estimate what each generation -// WOULD have cost if we'd called Anthropic instead of the local GPU. Real local -// compute cost is roughly $0 (sunk-cost GPU); the dashboard shows what we're -// saving by self-hosting + caching. +// The savings tracker compares against list-price Claude Sonnet 4.5. `models.generate()` +// reports no token usage, so both the counts and the dollars below are estimates from text +// length, not measurements — everything derived from them is labelled `est.` in the UI. const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens -// models.generate() returns only { content, finishReason } today — the -// backend's token usage isn't surfaced to callers. Approximate with the -// ~4-chars-per-token rule of thumb for English; close enough for a comparator. const estimateTokens = (text) => Math.max(1, Math.ceil((text?.length ?? 0) / 4)) const estimateClaudeCost = (promptTokens, completionTokens) => promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN -// Normalize text for embedding cache key — lowercase, strip punctuation, collapse whitespace const normalize = (s) => s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() +// Harper rejects a primary key over ~1978 bytes, and message text is unbounded, so the +// cache is keyed by a digest of the normalized text rather than the text itself. +const cacheKey = (text) => createHash('sha256').update(normalize(text)).digest('base64url') + // Cosine distance threshold for Harper's native HNSW vector search. // Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.15 ≈ cosine // similarity 0.85 — loose enough to catch rewordings and related phrasings @@ -31,9 +31,11 @@ const normalize = (s) => // that the matched reply is reasonably on-topic. const CACHE_DISTANCE_THRESHOLD = 0.15 -// HNSW search returns matches that satisfy the threshold but doesn't guarantee -// distance-ordered iteration. We compute distance ourselves and pick the closest. +// Vectors of differing length are not comparable — that happens when the host's embedding +// backend changes under stored vectors — so report maximum distance rather than silently +// comparing a prefix. function cosineDistance(a, b) { + if (a.length !== b.length) return 2 let dot = 0, na = 0, nb = 0 for (let i = 0; i < a.length; i++) { dot += a[i] * b[i] @@ -44,9 +46,9 @@ function cosineDistance(a, b) { return denom === 0 ? 1 : 1 - dot / denom } -// Get or compute an embedding, using Harper as a cache to skip the model on repeated text. +// Cache the embedding of repeated text so a rephrase-free repeat skips the backend call. async function cachedEmbed(text) { - const key = normalize(text) + const key = cacheKey(text) const cached = await tables.EmbeddingCache.get(key) if (cached?.embedding) return cached.embedding const embedding = await embed(text) @@ -99,9 +101,6 @@ export class Agent extends Resource { const tStore = Date.now() - t3 // 4. Semantic cache — Harper-native HNSW vector search with distance threshold. - // HNSW search returns matches under the threshold but iteration order isn't - // guaranteed to be distance-ascending, so we collect candidates, compute - // cosine distance ourselves, and pick the closest valid one. const t4 = Date.now() let cachedReply = null const nearbyMsgs = tables.Message.search({ @@ -121,8 +120,8 @@ export class Agent extends Resource { } candidates.sort((a, b) => a.distance - b.distance) - // Harper's HNSW `lt` filter doesn't always cull matches outside the threshold, - // so we apply a hard check using the distance we computed ourselves. + // HNSW iteration is not distance-ordered, and matches outside the `lt` bound have been + // observed to survive it, so rank and re-check against the distance computed here. const filtered = candidates.filter((c) => c.distance <= CACHE_DISTANCE_THRESHOLD) for (const { match } of filtered) { @@ -134,11 +133,10 @@ export class Agent extends Resource { for await (const m of matchHistory) matchConvMsgs.push(m) matchConvMsgs.sort((a, b) => a.createdAt.localeCompare(b.createdAt)) const midx = matchConvMsgs.findIndex((m) => m.id === match.id) - // The matched message's reply must be the IMMEDIATELY following message. - // `.find()` would walk past any subsequent user-msgs (cache hits that didn't - // generate a reply) and pull an unrelated answer from much later in the - // conversation — e.g. matching "is soccer fun" but returning the assistant - // reply to a later "what is 2 plus 3" question in the same conversation. + if (midx === -1) continue + // Only the IMMEDIATELY following message is this question's answer: scanning forward + // would skip past later user messages that produced no reply of their own and return + // an answer to a different question. const next = matchConvMsgs[midx + 1] if (next?.role === 'assistant') { cachedReply = next @@ -150,9 +148,8 @@ export class Agent extends Resource { const timing = { embedMs: tEmbed, convMs: tConv, storeMs: tStore, cacheSearchMs: tCache } console.log('[Agent] timing:', JSON.stringify(timing)) - // Return the cached answer — zero LLM call. We credit the original message's - // estimated cost to `totalSaved` so the dashboard shows the running benefit - // of the semantic cache (and self-hosting more broadly). + // Cache hit: no generation call. Credit the matched message's estimated cost to the + // running `totalSaved` the dashboard shows. if (cachedReply) { let savedCost = 0 try { @@ -192,15 +189,28 @@ export class Agent extends Resource { const latencyMs = Date.now() - startTime const assistantContent = result.content?.trim() ?? '' + // An empty or truncated reply must not be persisted: the semantic cache would serve it + // to every near-miss question from here on, and `Message` rows have no repair path. + if (!assistantContent) { + const err = new Error(`Model returned no content (finishReason: ${result.finishReason})`) + err.statusCode = 502 + throw err + } + if (result.finishReason === 'length') { + const err = new Error('Model response was truncated at maxTokens; not caching a partial answer') + err.statusCode = 502 + throw err + } const promptTokens = estimateTokens(SYSTEM_PROMPT + message) const completionTokens = estimateTokens(assistantContent) const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) - // 9. Store the assistant's response with its embedding. We persist the - // *hypothetical* Claude cost so a future cache-hit on this same message - // can credit that amount to `totalSaved`. + // 9. Store the assistant's response with its embedding. The estimated cost rides along + // so a later cache hit on this question can credit it to `totalSaved`. const assistantMsgId = crypto.randomUUID() - const assistantEmbedding = await cachedEmbed(assistantContent) + // Not `cachedEmbed`: a generated reply is unique text, so the cache lookup is a + // guaranteed miss and the write is a large row nothing will ever read. + const assistantEmbedding = await embed(assistantContent) await tables.Message.put({ id: assistantMsgId, conversationId, diff --git a/resources/Chat.js b/resources/Chat.js index c75cadb..a965ac9 100644 --- a/resources/Chat.js +++ b/resources/Chat.js @@ -195,8 +195,6 @@ const HTML = /* html */ ` .meta .pill.vector-miss .label { color: var(--muted); } .meta .pill.cache-hit { color: var(--btree-green); font-weight: 500; } .meta .pill.cache-hit .label { opacity: 1; } - .meta .pill.web-search { color: #a78bfa; } - .meta .pill.web-search .label { color: #a78bfa; opacity: 1; } /* ── Typing indicator ───────────────────────────────── */ .typing-wrap { align-self: flex-start; } @@ -330,7 +328,7 @@ const HTML = /* html */ ` Semantic Cache - cosine sim >= 0.88 + cosine sim >= 0.85 instant answers $0 LLM cost @@ -348,23 +346,23 @@ const HTML = /* html */ ` Response $0.00 • <50ms - + Shared model · nomic-embed-text models.embed() · GPU on Fabric · no API cost - + - + external - Claude Sonnet · Web Search - Anthropic API · response embedded by local SLM + models.generate() + host-configured backend · reply embedded in Harper - + embed & store in Harper @@ -462,14 +460,10 @@ const HTML = /* html */ ` const vecLabel = vectorContext.hit ? vectorContext.count + ' memor' + (vectorContext.count === 1 ? 'y' : 'ies') + ' recalled from Harper' : 'No vector context — LLM knowledge only' - const searchPill = (meta.webSearches > 0) - ? 'Web' + meta.webSearches + ' search' + (meta.webSearches > 1 ? 'es' : '') + '' - : '' div.innerHTML = 'Latency' + latency + '' + - 'Tokens' + tok + '' + - 'Cost' + usd + '' + - searchPill + + 'Tokens' + tok + ' (est.)' + + 'Cost' + usd + ' (est.)' + 'Vector' + vecLabel + '' } return div diff --git a/schemas/schema.graphql b/schemas/schema.graphql index 0eeade9..9b7e3d1 100644 --- a/schemas/schema.graphql +++ b/schemas/schema.graphql @@ -15,7 +15,7 @@ type Message @table(expiration: 3600) @export { createdAt: String } -type EmbeddingCache @table @export { +type EmbeddingCache @table(expiration: 3600) @export { id: ID @primaryKey embedding: [Float] } From 0c159ea6d07b87fa85a1fe4478cb252b9ae32275 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 25 Aug 2026 09:39:29 -0600 Subject: [PATCH 11/13] Address second-round review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `resources/Agent.js`: `normalize()` used `\w`, which is ASCII-only, so every message in a non-Latin script collapsed to the empty string and shared one cache key — a user asking a question in Japanese got back the vector cached for someone else's unrelated Japanese question, which was then indexed as their own. Now `\p{L}\p{N}` with the `u` flag. - `resources/Agent.js`: a truncated or content-filtered generation is returned to the caller (rejecting it outright made every long answer a hard failure) but not persisted, so it can never be served as a complete answer from the semantic cache. `meta.finishReason` exposes which happened. - `integrationTests/app.test.ts`: the success branch returned without asserting, so on a host that does have backends configured the test could not fail for the reason it exists. - `CLAUDE.md`: dropped the "Layer 1 exact match" and "Vector Context" sections, which described code that no longer exists, and described what the embedding cache actually is. Co-Authored-By: Claude Opus --- CLAUDE.md | 11 +++------ integrationTests/app.test.ts | 8 +++++- resources/Agent.js | 48 +++++++++++++++++++----------------- 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 16c2cd4..ab5220c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,14 +50,9 @@ lib/embeddings.js # Thin wrapper over `models.embed()` - `@indexed` on `conversationId` — secondary index for conversation lookups - `Stats` table has no TTL (cumulative savings persist indefinitely) -### Semantic Cache (two layers) -1. **Layer 1 — Exact match:** Normalize text (lowercase, strip punctuation, collapse whitespace) and compare against conversation history. No DB query needed. -2. **Layer 2 — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.15` (cosine distance). **Never scan a table and score it in JS** — the index does the filtering. `resources/Agent.js` does recompute cosine distance, but only over the ≤20 rows the index already returned, to rank them (HNSW iteration is not distance-ordered) and to re-check the bound; matches outside `lt` have been observed to survive it, which is worth confirming against harper core rather than leaving as app-side compensation. - -### Vector Context (for LLM prompt) -- Uses `sort: { attribute: 'embedding', target: userEmbedding }` with `limit: 10` — returns top 10 most similar messages -- Top 5 injected into system prompt as silent background context -- System prompt explicitly tells Claude NOT to repeat/summarize context in responses +### Semantic Cache +1. **Embedding cache (`EmbeddingCache`):** keyed by a SHA-256 digest of the normalized text (lowercased, punctuation stripped, whitespace collapsed) — a digest because Harper rejects a primary key over ~1978 bytes. Skips the embedding backend on exactly repeated text; it is not an answer cache. +2. **Answer cache — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.15` (cosine distance). **Never scan a table and score it in JS** — the index does the filtering. `resources/Agent.js` does recompute cosine distance, but only over the ≤20 rows the index already returned, to rank them (HNSW iteration is not distance-ordered) and to re-check the bound; matches outside `lt` have been observed to survive it, which is worth confirming against harper core rather than leaving as app-side compensation. ### Models API access - `import { models } from 'harper'` — `models` is a process-wide singleton, exported by the `harper` package and also available as a bare global. It is the same object as `scope.models`, so a `handleApplication(scope)` plugin that stashes the Scope on `globalThis` is not needed. diff --git a/integrationTests/app.test.ts b/integrationTests/app.test.ts index b1fa2a4..e6b91be 100644 --- a/integrationTests/app.test.ts +++ b/integrationTests/app.test.ts @@ -74,8 +74,14 @@ void suite('agent-example-harper loads', (ctx: ContextWithHarper) => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ message: 'hello there' }), }); - if (res.status === 200) return; const body = await res.json(); + if (res.status === 200) { + // A host with backends configured: assert the success shape rather than passing vacuously. + strictEqual(typeof body.conversationId, 'string'); + strictEqual(body.message.role, 'assistant'); + ok(body.message.content.length > 0, 'assistant content must be non-empty'); + return; + } strictEqual(body.code, 'ModelBackendNotFoundError', `unexpected error body ${JSON.stringify(body)}`); }); diff --git a/resources/Agent.js b/resources/Agent.js index 2cd6ecb..cd258be 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -17,8 +17,11 @@ const estimateTokens = (text) => Math.max(1, Math.ceil((text?.length ?? 0) / 4)) const estimateClaudeCost = (promptTokens, completionTokens) => promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN +// `\p{L}\p{N}` rather than `\w`: `\w` is ASCII-only, so every message written in a +// non-Latin script normalized to the empty string and shared one cache key with all the +// others — returning another user's vector, which then got indexed as this message's. const normalize = (s) => - s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() + s.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, '').replace(/\s+/g, ' ').trim() // Harper rejects a primary key over ~1978 bytes, and message text is unbounded, so the // cache is keyed by a digest of the normalized text rather than the text itself. @@ -189,37 +192,35 @@ export class Agent extends Resource { const latencyMs = Date.now() - startTime const assistantContent = result.content?.trim() ?? '' - // An empty or truncated reply must not be persisted: the semantic cache would serve it - // to every near-miss question from here on, and `Message` rows have no repair path. if (!assistantContent) { const err = new Error(`Model returned no content (finishReason: ${result.finishReason})`) err.statusCode = 502 throw err } - if (result.finishReason === 'length') { - const err = new Error('Model response was truncated at maxTokens; not caching a partial answer') - err.statusCode = 502 - throw err - } + // A truncated or filtered answer is still worth returning, but persisting it would seed + // the semantic cache: every near-miss question from here on would be served the partial + // text as a complete answer, and `Message` rows have no repair path short of the TTL. + const isComplete = result.finishReason === 'stop' const promptTokens = estimateTokens(SYSTEM_PROMPT + message) const completionTokens = estimateTokens(assistantContent) const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) - // 9. Store the assistant's response with its embedding. The estimated cost rides along - // so a later cache hit on this question can credit it to `totalSaved`. - const assistantMsgId = crypto.randomUUID() - // Not `cachedEmbed`: a generated reply is unique text, so the cache lookup is a - // guaranteed miss and the write is a large row nothing will ever read. - const assistantEmbedding = await embed(assistantContent) - await tables.Message.put({ - id: assistantMsgId, - conversationId, - role: 'assistant', - content: assistantContent, - cost: estimatedCost, - embedding: assistantEmbedding, - createdAt: new Date().toISOString(), - }) + // 9. Store the assistant's response. The estimated cost rides along so a later cache hit + // on this question can credit it to `totalSaved`. + if (isComplete) { + // Not `cachedEmbed`: a generated reply is unique text, so the lookup is a guaranteed + // miss and the write is a large row nothing will ever read. + const assistantEmbedding = await embed(assistantContent) + await tables.Message.put({ + id: crypto.randomUUID(), + conversationId, + role: 'assistant', + content: assistantContent, + cost: estimatedCost, + embedding: assistantEmbedding, + createdAt: new Date().toISOString(), + }) + } // 10. Update conversation timestamp await tables.Conversation.put({ @@ -246,6 +247,7 @@ export class Agent extends Resource { saved: 0, }, vectorContext: { hit: false, count: 0, cached: false }, + finishReason: result.finishReason, }, } } From 78d41332af06c102d341ceb7399c7dbffbd8c4c8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 25 Aug 2026 09:58:47 -0600 Subject: [PATCH 12/13] Use the backend's reported token usage; keep the cache key identity-preserving - `models.generate()` does pass the backend's `usage` through (harper's `GenerateResult.usage`, injected by `Models.generate` from the backend's `ModelCallResult.usage`). Token counts now come from it, falling back to the length estimate only when a backend reports none; `meta.tokensAreMeasured` says which, and the UI marks only estimates. The previous comments and docs asserted the field did not exist. - `normalize()` no longer strips punctuation. The digest it feeds is an embedding-cache key, so it must preserve identity: `What is C++?` and `What is C#?` shared a key, so the second asker received the first's vector and it was indexed as their own message. - Explained the JS distance re-check rather than attributing it to an HNSW defect: core's cosine helper zero-pads to the longer vector (`resources/indexes/vector.ts`) instead of rejecting a length mismatch, so a row left from a different embedding backend can pass a `lt` bound. Co-Authored-By: Claude Opus --- CLAUDE.md | 6 +++--- README.md | 6 ++++-- resources/Agent.js | 30 ++++++++++++++++-------------- resources/Chat.js | 2 +- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ab5220c..835233c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,12 +51,12 @@ lib/embeddings.js # Thin wrapper over `models.embed()` - `Stats` table has no TTL (cumulative savings persist indefinitely) ### Semantic Cache -1. **Embedding cache (`EmbeddingCache`):** keyed by a SHA-256 digest of the normalized text (lowercased, punctuation stripped, whitespace collapsed) — a digest because Harper rejects a primary key over ~1978 bytes. Skips the embedding backend on exactly repeated text; it is not an answer cache. +1. **Embedding cache (`EmbeddingCache`):** keyed by a SHA-256 digest of the text with case and whitespace normalized (punctuation is preserved — the key selects a vector, so it must preserve identity) — a digest because Harper rejects a primary key over ~1978 bytes. Skips the embedding backend on exactly repeated text; it is not an answer cache. 2. **Answer cache — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.15` (cosine distance). **Never scan a table and score it in JS** — the index does the filtering. `resources/Agent.js` does recompute cosine distance, but only over the ≤20 rows the index already returned, to rank them (HNSW iteration is not distance-ordered) and to re-check the bound; matches outside `lt` have been observed to survive it, which is worth confirming against harper core rather than leaving as app-side compensation. ### Models API access - `import { models } from 'harper'` — `models` is a process-wide singleton, exported by the `harper` package and also available as a bare global. It is the same object as `scope.models`, so a `handleApplication(scope)` plugin that stashes the Scope on `globalThis` is not needed. -- `models.generate()` currently returns `{ content, finishReason }` only — no token usage — so `resources/Agent.js` estimates token counts from text length for the cost comparator. +- `models.generate()` returns `usage` (`promptTokens` / `completionTokens`) passed through from the backend, but the field is optional — `resources/Agent.js` falls back to a length estimate and reports which it used as `meta.tokensAreMeasured`. ### Chat UI (resources/Chat.js) - Full HTML/CSS/JS served from a single template literal via `new Response(HTML, ...)` @@ -109,5 +109,5 @@ curl http://localhost:9926/PublicStats/global 1. **Template literal backslashes** — `\n` inside a JS template literal becomes a real newline. Use `\\n` in Chat.js script sections. Same for `\d`, `\s`, `\*` in regex patterns. 2. **Resource class naming** — naming a class `Stats` when there's a `Stats` table shadows `tables.Stats`. Always use a different name (e.g. `PublicStats`). 3. **`tables.Stats.get()` on empty DB** — returns `null`, not `{}`. Always provide a fallback: `?? { id: 'global', totalSaved: 0, cacheHits: 0 }`. -4. **No token usage from `models.generate()`** — it returns `{ content, finishReason }` only. Token counts in `meta` are length-based estimates, not billed counts. +4. **`models.generate().usage` is optional** — a backend that reports none leaves it undefined, so `meta.tokens` may be a length estimate. `meta.tokensAreMeasured` says which. Dollar amounts are always list-price Claude Sonnet, never the backend's real cost. 5. **V2 auth** — `target.checkPermission = false` is the only way to allow unauthenticated access when `loadAsInstance = false`. V1 methods (`allowRead`) are silently ignored. diff --git a/README.md b/README.md index 531f90d..5d6e239 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ curl -X POST http://localhost:9926/Agent \ -d '{"message": "What is Harper?"}' ``` -Response (`models.generate()` reports no token usage, so `tokens` and `cost` are length-based estimates of what the same call would have cost on Claude Sonnet — the comparator behind the savings tracker, not a bill): +Response. `tokens` comes from the backend's reported usage when it reports any, and falls back to a length estimate otherwise — `meta.tokensAreMeasured` says which. `cost` is always what the same call would have cost at list-price Claude Sonnet, the comparator behind the savings tracker, never the backend's real cost: ```json { @@ -154,7 +154,9 @@ Response (`models.generate()` reports no token usage, so `tokens` and `cost` are "latencyMs": 1842, "tokens": { "input": 312, "output": 148, "total": 460 }, "cost": { "input": 0.000936, "output": 0.00222, "total": 0.003156, "saved": 0 }, - "vectorContext": { "hit": false, "count": 0, "cached": false } + "vectorContext": { "hit": false, "count": 0, "cached": false }, + "finishReason": "stop", + "tokensAreMeasured": true } } ``` diff --git a/resources/Agent.js b/resources/Agent.js index cd258be..365642e 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -6,25 +6,24 @@ const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the use Do NOT summarize, repeat, or reference prior conversation context in your response — use it silently \ as background knowledge only if it is directly relevant. Never recite or recap previous answers.` -// The savings tracker compares against list-price Claude Sonnet 4.5. `models.generate()` -// reports no token usage, so both the counts and the dollars below are estimates from text -// length, not measurements — everything derived from them is labelled `est.` in the UI. +// The savings tracker prices every generation at list-price Claude Sonnet 4.5 whichever +// backend actually ran it, so the dollars are a comparator, never a bill. const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens +// Fallback only: `models.generate()` passes the backend's token usage through, but the +// field is optional and a backend that reports none leaves it undefined. const estimateTokens = (text) => Math.max(1, Math.ceil((text?.length ?? 0) / 4)) const estimateClaudeCost = (promptTokens, completionTokens) => promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN -// `\p{L}\p{N}` rather than `\w`: `\w` is ASCII-only, so every message written in a -// non-Latin script normalized to the empty string and shared one cache key with all the -// others — returning another user's vector, which then got indexed as this message's. -const normalize = (s) => - s.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, '').replace(/\s+/g, ' ').trim() +// Case and whitespace only. This key selects a stored *vector*, so it has to preserve +// identity: stripping punctuation collided `What is C++?` with `What is C#?`, handing the +// second asker the first one's embedding to be indexed as their own message. +const normalize = (s) => s.toLowerCase().replace(/\s+/g, ' ').trim() -// Harper rejects a primary key over ~1978 bytes, and message text is unbounded, so the -// cache is keyed by a digest of the normalized text rather than the text itself. +// Harper rejects a primary key over ~1978 bytes and message text is unbounded. const cacheKey = (text) => createHash('sha256').update(normalize(text)).digest('base64url') // Cosine distance threshold for Harper's native HNSW vector search. @@ -123,8 +122,9 @@ export class Agent extends Resource { } candidates.sort((a, b) => a.distance - b.distance) - // HNSW iteration is not distance-ordered, and matches outside the `lt` bound have been - // observed to survive it, so rank and re-check against the distance computed here. + // HNSW iteration is not distance-ordered, so rank here. The re-check is not redundant: + // core's cosine helper zero-pads to the longer vector rather than rejecting a length + // mismatch, so a row left from a different embedding backend can pass `lt`. const filtered = candidates.filter((c) => c.distance <= CACHE_DISTANCE_THRESHOLD) for (const { match } of filtered) { @@ -201,8 +201,9 @@ export class Agent extends Resource { // the semantic cache: every near-miss question from here on would be served the partial // text as a complete answer, and `Message` rows have no repair path short of the TTL. const isComplete = result.finishReason === 'stop' - const promptTokens = estimateTokens(SYSTEM_PROMPT + message) - const completionTokens = estimateTokens(assistantContent) + const promptTokens = result.usage?.promptTokens ?? estimateTokens(SYSTEM_PROMPT + message) + const completionTokens = result.usage?.completionTokens ?? estimateTokens(assistantContent) + const tokensAreMeasured = result.usage?.promptTokens !== undefined const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) // 9. Store the assistant's response. The estimated cost rides along so a later cache hit @@ -248,6 +249,7 @@ export class Agent extends Resource { }, vectorContext: { hit: false, count: 0, cached: false }, finishReason: result.finishReason, + tokensAreMeasured, }, } } diff --git a/resources/Chat.js b/resources/Chat.js index a965ac9..2fd05eb 100644 --- a/resources/Chat.js +++ b/resources/Chat.js @@ -462,7 +462,7 @@ const HTML = /* html */ ` : 'No vector context — LLM knowledge only' div.innerHTML = 'Latency' + latency + '' + - 'Tokens' + tok + ' (est.)' + + 'Tokens' + tok + (meta.tokensAreMeasured ? '' : ' (est.)') + '' + 'Cost' + usd + ' (est.)' + 'Vector' + vecLabel + '' } From 44a47f8033dca2863d903e55fb4832d8a43ef6b7 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 25 Aug 2026 10:04:27 -0600 Subject: [PATCH 13/13] Drop a redundant read, keep a paid-for answer on embed failure, use the logger - The cache-hit path re-read the matched row to get its `cost`; `cachedReply` already carries it. - `embed(assistantContent)` throwing after `models.generate()` discarded an answer that had already been paid for. The row is now stored unembedded instead; the candidate loop already skips rows without an embedding. - Per-request `console.log` + `JSON.stringify` became `logger.debug`, and the savings-counter `catch {}` no longer hides a persistent write failure. Co-Authored-By: Claude Opus --- resources/Agent.js | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/resources/Agent.js b/resources/Agent.js index 365642e..4c82e0f 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto' -import { models, Resource, tables } from 'harper' +import { logger, models, Resource, tables } from 'harper' import { embed } from '../lib/embeddings.js' const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the user's current question. \ @@ -33,9 +33,8 @@ const cacheKey = (text) => createHash('sha256').update(normalize(text)).digest(' // that the matched reply is reasonably on-topic. const CACHE_DISTANCE_THRESHOLD = 0.15 -// Vectors of differing length are not comparable — that happens when the host's embedding -// backend changes under stored vectors — so report maximum distance rather than silently -// comparing a prefix. +// Unequal lengths mean the host's embedding backend changed under the stored vectors. +// Report maximum distance rather than scoring a prefix. function cosineDistance(a, b) { if (a.length !== b.length) return 2 let dot = 0, na = 0, nb = 0 @@ -48,7 +47,6 @@ function cosineDistance(a, b) { return denom === 0 ? 1 : 1 - dot / denom } -// Cache the embedding of repeated text so a rephrase-free repeat skips the backend call. async function cachedEmbed(text) { const key = cacheKey(text) const cached = await tables.EmbeddingCache.get(key) @@ -149,15 +147,11 @@ export class Agent extends Resource { const tCache = Date.now() - t4 const timing = { embedMs: tEmbed, convMs: tConv, storeMs: tStore, cacheSearchMs: tCache } - console.log('[Agent] timing:', JSON.stringify(timing)) + logger.debug('[Agent] timing:', timing) - // Cache hit: no generation call. Credit the matched message's estimated cost to the - // running `totalSaved` the dashboard shows. if (cachedReply) { - let savedCost = 0 + const savedCost = cachedReply.cost ?? 0 try { - const origMsg = await tables.Message.get(cachedReply.id) - savedCost = origMsg?.cost ?? 0 const stats = await tables.Stats.get('global') await tables.Stats.put({ id: 'global', @@ -165,7 +159,9 @@ export class Agent extends Resource { cacheHits: ((stats?.cacheHits) ?? 0) + 1, updatedAt: new Date().toISOString(), }) - } catch {} + } catch (err) { + logger.warn('[Agent] savings counter update failed', err) + } return { conversationId, message: { role: 'assistant', content: cachedReply.content }, @@ -198,20 +194,25 @@ export class Agent extends Resource { throw err } // A truncated or filtered answer is still worth returning, but persisting it would seed - // the semantic cache: every near-miss question from here on would be served the partial - // text as a complete answer, and `Message` rows have no repair path short of the TTL. + // the cache: every near-miss question thereafter is served the partial text as complete. const isComplete = result.finishReason === 'stop' const promptTokens = result.usage?.promptTokens ?? estimateTokens(SYSTEM_PROMPT + message) const completionTokens = result.usage?.completionTokens ?? estimateTokens(assistantContent) const tokensAreMeasured = result.usage?.promptTokens !== undefined const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) - // 9. Store the assistant's response. The estimated cost rides along so a later cache hit - // on this question can credit it to `totalSaved`. + // 9. Store the assistant's response. The cost rides along so a later cache hit on this + // question can credit it to `totalSaved`. if (isComplete) { - // Not `cachedEmbed`: a generated reply is unique text, so the lookup is a guaranteed - // miss and the write is a large row nothing will ever read. - const assistantEmbedding = await embed(assistantContent) + // Not `cachedEmbed`: a generated reply is unique text, so the lookup always misses. + // A failure here must not discard an answer already paid for, so the row is stored + // unembedded; the candidate loop skips rows without an embedding. + let assistantEmbedding + try { + assistantEmbedding = await embed(assistantContent) + } catch (err) { + logger.warn('[Agent] reply embedding failed; storing message unindexed', err) + } await tables.Message.put({ id: crypto.randomUUID(), conversationId,