diff --git a/.gitignore b/.gitignore index ca8a006..cb565e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ node_modules/ -models/ .env CONFIG.env *.env diff --git a/CLAUDE.md b/CLAUDE.md index f0c8227..835233c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,16 +12,15 @@ These skills provide LLM-consumable guidelines at https://github.com/HarperFast/ ## Project Overview -Harper Demo Agent — a conversational AI agent running entirely on Harper with Claude as the LLM. Harper provides the database, vector index, semantic cache, API server, and deployment runtime in a single process. +Harper Demo Agent — a conversational AI agent running entirely on Harper. Harper provides the database, vector index, semantic cache, API server, model gateway, and deployment runtime in a single process; generation and embedding both go through Harper's `models` API, so the app carries no LLM SDK and no API key. Live demo: https://agent-example.stephen-demo-org.harperfabric.com/Chat ## Tech Stack - **Runtime:** Harper (harperdb) — unified DB/cache/vector/API -- **LLM:** Claude Sonnet via Anthropic SDK (`@anthropic-ai/sdk`) or Google Cloud Vertex AI (`@anthropic-ai/vertex-sdk`) -- **Embeddings:** `bge-small-en-v1.5` running locally via `harper-fabric-embeddings` (llama.cpp) — no embedding API -- **Web Search:** Anthropic's built-in server-side `web_search_20250305` tool +- **LLM:** `models.generate()` — routes to the host's configured `models.generative.default` backend (the shared inference process on Fabric GPU hosts; Ollama / OpenAI / Anthropic / Bedrock elsewhere) +- **Embeddings:** `models.embed()` — routes to the host's configured `models.embedding.default` backend - **Language:** JavaScript (ES modules, `"type": "module"`) - **License:** Apache 2.0 @@ -32,10 +31,8 @@ config.yaml # Harper app config (rest, schema, resources) schemas/schema.graphql # Database schema — 3 tables, HNSW vector index, TTL resources/Agent.js # Agent endpoint (POST /Agent) + PublicStats (GET /PublicStats/global) resources/Chat.js # Chat UI (GET /Chat) — full HTML/CSS/JS served from a Resource -lib/config.js # Environment variable helpers -lib/embeddings.js # Local SLM embeddings (bge-small-en-v1.5 via llama.cpp) -models/ # Auto-downloaded GGUF model (gitignored) -.env # ANTHROPIC_API_KEY (not committed) +lib/embeddings.js # Thin wrapper over `models.embed()` +.env # Deploy credentials only (not committed) ``` ## Key Architecture Decisions @@ -53,19 +50,13 @@ models/ # Auto-downloaded GGUF model (gitignored) - `@indexed` on `conversationId` — secondary index for conversation lookups - `Stats` table has no TTL (cumulative savings persist indefinitely) -### Semantic Cache (two layers) -1. **Layer 1 — Exact match:** Normalize text (lowercase, strip punctuation, collapse whitespace) and compare against conversation history. No DB query needed. -2. **Layer 2 — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.12` (cosine distance). **Never do manual cosine similarity in JS** — always use Harper's native HNSW index for distance filtering. +### Semantic Cache +1. **Embedding cache (`EmbeddingCache`):** keyed by a SHA-256 digest of the text with case and whitespace normalized (punctuation is preserved — the key selects a vector, so it must preserve identity) — a digest because Harper rejects a primary key over ~1978 bytes. Skips the embedding backend on exactly repeated text; it is not an answer cache. +2. **Answer cache — HNSW vector search:** Use Harper's native `conditions` search with `comparator: 'lt'` and `value: 0.15` (cosine distance). **Never scan a table and score it in JS** — the index does the filtering. `resources/Agent.js` does recompute cosine distance, but only over the ≤20 rows the index already returned, to rank them (HNSW iteration is not distance-ordered) and to re-check the bound; matches outside `lt` have been observed to survive it, which is worth confirming against harper core rather than leaving as app-side compensation. -### Vector Context (for LLM prompt) -- Uses `sort: { attribute: 'embedding', target: userEmbedding }` with `limit: 10` — returns top 10 most similar messages -- Top 5 injected into system prompt as silent background context -- System prompt explicitly tells Claude NOT to repeat/summarize context in responses - -### Web Search Response Handling -- Anthropic API returns multiple `text` blocks (sentence fragments) mixed with `server_tool_use` and `web_search_tool_result` blocks -- **Always join text blocks that appear AFTER the last non-text block** — text before tool calls is narration ("Let me search for that..."), not the answer -- Handle `pause_turn` stop reason by continuing with partial response as assistant message +### Models API access +- `import { models } from 'harper'` — `models` is a process-wide singleton, exported by the `harper` package and also available as a bare global. It is the same object as `scope.models`, so a `handleApplication(scope)` plugin that stashes the Scope on `globalThis` is not needed. +- `models.generate()` returns `usage` (`promptTokens` / `completionTokens`) passed through from the backend, but the field is optional — `resources/Agent.js` falls back to a length estimate and reports which it used as `meta.tokensAreMeasured`. ### Chat UI (resources/Chat.js) - Full HTML/CSS/JS served from a single template literal via `new Response(HTML, ...)` @@ -82,28 +73,15 @@ npm run start # Start production server npm run deploy # Deploy to Harper Fabric ``` -## Environment Variables - -``` -LLM_PROVIDER # Optional — "anthropic" (default) or "vertex" (Google Cloud Vertex AI) -ANTHROPIC_API_KEY # Required when LLM_PROVIDER=anthropic — Anthropic API key -CLAUDE_MODEL # Optional — defaults to claude-sonnet-4-5-20250929 (anthropic) or claude-sonnet-4-5@20250929 (vertex) -VERTEX_PROJECT_ID # Required when LLM_PROVIDER=vertex — Google Cloud project ID -VERTEX_REGION # Optional — Vertex AI region, defaults to "global" -``` - -### Vertex AI Setup +## Model Configuration -To use Claude through Google Cloud Vertex AI instead of the direct Anthropic API: +The app names no model and holds no provider credentials. Backends are configured on the +Harper *host* — the `models:` block of `harperdb-config.yaml`, or the equivalent env vars — +under `models.embedding.default` and `models.generative.default`. With neither configured, +`POST /Agent` returns a `ModelBackendNotFoundError`. -1. Set up GCP credentials: `gcloud auth application-default login` -2. Configure env vars: - ``` - LLM_PROVIDER=vertex - VERTEX_PROJECT_ID=my-gcp-project - VERTEX_REGION=global - ``` -3. Note: Vertex model IDs use `@` version suffixes (e.g. `claude-sonnet-4-5@20250929`) while direct API uses `-` (e.g. `claude-sonnet-4-5-20250929`) +`.env` is still read (see `loadEnv` in `config.yaml`) but only carries the Fabric deploy +credentials: `CLI_TARGET`, `CLI_TARGET_USERNAME`, `CLI_TARGET_PASSWORD`. ## Common Tasks @@ -131,5 +109,5 @@ curl http://localhost:9926/PublicStats/global 1. **Template literal backslashes** — `\n` inside a JS template literal becomes a real newline. Use `\\n` in Chat.js script sections. Same for `\d`, `\s`, `\*` in regex patterns. 2. **Resource class naming** — naming a class `Stats` when there's a `Stats` table shadows `tables.Stats`. Always use a different name (e.g. `PublicStats`). 3. **`tables.Stats.get()` on empty DB** — returns `null`, not `{}`. Always provide a fallback: `?? { id: 'global', totalSaved: 0, cacheHits: 0 }`. -4. **Web search text blocks** — join only text blocks after the last tool block. Joining all text blocks concatenates narration with the answer. +4. **`models.generate().usage` is optional** — a backend that reports none leaves it undefined, so `meta.tokens` may be a length estimate. `meta.tokensAreMeasured` says which. Dollar amounts are always list-price Claude Sonnet, never the backend's real cost. 5. **V2 auth** — `target.checkPermission = false` is the only way to allow unauthenticated access when `loadAsInstance = false`. V1 methods (`allowRead`) are silently ignored. diff --git a/README.md b/README.md index 0b0f782..5d6e239 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,16 @@ # Harper Demo Agent -A conversational AI agent with persistent semantic memory, a two-layer semantic cache, web search, cost tracking, and a browser chat UI — all running on [Harper](https://harper.fast) with Claude (via the Anthropic API or Google Cloud Vertex AI). +A conversational AI agent with persistent semantic memory, a two-layer semantic cache, cost tracking, and a browser chat UI — all running on [Harper](https://harper.fast). Generation and embedding go through Harper's `models` API, so the app ships no LLM SDK and holds no provider credentials. Live demo: **[agent-example.stephen-demo-org.harperfabric.com/Chat](https://agent-example.stephen-demo-org.harperfabric.com/Chat)** ## What It Does -- **Chat with Claude** via a REST endpoint (`POST /Agent`) or the built-in browser chat UI (`GET /Chat`) -- **Semantic cache** — two-layer cache catches repeated and rephrased questions before they reach Claude, returning answers instantly at zero LLM cost -- **Web search** — Anthropic's built-in server-side web search (`web_search_20250305`, up to 5 uses per turn); no external API key required +- **Chat** via a REST endpoint (`POST /Agent`) or the built-in browser chat UI (`GET /Chat`) +- **Semantic cache** — two-layer cache catches repeated and rephrased questions before they reach the model, returning answers instantly at zero LLM cost - **Persistent memory** — every message is embedded and stored in Harper; semantic recall surfaces relevant context from past conversations automatically -- **Local embeddings** — `bge-small-en-v1.5` runs via `harper-fabric-embeddings` (llama.cpp wrapper), entirely in-process; no embedding API key or billing -- **Per-response metadata** — every API response includes latency, token counts, cost breakdown, web searches used, and vector context stats +- **Host-provided models** — `models.generate()` and `models.embed()` resolve to whatever backend the Harper host has configured (the shared inference process on Fabric GPU hosts; Ollama / OpenAI / Anthropic / Bedrock elsewhere). No SDK dependency, no API key in the app +- **Per-response metadata** — every API response includes latency, token estimates, cost breakdown, and vector context stats - **Global savings tracker** — cache hits accumulate a running total of USD saved and hit count in a `Stats` table, displayed live in the chat sidebar - **Auto-generated REST APIs** — full CRUD on `Conversation`, `Message`, and `Stats` tables, generated from the GraphQL schema with zero route code @@ -27,55 +26,52 @@ User Query │ 1. Embed user message │ │ ┌─────────────────────┐ │ │ │ EmbeddingCache │ ← normalized text → vector │ -│ │ hit: ~1ms lookup │ miss: SLM generates it, │ -│ │ (skip SLM) │ then stores for next time │ +│ │ hit: ~1ms lookup │ miss: models.embed(), │ +│ │ (skip the model) │ then stores for next time │ │ └─────────────────────┘ │ -│ Local SLM: bge-small-en-v1.5 (llama.cpp, in-process)│ │ │ │ 2. Store user message + embedding │ -│ 3. HNSW semantic cache check (cosine distance < 0.12) │ +│ 3. HNSW semantic cache check (cosine distance < 0.15) │ │ │ │ │ │ Cache HIT Cache MISS │ │ │ │ │ -│ Return $0.00 Call Claude ──────────────────────┼──► Anthropic API -│ + saved $X │ │ + Web Search -│ Embed response (via cache/SLM) │◄──────────┘ -│ Store in Harper │ +│ Return $0.00 models.generate() ───────────────────┼──► host-configured +│ + saved $X │ │ model backend +│ Embed response (cache/embed) │◄──────────┘ +│ Store in Harper │ │ │ └──────────────────────────────────────────────────────────┘ ``` -Every request is standalone. Ask once, pay for Claude. Ask again — or rephrase the same question — and Harper serves the cached answer instantly at $0. The embedding cache eliminates the SLM cost on repeated text (~2.3s on Fabric → ~1ms). +Every request is standalone. Ask once, pay for the generation. Ask again — or rephrase the same question — and Harper serves the cached answer instantly at $0. The embedding cache eliminates the embedding round-trip on repeated text. ## How the Semantic Cache Works -Before calling Claude, the agent searches Harper's HNSW vector index for semantically similar past questions: +Before calling the model, the agent searches Harper's HNSW vector index for semantically similar past questions: ```javascript tables.Message.search({ conditions: { attribute: 'embedding', comparator: 'lt', - value: 0.12, // cosine distance < 0.12 ≡ cosine similarity ≥ 0.88 + value: 0.15, // cosine distance < 0.15 ≡ cosine similarity ≥ 0.85 target: userEmbedding, }, - limit: 10, + limit: 20, }) ``` -Harper's HNSW index evaluates the distance threshold internally — no full table scan, no in-memory cosine math. When a match is found, the agent looks up the assistant reply that followed it and returns that directly. No Claude call, no tokens, no cost. +Harper's HNSW index evaluates the distance threshold internally — no full table scan, no vector DB round-trip. The agent then ranks the returned candidates by cosine distance and takes the closest whose *immediately* following message is an assistant reply, and returns that directly. No generation call, no tokens, no cost. Cache hits return `cost.total: 0` and include a `cost.saved` field showing what the call would have cost. The saved amount is added to the global `Stats` record (`totalSaved`, `cacheHits`). ## Prerequisites - [Node.js](https://nodejs.org/) 22+ -- [Harper CLI](https://www.npmjs.com/package/harper): `npm install -g harper` -- **One of:** - - [Anthropic API key](https://console.anthropic.com/) (direct API — default) - - [Google Cloud project](https://console.cloud.google.com/) with Vertex AI enabled (GCP Vertex AI) +- [Harper](https://www.npmjs.com/package/harper) 5.2+: `npm install -g harper` +- A Harper host with an embedding backend and a generative backend configured (see [Model Configuration](#model-configuration)) -No embedding API key needed — embeddings run in-process. +No API key lives in this app — credentials, if the chosen backend needs any, belong to the host's model configuration. ## Quick Start @@ -87,69 +83,48 @@ cd agent-example-harper # Install dependencies npm install -# Configure environment +# Configure environment (deploy credentials only) cp dot-env.example .env -# Edit .env — see "LLM Provider Setup" below # Start the dev server npm run dev ``` -## LLM Provider Setup - -This agent supports two LLM backends — the direct Anthropic API and Google Cloud Vertex AI. Set `LLM_PROVIDER` in your `.env` to choose which one to use. - -### Option A: Anthropic API (default) - -The simplest path. You just need an API key from [console.anthropic.com](https://console.anthropic.com/). - -```env -LLM_PROVIDER=anthropic -ANTHROPIC_API_KEY=sk-ant-... -``` - -Web search is included automatically via Anthropic's server-side `web_search_20250305` tool — no additional API keys required. - -### Option B: Google Cloud Vertex AI - -Run Claude through your own GCP project. Useful for enterprise environments, org-level billing, data residency, and keeping everything inside Google Cloud. - -**1. Enable the Vertex AI API** in your GCP project: - -``` -https://console.developers.google.com/apis/api/aiplatform.googleapis.com/overview?project=YOUR_PROJECT_ID -``` - -**2. Enable a Claude model** in the [Vertex AI Model Garden](https://console.cloud.google.com/vertex-ai/model-garden) — search for "Claude" and enable the model you want. - -**3. Request quota** — new projects start with 0 tokens/min. Go to [IAM & Admin → Quotas](https://console.cloud.google.com/iam-admin/quotas), filter for your Claude model, and request an increase. - -**4. Create a service account** with the **Vertex AI User** role, download the JSON key, and place it in the project root. - -**5. Configure `.env`:** - -```env -LLM_PROVIDER=vertex -VERTEX_PROJECT_ID=my-gcp-project -VERTEX_REGION=us-east5 -GOOGLE_APPLICATION_CREDENTIALS=./your-service-account-key.json +## Model Configuration + +This app never names a model or holds a credential. `models.generate()` and `models.embed()` +resolve the logical names `models.generative.default` and `models.embedding.default` from the +**host's** configuration — the top-level `models:` block of `harperdb-config.yaml` at the +instance root. Change the backend there and the app is unchanged. + +```yaml +models: + embedding: + default: + backend: ollama + host: http://localhost:11434 + model: nomic-embed-text + generative: + default: + backend: anthropic + model: claude-sonnet-4-5 + apiKey: ${ANTHROPIC_API_KEY} ``` -> **Note:** Web search is not available on Vertex AI by default (requires an org policy change). The agent automatically disables it when running on Vertex. +Built-in backends: `ollama`, `openai`, `anthropic`, `bedrock`. Any other `backend` value is +resolved as a module specifier and imported, so a custom backend needs no core change. -### Environment Variable Reference +Two things worth knowing: -| Variable | Required | Default | Description | -|---|---|---|---| -| `LLM_PROVIDER` | No | `anthropic` | `anthropic` or `vertex` | -| `ANTHROPIC_API_KEY` | When `anthropic` | — | Anthropic API key | -| `VERTEX_PROJECT_ID` | When `vertex` | — | GCP project ID | -| `VERTEX_REGION` | No | `global` | Vertex AI region (e.g. `us-east5`, `global`) | -| `VERTEX_MODEL` | No | `claude-sonnet-4-6` | Vertex model ID | -| `GOOGLE_APPLICATION_CREDENTIALS` | When `vertex` | — | Path to GCP service account JSON key | -| `CLAUDE_MODEL` | No | `claude-sonnet-4-5-20250929` | Anthropic direct API model ID | +- **Keep credentials out of the YAML.** String leaves are env-expanded before they reach the + backend, so write `apiKey: ${ANTHROPIC_API_KEY}` rather than the literal key — Harper warns + at boot when it sees a literal in a credential field. +- **A misconfigured entry is logged and skipped, not fatal.** Harper still boots; the failure + surfaces on first use as `ModelBackendNotFoundError: No backend registered for + 'embedding.default'` from `POST /Agent`. -> **First run:** On startup, `bge-small-en-v1.5` (~24 MB) is auto-downloaded into `./models/`. This only happens once. +On Fabric GPU hosts the host-manager configures these entries for you against the shared +inference process, and no local setup is needed. The server starts at `http://localhost:9926`. Open `http://localhost:9926/Chat` in your browser. @@ -169,7 +144,7 @@ curl -X POST http://localhost:9926/Agent \ -d '{"message": "What is Harper?"}' ``` -Response: +Response. `tokens` comes from the backend's reported usage when it reports any, and falls back to a length estimate otherwise — `meta.tokensAreMeasured` says which. `cost` is always what the same call would have cost at list-price Claude Sonnet, the comparator behind the savings tracker, never the backend's real cost: ```json { @@ -178,9 +153,10 @@ Response: "meta": { "latencyMs": 1842, "tokens": { "input": 312, "output": 148, "total": 460 }, - "cost": { "input": 0.000936, "output": 0.00222, "search": 0, "total": 0.003156 }, - "webSearches": 0, - "vectorContext": { "hit": false, "count": 0, "cached": false } + "cost": { "input": 0.000936, "output": 0.00222, "total": 0.003156, "saved": 0 }, + "vectorContext": { "hit": false, "count": 0, "cached": false }, + "finishReason": "stop", + "tokensAreMeasured": true } } ``` @@ -226,18 +202,16 @@ curl "http://localhost:9926/Message?conversationId=abc-123" ## Project Structure ``` -├── config.yaml # Harper app configuration (6 lines) +├── config.yaml # Harper app configuration ├── schemas/ │ └── schema.graphql # Database schema (Conversation, Message, Stats + HNSW index) ├── resources/ -│ ├── Agent.js # POST /Agent (agent loop + semantic cache + web search) +│ ├── Agent.js # POST /Agent (agent loop + semantic cache) │ │ # GET /PublicStats/:id (public stats endpoint) │ └── Chat.js # GET /Chat (full browser chat UI served as HTML) ├── lib/ -│ ├── config.js # Environment variable helpers -│ └── embeddings.js # Local llama.cpp embeddings via harper-fabric-embeddings -├── models/ # Auto-downloaded GGUF model (gitignored) -├── .env.example # Environment template +│ └── embeddings.js # Thin wrapper over `models.embed()` +├── dot-env.example # Environment template (deploy credentials) └── package.json ``` @@ -297,7 +271,7 @@ Rolling restarts and replication are handled automatically. | Semantic cache | Redis + custom logic | Built in (native HNSW threshold filter) | | API server | Express / Fastify | Auto-generated from schema | | Chat UI server | Vite / Next.js | Resource returning `Response(html)` | -| Embeddings | Voyage / OpenAI API | Local via `harper-fabric-embeddings` (24 MB, in-process) | +| Model access | Per-provider SDK + key per app | `models.embed()` / `models.generate()`, backend configured on the host | | Deployment | Docker + K8s + cloud | `harper deploy .` | **Key insights from building this:** @@ -306,7 +280,7 @@ Rolling restarts and replication are handled automatically. - **Everything in one process means no network hops.** Database, vector index, cache, API, and agent code share the same runtime. No Redis round-trip, no vector DB round-trip. - **The schema is the only config you need.** One `@indexed(type: "HNSW", distance: "cosine")` directive creates the vector index. One `@export` generates the CRUD API. One `@indexed` on `conversationId` creates the secondary index. - **Resources can return anything.** A `Resource` subclass can return a `Response` with any content type — JSON, HTML, plain text. The chat UI lives in the same project and deploy as the agent logic. -- **Local embeddings eliminate a cost center.** `bge-small-en-v1.5` runs in-process via llama.cpp. No per-embedding billing, no embedding service SLA to worry about. +- **The model backend is host configuration, not app code.** `import { models } from 'harper'` gives a resource the process-wide models singleton — the same object as `scope.models`, so no `handleApplication(scope)` shim is needed. Swapping Ollama for Bedrock is a YAML edit on the host; this app does not change. ## License diff --git a/dot-env.example b/dot-env.example index 47133be..6591575 100644 --- a/dot-env.example +++ b/dot-env.example @@ -1,14 +1,7 @@ -#MUST USE ANTHROPIC OR GOOGLE VERTEX AI - -# Anthropic (Optional) -ANTHROPIC_API_KEY=sk-ant-your-key-here -CLAUDE_MODEL=claude-sonnet-4-6 - -#Google Vertex AI (optional) -#LLM_PROVIDER=vertex -#VERTEX_PROJECT_ID=your-gcp-project -#VERTEX_REGION=global - +# The agent holds no model credentials. Generation and embedding go through +# Harper's `models` API, which resolves `models.generative.default` and +# `models.embedding.default` from the HOST's configuration (the `models:` block +# of harperdb-config.yaml, or the equivalent env vars) — not from this file. # Harper Fabric deployment (optional, only needed for deploy) # CLI_TARGET=https://your-instance.your-org.harperfabric.com:9925/ diff --git a/integrationTests/app.test.ts b/integrationTests/app.test.ts index 9e4e1e4..e6b91be 100644 --- a/integrationTests/app.test.ts +++ b/integrationTests/app.test.ts @@ -1,8 +1,8 @@ /** * Integration tests for agent-example-harper. * Tests that the app starts and key endpoints respond correctly. - * Note: Full agent functionality requires Anthropic/Vertex API keys - * which are not available in CI, so we only test structural correctness. + * Note: Full agent functionality requires a configured `models` backend, which is not + * available in CI, so we only test structural correctness. */ import { suite, test, before, after } from 'node:test'; import { strictEqual, ok } from 'node:assert/strict'; @@ -66,6 +66,25 @@ void suite('agent-example-harper loads', (ctx: ContextWithHarper) => { ok([200, 404].includes(res.status), `unexpected status ${res.status}`); }); + // Guards `import { models } from 'harper'`: with no backend configured the request must + // fail in the models layer, not on an undefined import. + void test('POST /Agent reaches the models layer', async () => { + const res = await authFetch(ctx, '/Agent', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message: 'hello there' }), + }); + const body = await res.json(); + if (res.status === 200) { + // A host with backends configured: assert the success shape rather than passing vacuously. + strictEqual(typeof body.conversationId, 'string'); + strictEqual(body.message.role, 'assistant'); + ok(body.message.content.length > 0, 'assistant content must be non-empty'); + return; + } + strictEqual(body.code, 'ModelBackendNotFoundError', `unexpected error body ${JSON.stringify(body)}`); + }); + void test('POST /Conversation/ creates a conversation record', async () => { const res = await authFetch(ctx, '/Conversation/', { method: 'POST', diff --git a/lib/config.js b/lib/config.js deleted file mode 100644 index d1e1885..0000000 --- a/lib/config.js +++ /dev/null @@ -1,21 +0,0 @@ -const required = (name) => { - const value = process.env[name] - if (!value) throw new Error(`Missing required env var: ${name}`) - return value -} - -const optional = (name, fallback) => process.env[name] ?? fallback - -export const config = { - // "anthropic" (direct API) or "vertex" (Google Cloud Vertex AI) - provider: () => optional('LLM_PROVIDER', 'anthropic'), - anthropic: { - apiKey: () => required('ANTHROPIC_API_KEY'), - model: () => optional('CLAUDE_MODEL', 'claude-sonnet-4-5-20250929'), - }, - vertex: { - projectId: () => required('VERTEX_PROJECT_ID'), - region: () => optional('VERTEX_REGION', 'global'), - model: () => optional('VERTEX_MODEL', 'claude-sonnet-4-6'), - }, -} diff --git a/lib/embeddings.js b/lib/embeddings.js index 0b3102e..75ca762 100644 --- a/lib/embeddings.js +++ b/lib/embeddings.js @@ -1,14 +1,9 @@ -import { init, embed as llamaEmbed } from 'harper-fabric-embeddings' -import { resolve } from 'path' -import { fileURLToPath } from 'url' +import { models } from 'harper'; -const __dirname = fileURLToPath(new URL('.', import.meta.url)) -const modelPath = resolve(__dirname, '../models/bge-small-en-v1.5-q4_k_m.gguf') - -// Model is pre-downloaded by the predev/prestart npm hook (scripts/download-model.js) -const initPromise = init({ modelPath }) +// `models.embed()` (harper#510) dispatches to the host's configured embedding backend. +// Harper's HNSW index stores arrays, so the returned Float32Array is converted. export async function embed(text) { - await initPromise - return llamaEmbed(text) + const [vector] = await models.embed(text); + return Array.from(vector); } diff --git a/package-lock.json b/package-lock.json index 1142a3b..cf0c565 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,23 +8,15 @@ "name": "agent-example-harper", "version": "1.0.0", "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", - "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", - "harper": "^5.2.1", - "harper-fabric-embeddings": "^0.2.2" + "harper": "^5.2.1" }, "devDependencies": { "@harperfast/integration-testing": "^0.3.1", "@types/node": "^22.19.19" }, "engines": { - "harper": "^5.0" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-x64": "3.17.1", - "@node-llama-cpp/mac-arm64-metal": "3.17.1", - "@node-llama-cpp/mac-x64": "3.17.1" + "harper": "^5.2" } }, "node_modules/@agoric/babel-generator": { @@ -41,67 +33,6 @@ "node": ">=6.9.0" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.39.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.39.0.tgz", - "integrity": "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg==", - "license": "MIT", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { - "version": "18.19.130", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", - "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, - "node_modules/@anthropic-ai/vertex-sdk": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/vertex-sdk/-/vertex-sdk-0.15.0.tgz", - "integrity": "sha512-i2LDdu6VB8Lqqip+kbNSXRxQgFsCg6GPBO/X2zRJwLl99dNzf28nb6Rdi0EodONXsyJfY2TKdGR+y5l1/AKFEg==", - "license": "MIT", - "dependencies": { - "@anthropic-ai/sdk": ">=0.50.3 <1", - "google-auth-library": "^9.4.2" - } - }, - "node_modules/@anthropic-ai/vertex-sdk/node_modules/@anthropic-ai/sdk": { - "version": "0.116.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.116.0.tgz", - "integrity": "sha512-4UEapYQ+epLEMsAuLZDvW8ExVSOtHD8a7zTyLzhw0H9RXJ1eilPgmqhjwgcdg22diwx13spw6fJ4rONZ+bS7Ww==", - "license": "MIT", - "dependencies": { - "json-schema-to-ts": "^3.1.1", - "standardwebhooks": "^1.0.0" - }, - "bin": { - "anthropic-ai-sdk": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } - } - }, "node_modules/@aws-sdk/checksums": { "version": "3.1000.26", "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.26.tgz", @@ -1987,131 +1918,6 @@ "url": "https://paulmillr.com/funding/" } }, - "node_modules/@node-llama-cpp/linux-arm64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-arm64/-/linux-arm64-3.18.1.tgz", - "integrity": "sha512-rXMgZxUay78FOJV/fJ67apYP9eElH5jd4df5YRKPlLhLHHchuOSyDn+qtyW/L/EnPzpogoLkmULqCkdXU39XsQ==", - "cpu": [ - "arm64", - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/linux-armv7l": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-armv7l/-/linux-armv7l-3.18.1.tgz", - "integrity": "sha512-BrJL2cGo0pN5xd5nw+CzTn2rFMpz9MJyZZPUY81ptGkF2uIuXT2hdCVh56i9ImQrTwBfq1YcZL/l/Qe/1+HR/Q==", - "cpu": [ - "arm", - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/linux-x64": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-x64/-/linux-x64-3.17.1.tgz", - "integrity": "sha512-/o/UoqAdslg4ExdKYyYPqbw+21Dr4cQ2JgouXg8Ji3opRKoTMrlUNfrMwIsYZfbDDJ8l7xFnwfGIwdlQ5RPwJg==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/mac-arm64-metal": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-arm64-metal/-/mac-arm64-metal-3.17.1.tgz", - "integrity": "sha512-oRq6/7qCMsazO2Cw0oCyiILZmMvejKJgLAIG60E00WOZWhpJGjh71JGnOybRycKA015mFPNDHzT3SDdUZtZBew==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/mac-x64": { - "version": "3.17.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-x64/-/mac-x64-3.17.1.tgz", - "integrity": "sha512-3L0nFVi70j+Qk7Xb8p/RQVMU0E28G0xXX0YL6Vzkirq3DazPYhWOLWUUs9MtGW2FrBg/6PLqyddmxwBfCpjm3w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/win-arm64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/win-arm64/-/win-arm64-3.18.1.tgz", - "integrity": "sha512-S05YUzBMVSRS5KNbOS26cDYugeQHqogI3uewtTUBVC0tPbTHRSKjsdicmgWru1eNAry399LWWhzOf/3St/qsAw==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@node-llama-cpp/win-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/win-x64/-/win-x64-3.18.1.tgz", - "integrity": "sha512-QLDVphPl+YDI+x/VYYgIV1N9g0GMXk3PqcoopOUG3cBRUtce7FO+YX903YdRJezs4oKbIp8YaO+xYBgeUSqhpA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2556,12 +2362,6 @@ "node": ">=18.0.0" } }, - "node_modules/@stablelib/base64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", - "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", - "license": "MIT" - }, "node_modules/@turf/area": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@turf/area/-/area-6.5.0.tgz", @@ -2922,16 +2722,6 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.4" - } - }, "node_modules/@types/readable-stream": { "version": "4.0.24", "resolved": "https://registry.npmjs.org/@types/readable-stream/-/readable-stream-4.0.24.tgz", @@ -3038,22 +2828,12 @@ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">= 14" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -3248,12 +3028,6 @@ "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==", "license": "MIT" }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -4075,18 +3849,6 @@ "node": ">=0.1.90" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/commander": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", @@ -4341,15 +4103,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -4560,21 +4313,6 @@ "node": ">= 0.4" } }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -4664,12 +4402,6 @@ "optional": true, "peer": true }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, "node_modules/eyes": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", @@ -4762,12 +4494,6 @@ "node": ">=6" } }, - "node_modules/fast-sha256": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", - "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", - "license": "Unlicense" - }, "node_modules/fast-uri": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-4.1.2.tgz", @@ -5052,41 +4778,6 @@ "optional": true, "peer": true }, - "node_modules/form-data": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.6.tgz", - "integrity": "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.4", - "mime-types": "^2.1.35" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/fraction.js": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.4.tgz", @@ -5164,36 +4855,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/gaxios": { - "version": "6.7.1", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", - "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", - "license": "Apache-2.0", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gcp-metadata": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", - "integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==", - "license": "Apache-2.0", - "dependencies": { - "gaxios": "^6.1.1", - "google-logging-utils": "^0.0.2", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -5325,32 +4986,6 @@ "node": ">= 6" } }, - "node_modules/google-auth-library": { - "version": "9.15.1", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", - "integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==", - "license": "Apache-2.0", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/google-logging-utils": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz", - "integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=14" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -5393,19 +5028,6 @@ "graphql": ">=0.11 <=17" } }, - "node_modules/gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "license": "MIT", - "dependencies": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, "node_modules/gunzip-maybe": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/gunzip-maybe/-/gunzip-maybe-1.4.2.tgz", @@ -5558,77 +5180,6 @@ } } }, - "node_modules/harper-fabric-embeddings": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/harper-fabric-embeddings/-/harper-fabric-embeddings-0.2.3.tgz", - "integrity": "sha512-25F1xzRTJ+19NlDiMI0RLF47u4Fwd5Ve/V03q06kJjCioqvEc2yrAASQ3NY4O+LJDU9rNq9WQivFeU+9JKt4IA==", - "hasInstallScript": true, - "license": "MIT", - "engines": { - "node": ">=22" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-arm64": "3.18.1", - "@node-llama-cpp/linux-armv7l": "3.18.1", - "@node-llama-cpp/linux-x64": "3.18.1", - "@node-llama-cpp/mac-arm64-metal": "3.18.1", - "@node-llama-cpp/mac-x64": "3.18.1", - "@node-llama-cpp/win-arm64": "3.18.1", - "@node-llama-cpp/win-x64": "3.18.1" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/linux-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/linux-x64/-/linux-x64-3.18.1.tgz", - "integrity": "sha512-tRmWcsyvAcqJHQHXHsaOkx6muGbcirA9nRdNgH6n7bjGUw4VuoBD3dChyNF3/Ktt7ohB9kz+XhhyZjbDHpXyMA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/mac-arm64-metal": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-arm64-metal/-/mac-arm64-metal-3.18.1.tgz", - "integrity": "sha512-cyZTdsUMlvuRlGmkkoBbN3v/DT6NuruEqoQYd9CqIrPyLa1xLNBTSKIZ9SgRnw23iCOj4URfITvRP+2pu63LuQ==", - "cpu": [ - "arm64", - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/harper-fabric-embeddings/node_modules/@node-llama-cpp/mac-x64": { - "version": "3.18.1", - "resolved": "https://registry.npmjs.org/@node-llama-cpp/mac-x64/-/mac-x64-3.18.1.tgz", - "integrity": "sha512-GfCPgdltaIpBhEnQ7WfsrRXrZO9r9pBtDUAQMXRuJwOPP5q7xKrQZUXI6J6mpc8tAG0//CTIuGn4hTKoD/8V8w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.0.0" - } - }, "node_modules/harper/node_modules/uuid": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", @@ -5754,6 +5305,8 @@ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "4" @@ -5771,15 +5324,6 @@ "knuth-shuffle": "^1.0.0" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -6127,18 +5671,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/is-unicode-supported": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-2.1.0.tgz", @@ -6475,15 +6007,6 @@ "node": ">=4" } }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "license": "MIT", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, "node_modules/json-bigint-fixes": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/json-bigint-fixes/-/json-bigint-fixes-1.1.0.tgz", @@ -6512,19 +6035,6 @@ "dequal": "^2.0.3" } }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -7448,27 +6958,6 @@ "node": ">= 0.6" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types/node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -7668,26 +7157,6 @@ "node": "^18 || ^20 || >= 21" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -9656,16 +9125,6 @@ "node": ">=8" } }, - "node_modules/standardwebhooks": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", - "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", - "license": "MIT", - "dependencies": { - "@stablelib/base64": "^1.0.0", - "fast-sha256": "^1.3.0" - } - }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -10147,12 +9606,6 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "license": "MIT" }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "license": "MIT" - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -10294,20 +9747,6 @@ "node": ">= 0.4.0" } }, - "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/validate.js": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/validate.js/-/validate.js-0.13.1.tgz", @@ -10348,15 +9787,6 @@ "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==", "license": "MIT" }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/package.json b/package.json index 2ea6531..d1a35b6 100644 --- a/package.json +++ b/package.json @@ -1,15 +1,12 @@ { "name": "agent-example-harper", "version": "1.0.0", - "description": "A conversational AI agent with persistent memory, built on Harper and Claude", + "description": "A conversational AI agent with persistent memory, built on Harper with the harper models API", "type": "module", "engines": { - "harper": "^5.0" + "harper": "^5.2" }, "scripts": { - "setup": "node scripts/download-model.js", - "predev": "node scripts/download-model.js", - "prestart": "node scripts/download-model.js", "start": "npx -y dotenv-cli -e .env -o -- harper run .", "dev": "npx -y dotenv-cli -e .env -o -- harper dev .", "login": "node login.js", @@ -17,19 +14,11 @@ "test:integration": "harper-integration-test-run 'integrationTests/**/*.test.ts'" }, "dependencies": { - "@anthropic-ai/sdk": "^0.39.0", - "@anthropic-ai/vertex-sdk": "^0.15.0", "graphql": "^16.8.1", - "harper": "^5.2.1", - "harper-fabric-embeddings": "^0.2.2" + "harper": "^5.2.1" }, "devDependencies": { "@harperfast/integration-testing": "^0.3.1", "@types/node": "^22.19.19" - }, - "optionalDependencies": { - "@node-llama-cpp/linux-x64": "3.17.1", - "@node-llama-cpp/mac-arm64-metal": "3.17.1", - "@node-llama-cpp/mac-x64": "3.17.1" } } diff --git a/resources/Agent.js b/resources/Agent.js index 02e9310..4c82e0f 100644 --- a/resources/Agent.js +++ b/resources/Agent.js @@ -1,53 +1,54 @@ -import { Resource, tables } from 'harper' -import Anthropic from '@anthropic-ai/sdk' -import { AnthropicVertex } from '@anthropic-ai/vertex-sdk' -import { config } from '../lib/config.js' +import { createHash } from 'node:crypto' +import { logger, models, Resource, tables } from 'harper' import { embed } from '../lib/embeddings.js' -let _client -const getClient = () => { - if (_client) return _client - if (config.provider() === 'vertex') { - _client = new AnthropicVertex({ - projectId: config.vertex.projectId(), - region: config.vertex.region(), - }) - } else { - _client = new Anthropic({ apiKey: config.anthropic.apiKey() }) - } - return _client -} - -const getModel = () => - config.provider() === 'vertex' ? config.vertex.model() : config.anthropic.model() - const SYSTEM_PROMPT = `You are a helpful, concise assistant. Answer only the user's current question. \ Do NOT summarize, repeat, or reference prior conversation context in your response — use it silently \ as background knowledge only if it is directly relevant. Never recite or recap previous answers.` -// Approximate pricing for Claude Sonnet 4.5 (per token) -const COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens -const COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens -const COST_PER_WEB_SEARCH = 10 / 1_000 // $10 / 1K searches +// The savings tracker prices every generation at list-price Claude Sonnet 4.5 whichever +// backend actually ran it, so the dollars are a comparator, never a bill. +const CLAUDE_COST_INPUT_PER_TOKEN = 3 / 1_000_000 // $3 / 1M input tokens +const CLAUDE_COST_OUTPUT_PER_TOKEN = 15 / 1_000_000 // $15 / 1M output tokens -// Anthropic web search tool — executed server-side, no external API key needed. -// Not available on Vertex AI without an org policy change. -const WEB_SEARCH_TOOL = { type: 'web_search_20250305', name: 'web_search', max_uses: 5 } -const isVertex = () => config.provider() === 'vertex' +// Fallback only: `models.generate()` passes the backend's token usage through, but the +// field is optional and a backend that reports none leaves it undefined. +const estimateTokens = (text) => Math.max(1, Math.ceil((text?.length ?? 0) / 4)) -// Normalize text for embedding cache key — lowercase, strip punctuation, collapse whitespace -const normalize = (s) => - s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim() +const estimateClaudeCost = (promptTokens, completionTokens) => + promptTokens * CLAUDE_COST_INPUT_PER_TOKEN + completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN + +// Case and whitespace only. This key selects a stored *vector*, so it has to preserve +// identity: stripping punctuation collided `What is C++?` with `What is C#?`, handing the +// second asker the first one's embedding to be indexed as their own message. +const normalize = (s) => s.toLowerCase().replace(/\s+/g, ' ').trim() + +// Harper rejects a primary key over ~1978 bytes and message text is unbounded. +const cacheKey = (text) => createHash('sha256').update(normalize(text)).digest('base64url') // Cosine distance threshold for Harper's native HNSW vector search. -// Harper uses cosine *distance* (0 = identical, 2 = opposite), so this is -// equivalent to cosine similarity >= 0.88 (distance = 1 - similarity = 0.12). -const CACHE_DISTANCE_THRESHOLD = 0.12 +// Harper uses cosine *distance* (0 = identical, 2 = opposite). 0.15 ≈ cosine +// similarity 0.85 — loose enough to catch rewordings and related phrasings +// ("describe the moon landing" / "tell me about apollo 11"), tight enough +// that the matched reply is reasonably on-topic. +const CACHE_DISTANCE_THRESHOLD = 0.15 + +// Unequal lengths mean the host's embedding backend changed under the stored vectors. +// Report maximum distance rather than scoring a prefix. +function cosineDistance(a, b) { + if (a.length !== b.length) return 2 + let dot = 0, na = 0, nb = 0 + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i] + na += a[i] * a[i] + nb += b[i] * b[i] + } + const denom = Math.sqrt(na) * Math.sqrt(nb) + return denom === 0 ? 1 : 1 - dot / denom +} -// Get or compute an embedding, using Harper as a cache to skip the SLM on repeated text. -// On Fabric, the SLM takes ~2.3s per embedding — this cache makes repeat queries instant. async function cachedEmbed(text) { - const key = normalize(text) + const key = cacheKey(text) const cached = await tables.EmbeddingCache.get(key) if (cached?.embedding) return cached.embedding const embedding = await embed(text) @@ -109,11 +110,22 @@ export class Agent extends Resource { value: CACHE_DISTANCE_THRESHOLD, target: userEmbedding, }, - limit: 10, + limit: 20, }) + const candidates = [] for await (const match of nearbyMsgs) { - if (match.id === userMsgId || match.role !== 'user') continue + if (match.id === userMsgId || match.role !== 'user' || !match.embedding) continue + candidates.push({ match, distance: cosineDistance(userEmbedding, match.embedding) }) + } + candidates.sort((a, b) => a.distance - b.distance) + + // HNSW iteration is not distance-ordered, so rank here. The re-check is not redundant: + // core's cosine helper zero-pads to the longer vector rather than rejecting a length + // mismatch, so a row left from a different embedding backend can pass `lt`. + const filtered = candidates.filter((c) => c.distance <= CACHE_DISTANCE_THRESHOLD) + + for (const { match } of filtered) { const matchConvMsgs = [] const matchHistory = tables.Message.search({ conditions: [{ attribute: 'conversationId', value: match.conversationId }], @@ -122,34 +134,34 @@ export class Agent extends Resource { for await (const m of matchHistory) matchConvMsgs.push(m) matchConvMsgs.sort((a, b) => a.createdAt.localeCompare(b.createdAt)) const midx = matchConvMsgs.findIndex((m) => m.id === match.id) - const reply = matchConvMsgs.slice(midx + 1).find((m) => m.role === 'assistant') - if (reply) { - cachedReply = reply + if (midx === -1) continue + // Only the IMMEDIATELY following message is this question's answer: scanning forward + // would skip past later user messages that produced no reply of their own and return + // an answer to a different question. + const next = matchConvMsgs[midx + 1] + if (next?.role === 'assistant') { + cachedReply = next break } } const tCache = Date.now() - t4 const timing = { embedMs: tEmbed, convMs: tConv, storeMs: tStore, cacheSearchMs: tCache } - console.log('[Agent] timing:', JSON.stringify(timing)) + logger.debug('[Agent] timing:', timing) - // Return the cached answer — zero LLM cost if (cachedReply) { - const t5 = Date.now() - let savedCost = 0 + const savedCost = cachedReply.cost ?? 0 try { - const origMsg = await tables.Message.get(cachedReply.id) - savedCost = origMsg?.cost ?? 0 const stats = await tables.Stats.get('global') await tables.Stats.put({ id: 'global', - totalSaved: ((stats?.totalSaved) ?? 0) + savedCost, - cacheHits: ((stats?.cacheHits) ?? 0) + 1, - updatedAt: new Date().toISOString(), + totalSaved: (stats?.totalSaved ?? 0) + savedCost, + cacheHits: ((stats?.cacheHits) ?? 0) + 1, + updatedAt: new Date().toISOString(), }) - } catch {} - const tStats = Date.now() - t5 - console.log('[Agent] cache hit stats update:', tStats + 'ms') + } catch (err) { + logger.warn('[Agent] savings counter update failed', err) + } return { conversationId, message: { role: 'assistant', content: cachedReply.content }, @@ -157,69 +169,60 @@ export class Agent extends Resource { latencyMs: Date.now() - startTime, timing, tokens: { input: 0, output: 0, total: 0 }, - cost: { input: 0, output: 0, total: 0, saved: savedCost }, + cost: { input: 0, output: 0, total: 0, saved: savedCost }, vectorContext: { hit: true, count: 1, cached: true }, }, } } - // 5. Call Claude with web search enabled — standalone question, no conversation history. - // Anthropic executes searches server-side, no external search API or key required. - const messages = [{ role: 'user', content: message }] - - const tools = isVertex() ? [] : [WEB_SEARCH_TOOL] - - let apiResponse = await getClient().messages.create({ - model: getModel(), - max_tokens: 1024, - ...(tools.length && { tools }), - system: SYSTEM_PROMPT, - messages, - }) - - // Handle pause_turn — server hit the max_uses limit mid-response; continue once - if (apiResponse.stop_reason === 'pause_turn') { - apiResponse = await getClient().messages.create({ - model: getModel(), - max_tokens: 1024, - ...(tools.length && { tools }), + // 5. Generate via models.generate() — routes to whatever backend the host + // has configured for `models.generative.default` (the shared inference process + // on Fabric GPU hosts, Ollama / OpenAI / Anthropic / Bedrock elsewhere). + const result = await models.generate( + { + messages: [{ role: 'user', content: message }], system: SYSTEM_PROMPT, - messages: [...messages, { role: 'assistant', content: apiResponse.content }], - }) - } + }, + { maxTokens: 1024 }, + ) const latencyMs = Date.now() - startTime + const assistantContent = result.content?.trim() ?? '' + if (!assistantContent) { + const err = new Error(`Model returned no content (finishReason: ${result.finishReason})`) + err.statusCode = 502 + throw err + } + // A truncated or filtered answer is still worth returning, but persisting it would seed + // the cache: every near-miss question thereafter is served the partial text as complete. + const isComplete = result.finishReason === 'stop' + const promptTokens = result.usage?.promptTokens ?? estimateTokens(SYSTEM_PROMPT + message) + const completionTokens = result.usage?.completionTokens ?? estimateTokens(assistantContent) + const tokensAreMeasured = result.usage?.promptTokens !== undefined + const estimatedCost = estimateClaudeCost(promptTokens, completionTokens) - // The API can split the answer across multiple text blocks (sentence fragments joined - // without separators) and may emit a text block BEFORE the web search tool call. - // Strategy: find the last non-text block (tool use / search result) and take only the - // text blocks that follow it — these form the actual answer. Join with '' since the - // fragments are already continuous prose. Falls back to all text blocks if no tools used. - const lastToolIdx = apiResponse.content.reduce((acc, b, i) => b.type !== 'text' ? i : acc, -1) - const assistantContent = apiResponse.content - .slice(lastToolIdx + 1) - .filter((b) => b.type === 'text') - .map((b) => b.text) - .join('') - .trim() - - const { input_tokens, output_tokens } = apiResponse.usage - const webSearches = apiResponse.usage?.server_tool_use?.web_search_requests ?? 0 - - // 9. Store the assistant's response with its embedding - const assistantMsgId = crypto.randomUUID() - const assistantEmbedding = await cachedEmbed(assistantContent) - const searchCost = webSearches * COST_PER_WEB_SEARCH - const totalCost = (input_tokens * COST_INPUT_PER_TOKEN) + (output_tokens * COST_OUTPUT_PER_TOKEN) + searchCost - await tables.Message.put({ - id: assistantMsgId, - conversationId, - role: 'assistant', - content: assistantContent, - cost: totalCost, - embedding: assistantEmbedding, - createdAt: new Date().toISOString(), - }) + // 9. Store the assistant's response. The cost rides along so a later cache hit on this + // question can credit it to `totalSaved`. + if (isComplete) { + // Not `cachedEmbed`: a generated reply is unique text, so the lookup always misses. + // A failure here must not discard an answer already paid for, so the row is stored + // unembedded; the candidate loop skips rows without an embedding. + let assistantEmbedding + try { + assistantEmbedding = await embed(assistantContent) + } catch (err) { + logger.warn('[Agent] reply embedding failed; storing message unindexed', err) + } + await tables.Message.put({ + id: crypto.randomUUID(), + conversationId, + role: 'assistant', + content: assistantContent, + cost: estimatedCost, + embedding: assistantEmbedding, + createdAt: new Date().toISOString(), + }) + } // 10. Update conversation timestamp await tables.Conversation.put({ @@ -232,19 +235,22 @@ export class Agent extends Resource { message: { role: 'assistant', content: assistantContent }, meta: { latencyMs, + timing, tokens: { - input: input_tokens, - output: output_tokens, - total: input_tokens + output_tokens, + input: promptTokens, + output: completionTokens, + total: promptTokens + completionTokens, }, cost: { - input: +(input_tokens * COST_INPUT_PER_TOKEN).toFixed(6), - output: +(output_tokens * COST_OUTPUT_PER_TOKEN).toFixed(6), - search: +searchCost.toFixed(6), - total: +totalCost.toFixed(6), + input: +(promptTokens * CLAUDE_COST_INPUT_PER_TOKEN).toFixed(6), + output: +(completionTokens * CLAUDE_COST_OUTPUT_PER_TOKEN).toFixed(6), + total: +estimatedCost.toFixed(6), + // `saved` is what cache hits credit; on a real generation it stays 0. + saved: 0, }, - webSearches, vectorContext: { hit: false, count: 0, cached: false }, + finishReason: result.finishReason, + tokensAreMeasured, }, } } diff --git a/resources/Chat.js b/resources/Chat.js index 41ce403..2fd05eb 100644 --- a/resources/Chat.js +++ b/resources/Chat.js @@ -195,8 +195,6 @@ const HTML = /* html */ ` .meta .pill.vector-miss .label { color: var(--muted); } .meta .pill.cache-hit { color: var(--btree-green); font-weight: 500; } .meta .pill.cache-hit .label { opacity: 1; } - .meta .pill.web-search { color: #a78bfa; } - .meta .pill.web-search .label { color: #a78bfa; opacity: 1; } /* ── Typing indicator ───────────────────────────────── */ .typing-wrap { align-self: flex-start; } @@ -330,7 +328,7 @@ const HTML = /* html */ ` Semantic Cache - cosine sim >= 0.88 + cosine sim >= 0.85 instant answers $0 LLM cost @@ -348,23 +346,23 @@ const HTML = /* html */ ` Response $0.00 • <50ms - + - Local SLM · bge-small-en-v1.5 - embeddings run in Harper · no API cost + Shared model · nomic-embed-text + models.embed() · GPU on Fabric · no API cost - + - + external - Claude Sonnet · Web Search - Anthropic API · response embedded by local SLM + models.generate() + host-configured backend · reply embedded in Harper - + embed & store in Harper @@ -462,14 +460,10 @@ const HTML = /* html */ ` const vecLabel = vectorContext.hit ? vectorContext.count + ' memor' + (vectorContext.count === 1 ? 'y' : 'ies') + ' recalled from Harper' : 'No vector context — LLM knowledge only' - const searchPill = (meta.webSearches > 0) - ? 'Web' + meta.webSearches + ' search' + (meta.webSearches > 1 ? 'es' : '') + '' - : '' div.innerHTML = 'Latency' + latency + '' + - 'Tokens' + tok + '' + - 'Cost' + usd + '' + - searchPill + + 'Tokens' + tok + (meta.tokensAreMeasured ? '' : ' (est.)') + '' + + 'Cost' + usd + ' (est.)' + 'Vector' + vecLabel + '' } return div diff --git a/schemas/schema.graphql b/schemas/schema.graphql index 0eeade9..9b7e3d1 100644 --- a/schemas/schema.graphql +++ b/schemas/schema.graphql @@ -15,7 +15,7 @@ type Message @table(expiration: 3600) @export { createdAt: String } -type EmbeddingCache @table @export { +type EmbeddingCache @table(expiration: 3600) @export { id: ID @primaryKey embedding: [Float] } diff --git a/scripts/download-model.js b/scripts/download-model.js deleted file mode 100644 index a8aeb63..0000000 --- a/scripts/download-model.js +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env node -// Downloads the bge-small-en-v1.5 embedding model if not already present. -// Run automatically via the predev / prestart npm hooks. - -import { createWriteStream, existsSync, mkdirSync } from 'fs' -import { pipeline } from 'stream/promises' -import { resolve } from 'path' -import { fileURLToPath } from 'url' - -const __dirname = fileURLToPath(new URL('.', import.meta.url)) -const modelsDir = resolve(__dirname, '../models') -const modelPath = resolve(modelsDir, 'bge-small-en-v1.5-q4_k_m.gguf') -const MODEL_URL = - 'https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf/resolve/main/bge-small-en-v1.5-q4_k_m.gguf' - -if (existsSync(modelPath)) { - console.log('✓ Embedding model already downloaded.') - process.exit(0) -} - -console.log('Downloading bge-small-en-v1.5 embedding model (~24 MB)...') -mkdirSync(modelsDir, { recursive: true }) - -const response = await fetch(MODEL_URL) -if (!response.ok) { - console.error(`Download failed: ${response.status} ${response.statusText}`) - process.exit(1) -} - -const total = Number(response.headers.get('content-length') || 0) -let downloaded = 0 - -const progress = new TransformStream({ - transform(chunk, controller) { - downloaded += chunk.byteLength - if (total) { - const pct = Math.round((downloaded / total) * 100) - process.stdout.write(`\r ${pct}% (${(downloaded / 1024 / 1024).toFixed(1)} MB)`) - } - controller.enqueue(chunk) - }, -}) - -await pipeline(response.body.pipeThrough(progress), createWriteStream(modelPath)) -console.log('\n✓ Model ready.')