diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d8cb3b..cd9084f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,9 +134,3 @@ jobs: - name: Run dependency review uses: actions/dependency-review-action@v4 if: github.event_name == 'pull_request' - with: - # Upstream baseline from @huggingface/transformers 4.2.0 native runtime deps. - # Keep this narrow and remove entries when HF/onnxruntime/sharp ship patched versions. - allow-ghsas: >- - GHSA-xcpc-8h2w-3j85, - GHSA-f88m-g3jw-g9cj diff --git a/README.md b/README.md index 79234c6..b72aa59 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Use it when you want: -- one indexing/search API across local and hosted embedding providers +- one indexing/search API across external embedding providers - direct control over vector dimensions, table names, and deployment shape - a lightweight library instead of a hosted search product @@ -45,7 +45,7 @@ Note for `0.17.x`: the client no longer exports `./package.json`, so `require("@ ## Quick Start -This example uses the default local provider. Local embeddings run in-process after the initial model download and cache warmup; they are not automatically air-gapped. +This example uses a separately deployed OpenAI-compatible embedding service. `libsql-search` never loads or hosts an embedding model in-process. ```ts import { createClient } from "@libsql/client"; @@ -56,25 +56,29 @@ const client = createClient({ authToken: "your-auth-token", }); -await createTable(client, "articles_local_384", 384); +const embeddingOptions = { + provider: "openai-compatible" as const, + baseUrl: process.env.EMBEDDING_BASE_URL!, + apiKey: process.env.EMBEDDING_API_KEY, + model: "bge-large-en-v1.5", + dimensions: 1024, +}; + +await createTable(client, "articles_bge_1024", 1024); await indexContent({ client, contentPath: "./content", - tableName: "articles_local_384", - embeddingOptions: { - provider: "local", - }, + tableName: "articles_bge_1024", + embeddingOptions, }); const results = await search({ client, query: "how do I deploy my docs site", - tableName: "articles_local_384", + tableName: "articles_bge_1024", limit: 5, - embeddingOptions: { - provider: "local", - }, + embeddingOptions, }); console.log(results.map((result) => ({ @@ -91,7 +95,7 @@ Important behavior: - `indexContent()` embeds every document before it touches the database, then replaces the table in one transaction, so a failed rebuild leaves the previous index intact. - `indexContent()` throws `IndexingError` when a file fails; pass `failurePolicy: "skip"` to rebuild from the remaining files. - `indexContent()` throws `IndexingError` when no source files are found; pass `allowEmptyIndex: true` to intentionally empty the index. -- Hosted providers send indexed and queried text to external services and may incur provider charges. +- Every provider sends indexed and queried text to an external service; review that service's privacy, retention, and pricing terms. ## Search Accuracy And Performance @@ -103,10 +107,10 @@ Two options control the trade-off: ```ts // Widen the index probe to raise recall (default: max(limit * 4, 32)) -await search({ client, query, limit: 10, candidates: 200 }); +await search({ client, query, embeddingOptions, limit: 10, candidates: 200 }); // Bypass the index entirely: exact, but linear in table size -await search({ client, query, exact: true }); +await search({ client, query, embeddingOptions, exact: true }); ``` `exact: true` is the only way to guarantee exactness. Use it for small corpora, for correctness checks against the index path, and for tables that have no vector index. @@ -117,7 +121,6 @@ Requirements: `vector_top_k()` and `libsql_vector_idx()` need a libSQL build wit Built-in providers: -- `local` with `Xenova/all-MiniLM-L6-v2` at 384 dimensions - `cloudflare` with `@cf/baai/bge-m3` at 1024 dimensions - `mistral` with `mistral-embed` at 1024 dimensions - `gemini` with `gemini-embedding-2` at 128-3072 dimensions, default 3072 diff --git a/docs/API.md b/docs/API.md index f853679..40f0272 100644 --- a/docs/API.md +++ b/docs/API.md @@ -113,7 +113,7 @@ Indexes Markdown files from a directory on disk. interface IndexerOptions { client: Client | DatabaseAdapter; contentPath: string; - embeddingOptions?: EmbeddingOptions; + embeddingOptions: EmbeddingOptions; fileExtensions?: string[]; exclude?: string[]; tableName?: string; @@ -187,7 +187,16 @@ class IndexingError extends Error { import { indexContent, IndexingError } from "libsql-search"; try { - await indexContent({ client, contentPath: "./content" }); + await indexContent({ + client, + contentPath: "./content", + embeddingOptions: { + provider: "openai-compatible", + baseUrl: process.env.EMBEDDING_BASE_URL!, + model: "bge-large-en-v1.5", + dimensions: 1024, + }, + }); } catch (error) { if (error instanceof IndexingError) { console.error(error.phase, error.failures); @@ -212,7 +221,7 @@ interface SearchOptions { query: string; limit?: number; tableName?: string; - embeddingOptions?: EmbeddingOptions; + embeddingOptions: EmbeddingOptions; candidates?: number; exact?: boolean; } @@ -246,7 +255,7 @@ Controls how many rows the index returns for the exact re-rank. It must be an in ```ts // Trade query cost for recall on a large corpus -const results = await search({ client, query, limit: 10, candidates: 200 }); +const results = await search({ client, query, embeddingOptions, limit: 10, candidates: 200 }); ``` `candidates` has no effect when `exact` is `true` — that path scans every row — but it is **still validated**. `search({ exact: true, limit: 10, candidates: 5 })` throws, exactly as it would on the index path. Validity does not depend on which path a call happens to take. @@ -262,7 +271,7 @@ Related exported constants: Set `exact: true` to bypass the index and score every row in the table. ```ts -const results = await search({ client, query, exact: true }); +const results = await search({ client, query, embeddingOptions, exact: true }); ``` This is the guaranteed-exact path: it computes `vector_distance_cos` for every row with a non-`NULL` embedding, sorts by `(distance, id)`, and trims to `limit`. Cost grows linearly with table size, so it is intended for small corpora, correctness checks against the index path, and tables that have no vector index. @@ -330,8 +339,7 @@ All retrieval helpers validate `tableName` before executing SQL, and all of them ```ts interface EmbeddingOptions { - provider?: - | "local" + provider: | "cloudflare" | "mistral" | "gemini" @@ -353,7 +361,7 @@ interface EmbeddingOptions { Important option rules: -- `provider` defaults to `local` +- `provider` is required; every provider is an external service - `maxLength` defaults to `8000` - `timeoutMs` defaults to `30000` - `model` is only used by `openai-compatible` @@ -364,7 +372,6 @@ Important option rules: Dimension rules: -- local: fixed `384` - Cloudflare: fixed `1024` - Mistral: fixed `1024` - Gemini: default `3072`, allowed integer range `128-3072` @@ -373,7 +380,7 @@ Dimension rules: See [Provider matrix and credential rules](./PROVIDERS.md) for the canonical provider table. -### `generateEmbedding(text, options?)` +### `generateEmbedding(text, options)` Generates one embedding vector. @@ -385,15 +392,15 @@ const embedding = await generateEmbedding("deploy docs", { }); ``` -### `generateEmbeddings(texts, options?)` +### `generateEmbeddings(texts, options)` Generates an ordered batch of embeddings. -- empty batches return `[]` without loading the local model or making a hosted call +- empty batches return `[]` without configuring credentials or making a provider call - OpenAI batches above `2048` inputs are rejected before network work - `openai-compatible` batches are chunked sequentially according to `batchSize` -### `createEmbeddingProvider(options?)` +### `createEmbeddingProvider(options)` Creates a provider client with immutable metadata and an `embed(texts, options?)` method. @@ -413,7 +420,7 @@ Provider clients return a rich `EmbeddingBatchResult`; the compatibility helpers Hosted provider clients are scoped to their current options. The library does not reuse a Cloudflare, Mistral, Gemini, or OpenAI client across different credential sets or configurations. -### `getEmbeddingProviderMetadata(options?)` +### `getEmbeddingProviderMetadata(options)` Returns the same metadata exposed by `createEmbeddingProvider(options).metadata` without resolving hosted-provider credentials. @@ -422,7 +429,6 @@ Metadata shape: ```ts interface EmbeddingProviderMetadata { name: - | "local" | "cloudflare" | "mistral" | "gemini" @@ -451,7 +457,6 @@ Batch interpretation: interface EmbeddingBatchResult { embeddings: number[][]; provider: - | "local" | "cloudflare" | "mistral" | "gemini" @@ -474,7 +479,7 @@ Validates provider responses before they reach the database: ### `padEmbedding(embedding, targetDimensions)` -Pads or truncates a vector to the target width. This is exported for compatibility and migration workflows, but the current local provider uses its native `384` dimensions rather than padding by default. +Pads or truncates a vector to the target width. This is exported for compatibility and migration workflows; provider adapters otherwise validate and preserve the vectors returned by their external service. ### `prepareTextForEmbedding(fields)` diff --git a/docs/INDEXING.md b/docs/INDEXING.md index 2a72888..22cac66 100644 --- a/docs/INDEXING.md +++ b/docs/INDEXING.md @@ -18,13 +18,18 @@ The slug is derived from the file path relative to `contentPath`. `indexContent()` replaces the whole target table: ```ts +const embeddingOptions = { + provider: "openai-compatible" as const, + baseUrl: process.env.EMBEDDING_BASE_URL!, + model: "bge-large-en-v1.5", + dimensions: 1024, +}; + await indexContent({ client, contentPath: "./content", - tableName: "articles_local_384", - embeddingOptions: { - provider: "local", - }, + tableName: "articles_bge_1024", + embeddingOptions, }); ``` @@ -69,7 +74,7 @@ Both are governed by `failurePolicy` like any other build failure, so they abort import { indexContent, IndexingError } from "libsql-search"; try { - await indexContent({ client, contentPath: "./content" }); + await indexContent({ client, contentPath: "./content", embeddingOptions }); } catch (error) { if (error instanceof IndexingError) { for (const failure of error.failures) { @@ -87,6 +92,7 @@ By default one bad file aborts the whole rebuild. To index everything that can b const result = await indexContent({ client, contentPath: "./content", + embeddingOptions, failurePolicy: "skip", }); @@ -105,6 +111,7 @@ An empty source directory throws by default, because silently leaving stale rows await indexContent({ client, contentPath: "./content", + embeddingOptions, allowEmptyIndex: true, }); ``` @@ -173,7 +180,6 @@ Many projects wire indexing into a dedicated script and call it before their sit ## Runtime Notes -- local embeddings may download and cache a model on the first run - Node users need `@libsql/client` installed alongside the package, at `^0.15.0 || ^0.17.0`; the packaged build is smoke-tested against both arms (`0.15.15` and `0.17.4`), which covers table and vector-index creation. `batch()` rollback behaves identically on both at the contract level, though its error text differs — see [Version differences](./TROUBLESHOOTING.md#libsqlclient-version-differences). Upgrading the client is not a prerequisite for upgrading this package. Deno/JSR users are not covered by that range and should pin the client themselves — see [Install](../README.md#install) -- hosted providers send indexed or queried text to external services +- all embedding providers send indexed or queried text to external services; this library never loads a model in-process - the repository validates package build and `deno check`, but indexing still depends on filesystem access diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 03208a3..b735fd0 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -55,13 +55,6 @@ These presets keep credentials out of the source file while making dimensions an ```ts const providerPresets = { - local: { - tableName: "articles_local_384", - dimensions: 384, - embeddingOptions: { - provider: "local" as const, - }, - }, cloudflare: { tableName: "articles_cf_bgem3_1024", dimensions: 1024, @@ -137,9 +130,13 @@ export const POST: APIRoute = async ({ request }) => { client, query, limit, - tableName: "articles_local_384", + tableName: "articles_tei_1024", embeddingOptions: { - provider: "local", + provider: "openai-compatible", + baseUrl: process.env.EMBEDDING_BASE_URL!, + model: process.env.EMBEDDING_MODEL!, + dimensions: 1024, + apiKey: process.env.EMBEDDING_API_KEY, intent: "query", }, }); @@ -162,14 +159,14 @@ const client = createClient({ }); export async function getStaticPaths() { - const articles = await getAllArticles(client, "articles_local_384"); + const articles = await getAllArticles(client, "articles_tei_1024"); return articles.map((article) => ({ params: { slug: article.slug }, })); } -const article = await getArticleBySlug(client, "guides/getting-started", "articles_local_384"); +const article = await getArticleBySlug(client, "guides/getting-started", "articles_tei_1024"); ``` ## Next.js Route Handler @@ -191,9 +188,13 @@ export async function POST(request: NextRequest) { client, query, limit, - tableName: "articles_local_384", + tableName: "articles_tei_1024", embeddingOptions: { - provider: "local", + provider: "openai-compatible", + baseUrl: process.env.EMBEDDING_BASE_URL!, + model: process.env.EMBEDDING_MODEL!, + dimensions: 1024, + apiKey: process.env.EMBEDDING_API_KEY, intent: "query", }, }); @@ -214,7 +215,7 @@ const client = createClient({ }); export async function generateStaticParams() { - const articles = await getAllArticles(client, "articles_local_384"); + const articles = await getAllArticles(client, "articles_tei_1024"); return articles.map((article) => ({ slug: article.slug, @@ -223,7 +224,7 @@ export async function generateStaticParams() { export default async function Page({ params }: { params: Promise<{ slug: string }> }) { const { slug } = await params; - const article = await getArticleBySlug(client, slug, "articles_local_384"); + const article = await getArticleBySlug(client, slug, "articles_tei_1024"); return
{article?.title}
; } @@ -241,13 +242,6 @@ const client = createClient({ }); const providerPresets = { - local: { - tableName: "articles_local_384", - dimensions: 384, - embeddingOptions: { - provider: "local" as const, - }, - }, cloudflare: { tableName: "articles_cf_bgem3_1024", dimensions: 1024, @@ -299,7 +293,7 @@ const providerPresets = { }, } as const; -const provider = process.env.EMBEDDING_PROVIDER ?? "local"; +const provider = process.env.EMBEDDING_PROVIDER ?? "openai-compatible"; if (!(provider in providerPresets)) { throw new Error( @@ -349,4 +343,4 @@ await generateEmbeddings(["doc one", "doc two"], { }); ``` -The repository test suite should not require real provider credentials. See [Testing guidance](./TESTING.md) for local model mocks, Gemini SDK mocks, and validation-before-network assertions. +The repository test suite should not require real provider credentials. See [Testing guidance](./TESTING.md) for embedding-service mocks, Gemini SDK mocks, and validation-before-network assertions. diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md index dea7c40..4773378 100644 --- a/docs/MIGRATIONS.md +++ b/docs/MIGRATIONS.md @@ -27,7 +27,7 @@ References: In practice, this means: -- `384 local` and `1024 Mistral` can never share a table +- a legacy `384` embedding space and `1024 Mistral` can never share a table - `1024 Cloudflare` and `1024 Mistral` still need separate rebuilds because equal width does not make the vectors compatible - a custom endpoint change at the same width still needs a new table because the model or serving stack may have changed @@ -84,14 +84,14 @@ Re-running `createTable()` with the table's existing width is the fix. It is ide ```ts // Same name and same width as the existing table -await createTable(client, "articles_local_384", 384); +await createTable(client, "articles_legacy_384", 384); ``` Equivalently, in SQL: ```sql -CREATE INDEX IF NOT EXISTS "articles_local_384_embedding_idx" -ON "articles_local_384"(libsql_vector_idx(embedding)); +CREATE INDEX IF NOT EXISTS "articles_legacy_384_embedding_idx" +ON "articles_legacy_384"(libsql_vector_idx(embedding)); ``` Reindexing does not create the index; `indexContent()` only replaces rows. Any new table created by `createTable()` as part of a migration already has it, so this applies only to pre-existing tables you are carrying forward. Until the index exists, `search({ ..., exact: true })` keeps queries working on the exact full-scan path. @@ -100,7 +100,7 @@ Reindexing does not create the index; `indexContent()` only replaces rows. Any n | From | To | Why a rebuild is required | Recommended table move | | --- | --- | --- | --- | -| Legacy padded local `768` | Native local `384` | Old tables stored `384` model values plus zero padding; current local provider is a native `384`-dimension space | Build into `articles_local_384`, validate, then retire the legacy table | +| Legacy in-process Transformers.js index | Any external provider | The runtime and provider were removed; every replacement service has its own embedding space | Build a parallel table named for the external provider/model, validate, then retire the legacy table | | Any `768` space | Any `1024` space | Width changes from `F32_BLOB(768)` to `F32_BLOB(1024)` | Create a new `1024` table and reindex | | Cloudflare `1024` | Mistral `1024` | Width stays the same, but provider/model space changes | Use a parallel `1024` table such as `articles_mistral_1024` | | Mistral `1024` | Cloudflare `1024` | Same reason in reverse | Use a parallel `1024` table such as `articles_cf_bgem3_1024` | @@ -111,21 +111,21 @@ Reindexing does not create the index; `indexContent()` only replaces rows. Any n ## Scenario Notes -### Legacy Local `768` To Native Local `384` +### Legacy In-Process Embeddings To An External Service -Earlier local migrations sometimes relied on zero padding to fit a `768`-wide table. The current local adapter emits the model's native `384` dimensions and rejects any other local dimension count. +Versions that exposed the in-process Transformers.js provider produced a separate embedding space that this release can no longer query. Choose an external provider or separately deployed OpenAI-compatible service and rebuild every vector into a new table. Safe path: ```ts -await createTable(client, "articles_local_384", 384); +await createTable(client, "articles_bge_1024", 1024); ``` -Reindex into `articles_local_384`; do not keep writing new local vectors into the legacy padded table. +Reindex into `articles_bge_1024` with the external service configuration; do not mix its vectors with the legacy table. ### `768` To `1024` -Any move from `768` dimensions to `1024` dimensions changes the schema width. Examples include a legacy local table moving to Cloudflare or Mistral. +Any move from `768` dimensions to `1024` dimensions changes the schema width. Examples include a legacy table moving to Cloudflare, Mistral, or a 1024-dimensional OpenAI-compatible service. ```ts await createTable(client, "articles_mistral_1024", 1024); diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 039e260..3a0ebc9 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -2,9 +2,8 @@ Use this page to choose an embedding provider, confirm the table width it needs, and understand what crosses a network boundary. -`libsql-search` supports these provider values: +`libsql-search` only talks to external embedding services; it never loads or hosts an embedding model in-process. It supports these provider values: -- `local` - `cloudflare` - `mistral` - `gemini` @@ -17,8 +16,7 @@ All providers share the same `EmbeddingOptions` surface: ```ts interface EmbeddingOptions { - provider?: - | "local" + provider: | "cloudflare" | "mistral" | "gemini" @@ -40,7 +38,7 @@ interface EmbeddingOptions { Shared defaults and rules: -- `provider` defaults to `local` +- `provider` is required; there is no implicit embedding runtime or service - `maxLength` defaults to `8000` code units - `timeoutMs` defaults to `30000` - `indexContent()` defaults to `intent: "document"` @@ -59,7 +57,6 @@ The `model` option is only used by `openai-compatible`. | Provider | Literal | Upstream model used by this adapter | Dimensions | Credentials | Batching | Network and privacy boundary | Cost and table planning | | --- | --- | --- | --- | --- | --- | --- | --- | -| Local | `local` | `Xenova/all-MiniLM-L6-v2` | Fixed `384` | None | Sequential in-process | No hosted API call. First use may download model artifacts and cache them locally. | No hosted API bill. Table must be `F32_BLOB(384)`. | | Cloudflare Workers AI | `cloudflare` | `@cf/baai/bge-m3` | Fixed `1024` | `accountId` and `apiToken`, or `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` | Native batch in one request | Indexed and queried text is sent to Cloudflare. | Check Cloudflare pricing before large rebuilds. Table must be `F32_BLOB(1024)`. | | Mistral | `mistral` | `mistral-embed` | Fixed `1024` | `apiKey`, or `MISTRAL_API_KEY` | Native batch in one request | Indexed and queried text is sent to Mistral. | Check Mistral pricing before rebuilds. Table must be `F32_BLOB(1024)`. | | Gemini | `gemini` | `gemini-embedding-2` | Default `3072`; allowed integers `128-3072` | `apiKey`, or `GEMINI_API_KEY` | Sequential SDK request per input | Indexed and queried text is sent to Google. The adapter currently rewrites payload text by intent. | Check Gemini pricing before rebuilds. Table width must match the chosen dimension count exactly. | @@ -68,23 +65,6 @@ The `model` option is only used by `openai-compatible`. ## Provider Notes -### Local - -```ts -embeddingOptions: { - provider: "local", -} -``` - -- fixed at `384` dimensions -- rejects any other `dimensions` value before loading the runtime -- uses `@huggingface/transformers` lazily and caches the local pipeline by model name - -References: - -- [Transformers.js in Node.js](https://huggingface.co/docs/transformers.js/en/tutorials/node) -- [Transformers.js environment and cache controls](https://huggingface.co/docs/transformers.js/en/api/env) - ### Cloudflare Workers AI ```ts diff --git a/docs/README.md b/docs/README.md index da82d4a..c7bf13a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,7 @@ This directory holds the longer-form reference material for `libsql-search`. -- [Provider selection and configuration](./PROVIDERS.md): compare local, hosted, and custom embedding providers before you build an index +- [Provider selection and configuration](./PROVIDERS.md): compare external and custom embedding providers before you build an index - [Integration examples](./INTEGRATIONS.md): reusable provider flow plus Astro and Next.js examples - [Migration and reindexing guide](./MIGRATIONS.md): table-width changes, provider/model swaps, and safe cutovers - [API reference](./API.md): exported functions, option shapes, and result data diff --git a/docs/TESTING.md b/docs/TESTING.md index aa6f84f..cd6bb62 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -9,30 +9,31 @@ Routine unit tests and CI should not make live embedding-provider calls and shou - validate option handling and response parsing with mocks first - assert failures happen before network calls when configuration is invalid -The current test suite follows that pattern in `tests/embeddings.test.ts` and [`tests/huggingface-transformers.mock.ts`](../tests/huggingface-transformers.mock.ts). +The current test suite follows that pattern in `tests/embeddings.test.ts` and [`tests/embedding-service.mock.ts`](../tests/embedding-service.mock.ts). -## Local Provider Mocks +## Shared Embedding Service Mock -The local provider should use a lightweight Transformers.js mock instead of downloading the real model during routine tests. +Indexer, search, and database tests use a deterministic OpenAI-compatible service mock. This keeps the tests on the same external-service boundary as production without making network calls. ```ts import { - huggingFaceTransformersMock, - resetHuggingFaceTransformersMock, -} from "./huggingface-transformers.mock.js"; + embeddingServiceMock, + resetEmbeddingServiceMock, + TEST_EMBEDDING_OPTIONS, +} from "./embedding-service.mock.js"; beforeEach(() => { - resetHuggingFaceTransformersMock(); + resetEmbeddingServiceMock(); }); ``` -The repository source file is `huggingface-transformers.mock.ts`. The example keeps the `.js` import suffix because this repo's ESM TypeScript source uses explicit `.js` relative imports that resolve after compilation. +The example keeps the `.js` import suffix because this repo's ESM TypeScript source uses explicit `.js` relative imports that resolve after compilation. Test the contract you care about: -- the library requests `Xenova/all-MiniLM-L6-v2` -- the call uses `pooling: "mean"` and `normalize: true` -- non-`384` local dimensions fail before runtime loading +- indexing and querying use the same endpoint, model, and dimensions +- provider failures remain classified as build-stage failures +- queued deterministic vectors exercise exact ranking and tie behavior ## HTTP Provider Mocks @@ -116,9 +117,9 @@ Apply the same pattern to `GEMINI_API_KEY`, `MISTRAL_API_KEY`, `CLOUDFLARE_ACCOU Prefer tests that prove bad inputs fail locally: - unknown provider +- missing provider - missing provider credentials - blank credentials where trimming is expected -- invalid local dimensions - invalid Gemini dimensions - invalid `openai-compatible` `baseUrl` - invalid `openai-compatible` `batchSize` diff --git a/docs/TROUBLESHOOTING-SHARP.md b/docs/TROUBLESHOOTING-SHARP.md deleted file mode 100644 index 1cad01e..0000000 --- a/docs/TROUBLESHOOTING-SHARP.md +++ /dev/null @@ -1,65 +0,0 @@ -# Troubleshooting: Transitive `sharp` Install Errors - -`libsql-search` does not directly import `sharp`, but local embeddings use -`@huggingface/transformers`, which currently brings in `sharp` as a transitive -runtime dependency. If you see an install error mentioning `sharp`, it is -usually a native-package install or approval issue. - -This page exists because the error can show up before your application reaches -any `libsql-search` code. - -## Typical Error - -```text -Cannot find module '../build/Release/sharp-*.node' -``` - -Or: - -```text -Error: Something went wrong installing the "sharp" module -``` - -## Why It Happens - -With pnpm, native packages may need explicit build-script approval. If the -relevant install script is blocked, the native binary is never downloaded or -built. - -## What To Do - -First inspect which build scripts pnpm blocked: - -```bash -pnpm ignored-builds -``` - -Then approve the package that is actually failing and reinstall: - -```bash -pnpm approve-builds -pnpm install -``` - -In the interactive `pnpm approve-builds` prompt, select `sharp` if that is the -package reporting the native-module failure. - -For a committed repository-level fix, you can also allow the package explicitly -in `pnpm-workspace.yaml` with `onlyBuiltDependencies`. - -## Relation To `libsql-search` - -- local embeddings use `@huggingface/transformers` -- the first local embedding run may download a model at runtime -- that runtime model download is separate from a pnpm native-module install - failure - -## Verification - -After reinstalling, rerun the command that originally failed. If your app uses -`sharp` directly, verify that import in your own project context. - -## Additional Resources - -- [pnpm approve-builds](https://pnpm.io/10.x/cli/approve-builds) -- [Sharp installation docs](https://sharp.pixelplumbing.com/install) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 808b807..967495f 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -1,9 +1,5 @@ # Troubleshooting -Use the page that matches the failure mode: - -- [Sharp native module issues](./TROUBLESHOOTING-SHARP.md) - Common operational checks: - verify you called `createTable()` before indexing or searching @@ -15,8 +11,6 @@ Common operational checks: - after upgrading an existing Gemini index, fully re-embed with `gemini-embedding-2`; for 3072-dimensional Gemini indexes, recreate the table or use a new table name before rebuilding -- after upgrading an existing local 768-dimensional padded index, create or - recreate a 384-dimensional table and fully re-index before querying it - if `search()` reports that the `_embedding_idx` vector index could not be used, the table has no embedding vector index: re-run `createTable()` with the table's existing name and width to add it without touching rows, or diff --git a/docs/TURSO.md b/docs/TURSO.md index 3de7c5a..9dee265 100644 --- a/docs/TURSO.md +++ b/docs/TURSO.md @@ -44,19 +44,26 @@ import { createTable, indexContent, search } from "libsql-search"; const database = await connect("./local.db"); const client = tursoAdapter(database); -await createTable(client, "articles", 384); +const embeddingOptions = { + provider: "openai-compatible" as const, + baseUrl: process.env.EMBEDDING_BASE_URL!, + model: "bge-large-en-v1.5", + dimensions: 1024, +}; + +await createTable(client, "articles", 1024); await indexContent({ client, contentPath: "./content", - embeddingOptions: { provider: "local" }, + embeddingOptions, }); const results = await search({ client, query: "how do I deploy my docs site", limit: 5, - embeddingOptions: { provider: "local" }, + embeddingOptions, }); ``` diff --git a/package.json b/package.json index ef31a34..758e794 100644 --- a/package.json +++ b/package.json @@ -75,7 +75,6 @@ } }, "dependencies": { - "@huggingface/transformers": "4.2.0", "gray-matter": "^4.0.3" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e18ef1e..5a2de6f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,9 +8,6 @@ importers: .: dependencies: - '@huggingface/transformers': - specifier: 4.2.0 - version: 4.2.0 gray-matter: specifier: ^4.0.3 version: 4.0.3 @@ -90,9 +87,6 @@ packages: resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} - '@emnapi/runtime@1.11.3': - resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} - '@esbuild/aix-ppc64@0.28.2': resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==} engines: {node: '>=18'} @@ -258,169 +252,6 @@ packages: '@modelcontextprotocol/sdk': optional: true - '@huggingface/jinja@0.5.9': - resolution: {integrity: sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==} - engines: {node: '>=18'} - - '@huggingface/tokenizers@0.1.3': - resolution: {integrity: sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==} - - '@huggingface/transformers@4.2.0': - resolution: {integrity: sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==} - - '@img/colour@1.1.0': - resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} - engines: {node: '>=18'} - - '@img/sharp-darwin-arm64@0.34.5': - resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [darwin] - - '@img/sharp-darwin-x64@0.34.5': - resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-darwin-arm64@1.2.4': - resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} - cpu: [arm64] - os: [darwin] - - '@img/sharp-libvips-darwin-x64@1.2.4': - resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-linux-arm64@1.2.4': - resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-arm@1.2.4': - resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-ppc64@1.2.4': - resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-riscv64@1.2.4': - resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-s390x@1.2.4': - resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-x64@1.2.4': - resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linuxmusl-arm64@1.2.4': - resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@img/sharp-libvips-linuxmusl-x64@1.2.4': - resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@img/sharp-linux-arm64@0.34.5': - resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-arm@0.34.5': - resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-ppc64@0.34.5': - resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-riscv64@0.34.5': - resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-s390x@0.34.5': - resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-x64@0.34.5': - resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@img/sharp-linuxmusl-arm64@0.34.5': - resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@img/sharp-linuxmusl-x64@0.34.5': - resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - libc: [musl] - - '@img/sharp-wasm32@0.34.5': - resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [wasm32] - - '@img/sharp-win32-arm64@0.34.5': - resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [win32] - - '@img/sharp-win32-ia32@0.34.5': - resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ia32] - os: [win32] - - '@img/sharp-win32-x64@0.34.5': - resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [win32] - '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -797,10 +628,6 @@ packages: '@vitest/utils@4.1.11': resolution: {integrity: sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==} - adm-zip@0.5.18: - resolution: {integrity: sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==} - engines: {node: '>=12.0'} - agent-base@7.1.4: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} @@ -817,10 +644,6 @@ packages: bignumber.js@9.3.1: resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} - boolean@3.2.0: - resolution: {integrity: sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. - buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} @@ -851,54 +674,24 @@ packages: resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} engines: {node: '>=0.10.0'} - define-data-property@1.1.4: - resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} - engines: {node: '>= 0.4'} - - define-properties@1.2.1: - resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} - engines: {node: '>= 0.4'} - detect-libc@2.0.2: resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} engines: {node: '>=8'} - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - detect-node@2.1.0: - resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} - ecdsa-sig-formatter@1.0.11: resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - es-define-property@1.0.1: - resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} - engines: {node: '>= 0.4'} - - es-errors@1.3.0: - resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} - engines: {node: '>= 0.4'} - es-module-lexer@1.7.0: resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} es-module-lexer@2.3.2: resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} - es6-error@4.1.1: - resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} - esbuild@0.28.2: resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==} engines: {node: '>=18'} hasBin: true - escape-string-regexp@4.0.0: - resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} - engines: {node: '>=10'} - esprima@4.0.1: resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} engines: {node: '>=4'} @@ -937,9 +730,6 @@ packages: fflate@0.8.2: resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} - flatbuffers@25.9.23: - resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} - flatted@3.4.4: resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} @@ -966,14 +756,6 @@ packages: get-tsconfig@4.14.3: resolution: {integrity: sha512-++QEw4DIY7WGoukz+/+A/8dGYPT9l9yIadnmSgZ8Rjr3YVSVDipQSO9CdnJo9ePqFqUUqh+wk9uIaoiAwsiPkA==} - global-agent@3.0.0: - resolution: {integrity: sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==} - engines: {node: '>=10.0'} - - globalthis@1.0.4: - resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} - engines: {node: '>= 0.4'} - google-auth-library@10.9.1: resolution: {integrity: sha512-i1ydyHrqcIxXkWh/uBmVkzCvIuq5yiK2ATndIe5XxKholrG/MTYP9xGYka4sQhrbIAgGjL2B6NOE7rFaiF3fXw==} engines: {node: '>=18'} @@ -982,24 +764,14 @@ packages: resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} engines: {node: '>=14'} - gopd@1.2.0: - resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} - engines: {node: '>= 0.4'} - gray-matter@4.0.3: resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} engines: {node: '>=6.0'} - guid-typescript@1.0.9: - resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} - has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} - has-property-descriptors@1.0.2: - resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} - hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} @@ -1050,9 +822,6 @@ packages: json-bigint@1.0.0: resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} - json-stringify-safe@5.0.1: - resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} - jwa@2.0.1: resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} @@ -1081,10 +850,6 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} - matcher@3.0.0: - resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==} - engines: {node: '>=10'} - mrmime@2.0.1: resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==} engines: {node: '>=10'} @@ -1106,27 +871,10 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - object-keys@1.1.1: - resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} - engines: {node: '>= 0.4'} - obug@2.1.4: resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} engines: {node: '>=12.20.0'} - onnxruntime-common@1.24.0-dev.20251116-b39e144322: - resolution: {integrity: sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==} - - onnxruntime-common@1.24.3: - resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==} - - onnxruntime-node@1.24.3: - resolution: {integrity: sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==} - os: [win32, darwin, linux] - - onnxruntime-web@1.26.0-dev.20260416-b7804b056c: - resolution: {integrity: sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==} - p-retry@4.6.2: resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} engines: {node: '>=8'} @@ -1144,9 +892,6 @@ packages: resolution: {integrity: sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==} engines: {node: '>=12'} - platform@1.3.6: - resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} - postcss@8.5.26: resolution: {integrity: sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==} engines: {node: ^10 || ^12 || >=14} @@ -1170,10 +915,6 @@ packages: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} - roarr@2.15.4: - resolution: {integrity: sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==} - engines: {node: '>=8.0'} - rollup-plugin-dts@6.5.1: resolution: {integrity: sha512-jODTXp3H7MK/Ur/ErtsrQ0G1GvaCmc3du+y5pNrdBMf6d7HlL2Nd/N6TkEr+f75CkUj01zEoEd7y2elH0eHi1Q==} engines: {node: '>=20'} @@ -1204,22 +945,11 @@ packages: resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} engines: {node: '>=4'} - semver-compare@1.0.0: - resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} - semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} hasBin: true - serialize-error@7.0.1: - resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==} - engines: {node: '>=10'} - - sharp@0.34.5: - resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -1234,9 +964,6 @@ packages: sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - sprintf-js@1.1.3: - resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} - stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -1277,10 +1004,6 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - type-fest@0.13.1: - resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} - engines: {node: '>=10'} - typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -1421,11 +1144,6 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} - '@emnapi/runtime@1.11.3': - dependencies: - tslib: 2.8.1 - optional: true - '@esbuild/aix-ppc64@0.28.2': optional: true @@ -1516,114 +1234,6 @@ snapshots: - utf-8-validate optional: true - '@huggingface/jinja@0.5.9': {} - - '@huggingface/tokenizers@0.1.3': {} - - '@huggingface/transformers@4.2.0': - dependencies: - '@huggingface/jinja': 0.5.9 - '@huggingface/tokenizers': 0.1.3 - onnxruntime-node: 1.24.3 - onnxruntime-web: 1.26.0-dev.20260416-b7804b056c - sharp: 0.34.5 - - '@img/colour@1.1.0': {} - - '@img/sharp-darwin-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-darwin-arm64': 1.2.4 - optional: true - - '@img/sharp-darwin-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-darwin-x64': 1.2.4 - optional: true - - '@img/sharp-libvips-darwin-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-darwin-x64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-arm@1.2.4': - optional: true - - '@img/sharp-libvips-linux-ppc64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-riscv64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-s390x@1.2.4': - optional: true - - '@img/sharp-libvips-linux-x64@1.2.4': - optional: true - - '@img/sharp-libvips-linuxmusl-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-linuxmusl-x64@1.2.4': - optional: true - - '@img/sharp-linux-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm64': 1.2.4 - optional: true - - '@img/sharp-linux-arm@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm': 1.2.4 - optional: true - - '@img/sharp-linux-ppc64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-ppc64': 1.2.4 - optional: true - - '@img/sharp-linux-riscv64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-riscv64': 1.2.4 - optional: true - - '@img/sharp-linux-s390x@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-s390x': 1.2.4 - optional: true - - '@img/sharp-linux-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-x64': 1.2.4 - optional: true - - '@img/sharp-linuxmusl-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 - optional: true - - '@img/sharp-linuxmusl-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-x64': 1.2.4 - optional: true - - '@img/sharp-wasm32@0.34.5': - dependencies: - '@emnapi/runtime': 1.11.3 - optional: true - - '@img/sharp-win32-arm64@0.34.5': - optional: true - - '@img/sharp-win32-ia32@0.34.5': - optional: true - - '@img/sharp-win32-x64@0.34.5': - optional: true - '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -1708,25 +1318,34 @@ snapshots: '@polka/url@1.0.0-next.29': {} - '@protobufjs/aspromise@1.1.2': {} + '@protobufjs/aspromise@1.1.2': + optional: true - '@protobufjs/base64@1.1.2': {} + '@protobufjs/base64@1.1.2': + optional: true - '@protobufjs/codegen@2.0.5': {} + '@protobufjs/codegen@2.0.5': + optional: true - '@protobufjs/eventemitter@1.1.1': {} + '@protobufjs/eventemitter@1.1.1': + optional: true '@protobufjs/fetch@1.1.1': dependencies: '@protobufjs/aspromise': 1.1.2 + optional: true - '@protobufjs/float@1.0.2': {} + '@protobufjs/float@1.0.2': + optional: true - '@protobufjs/path@1.1.2': {} + '@protobufjs/path@1.1.2': + optional: true - '@protobufjs/pool@1.1.0': {} + '@protobufjs/pool@1.1.0': + optional: true - '@protobufjs/utf8@1.1.2': {} + '@protobufjs/utf8@1.1.2': + optional: true '@rollup/plugin-commonjs@29.0.3(rollup@4.62.5)': dependencies: @@ -1947,8 +1566,6 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.1 - adm-zip@0.5.18: {} - agent-base@7.1.4: optional: true @@ -1968,8 +1585,6 @@ snapshots: bignumber.js@9.3.1: optional: true - boolean@3.2.0: {} - buffer-equal-constant-time@1.0.1: optional: true @@ -1988,39 +1603,17 @@ snapshots: deepmerge@4.3.1: {} - define-data-property@1.1.4: - dependencies: - es-define-property: 1.0.1 - es-errors: 1.3.0 - gopd: 1.2.0 - - define-properties@1.2.1: - dependencies: - define-data-property: 1.1.4 - has-property-descriptors: 1.0.2 - object-keys: 1.1.1 - detect-libc@2.0.2: {} - detect-libc@2.1.2: {} - - detect-node@2.1.0: {} - ecdsa-sig-formatter@1.0.11: dependencies: safe-buffer: 5.2.1 optional: true - es-define-property@1.0.1: {} - - es-errors@1.3.0: {} - es-module-lexer@1.7.0: {} es-module-lexer@2.3.2: {} - es6-error@4.1.1: {} - esbuild@0.28.2: optionalDependencies: '@esbuild/aix-ppc64': 0.28.2 @@ -2050,8 +1643,6 @@ snapshots: '@esbuild/win32-ia32': 0.28.2 '@esbuild/win32-x64': 0.28.2 - escape-string-regexp@4.0.0: {} - esprima@4.0.1: {} estree-walker@2.0.2: {} @@ -2081,8 +1672,6 @@ snapshots: fflate@0.8.2: {} - flatbuffers@25.9.23: {} - flatted@3.4.4: {} formdata-polyfill@4.0.10: @@ -2117,20 +1706,6 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 - global-agent@3.0.0: - dependencies: - boolean: 3.2.0 - es6-error: 4.1.1 - matcher: 3.0.0 - roarr: 2.15.4 - semver: 7.8.5 - serialize-error: 7.0.1 - - globalthis@1.0.4: - dependencies: - define-properties: 1.2.1 - gopd: 1.2.0 - google-auth-library@10.9.1: dependencies: base64-js: 1.5.1 @@ -2146,8 +1721,6 @@ snapshots: google-logging-utils@1.1.3: optional: true - gopd@1.2.0: {} - gray-matter@4.0.3: dependencies: js-yaml: 3.15.1 @@ -2155,14 +1728,8 @@ snapshots: section-matter: 1.0.0 strip-bom-string: 1.0.0 - guid-typescript@1.0.9: {} - has-flag@4.0.0: {} - has-property-descriptors@1.0.2: - dependencies: - es-define-property: 1.0.1 - hasown@2.0.2: dependencies: function-bind: 1.1.2 @@ -2216,8 +1783,6 @@ snapshots: bignumber.js: 9.3.1 optional: true - json-stringify-safe@5.0.1: {} - jwa@2.0.1: dependencies: buffer-equal-constant-time: 1.0.1 @@ -2248,7 +1813,8 @@ snapshots: '@libsql/linux-x64-musl': 0.5.29 '@libsql/win32-x64-msvc': 0.5.29 - long@5.3.2: {} + long@5.3.2: + optional: true magic-string@0.30.21: dependencies: @@ -2264,10 +1830,6 @@ snapshots: dependencies: semver: 7.8.5 - matcher@3.0.0: - dependencies: - escape-string-regexp: 4.0.0 - mrmime@2.0.1: {} ms@2.1.3: {} @@ -2284,29 +1846,8 @@ snapshots: formdata-polyfill: 4.0.10 optional: true - object-keys@1.1.1: {} - obug@2.1.4: {} - onnxruntime-common@1.24.0-dev.20251116-b39e144322: {} - - onnxruntime-common@1.24.3: {} - - onnxruntime-node@1.24.3: - dependencies: - adm-zip: 0.5.18 - global-agent: 3.0.0 - onnxruntime-common: 1.24.3 - - onnxruntime-web@1.26.0-dev.20260416-b7804b056c: - dependencies: - flatbuffers: 25.9.23 - guid-typescript: 1.0.9 - long: 5.3.2 - onnxruntime-common: 1.24.0-dev.20251116-b39e144322 - platform: 1.3.6 - protobufjs: 7.6.5 - p-retry@4.6.2: dependencies: '@types/retry': 0.12.0 @@ -2321,8 +1862,6 @@ snapshots: picomatch@4.0.5: {} - platform@1.3.6: {} - postcss@8.5.26: dependencies: nanoid: 3.3.18 @@ -2344,6 +1883,7 @@ snapshots: '@protobufjs/utf8': 1.1.2 '@types/node': 24.13.3 long: 5.3.2 + optional: true resolve-pkg-maps@1.0.0: {} @@ -2356,15 +1896,6 @@ snapshots: retry@0.13.1: optional: true - roarr@2.15.4: - dependencies: - boolean: 3.2.0 - detect-node: 2.1.0 - globalthis: 1.0.4 - json-stringify-safe: 5.0.1 - semver-compare: 1.0.0 - sprintf-js: 1.1.3 - rollup-plugin-dts@6.5.1(rollup@4.62.5)(typescript@5.9.3): dependencies: '@jridgewell/remapping': 2.3.5 @@ -2427,45 +1958,8 @@ snapshots: extend-shallow: 2.0.1 kind-of: 6.0.3 - semver-compare@1.0.0: {} - semver@7.8.5: {} - serialize-error@7.0.1: - dependencies: - type-fest: 0.13.1 - - sharp@0.34.5: - dependencies: - '@img/colour': 1.1.0 - detect-libc: 2.1.2 - semver: 7.8.5 - optionalDependencies: - '@img/sharp-darwin-arm64': 0.34.5 - '@img/sharp-darwin-x64': 0.34.5 - '@img/sharp-libvips-darwin-arm64': 1.2.4 - '@img/sharp-libvips-darwin-x64': 1.2.4 - '@img/sharp-libvips-linux-arm': 1.2.4 - '@img/sharp-libvips-linux-arm64': 1.2.4 - '@img/sharp-libvips-linux-ppc64': 1.2.4 - '@img/sharp-libvips-linux-riscv64': 1.2.4 - '@img/sharp-libvips-linux-s390x': 1.2.4 - '@img/sharp-libvips-linux-x64': 1.2.4 - '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 - '@img/sharp-libvips-linuxmusl-x64': 1.2.4 - '@img/sharp-linux-arm': 0.34.5 - '@img/sharp-linux-arm64': 0.34.5 - '@img/sharp-linux-ppc64': 0.34.5 - '@img/sharp-linux-riscv64': 0.34.5 - '@img/sharp-linux-s390x': 0.34.5 - '@img/sharp-linux-x64': 0.34.5 - '@img/sharp-linuxmusl-arm64': 0.34.5 - '@img/sharp-linuxmusl-x64': 0.34.5 - '@img/sharp-wasm32': 0.34.5 - '@img/sharp-win32-arm64': 0.34.5 - '@img/sharp-win32-ia32': 0.34.5 - '@img/sharp-win32-x64': 0.34.5 - siginfo@2.0.0: {} sirv@3.0.2: @@ -2478,8 +1972,6 @@ snapshots: sprintf-js@1.0.3: {} - sprintf-js@1.1.3: {} - stackback@0.0.2: {} std-env@4.2.0: {} @@ -2507,8 +1999,6 @@ snapshots: tslib@2.8.1: {} - type-fest@0.13.1: {} - typescript@5.9.3: {} undici-types@7.18.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index f49ca84..efc037a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,11 +1,2 @@ onlyBuiltDependencies: - esbuild - - onnxruntime-node - - sharp - -auditConfig: - # Upstream baseline from @huggingface/transformers 4.2.0 native runtime deps. - # Keep this narrow and remove entries when HF/onnxruntime/sharp ship patched versions. - ignoreGhsas: - - GHSA-xcpc-8h2w-3j85 - - GHSA-f88m-g3jw-g9cj diff --git a/rollup.config.js b/rollup.config.js index ac834e4..65e5647 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -4,16 +4,12 @@ import esbuild from 'rollup-plugin-esbuild'; const external = [ '@libsql/client', - '@huggingface/transformers', 'gray-matter', '@google/genai', // Never bundled, and never imported by our source either. Listed so that a // future import of it fails the build loudly instead of being inlined into // an entry point that must not resolve a native package. '@tursodatabase/database', - 'onnxruntime-node', - 'onnxruntime-web', - 'sharp', 'fs', 'path', 'fs/promises', diff --git a/rollup.dts.config.js b/rollup.dts.config.js index e5b256d..57a74a6 100644 --- a/rollup.dts.config.js +++ b/rollup.dts.config.js @@ -2,13 +2,9 @@ import dts from 'rollup-plugin-dts'; const external = [ '@libsql/client', - '@huggingface/transformers', 'gray-matter', '@google/genai', '@tursodatabase/database', - 'onnxruntime-node', - 'onnxruntime-web', - 'sharp', 'fs', 'path', 'fs/promises', diff --git a/scripts/smoke-package.mjs b/scripts/smoke-package.mjs index 4fbd710..d5c35e7 100644 --- a/scripts/smoke-package.mjs +++ b/scripts/smoke-package.mjs @@ -115,7 +115,6 @@ async function writeConsumerProject(directory) { 'onlyBuiltDependencies:', ' - esbuild', ' - protobufjs', - ' - sharp', '', ].join('\n'), ); @@ -319,7 +318,6 @@ async function smokeAgainstClientArm(options) { 'add', '--allow-build=esbuild', '--allow-build=protobufjs', - '--allow-build=sharp', tarballPath, arm.spec, ], diff --git a/src/embeddings.ts b/src/embeddings.ts index 13e0a9c..e3422a5 100644 --- a/src/embeddings.ts +++ b/src/embeddings.ts @@ -1,11 +1,10 @@ /** * Multi-provider embedding generation - * Supports local Hugging Face Transformers, Gemini, OpenAI, Mistral, - * Cloudflare Workers AI, and custom OpenAI-compatible endpoints + * Supports Gemini, OpenAI, Mistral, Cloudflare Workers AI, and custom + * OpenAI-compatible endpoints. */ export type EmbeddingProvider = - | 'local' | 'gemini' | 'openai' | 'mistral' @@ -45,7 +44,7 @@ export interface EmbeddingBatchResult { } export interface EmbeddingOptions { - provider?: EmbeddingProvider; + provider: EmbeddingProvider; apiKey?: string; accountId?: string; apiToken?: string; @@ -59,28 +58,6 @@ export interface EmbeddingOptions { signal?: AbortSignal; } -interface LocalEmbeddingOutput { - data: ArrayLike; -} - -type LocalFeatureExtractionPipeline = ( - text: string, - options: { pooling: 'mean'; normalize: true } -) => LocalEmbeddingOutput | Promise; - -interface HuggingFaceTransformersModule { - pipeline: ( - task: 'feature-extraction', - model: string - ) => Promise; -} - -interface LocalModelCacheEntry { - promise: Promise; - settled: boolean; - waiters: number; -} - interface RuntimeEnvironment { process?: { env?: Record }; Deno?: { env?: { get?: (name: string) => string | undefined } }; @@ -110,10 +87,8 @@ export type EmbeddingBatchItem = number[] | EmbeddingBatchItemResult; const OPENAI_DEFAULT_DIMENSIONS = 768; const OPENAI_COMPATIBLE_DEFAULT_BATCH_SIZE = 32; -const LOCAL_DIMENSIONS = 384; const DEFAULT_MAX_LENGTH = 8000; const DEFAULT_TIMEOUT_MS = 30_000; -const LOCAL_MODEL = 'Xenova/all-MiniLM-L6-v2'; const GEMINI_MODEL = 'gemini-embedding-2'; const GEMINI_DIMENSIONS = 3072; const GEMINI_MIN_DIMENSIONS = 128; @@ -127,8 +102,6 @@ const MISTRAL_DIMENSIONS = 1024; const CLOUDFLARE_MODEL = '@cf/baai/bge-m3'; const CLOUDFLARE_DIMENSIONS = 1024; -const localModelCacheByModel = new Map(); - function getEnvironmentVariable(name: string): string | undefined { const runtime = globalThis as typeof globalThis & RuntimeEnvironment; const nodeValue = runtime.process?.env?.[name]; @@ -143,84 +116,6 @@ function getEnvironmentVariable(name: string): string | undefined { } } -function deletePendingLocalModelCache(modelName: string, entry: LocalModelCacheEntry): void { - if (!entry.settled && entry.waiters === 0 && localModelCacheByModel.get(modelName) === entry) { - localModelCacheByModel.delete(modelName); - } -} - -async function getLocalEmbeddingModel( - modelName: string, - signal: AbortSignal -): Promise { - const cached = localModelCacheByModel.get(modelName); - if (cached) { - cached.waiters++; - try { - return await waitForLocalEmbeddingModel(modelName, cached, signal); - } finally { - cached.waiters--; - deletePendingLocalModelCache(modelName, cached); - } - } - - const modelPromise = (async () => { - console.log(`Loading local embedding model (${modelName})...`); - const { pipeline } = await import('@huggingface/transformers') as HuggingFaceTransformersModule; - const model = await pipeline('feature-extraction', modelName); - console.log('Local model loaded successfully'); - return model; - })(); - - const entry: LocalModelCacheEntry = { - promise: modelPromise, - settled: false, - waiters: 1 - }; - localModelCacheByModel.set(modelName, entry); - - modelPromise - .then(() => { - entry.settled = true; - }) - .catch(() => { - localModelCacheByModel.delete(modelName); - }); - - try { - return await waitForLocalEmbeddingModel(modelName, entry, signal); - } finally { - entry.waiters--; - deletePendingLocalModelCache(modelName, entry); - } -} - -async function waitForLocalEmbeddingModel( - modelName: string, - entry: LocalModelCacheEntry, - signal: AbortSignal -): Promise { - if (signal.aborted) { - deletePendingLocalModelCache(modelName, entry); - throw providerError('local', 'model inference was aborted'); - } - - let rejectAbort: (error: Error) => void = () => {}; - const abortPromise = new Promise((_resolve, reject) => { - rejectAbort = reject; - }); - const onAbort = (): void => { - rejectAbort(providerError('local', 'model inference was aborted')); - }; - - signal.addEventListener('abort', onAbort, { once: true }); - try { - return await Promise.race([entry.promise, abortPromise]); - } finally { - signal.removeEventListener('abort', onAbort); - } -} - function getPositiveInteger(value: number, optionName: string): number { if (!Number.isInteger(value) || value <= 0) { throw new Error(`Invalid ${optionName}: expected a positive integer`); @@ -234,15 +129,17 @@ function getTimeoutMs(value: number | undefined): number { } function resolveProviderName(provider: EmbeddingProvider | undefined): EmbeddingProvider { - switch (provider ?? 'local') { - case 'local': + switch (provider) { case 'gemini': case 'openai': case 'mistral': case 'cloudflare': case 'openai-compatible': - return provider ?? 'local'; + return provider; default: + if (provider === undefined) { + throw new Error('Embedding provider is required'); + } throw new Error(`Unknown embedding provider: ${String(provider)}`); } } @@ -553,13 +450,6 @@ function createProviderMetadata( model?: string ): EmbeddingProviderMetadata { switch (provider) { - case 'local': - return Object.freeze({ - name: 'local' as const, - model: LOCAL_MODEL, - dimensions, - batch: Object.freeze({ mode: 'sequential' as const }) - }); case 'gemini': return Object.freeze({ name: 'gemini' as const, @@ -602,17 +492,6 @@ function getEffectiveDimensions( provider: EmbeddingProvider, dimensions: number | undefined ): number { - if (provider === 'local') { - if (dimensions !== undefined && dimensions !== LOCAL_DIMENSIONS) { - throw providerError( - 'local', - `${LOCAL_MODEL} returns ${LOCAL_DIMENSIONS} dimensions; received dimensions ${String(dimensions)}` - ); - } - - return LOCAL_DIMENSIONS; - } - if (provider === 'gemini') { const effectiveDimensions = dimensions ?? GEMINI_DIMENSIONS; if ( @@ -694,47 +573,6 @@ function createEmbeddingBatchResult( }); } -class LocalEmbeddingProvider implements EmbeddingProviderClient { - readonly metadata: EmbeddingProviderMetadata; - readonly #timeoutMs: number; - - constructor(metadata: EmbeddingProviderMetadata, timeoutMs: number) { - this.metadata = metadata; - this.#timeoutMs = timeoutMs; - } - - async embed(texts: string[], options: EmbeddingRequestOptions = {}): Promise { - const intent = resolveIntent(options.intent); - if (texts.length === 0) { - return createEmbeddingBatchResult(this.metadata, intent, []); - } - - assertBatchSize(this.metadata, texts.length); - const vectors = await withTimeout( - 'local', - 'model inference', - this.#timeoutMs, - async (signal) => { - const model = await getLocalEmbeddingModel(this.metadata.model, signal); - return embedSequentially('local', 'model inference', texts, signal, async (text) => { - const output = await model(text, { - pooling: 'mean', - normalize: true - }); - return Array.from(output.data); - }); - }, - options.signal - ); - - return createEmbeddingBatchResult( - this.metadata, - intent, - validateEmbeddingBatch(vectors, texts.length, this.metadata.dimensions, 'local') - ); - } -} - class GeminiEmbeddingProvider implements EmbeddingProviderClient { readonly metadata: EmbeddingProviderMetadata; readonly #apiKey: string; @@ -954,14 +792,12 @@ class OpenAICompatibleEmbeddingProvider implements EmbeddingProviderClient { } } -export function createEmbeddingProvider(options: EmbeddingOptions = {}): EmbeddingProviderClient { +export function createEmbeddingProvider(options: EmbeddingOptions): EmbeddingProviderClient { const provider = resolveProviderName(options.provider); const metadata = getEmbeddingProviderMetadata(options); const timeoutMs = getTimeoutMs(options.timeoutMs); switch (provider) { - case 'local': - return new LocalEmbeddingProvider(metadata, timeoutMs); case 'gemini': { const key = getOptionalTrimmedCredential( options.apiKey ?? getEnvironmentVariable('GEMINI_API_KEY') @@ -1068,7 +904,7 @@ export function createEmbeddingProvider(options: EmbeddingOptions = {}): Embeddi } } -export function getEmbeddingProviderMetadata(options: EmbeddingOptions = {}): EmbeddingProviderMetadata { +export function getEmbeddingProviderMetadata(options: EmbeddingOptions): EmbeddingProviderMetadata { const provider = resolveProviderName(options.provider); if (provider === 'openai-compatible') { normalizeOpenAICompatibleEmbeddingsUrl(getRequiredTrimmedString(options.baseUrl, 'baseUrl')); @@ -1085,7 +921,7 @@ export function getEmbeddingProviderMetadata(options: EmbeddingOptions = {}): Em */ export async function generateEmbeddings( texts: string[], - options: EmbeddingOptions = {} + options: EmbeddingOptions ): Promise { const maxLength = getPositiveInteger(options.maxLength ?? DEFAULT_MAX_LENGTH, 'maxLength'); const intent = resolveIntent(options.intent); @@ -1107,7 +943,7 @@ export async function generateEmbeddings( */ export async function generateEmbedding( text: string, - options: EmbeddingOptions = {} + options: EmbeddingOptions ): Promise { const [embedding] = await generateEmbeddings([text], options); if (!embedding) { diff --git a/src/indexer.ts b/src/indexer.ts index 61b652c..163cb99 100644 --- a/src/indexer.ts +++ b/src/indexer.ts @@ -42,7 +42,7 @@ export interface IndexerOptions { */ client: DatabaseClient; contentPath: string; - embeddingOptions?: EmbeddingOptions; + embeddingOptions: EmbeddingOptions; fileExtensions?: string[]; exclude?: string[]; tableName?: string; @@ -133,7 +133,7 @@ export async function indexContent(options: IndexerOptions): Promise { query, limit = 10, tableName = 'articles', - embeddingOptions = {}, + embeddingOptions, candidates, exact = false } = options; @@ -145,6 +145,10 @@ export async function search(options: SearchOptions): Promise { ); const candidateCount = normalizeSearchCandidates(candidates, resultLimit); + if (embeddingOptions === undefined) { + throw new TypeError('embeddingOptions is required'); + } + // Generate embedding for query const queryEmbedding = await generateEmbedding(query, { ...embeddingOptions, diff --git a/tests/database.test.ts b/tests/database.test.ts index 5d9ef9d..249b78b 100644 --- a/tests/database.test.ts +++ b/tests/database.test.ts @@ -21,7 +21,10 @@ import { import { createTable, indexContent } from '../src/indexer.js'; import { search, getFolders } from '../src/search.js'; import { tursoAdapter } from '../src/turso.js'; -import { resetHuggingFaceTransformersMock } from './huggingface-transformers.mock.js'; +import { + resetEmbeddingServiceMock, + TEST_EMBEDDING_OPTIONS +} from './embedding-service.mock.js'; interface RecordedWrite { sql: string; @@ -65,6 +68,10 @@ function createRecordingAdapter(supportsVectorIndex: boolean): DatabaseAdapter & } describe('database boundary', () => { + beforeEach(() => { + resetEmbeddingServiceMock(); + }); + describe('isDatabaseAdapter', () => { it('should recognize adapters built by this package', () => { expect(isDatabaseAdapter(createLibsqlAdapter({} as Client))).toBe(true); @@ -190,7 +197,7 @@ describe('database boundary', () => { await search({ client: adapter, query: 'TypeScript', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(adapter.queries).toHaveLength(1); @@ -203,7 +210,7 @@ describe('database boundary', () => { await search({ client: adapter, query: 'TypeScript', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(adapter.queries).toHaveLength(1); @@ -469,13 +476,13 @@ describe('database boundary', () => { const testDir = join(process.cwd(), 'test-content-adapter'); beforeEach(async () => { - resetHuggingFaceTransformersMock(); + resetEmbeddingServiceMock(); await mkdir(testDir, { recursive: true }); }); afterEach(async () => { await rm(testDir, { recursive: true, force: true }); - resetHuggingFaceTransformersMock(); + resetEmbeddingServiceMock(); }); it('should hand the whole replacement to the adapter as one atomic write', async () => { @@ -487,7 +494,7 @@ describe('database boundary', () => { const result = await indexContent({ client: adapter, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(result.success).toBe(2); diff --git a/tests/embedding-service.mock.ts b/tests/embedding-service.mock.ts new file mode 100644 index 0000000..94ef162 --- /dev/null +++ b/tests/embedding-service.mock.ts @@ -0,0 +1,81 @@ +import { vi } from 'vitest'; +import type { EmbeddingOptions } from '../src/embeddings.js'; + +export const TEST_EMBEDDING_DIMENSIONS = 384; + +export const TEST_EMBEDDING_OPTIONS = Object.freeze({ + provider: 'openai-compatible' as const, + baseUrl: 'https://embeddings.example.test/v1', + model: 'test-embedding-model', + dimensions: TEST_EMBEDDING_DIMENSIONS +}) satisfies EmbeddingOptions; + +function deterministicVector(text: string): number[] { + const lower = text.toLowerCase(); + const vector = new Array(TEST_EMBEDDING_DIMENSIONS).fill(0); + + vector[0] = 0.01; + + if (lower.includes('static') || lower.includes('astro')) { + vector[1] += 2; + } + + if (lower.includes('react')) { + vector[2] += 2; + } + + if (lower.includes('typescript')) { + vector[3] += 3; + } + + if (lower.includes('programming')) { + vector[4] += 1; + } + + if (lower.includes('python')) { + vector[5] += 2; + } + + if (lower.includes('javascript')) { + vector[6] += 1; + } + + if (lower.includes('article') || lower.includes('content')) { + vector[7] += 1; + } + + return vector; +} + +interface EmbeddingRequestBody { + input: string[]; +} + +export const embeddingServiceMock = { + texts: [] as string[], + queuedVectors: [] as number[][], + fetch: vi.fn() +}; + +export function resetEmbeddingServiceMock(): void { + embeddingServiceMock.texts = []; + embeddingServiceMock.queuedVectors = []; + embeddingServiceMock.fetch.mockReset(); + embeddingServiceMock.fetch.mockImplementation(async (_input: unknown, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as EmbeddingRequestBody; + embeddingServiceMock.texts.push(...body.input); + + return { + ok: true, + status: 200, + headers: new Headers(), + json: async () => ({ + data: body.input.map((text, index) => ({ + index, + embedding: embeddingServiceMock.queuedVectors.shift() ?? deterministicVector(text) + })) + }) + }; + }); + vi.stubGlobal('fetch', embeddingServiceMock.fetch); +} diff --git a/tests/embeddings.test.ts b/tests/embeddings.test.ts index 905a9d8..7f2b18e 100644 --- a/tests/embeddings.test.ts +++ b/tests/embeddings.test.ts @@ -1,10 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import type { EmbeddingOptions } from '../src/embeddings.js'; -import { - LOCAL_TEST_DIMENSIONS, - huggingFaceTransformersMock, - resetHuggingFaceTransformersMock -} from './huggingface-transformers.mock.js'; type EmbeddingsModule = typeof import('../src/embeddings.js'); @@ -76,7 +71,6 @@ describe('embeddings', () => { delete process.env.CLOUDFLARE_ACCOUNT_ID; delete process.env.CLOUDFLARE_API_TOKEN; vi.resetModules(); - resetHuggingFaceTransformersMock(); ({ createEmbeddingProvider, generateEmbedding, @@ -192,37 +186,32 @@ describe('embeddings', () => { }); describe('generateEmbedding', () => { - it('should generate local embeddings with native dimensions', async () => { - const text = 'This is a test sentence for embedding generation'; - const embedding = await generateEmbedding(text, { - provider: 'local' - }); - - expect(embedding).toBeInstanceOf(Array); - expect(embedding.length).toBe(LOCAL_TEST_DIMENSIONS); - expect(embedding.every(n => typeof n === 'number')).toBe(true); - expect(huggingFaceTransformersMock.pipeline).toHaveBeenCalledWith( - 'feature-extraction', - 'Xenova/all-MiniLM-L6-v2' - ); - expect(huggingFaceTransformersMock.calls[0]).toEqual({ - text, - options: { - pooling: 'mean', - normalize: true - } - }); - }); - it('should truncate long text to maxLength', async () => { const longText = 'a'.repeat(10000); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers(), + json: async () => ({ data: [{ index: 0, embedding: [1, 2] }] }) + }); + vi.stubGlobal('fetch', fetchMock); + const embedding = await generateEmbedding(longText, { - provider: 'local', + provider: 'openai-compatible', + baseUrl: 'https://embeddings.example.test/v1', + model: 'test-model', + dimensions: 2, maxLength: 100 }); - expect(embedding).toBeInstanceOf(Array); - expect(huggingFaceTransformersMock.calls[0]?.text).toHaveLength(100); + expect(embedding).toEqual([1, 2]); + const request = JSON.parse(fetchMock.mock.calls[0]?.[1]?.body as string) as { input: string[] }; + expect(request.input[0]).toHaveLength(100); + }); + + it('should require an explicit provider', async () => { + await expect(generateEmbedding('test', {} as EmbeddingOptions)).rejects.toThrow( + 'Embedding provider is required' + ); }); it('should throw error for unknown provider', async () => { @@ -453,25 +442,6 @@ describe('embeddings', () => { describe('provider contract', () => { it('exposes stable provider metadata without provider calls', () => { - expect(getEmbeddingProviderMetadata({ - provider: 'local' - })).toEqual({ - name: 'local', - model: 'Xenova/all-MiniLM-L6-v2', - dimensions: 384, - batch: { mode: 'sequential' } - }); - - expect(getEmbeddingProviderMetadata({ - provider: 'local', - dimensions: 384 - })).toEqual({ - name: 'local', - model: 'Xenova/all-MiniLM-L6-v2', - dimensions: 384, - batch: { mode: 'sequential' } - }); - expect(getEmbeddingProviderMetadata({ provider: 'gemini' })).toEqual({ @@ -523,24 +493,20 @@ describe('embeddings', () => { }); it('returns immutable provider metadata', () => { - const metadata = getEmbeddingProviderMetadata({ provider: 'local' }); + const metadata = getEmbeddingProviderMetadata({ provider: 'gemini' }); expect(Object.isFrozen(metadata)).toBe(true); expect(Object.isFrozen(metadata.batch)).toBe(true); expect(() => { (metadata as { dimensions: number }).dimensions = 768; }).toThrow(TypeError); - expect(metadata.dimensions).toBe(384); + expect(metadata.dimensions).toBe(3072); }); it('returns empty batches without provider setup or network work', async () => { const fetchMock = vi.fn(); vi.stubGlobal('fetch', fetchMock); - await expect(generateEmbeddings([], { - provider: 'local' - })).resolves.toEqual([]); - await expect(generateEmbeddings([], { provider: 'openai' })).resolves.toEqual([]); @@ -563,137 +529,6 @@ describe('embeddings', () => { expect(fetchMock).not.toHaveBeenCalled(); expect(geminiMock.keys).toEqual([]); - expect(huggingFaceTransformersMock.pipeline).not.toHaveBeenCalled(); - }); - - it.each([768, 383, 385, 384.5, Number.NaN])('rejects invalid local dimensions %s before provider setup', async (dimensions) => { - const expected = /Xenova\/all-MiniLM-L6-v2 returns 384 dimensions/; - - expect(() => getEmbeddingProviderMetadata({ - provider: 'local', - dimensions - })).toThrow(expected); - - await expect(generateEmbedding('test', { - provider: 'local', - dimensions - })).rejects.toThrow(expected); - - expect(huggingFaceTransformersMock.pipeline).not.toHaveBeenCalled(); - }); - - it('returns native local vectors exactly without zero padding', async () => { - const vector = Array.from({ length: LOCAL_TEST_DIMENSIONS }, (_value, index) => index + 1); - huggingFaceTransformersMock.queuedVectors = [vector]; - - await expect(generateEmbedding('native vector', { - provider: 'local' - })).resolves.toEqual(vector); - }); - - it.each([ - { - name: '383 dimensions', - vector: new Array(383).fill(1), - expected: /embedding 0 has 383 dimensions, expected 384/ - }, - { - name: '385 dimensions', - vector: new Array(385).fill(1), - expected: /embedding 0 has 385 dimensions, expected 384/ - }, - { - name: 'NaN value', - vector: [Number.NaN, ...new Array(383).fill(1)], - expected: /embedding 0 contains a non-finite value at dimension 0/ - }, - { - name: 'Infinity value', - vector: [Number.POSITIVE_INFINITY, ...new Array(383).fill(1)], - expected: /embedding 0 contains a non-finite value at dimension 0/ - } - ])('rejects local model output with $name', async ({ vector, expected }) => { - huggingFaceTransformersMock.queuedVectors = [vector]; - - await expect(generateEmbedding('bad vector', { - provider: 'local' - })).rejects.toThrow(expected); - }); - - it('preserves sequential local batch order', async () => { - const first = new Array(LOCAL_TEST_DIMENSIONS).fill(1); - const second = new Array(LOCAL_TEST_DIMENSIONS).fill(2); - huggingFaceTransformersMock.queuedVectors = [first, second]; - - await expect(generateEmbeddings(['first', 'second'], { - provider: 'local' - })).resolves.toEqual([first, second]); - expect(huggingFaceTransformersMock.calls.map(call => call.text)).toEqual(['first', 'second']); - }); - - it('loads the local pipeline once for concurrent first calls', async () => { - let resolvePipeline: (model: typeof huggingFaceTransformersMock.model) => void = () => {}; - huggingFaceTransformersMock.pipeline.mockImplementationOnce(async () => - new Promise(resolve => { - resolvePipeline = resolve; - }) - ); - - const first = generateEmbedding('first', { provider: 'local' }); - const second = generateEmbedding('second', { provider: 'local' }); - await vi.waitFor(() => { - expect(huggingFaceTransformersMock.pipeline).toHaveBeenCalledTimes(1); - }); - - resolvePipeline(huggingFaceTransformersMock.model); - await expect(Promise.all([first, second])).resolves.toHaveLength(2); - expect(huggingFaceTransformersMock.pipeline).toHaveBeenCalledTimes(1); - }); - - it('evicts failed local pipeline loads so a later call can retry', async () => { - huggingFaceTransformersMock.pipeline - .mockRejectedValueOnce(new Error('download failed')) - .mockResolvedValueOnce(huggingFaceTransformersMock.model); - - await expect(generateEmbedding('first', { - provider: 'local' - })).rejects.toThrow('download failed'); - - await expect(generateEmbedding('second', { - provider: 'local' - })).resolves.toHaveLength(LOCAL_TEST_DIMENSIONS); - expect(huggingFaceTransformersMock.pipeline).toHaveBeenCalledTimes(2); - }); - - it('aborts local inference before loading when the parent signal is already aborted', async () => { - const controller = new AbortController(); - controller.abort(); - - await expect(generateEmbedding('test', { - provider: 'local', - signal: controller.signal - })).rejects.toThrow('local embedding error: model inference was aborted'); - expect(huggingFaceTransformersMock.pipeline).not.toHaveBeenCalled(); - }); - - it('times out local model loading when the runtime does not settle', async () => { - vi.useFakeTimers(); - huggingFaceTransformersMock.pipeline.mockImplementationOnce(async () => new Promise(() => {})); - - const promise = generateEmbedding('test', { - provider: 'local', - timeoutMs: 10 - }); - const assertion = expect(promise).rejects.toThrow('local embedding error: model inference timed out after 10ms'); - await vi.advanceTimersByTimeAsync(10); - - await assertion; - vi.useRealTimers(); - - await expect(generateEmbedding('retry after timeout', { - provider: 'local' - })).resolves.toHaveLength(LOCAL_TEST_DIMENSIONS); - expect(huggingFaceTransformersMock.pipeline).toHaveBeenCalledTimes(2); }); it.each([128, 768, 1536, 3072])('accepts Gemini dimensions %i without credentials', (dimensions) => { diff --git a/tests/huggingface-transformers.mock.ts b/tests/huggingface-transformers.mock.ts deleted file mode 100644 index 1cf51d8..0000000 --- a/tests/huggingface-transformers.mock.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { vi } from 'vitest'; - -export const LOCAL_TEST_DIMENSIONS = 384; - -export interface LocalModelCall { - text: string; - options: { - pooling?: string; - normalize?: boolean; - }; -} - -function deterministicVector(text: string): number[] { - const lower = text.toLowerCase(); - const vector = new Array(LOCAL_TEST_DIMENSIONS).fill(0); - - vector[0] = 0.01; - - if (lower.includes('static') || lower.includes('astro')) { - vector[1] += 2; - } - - if (lower.includes('react')) { - vector[2] += 2; - } - - if (lower.includes('typescript')) { - vector[3] += 3; - } - - if (lower.includes('programming')) { - vector[4] += 1; - } - - if (lower.includes('python')) { - vector[5] += 2; - } - - if (lower.includes('javascript')) { - vector[6] += 1; - } - - if (lower.includes('article') || lower.includes('content')) { - vector[7] += 1; - } - - return vector; -} - -const huggingFaceTransformersMock = vi.hoisted(() => { - const globalState = globalThis as typeof globalThis & { - __LIBSQL_SEARCH_HF_TRANSFORMERS_MOCK__?: { - calls: LocalModelCall[]; - queuedVectors: number[][]; - pipeline: ReturnType; - model: ReturnType; - }; - }; - - if (globalState.__LIBSQL_SEARCH_HF_TRANSFORMERS_MOCK__) { - return globalState.__LIBSQL_SEARCH_HF_TRANSFORMERS_MOCK__; - } - - const state = { - calls: [] as LocalModelCall[], - queuedVectors: [] as number[][], - pipeline: vi.fn(), - model: vi.fn() - }; - - const createVector = (text: string): number[] => { - const next = state.queuedVectors.shift(); - if (next) { - return next; - } - - const lower = text.toLowerCase(); - const vector = new Array(384).fill(0); - - vector[0] = 0.01; - - if (lower.includes('static') || lower.includes('astro')) { - vector[1] += 2; - } - - if (lower.includes('react')) { - vector[2] += 2; - } - - if (lower.includes('typescript')) { - vector[3] += 3; - } - - if (lower.includes('programming')) { - vector[4] += 1; - } - - if (lower.includes('python')) { - vector[5] += 2; - } - - if (lower.includes('javascript')) { - vector[6] += 1; - } - - if (lower.includes('article') || lower.includes('content')) { - vector[7] += 1; - } - - return vector; - }; - - state.model.mockImplementation(async (text: string, options: LocalModelCall['options']) => { - state.calls.push({ text, options }); - return { - data: Float32Array.from(createVector(text)) - }; - }); - - state.pipeline.mockResolvedValue(state.model); - - globalState.__LIBSQL_SEARCH_HF_TRANSFORMERS_MOCK__ = state; - return state; -}); - -vi.mock('@huggingface/transformers', () => ({ - pipeline: huggingFaceTransformersMock.pipeline -})); - -export { huggingFaceTransformersMock }; - -export function resetHuggingFaceTransformersMock(): void { - huggingFaceTransformersMock.calls = []; - huggingFaceTransformersMock.queuedVectors = []; - huggingFaceTransformersMock.model.mockClear(); - huggingFaceTransformersMock.model.mockImplementation(async (text: string, options: LocalModelCall['options']) => { - huggingFaceTransformersMock.calls.push({ text, options }); - const next = huggingFaceTransformersMock.queuedVectors.shift(); - return { - data: Float32Array.from(next ?? deterministicVector(text)) - }; - }); - huggingFaceTransformersMock.pipeline.mockReset(); - huggingFaceTransformersMock.pipeline.mockResolvedValue(huggingFaceTransformersMock.model); -} diff --git a/tests/indexer.test.ts b/tests/indexer.test.ts index eaf009d..4384c7a 100644 --- a/tests/indexer.test.ts +++ b/tests/indexer.test.ts @@ -4,9 +4,10 @@ import { mkdir, writeFile, rm, symlink } from 'fs/promises'; import { join } from 'path'; import { createTable, indexContent, IndexingError } from '../src/indexer.js'; import { - huggingFaceTransformersMock, - resetHuggingFaceTransformersMock -} from './huggingface-transformers.mock.js'; + embeddingServiceMock, + resetEmbeddingServiceMock, + TEST_EMBEDDING_OPTIONS +} from './embedding-service.mock.js'; type BatchStatement = string | { sql: string; args?: unknown }; @@ -79,13 +80,13 @@ describe('indexer', () => { const testDir = join(process.cwd(), 'test-content'); beforeEach(async () => { + resetEmbeddingServiceMock(); client = createClient({ url: testDbUrl }); await mkdir(testDir, { recursive: true }); }); afterEach(async () => { await rm(testDir, { recursive: true, force: true }); - resetHuggingFaceTransformersMock(); vi.unstubAllGlobals(); }); @@ -143,7 +144,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(result.success).toBe(1); @@ -168,7 +169,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(result.success).toBe(1); @@ -186,7 +187,7 @@ describe('indexer', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); const articles = await client.execute('SELECT * FROM articles'); @@ -202,7 +203,7 @@ describe('indexer', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); const articles = await client.execute('SELECT * FROM articles'); @@ -216,7 +217,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(result.success).toBe(1); @@ -231,7 +232,7 @@ describe('indexer', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); await rm(join(testDir, 'first.md')); @@ -240,7 +241,7 @@ describe('indexer', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); const articles = await client.execute('SELECT * FROM articles'); @@ -258,7 +259,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, allowEmptyIndex: true }); @@ -273,7 +274,7 @@ describe('indexer', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, onProgress: (current, total, file) => { progressCalls.push({ current, total, file }); } @@ -291,7 +292,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -312,7 +313,7 @@ describe('indexer', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); await rm(join(testDir, 'first.md')); @@ -329,7 +330,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -342,7 +343,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, allowEmptyIndex: true }); @@ -361,12 +362,12 @@ describe('indexer', () => { await seedIndex(); await writeFile(join(testDir, 'second.md'), '---\ntitle: Second\n---\nContent'); - huggingFaceTransformersMock.model.mockRejectedValueOnce(new Error('provider unavailable')); + embeddingServiceMock.fetch.mockRejectedValueOnce(new Error('provider unavailable')); const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -387,7 +388,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -407,7 +408,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -424,12 +425,12 @@ describe('indexer', () => { await writeFile(join(testDir, 'alpha.md'), '---\ntitle: Alpha\n---\nContent'); await writeFile(join(testDir, 'beta.md'), '---\ntitle: Beta\n---\nContent'); - huggingFaceTransformersMock.model.mockRejectedValueOnce(new Error('provider unavailable')); + embeddingServiceMock.fetch.mockRejectedValueOnce(new Error('provider unavailable')); const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, failurePolicy: 'skip' }); @@ -453,13 +454,13 @@ describe('indexer', () => { await writeFile(join(testDir, 'alpha.md'), '---\ntitle: Alpha\n---\nContent'); await writeFile(join(testDir, 'beta.md'), '---\ntitle: Beta\n---\nContent'); - huggingFaceTransformersMock.model.mockRejectedValueOnce(new Error('provider unavailable')); - huggingFaceTransformersMock.model.mockRejectedValueOnce(new Error('provider unavailable')); + embeddingServiceMock.fetch.mockRejectedValueOnce(new Error('provider unavailable')); + embeddingServiceMock.fetch.mockRejectedValueOnce(new Error('provider unavailable')); const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, failurePolicy: 'skip' })); @@ -481,7 +482,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client: failingClient, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -502,7 +503,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client: rejectingClient, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -521,12 +522,12 @@ describe('indexer', () => { await seedIndex(); await writeFile(join(testDir, 'alpha.md'), '---\ntitle: Alpha\n---\nContent'); - huggingFaceTransformersMock.model.mockRejectedValueOnce('provider down'); + embeddingServiceMock.fetch.mockRejectedValueOnce('provider down'); const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -544,7 +545,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client, contentPath: join(testDir, 'does-not-exist'), - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -561,13 +562,13 @@ describe('indexer', () => { await writeFile(join(testDir, 'alpha.md'), '---\ntitle: Alpha\n---\nContent'); await writeFile(join(testDir, 'beta.md'), '---\ntitle: Beta\n---\nContent'); - huggingFaceTransformersMock.model.mockRejectedValueOnce(new Error('provider unavailable')); - huggingFaceTransformersMock.model.mockRejectedValueOnce(new Error('provider unavailable')); + embeddingServiceMock.fetch.mockRejectedValueOnce(new Error('provider unavailable')); + embeddingServiceMock.fetch.mockRejectedValueOnce(new Error('provider unavailable')); const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, failurePolicy: 'skip', allowEmptyIndex: true })); @@ -590,7 +591,7 @@ describe('indexer', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); await rm(join(testDir, 'first.md')); @@ -607,7 +608,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -628,7 +629,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, failurePolicy: 'skip' }); @@ -727,7 +728,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(result.success).toBe(1); @@ -742,7 +743,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -765,7 +766,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, failurePolicy: 'skip' }); @@ -784,7 +785,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(result.success).toBe(1); @@ -798,7 +799,7 @@ describe('indexer', () => { const error = await captureError(() => indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })); expect(error).toBeInstanceOf(IndexingError); @@ -822,7 +823,7 @@ describe('indexer', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 }, + embeddingOptions: TEST_EMBEDDING_OPTIONS, failurePolicy: 'skip' }); diff --git a/tests/search.test.ts b/tests/search.test.ts index f6a2ce0..748d49f 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -11,16 +11,17 @@ import { } from '../src/search.js'; import { generateEmbedding } from '../src/embeddings.js'; import { - huggingFaceTransformersMock, - resetHuggingFaceTransformersMock -} from './huggingface-transformers.mock.js'; + embeddingServiceMock, + resetEmbeddingServiceMock, + TEST_EMBEDDING_OPTIONS +} from './embedding-service.mock.js'; describe('search', () => { const testDbUrl = ':memory:'; let client: ReturnType; beforeEach(async () => { - resetHuggingFaceTransformersMock(); + resetEmbeddingServiceMock(); client = createClient({ url: testDbUrl }); await createTable(client); }); @@ -40,7 +41,7 @@ describe('search', () => { * derived from its text, so distances can be made to tie deliberately. */ async function insertArticleWithVector(slug: string, vector: number[]): Promise { - huggingFaceTransformersMock.queuedVectors.push(vector); + embeddingServiceMock.queuedVectors.push(vector); await insertTestArticle({ slug, @@ -56,10 +57,7 @@ describe('search', () => { folder?: string; tags?: string[]; }) { - const embedding = await generateEmbedding(data.content, { - provider: 'local', - dimensions: 384 - }); + const embedding = await generateEmbedding(data.content, TEST_EMBEDDING_OPTIONS); await client.execute({ sql: `INSERT INTO articles @@ -94,7 +92,7 @@ describe('search', () => { client, query: 'static site building', limit: 5, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results).toHaveLength(2); @@ -126,7 +124,7 @@ describe('search', () => { client, query: 'JavaScript', limit: 2, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results).toHaveLength(2); @@ -149,7 +147,7 @@ describe('search', () => { client, query: 'TypeScript programming', limit: 5, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results[0].slug).toBe('exact-match'); @@ -167,7 +165,7 @@ describe('search', () => { const results = await search({ client, query: 'article', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results[0].tags).toEqual(['tag1', 'tag2']); @@ -177,7 +175,7 @@ describe('search', () => { const results = await search({ client, query: 'anything', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results).toEqual([]); @@ -201,7 +199,7 @@ describe('search', () => { client, query: 'article', limit: 10, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results.map(result => result.slug)).toEqual(['embedded']); @@ -228,7 +226,7 @@ describe('search', () => { content: 'TypeScript is a typed superset of JavaScript' }); - const embeddingOptions = { provider: 'local' as const, dimensions: 384 }; + const embeddingOptions = TEST_EMBEDDING_OPTIONS; const indexed = await search({ client, @@ -264,13 +262,13 @@ describe('search', () => { // through. With the tiebreaker present the result is fully deterministic, // so a higher count adds no flake risk of its own. for (let run = 0; run < 25; run++) { - huggingFaceTransformersMock.queuedVectors.push(unitVector(12)); + embeddingServiceMock.queuedVectors.push(unitVector(12)); const results = await search({ client, query: 'tie breaker', limit: 3, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); runs.push(results.map(result => result.slug).join(',')); @@ -294,7 +292,7 @@ describe('search', () => { query: 'JavaScript', limit: 5, candidates: 50, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(executeSpy).toHaveBeenCalledWith(expect.objectContaining({ @@ -319,7 +317,7 @@ describe('search', () => { client, query: 'JavaScript', limit, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(executeSpy).toHaveBeenCalledWith(expect.objectContaining({ @@ -346,11 +344,11 @@ describe('search', () => { query: 'JavaScript', limit: 5, candidates: candidates as number, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })).rejects.toThrow('Invalid search candidates'); expect(executeSpy).not.toHaveBeenCalled(); - expect(huggingFaceTransformersMock.model).not.toHaveBeenCalled(); + expect(embeddingServiceMock.fetch).not.toHaveBeenCalled(); } ); @@ -369,10 +367,7 @@ describe('search', () => { ) `); - const embedding = await generateEmbedding('JavaScript programming', { - provider: 'local', - dimensions: 384 - }); + const embedding = await generateEmbedding('JavaScript programming', TEST_EMBEDDING_OPTIONS); await client.execute({ sql: `INSERT INTO unindexed @@ -388,7 +383,7 @@ describe('search', () => { client, query: 'JavaScript', tableName: 'unindexed', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); } catch (error) { thrown = error; @@ -432,10 +427,7 @@ describe('search', () => { ['exact-match', 'TypeScript is a typed superset of JavaScript'], ['partial-match', 'Python is a programming language'] ]) { - const embedding = await generateEmbedding(content, { - provider: 'local', - dimensions: 384 - }); + const embedding = await generateEmbedding(content, TEST_EMBEDDING_OPTIONS); await client.execute({ sql: `INSERT INTO legacy @@ -453,7 +445,7 @@ describe('search', () => { query: 'TypeScript programming', tableName: 'legacy', limit: 5, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); // The full ordered list, not just "not empty": both pre-existing rows @@ -479,7 +471,7 @@ describe('search', () => { await expect(search({ client, query: 'JavaScript', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })).rejects.toBe(failure); }, 30000); @@ -495,7 +487,7 @@ describe('search', () => { await search({ client, query: 'JavaScript', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); } catch (error) { thrown = error; @@ -511,7 +503,7 @@ describe('search', () => { }, 30000); it('should surface a real dimension mismatch instead of the missing-index message', async () => { - // Dimension drift: the table was created 4 wide, but the local provider + // Dimension drift: the table was created 4 wide, but the test provider // queries with its native 384-wide vector. The vector index exists here, // so the missing-index advice would be actively wrong. await createTable(client, 'narrow', 4); @@ -530,7 +522,7 @@ describe('search', () => { client, query: 'mismatched width', tableName: 'narrow', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); } catch (error) { thrown = error; @@ -570,10 +562,7 @@ describe('search', () => { ['exact-match', 'TypeScript is a typed superset of JavaScript'], ['partial-match', 'Python is a programming language'] ]) { - const embedding = await generateEmbedding(content, { - provider: 'local', - dimensions: 384 - }); + const embedding = await generateEmbedding(content, TEST_EMBEDDING_OPTIONS); await client.execute({ sql: `INSERT INTO unindexed @@ -589,7 +578,7 @@ describe('search', () => { tableName: 'unindexed', limit: 5, exact: true, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results.map(result => result.slug)).toEqual(['exact-match', 'partial-match']); @@ -610,7 +599,7 @@ describe('search', () => { query: 'JavaScript', exact: true, candidates: 64, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); const statement = executeSpy.mock.calls[0][0] as { sql: string; args: unknown }; @@ -634,11 +623,11 @@ describe('search', () => { limit: 10, candidates: 5, exact: true, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS })).rejects.toThrow('Invalid search candidates'); expect(executeSpy).not.toHaveBeenCalled(); - expect(huggingFaceTransformersMock.model).not.toHaveBeenCalled(); + expect(embeddingServiceMock.fetch).not.toHaveBeenCalled(); }, 30000); }); diff --git a/tests/sql-security.test.ts b/tests/sql-security.test.ts index 1034bb7..7dbc197 100644 --- a/tests/sql-security.test.ts +++ b/tests/sql-security.test.ts @@ -68,7 +68,8 @@ describe('SQL input security', () => { client, query: 'guide', tableName, - limit: 1 + limit: 1, + embeddingOptions: { provider: 'openai', apiKey: 'key' } })).resolves.toHaveLength(1); } ); @@ -117,6 +118,23 @@ describe('SQL input security', () => { expect(generateEmbedding).not.toHaveBeenCalled(); }); + it('requires explicit embedding options for indexing and search', async () => { + const client = createMockClient(); + + await expect(indexContent({ + client, + contentPath: '/path/that/should/not/be/read' + } as never)).rejects.toThrow('embeddingOptions is required'); + + await expect(search({ + client, + query: 'guide' + } as never)).rejects.toThrow('embeddingOptions is required'); + + expect(client.execute).not.toHaveBeenCalled(); + expect(generateEmbedding).not.toHaveBeenCalled(); + }); + it.each([ ['getAllArticles', (client: Client) => getAllArticles(client, 'bad-name')], ['getArticleBySlug', (client: Client) => getArticleBySlug(client, 'slug', 'bad-name')], @@ -157,10 +175,15 @@ describe('SQL input security', () => { await search({ client, query: 'guide', + embeddingOptions: { provider: 'openai', apiKey: 'key' }, ...(limit === undefined ? {} : { limit }) }); - expect(generateEmbedding).toHaveBeenCalledWith('guide', { intent: 'query' }); + expect(generateEmbedding).toHaveBeenCalledWith('guide', { + provider: 'openai', + apiKey: 'key', + intent: 'query' + }); expect(client.execute).toHaveBeenCalledWith(expect.objectContaining({ args: { queryVector: JSON.stringify([0.1, 0.2, 0.3]), diff --git a/tests/turso-database.test.ts b/tests/turso-database.test.ts index 5121dbb..d90ada6 100644 --- a/tests/turso-database.test.ts +++ b/tests/turso-database.test.ts @@ -28,7 +28,10 @@ import { type TursoDatabase, type TursoStatement } from '../src/turso.js'; -import { resetHuggingFaceTransformersMock } from './huggingface-transformers.mock.js'; +import { + resetEmbeddingServiceMock, + TEST_EMBEDDING_OPTIONS +} from './embedding-service.mock.js'; /** A connected handle, which additionally closes. */ type TursoConnection = TursoDatabase & { close(): void }; @@ -71,7 +74,7 @@ describeTurso('turso database backend', () => { let client: DatabaseAdapter; beforeEach(async () => { - resetHuggingFaceTransformersMock(); + resetEmbeddingServiceMock(); database = await connect!(':memory:'); client = tursoAdapter(database); await mkdir(testDir, { recursive: true }); @@ -80,7 +83,7 @@ describeTurso('turso database backend', () => { afterEach(async () => { await rm(testDir, { recursive: true, force: true }); database.close(); - resetHuggingFaceTransformersMock(); + resetEmbeddingServiceMock(); }); async function query(sql: string, args?: unknown): Promise>> { @@ -113,10 +116,7 @@ describeTurso('turso database backend', () => { folder?: string; tags?: string[]; }): Promise { - const embedding = await generateEmbedding(data.content, { - provider: 'local', - dimensions: 384 - }); + const embedding = await generateEmbedding(data.content, TEST_EMBEDDING_OPTIONS); await database .prepare( @@ -283,7 +283,7 @@ describeTurso('turso database backend', () => { const result = await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(result.success).toBe(2); @@ -300,13 +300,10 @@ describeTurso('turso database backend', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); - const embedding = await generateEmbedding('TypeScript', { - provider: 'local', - dimensions: 384 - }); + const embedding = await generateEmbedding('TypeScript', TEST_EMBEDDING_OPTIONS); const rows = await query( 'SELECT vector_distance_cos(embedding, vector32(?)) AS distance FROM articles', @@ -322,7 +319,7 @@ describeTurso('turso database backend', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); await rm(join(testDir, 'first.md')); @@ -330,7 +327,7 @@ describeTurso('turso database backend', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(await indexedTitles()).toEqual(['Second']); @@ -346,7 +343,7 @@ describeTurso('turso database backend', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); await rm(join(testDir, 'first.md')); @@ -359,7 +356,7 @@ describeTurso('turso database backend', () => { indexContent({ client: failing, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }) ); @@ -378,7 +375,7 @@ describeTurso('turso database backend', () => { await indexContent({ client, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); await rm(join(testDir, 'first.md')); @@ -390,7 +387,7 @@ describeTurso('turso database backend', () => { indexContent({ client: rejecting, contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }) ); @@ -426,7 +423,7 @@ describeTurso('turso database backend', () => { await indexContent({ client: tursoAdapter(recording), contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(batchCalls).toHaveLength(0); @@ -456,7 +453,7 @@ describeTurso('turso database backend', () => { const result = await indexContent({ client: tursoAdapter(recording), contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(result.success).toBe(5); @@ -484,7 +481,7 @@ describeTurso('turso database backend', () => { indexContent({ client: tursoAdapter(recording), contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }) ); @@ -539,7 +536,7 @@ describeTurso('turso database backend', () => { await search({ client: tursoAdapter(tracking.handle), query: 'TypeScript', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); // search() issues exactly one query, so one prepare and one close. An SSR @@ -569,7 +566,7 @@ describeTurso('turso database backend', () => { await indexContent({ client: tursoAdapter(tracking.handle), contentPath: testDir, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); // One DELETE plus one cached INSERT, both closed after COMMIT. Closing a @@ -619,7 +616,7 @@ describeTurso('turso database backend', () => { const results = await search({ client, query: 'TypeScript programming', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results.map(result => result.slug)).toEqual(['exact-match', 'unrelated']); @@ -645,7 +642,7 @@ describeTurso('turso database backend', () => { await search({ client: tursoAdapter(recording), query: 'JavaScript', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(prepared).toHaveLength(1); @@ -662,7 +659,7 @@ describeTurso('turso database backend', () => { client, query: 'TypeScript', limit: 2, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results).toHaveLength(2); @@ -675,7 +672,7 @@ describeTurso('turso database backend', () => { query: 'TypeScript', limit: 10, candidates: 5, - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }) ).rejects.toThrow('Invalid search candidates'); }, 30000); @@ -691,7 +688,7 @@ describeTurso('turso database backend', () => { const results = await search({ client, query: 'TypeScript', - embeddingOptions: { provider: 'local', dimensions: 384 } + embeddingOptions: TEST_EMBEDDING_OPTIONS }); expect(results[0].tags).toEqual(['ts', 'guide']); diff --git a/vitest.config.ts b/vitest.config.ts index 805f62b..9aae56d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,7 +4,6 @@ export default defineConfig({ test: { globals: true, environment: 'node', - setupFiles: ['./tests/huggingface-transformers.mock.ts'], coverage: { provider: 'v8', include: ['src/**/*.ts'],