From a77f04e0fd7c231688d95dcf5d09f2ab85fad3a1 Mon Sep 17 00:00:00 2001 From: Logan Lindquist Land Date: Sun, 23 Aug 2026 19:52:14 -0500 Subject: [PATCH] fix: serialize embeddings during index build Keep JSON serialization failures inside the per-file build phase so malformed provider output follows the configured skip or abort policy. This preserves the existing index replacement guarantees instead of letting an embedding value fail later during insert binding after the rebuild is already underway. --- src/indexer.ts | 15 +++++--- tests/indexer.test.ts | 84 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/src/indexer.ts b/src/indexer.ts index a5316fc..61b652c 100644 --- a/src/indexer.ts +++ b/src/indexer.ts @@ -107,13 +107,15 @@ interface ContentFile { /** * Internal build-phase view of a document. * - * `serializedTags` is derived from `tags` during the parse stage so that - * unserializable tags fail the file there rather than at bind time. It is an - * implementation detail of the build-to-replace handoff and deliberately kept - * off the exported `IndexedDocument`. + * Serialized values are derived during their corresponding build stages so + * that serialization failures remain attributable to a source file rather + * than escaping into the replacement phase. They are implementation details + * of the build-to-replace handoff and deliberately kept off the exported + * `IndexedDocument`. */ interface BuiltDocument extends IndexedDocument { serializedTags: string; + serializedEmbedding: string; } type BuildOutcome = @@ -311,11 +313,13 @@ async function buildDocument( } let embedding: number[]; + let serializedEmbedding: string; try { embedding = await generateEmbedding(parsed.embeddingText, { ...embeddingOptions, intent: embeddingOptions.intent ?? 'document' }); + serializedEmbedding = JSON.stringify(embedding); } catch (error) { return { ok: false, stage: 'embed', error: toError(error) }; } @@ -330,6 +334,7 @@ async function buildDocument( tags: parsed.tags, serializedTags: parsed.serializedTags, embedding, + serializedEmbedding, metadata: parsed.metadata } }; @@ -509,7 +514,7 @@ function createInsertStatement( document.content, document.folder, document.serializedTags, - JSON.stringify(document.embedding) + document.serializedEmbedding ] }; } diff --git a/tests/indexer.test.ts b/tests/indexer.test.ts index 629d434..eaf009d 100644 --- a/tests/indexer.test.ts +++ b/tests/indexer.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { createClient, type Client } from '@libsql/client'; import { mkdir, writeFile, rm, symlink } from 'fs/promises'; import { join } from 'path'; @@ -86,6 +86,7 @@ describe('indexer', () => { afterEach(async () => { await rm(testDir, { recursive: true, force: true }); resetHuggingFaceTransformersMock(); + vi.unstubAllGlobals(); }); describe('createTable', () => { @@ -639,6 +640,87 @@ describe('indexer', () => { expect(await indexedTitles()).toEqual(['Valid']); }, 30000); + it('should treat an unserializable embedding as an embed failure', async () => { + await writeFile(join(testDir, 'broken.md'), '---\ntitle: Broken\n---\nContent'); + + const embedding = new Array(384).fill(1); + const circular: { self?: unknown } = {}; + circular.self = circular; + Object.defineProperty(embedding, 'toJSON', { value: () => circular }); + + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers(), + json: async () => ({ data: [{ index: 0, embedding }] }) + })); + + const error = await captureError(() => indexContent({ + client, + contentPath: testDir, + embeddingOptions: { + provider: 'openai-compatible', + baseUrl: 'https://example.com/v1', + model: 'test-model', + dimensions: 384 + } + })); + + expect(error).toBeInstanceOf(IndexingError); + const indexingError = error as IndexingError; + expect(indexingError.phase).toBe('build'); + expect(indexingError.failures).toHaveLength(1); + expect(indexingError.failures[0].file).toBe('broken.md'); + expect(indexingError.failures[0].stage).toBe('embed'); + expect(indexingError.failures[0].error.message).toContain('circular'); + + expect(await indexedTitles()).toEqual(['First']); + }, 30000); + + it('should skip an unserializable embedding under skip policy', async () => { + await writeFile(join(testDir, 'broken.md'), '---\ntitle: Broken\n---\nContent'); + await writeFile(join(testDir, 'valid.md'), '---\ntitle: Valid\n---\nContent'); + + const embedding = new Array(384).fill(1); + const circular: { self?: unknown } = {}; + circular.self = circular; + Object.defineProperty(embedding, 'toJSON', { value: () => circular }); + + const fetchMock = vi.fn() + .mockResolvedValueOnce({ + ok: true, + headers: new Headers(), + json: async () => ({ data: [{ index: 0, embedding }] }) + }) + .mockResolvedValueOnce({ + ok: true, + headers: new Headers(), + json: async () => ({ + data: [{ index: 0, embedding: new Array(384).fill(1) }] + }) + }); + vi.stubGlobal('fetch', fetchMock); + + const result = await indexContent({ + client, + contentPath: testDir, + embeddingOptions: { + provider: 'openai-compatible', + baseUrl: 'https://example.com/v1', + model: 'test-model', + dimensions: 384 + }, + failurePolicy: 'skip' + }); + + expect(result.success).toBe(1); + expect(result.failed).toBe(1); + expect(result.partial).toBe(true); + expect(result.failures[0].file).toBe('broken.md'); + expect(result.failures[0].stage).toBe('embed'); + + expect(await indexedTitles()).toEqual(['Valid']); + }, 30000); + it('should accept scalar frontmatter titles', async () => { await writeFile(join(testDir, 'numeric.md'), '---\ntitle: 2024\n---\nContent');