diff --git a/src/indexer.ts b/src/indexer.ts index a5316fc..61b652c 100644 --- a/src/indexer.ts +++ b/src/indexer.ts @@ -107,13 +107,15 @@ interface ContentFile { /** * Internal build-phase view of a document. * - * `serializedTags` is derived from `tags` during the parse stage so that - * unserializable tags fail the file there rather than at bind time. It is an - * implementation detail of the build-to-replace handoff and deliberately kept - * off the exported `IndexedDocument`. + * Serialized values are derived during their corresponding build stages so + * that serialization failures remain attributable to a source file rather + * than escaping into the replacement phase. They are implementation details + * of the build-to-replace handoff and deliberately kept off the exported + * `IndexedDocument`. */ interface BuiltDocument extends IndexedDocument { serializedTags: string; + serializedEmbedding: string; } type BuildOutcome = @@ -311,11 +313,13 @@ async function buildDocument( } let embedding: number[]; + let serializedEmbedding: string; try { embedding = await generateEmbedding(parsed.embeddingText, { ...embeddingOptions, intent: embeddingOptions.intent ?? 'document' }); + serializedEmbedding = JSON.stringify(embedding); } catch (error) { return { ok: false, stage: 'embed', error: toError(error) }; } @@ -330,6 +334,7 @@ async function buildDocument( tags: parsed.tags, serializedTags: parsed.serializedTags, embedding, + serializedEmbedding, metadata: parsed.metadata } }; @@ -509,7 +514,7 @@ function createInsertStatement( document.content, document.folder, document.serializedTags, - JSON.stringify(document.embedding) + document.serializedEmbedding ] }; } diff --git a/tests/indexer.test.ts b/tests/indexer.test.ts index 629d434..eaf009d 100644 --- a/tests/indexer.test.ts +++ b/tests/indexer.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { createClient, type Client } from '@libsql/client'; import { mkdir, writeFile, rm, symlink } from 'fs/promises'; import { join } from 'path'; @@ -86,6 +86,7 @@ describe('indexer', () => { afterEach(async () => { await rm(testDir, { recursive: true, force: true }); resetHuggingFaceTransformersMock(); + vi.unstubAllGlobals(); }); describe('createTable', () => { @@ -639,6 +640,87 @@ describe('indexer', () => { expect(await indexedTitles()).toEqual(['Valid']); }, 30000); + it('should treat an unserializable embedding as an embed failure', async () => { + await writeFile(join(testDir, 'broken.md'), '---\ntitle: Broken\n---\nContent'); + + const embedding = new Array(384).fill(1); + const circular: { self?: unknown } = {}; + circular.self = circular; + Object.defineProperty(embedding, 'toJSON', { value: () => circular }); + + vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ + ok: true, + headers: new Headers(), + json: async () => ({ data: [{ index: 0, embedding }] }) + })); + + const error = await captureError(() => indexContent({ + client, + contentPath: testDir, + embeddingOptions: { + provider: 'openai-compatible', + baseUrl: 'https://example.com/v1', + model: 'test-model', + dimensions: 384 + } + })); + + expect(error).toBeInstanceOf(IndexingError); + const indexingError = error as IndexingError; + expect(indexingError.phase).toBe('build'); + expect(indexingError.failures).toHaveLength(1); + expect(indexingError.failures[0].file).toBe('broken.md'); + expect(indexingError.failures[0].stage).toBe('embed'); + expect(indexingError.failures[0].error.message).toContain('circular'); + + expect(await indexedTitles()).toEqual(['First']); + }, 30000); + + it('should skip an unserializable embedding under skip policy', async () => { + await writeFile(join(testDir, 'broken.md'), '---\ntitle: Broken\n---\nContent'); + await writeFile(join(testDir, 'valid.md'), '---\ntitle: Valid\n---\nContent'); + + const embedding = new Array(384).fill(1); + const circular: { self?: unknown } = {}; + circular.self = circular; + Object.defineProperty(embedding, 'toJSON', { value: () => circular }); + + const fetchMock = vi.fn() + .mockResolvedValueOnce({ + ok: true, + headers: new Headers(), + json: async () => ({ data: [{ index: 0, embedding }] }) + }) + .mockResolvedValueOnce({ + ok: true, + headers: new Headers(), + json: async () => ({ + data: [{ index: 0, embedding: new Array(384).fill(1) }] + }) + }); + vi.stubGlobal('fetch', fetchMock); + + const result = await indexContent({ + client, + contentPath: testDir, + embeddingOptions: { + provider: 'openai-compatible', + baseUrl: 'https://example.com/v1', + model: 'test-model', + dimensions: 384 + }, + failurePolicy: 'skip' + }); + + expect(result.success).toBe(1); + expect(result.failed).toBe(1); + expect(result.partial).toBe(true); + expect(result.failures[0].file).toBe('broken.md'); + expect(result.failures[0].stage).toBe('embed'); + + expect(await indexedTitles()).toEqual(['Valid']); + }, 30000); + it('should accept scalar frontmatter titles', async () => { await writeFile(join(testDir, 'numeric.md'), '---\ntitle: 2024\n---\nContent');