fix: serialize embeddings during index build

src/indexer.ts

@@ -107,13 +107,15 @@ interface ContentFile {
 /**
  * Internal build-phase view of a document.
  *
- * `serializedTags` is derived from `tags` during the parse stage so that
- * unserializable tags fail the file there rather than at bind time. It is an
- * implementation detail of the build-to-replace handoff and deliberately kept
- * off the exported `IndexedDocument`.
+ * Serialized values are derived during their corresponding build stages so
+ * that serialization failures remain attributable to a source file rather
+ * than escaping into the replacement phase. They are implementation details
+ * of the build-to-replace handoff and deliberately kept off the exported
+ * `IndexedDocument`.
  */
 interface BuiltDocument extends IndexedDocument {
   serializedTags: string;
+  serializedEmbedding: string;
 }
 
 type BuildOutcome =
@@ -311,11 +313,13 @@ async function buildDocument(
   }
 
   let embedding: number[];
+  let serializedEmbedding: string;
   try {
     embedding = await generateEmbedding(parsed.embeddingText, {
       ...embeddingOptions,
       intent: embeddingOptions.intent ?? 'document'
     });
+    serializedEmbedding = JSON.stringify(embedding);
   } catch (error) {
     return { ok: false, stage: 'embed', error: toError(error) };
   }
@@ -330,6 +334,7 @@ async function buildDocument(
       tags: parsed.tags,
       serializedTags: parsed.serializedTags,
       embedding,
+      serializedEmbedding,
       metadata: parsed.metadata
     }
   };
@@ -509,7 +514,7 @@ function createInsertStatement(
       document.content,
       document.folder,
       document.serializedTags,
-      JSON.stringify(document.embedding)
+      document.serializedEmbedding
     ]
   };
 }

tests/indexer.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 import { createClient, type Client } from '@libsql/client';
 import { mkdir, writeFile, rm, symlink } from 'fs/promises';
 import { join } from 'path';
@@ -86,6 +86,7 @@ describe('indexer', () => {
   afterEach(async () => {
     await rm(testDir, { recursive: true, force: true });
     resetHuggingFaceTransformersMock();
+    vi.unstubAllGlobals();
   });
 
   describe('createTable', () => {
@@ -639,6 +640,87 @@ describe('indexer', () => {
       expect(await indexedTitles()).toEqual(['Valid']);
     }, 30000);
 
+    it('should treat an unserializable embedding as an embed failure', async () => {
+      await writeFile(join(testDir, 'broken.md'), '---\ntitle: Broken\n---\nContent');
+
+      const embedding = new Array(384).fill(1);
+      const circular: { self?: unknown } = {};
+      circular.self = circular;
+      Object.defineProperty(embedding, 'toJSON', { value: () => circular });
+
+      vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
+        ok: true,
+        headers: new Headers(),
+        json: async () => ({ data: [{ index: 0, embedding }] })
+      }));
+
+      const error = await captureError(() => indexContent({
+        client,
+        contentPath: testDir,
+        embeddingOptions: {
+          provider: 'openai-compatible',
+          baseUrl: 'https://example.com/v1',
+          model: 'test-model',
+          dimensions: 384
+        }
+      }));
+
+      expect(error).toBeInstanceOf(IndexingError);
+      const indexingError = error as IndexingError;
+      expect(indexingError.phase).toBe('build');
+      expect(indexingError.failures).toHaveLength(1);
+      expect(indexingError.failures[0].file).toBe('broken.md');
+      expect(indexingError.failures[0].stage).toBe('embed');
+      expect(indexingError.failures[0].error.message).toContain('circular');
+
+      expect(await indexedTitles()).toEqual(['First']);
+    }, 30000);
+
+    it('should skip an unserializable embedding under skip policy', async () => {
+      await writeFile(join(testDir, 'broken.md'), '---\ntitle: Broken\n---\nContent');
+      await writeFile(join(testDir, 'valid.md'), '---\ntitle: Valid\n---\nContent');
+
+      const embedding = new Array(384).fill(1);
+      const circular: { self?: unknown } = {};
+      circular.self = circular;
+      Object.defineProperty(embedding, 'toJSON', { value: () => circular });
+
+      const fetchMock = vi.fn()
+        .mockResolvedValueOnce({
+          ok: true,
+          headers: new Headers(),
+          json: async () => ({ data: [{ index: 0, embedding }] })
+        })
+        .mockResolvedValueOnce({
+          ok: true,
+          headers: new Headers(),
+          json: async () => ({
+            data: [{ index: 0, embedding: new Array(384).fill(1) }]
+          })
+        });
+      vi.stubGlobal('fetch', fetchMock);
+
+      const result = await indexContent({
+        client,
+        contentPath: testDir,
+        embeddingOptions: {
+          provider: 'openai-compatible',
+          baseUrl: 'https://example.com/v1',
+          model: 'test-model',
+          dimensions: 384
+        },
+        failurePolicy: 'skip'
+      });
+
+      expect(result.success).toBe(1);
+      expect(result.failed).toBe(1);
+      expect(result.partial).toBe(true);
+      expect(result.failures[0].file).toBe('broken.md');
+      expect(result.failures[0].stage).toBe('embed');
+
+      expect(await indexedTitles()).toEqual(['Valid']);
+    }, 30000);
+
     it('should accept scalar frontmatter titles', async () => {
       await writeFile(join(testDir, 'numeric.md'), '---\ntitle: 2024\n---\nContent');