diff --git a/.gitattributes b/.gitattributes index a296bc07..354b696c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,4 @@ -deps/** filter=lfs diff=lfs merge=lfs -text +deps/** filter=lfs diff=lfs merge=lfs -text # Embedded prebuilt FST dictionary (go:embed build asset) -core/tokenizer/fstcjk/dict.fst filter=lfs diff=lfs merge=lfs -text +packages/core/tokenizer/fstcjk/dict.fst filter=lfs diff=lfs merge=lfs -text diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 696bff89..0aea5a7b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -43,14 +43,4 @@ Haystack is a local code search indexer tool designed to create and query search - Pebble DB (storage) - Git integration -### Project Documentation Structure -The project contains multiple README.md files throughout the directory structure, providing specific documentation for different components: -- Root README.md: Overall project description, installation and setup -- internal/server/README.md: Server component implementation details -- internal/core/invertedindex/README.md: Documentation for the storage subsystem and Pebble DB usage -- internal/server/indexer/README.md: Explanation of the indexing mechanism -- internal/server/searcher/README.md: Details on search functionality and query syntax - -These README.md files serve as component-specific documentation and should be consulted when working on the respective parts of the codebase. - This context should help you provide more relevant and aligned suggestions when working with the Haystack codebase. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bed9316b..98fc9633 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,20 +43,20 @@ jobs: run: go version - name: Build - working-directory: core + working-directory: packages/core run: go build ./... - name: Coverage gate (Linux) # go-cov runs core's full suite and enforces the same bar as the # core module. if: matrix.os == 'ubuntu-latest' - working-directory: core + working-directory: packages/core run: go run github.com/codetreker/go-cov/cmd/go-cov@v0.1.2 - name: Test (macOS/Windows) if: matrix.os != 'ubuntu-latest' shell: bash - working-directory: core + working-directory: packages/core run: | # -v prints each case's execution time. Show only result lines, and # drop the 0.00s cases (noise) — keep all FAILs and the full log on @@ -110,7 +110,7 @@ jobs: run: ./.github/workflows/scripts/check-fmt.sh - name: Build - run: go build ./... + run: go build ./packages/server/... ./packages/core/... - name: Test and Coverage run: ./.github/workflows/scripts/test_and_coverage.sh diff --git a/.github/workflows/fidelity.yml b/.github/workflows/fidelity.yml index 01ed276a..c47c441f 100644 --- a/.github/workflows/fidelity.yml +++ b/.github/workflows/fidelity.yml @@ -9,16 +9,16 @@ name: FST Fidelity on: pull_request: paths: - - 'core/tokenizer/fstcjk/**' - - 'core/tokenizer/cjk_fidelity_test.go' - - 'core/go.mod' + - 'packages/core/tokenizer/fstcjk/**' + - 'packages/core/tokenizer/cjk_fidelity_test.go' + - 'packages/core/go.mod' - '.github/workflows/fidelity.yml' push: branches: [main] paths: - - 'core/tokenizer/fstcjk/**' - - 'core/tokenizer/cjk_fidelity_test.go' - - 'core/go.mod' + - 'packages/core/tokenizer/fstcjk/**' + - 'packages/core/tokenizer/cjk_fidelity_test.go' + - 'packages/core/go.mod' jobs: fidelity: @@ -34,11 +34,11 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.24.2' # keep in sync with core/go.mod toolchain + go-version: '1.24.2' # keep in sync with packages/core/go.mod toolchain cache: true - name: Exhaustive FST-vs-gse byte-fidelity - working-directory: core + working-directory: packages/core env: HAYSTACK_FIDELITY: '1' # enables the live-gse golden suite HAYSTACK_FIDELITY_FULL: '1' # full fuzz counts (~60k inputs) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3d4a2f9d..8b434fed 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,12 +22,12 @@ jobs: git lfs pull - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version: '1.24.2' # keep in sync with the go.work / packages/*/go.mod toolchain - name: Run tests - run: go test ./... + run: go test ./packages/server/... ./packages/core/... - name: Run build.go run: go run build.go diff --git a/.github/workflows/scripts/test_and_coverage.sh b/.github/workflows/scripts/test_and_coverage.sh index 886cfa5a..773b0e0c 100755 --- a/.github/workflows/scripts/test_and_coverage.sh +++ b/.github/workflows/scripts/test_and_coverage.sh @@ -4,4 +4,6 @@ set -e +# The App CI job gates the SERVER module (core has its own job). +cd "$(dirname "$0")/../../../packages/server" go run github.com/codetreker/go-cov/cmd/go-cov@v0.1.2 diff --git a/.gitignore b/.gitignore index cb369af2..ea5cac6d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,12 +17,10 @@ node_modules/ /haystack # Generated test fixtures (large binary files) -core/vectorindex/testdata/*.bin *.out cover.html scripts/lib/coverage/coverage -core/vectorindex/testdata/sift/ -/cmd/bench50k* +packages/server/cmd/bench50k* .tmp/ tmp/ .bf/ diff --git a/.vscode/launch.json b/.vscode/launch.json index 438d624f..7bef5077 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,7 @@ "type": "go", "request": "launch", "mode": "auto", - "program": "${workspaceFolder}/cmd/haystack/", + "program": "${workspaceFolder}/packages/server/cmd/haystack/", "env": {}, "args": [ "server", diff --git a/Dockerfile.test b/Dockerfile.test deleted file mode 100644 index 6dc73981..00000000 --- a/Dockerfile.test +++ /dev/null @@ -1,9 +0,0 @@ -FROM golang:1.23 - -WORKDIR /app - -# Copy dependency files only (source mounted at runtime) -COPY go.mod go.sum ./ -RUN go mod download - -CMD ["bash", "-c", "ulimit -u 256 && go test ./... -count=1 -timeout 5m"] diff --git a/Makefile b/Makefile index f24d597f..2d98bc8e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test coverage fmt clean gen-testdata test-docker-build test-docker-ensure test-safe test-safe-race +.PHONY: build test coverage fmt clean test-docker-build test-docker-ensure test-safe test-safe-race APP_NAME=haystack BUILD_DIR=build @@ -20,12 +20,12 @@ APP_BIN=$(BUILD_DIR)/$(APP_NAME)$(EXE_EXT) build: @$(MKDIR_P) @echo "Building $(APP_NAME)..." - @go build -o $(APP_BIN) ./cmd/haystack/ + @go build -o $(APP_BIN) ./packages/server/cmd/haystack/ @echo "Build complete: $(APP_BIN)" test: @echo "Running tests..." - @go test ./... -count=1 + @go test ./packages/core/... ./packages/server/... -count=1 coverage: @echo "Running tests with coverage..." @@ -40,14 +40,10 @@ clean: @echo "Cleaning..." @$(RM_RF) -gen-testdata: - @echo "Generating test fixtures (this takes ~15 min)..." - @cd core && go run -tags tools ./cmd/gen-testdata/ - DOCKER_TEST_IMAGE=haystack-test test-docker-build: - @docker build -f Dockerfile.test -t $(DOCKER_TEST_IMAGE) . + @docker build -f ./packages/server/Dockerfile.test -t $(DOCKER_TEST_IMAGE) . test-docker-ensure: @if [ -z "$$(docker images -q $(DOCKER_TEST_IMAGE) 2>/dev/null)" ]; then \ @@ -67,4 +63,4 @@ test-safe-race: test-docker-ensure @echo "Running tests with race detector in Docker (isolated)..." @docker run --rm --cpus=2 --memory=4g --pids-limit=256 --network=none \ -v $$(pwd):/app:ro \ - $(DOCKER_TEST_IMAGE) bash -c "ulimit -u 256 && go test -race ./... -count=1 -timeout 5m" + $(DOCKER_TEST_IMAGE) bash -c "ulimit -u 256 && go test -race ./packages/server/... ./packages/core/... -count=1 -timeout 5m" diff --git a/README.md b/README.md index e3f9bf47..ff624093 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ The Search Indexer is structured with the following components: ### Prerequisites -- Go 1.23+ +- Go 1.24+ - Git ### Installation @@ -40,21 +40,20 @@ Clone the repository: ```bash git clone https://github.com/codetrek/haystack.git -cd haystack/src +cd haystack ``` -Install dependencies: - -```bash -go mod download -``` +The repository is a Go **workspace** (`go.work`) spanning two modules — +`packages/core` (the search/index library) and `packages/server` (the server + CLI +app). Dependencies are vendored under `vendor/`, so there is no separate +dependency-install step. ### Configuration Copy the example configuration and modify as needed: ```bash -cp config.example.yaml config.local.yaml +cp packages/server/config.example.yaml config.local.yaml ``` Edit `config.local.yaml` to configure your indexing preferences. @@ -62,14 +61,14 @@ Edit `config.local.yaml` to configure your indexing preferences. ### Running the Server ```bash -go run ./ server run +go run ./packages/server/cmd/haystack server run ``` ### Using the Client ```bash # Example query command -go run ./ search "your search query" +go run ./packages/server/cmd/haystack search "your search query" ``` ## Development @@ -79,13 +78,13 @@ go run ./ search "your search query" Run the test suite with: ```bash -go test ./... +make test # or: go test ./packages/core/... ./packages/server/... ``` ### Building ```bash -go build ./ +make build # or: go build -o build/haystack ./packages/server/cmd/haystack/ ``` ## Contributing diff --git a/build.go b/build.go index 1a3aee36..4e8a8816 100644 --- a/build.go +++ b/build.go @@ -50,7 +50,7 @@ func main() { "-ldflags", ldflags, "-gcflags=all=-l", "-o", binPath, - "./cmd/haystack/", + "./packages/server/cmd/haystack/", } cmd := exec.Command("go", args...) diff --git a/core/.gitignore b/core/.gitignore deleted file mode 100644 index 003eab11..00000000 --- a/core/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.core/ diff --git a/docs/research/mmap-storage-design-2.md b/docs/research/mmap-storage-design-2.md deleted file mode 100644 index eac9aaee..00000000 --- a/docs/research/mmap-storage-design-2.md +++ /dev/null @@ -1,529 +0,0 @@ -# HNSW Mmap Storage: Industry Survey & Haystack Design - -**Date:** 2026-04-17 -**Scope:** How production vector databases store HNSW indexes with mmap, supporting incremental insert/delete/upsert — informing Haystack's own mmap flat file design. - ---- - -## 1. Weaviate (Go) - -**Source:** `weaviate/weaviate` — `adapters/repos/db/vector/hnsw/` - -### File Format - -| Component | Format | -|-----------|--------| -| **Mutations** | Append-only binary commit log (WAL) | -| **Persisted graph** | Condensed mmap flat file (`.condensed`) | -| **Vectors** | Stored separately from HNSW graph structure | - -**Commit log record types** (16 total): Each record = 1 byte type tag + little-endian payload: - -| Record | Layout | Size | -|--------|--------|------| -| `AddNode` | type(1B) + ID(8B) + level(2B) | 11B | -| `AddLinkAtLevel` | type(1B) + src(8B) + level(2B) + target(8B) | 19B | -| `ReplaceLinksAtLevel` | type(1B) + src(8B) + level(2B) + count(2B) + targets(8B * N) | variable | -| `AddTombstone` | type(1B) + ID(8B) | 9B | -| `RemoveTombstone` | type(1B) + ID(8B) | 9B | -| `SetEntryPointMaxLevel` | type(1B) + ID(8B) + level(2B) | 11B | -| `DeleteNode` / `ClearLinks` / `ResetIndex` | similar compact encodings | — | - -**Condensed flat file layout:** The `MmapCondensorAnalyzer` replays commit logs and builds an index of `{id uint64, offset uint64, maxLevel uint16}` entries sorted by node ID. Each node's block is sized as: - -``` -nodeBlockSize = overhead(uint16 length indicators) + connectionsPerLevel * (maxLevel + 1) -// level 0 uses maxM0 = 2*M (standard HNSW) -``` - -Mapped with `mmap.MapRegion` using copy-on-write semantics. - -### Incremental Update - -- **Insert:** Appends `AddNode` + `AddLinkAtLevel`/`ReplaceLinksAtLevel` to commit log. Graph built in-memory. -- **Delete:** Two-phase tombstone system: - 1. `addTombstone()` — writes to both in-memory set and commit log (fast, O(1)) - 2. `CleanUpTombstonedNodes()` — periodic background job with parallel workers: - - Reassigns neighbor edges (removes dangling links, reconnects orphaned neighbors) - - Replaces entry point if tombstoned - - Removes tombstones from memory and commit log - - Aborts if memory pressure exceeds 100MB - -### Crash Recovery - -Replays commit logs sequentially on startup to reconstruct in-memory HNSW graph. Commit log is the source of truth. - -### Compaction - -Two-level pipeline: - -1. **Condensation:** Replay commit log into `.condensed` mmap flat file (net state only, no tombstones) -2. **Combination:** `CommitLogCombiner` merges consecutive `.condensed` files when `file1.size + file2.size <= threshold`. Writes to temp file, fsyncs, atomic rename, delete originals. - -### Mmap Usage - -Graph link structure is mmap'd after condensation. Vectors stored separately with configurable in-memory cache (`vectorCacheMaxObjects`). The HNSW index itself is primarily in-memory; condensed files provide persistence. - -### Lock Discipline - -`compressActionLock`, `deleteVsInsertLock`, `deleteLock` coordinate concurrent insert/delete/compaction. - ---- - -## 2. Qdrant (Rust) - -**Source:** `qdrant/qdrant` — `lib/segment/src/index/hnsw_index/`, `lib/segment/src/vector_storage/` - -### File Format - -**Segment architecture:** All data divided into independent segments, each with its own vector storage, payload index, and HNSW index. - -| Segment type | Properties | -|-------------|-----------| -| **Appendable** | Read/write, accepts new inserts | -| **Non-appendable** | Read + soft delete only, optimized for search | - -**Vector storage backends:** -- In-memory (backed by RocksDB for persistence) -- **Memmap:** Chunked appendable mmap format (`chunked_vectors`). Creates virtual address space mapped to disk. - -**Graph links:** Flattened hierarchical structure. `GraphLinksEnum` has two backends: -- `Ram(Vec)` — fully in-memory -- `Mmap(Arc)` — memory-mapped, with `Advice::Random` access pattern and optional `populate` for page cache preloading - -Three compression formats: Plain, Compressed, CompressedWithVectors. - -### Incremental Update - -- **Insert:** Written to appendable segment. When segment size crosses `indexing_threshold_kb`, indexing optimizer triggers HNSW construction. -- **Delete:** Soft delete via bit markers in ID tracker. Mutable and immutable tracker variants. Deleted records accumulate until vacuum threshold reached. -- **Upsert:** Insert with version tracking — newer version wins. - -### Crash Recovery - -**WAL with version tracking:** All operations get sequential numbers. On recovery, WAL is replayed; version numbers prevent applying stale operations to segments that already contain newer data. - -### Compaction — Three Optimizers - -| Optimizer | Trigger | Action | -|-----------|---------|--------| -| **Vacuum** | `deleted_count / total > deleted_threshold` | Rebuild segment excluding deleted records | -| **Merge** | `segment_count > default_segment_number` | Consolidate small segments | -| **Indexing** | Segment size exceeds `indexing_threshold_kb` | Migrate brute-force segment to HNSW + memmap | - -All optimizers use **copy-on-write** to maintain read availability during optimization. - -### Mmap Usage - -Extensive. Both vector data and graph links support mmap backends. Configurable advice (Random, Sequential) and populate/prefetch settings. Chunked mmap format allows appending new vectors without remapping entire file. - ---- - -## 3. hnswlib (C++) - -**Source:** `nmslib/hnswlib` — `hnswlib/hnswalg.h` - -### File Format - -Single binary file, sequential layout: - -``` -[Metadata header] - offsetLevel0_, max_elements_, cur_element_count, - size_data_per_element_, label_offset_, offsetData_, - maxlevel_, enterpoint_node_, maxM_, maxM0_, M_, - mult_, ef_construction_ - (raw POD types, no versioning) - -[Level-0 data block] - cur_element_count * size_data_per_element_ bytes - Each element = [links_level0 | vector_data | label] - Links and vectors are INTERLEAVED in same flat array - -[Higher-level link lists] - For each element: - uint size (0 if element has no higher levels) - link list data (variable length) -``` - -Vectors are **inline** with level-0 links — no separation. - -### Incremental Update - -- **Insert:** `addPoint()` up to pre-allocated `max_elements`. **Cannot grow beyond this limit** without full rebuild. -- **Delete:** `markDelete()` sets a bit in byte 2 of level-0 link list memory: `*(ll_cur + 2) |= 0x01`. Soft delete only — **no graph repair**, edges to deleted nodes persist. Deleted nodes skipped during search. -- **No incremental persistence.** `saveIndex()` / `loadIndex()` are full serialization. No WAL. - -### Crash Recovery - -None. Crash during `saveIndex()` corrupts the file. No checksums, no WAL. - -### Compaction - -None. Deleted elements waste space permanently until full rebuild. - -### Mmap Usage - -None. All memory allocated via `malloc`. Entire index must fit in RAM. - -### Assessment - -hnswlib is a reference algorithm implementation, not a storage engine. Its format is the simplest to understand but lacks every production feature: no WAL, no mmap, no incremental persistence, no compaction, fixed capacity. - ---- - -## 4. Vald (Go) - -**Source:** `vdaas/vald` — `pkg/agent/core/ngt/service/` - -### Architecture - -Distributed vector database on Kubernetes, built on Yahoo Japan's **NGT** (Neighborhood Graph and Tree), **not HNSW**. Microservice architecture with agent pods. - -### Storage - -| Component | Purpose | -|-----------|---------| -| `kvs` (BidiMap) | Bidirectional UUID <-> internal object ID mapping | -| `vqueue` | Virtual queue buffering insert/delete before indexing | -| `core` | NGT index (vectors + graph) | -| `fmap` | Failed operation tracker for recovery | - -### Insert/Delete - -- **Insert:** Validates no duplicate UUID, pushes to `vqueue`. Periodically, vectors extracted from queue, inserted into NGT core, UUID-to-ID mapping stored in KVS. -- **Delete:** Also queued in `vqueue`, processed during periodic index creation. Removes both KVS entry and NGT core object. - -**Key pattern:** Mutations are buffered in a virtual queue and batch-applied during periodic index creation, reducing rebuild frequency. - -### Persistence - -- Serialization via **GOB encoding** (Go-native binary format) -- **Copy-on-write save:** Write to temp directory, then atomically swap to production path -- On startup, attempts loading from multiple fallback locations -- Distributed resilience via Kubernetes replication - -### Mmap Usage - -None. Full in-memory index with periodic full serialization snapshots. - ---- - -## 5. Chroma - -**Source:** `chroma-core/chroma` — `rust/index/src/` - -### Architecture - -Wraps hnswlib through a Rust `HnswIndexProvider` that manages index lifecycle. Recent versions moved from Python hnswlib bindings to Rust layer. - -### File Format - -Splits hnswlib's single file into **four binary files:** - -| File | Content | -|------|---------| -| `header.bin` | Index metadata (M, ef, dimensions, etc.) | -| `data_level0.bin` | Level-0 data (vectors + level-0 links) | -| `length.bin` | Element count/sizing info | -| `link_lists.bin` | Higher-level link lists | - -Stored in S3 or local filesystem with optional encryption. - -### Insert/Delete - -Inherits hnswlib's mechanisms: -- Insert adds to in-memory index -- Delete uses `markDelete` (soft delete, no graph repair) -- **No incremental persistence** — index must be fully flushed - -### Crash Recovery - -- **Flush:** Serialize in-memory state to four files, upload with priority, supports parallel uploads -- **Load:** Double-checked locking to avoid redundant loads -- **Fork:** Copy serialized data from source with new UUID -- No WAL. Crash loses unflushed data. - -### Mmap Usage - -None. Full deserialization into memory. - -### Assessment - -Chroma's main contribution is splitting the single hnswlib file into four parallel-loadable files. Otherwise inherits all of hnswlib's limitations. - ---- - -## Comparative Summary - -| Feature | Weaviate | Qdrant | hnswlib | Vald | Chroma | -|---------|----------|--------|---------|------|--------| -| Language | Go | Rust | C++ | Go | Rust | -| Algorithm | HNSW | HNSW | HNSW | NGT | HNSW (via hnswlib) | -| Graph persist | Commit log -> mmap flat | Mmap or RAM, flattened | Single binary blob | GOB snapshot | 4 binary files | -| Vector persist | Separate store | Mmap chunks or RAM | Inline with level-0 | GOB snapshot | Inline with level-0 | -| Delete | Tombstone + background edge repair | Soft delete + vacuum rebuild | Bit flag, no repair | Queue batch | Bit flag, no repair | -| WAL | Typed binary commit log | Versioned WAL | None | None | None | -| Crash recovery | Commit log replay | WAL replay with versions | None | COW snapshots | None | -| Compaction | Condense + combine | Vacuum/merge/indexing optimizers | None | Full rebuild | None | -| Mmap graph | Yes (after condensation) | Yes (optional backend) | No | No | No | -| Mmap vectors | No (separate cache) | Yes (chunked) | No | No | No | -| Dynamic capacity | Yes | Yes (segment growth) | No (fixed max_elements) | Yes | No (hnswlib limit) | - ---- - -## Recommended Design for Haystack - -### Principles - -1. **Separate vectors from graph structure** — different access patterns, different file management -2. **WAL-first** — all mutations hit the write-ahead log before anything else -3. **Two-phase delete** — tombstone immediately, repair graph in background -4. **Condense for mmap** — replay WAL into position-indexed flat files for mmap access -5. **Grow incrementally** — no fixed max_elements; use chunked or segment-based expansion - -### File Layout - -``` -index_dir/ - wal/ - 000001.wal # append-only binary WAL segments - 000002.wal - vectors.dat # mmap'd flat vector storage - vectors.idx # ID -> offset mapping for vectors - graph_l0.dat # mmap'd level-0 connections (fixed-size blocks) - graph_upper.dat # mmap'd upper-level connections - graph_upper.idx # node ID -> offset into graph_upper.dat - meta.json # index metadata (M, efConstruction, dim, entryPoint, maxLevel) - tombstones.bin # active tombstone set -``` - -### File Formats — Detail - -#### WAL Records - -Same philosophy as Weaviate's commit log: typed binary records, append-only. - -``` -Record = TypeTag(1B) + Payload - -Types: - 0x01 InsertVector nodeID(8B) + level(2B) + vector(dim*4 B) - 0x02 SetLinks nodeID(8B) + level(2B) + count(2B) + targetIDs(8B * count) - 0x03 SetEntryPoint nodeID(8B) + maxLevel(2B) - 0x04 AddTombstone nodeID(8B) - 0x05 RemoveTombstone nodeID(8B) - 0x06 DeleteNode nodeID(8B) - -Each WAL segment = sequence of records + CRC32 per record for integrity -``` - -WAL segments are numbered sequentially. Fsync after each write (or batch of writes with configurable flush interval for throughput). - -#### vectors.dat (Mmap'd) - -Flat file of fixed-size vector blocks: - -``` -[Header: 32B] - magic(4B) + version(2B) + dimensions(4B) + count(8B) + reserved(14B) - -[Vector blocks] - Block N at offset = 32 + N * (dim * 4) - Each block = float32[dim] -``` - -Node IDs map directly to block offsets: `offset = headerSize + nodeID * dim * sizeof(float32)`. This gives O(1) random access. Deleted vectors leave gaps (reclaimed by compaction). - -For growth beyond initial allocation: **chunked mmap** (inspired by Qdrant). Each chunk is a fixed-size mmap region (e.g., 64MB). New chunks allocated as needed. A chunk table maps nodeID ranges to chunks. - -#### graph_l0.dat (Mmap'd) - -Fixed-size blocks for level-0 connections: - -``` -[Header: 16B] - maxM0(2B) + count(8B) + reserved(6B) - -[Connection blocks] - Block N at offset = 16 + N * blockSize - blockSize = 2B (numLinks) + maxM0 * 8B (neighborIDs) + 1B (flags: tombstoned, etc.) -``` - -O(1) access: `offset = headerSize + nodeID * blockSize`. Same dense layout as hnswlib level-0 but separated from vectors. - -#### graph_upper.dat + graph_upper.idx (Mmap'd) - -Upper levels are sparse (few nodes have level > 0), so use offset index: - -``` -graph_upper.idx: - [nodeID(8B) + offset(8B) + numLevels(2B)] * count - Sorted by nodeID for binary search - -graph_upper.dat: - Per node: [level1_links | level2_links | ...] - level_N_links = count(2B) + neighborIDs(8B * count) -``` - -### Insert Flow - -``` -1. Acquire insert lock (shared, allows concurrent inserts) -2. Append InsertVector to WAL -3. Write vector to vectors.dat (extend or fill gap from free list) -4. Run HNSW neighbor search (in-memory graph + mmap reads) -5. For each affected node, append SetLinks to WAL -6. Update in-memory graph (level-0 and upper-level connections) -7. Update graph_l0.dat and graph_upper.dat in-place (mmap writes) -``` - -**Batch insert optimization:** Buffer multiple inserts, write WAL records in batch, fsync once. - -### Delete Flow - -``` -Phase 1 — Mark (immediate, O(1)): - 1. Append AddTombstone to WAL - 2. Add nodeID to in-memory tombstone set - 3. Set tombstone flag in graph_l0.dat block - -Phase 2 — Cleanup (background, periodic): - 1. For each tombstoned node: - a. Collect its neighbors - b. For each neighbor, remove the tombstoned node from their link list - c. Reconnect orphaned neighbors to maintain graph quality - (find best replacement links via local search) - d. If tombstoned node was entry point, elect new entry point - 2. Add nodeID to free list for vector slot reuse - 3. Append RemoveTombstone + DeleteNode to WAL - 4. Remove from in-memory tombstone set -``` - -Cleanup runs with configurable parallelism. Respects memory pressure. Can be interrupted and resumed. - -### Upsert Flow - -``` -1. If nodeID exists and not tombstoned: - a. Update vector in vectors.dat (overwrite in-place) - b. Append InsertVector to WAL (marks as update) - c. If vector changed significantly, re-link: - - Run HNSW search with new vector - - Update connections for affected nodes -2. If nodeID doesn't exist or is tombstoned: - a. Remove tombstone if present - b. Insert as new (standard insert flow) -``` - -### Crash Recovery - -``` -On startup: - 1. Read meta.json for index parameters - 2. Mmap vectors.dat, graph_l0.dat, graph_upper.dat (read-only initially) - 3. Load tombstones.bin into memory - 4. Replay WAL from last checkpoint: - - For each record, apply to in-memory state - - Verify against mmap'd files (skip if already applied) - 5. Re-mmap files as read-write - 6. Resume normal operation -``` - -**Checkpoint mechanism:** After condensation/compaction, record the WAL offset in meta.json. On recovery, only replay from checkpoint forward. - -**CRC32 per WAL record** detects partial writes from crashes. Truncate WAL at first corrupted record. - -### Compaction Strategy - -Inspired by Weaviate's condense + combine, simplified: - -**Level 1 — WAL Condensation (frequent):** -``` -1. Snapshot current WAL position -2. Replay WAL records into mmap files (apply net state) -3. Fsync all mmap files -4. Update checkpoint in meta.json -5. Delete WAL segments before checkpoint -``` - -**Level 2 — Space Reclamation (periodic):** -``` -Trigger: free_slots / total_slots > 20% (configurable) -1. Create new vectors.dat, graph_l0.dat, graph_upper.dat -2. Copy live (non-tombstoned) nodes with compacted IDs -3. Build ID remapping table (old ID -> new ID) -4. Fsync new files -5. Atomic swap (rename) new files over old -6. Delete old files -``` - -This is the most expensive operation. Run during low-traffic periods. Use copy-on-write during compaction so reads continue from old files. - -**Level 3 — Upper-level defrag (rare):** -``` -Rewrite graph_upper.dat + graph_upper.idx to remove gaps from deleted nodes. -Much cheaper than full compaction since upper levels are small. -``` - -### Mmap Strategy - -| File | Access pattern | Mmap advice | Writability | -|------|---------------|-------------|-------------| -| vectors.dat | Random (search), sequential (scan) | `MADV_RANDOM` | Read-write | -| graph_l0.dat | Random (graph traversal) | `MADV_RANDOM` | Read-write | -| graph_upper.dat | Random, infrequent | `MADV_RANDOM` | Read-write | -| WAL segments | Sequential append | `MADV_SEQUENTIAL` | Append-only | - -**Growth strategy:** Use Go's `syscall.Mmap` or `golang.org/x/exp/mmap`. For growth: -- Pre-allocate with headroom (e.g., 2x current size) -- When full, unmap, truncate file to new size (ftruncate), remap -- Or use chunked approach: multiple mmap regions of fixed size - -**Page cache management:** For large indexes exceeding RAM, the OS page cache handles eviction. `MADV_DONTNEED` can be used to hint that compaction source pages are no longer needed after copying. - -### Concurrency - -``` -Locks: - insertMu sync.RWMutex // shared for concurrent inserts - deleteMu sync.Mutex // exclusive for tombstone cleanup - compactMu sync.RWMutex // write-lock during file swap, read-lock for normal ops -``` - -Insert and search can proceed concurrently. Tombstone cleanup acquires deleteMu to prevent concurrent cleanups. Compaction file swap briefly acquires compactMu write-lock. - -### What We Borrow From Each Project - -| Source | Borrowed idea | -|--------|--------------| -| **Weaviate** | Typed binary commit log, two-phase tombstone delete with background edge repair, condense + combine compaction pipeline, lock discipline pattern | -| **Qdrant** | Separate mutable/immutable storage concepts, chunked mmap for growth, mmap advice configuration, version-tracked WAL replay, three-tier optimization | -| **hnswlib** | Fixed-size level-0 block layout (O(1) offset calculation), interleaved link + metadata format as baseline reference | -| **Vald** | Virtual queue pattern for batch mutations, COW directory swap for atomic persistence, BidiMap for external-to-internal ID mapping | -| **Chroma** | Splitting index into multiple files for parallel I/O, provider/cache lifecycle management | - -### Implementation Priority - -1. **Phase 1 — WAL + in-memory graph** (foundation) - - Binary WAL with typed records and CRC32 - - In-memory HNSW graph (current implementation) - - WAL replay on startup - -2. **Phase 2 — Mmap vector storage** - - vectors.dat with O(1) access by nodeID - - Chunked growth support - -3. **Phase 3 — Mmap graph storage** - - graph_l0.dat with fixed-size blocks - - graph_upper.dat with offset index - - Transition from fully in-memory to mmap-backed - -4. **Phase 4 — Compaction** - - WAL condensation (level 1) - - Space reclamation (level 2) - -5. **Phase 5 — Production hardening** - - Concurrent insert/delete/search stress testing - - Crash recovery fuzz testing - - Memory pressure handling diff --git a/docs/research/storage-backend-research-1.md b/docs/research/storage-backend-research-1.md deleted file mode 100644 index a7c12370..00000000 --- a/docs/research/storage-backend-research-1.md +++ /dev/null @@ -1,374 +0,0 @@ -# HNSW 向量索引存储后端选型研究 - -> **日期**: 2026-04-17 -> **关联**: HAY-007 PebbleStore 构建性能优化 -> **状态**: 调研完成 - ---- - -## 1. 问题陈述 - -Haystack 的 HNSW 向量索引在构建阶段是**读密集型**工作负载： - -| 指标 | 数值 | -|------|------| -| 每次 Insert 平均读向量 | ~3,500 次 | -| 每次 Insert 平均写操作 | ~37 次 | -| **读写比** | **86:1** | -| 50K 向量总随机读 | 1.6 亿次 | -| 50K 向量总写入 | 186 万次 | -| 单条向量大小 (128d float32) | 512 bytes | -| 单条向量大小 (768d float32) | 3,072 bytes | - -当前 PebbleStore（LSM-tree）是**写优化**结构，与 HNSW 构建的读密集模式严重不匹配： - -| 后端 | 50K 耗时 | ops/sec | 相对速度 | -|------|----------|---------|----------| -| MemStore | 59s (1.18ms/op) | 847 | **1x (基准)** | -| PebbleStore | 18min (21.85ms/op) | 46 | **18.5x 慢** | - ---- - -## 2. 当前 Pebble 配置分析 - -``` -internal/core/pebble/db.go: - Cache: 调用方传入 cacheSize - BlockSize: 32 KB - FilterPolicy: Bloom(10) - MemTableSize: 4 MB - MaxOpenFiles: 8192 - L0CompactionThreshold: 12 - MaxConcurrentCompactions: 2 -``` - -``` -internal/core/vectorindex/store.go: - LRU cache 容量: 10,000 条向量 - GetVectorRef(): 零拷贝读取 -``` - -**瓶颈诊断**： -- 32KB BlockSize 意味着读一个 512B 向量要解压一整个 32KB block -- LRU cache 10K 条在 50K 数据集下命中率仅约 20% -- LSM 多层查找：Bloom filter → L0 → L1 → ... 每次随机读可能触发多次磁盘 I/O -- 每次 Get 返回的 value 需要 Pebble 内部拷贝（即使 GetVectorRef 做了应用层零拷贝） - ---- - -## 3. 方案详细评估 - -### 方案 A：MemStore 构建 + Pebble 持久化（混合模式） - -**架构**： -``` -[构建阶段] Insert → MemNodeStore (全内存) -[持久化] 构建完成 → 批量 dump 到 Pebble -[启动] Pebble scan → 加载到 MemNodeStore -[搜索] 查询 → MemNodeStore (内存) -``` - -**性能预期**： - -| 指标 | 估算 | -|------|------| -| 50K 构建耗时 | ~59s（等同 MemStore） | -| 持久化耗时 (50K) | ~2-5s（顺序批量写入） | -| 启动加载耗时 (50K) | ~3-8s（Pebble scan + 反序列化） | -| 搜索延迟 | <1ms（纯内存） | - -**内存占用**： - -| 规模 | 128d | 768d | -|------|------|------| -| 10K | ~9 MB | ~35 MB | -| 50K | ~44 MB | ~175 MB | -| 100K | ~88 MB | ~350 MB | -| 500K | ~440 MB | ~1.75 GB | - -> 含向量 + neighbors + metadata 开销，约为纯向量大小的 1.5-1.7x - -**优点**： -- 构建速度等同纯内存，18.5x 提升 -- 实现简单：复用现有 MemNodeStore + PebbleNodeStore -- 搜索延迟最优 -- 渐进式实现，不破坏现有接口 -- Pebble 批量写入非常高效（顺序写，一次 fsync） - -**缺点**： -- 内存占用与数据规模线性增长，500K x 768d 需要 ~1.75 GB -- 启动时需要全量加载，冷启动较慢 -- 构建过程中宕机丢失未持久化数据（可通过 checkpoint 缓解） -- 增量更新（单条 upsert）需要特殊处理 - -**实现复杂度**：**低** (~200-300 行代码) -- 需要实现 `MemNodeStore.DumpTo(PebbleNodeStore)` 方法 -- 需要实现 `PebbleNodeStore.LoadInto(MemNodeStore)` 方法 -- 修改 HNSWIndex 初始化逻辑，支持两阶段模式 -- 已有 MemNodeStore 和 PebbleNodeStore 完整实现 - ---- - -### 方案 B：mmap 文件存储 - -**架构**： -``` -vectors.bin: [vec_0: 512B][vec_1: 512B]...[vec_N: 512B] ← 固定大小记录 -neighbors.bin: [node_0_layer_0][node_0_layer_1]... ← 变长，需索引 -meta.bin: entry point, levels, mappings ← 小文件 -``` - -向量按 ID 偏移访问：`offset = id * dim * 4`，O(1) 随机读。 - -**性能预期**： - -| 指标 | 估算 | -|------|------| -| 热读延迟 | <1 us（等同内存指针访问） | -| 冷读延迟 | ~10-100 us（page fault） | -| 50K 构建耗时 | ~60-70s（接近 MemStore，page cache 覆盖） | -| 写入延迟 | 需 munmap/mmap 周期扩展文件 | - -**参考实现**：Weaviate 使用 mmap 存储 HNSW 图和向量数据，读延迟 <1us。 - -**优点**： -- 随机读性能理论最优 -- OS page cache 自动管理热数据 -- 内存占用由 OS 管理，不占 Go heap（无 GC 压力） -- 500K 向量在 page cache 热时等同内存访问 - -**缺点**： -- **实现复杂度高**：固定大小向量尚可，变长 neighbors 需要复杂索引 -- **并发写入困难**：扩展文件需要 munmap→truncate→mmap，期间所有读失效 -- **跨平台差异**：Windows 的 mmap 行为不同（不能 truncate mmap'd 文件） -- **崩溃恢复复杂**：无 ACID 保证，需自行实现 WAL 或 checkpoint -- **删除/upsert 复杂**：需要空洞管理或紧凑化 -- golang.org/x/exp/mmap 只支持只读，写入必须用 syscall - -**实现复杂度**：**高** (~1000-1500 行代码) -- 向量文件 + neighbors 文件 + metadata 文件 -- 空间分配器（处理删除产生的空洞） -- 跨平台 mmap 封装（build tags for linux/darwin/windows） -- 崩溃恢复机制 -- 文件增长策略 - ---- - -### 方案 C：bbolt (B+tree) - -**架构**：单文件 B+tree，mmap 读取，copy-on-write 写入。 - -```go -db, _ := bbolt.Open("index.db", 0600, nil) -db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte("vectors")) - vec := b.Get(uint64ToBytes(id)) // mmap 直接读取 - return nil -}) -``` - -**性能预期**： - -| 指标 | 估算 | -|------|------| -| 热读延迟 | 1-5 us（mmap B+tree 查找） | -| 冷读延迟 | 50-200 us | -| 写入延迟 | 50-200 us/tx（含 fsync） | -| 50K 构建耗时 | ~90-150s（估算，batch 写入 + mmap 读） | - -**优点**： -- B+tree 读优化，比 LSM 随机读快 -- 单文件存储，部署简单 -- mmap 读取，OS page cache 自动管理 -- 成熟稳定（etcd 核心组件） -- API 简洁，事务模型清晰 - -**缺点**： -- **单写者模型**：写事务互斥，虽然 86:1 比例可接受但构建时仍有写 -- **写入较慢**：每个写事务 fsync，批量写需要大事务 -- B+tree 开销比 mmap 直接偏移访问大（需要树遍历） -- 文件只增不缩（需要 Compact 回收空间） -- 不支持压缩 - -**实现复杂度**：**中** (~400-500 行代码) -- 实现新的 BboltNodeStore -- Bucket 设计（vectors / neighbors / metadata / mappings） -- 读事务池化（避免每次分配） -- 写事务批量化 - ---- - -### 方案 D：modernc.org/sqlite（纯 Go SQLite） - -**架构**：纯 Go 转译的 SQLite，B-tree 存储，WAL 模式。 - -```sql -CREATE TABLE vectors (id INTEGER PRIMARY KEY, data BLOB); -CREATE TABLE neighbors (node_id INTEGER, layer INTEGER, neighbors BLOB, PRIMARY KEY(node_id, layer)); -``` - -**性能预期**： - -| 指标 | 估算 | -|------|------| -| 热读延迟 | 5-15 us（SQL 解析 + B-tree 查找） | -| 冷读延迟 | 100-500 us | -| 50K 构建耗时 | ~150-300s（估算） | - -**优点**： -- SQL 查询灵活，方便调试 -- WAL 模式支持并发读写 -- 成熟的 ACID 保证 -- 单文件部署 - -**缺点**： -- **纯 Go 版比 CGo 慢 2-4x**：c2go 转译的代码无法享受 Go 编译器优化 -- SQL 解析开销：KV 场景下 SQL 层是纯开销 -- 内存分配多（C-to-Go 内存模型转换） -- 对 KV 场景而言过于重量级 -- 社区反馈性能波动较大 - -**实现复杂度**：**中** (~400-500 行代码) -- Schema 设计 + prepared statements -- 事务管理 -- Blob 编解码 - ---- - -### 方案 E：Pebble 优化（当前架构调优） - -**可调参数**： - -| 参数 | 当前值 | 建议值 | 影响 | -|------|--------|--------|------| -| Cache | 调用方决定 | 256-512 MB | 全数据集缓存 | -| BlockSize | 32 KB | 4 KB | 减少读放大 (32KB→4KB = 8x) | -| Compression | Snappy (默认) | NoCompression | 省去解压 CPU | -| LRU cache (应用层) | 10,000 | 50,000-100,000 | 直接命中率翻倍 | -| BloomFilter | 10 bits | 保持 | 已足够 | - -**性能预期（调优后）**： - -| 场景 | 预期延迟 | -|------|----------| -| 全数据集缓存热 | 1-3 us/read | -| 50K 构建耗时（预热后） | ~120-180s（2-3x 提升，从 18min 降到 ~3min） | -| 首次冷启动 | 仍然 18min | - -**关键问题**：构建阶段是逐步插入新向量，cache 从冷到热是渐进的。前期数据少时 cache 命中率高，后期数据增长超过 cache 容量后性能下降。**无法根本解决构建阶段的性能问题**。 - -**优点**： -- 改动最小，只调参数 -- 搜索阶段（cache 热）性能接近内存 -- 不需要新的 Store 实现 - -**缺点**： -- 构建阶段改善有限（3-5x 提升 vs 方案 A 的 18.5x） -- 大规模场景 cache 内存开销与方案 A 相当，但性能更差 -- BlockSize 修改需要重建数据库 -- 不解决根本的架构不匹配问题 - -**实现复杂度**：**极低** (~20-50 行代码) - ---- - -### 方案 F：BadgerDB（KV 分离 LSM） - -**架构**：Key-value 分离，key 在 LSM-tree，value 在 append-only vlog。 - -**评估**： -- 512B 向量低于 BadgerDB v4 的 ValueThreshold (1MB)，会被 **inline 存储在 LSM 中** -- 等同于另一个 LSM-tree，随机读性能与 Pebble 相当（3-10 us 热读） -- **无优势**，额外增加 vlog GC 的运维复杂度 -- 社区活跃度下降，Pebble 是更好的 LSM 选择 - -**结论**：**不推荐**，对本场景无收益。 - ---- - -## 4. 方案对比总表 - -| 维度 | A: 混合模式 | B: mmap | C: bbolt | D: SQLite | E: Pebble 调优 | -|------|-------------|---------|----------|-----------|----------------| -| **构建速度** | ★★★★★ ~59s | ★★★★★ ~65s | ★★★☆☆ ~120s | ★★☆☆☆ ~200s | ★★★☆☆ ~150s | -| **搜索延迟** | ★★★★★ <1ms | ★★★★★ <1ms | ★★★★☆ <2ms | ★★★☆☆ <5ms | ★★★★☆ <2ms | -| **内存占用** | ★★☆☆☆ 全量 | ★★★★☆ OS 管理 | ★★★★☆ OS 管理 | ★★★☆☆ 中等 | ★★★☆☆ cache 依赖 | -| **持久化安全** | ★★★★☆ Pebble | ★★☆☆☆ 需自建 | ★★★★★ ACID | ★★★★★ ACID | ★★★★★ Pebble | -| **实现复杂度** | ★★★★★ 低 | ★★☆☆☆ 高 | ★★★☆☆ 中 | ★★★☆☆ 中 | ★★★★★ 极低 | -| **跨平台** | ★★★★★ | ★★★☆☆ | ★★★★★ | ★★★★★ | ★★★★★ | -| **500K 可行性** | ★★★☆☆ ~1.7GB | ★★★★★ | ★★★★☆ | ★★★★☆ | ★★★☆☆ cache 大 | -| **增量更新** | ★★★☆☆ 需特殊处理 | ★★☆☆☆ 复杂 | ★★★★☆ 自然支持 | ★★★★☆ 自然支持 | ★★★★★ 自然支持 | - ---- - -## 5. 推荐排序 - -### 第一推荐：方案 A — MemStore 构建 + Pebble 持久化 - -**理由**： -1. **性能最优**：构建速度直接等同 MemStore（18.5x 提升），这是理论上限 -2. **实现最简**：200-300 行代码，复用现有 MemNodeStore + PebbleNodeStore -3. **风险最低**：两个 Store 实现已经过充分测试（HAY-006 完整覆盖） -4. **渐进式**：不破坏现有 NodeStore 接口，可与当前代码并存 - -**适用规模**：10K-100K 向量（128d），10K-50K 向量（768d）。超过此规模内存成为瓶颈。 - -**增量更新策略**： -- 少量 upsert（<100 条）：直接操作 MemStore（已在内存中） -- 大批量更新：重建索引（HNSW 大批量更新本身效率低） - -### 第二推荐：方案 A + E 组合 — 混合模式 + Pebble 参数调优 - -**理由**： -- 方案 A 负责构建阶段性能 -- Pebble 参数调优（BlockSize 4KB、NoCompression）优化持久化层效率 -- 两者互补，无冲突 - -### 第三推荐：方案 B — mmap（长期演进方向） - -**理由**： -- 如果 500K x 768d 规模成为刚需，mmap 是唯一不需要全量内存的高性能方案 -- 可作为 v2 架构，在方案 A 验证后再启动 -- 参考 Weaviate 的成熟实践 - -### 不推荐：方案 C (bbolt)、方案 D (SQLite)、方案 F (BadgerDB) - -**理由**： -- bbolt 和 SQLite 比 Pebble 快但比 MemStore 慢很多，属于"中间地带"——付出迁移成本却拿不到最优性能 -- BadgerDB 对此场景无收益 -- 如果要换存储引擎，不如直接跳到 mmap（方案 B）获取根本性提升 - ---- - -## 6. 实施路线图 - -``` -Phase 1 (HAY-007): 方案 A — MemStore + Pebble 混合模式 -├── 实现 DumpTo / LoadFrom 方法 -├── 修改 HNSWIndex 支持双阶段模式 -├── 基准测试验证 50K 性能 -└── 预期: 构建 18min → 60s - -Phase 1.5: 方案 E — Pebble 参数调优 -├── BlockSize 32KB → 4KB -├── 禁用向量数据压缩 -├── 优化应用层 LRU cache 大小 -└── 预期: 持久化/加载 2-3x 提升 - -Phase 2 (未来): 方案 B — mmap 存储（如需 500K+ 规模） -├── 固定大小向量文件 + mmap -├── 替换 Pebble 作为向量存储层 -├── 保留 Pebble 存 metadata/mappings -└── 预期: 内存降至 OS page cache 管理 -``` - ---- - -## 7. 关键风险与缓解 - -| 风险 | 影响 | 缓解 | -|------|------|------| -| 方案 A 内存不足 (500K x 768d = 1.75GB) | 无法在小内存机器运行 | 设置内存上限，超限时回退到 Pebble 直接构建 | -| 构建中宕机丢失数据 | 需要重新构建全部索引 | 每 N 条 checkpoint 一次到 Pebble | -| MemStore → Pebble dump 期间阻塞搜索 | 搜索不可用 | dump 在后台执行，搜索仍走 MemStore | -| 冷启动加载时间随数据增长 | 大数据集启动慢 | 并行加载 + 进度反馈；考虑 mmap 作为长期方案 | diff --git a/docs/research/storage-backend-research-2.md b/docs/research/storage-backend-research-2.md deleted file mode 100644 index ca68c0cb..00000000 --- a/docs/research/storage-backend-research-2.md +++ /dev/null @@ -1,247 +0,0 @@ -# HNSW 存储后端选型调研 - -> Date: 2026-04-17 | Context: HAY-007 PebbleStore build performance - -## 1. 问题本质 - -HNSW 构建的 I/O 模式极其特殊：**随机点查为主，写极少**。 - -| 指标 | 数值 | -|------|------| -| 50K 向量总读次数 | ~1.6 亿次 | -| 每次 insert 平均读 | 3,500 次向量 | -| 每次 insert 平均写 | 37 次 | -| 读写比 | **86:1** | -| 单次读取大小 | 512 bytes (128d × 4B) | -| MemStore 耗时 | 59s (1.18ms/op) | -| PebbleStore 耗时 | 18min (21.85ms/op) | -| 性能差距 | **18.5x** | - -核心瓶颈：Pebble 的 LSM-tree 为写优化设计，每次随机读需要遍历 memtable + 多级 SST，对 HNSW 的随机点查 pattern 天然不利。 - -## 2. 业界 HNSW 实现的存储方案 - -### hnswlib (原始实现, C++) -- **方案**: 单一连续内存块，向量按 ID 顺序排列 -- **持久化**: 整体 `save_index()` / `load_index()` 写入/读取单个二进制文件 -- **特点**: 纯内存操作，O(1) 向量访问（基地址 + offset），加载时整块读入内存 -- **结论**: 全量内存，不做增量持久化 - -### Weaviate (Go) -- **方案**: 自研 mmap 文件格式 - - 向量存储：固定大小记录的 flat file，mmap 映射，按 ID 直接偏移寻址 - - 图连接：commitlog + 内存中的 slice-of-slices - - HNSW 层级数据全部在内存 -- **持久化**: commitlog (WAL) 用于 crash recovery，定期 compaction -- **特点**: 向量通过 mmap 实现 O(1) 访问，内存由 OS page cache 管理 -- **启动**: mmap 映射即可，无需反序列化 - -### Milvus (Go + C++) -- **方案**: Segment-based 架构 - - Growing segment（写入缓冲）在内存 - - Sealed segment（只读）flush 到对象存储/本地文件 - - 向量数据以列式存储，连续排列 -- **特点**: 读写分离，sealed segment 可 mmap 加载 - -### Qdrant (Rust) -- **方案**: 自研存储 - - 向量: mmap'd flat file，固定大小记录 - - Graph: mmap'd flat file - - Payload: RocksDB (仅用于元数据过滤，不用于向量) -- **特点**: 热数据内存，冷数据 mmap，向量绝不经过 KV store - -### Chroma / LanceDB -- **方案**: 基于 Arrow/Lance 列式格式存储向量 -- **特点**: 批量加载高效，但非 HNSW 的主流选择 - -### 关键共识 - -> **没有一个成熟的向量数据库用通用 KV store (LSM-tree 或 B+tree) 存储向量数据。** -> -> 所有高性能实现都选择：连续内存/文件 + O(1) offset 寻址。 - -## 3. 候选方案评估 - -### 方案 A：MemStore + Snapshot（推荐 #1） - -**原理**: 构建和搜索全在内存，后台定期序列化到文件。 - -| 维度 | 评价 | -|------|------| -| 随机点查 | **最优** — 直接 map 查找，~50ns | -| 构建性能 | **最优** — 与当前 MemStore 一致 (59s/50K) | -| 内存占用 | 50K×128d ≈ 25MB 向量 + 图结构 ≈ **40-50MB** | -| 500K×768d ≈ | **~1.5GB 向量 + 图 ≈ 2GB** | -| 持久化 | snapshot 文件，gob/binary 序列化 | -| 启动时间 | 反序列化加载，50K ≈ <1s, 500K ≈ 5-10s | -| 增量更新 | 原生支持（内存 map 操作） | -| 实现复杂度 | **低** — 新增 ~200 行 snapshot 逻辑 | -| 风险 | crash 时丢失最近写入（可接受 WAL 缓解） | - -**内存占用分析**（目标规模）: - -| 规模 | 维度 | 向量内存 | 图+元数据 | 总计 | -|------|------|----------|-----------|------| -| 10K | 128 | 5 MB | 5 MB | ~10 MB | -| 50K | 128 | 25 MB | 15 MB | ~40 MB | -| 100K | 384 | 150 MB | 40 MB | ~190 MB | -| 500K | 768 | 1.5 GB | 200 MB | ~1.7 GB | - -对于代码搜索工具，10K-100K 是最常见规模，内存完全可接受。500K×768d 的 1.7GB 在现代开发机上也合理。 - -### 方案 B：mmap 自定义文件格式（推荐 #2） - -**原理**: 向量连续存于 flat file，mmap 映射，O(1) 偏移寻址。图结构单独文件。 - -``` -vectors.bin: [header][vec_0: 512B][vec_1: 512B]...[vec_N] -graph.bin: [header][node_0: level + neighbors per layer]... -meta.bin: [entry_point][doc_id mappings]... -``` - -| 维度 | 评价 | -|------|------| -| 随机点查 | **极优** — mmap offset, ~100-200ns (page cache 命中) | -| 构建性能 | 接近 MemStore（热数据在 page cache） | -| 内存占用 | OS 管理 page cache，实际驻留按需，低于全内存方案 | -| 持久化 | 天然持久，写即落盘 | -| 启动时间 | **最优** — mmap 映射 <10ms，无需反序列化 | -| 增量更新 | 中等 — 向量可原地更新（固定大小），图结构需要处理变长 neighbor list | -| 实现复杂度 | **中高** — ~500-800 行，需处理文件增长、delete 空洞、graph 变长记录 | -| 风险 | 跨平台 mmap 行为差异；变长 graph 记录需要设计 | - -**graph 变长问题解法**: 按 Mmax0 预分配固定大小 slot（每个 node = 4 + 32×8 + 16×8×maxLayer ≈ 640B），浪费一些空间换取 O(1) 寻址。 - -### 方案 C：bbolt (B+tree) - -| 维度 | 评价 | -|------|------| -| 随机点查 | **中** — B+tree 读优于 LSM，但仍有 page 遍历开销，~1-5μs | -| 构建性能 | 预估 3-5x 慢于 MemStore（好于 Pebble 的 18.5x） | -| 内存占用 | 低 — mmap 读，OS page cache 管理 | -| 持久化 | 原生 | -| 启动时间 | 快 — mmap 打开文件即可 | -| 增量更新 | 原生支持（事务） | -| 实现复杂度 | **低** — 接口适配 ~300 行 | -| 风险 | 写性能差（单 writer 锁）；1.6亿次点查仍有显著开销 | - -**数据**: bbolt 随机读 ~1-5μs/op（page cache 热时），50K 构建预估 3-5 分钟。比 Pebble 好但远逊内存方案。 - -### 方案 D：Badger (LSM, value 分离) - -| 维度 | 评价 | -|------|------| -| 随机点查 | **中** — value log 直接 seek 读，~2-10μs，优于 Pebble 的多级查找 | -| 构建性能 | 预估 5-8x 慢于 MemStore | -| 内存占用 | 低-中 — key 在内存，value 在 vlog | -| 持久化 | 原生 | -| 启动时间 | 中 — 需要加载 key 索引 | -| 增量更新 | 原生支持 | -| 实现复杂度 | **低** — 接口适配 ~300 行 | -| 风险 | GC 带来的写放大；仍是通用 KV 的开销 | - -Badger 的 value 分离设计（WiscKey 论文）对大 value 随机读有帮助，但对 HNSW 的亿级点查仍不够快。 - -### 方案 E：自研 flat file + index - -| 维度 | 评价 | -|------|------| -| 随机点查 | **最优** — 等同方案 B | -| 构建性能 | **最优** | -| 内存占用 | 可控 | -| 持久化 | 自行实现 | -| 启动时间 | 取决于实现 | -| 增量更新 | 需要自行实现所有逻辑 | -| 实现复杂度 | **极高** — 1000+ 行，需实现 crash recovery、compaction、并发控制 | -| 风险 | 大量 edge case；实质上在写一个专用数据库 | - -除非 mmap 方案无法满足需求，否则不建议走这条路。方案 B 的 mmap 已覆盖其核心优势。 - -### 方案 F：保持 Pebble + 大 cache - -| 维度 | 评价 | -|------|------| -| 随机点查 | 取决于 cache 命中率 | -| 构建性能 | cache 全命中 → 接近 MemStore；cache miss → 回退到 21ms/op | -| 内存占用 | cache 50K 向量 ≈ 25MB（可接受），但此时与方案 A 等价 | -| 持久化 | 原生 | -| 启动时间 | 中 — 需要预热 cache | -| 增量更新 | 原生支持 | -| 实现复杂度 | **极低** — 调大 cache size | -| 风险 | 如果 cache 住所有向量，Pebble 只是个昂贵的持久化层 | - -**关键洞察**: 如果 cache 足够大到覆盖所有向量（构建时 100% 命中率所需），那么 Pebble 退化为纯粹的持久化写入层。此时不如直接用方案 A（MemStore + Snapshot），更简单且无 Pebble 的 compaction/WAL 开销。 - -## 4. 对比总结 - -| 方案 | 构建性能 | 随机读延迟 | 内存效率 | 启动速度 | 增量更新 | 实现复杂度 | 综合评分 | -|------|----------|-----------|----------|----------|---------|-----------|---------| -| **A: MemStore+Snapshot** | ★★★★★ | ★★★★★ | ★★★☆☆ | ★★★★☆ | ★★★★★ | ★★★★★ | **#1** | -| **B: mmap flat file** | ★★★★★ | ★★★★☆ | ★★★★★ | ★★★★★ | ★★★☆☆ | ★★★☆☆ | **#2** | -| C: bbolt | ★★★☆☆ | ★★★☆☆ | ★★★★☆ | ★★★★☆ | ★★★★☆ | ★★★★☆ | #4 | -| D: Badger | ★★☆☆☆ | ★★★☆☆ | ★★★★☆ | ★★★☆☆ | ★★★★☆ | ★★★★☆ | #5 | -| E: 自研 flat file | ★★★★★ | ★★★★★ | ★★★★★ | ★★★★★ | ★★★☆☆ | ★☆☆☆☆ | #3 | -| F: Pebble+大cache | ★★★★☆ | ★★★★☆ | ★★★☆☆ | ★★★☆☆ | ★★★★★ | ★★★★★ | #6 | - -## 5. 推荐路径 - -### 阶段 1（立即）：方案 A — MemStore + Snapshot - -**理由**: -1. **最小改动，最大收益** — 18.5x 性能提升，~200 行代码 -2. **已验证** — MemStore 已经是生产质量代码，只需加持久化 -3. **内存可接受** — 目标规模 10K-100K，内存 10-190MB -4. 所有成熟向量数据库的 HNSW 实现本质上都是内存方案 - -**实现计划**: -```go -type SnapshotStore struct { - mem *MemNodeStore - path string // snapshot 文件路径 - dirty atomic.Bool // 有未持久化的变更 - interval time.Duration // snapshot 间隔 -} - -// 启动: 从文件加载 → MemNodeStore -// 运行: 所有读写走 MemNodeStore -// 持久化: 后台 goroutine 定期 binary 序列化到文件 -// 关闭: 最终 snapshot + close -``` - -Snapshot 格式（binary，非 gob —— 更快更紧凑）: -``` -[magic: 4B][version: 4B][entryID: 8B][maxLayer: 4B][nodeCount: 4B] -[vectors section: id + dim + float32 data...] -[levels section: id + level...] -[norms section: id + float32...] -[neighbors section: id + layer + count + neighbor_ids...] -[mappings section: docId + nodeId...] -[checksum: 32B SHA-256] -``` - -### 阶段 2（如果需要）：方案 B — mmap 迁移 - -仅在以下情况考虑： -- 500K+ 向量场景内存不足 -- 启动时间要求 <100ms -- 需要多进程共享索引 - -## 6. 快速 ROI 分析 - -| | 现状 (Pebble) | 方案 A (MemStore+Snapshot) | 提升 | -|--|--------------|--------------------------|------| -| 50K 构建 | 18 min | ~59s | **18x** | -| 100K 构建 | ~40 min (预估) | ~3 min | **13x** | -| 搜索延迟 | ~2ms (cache miss 更高) | ~0.1ms | **20x** | -| 启动时间 | ~1s (打开 DB) | ~1-2s (加载 snapshot) | 持平 | -| 内存 (50K) | 88MB + Pebble overhead | ~40MB | **更低** | -| 磁盘 (50K) | 36MB (Pebble SST) | ~30MB (binary snapshot) | 持平 | -| 代码变更 | 0 | ~200-300 行 | 极低 | - -## 7. 结论 - -**用通用 KV store 存储 HNSW 向量是一个架构错配。** 业界共识明确：HNSW 需要 O(1) 随机访问，应该用内存或 mmap flat file，不应用 LSM-tree 或 B+tree。 - -对于 Haystack 的规模（10K-500K 向量），**MemStore + Snapshot 是性价比最高的方案**：实现简单、性能最优、已有成熟代码基础。mmap 方案作为未来演进路径保留。 - -不建议在 KV store 选型上花更多时间（bbolt/Badger/Pebble 调优），因为这是在错误的方向上优化。 diff --git a/docs/research/storage-backend-research-3.md b/docs/research/storage-backend-research-3.md deleted file mode 100644 index fc8499b6..00000000 --- a/docs/research/storage-backend-research-3.md +++ /dev/null @@ -1,182 +0,0 @@ -# Storage Backend Research for HNSW Vector Index (HAY-007) - -> Date: 2026-04-17 -> Context: Haystack HNSW index currently uses Pebble; random point-read performance is the bottleneck (read:write = 86:1, ~3500 reads/insert, 1.6 亿 total reads for 50K build) - -## Workload Profile - -| Metric | Value | -|---|---| -| Dataset | 50K vectors, 128-dim | -| Value size | 512 B (128-dim) – 3072 B (768-dim) | -| Reads/insert | ~3,500 (neighbor traversal) | -| Writes/insert | ~37 | -| Total reads (50K build) | ~160,000,000 | -| Target read latency | < 1 μs (memory), < 10 μs (disk+cache) | -| Total dataset size | 50K × 3 KB ≈ 150 MB | - -## Candidate Comparison - -| | Pebble | bbolt | Badger v4 | BuntDB | NutsDB | Custom mmap | -|---|---|---|---|---|---|---| -| **Engine** | LSM-tree | B+ tree (mmap) | LSM + value log | In-memory + AOF | Bitcask variant | Flat file + mmap | -| **GitHub stars** | ~5.8K | ~9.5K | ~15.6K | ~4.8K | ~3.6K | N/A | -| **Last release** | Active (CockroachDB core) | v1.4.x (2025) | v4.9.1 (2026-02) | Maintenance mode | v1.1.0 (2025-12) | N/A | -| **Pure Go (zero CGo)** | Yes (default build) | Yes | Yes | Yes | Yes | Yes | -| **Random read latency** | 1–5 μs (warm cache), 10–50 μs (cold) | 0.5–2 μs (mmap warm) | 2–10 μs (value indirection) | < 0.5 μs (all in RAM) | 1–3 μs (key index in RAM) | 0.1–0.5 μs (direct pointer) | -| **Write latency** | ~2–5 μs (WAL batch) | 50–200 μs (COW + fsync) | ~3–8 μs (WAL) | ~1–3 μs (append AOF) | ~2–5 μs | ~1 μs (msync deferred) | -| **Memory efficiency** | Good (block cache tunable) | Good (OS page cache) | Poor (high base overhead) | Poor (all data in RAM) | OK (key index in RAM) | Excellent (OS page cache) | -| **Crash safety** | WAL + manifest | COW B+ tree (excellent) | WAL + value log | AOF replay | Bitcask merge | Manual (msync) | -| **API complexity** | High (many options) | Simple | Medium | Very simple | Simple | DIY | -| **Dep tree weight** | Heavy (CockroachDB) | Light | Medium | Minimal | Light | Zero | - -## Per-Library Analysis - -### 1. Pebble (current) — github.com/cockroachdb/pebble - -**Architecture:** LSM-tree with WAL, bloom filters, block cache. - -**Why it underperforms for HNSW:** -- Random point reads must check bloom filter → index block → data block per SST level -- 50K keys likely spans 1–3 SST levels; each read touches 2–4 blocks -- Block cache helps, but HNSW traversal has poor locality (random neighbor hops) -- Compaction background work competes for I/O - -**When it's still OK:** -- If the entire dataset fits in block cache (~150 MB, easily tunable), reads hit cached blocks -- Already integrated — switching has engineering cost - -**Verdict:** Solid but architecturally mismatched for random-read-dominant workloads. - -### 2. bbolt — go.etcd.io/bbolt - -**Architecture:** Single-file B+ tree, mmap'd pages, COW writes. - -**Why it's better for reads:** -- B+ tree lookup = O(log_B N) page accesses; for 50K keys with 4 KB pages, ~3–4 page touches -- Entire file is mmap'd — reads go through OS page cache, no userspace buffering layer -- No bloom filter overhead, no SST level multiplier -- 150 MB file easily stays in page cache - -**Write concern:** -- COW semantics: every write copies touched pages → new root -- Must `fsync` on commit → 50–200 μs per write tx -- Mitigated by batching: accumulate neighbor updates in one write tx per insert - -**Concurrency model:** -- Single writer, multiple concurrent readers (MVCC via COW) -- Readers never block writers (and vice versa) — good for HNSW where reads dominate - -**Verdict:** Strong candidate. Natural fit for read-heavy + batched-write workload. - -### 3. Badger v4 — github.com/dgraph-io/badger - -**Architecture:** WiscKey — keys in LSM, values in separate value log. - -**Why it's NOT ideal here:** -- Values 512–3072 B sit at the awkward threshold boundary - - Below `ValueThreshold` → stored inline in LSM (same as Pebble, no benefit) - - Above → requires two seeks: LSM for key → value log for data -- Value log GC adds complexity and unpredictable latency spikes -- Higher base memory usage than Pebble or bbolt -- Known production stability issues in community reports - -**When value separation helps:** -- Very large values (>16 KB) where keeping them out of LSM compaction saves I/O -- Not this workload. - -**Verdict:** Not recommended. Adds complexity without read performance benefit at this value size. - -### 4. BuntDB — github.com/tidwall/buntdb - -**Architecture:** In-memory B-tree with optional AOF persistence. - -**Reads:** All data in RAM → sub-microsecond. Benchmarks report ~4.6M reads/sec. - -**Concerns:** -- **Maintenance:** 172 commits total, low recent activity, essentially one-person project -- **Persistence:** AOF append + periodic shrink. Not crash-safe in the same way as WAL-based stores -- **Memory:** Entire dataset in Go heap → GC pressure on larger datasets -- **Scale ceiling:** Fine at 150 MB, questionable at 1 GB+ - -**Verdict:** Fastest reads, but maintenance risk and GC pressure concern. Consider only if bbolt doesn't meet latency targets. - -### 5. NutsDB — github.com/nutsdb/nutsdb - -**Architecture:** Bitcask-inspired — all keys indexed in memory, values on disk. - -**Reads:** Key lookup in RAM hash map → one disk read at known offset. Fast (~1–3 μs warm). - -**Concerns:** -- Less battle-tested than bbolt/Pebble -- Compaction (merge) can cause latency spikes -- Smaller community, fewer production deployments - -**Verdict:** OK option but no advantage over bbolt for this workload. - -### 6. Custom mmap Flat File - -**Design:** -``` -vectors.dat: [vec_0 (512B padded)] [vec_1] ... [vec_49999] -graph.dat: [neighbors_0 (var)] [neighbors_1] ... -index: map[uint64]offset (in-memory or separate file) -``` - -Read path: `id → offset → mmap slice → done`. One pointer dereference + possible page fault. - -**Advantages:** -- Absolute minimum overhead: no serialization, no key comparison, no tree traversal -- 150 MB file trivially fits in OS page cache -- Zero library dependencies, zero background goroutines -- Predictable, no GC interaction (mmap is outside Go heap) - -**Disadvantages:** -- Must handle: variable-length records, crash safety, file growth, deletion/compaction -- No transactions, no key iteration, no range queries -- Testing and correctness burden is on you - -**Implementation options:** -- `syscall.Mmap` / `golang.org/x/sys/unix` — direct, low-level -- `github.com/edsrzf/mmap-go` (~4K stars) — thin cross-platform wrapper - -**Verdict:** Maximum performance, maximum engineering cost. Best when the KV store is proven to be the bottleneck. - -### 7. Others Considered - -| Library | Notes | Verdict | -|---|---|---| -| **RoseDB** (github.com/rosedblabs/rosedb, 5K stars) | Bitcask model, pure Go, clean API | Similar to NutsDB, no clear advantage | -| **LotusDB** (github.com/lotusdblabs/lotusdb, 2K stars) | B+ tree index + value log | Immature, small community | -| **go-memdb** (github.com/hashicorp/go-memdb, 3.5K stars) | In-memory immutable radix tree | No persistence — not suitable | -| **Bolt** (github.com/boltdb/bolt) | Original, archived | Use bbolt instead | - -## Recommendation Ranking - -### Tier 1: Recommended - -| Rank | Library | Rationale | -|---|---|---| -| **1** | **bbolt** | Best balance of read performance, simplicity, and reliability. Mmap B+ tree is architecturally matched to random-read workloads. Batch writes to amortize COW cost. Battle-tested (etcd core). | -| **2** | **Custom mmap** | Maximum performance ceiling. Choose this if bbolt benchmarks show B+ tree overhead is still too high, or if you want zero-dependency storage. | - -### Tier 2: Acceptable - -| Rank | Library | Rationale | -|---|---|---| -| **3** | **Pebble (tuned)** | Already integrated. Before switching, try: increase block cache to 256 MB, use bloom filters, measure actual hit rate. If cache-warm reads are <2 μs, the switch may not be worth the engineering cost. | -| **4** | **BuntDB** | Fastest reads (in-memory). Acceptable if dataset stays small and maintenance risk is tolerable. | - -### Tier 3: Not Recommended - -| Rank | Library | Rationale | -|---|---|---| -| **5** | NutsDB | No advantage over bbolt | -| **6** | Badger | Wrong architecture for this value size and access pattern | - -## Suggested Next Steps - -1. **Benchmark Pebble with tuned cache** — set block cache = 256 MB, measure p50/p99 read latency for HNSW traversal. This is the cheapest experiment. -2. **Prototype bbolt backend** — implement `Store` interface with bbolt, batch writes per insert (one write tx with all ~37 neighbor updates). -3. **Compare** — run 50K build benchmark with both backends. If bbolt is 2×+ faster, switch. -4. **Consider mmap later** — only if bbolt still doesn't meet targets and profiling confirms storage is the bottleneck (not HNSW algorithm overhead). diff --git a/docs/review/HAY-007-p1-fixes-review.md b/docs/review/HAY-007-p1-fixes-review.md deleted file mode 100644 index 8956a0a1..00000000 --- a/docs/review/HAY-007-p1-fixes-review.md +++ /dev/null @@ -1,118 +0,0 @@ -# HAY-007 P1 Fixes Review — PR #54 - -**Reviewer:** Claude (Opus 4) -**Date:** 2026-04-17 -**Commits reviewed:** 4ffb4aa..4fd88d0 (5 commits) - ---- - -## P1 Fix Verification - -### P1-1: Lock Ordering Documentation — ✅ FIXED - -**Commit:** 4ffb4aa - -Lock ordering comment `muGraph → muNodes → muVec` added to the `MmapStore` struct -doc-comment (`mmap_store.go:20-27`). Additionally, `getNeighborsUpper` now has an -inline comment (`mmap_store_read.go:65`) clarifying that `muNodes` is acquired while -`muGraph` is held by the caller, consistent with the documented order. - -**Verdict:** Clear, correct, sufficient. - ---- - -### P1-2: Offset int64 — ✅ FIXED - -**Commit:** e95e0ed - -All offset calculations in `mmap_store_read.go` converted from `int` to `int64`: -- `GetVector` (line 18-19) -- `getNeighborsL0` (line 49, loop var line 57) -- `getNeighborsUpper` (lines 86-87, loop var line 96) -- `readUpperSlot` (line 108) -- `GetNorm` (line 121) -- `GetNodeLevel` (line 135) - -All use the pattern `int64(pageSize) + int64(id)*int64(slotSize)` which prevents -overflow when `id * slotSize` exceeds 2³¹ on 32-bit or when intermediate `int` -multiplication wraps on platforms where `int` is 32-bit. - -**Verdict:** Thorough, all read-path offsets covered. - ---- - -### P1-3: Benchmark — ✅ FIXED - -**Commit:** 4fd88d0 - -`BenchmarkMmapStoreGetVector` added in `benchmark_test.go`. Creates 1000 vectors -(dim=128), exports to mmap, benchmarks `GetVector` in a tight loop. Target documented -as < 1μs per call. - -**Note:** No `BenchmarkMmapStoreGetNeighbors` was added, but the original P1 only -required `GetVector`. Adequate. - -**Verdict:** Meets requirement. - ---- - -### P1-4: mmapAll Error Cleanup — ✅ FIXED - -**Commit:** aa4787b - -`mmapAll()` now tracks `openedFiles` and `mappedRegions` in local slices. A `cleanup()` -closure calls `mmapFree` on all mapped regions and `Close` on all opened files. Every -error return in the loop calls `cleanup()` before returning. - -**Minor observation:** On error, struct fields (`s.vectors`, `s.l0File`, etc.) that -were already assigned still point to freed/closed resources. This is safe because -the caller (`OpenMmapStore`) returns the error and discards the struct. However, -nil-ing the struct fields inside `cleanup()` would be strictly more defensive. This -is cosmetic, not blocking. - -**Verdict:** Correct. Resource leak on partial failure is eliminated. - ---- - -### P1-5: docToNode RWMutex — ✅ FIXED - -**Commit:** e04b62d - -- New field `muDoc sync.RWMutex` added to `MmapStore` (`mmap_store.go:48`). -- `GetNodeId` now wraps the map read with `s.muDoc.RLock()` / `s.muDoc.RUnlock()` - (`mmap_store_read.go:154-156`). - -**Note:** Currently `docToNode` is only written during single-threaded initialization -(constructor) and in tests, so the write side doesn't need locking yet. The read lock -is still correct — it future-proofs the map for concurrent writes without imposing -measurable overhead (RLock is uncontended). If a write path is added later, it must -acquire `muDoc.Lock()`. - -**Verdict:** Correct and forward-looking. - ---- - -## Standard Checklist - -| # | Check | Status | -|---|-------|--------| -| 1 | **Test coverage** | ✅ Benchmark added (P1-3). Existing read tests cover the int64 offset paths. | -| 2 | **Error handling** | ✅ mmapAll cleanup is correct (P1-4). | -| 3 | **Code quality** | ✅ Changes are minimal and focused. Each commit addresses exactly one P1. | -| 4 | **New issues introduced?** | ⚠️ See observations below. | - -### Observations (non-blocking) - -1. **Stale struct fields on mmapAll error** — As noted in P1-4, struct fields point - to freed resources after cleanup. Not exploitable since the struct is discarded, - but nil-ing them would be safer. **Severity: informational.** - -2. **`muDoc` not in lock ordering comment** — The doc says `muGraph → muNodes → muVec` - but `muDoc` is independent (never held with others). Consider adding a note like - "muDoc is independent and may be acquired at any time." **Severity: informational.** - ---- - -## Conclusion - -**APPROVE** — All 5 P1 issues are correctly fixed. No new bugs or regressions introduced. Two informational observations noted for future consideration. diff --git a/docs/review/HAY-007-phase1-full-review.md b/docs/review/HAY-007-phase1-full-review.md deleted file mode 100644 index 33384a6f..00000000 --- a/docs/review/HAY-007-phase1-full-review.md +++ /dev/null @@ -1,172 +0,0 @@ -# HAY-007 Phase 1 Code Review — Full Report - -> Reviewer: Claude (strict review) -> Date: 2026-04-17 -> Branch: `feat/hay007-mmap-store` -> PR: #53 - ---- - -## Spec 对照表 - -| # | Spec 要求 (Phase 1) | 实现状态 | 备注 | -|---|---------------------|---------|------| -| 1 | vectors.dat / nodes.dat / graph_l0.dat mmap 读取 | ✅ 已实现 | `mmap_store.go:mmapAll()`, graph_upper.dat 也已实现（超出 Phase 1 scope，但无害） | -| 2 | 自封装 syscall.Mmap（Linux/macOS/Windows） | ✅ 已实现 | `mmap_unix.go` + `mmap_windows.go`，零第三方依赖 | -| 3 | GetVector / GetVectorRef | ✅ 已实现 | `mmap_store_read.go:10-31`，GetVectorRef 返回 copy（符合 spec） | -| 4 | GetNeighbors (L0 + upper) | ✅ 已实现 | `mmap_store_read.go:34-99` | -| 5 | GetNorm | ✅ 已实现 | `mmap_store_read.go:113-124` | -| 6 | 从 PebbleStore/MemStore 导出数据到 mmap 格式 | ✅ 已实现 | `mmap_export_test.go:exportMemStoreToMmap`（测试 helper） | -| 7 | Header 4096 page-aligned | ✅ 已实现 | `pageSize = 4096`，所有数据偏移从 `pageSize + id*slotSize` 开始 | -| 8 | MetaHeader 64 bytes 编译期检查 | ✅ 已实现 | `mmap_format.go:42` 编译期断言 | -| 9 | meta.bin 原子写（tmp+fsync+rename） | ✅ 已实现 | `mmap_format.go:110-142` | -| 10 | little-endian 字节序 | ✅ 已实现 | 全局使用 `binary.LittleEndian` | -| 11 | 零 CGo | ✅ 已实现 | 无 `import "C"`，纯 `syscall` 调用 | -| 12 | 50K 随机向量读 benchmark < 1μs | ⚠️ 未验证 | 无 benchmark 测试文件（_bench_test.go）| - ---- - -## 问题列表 - -### P0 — Must Fix - -**无 P0 问题。** 代码在 Phase 1 只读路径的 scope 内实现正确，无数据损坏或安全风险。 - ---- - -### P1 — Should Fix - -#### P1-1: `getNeighborsUpper` 在持有 `muGraph.RLock` 的同时再获取 `muNodes.RLock` — 存在锁顺序隐患 - -**文件**: `mmap_store_read.go:63-68` -**描述**: `GetNeighbors` 调用时先持有 `muGraph.RLock()`（第 35 行 defer），然后在 `getNeighborsUpper` 中又获取 `muNodes.RLock()`。Phase 2 写路径如果以相反顺序获取锁（先 muNodes 再 muGraph），将导致死锁。 -**建议**: 在 `getNeighborsUpper` 入口先读取 `upperSlot`（需要 muNodes），释放后再读 upper 数据（需要 muGraph），或者文档化锁获取顺序（muGraph → muNodes）并在所有写路径中遵守。 - -#### P1-2: `int(id) * s.vecSlotSize` 等 offset 计算在 32 位平台有溢出风险 - -**文件**: `mmap_store_read.go:18`, `mmap_store_read.go:49`, `mmap_store_read.go:107` -**描述**: `int(id)` 在 32 位平台上 int 为 32 位。当 `id` 接近 500K 且 `vecSlotSize` 为 3072 (768d) 时，`int(id) * vecSlotSize` = ~1.5GB，接近 int32 上限（2GB）。更大规模会溢出。 -**建议**: 使用 `int64` 进行 offset 计算：`offset := int64(pageSize) + int64(id)*int64(s.vecSlotSize)`，或在文件头注释中标注仅支持 64 位平台。 - -#### P1-3: 缺少 Benchmark 测试 - -**文件**: 无 -**描述**: Phase 1 验收标准要求 "50K 随机向量读 benchmark < 1μs"，但没有 `Benchmark*` 函数来验证。 -**建议**: 添加 `BenchmarkMmapStoreGetVector` 和 `BenchmarkMmapStoreGetNeighbors` 函数。 - -#### P1-4: `mmapAll` 错误路径未清理已打开的文件和 mmap - -**文件**: `mmap_store.go:189-232` -**描述**: 如果 `mmapAll` 在处理第 3 个文件时失败，前 2 个文件的 fd 和 mmap 映射不会被关闭/释放。 -**建议**: 在错误路径中调用 `mmapFree` 和 `f.Close()` 清理已成功映射的资源，或在 `OpenMmapStore` 的错误路径中调用 `s.Close()`。 - -#### P1-5: `GetNodeId` 缺少并发保护 - -**文件**: `mmap_store_read.go:152-155` -**描述**: `docToNode` map 读取没有任何锁保护。Phase 2 写路径会并发修改这个 map（SetNodeMapping），导致 race condition。 -**建议**: 虽然 Phase 1 只有读路径可能暂时安全，但建议现在就加 `sync.RWMutex` 保护或在 struct 中用 `sync.Map`，避免 Phase 2 遗忘。 - ---- - -### P2 — Nit / Nice to Have - -#### P2-1: `NodesHeader` 的 capacity 偏移硬编码 - -**文件**: `mmap_store.go:227` -**描述**: `s.nodeCapacity = binary.LittleEndian.Uint64(s.nodes[8:16])` 依赖 `NodesHeader` 结构体布局中 padding 的位置。如果 `NodesHeader` 结构体变化，这个偏移会悄悄出错。 -**建议**: 添加注释或使用 `unsafe.Offsetof` 编译期断言。 - -#### P2-2: `writeDataFileHeader` 的 `magic` 参数未使用 - -**文件**: `mmap_format.go:165` -**描述**: 函数签名接受 `magic [4]byte` 参数但从未使用——magic 写入依赖 `headerData` 结构体中的 Magic 字段。 -**建议**: 移除未使用的 `magic` 参数。 - -#### P2-3: `GraphUpperHeader` 缺少 `NextSlot` 从 mmap 的读取 - -**文件**: `mmap_store.go:229` -**描述**: `mmapAll` 只读取了 `Capacity` 字段，但 `GraphUpperHeader.NextSlot` 也应在打开时恢复（Phase 2 写路径需要）。 -**建议**: Phase 2 开始前补充读取 `NextSlot`。 - -#### P2-4: `Close()` 中 `writeMetaHeader` 在 munmap 之后 - -**文件**: `mmap_store.go:112-137` -**描述**: `Close` 先 munmap 所有数据，然后才写 meta。如果 munmap 成功但 writeMetaHeader 失败，meta 未持久化。逻辑上应先写 meta 再 munmap。 -**建议**: 将 `writeMetaHeader` 移到 munmap 之前。 - -#### P2-5: Windows `munmapPlatform` 中 `FILE_MAP_WRITE` 不含 `FILE_MAP_READ` - -**文件**: `mmap_windows.go:17` -**描述**: 当 `flags & mmapWrite != 0` 时，`dwAccess` 设为 `FILE_MAP_WRITE`，但 Windows 的 `FILE_MAP_WRITE` 不隐含读权限。如果代码同时需要读写，应使用 `FILE_MAP_WRITE | FILE_MAP_READ`。 -**建议**: 改为 `dwAccess = syscall.FILE_MAP_WRITE | syscall.FILE_MAP_READ`。 - -#### P2-6: 测试中 `TestMmapStoreInitSmallCap` 未真正测试 `upperCap < 64` 分支 - -**文件**: `mmap_store_test.go:106-123` -**描述**: 注释承认无法测试这个分支（因为 `initAllFiles` 未导出），实际只验证了 `upperCapacity >= 64`。 -**建议**: 导出一个 `initWithCapacity` 测试 helper 或通过 `_test.go` 包内访问直接测试小 cap。 - ---- - -## mmap 特定检查项 - -| 检查项 | 状态 | 备注 | -|--------|------|------| -| Header 4096 page-aligned | ✅ | `pageSize = 4096`，所有文件 header 占满 4096 bytes | -| GetVectorRef 返回 copy | ✅ | 调用 GetVector，make+copy | -| 自封装 syscall（非第三方） | ✅ | `syscall.Mmap` / `CreateFileMapping+MapViewOfFile` | -| 零 CGo | ✅ | 无 `import "C"` | -| little-endian | ✅ | 全部使用 `binary.LittleEndian` | -| MetaHeader 编译期大小检查 | ✅ | `var _ [64]byte = [unsafe.Sizeof(MetaHeader{})]byte{}` | -| Windows mmap 实现 | ✅ | `CreateFileMapping` + `MapViewOfFile` + `FlushViewOfFile`，注意 P2-5 | -| 独立锁（muVec/muGraph/muNodes） | ✅ | 三把 `sync.RWMutex` | -| bounds check (id < capacity) | ✅ | 所有读方法入口均检查 | - ---- - -## 算法/数值检查 - -| 检查项 | 状态 | 备注 | -|--------|------|------| -| offset 溢出 | ⚠️ P1-2 | 32 位平台 `int` 乘法可能溢出 | -| 锁顺序一致性 | ⚠️ P1-1 | `muGraph → muNodes` 嵌套，需文档化 | - ---- - -## 测试覆盖评估 - -| 路径 | 覆盖 | 备注 | -|------|------|------| -| 正常读 (vector/neighbors/norm/level) | ✅ | 多维度测试 | -| 边界: out-of-range ID | ✅ | 所有读方法均有边界测试 | -| 边界: deleted node | ✅ | `TestMmapStoreGetNodeLevelDeleted` | -| 边界: no entry point | ✅ | `TestMmapStoreGetEntryPoint` | -| 边界: upper slot = 0 (no alloc) | ✅ | `TestMmapStoreGetNeighborsUpperSlotZero` | -| 边界: layer > maxLayers | ✅ | `TestMmapStoreGetNeighborsUpperBadLayer` | -| 边界: upper slot out of range | ✅ | `TestMmapStoreGetNeighborsUpperSlotOutOfRange` | -| 集成: 手工构造文件 → OpenMmapStore → 全路径验证 | ✅ | `TestMmapStoreIntegration` | -| 集成: MemStore → export → MmapStore 1000 节点对比 | ✅ | `TestExportMemStoreToMmap` | -| 集成: Close → Reopen 持久化 | ✅ | `TestExportMemStoreToMmap` 尾部 | -| meta.bin: 原子写/读/bad magic/truncated | ✅ | `mmap_format_test.go` | -| mmap: alloc/free/zero-length/empty-free | ✅ | `mmap_test.go` | -| 参数校验: dim=0, M=0, mismatch | ✅ | `mmap_store_test.go` | -| **缺失: Benchmark** | ❌ | 验收标准要求 | - ---- - -## 结论 - -### **APPROVE (with non-blocking suggestions)** - -Phase 1 只读路径实现正确、完整，与设计文档一致。代码质量高：命名清晰、函数粒度合适、错误消息有意义、无安全问题。测试覆盖全面（正常/边界/错误/集成）。 - -**Must-address before merge (P1)**: -1. **P1-1**: 文档化锁顺序（muGraph → muNodes）或重构以避免嵌套锁——不修复会在 Phase 2 引入死锁风险 -2. **P1-3**: 添加 benchmark 测试以满足验收标准 - -**Recommended for Phase 2 准备**: -3. P1-2: offset 计算改用 int64（或标注 64-bit only） -4. P1-4: mmapAll 错误路径资源清理 -5. P1-5: docToNode map 加锁保护 - -以上 P1 项建议在 Phase 2 开始前修复。P2 项可后续处理。 diff --git a/docs/review/HAY-007-phase2-full-review-r2.md b/docs/review/HAY-007-phase2-full-review-r2.md deleted file mode 100644 index b263dc6a..00000000 --- a/docs/review/HAY-007-phase2-full-review-r2.md +++ /dev/null @@ -1,164 +0,0 @@ -# HAY-007 Phase 2 Code Review — Round 2 (Post P1 Fix) - -> Reviewer: 飞马（Pegasus） -> Date: 2026-04-18 -> PR: #55 -> Scope: Phase 2 写入路径 + WAL + Batch + Grow + Delete(tombstone) - ---- - -## Spec 对照表 - -| Spec 要求 | 实现状态 | 备注 | -|-----------|---------|------| -| PutNode | ✅ 已实现 | WAL → ensure capacity → write vec/node/upper | -| SetNeighbors | ✅ 已实现 | L0 + Upper 两条路径 | -| SetNorm | ✅ 已实现 | WAL + nodes.dat 写入 | -| SetNodeMapping | ✅ 已实现 | 内存 map + idmap.dat 追加（带 CRC） | -| WAL with LSN | ✅ 已实现 | append + CRC32 + LSN 单调递增 | -| WAL replay | ✅ 已实现 | 5 种 record type 全覆盖 | -| WAL CRC32 | ✅ 已实现 | 覆盖 LSN+Length+Type+Payload | -| BatchableStore | ✅ 已实现 | BeginBatch / CommitBatch / DiscardBatch | -| 文件增长 grow | ✅ 已实现 | munmap→truncate→re-mmap，独立锁 | -| DeleteNode tombstone | ✅ 已实现 | flags |= nodeFlagDeleted | -| NextNodeId | ✅ 已实现 | auto-increment，freelist TODO Phase 3 | -| rebuildNodeCount | ✅ 已实现（R1 P1 fix） | WAL replay 后扫描 nodes.dat | -| WAL payload length cap | ✅ 已实现（R1 P1 fix） | 64 MiB maxWalPayloadSize | -| 50K Insert < 90s | ✅ 0.38s | 远超目标 | -| WAL replay 正确 | ✅ | 测试覆盖 | - ---- - -## Findings - -### P1 — Must Fix - -#### P1-1: `scanLSN()` 缺少 `maxWalPayloadSize` 检查 - -**文件**: `mmap_wal.go:66-97` (scanLSN) - -`Replay()` 正确检查了 `length > maxWalPayloadSize`，但 `scanLSN()` 没有。如果 WAL 文件损坏导致 length 字段是一个巨大值，`scanLSN` 会 `make([]byte, length)` 导致 OOM。 - -```go -// scanLSN line ~82 -length := binary.LittleEndian.Uint32(header[8:12]) -// 缺少: if length > maxWalPayloadSize { break } -payload := make([]byte, length) // ← OOM if corrupted -``` - -**建议**: 在 `scanLSN` 中加入与 `Replay` 相同的 `maxWalPayloadSize` 检查。 - -#### P1-2: `PutNode` 用 `muVec.RLock()` 做写入 - -**文件**: `mmap_store_write.go:40-48` - -`PutNode` 写入 vectors.dat 时使用 `muVec.RLock()`，写入 nodes.dat 时使用 `muNodes.RLock()`。虽然 HNSW 的 `h.mu` 保证了写操作串行化，但这意味着写入期间 `growVectors` 可以获取 `muVec.Lock()` 而此时另一个 goroutine 正在写同一个 mmap region。 - -更根本的问题是：`RLock` 语义上表达的是"共享读"，但这里实际在做写入。这会误导未来维护者。且如果 grow 发生在写入中间（munmap 旧映射 → 写入到已 munmap 的内存 → SIGSEGV），虽然当前 `h.mu` 序列化了 Insert，但 `PutNode` 本身没有防御。 - -**建议**: 至少添加注释说明为什么 RLock 足够（依赖 h.mu 外部序列化），或改为用 Lock() 在写路径上。 - -#### P1-3: WAL replay INSERT 的 `NodeCount++` 不幂等 - -**文件**: `mmap_store.go:431` (replayWAL, WalInsert case) - -Replay 每遇到一条 INSERT record 就 `s.meta.NodeCount++`。如果 WAL replay 被执行两次（例如 replay 后写 meta 失败，再次重启），NodeCount 会被重复计数。 - -虽然 `rebuildNodeCount()` 在 replay 后修正了最终值（R1 P1-1 fix），**当前代码是安全的**。但 replay 中的 `NodeCount++` 是死代码——它的值总是被 `rebuildNodeCount` 覆盖。建议删除 replay 中的 `s.meta.NodeCount++` 和 `s.meta.NodeCount--` 以避免混淆。 - -降级为 P2（代码清洁度），因为 `rebuildNodeCount` 保证了正确性。 - -### P2 — Should Fix - -#### P2-1: `growUpper` 和 `growL0` 共用 `muGraph` 锁 - -**文件**: `mmap_store_grow.go:93,113` - -`growL0` 和 `growUpper` 都持有 `muGraph.Lock()`。如果 L0 需要 grow 而 upper 正在被读取，整个 graph 读取会被阻塞。Spec 说"独立锁"，但实际 L0 和 upper 共享同一把锁。 - -目前可接受（grow 频率极低），但与 spec 的"独立锁"描述不完全一致。 - -**建议**: 文档化这个决策，或未来拆分为 `muL0` 和 `muUpper`。 - -#### P2-2: `CommitBatch` 的 `sync` 参数被忽略 - -**文件**: `mmap_store_write.go:89-90` - -```go -func (s *MmapStore) CommitBatch(sync bool) error { - ... - sync = true // 参数被覆盖 -``` - -`sync` 参数声称"kept for interface compatibility"但直接被覆盖为 `true`。要么删除参数改为 `CommitBatch() error`，要么尊重参数。当前行为正确（always sync），但 API 有误导性。 - -#### P2-3: `idmapFile` 写入没有 fsync - -**文件**: `mmap_store_write.go:119-121` - -`SetNodeMapping` 追加写 `idmap.dat` 后没有 fsync（非 batch 模式下也没有）。WAL 已经记录了 INSERT（含 docId），所以 idmap 丢失可以从 WAL 恢复。但 `replayWAL` 中 INSERT case 没有调用 `SetNodeMapping`，意味着 replay 后 docToNode 映射不会被恢复。 - -**分析**: `loadIdmap` 在 `replayWAL` 之前执行，所以 idmap.dat 中已有的映射会被加载。但如果 crash 发生在 WAL INSERT 写入后、idmap.dat 写入前，该映射会丢失。WAL replay 没有恢复 id mapping 的逻辑。 - -**建议**: 在 `replayWAL` 的 INSERT case 中恢复 docToNode/nodeToDoc 映射。 - -#### P2-4: `replayWAL` DELETE case 没有清理 id mapping - -**文件**: `mmap_store.go:473-479` - -Replay DELETE 只设置 tombstone flag 和减 NodeCount，没有从 docToNode/nodeToDoc 中删除映射。如果 crash 发生在 DeleteNode WAL 之后、内存映射删除之前，replay 后会有 stale mapping 指向 tombstoned node。 - -#### P2-5: 测试缺少 WAL replay 完整性测试 - -没有测试验证：写入 N 条数据 → Close → Reopen → 数据完整（端到端 crash recovery 测试）。`TestMmapStoreNodeMappingPersistence` 和 `TestMmapStoreNextNodeIdPersistence` 覆盖了部分，但缺少 PutNode + SetNeighbors 的完整 reopen 验证。 - -### P3 — Nice to Have - -#### P3-1: `EncodeInsert` 和 `DecodeInsert` 没有边界检查 - -Decode 函数假设 payload 格式正确。如果 payload 被截断（虽然 CRC 应该已经拦截），会 panic。防御性编程建议加 bounds check。 - -#### P3-2: `mmapSync` 每次 sync 整个 region - -非 batch 模式下，每次 PutNode/SetNeighbors/SetNorm 都 msync 整个 mmap region（可能 24MB+）。Linux 的 `MS_SYNC` 会 flush 所有 dirty pages，不仅仅是刚写的那一页。在 batch 模式下无影响（延迟到 CommitBatch），但非 batch 单条写入会很慢。 - ---- - -## 第 1 轮 P1 修复验证 - -| R1 Finding | 修复状态 | 验证 | -|------------|---------|------| -| P1-1: rebuildNodeCount | ✅ 已修复 | `rebuildNodeCount()` 在 replayWAL 后调用，扫描 nodes.dat 重建 | -| P1-5: WAL payload length cap | ✅ 已修复 | `maxWalPayloadSize = 64 MiB`，Replay 中检查 | - ---- - -## Checklist 总结 - -| 检查项 | 结论 | -|--------|------| -| 符合设计 | ✅ Phase 2 所有功能已实现 | -| 测试覆盖 | ⚠️ 缺少 PutNode reopen 端到端测试 | -| 错误处理 | ✅ 错误信息清晰，逐层 wrap | -| 代码质量 | ⚠️ replay 中有死代码（NodeCount++），sync 参数被忽略 | -| 性能 | ✅ 50K Insert 0.38s 远超目标 | -| 安全 | ⚠️ scanLSN 缺少 payload size cap（P1-1） | -| 文档 | ✅ Spec 一致 | -| 锁顺序 | ✅ muGraph→muNodes 在 PutNode 中遵守 | -| WAL replay 幂等 | ⚠️ 形式上不幂等（NodeCount++），但 rebuildNodeCount 兜底 | -| grow 安全 | ✅ munmap→truncate→re-mmap，double-check under lock | -| 并发安全 | ⚠️ PutNode 用 RLock 写入（依赖外部 h.mu，需文档化） | - ---- - -## 结论 - -**REQUEST CHANGES** — 1 个 P1 必须修复： - -1. **P1-1**: `scanLSN` 缺少 `maxWalPayloadSize` 检查 → OOM 风险 - -P1-2 和 P1-3 降级为 P2（P1-2 因有 h.mu 保护实际安全，P1-3 因 rebuildNodeCount 兜底正确）。 - -P2-3（replay 不恢复 id mapping）值得在 Phase 3 之前修复，但不阻塞合并。 - -修复 P1-1 后可 APPROVE。 diff --git a/docs/review/HAY-007-phase2-full-review-r3.md b/docs/review/HAY-007-phase2-full-review-r3.md deleted file mode 100644 index a44c077b..00000000 --- a/docs/review/HAY-007-phase2-full-review-r3.md +++ /dev/null @@ -1,113 +0,0 @@ -# HAY-007 Phase 2 Full Review — Round 3 - -**PR**: #55 `feat(vectorindex): HAY-007 Phase 2 - Write paths, WAL, Batch, Grow` -**Date**: 2026-04-18 -**Reviewer**: Claude Opus 4 (code review) -**Verdict**: **REQUEST CHANGES** - ---- - -## Targeted Check: scanLSN + maxWalPayloadSize - -**Status: FIXED** (commit `3637ad3`) - -`scanLSN()` now checks `if length > maxWalPayloadSize { break }` before allocating payload buffer, matching the same guard in `replayWAL()`. This closes the OOM vector on corrupted WAL during open. - ---- - -## 7-Point Checklist - -### 1. Design Compliance — PARTIAL PASS - -| Requirement | Status | -|---|---| -| WAL with LSN + CRC32 | PASS | -| Batch mode (deferred sync) | PASS | -| File grow (2x expansion) | PASS | -| 50K insert < 90s benchmark | PASS (0.38s reported) | -| Lock ordering: muGraph -> muNodes -> muVec | **FAIL** — PutNode acquires muVec.RLock (line 41) before muGraph.Lock (line 55) | -| Crash recovery via WAL replay | PARTIAL — replay works but metadata updates unprotected | - -### 2. Test Coverage — PARTIAL PASS - -**Present**: WAL roundtrip (5 record types), WAL corruption/truncation recovery, write roundtrip, batch nesting, grow preservation, concurrent grow+read, 50K benchmark, node mapping persistence, NextNodeId persistence. - -**Missing**: -- No crash recovery end-to-end test (kill mid-write, reopen, verify) -- No concurrent write stress test (would expose metadata races) -- No concurrent Append+Replay WAL test -- No grow failure / error path test - -### 3. Error Handling — PARTIAL PASS - -- WAL CRC mismatch -> truncate at last valid record: good -- WAL payload > 64MiB -> reject: good (both scanLSN and replayWAL) -- Close() collects first error but continues cleanup: acceptable -- **Gap**: grow failure (mmapAlloc fails after Truncate) leaves file truncated with no mmap — unrecoverable without rollback -- **Gap**: replayWAL callback failure leaves metadata inconsistent - -### 4. Code Quality — PARTIAL PASS - -- Clean file decomposition (wal, write, grow, bench — each focused) -- Consistent encoding (little-endian everywhere) -- Good use of bufio for WAL I/O - -**Issues**: - -| # | Severity | File:Line | Issue | -|---|---|---|---| -| C1 | CRITICAL | mmap_store_write.go:41-61 | **Lock ordering violation**: muVec.RLock before muGraph.Lock — deadlock risk with concurrent grow | -| C2 | CRITICAL | mmap_store_write.go:79-86,193-196,259-260,341-342 | **s.meta unprotected**: PutNode, SetEntryPoint, DeleteNode, NextNodeId all mutate s.meta without any lock | -| C3 | CRITICAL | mmap_store_write.go:30-49 | **Capacity stale after ensure**: ensureVecCapacity called outside muVec; grow unmaps old region before write acquires RLock -> potential SIGSEGV | -| C4 | HIGH | mmap_store_write.go:215-222 | **idmapFile.Write outside muDoc** -> concurrent SetNodeMapping interleaves file writes | -| C5 | HIGH | mmap_wal.go:195-258 | **Replay concurrent with Append**: Replay seeks to 0 and resets bufio while Append may be writing. In practice only called during Open (single-threaded), but API doesn't enforce this | -| C6 | MEDIUM | mmap_windows.go:40-43 | **munmapPlatform crash on empty slice**: &data[0] panics when len(data)==0 (Unix side is safe) | -| C7 | MEDIUM | mmap_wal.go:78-92 | **scanLSN allocates per-record**: header/payload/crc buffers allocated in every loop iteration — GC pressure on large WAL | - -### 5. Performance — PASS - -- 50K x 128d in 0.38s (batch mode) — well within 90s target -- 2x file growth amortizes resize cost -- WAL buffered writer reduces syscalls -- rebuildNodeCount is O(N) on open but acceptable for current scale - -### 6. Security — PASS (with notes) - -- maxWalPayloadSize (64 MiB) prevents OOM on malicious WAL — good -- CRC32 is not cryptographic; acceptable for integrity, not tamper-proofing -- docId encoded as uint16 length — silently truncates at 65535 chars (document or validate) - -### 7. Documentation — PASS - -- Design doc (HAY-007-mmap-store.md) is thorough and matches implementation intent -- Commit messages are clear and well-structured -- Lock ordering documented in design but should be added as code comments - ---- - -## Required Changes Before Merge - -### Must Fix (3 critical) - -1. **C1 — Lock ordering**: Reorder PutNode to acquire muGraph.Lock before muVec.RLock -2. **C2 — Metadata protection**: Add a mutex (or use existing locks) to protect all s.meta mutations -3. **C3 — Capacity re-check under lock**: After ensureVecCapacity, re-verify capacity under muVec.RLock before writing - -### Should Fix (2 high) - -4. **C4 — idmapFile write race**: Extend muDoc lock to cover idmapFile.Write -5. **C6 — Windows munmap empty check**: Add `if len(data) == 0 { return nil }` guard - -### Nice to Have - -6. C5 — Document that Replay must only be called during single-threaded Open -7. C7 — Reuse buffers in scanLSN (perf, not correctness) -8. Add crash recovery integration test -9. Add concurrent PutNode stress test with `-race` -10. Add lock ordering comment in code (e.g., top of mmap_store.go) - ---- - -## Verdict - -**REQUEST CHANGES** — 3 critical concurrency issues (lock ordering violation, unprotected metadata, stale capacity after grow) create deadlock and data corruption risk under concurrent load. The scanLSN fix (original review target) is confirmed correct. Estimated fix effort: 2-3 hours. diff --git a/docs/review/HAY-007-phase2-full-review-r4.md b/docs/review/HAY-007-phase2-full-review-r4.md deleted file mode 100644 index 33f0d82b..00000000 --- a/docs/review/HAY-007-phase2-full-review-r4.md +++ /dev/null @@ -1,147 +0,0 @@ -# HAY-007 Phase 2 Full Review — Round 4 - -**PR**: #55 `feat(vectorindex): HAY-007 Phase 2 - Write paths, WAL, Batch, Grow` -**Date**: 2026-04-18 -**Reviewer**: Claude Opus 4 (code review) -**Verdict**: **APPROVE** (with minor items) - ---- - -## Round 3 → Round 4 Fix Verification - -### C1 (CRITICAL): Lock ordering → muWrite serialisation -**Status: FIXED** (commit `b80b104`) - -The old approach used fine-grained lock ordering (muGraph → muNodes → muVec) for writes. This has been replaced with a single `muWrite sync.RWMutex` that serialises **all** write methods. Verified: - -- `PutNode`: `s.muWrite.Lock()` at entry (mmap_store_write.go:17) -- `SetNeighbors`: `s.muWrite.Lock()` at entry (mmap_store_write.go:86) -- `SetNorm`: `s.muWrite.Lock()` at entry (mmap_store_write.go:159) -- `SetEntryPoint`: `s.muWrite.Lock()` at entry (mmap_store_write.go:180) -- `DeleteNode`: `s.muWrite.Lock()` at entry (mmap_store_write.go:241) -- `SetNodeMapping`: `s.muWrite.Lock()` at entry (mmap_store_write.go:197) -- `DeleteNodeMapping`: `s.muWrite.Lock()` at entry (mmap_store_write.go:226) -- `NextNodeId`: `s.muWrite.Lock()` at entry (mmap_store_write.go:348) -- `BeginBatch`: `s.muWrite.Lock()` at entry (mmap_store_write.go:280) -- `CommitBatch`: `s.muWrite.Lock()` at entry (mmap_store_write.go:291) -- `DiscardBatch`: `s.muWrite.Lock()` at entry (mmap_store_write.go:319) - -No nested lock acquisition within write paths — `ensureCapacity` and `growFile` run under muWrite without acquiring muVec/muGraph/muNodes. **Deadlock risk eliminated.** - -Read paths remain lock-free w.r.t. muWrite: -- `GetVector`: `muVec.RLock` only -- `GetNeighbors`: `muGraph.RLock` (+ `muNodes.RLock` for upper) -- `GetNorm` / `GetNodeLevel`: `muNodes.RLock` only -- `GetNodeId`: `muDoc.RLock` only -- `GetEntryPoint`: `muWrite.RLock` (lightweight, no contention with reads) - -**Concern**: Read paths (e.g., `GetVector`) still hold `muVec.RLock` while a grow under `muWrite.Lock` does `munmap` + `remapFile` which writes to `s.vectors` directly — but grow does NOT acquire `muVec.Lock`. This means a concurrent reader could be reading from a stale/unmapped `s.vectors` slice during grow. See P1-1 below. - -### C2 (CRITICAL): s.meta unprotected -**Status: FIXED** - -All meta mutations (`PutNode`, `SetEntryPoint`, `DeleteNode`, `NextNodeId`) are now under `muWrite.Lock()`. The only reader of meta is `GetEntryPoint`, which uses `muWrite.RLock()` (mmap_store_read.go:146-149) — correct atomic snapshot of EntryPoint + EntryLevel. - -### C3 (CRITICAL): ensureCapacity race -**Status: FIXED** - -`ensureCapacity` + `growFile` + `remapFile` all run under `muWrite.Lock()`. No separate lock acquisition needed. The stale-capacity-after-grow race is eliminated because only one writer can exist. - -### C4 (HIGH): idmapFile write race -**Status: FIXED** - -`SetNodeMapping` acquires both `muWrite.Lock()` and `muDoc.Lock()` (mmap_store_write.go:197-201). The `idmapFile.Write` at line 217 is now protected by both locks. - -### C6 (MEDIUM): Windows munmap empty slice -**Status: FIXED** - -`munmapPlatform` on Windows still has `&data[0]` without len check (mmap_windows.go:41), BUT `mmapSyncPlatform` has the `len(data) == 0` guard (mmap_windows.go:49-51). The `mmapFree` wrapper in mmap.go:25 already guards: `if data == nil { return nil }`. However, `munmapPlatform` itself would panic on `[]byte{}` (non-nil, zero-length). See P2-1. - ---- - -## 7-Point Checklist - -### 1. Design Compliance — PASS - -| Requirement | Status | -|---|---| -| WAL with LSN + CRC32 | PASS — 5 record types, CRC covers header+payload | -| Batch mode (deferred sync) | PASS — BeginBatch/CommitBatch/DiscardBatch with nesting | -| File grow (2x expansion) | PASS — remapFile: munmap→truncate→mmap | -| 50K insert < 90s | PASS — 0.38s reported (2400x vs Pebble) | -| Concurrency: serial writes, concurrent reads | PASS — muWrite for writes, fine-grained RLocks for reads | -| Crash recovery via WAL replay | PASS — replayWAL handles all 5 types, rebuildNodeCount for idempotency | -| idmap persistence with CRC | PASS — append with CRC32, loadIdmap validates | - -### 2. Test Coverage — PASS (adequate for Phase 2) - -**Present**: WAL roundtrip (5 types), corruption/truncation recovery, LSN continuity, write roundtrip (PutNode/SetNeighbors/SetNorm/SetEntryPoint), batch nesting, grow (single/multiple/preserves data/concurrent), node mapping persistence, NextNodeId persistence, 50K insert benchmark. - -**Still missing** (acceptable as Phase 3/4 scope): -- Crash recovery integration test (Phase 4) -- Concurrent write + read stress test under -race - -### 3. Error Handling — PASS - -- WAL: CRC mismatch → truncate; payload > 64MiB → reject -- Grow: munmap → truncate → mmap sequence (P2-2 notes partial failure risk) -- Close: ordered cleanup (msync → WAL sync → meta → WAL close → idmap close → munmap → file close) - -### 4. Code Quality — PASS - -- Clean file decomposition: mmap_store.go (core), _write.go, _read.go, _grow.go, mmap_wal.go -- Consistent little-endian encoding -- Good concurrency model documentation in struct comment -- WAL buffered I/O for performance - -### 5. Performance — PASS - -- 50K×128d in 0.38s (batch mode) -- 2x growth amortizes resize -- WAL buffered writer reduces syscalls in batch mode -- rebuildNodeCount O(N) on open — acceptable at current scale - -### 6. Security — PASS - -- maxWalPayloadSize prevents OOM -- CRC32 for integrity (not tamper-proofing — acceptable for local storage) -- docId uint16 length cap: 65535 chars (sufficient, no validation needed) - -### 7. Documentation — PASS - -- Concurrency model clearly documented in MmapStore struct comment -- Design doc matches implementation -- Commit messages well-structured - ---- - -## Remaining Issues - -### P1 — Should Fix (non-blocking for merge, fix before Phase 5 integration) - -| # | File:Line | Issue | -|---|---|---| -| P1-1 | mmap_store_grow.go:137-157 | **Grow/read race on mmap slice pointer**: `remapFile` writes to `*data` (e.g., `s.vectors`) under `muWrite.Lock()`, but concurrent readers hold only `muVec.RLock()` — they can read a stale slice header pointing to unmapped memory. In practice, HNSW Insert holds `h.mu` which serialises insert+search, so concurrent Search during grow is unlikely. But if Search runs without `h.mu` (pure read path), this is a SIGSEGV risk. **Fix**: acquire the per-file write lock (e.g., `muVec.Lock()`) inside `remapFile` around the pointer swap, OR document that grow is only safe when no concurrent reads are possible. | -| P1-2 | mmap_store_write.go:327-329 | **BatchDepth() unsynchronised**: reads `s.batchDepth` without any lock. If called from a different goroutine than the writer, this is a data race. Low risk since it's only used in tests. **Fix**: add `muWrite.RLock()`. | - -### P2 — Nice to Have - -| # | File:Line | Issue | -|---|---|---| -| P2-1 | mmap_windows.go:41 | `munmapPlatform` panics on `[]byte{}` (non-nil, zero-length). The `mmapFree` nil guard doesn't catch this. **Fix**: add `if len(data) == 0 { return nil }`. | -| P2-2 | mmap_store_grow.go:137-157 | `remapFile` has no rollback if `mmapAlloc` fails after `Truncate` succeeds — the old mmap is already freed, file is extended, but no new mapping exists. The store is in a broken state. Acceptable for now since mmap failure is extremely rare (OOM or fd exhaustion). Phase 4 crash recovery should cover this. | -| P2-3 | mmap_wal.go:216-217 | `Replay` returns hard error on `length > maxWalPayloadSize` but `scanLSN` just breaks. Inconsistent — scanLSN's behavior is more forgiving (treats as corruption boundary). Consider making Replay also just break (truncate at corruption). | -| P2-4 | mmap_store.go:356 (loadIdmap) | `os.ReadFile(path)` re-reads the file that's already open via `f`. Could use `io.ReadAll(f)` after seeking to 0, avoiding double-open. Minor. | -| P2-5 | mmap_store_write.go:300-301 | `sync = true` overwrites the parameter then checks `if sync` — dead branch. Remove the parameter or the override. | - ---- - -## Verdict - -**APPROVE** - -All 3 CRITICAL and 2 HIGH issues from Round 3 are fixed. The muWrite serialisation approach is clean and eliminates the entire class of lock-ordering and metadata-race bugs. The code is well-structured, tested, and meets the Phase 2 performance target by a wide margin. - -P1-1 (grow/read race on mmap slice pointer) is the most significant remaining issue but is mitigated by HNSW's own `h.mu` serialisation in practice. It should be addressed before Phase 5 integration when Search may run concurrently with Insert without `h.mu`. - -P1-2 and P2-* items are minor and can be addressed in follow-up commits. diff --git a/docs/review/HAY-007-phase2-full-review-r5.md b/docs/review/HAY-007-phase2-full-review-r5.md deleted file mode 100644 index 2fa1fd8f..00000000 --- a/docs/review/HAY-007-phase2-full-review-r5.md +++ /dev/null @@ -1,264 +0,0 @@ -# HAY-007 Phase 2 Full Review -- Round 5 (Final) - -**PR**: #55 `feat(vectorindex): HAY-007 Phase 2 - Write paths, WAL, Batch, Grow` -**Date**: 2026-04-18 -**Reviewer**: Claude Opus 4 (strict code review) -**Scope**: DATA RACE fix + new tests + Windows fix + 3-platform CI -- final version -**Verdict**: **APPROVE** - ---- - -## Round 4 Fix Verification - -### R4 P1-1 (grow/read race on mmap slice pointer) -**Status: FIXED** (commit `b4c2eb2`) - -Each grow function now acquires the corresponding region write lock around `remapFile`: -- `growVectors()`: `muVec.Lock()` at line 83, unlock at line 85 -- `growNodes()`: `muNodes.Lock()` at line 104, unlock at line 106 -- `growL0()`: `muGraph.Lock()` at line 125, unlock at line 127 -- `growUpper()`: `muGraph.Lock()` at line 146, unlock at line 148 - -This blocks concurrent readers (who hold RLock) during the critical munmap-remap window. Verified: `remapFile` (lines 153-173) runs entirely within the region lock scope. The double-check pattern (re-check capacity under lock, lines 72-74, 93-95, 114-116, 135-137) prevents TOCTOU races. **Correct.** - -### R4 P2-1 (Windows munmapPlatform empty slice) -**Status: FIXED** (commit `499e709`) - -`munmapPlatform` on Windows now has `if len(data) == 0 { return nil }` guard at line 41-42. `mmapSyncWindows` already had the guard at lines 53-54. Both paths are safe. **Correct.** - -### R4 P2-5 (CommitBatch sync parameter dead branch) -**Status: NOT FIXED** -- still present at `mmap_store_write.go:300-301`. `sync = true` overwrites the parameter. **Severity: P2 -- non-blocking.** - ---- - -## 7-Point Checklist - -### 1. Design Compliance -- PASS - -| Requirement | Status | Evidence | -|---|---|---| -| WAL with LSN + CRC32 | PASS | `mmap_wal.go`: 5 record types, CRC covers LSN+Length+Type+Payload | -| WAL replay (all 5 types) | PASS | `mmap_store.go:394-496`: INSERT writes vec/node/upper, SET_NEIGHBORS dispatches L0/upper, SET_NORM writes norm, SET_ENTRY restores meta, DELETE sets tombstone | -| Batch mode (deferred sync) | PASS | BeginBatch/CommitBatch/DiscardBatch with nesting depth counter | -| File grow (2x expansion) | PASS | `mmap_store_grow.go`: munmap->truncate->remap per region, region lock held | -| 50K insert < 90s | PASS | 0.38s reported (132K inserts/sec), 237x under target | -| Concurrency: serial writes, concurrent reads | PASS | `muWrite` serializes all writes; fine-grained RLocks for reads; region locks in grow block readers during remap | -| Crash recovery via WAL replay | PASS | replayWAL handles all 5 types + rebuildNodeCount for idempotency | -| idmap persistence with CRC | PASS | append with CRC32, loadIdmap validates per-entry | -| DeleteNode (tombstone) | PASS | WAL DELETE + tombstone flag in nodes.dat | -| NextNodeId (auto-increment) | PASS | freelist TODO Phase 3, simple increment for now | -| rebuildNodeCount after replay | PASS | scans nodes.dat, counts non-deleted + norm != 0 | -| WAL payload size cap (64 MiB) | PASS | Both scanLSN (line 84-86) and Replay (line 217) check maxWalPayloadSize | - -### 2. Test Coverage -- PASS - -**New tests added (commit `4c3cd35`, file `mmap_phase2_test.go`):** - -| Test | Coverage Target | -|---|---| -| `TestMmapStorePutNodeWritesMmapContents` | Raw mmap byte verification after PutNode | -| `TestMmapStorePutNodeWithUpperLevel` | Upper slot allocation for level > 0 | -| `TestMmapStoreSetNeighborsUpperMultipleLayers` | Multi-layer upper neighbor write + readback | -| `TestMmapStoreGrowUpperGraph` | Upper graph grow triggered by many level>0 inserts | -| `TestMmapStoreCloseReopenPersistence` | End-to-end: write -> Close -> Reopen -> verify all paths (vec, level, entry, mapping, L0 neighbors) | -| `TestMmapStoreLoadIdmap` | 20-entry idmap load after reopen | -| `TestMmapStoreLoadIdmapCorrupt` | Corrupt CRC -> second entry skipped, first survives | -| `TestMmapStoreSyncAll` | syncAll doesn't panic | -| `TestMmapStoreDeleteNodeAndReopen` | Delete + reopen -> tombstone persists via WAL replay | -| `TestMmapStoreRebuildNodeCount` | 5 inserts - 1 delete = 4 after replay | -| `TestMmapStoreCommitBatchSyncs` | CommitBatch flushes + data readable | - -**Existing test coverage (mmap_store_write_test.go, mmap_wal_test.go, mmap_store_grow_test.go):** -- WAL: roundtrip all 5 types, truncation recovery, CRC corruption, LSN continuity, afterLSN filtering -- Write: PutNode+Get*, SetNeighbors L0/Upper, SetNorm, SetEntryPoint, NodeMapping CRUD + persistence, Batch nesting, DiscardBatch, NextNodeId + persistence -- Grow: single grow, multiple grows, data preservation, concurrent read+grow - -**Assessment**: Coverage is comprehensive. The `TestMmapStoreCloseReopenPersistence` test is the end-to-end crash recovery test that was missing in R2-R4. DeleteNode now has explicit test coverage via `TestMmapStoreDeleteNodeAndReopen`. - -### 3. Error Handling -- PASS - -- WAL CRC mismatch -> truncate at last valid record (scanLSN) -- WAL payload > 64 MiB -> reject (both scanLSN and Replay) -- Close() ordered cleanup: syncAll -> WAL sync -> meta write -> WAL close -> idmap close -> closeMmaps -- mmapAll cleanup on partial failure: tracks openedFiles + mappedRegions, cleanup closure frees all on error -- loadIdmap: per-entry CRC, skips corrupt entries (doesn't abort) - -### 4. Code Quality -- PASS - -- Clean file decomposition: _write.go (write methods), _grow.go (grow + remap), _read.go (read methods), mmap_wal.go (WAL) -- Consistent little-endian encoding throughout -- Concurrency model well-documented: muWrite for writes, per-region RLocks for reads -- WAL buffered I/O for batch performance -- Lock acquisition in grow functions has clear comments explaining the two-lock pattern (muWrite + region lock) - -### 5. Performance -- PASS - -- 50K x 128d in 0.38s (batch mode) -- 132K inserts/sec -- 2x growth amortizes resize cost -- WAL buffered writer reduces syscalls in batch mode -- rebuildNodeCount O(N) on open -- acceptable at current scale (< 500K nodes) - -### 6. Security -- PASS - -- maxWalPayloadSize (64 MiB) prevents OOM on corrupted/malicious WAL -- CRC32 for integrity detection (acceptable for local storage) -- idmap CRC32 per-entry validation -- No user-controlled input reaches mmap offsets without bounds checking - -### 7. Documentation -- PASS - -- Design doc (HAY-007-mmap-store.md) matches implementation -- Concurrency model documented in MmapStore struct comments -- Lock ordering documented (muGraph -> muNodes -> muVec) -- Commit messages well-structured and descriptive - ---- - -## DATA RACE Fix Deep Dive - -### muWrite Serialisation Model (commit `b80b104`) - -**Design**: Single `muWrite sync.RWMutex` serializes all write methods. Read methods retain fine-grained per-region RLocks. - -**Write methods verified under muWrite.Lock():** -- PutNode, SetNeighbors, SetNorm, SetEntryPoint, DeleteNode, SetNodeMapping, DeleteNodeMapping, NextNodeId, BeginBatch, CommitBatch, DiscardBatch - -**Read methods verified with region RLocks only:** -- GetVector: muVec.RLock -- GetNeighbors: muGraph.RLock (+ muNodes.RLock for upper) -- GetNorm / GetNodeLevel: muNodes.RLock -- GetNodeId: muDoc.RLock -- GetEntryPoint: muWrite.RLock (reads s.meta) - -**Grow functions verified with muWrite + region Lock:** -- growVectors: muWrite (caller) + muVec.Lock -- growNodes: muWrite (caller) + muNodes.Lock -- growL0: muWrite (caller) + muGraph.Lock -- growUpper: muWrite (caller) + muGraph.Lock - -**Assessment**: The two-level locking model is sound: -1. muWrite serializes all mutations (no write-write races) -2. Region locks block readers during grow's remap phase (no read-during-remap races) -3. Readers don't block each other (RLock concurrency preserved) -4. No nested lock acquisition within write path (deadlock-free) - ---- - -## Windows munmapPlatform Guard - -**Commit**: `499e709` -**File**: `mmap_windows.go:41-42` - -```go -func munmapPlatform(data []byte) error { - if len(data) == 0 { - return nil - } - // ... UnmapViewOfFile(&data[0]) -``` - -Both `munmapPlatform` and `mmapSyncWindows` now guard against empty slices before dereferencing `&data[0]`. The `mmapFree` wrapper in `mmap.go:25` guards against nil, but non-nil zero-length slices (`[]byte{}`) were the gap. **Fixed correctly.** - ---- - -## CI 3-Platform Matrix - -**File**: `.github/workflows/ci.yml` - -| Platform | Build | Test | Coverage | -|---|---|---|---| -| ubuntu-latest | `go build ./...` | `test_and_coverage.sh` (full suite + race + coverage) | Yes | -| macos-latest | `go build ./...` | `go test -timeout 15m ./internal/core/vectorindex/...` | No | -| windows-latest | `go build ./...` | `go test -timeout 15m ./internal/core/vectorindex/...` | No | - -**Observations**: -- Linux gets full test suite + coverage; macOS/Windows get vectorindex-specific tests with 15m timeout -- Git LFS install only on Linux (macOS/Windows don't need SIFT test data) -- Format check only on Linux (reasonable -- formatting is platform-independent) -- Build changed from `make build` to `go build ./...` for cross-platform compatibility - -**Assessment**: Reasonable configuration. Cross-platform tests focus on the mmap code (platform-specific syscalls) where differences matter most. Full coverage tracking on Linux prevents regression. - ---- - -## nocov Usage Audit - -| Location | Annotation | Legitimate? | -|---|---|---| -| `mmap_store.go:145` | `replayWAL()` error during Open | **YES** -- requires WAL file corruption during running test; defensive | -| `mmap_store.go:275` | `mmapAll` cleanup closure | **YES** -- requires partial mmap failure (e.g., fd exhaustion mid-open); defensive | -| `mmap_store.go:320` | `closeMmaps()` | **YES** -- cleanup helper called from Close/error paths; tested indirectly but coverage tool can't always trace | - -**EXCLUDE_FUNCS in coverage script:** -``` -PutNode, DeleteNode, SetNodeMapping, DeleteNodeMapping, writeMetaHeader, -writeDataFileHeader, initAllFiles, mmapAll, Replay, setNeighborsUpper, -remapFile, Close, OpenWAL, growFile, ensureUpperCapacity, OpenMmapStore, -syncAll, closeMmaps -``` - -**Assessment**: The EXCLUDE_FUNCS list is large (18 functions). Many are write-path functions with error branches that are difficult to trigger in unit tests (e.g., mmap allocation failure, fsync failure, file truncation failure). This is acceptable for Phase 2 given: -1. The core logic paths ARE tested (write + read roundtrip, WAL replay, grow, persistence) -2. The excluded paths are predominantly OS-level error branches (mmap/msync/fsync failures) -3. Coverage threshold is still met at 88% with these exclusions - -**Concern**: Some excluded functions contain non-trivial logic beyond OS error handling (e.g., `OpenMmapStore`, `Replay`, `Close`). The happy paths of these functions ARE tested via integration tests, but the exclusion means regressions in their error handling won't trigger coverage alerts. **Acceptable for Phase 2, recommend tightening in Phase 4.** - ---- - -## Remaining Issues - -### P1 -- Should Fix (non-blocking, fix before Phase 5) - -| # | File:Line | Issue | Risk | -|---|---|---|---| -| P1-1 | mmap_store_write.go:23, :245 | **nodeToDoc read without muDoc**: `PutNode` (line 23) and `DeleteNode` (line 245) read `s.nodeToDoc[id]` under `muWrite.Lock()` but without `muDoc.Lock()`. `SetNodeMapping` (line 197-201) writes to `nodeToDoc` under both `muWrite.Lock()` + `muDoc.Lock()`. Since writes are serialized by `muWrite`, there is NO data race between `PutNode` and `SetNodeMapping` -- they can't execute concurrently. However, `GetNodeId` (read path) acquires only `muDoc.RLock()`, not `muWrite.RLock()`. If `GetNodeId` runs concurrently with `PutNode`, there is a theoretical race on `nodeToDoc`. In practice, `PutNode` only reads the map (not writes), and `GetNodeId` only reads via `docToNode` (different map), so **no actual race exists**. But the asymmetric locking is confusing. | Low | -| P1-2 | mmap_store.go:508 | **rebuildNodeCount assumes norm != 0 for all real nodes**: If a vector has true L2 norm = 0 (zero vector), it will be counted as "never written" and excluded from NodeCount. `vek32.Norm` returns 0 for zero vectors. | Low (zero vectors rare in practice) | - -### P2 -- Nice to Have - -| # | File:Line | Issue | -|---|---|---| -| P2-1 | mmap_store_write.go:300-301 | `CommitBatch` `sync` parameter overwritten to `true` -- dead branch, misleading API | -| P2-2 | mmap_store.go:445, :483-484 | Replay loop has `NodeCount++` and `NodeCount--` that are always overwritten by `rebuildNodeCount` -- dead code | -| P2-3 | mmap_store.go:340-352 | `loadIdmap` opens file then uses `os.ReadFile` on same path -- double fd; should use `io.ReadAll(f)` | -| P2-4 | mmap_store_bench_test.go | Benchmark uses `time.Now()` instead of `b.ResetTimer()` -- doesn't follow Go benchmark conventions | -| P2-5 | test_and_coverage.sh | 18 functions in EXCLUDE_FUNCS is large; consider tightening in Phase 4 | - ---- - -## New Test Quality Assessment - -The `mmap_phase2_test.go` tests (commit `4c3cd35`) are well-written: - -**Strengths:** -- `TestMmapStoreCloseReopenPersistence` is a comprehensive end-to-end test covering vectors, node levels, entry point, node mappings, and L0 neighbors across Close/Reopen -- `TestMmapStoreLoadIdmapCorrupt` properly corrupts CRC and verifies partial recovery -- `TestMmapStoreDeleteNodeAndReopen` verifies tombstone persistence through WAL replay -- `TestMmapStoreRebuildNodeCount` verifies correct count after insert + delete + replay -- Raw mmap byte assertions in `TestMmapStorePutNodeWritesMmapContents` verify the actual on-disk format - -**Gaps (acceptable for Phase 2):** -- No concurrent write stress test with `-race` flag -- No test verifying WAL replay idempotency (replay same WAL twice) -- No test for oversized WAL payload rejection - ---- - -## Conclusion - -**APPROVE** - -All critical and high issues from Rounds 1-4 are fixed. The concurrency model is sound: `muWrite` serializes all writes, per-region RLocks enable concurrent reads, and grow functions properly acquire region write locks to protect the remap window. The Windows empty-slice guard is correct. The 3-platform CI matrix is reasonable. Test coverage is comprehensive with the new Phase 2 tests pushing coverage to 88%. - -P1-1 (asymmetric locking on nodeToDoc) and P1-2 (zero-vector norm assumption) are minor and non-blocking. P2 items are code cleanliness issues suitable for follow-up. - -**Summary of all 5 rounds:** - -| Round | Verdict | Key Issues | Resolution | -|---|---|---|---| -| R1 | REQUEST CHANGES | NodeCount drift, WAL payload cap | Fixed (rebuildNodeCount, maxWalPayloadSize) | -| R2 | REQUEST CHANGES | scanLSN missing size check | Fixed (commit 3637ad3) | -| R3 | REQUEST CHANGES | Lock ordering violation (C1), meta unprotected (C2), capacity TOCTOU (C3), idmap race (C4), Windows munmap (C6) | Fixed (muWrite serialisation, commit b80b104) | -| R4 | APPROVE (with items) | Grow/read race on slice pointer (P1-1), Windows munmap (P2-1) | Fixed (region locks in grow, commit b4c2eb2; empty slice guard, commit 499e709) | -| R5 | **APPROVE** | Asymmetric locking (P1), dead code (P2) | Non-blocking | diff --git a/docs/review/HAY-007-phase2-p1-fixes-review.md b/docs/review/HAY-007-phase2-p1-fixes-review.md deleted file mode 100644 index 8e532382..00000000 --- a/docs/review/HAY-007-phase2-p1-fixes-review.md +++ /dev/null @@ -1,56 +0,0 @@ -# HAY-007 Phase 2 P1 Fixes Review - -PR: codetrek/haystack#55 -Reviewer: Pegasus -Date: 2026-04-18 - ---- - -## P1-1: replayWAL NodeCount 盲增/盲减 - -**结论: 已修复** - -| 检查项 | 状态 | 说明 | -|--------|------|------| -| `rebuildNodeCount` 函数存在 | OK | `mmap_store.go:493` | -| replay 后调用 | OK | `mmap_store.go:487`，在 `replayWAL()` 末尾、return 前调用 | -| 扫描逻辑正确 | OK | 遍历 `[0, TotalSlots)` 所有 slot，检查 `nodeFlagDeleted` 位，排除 tombstone 后计数 | -| 测试覆盖 | **缺失** | 无专门测试验证重复 replay 后 NodeCount 幂等性 | - -**代码质量**: 实现简洁正确。`rebuildNodeCount` 无条件覆盖 `meta.NodeCount`，无论 replay 期间累加了什么值，最终都以扫描结果为准——这是正确的幂等策略。 - -**遗留问题**: replay 循环体中仍有 `s.meta.NodeCount++` 和 `s.meta.NodeCount--`（`mmap_store.go:468, 483`），虽然被 `rebuildNodeCount` 覆盖不会造成 bug，但属于死代码。建议后续清理。严重程度: LOW。 - ---- - -## P1-5: WAL payload length 无上限校验 - -**结论: 已修复** - -| 检查项 | 状态 | 说明 | -|--------|------|------| -| 常量定义 | OK | `maxWalPayloadSize = 64 << 20` (64 MiB), `mmap_wal.go:33` | -| replay 时校验 | OK | `mmap_wal.go:213`: `if length > maxWalPayloadSize` 在 `make([]byte, length)` 之前 | -| 超限处理 | OK | 返回 `fmt.Errorf` 错误，终止 replay。不截断、不跳过——正确选择 | -| 测试覆盖 | **缺失** | 无测试构造 length > 64MiB 的损坏 WAL 验证拒绝行为 | - -**代码质量**: 防御位置正确（allocate 前检查），阈值合理（64 MiB 远超正常 record），错误信息包含实际值和上限便于诊断。 - -**无新问题引入。** - ---- - -## 总结 - -| 修复 | 状态 | 测试 | -|------|------|------| -| P1-1 rebuildNodeCount | 已修复，逻辑正确 | 缺专项测试 | -| P1-5 WAL payload cap | 已修复，逻辑正确 | 缺专项测试 | - -两个 P1 修复实现正确，无新 bug 引入。测试覆盖不足但不阻塞合入。 - -## 结论: APPROVE - -附带建议: -1. (LOW) 清理 replay 循环中的死代码 `NodeCount++/--` -2. (LOW) 补充 `rebuildNodeCount` 幂等性测试和 WAL oversized payload 拒绝测试 diff --git a/docs/review/HAY-007-phase2-review.md b/docs/review/HAY-007-phase2-review.md deleted file mode 100644 index cd1f54c3..00000000 --- a/docs/review/HAY-007-phase2-review.md +++ /dev/null @@ -1,212 +0,0 @@ -# HAY-007 Phase 2 Code Review — PR #55 - -> Reviewer: Claude Code (strict mode) -> Date: 2026-04-18 -> Branch: `feat/hay007-phase2-write` -> Commits: b44a657 → 8044c86 (5 commits, post-review fixes included) - ---- - -## Spec 对照表 - -| Spec 要求 | 实现状态 | 文件 | 备注 | -|-----------|---------|------|------| -| PutNode 写入路径 | ✅ 已实现 | mmap_store_write.go:16-89 | WAL → ensureCapacity → 写 vector/node/upper slot | -| SetNeighbors 写入路径 | ✅ 已实现 | mmap_store_write.go:93-164 | L0 + Upper 分支，含容量校验 | -| SetNorm 写入路径 | ✅ 已实现 | mmap_store_write.go:167-185 | WAL → 写 nodes.dat | -| SetNodeMapping 写入路径 | ✅ 已实现 | mmap_store_write.go:199-222 | 内存 map + idmap.dat append + CRC32 | -| WAL (LSN + append) | ✅ 已实现 | mmap_wal.go | 5 种 record type，LSN 单调递增 | -| WAL replay + CRC32 | ✅ 已实现 | mmap_wal.go:173-232, mmap_store.go:340-484 | 不完整/CRC 不匹配截断 | -| WAL record 格式: LSN+Length+Type+Payload+CRC32 | ✅ 符合 | mmap_wal.go:30-31 | 13 字节 header + payload + 4 字节 CRC | -| BatchableStore (BeginBatch/CommitBatch/DiscardBatch) | ✅ 已实现 | mmap_store_write.go:280-319 | 嵌套 depth，CommitBatch 始终 sync | -| 文件增长 (grow，独立锁) | ✅ 已实现 | mmap_store_grow.go | munmap → truncate → remap，每文件独立锁 | -| DeleteNode (墓碑) | ✅ 已实现 | mmap_store_write.go:239-262 | WAL DELETE + tombstone 标志 | -| NextNodeId (freelist pop 或新分配) | ⚠️ 部分 | mmap_store_write.go:337-344 | freelist TODO Phase 3，仅自增 | -| freelist 启动扫描重建 | ❌ 未实现 | — | 标注 Phase 3，**符合计划** | - -**总结**：Phase 2 spec 10/12 项完成，2 项（freelist）明确标注 Phase 3，符合设计文档分阶段规划。 - ---- - -## 问题清单 - -### P0 — 必须修复（数据正确性/安全） - -**无 P0 问题。** 此前 4 个 P0（allocUpperSlot race、DeleteNode WAL、CommitBatch sync、NextNodeId freelist）已在 b44a657–effd747 修复。 - -### P1 — 强烈建议修复 - -#### P1-1: replayWAL 中 NodeCount 盲增/盲减可能导致计数漂移 - -**文件**: `mmap_store.go:440`, `mmap_store.go:478-479` - -replay 时对每条 INSERT 执行 `s.meta.NodeCount++`，对 DELETE 执行 `NodeCount--`。但 meta.bin 中已保存了 checkpoint 时的 NodeCount。如果 WAL 中部分记录在 checkpoint 前已执行（数据已写入 mmap 且 meta 已更新），replay 会重复计数。 - -**风险**：Close() 写 meta 时 NodeCount 偏高/偏低。当前 Phase 2 无 checkpoint 逻辑（Phase 4），所以 `WalCheckpointLSN=0` 意味着全量 replay，此时 meta.NodeCount 应在 replay 前重置为 0。 - -**建议**：在 `replayWAL()` 开头重置 `s.meta.NodeCount = 0`、`s.meta.TotalSlots = 0`、`s.meta.NextNodeId = 0`，让 replay 完全重建 meta 状态。或者，在 replay 完成后扫描 nodes.dat 计算准确值。 - -#### P1-2: PutNode 写 mmap 时使用 RLock 但实际在修改数据 - -**文件**: `mmap_store_write.go:41-49` (muVec.RLock), `mmap_store_write.go:65-78` (muNodes.RLock) - -PutNode 在写入 vectors 和 nodes 时获取的是 **RLock**（读锁），不是 Lock（写锁）。注释说"HNSW h.mu 保证串行"所以不需要写锁。 - -**问题**：RLock 阻止 grow（需要 Lock），但不阻止并发 reader。多个 goroutine 同时读和写同一 mmap 区域虽然在 OS 层面不会 crash（mmap 写是原子的对 aligned word），但 Go 的 race detector 会报 data race。`SetNeighbors` 同样使用 `muGraph.RLock` 写数据。 - -**建议**：这是有意为之的设计权衡（避免读操作阻塞），在代码注释中明确记录此决策及其前提条件（HNSW h.mu 串行化写入）。如果未来允许并发写入，必须升级为写锁。 - -#### P1-3: loadIdmap 打开文件后又用 ReadFile 重新读取 - -**文件**: `mmap_store.go:336-352` - -`loadIdmap` 先 `os.OpenFile` 获得 handle，然后用 `os.ReadFile` 再次打开同一路径读取内容。这会创建两个独立的 fd。 - -**建议**：直接从已打开的 `f` 读取（`io.ReadAll(f)` 然后 `f.Seek(0, 2)`），避免多余的 open/close。 - -#### P1-4: idmap.dat 无 header/magic - -**文件**: `mmap_store.go:336-386` - -设计文档指定 idmap.dat 有 16 字节 header（Magic "IDMP" + Count + padding），但实现直接从 offset 0 开始写 entry。缺少 magic 校验意味着无法检测文件损坏或格式不匹配。 - -**建议**：加 header 或从 spec 删除此要求，保持文档与代码一致。 - -#### P1-5: WAL payload length 无上限校验 - -**文件**: `mmap_wal.go:156-160` - -`scanLSN` 和 `Replay` 从文件读取 `length` 字段后直接 `make([]byte, length)` 分配。若文件损坏导致 length 为极大值（如 0xFFFFFFFF = 4GB），会导致 OOM panic。 - -**建议**：加上 length 上限校验（如 `< 64MB`），超出则视为损坏记录。 - -### P2 — 建议改进 - -#### P2-1: CommitBatch 中 `sync = true` 覆盖使 `if sync` 分支成为死代码 - -**文件**: `mmap_store_write.go:296-310` - -```go -sync = true // line 296 -if sync { // line 301 — always true -``` - -这是 CRITICAL-2 修复的结果。代码正确但可读性差——参数 `sync bool` 的签名暗示调用者可以控制，实际无效。 - -**建议**：去掉参数，或加注释说明为什么忽略。 - -#### P2-2: mmapSyncPlatform (Unix) 未做 page 对齐 - -**文件**: `mmap_unix.go:27-36` - -`msync(2)` 要求 addr 是 page-aligned。`mmap` 返回的基地址通常是对齐的，但规范要求显式保证。如果 `data` 是对 mmap 区域的 sub-slice，addr 可能不对齐。 - -**建议**：当前所有调用传入完整 mmap slice，所以实际安全。加个注释说明此前提。 - -#### P2-3: grow 后 header capacity 已更新但未 msync - -**文件**: `mmap_store_grow.go:146-148` - -`remapFile` 更新 header 中的 capacity 字段但不 msync。如果随后 crash，重新 mmap 时 header 中的 capacity 可能与文件实际大小不一致。 - -**建议**：grow 后 msync header page，或在 replay 时根据文件大小重算 capacity。 - -#### P2-4: Benchmark 使用 `time.Now()` 而非 `b.ResetTimer()` - -**文件**: `mmap_store_bench_test.go:39-42` - -benchmark 在循环内用 `time.Now()` 手动计时，但没有调用 `b.ResetTimer()` 排除 setup 开销。不影响正确性但不符合 Go benchmark 惯例。 - -#### P2-5: replayWAL INSERT 不恢复 idmap 映射 - -**文件**: `mmap_store.go:396-444` - -replay INSERT 时 `docId` 被 `_` 忽略（line 398），不重建 `docToNode`/`nodeToDoc` 映射。如果 crash 发生在 PutNode 之后、SetNodeMapping 之前，WAL 中有 INSERT 但 idmap.dat 中无对应 entry，重启后丢失映射。 - -**建议**：replay INSERT 时也重建 id 映射（`s.docToNode[docId] = nodeId` if docId != ""）。 - -#### P2-6: SetNodeMapping 无 WAL 保护 - -**文件**: `mmap_store_write.go:199-222` - -SetNodeMapping 写 idmap.dat 但不写 WAL。crash 时 idmap.dat 可能部分写入（CRC 校验可以捕获不完整 entry），但与 WAL 的 INSERT record 中的 docId 存在冗余——两者可能不一致。 - -当前设计可接受（idmap CRC 保护 + replay 可补全），但应在注释中明确说明 recovery 策略。 - ---- - -## 测试覆盖评估 - -| 测试类别 | 覆盖情况 | 缺失 | -|---------|---------|------| -| PutNode + GetVector 正常路径 | ✅ | — | -| PutNode + GetNorm | ✅ | — | -| PutNode with level > 0 | ✅ | — | -| SetNeighbors L0 | ✅ | — | -| SetNeighbors Upper | ✅ | — | -| SetNorm | ✅ | — | -| SetEntryPoint | ✅ | — | -| NodeMapping CRUD | ✅ | — | -| NodeMapping 持久化 | ✅ | — | -| Batch write + read | ✅ | — | -| Batch 嵌套 | ✅ | — | -| DiscardBatch | ✅ | — | -| NextNodeId 自增 | ✅ | — | -| NextNodeId 持久化 | ✅ | — | -| WAL append + replay 全类型 | ✅ | — | -| WAL afterLSN 过滤 | ✅ | — | -| WAL 截断记录恢复 | ✅ | — | -| WAL CRC 损坏恢复 | ✅ | — | -| WAL LSN 跨重启连续 | ✅ | — | -| Grow 触发 | ✅ | — | -| Grow 多次翻倍 | ✅ | — | -| Grow 保留旧数据 | ✅ | — | -| Grow 并发读写 | ✅ | — | -| 50K insert benchmark | ✅ | — | -| WAL replay 恢复 mmap 数据 | ⚠️ 集成覆盖 | 缺独立 replayWAL 单测 | -| DeleteNode + tombstone 验证 | ❌ | 无直接 DeleteNode 测试 | -| 边界: 空向量/零维 | ❌ | — | -| 边界: docId 极长（>65535 bytes） | ❌ | uint16 溢出 | - -**测试质量评分**: 8/10 — 正常路径和错误路径覆盖良好，缺少部分边界和 DeleteNode 独立测试。 - ---- - -## 性能评估 - -**50K Insert 0.38s** — 可信。 - -理由： -- 这是纯写路径（PutNode + SetNeighbors），不含 HNSW 图搜索（占 Insert 95%+ 时间） -- Batch 模式下 WAL 使用 bufio.Writer，mmapSync 延迟到 CommitBatch -- mmap 写入 = 内存 copy，50K × (512B vec + 16B node + 260B L0) ≈ 38MB，现代 SSD 顺序写 > 1GB/s -- WAL 顺序追加 50K records ≈ 30MB，单次 fsync -- 0.38s 合理，瓶颈应在 WAL 最终 fsync - ---- - -## 代码质量 - -- **可读性**: 良好。文件按职责拆分（write/grow/wal），函数命名清晰 -- **命名**: 一致，遵循 Go 惯例 -- **函数大小**: PutNode ~70 行偏长但逻辑连贯，replayWAL ~80 行 switch 可接受 -- **重复代码**: grow 函数有些重复但通过 `remapFile` 提取了公共逻辑 -- **锁顺序**: PutNode 中显式遵守 muGraph → muNodes 顺序（line 53-69 注释），符合 Phase 1 review 要求 - ---- - -## 结论 - -**建议：Approve with conditions** - -Phase 2 实现完整度高，核心写入路径 + WAL + Batch + Grow 均已实现且测试覆盖良好。此前 4 个 P0 已在本 PR 中修复。 - -**合并前须修复：** -1. P1-1: replayWAL NodeCount 漂移（数据正确性风险） -2. P1-5: WAL payload length 无上限校验（OOM 风险） - -**建议后续修复（可在 Phase 3/4）：** -- P1-3: loadIdmap 双重 open -- P1-4: idmap.dat 缺少 header -- P2-5: replay INSERT 不恢复 idmap -- P2-6: SetNodeMapping 无 WAL 保护（在 Phase 4 crash recovery 中统一处理） diff --git a/docs/review/HAY-007-phase3-review-r2.md b/docs/review/HAY-007-phase3-review-r2.md deleted file mode 100644 index 778f0bcb..00000000 --- a/docs/review/HAY-007-phase3-review-r2.md +++ /dev/null @@ -1,102 +0,0 @@ -# HAY-007 Phase 3 — Code Review R2 (Post P1 Fixes) - -**PR**: #56 -**Branch**: `feat/hay-007-phase3-checkpoint` -**Reviewer**: Pegasus (Claude) -**Date**: 2026-04-18 -**Verdict**: **APPROVE** - ---- - -## Checklist Summary - -| # | Area | Verdict | Notes | -|---|------|---------|-------| -| 1 | Design (Checkpoint + Recovery) | PASS | msync → writeMeta → WAL Reset → compactIdmap ordering correct | -| 2 | Test Coverage (8 crash + crashAfterMeta) | PASS | All 8 crash scenarios + dedicated crashAfterMeta test | -| 3 | Error Handling | PASS | P1 fix: idmapFile.Close() error handled | -| 4 | Code Quality | PASS | Clean, well-structured | -| 5 | Performance | PASS | No unnecessary allocations in hot path | -| 6 | Security | PASS | No injection vectors; crash hooks unexported + nil-guarded | -| 7 | Documentation | PASS | Comments adequate | - ---- - -## P1 Fix Verification - -### 1. crashAfterMeta hook has test? **YES** - -`mmap_checkpoint_test.go:614` — `TestCrashPoint_AfterMeta` explicitly sets `s.crashAfterMeta` and panics after `writeMetaHeader` but before WAL truncate. Correctly split from the old `TestCrashPoint_BeforeTruncate` (now at line 659) which tests `crashBeforeTruncate`. Both hooks are exercised independently. - -### 2. compactIdmap Close error handling? **YES** - -`mmap_store_write.go:468` — `s.idmapFile.Close()` error is checked. On failure, `nf` (new file handle) is closed to prevent leak, and the error is returned wrapped. Correct. - -### 3. muDoc.Lock during idmapFile replacement? **YES** - -`mmap_store_write.go:467-474` — `s.muDoc.Lock()` is acquired before `s.idmapFile.Close()` + `s.idmapFile = nf` and released after. This prevents a concurrent `SetNodeMapping` from writing to the stale fd. The earlier read phase correctly uses `s.muDoc.RLock()` (line 420). - ---- - -## Detailed Review - -### Design Correctness - -**Checkpoint ordering** (`checkpointLocked`): -1. `syncAll()` — msync all 4 mmap regions -2. `writeMetaHeader` — persist `WalCheckpointLSN` -3. `wal.Reset()` — truncate WAL (LSN preserved in memory) -4. `compactIdmap()` — rewrite idmap.dat atomically via tmp+rename - -This is the correct crash-safe ordering. If crash occurs: -- After step 1 but before 2: WAL still has records, they replay idempotently (mmap already has the data, meta still has old LSN) -- After step 2 but before 3: WAL has stale records, LSN filter in `Replay` skips them (LSN <= WalCheckpointLSN) -- After step 3 but before 4: idmap is stale but still valid (append-only, compaction is optimization only) - -**Post-replay checkpoint** (`mmap_store.go:167-174`): After WAL replay during Open, if `wal.LSN() > meta.WalCheckpointLSN`, a checkpoint is triggered. This persists recovered state and prevents re-replay on next Open. Error handling properly cleans up all resources. - -**Auto-checkpoint** (`maybeCheckpoint`): Increments ops counter, triggers checkpoint when threshold reached. Skipped in batch mode (deferred to `CommitBatch`). `CommitBatch` checks threshold after sync. - -**Close** delegates to `checkpointLocked()` — clean, no duplication of msync/writeMeta logic. - -### Test Coverage - -| Test | Crash Point | Hook | Verified | -|------|------------|------|----------| -| 6a `TestCrashPoint_AfterWALWrite` | After WAL append, before mmap write | `crashAfterWALWrite` | WAL replay recovers node | -| 6b `TestCrashPoint_AfterMsync` | After msync, before meta write | `crashAfterMsync` | WAL replays all data | -| 6c `TestCrashPoint_AfterMeta` | After meta write, before WAL truncate | `crashAfterMeta` | LSN filter skips stale WAL records | -| 6c-b `TestCrashPoint_BeforeTruncate` | Before WAL truncate | `crashBeforeTruncate` | Same as 6c, different hook | -| 6d `TestCrashPoint_PartialWALRecord` | Partial bytes appended | Manual corruption | Good records recovered, junk ignored | -| 6e `TestCrashPoint_PartialMeta` | meta.bin truncated to 32 bytes | Manual corruption | Open returns error | -| 6f `TestCrashPoint_GrowMidWrite` | 1030 nodes (triggers grow), crash | simulateCrash | Replay re-grows and recovers | -| 6g `TestCrashPoint_SetNeighborsCrash` | After SetNeighbors WAL | simulateCrash | Neighbors recovered | -| 6h `TestCrashPoint_DeleteNodeCrash` | After DeleteNode WAL | simulateCrash | Tombstone recovered | - -Additional: -- `TestKill9Recovery_E2E` — 1000 vectors + neighbors + deletes, skip Close, full verification -- `TestCheckpoint_MetaAndWALTruncated` — basic checkpoint behavior -- `TestCheckpoint_ContinueWriteAndReplay` — write after checkpoint, verify replay -- `TestCheckpoint_LSNMonotonic` — LSN increases after Reset -- `TestAutoCheckpoint_*` (3 tests) — interval trigger, below threshold, batch mode -- `TestCrashRecovery_BasicReplay` / `AfterCheckpoint` — integration recovery tests - -### Potential Issues (All P2 — Non-blocking) - -**P2-1: `compactIdmap` tmp file not cleaned on rename failure** -`mmap_store_write.go:458` — if `os.Rename(tmp, path)` fails, the `.tmp` file is left on disk. Minor: rename rarely fails on same filesystem. - -**P2-2: `Close()` calls `checkpointLocked()` without holding `muWrite`** -`mmap_store.go:189` — `Close()` does not acquire `muWrite`. If a concurrent write is in progress when Close is called, there's a potential race. This is acceptable if the contract is "no concurrent operations during Close" (standard Go pattern), but worth documenting. - -**P2-3: `rebuildNodeCount` on every replay** -`mmap_store.go:505-507` — After replay, `rebuildNodeCount` scans all slots. For large indexes this could be slow. Acceptable for correctness-first approach; can optimize later with WAL-based count tracking. - -**P2-4: Double sync in CommitBatch + auto-checkpoint path** -When `CommitBatch` syncs (line 322) and then triggers `checkpointLocked` (line 327), `syncAll()` runs again inside checkpoint. Minor performance cost, not a correctness issue. - ---- - -## Verdict - -**APPROVE** — All P1 fixes verified. Design is crash-safe with correct ordering. Test coverage is comprehensive across all 8 crash scenarios plus the new `crashAfterMeta` hook. No P0 or P1 issues remaining. P2 items are non-blocking and can be addressed in future iterations. diff --git a/docs/review/HAY-007-phase3-review.md b/docs/review/HAY-007-phase3-review.md deleted file mode 100644 index f2a84f6e..00000000 --- a/docs/review/HAY-007-phase3-review.md +++ /dev/null @@ -1,125 +0,0 @@ -# HAY-007 Phase 3 Review — 崩溃恢复 + Checkpoint - -> Reviewer: Claude (strict) -> PR: #56 -> Date: 2026-04-18 -> Branch: `feat/hay-007-phase3-checkpoint` - ---- - -## Spec 对照 - -| Spec 要求 | 实现状态 | 备注 | -|-----------|---------|------| -| WAL checkpoint 机制（LSN-based） | ✅ 已实现 | `checkpointLocked()` — msync → writeMeta → WAL Reset | -| meta.bin 原子写（tmp+fsync+rename） | ✅ 已实现 | `writeMetaHeader`: Create tmp → binary.Write → Sync → Close → Rename | -| 崩溃恢复：读 meta → 从 lastLSN+1 replay WAL → CRC 校验 | ✅ 已实现 | `OpenMmapStore` → `replayWAL` → post-replay checkpoint | -| 5 类 crash point 测试 | ✅ 8 类（超出 spec） | 6a–6h 覆盖 WAL write / msync / meta / partial WAL / partial meta / grow / SetNeighbors / DeleteNode | -| kill -9 后重启索引完整 | ✅ 已实现 | `TestKill9Recovery_E2E` — 1000 vectors + neighbors + deletes | -| Checkpoint 在 Close 时触发 | ✅ 已实现 | `Close()` 调用 `checkpointLocked()` | -| Checkpoint 在 N 次操作后触发 | ✅ 已实现 | `maybeCheckpoint()` + `CheckpointInterval` option（默认 1000） | -| WAL 截断安全 | ✅ 已实现 | `WAL.Reset()` — Flush → Truncate(0) → Seek(0) → buf.Reset，LSN 保持单调 | -| CRC32 校验 | ✅ 已实现 | WAL record CRC + idmap entry CRC | - ---- - -## 发现项 - -### P0 — 无 - -无阻塞性问题。 - -### P1 — 需要关注 - -#### P1-1: `crashAfterMeta` hook 已声明但未测试 - -`mmap_store.go:80` 声明了 `crashAfterMeta func()`，`checkpointLocked()` 中已 wired（line 391-392），但 **没有任何测试用例实际设置并触发该 hook**。Test 6c (`TestCrashPoint_AfterMeta`) 测试的是 `crashBeforeTruncate`（meta 写完、WAL 未截断），而非 `crashAfterMeta`。 - -这意味着 "meta 写完但 WAL 截断前崩溃" 和 "msync 后 meta 写完前崩溃" 两个场景都有测试（6b, 6c），但 `crashAfterMeta` 这个 hook 本身是死代码（测试角度）。 - -**建议**: 要么在某个测试中使用 `crashAfterMeta`，要么删除它。当前 6c 用 `crashBeforeTruncate` 已覆盖了相同的时间窗口（meta 完成后、truncate 前），所以逻辑上没有遗漏，但 hook 是多余的。 - -#### P1-2: `compactIdmap` 先 Close 旧 file 后赋值新 file 的顺序风险 - -```go -// mmap_store_write.go:467-468 -s.idmapFile.Close() -s.idmapFile = nf -``` - -如果 `Close()` 返回错误（如 EIO），代码忽略了该错误。虽然 idmap 的 tmp+fsync+rename 保证了数据完整性，但 **Close 错误应该被记录或返回**，尤其在 fsync-on-close 的文件系统上。 - -#### P1-3: Checkpoint 期间的并发读安全 - -`checkpointLocked()` 持有 `muWrite`，但 `compactIdmap` 中对 `s.idmapFile` 的替换（close 旧 → 赋值新）不受 `muDoc` 写锁保护。如果有并发的 `SetNodeMapping` 调用在 `compactIdmap` 执行期间写入旧的 `idmapFile`，可能写入已关闭的 fd。 - -当前由于 HNSW `h.mu` 串行化了所有写操作，这不会在生产中触发。但 `checkpointLocked` 是一个 exported 行为（通过 `Checkpoint()`），如果未来有人在 HNSW 锁外调用，就会有问题。 - -**建议**: 在 `compactIdmap` 替换 `idmapFile` 时持有 `muDoc` 写锁。 - -### P2 — 建议 - -#### P2-1: `TestAutoCheckpoint_TriggeredAtInterval` WAL size 断言不够严格 - -```go -// mmap_checkpoint_test.go:239-242 -if info.Size() == 0 { - // WAL could be 0 if exactly at threshold ... -} -``` - -这个 if 分支体为空，既不 fail 也不 skip。如果 WAL 意外为空，测试静默通过。应断言 WAL size > 0（因为 ops 11-15 应在 WAL 中），或至少加个 `t.Log` 说明。 - -#### P2-2: `simulateCrash` 中 `wal.Sync()` 可能掩盖真实崩溃场景 - -所有 crash 模拟都先 `s.wal.Sync()`，确保 WAL 数据落盘。真实 kill -9 场景中，如果 WAL 使用了 buffered writer，最后几条记录可能丢失。当前测试保守偏向 "WAL 完整" 场景，缺少 "WAL 尾部丢失" 的测试（6d 部分 WAL 只测了追加垃圾字节，没测 truncated-mid-record）。 - -实际上 6d 追加了 5 字节垃圾模拟 partial record，这已经覆盖了部分场景。但真正的 buffered write 丢失（WAL 合法记录被截断到一半）没有直接测试。 - -#### P2-3: `WAL.Reset()` 未 fsync truncate - -```go -func (w *WAL) Reset() error { - w.buf.Flush() - w.file.Truncate(0) - w.file.Seek(0, io.SeekStart) - w.buf.Reset(w.file) - return nil -} -``` - -Truncate 后没有 `Sync()`。如果 Reset 后立即崩溃，文件系统可能还保留旧的 WAL 数据。由于 meta.bin 已经记录了新的 checkpoint LSN，replay 时会通过 LSN 过滤跳过旧记录，所以**功能上不受影响**。但如果追求 defense-in-depth，可以加个 `w.file.Sync()`。 - -#### P2-4: `nocov` 使用合规 - -PR 中新增代码没有添加新的 `nocov` 标记。已有的 `nocov` 标记（`replayWAL` error path、`cleanup`、`closeMmaps`）均在不可达/极难触发的错误路径上，合规。 - -#### P2-5: 测试中直接访问内部字段 - -`TestCrashPoint_DeleteNodeCrash` 和 `TestKill9Recovery_E2E` 直接读取 `s2.nodes[offset]` 来检查 tombstone flags。这使测试与内部布局紧耦合。建议考虑添加一个 `IsDeleted(id)` 方法供测试使用。不阻塞合入。 - ---- - -## 性能 - -- **Checkpoint 开销**: msync 4 个 mmap region + 1 次 meta.bin 原子写 + 1 次 WAL truncate + 1 次 idmap compact（排序 + 顺序写）。对 50K 节点，idmap compact 约 50K × ~30B = 1.5MB 顺序写，< 10ms。 -- **默认间隔 1000 ops**: HNSW Insert 产生 ~10 WAL records（1 INSERT + ~8 SET_NEIGHBORS + 1 SET_ENTRY），约每 100 次 Insert 触发一次 checkpoint。合理。 -- **Batch mode**: checkpoint 延迟到 CommitBatch，避免 batch 内多次 checkpoint。正确。 - ---- - -## 安全 - -- CRC32 校验覆盖 WAL record 全部字段（LSN + Length + Type + Payload）。✅ -- CRC32 校验覆盖 idmap entry。✅ -- meta.bin 原子写防止部分写入。✅ -- 不完整/CRC 失败的 WAL record 在 replay 时被丢弃（truncate at last valid offset）。✅ -- crash hook 字段为 `func()` 类型，nil 检查保护，生产零开销。✅ - ---- - -## 结论 - -**LGTM with minor comments**。P1-1 是死代码（未使用的 hook），P1-2/P1-3 是防御性编程建议，不影响当前正确性。8 类 crash test + kill-9 E2E 验收测试覆盖充分，超出 spec 要求的 5 类。Checkpoint 机制（msync → atomic meta → WAL truncate）顺序正确，crash 在任何步骤之间都不会导致数据丢失。 - -建议合入后跟进 P1 items。 diff --git a/docs/review/HAY-007-phase4-review.md b/docs/review/HAY-007-phase4-review.md deleted file mode 100644 index 2c3823cd..00000000 --- a/docs/review/HAY-007-phase4-review.md +++ /dev/null @@ -1,193 +0,0 @@ -# HAY-007 Phase 4 Review — HNSW + MmapStore Integration & E2E - -> Reviewer: Claude Code (Opus 4) -> PR: #57 -> Date: 2026-04-18 -> Files: `docs/tasks/HAY-007-phase4-tasks.md`, `mmap_store_hnsw_test.go` (new, +1071), `mmap_store_bench_test.go` (+357/-0) -> Verdict: **REQUEST CHANGES** (2 P0, 5 P1) - ---- - -## Summary - -Tests-only PR. No production code changes. The PR adds comprehensive integration tests for MmapStore as an HNSW backend, covering CRUD, persistence, WAL replay crash recovery, upper-graph multi-layer verification, recall@10 validation, export integration, and SIFT-128 50K benchmarks. Test design quality is high — real assertions, proper baseline comparisons, and realistic crash simulation. - ---- - -## 1. Design Conformance (vs. docs/design/HAY-007-mmap-store.md Phase 5) - -| Phase 5 Requirement | Covered? | Notes | -|---|---|---| -| graph_upper.dat 按需分配验证 | Yes | Task 5: multi-layer, persistence, grow+crash | -| 替换 HNSW 中的 PebbleStore 为 MmapStore | Deferred (Task 7) | Correct — scope declaration explicitly defers production switch | -| E2E 测试 (SIFT benchmark) | Yes | Task 3: 50K SIFT fvecs/ivecs loader, MemStore vs MmapStore | -| Recall 验证 | Yes | Task 4: brute-force ground truth, >0.95 threshold, <0.01 diff | -| 全部现有测试通过 | Not verified | No CI evidence in PR; Go not available in review env | - -**Assessment**: Good alignment. The scope split (Task 7 deferral) is explicitly documented and reasonable. - ---- - -## 2. Test Coverage Analysis - -### Task 1: HNSW + MmapStore CRUD -| Test | Real assertions? | Verdict | -|---|---|---| -| `TestMmapHNSW_InsertSearch` | 100 inserts, k=10, sorted distance check | Good | -| `TestMmapHNSW_InsertDeleteSearch` | Deleted doc mappings verified via `GetNodeId` | Good | -| `TestMmapHNSW_DeleteReinsert` | Freelist reuse, search correctness post-reinsert | Good | -| `TestMmapHNSW_Upsert` | Upsert same docId, distance ~0 check | Good | - -### Task 2: Persistence -| Test | Real assertions? | Verdict | -|---|---|---| -| `TestMmapHNSW_PersistenceReopen` | Bit-exact result comparison pre/post close | Good | -| `TestMmapHNSW_PersistenceReopenContinueInsert` | Node count assertion, search post-reopen | Good | -| `TestMmapHNSW_PersistenceReopenDelete` | Delete after reopen, mapping verification | Good | -| `TestMmapHNSW_WALReplayE2E` | Crash sim (no Close), WAL replay, bit-exact match | Good | - -### Task 3: 50K SIFT-128 Benchmark -| Test | Real assertions? | Verdict | -|---|---|---| -| `BenchmarkHNSW_MemStore_50K_Insert` | Reports total_sec, inserts/sec | Good | -| `BenchmarkHNSW_MmapStore_50K_Insert` | 90s hard-fail gate | Good | -| `BenchmarkHNSW_Search_MemStore_vs_MmapStore` | p50/p99 latencies reported | See P1-3 | - -### Task 4: Recall@10 -| Test | Real assertions? | Verdict | -|---|---|---| -| `TestMmapHNSW_RecallAt10` | Brute-force ground truth, >0.95, <0.01 diff | Good | -| `TestRecallAt10_SIFT` (bench file) | SIFT ground truth, same assertions | Good | - -### Task 5: Upper Graph -| Test | Real assertions? | Verdict | -|---|---|---| -| `TestMmapHNSW_UpperGraph_MultiLayer` | Neighbor level invariant, MemStore baseline compare | Good | -| `TestMmapHNSW_UpperGraph_PersistenceReopen` | EP/maxLevel restore, bit-exact results | Good | -| `TestMmapHNSW_UpperGraph_GrowCrashRecovery` | WAL replay, upper slot leak check | Good | - -### Task 6: Export Integration -| Test | Real assertions? | Verdict | -|---|---|---| -| `TestMmapHNSW_ExportRecall` | Recall >0.95, bit-exact vs MemStore | Good | -| `TestMmapHNSW_ExportThenInsertDelete` | Insert+delete post-export, node count | Good | - ---- - -## 3. Findings - -### P0 — Must Fix - -**P0-1: WAL replay crash simulation writes meta but doesn't test partial-meta crash** - -`TestMmapHNSW_WALReplayE2E` (line ~490-530) and `TestMmapHNSW_UpperGraph_GrowCrashRecovery` both call `writeMetaHeader()` + `syncAll()` before simulating crash. This means the mmap data files AND meta are fully consistent — the WAL replay is effectively a no-op (all data is already in the mmap files). The test passes vacuously. - -A real crash scenario would have WAL entries that are NOT yet reflected in the data files. The current simulation doesn't test WAL replay at all — it tests "reopen with fully-synced data." - -**Fix**: Either (a) don't call `syncAll()` — only flush the WAL, so mmap pages may not be persisted; or (b) insert additional records AFTER the syncAll/writeMeta to create genuine un-checkpointed WAL entries; or (c) add a test that explicitly corrupts/truncates a data file region to force WAL replay to reconstruct it. - -**P0-2: `TestMmapHNSW_UpperGraph_MultiLayer` comment says 5000 but n=2000** - -Line ~541 comment: `"with 5000 nodes, maxLevel should be > 0"` but `n = 2000`. The assertion still likely passes (2000 nodes almost always produces level>0 nodes), but the misleading comment suggests the constant was changed without updating the comment, raising the question of whether test parameters were tuned down to hide a failure. - -**Fix**: Update comment to match `n=2000`, or restore `n=5000` per the task spec which says "≥5000 vectors." - -### P1 — Should Fix - -**P1-1: bench_test.go `TestRecallAt10_SIFT` is in benchmark build tag but is a `Test` function** - -`mmap_store_bench_test.go` has `//go:build benchmark` at the top. The function `TestRecallAt10_SIFT` is a regular test, not a benchmark. It will never run in `go test ./...` (no benchmark tag) and will never run via `go test -bench=.` (it's a Test, not Benchmark). This test is dead code in normal CI. - -**Fix**: Move `TestRecallAt10_SIFT` to `mmap_store_hnsw_test.go` (no build tag), or change the build constraint. - -**P1-2: `recallAtK` in bench file vs `recallAtKMapped` in hnsw_test — duplicate logic** - -Two recall-at-K implementations exist: `recallAtK` (bench_test.go:406) uses `SearchResult.ID` directly as index, while `recallAtKMapped` (hnsw_test.go:838) uses a node→baseIdx mapping. The direct-ID version (`recallAtK`) is incorrect when node IDs don't equal base vector indices (e.g., after deletes or freelist reuse), making `TestRecallAt10_SIFT` results unreliable. - -**Fix**: Use `recallAtKMapped` consistently, or verify that in the SIFT benchmark node IDs always equal insertion order (and add an assertion for this invariant). - -**P1-3: Search benchmark doesn't assert p99 < 5ms** - -`BenchmarkHNSW_Search_MemStore_vs_MmapStore` reports p99 but doesn't fail if it exceeds the 5ms target from the design doc. The insert benchmark has a hard 90s gate; the search benchmark should have an equivalent. - -**Fix**: Add `if p99 > 5*time.Millisecond { b.Fatalf(...) }` for the MmapStore sub-benchmark. - -**P1-4: Crash simulation leaks resources without `t.Cleanup`** - -In `TestMmapHNSW_WALReplayE2E` and `TestMmapHNSW_UpperGraph_GrowCrashRecovery`, the manual resource cleanup (`mmapFree`, `file.Close`) can leave leaked file descriptors if any step panics before all cleanup calls execute. Using `t.Cleanup` registered before the crash sim block would be more robust. - -**Fix**: Register cleanup via `t.Cleanup(func() { ... })` before opening the store, or accept the risk (test-only code). - -**P1-5: `TestMmapHNSW_RecallAt10` uses n=2000 not n=5000 as stated in task spec** - -Task spec says "Uses 5000 random 128d vectors" but the test uses `n = 2000`. Recall with 2000 vectors is a weaker signal than with 5000+. The smaller dataset may mask issues that only manifest at scale (e.g., grow-related corruption). - -**Fix**: Increase to `n = 5000` per spec, or document why 2000 was chosen. - -### P2 — Suggestions - -**P2-1: Missing `//go:build !windows` or equivalent for crash simulation tests** - -The crash simulation tests use `mmapFree` and direct file handle manipulation that may behave differently on Windows (mmap files can't be deleted while mapped). Consider adding a build tag or `runtime.GOOS` skip. - -**P2-2: Benchmark data dependency on external SIFT dataset** - -SIFT benchmarks skip silently when data is absent (`b.Skip`). The task spec says "禁止使用合成数据" but there's no CI step to download SIFT data. Consider documenting the download step in a Makefile target or `testdata/sift/README.md`. - -**P2-3: `runtime.GC()` in searchBench** - -Calling `runtime.GC()` inside the benchmark loop (line ~399) can skew latency measurements if GC runs during timed operations. Move it outside the timed section or remove it. - ---- - -## 4. nocov Usage Audit - -All `nocov` annotations are in production code (`mmap_store.go`, `store.go`), not in test files. Each is on error paths that require hardware/OS-level failures: - -| Location | Justification | Verdict | -|---|---|---| -| `mmap_store.go:158` WAL replay error | Requires corrupt WAL that passes CRC but fails replay | Acceptable | -| `mmap_store.go:168` post-replay checkpoint | Requires msync/rename to fail | Acceptable | -| `mmap_store.go:288` partial mmap cleanup | Requires partial mmap alloc failure | Acceptable | -| `mmap_store.go:333` closeMmaps helper | Called from Close/error; covered indirectly | Acceptable | -| `store.go:510,552,570` PebbleStore errors | Pebble internal error (not ErrNotFound) | Acceptable | - -**Verdict**: nocov usage is compliant — all on genuinely untestable error paths. - ---- - -## 5. Performance Assessment - -- 50K insert benchmark has a hard 90s gate — good. -- Search benchmark reports p50/p99 but doesn't enforce the 5ms target — see P1-3. -- SIFT data loaders (`loadSiftFvecs`, `loadSiftIvecs`) are correctly implemented per standard fvecs/ivecs format. -- `runtime.GC()` in search benchmark may introduce noise — see P2-3. - ---- - -## 6. Security - -No concerns. Tests-only PR, no user input handling, no network access, no credential handling. - ---- - -## 7. Documentation - -- Task breakdown doc (`HAY-007-phase4-tasks.md`) is thorough and well-structured. -- Task 7 scope declaration is explicit and appropriate. -- Comment/code mismatch in P0-2 should be fixed. - ---- - -## Verdict: **REQUEST CHANGES** - -### Must Fix Before Merge -1. **P0-1**: WAL replay crash sim tests don't actually test WAL replay (data is fully synced before "crash") -2. **P0-2**: n=2000 vs comment says 5000 — verify intent and fix - -### Should Fix -3. **P1-1**: `TestRecallAt10_SIFT` is dead code under `//go:build benchmark` -4. **P1-2**: `recallAtK` may compute incorrect recall when nodeID != insertion index -5. **P1-3**: Search benchmark needs p99 < 5ms hard gate -6. **P1-4**: Crash sim resource cleanup should use `t.Cleanup` -7. **P1-5**: Recall test uses n=2000, spec says 5000 diff --git a/docs/tasks/BOARD.md b/docs/tasks/BOARD.md deleted file mode 100644 index 1aec2910..00000000 --- a/docs/tasks/BOARD.md +++ /dev/null @@ -1,32 +0,0 @@ -# Haystack Task Board - -## Done ✅ -| ID | 任务 | Owner | PR | 完成日期 | -|----|------|-------|----|---------| -| HAY-001 | 覆盖度提升 90% | Dev | — | 2026-04-04 | -| HAY-002 | 多语言索引（gse + 停用词） | Dev | #35/#37/#38/#39/#40 | 2026-04-06 | -| HAY-003 | 文件统计不准（状态管理重构） | Dev | #31/#32/#33/#34 | 2026-04-05 | -| HAY-004a | HNSW 设计定稿 | 飞马 | — | 2026-04-14 | -| HAY-004b | NodeStore + Pebble 持久化 | Dev | #42 | 2026-04-14 | -| HAY-004c | HNSW 算法核心（内存 mock） | Dev | #42 | 2026-04-14 | -| HAY-004d | Pebble 替换 mock | Dev | #44 | 2026-04-14 | -| HAY-004e | 并发安全 | Dev | #46 | 2026-04-14 | -| HAY-004f | 性能验证 & E2E（参数化 benchmark + SIFT 真实数据验证） | Dev | #49 | 2026-04-17 | -| HAY-004f-qa | QA 验收：HNSW 性能 & E2E | QA | #50 | 2026-04-17 | -| HAY-004g | HNSW Insert upsert 修补（重复 docId 孤儿节点） | Dev | #51 | 2026-04-17 | -| HAY-006 | 补充 upsert 测试场景 | Dev | #52 | 2026-04-17 | -| HAY-008 | core 剥离（搜索内核独立 module，P1-P5） | Dev | PR #62 | 2026-06-12 | - -## In Progress 🔵 -| ID | 任务 | Owner | Branch | 说明 | -|----|------|-------|--------|------| -| HAY-007 | mmap flat file 存储引擎 | Dev | PR #57 | Phase 4 完成（测试/E2E/集成），等飞马 review | - -## Backlog 📋 -（空） - -## 设计文档 -- [HAY-004 HNSW 设计](HAY-004-hnsw/design.md) -- [coder/hnsw 审查报告](HAY-004-hnsw/hnsw-deep-review.md) -- [HAY-007 MmapStore 设计](../design/HAY-007-mmap-store.md) -- HAY-008 core 剥离：见 PR #62（持久设计文档待补,详见 §docs 重整） diff --git a/docs/tasks/HAY-004-hnsw/TASK.md b/docs/tasks/HAY-004-hnsw/TASK.md deleted file mode 100644 index fa9d9fcc..00000000 --- a/docs/tasks/HAY-004-hnsw/TASK.md +++ /dev/null @@ -1,39 +0,0 @@ -# HAY-004 HNSW 向量搜索引擎 - -## 状态 -- Status: design -- Owner: Architect（负责方案设计） -- Created: 2026-04-06 - -## 背景 -建军在 2026-04-04 提出：引入 HNSW 数据库用于向量搜索。 - -## 需求 -- Go 原生 HNSW 实现 -- 内存占用低 -- 数据持久化到磁盘（Pebble） -- 用时动态加载 - -## 目标 -- 设计生产可用的 HNSW + Pebble 持久化方案 -- 明确性能目标和验收标准 - -## 进展 -- haystack-hnsw 项目不存在，确认为从零设计 -- 完成架构评估：haystack 现有存储层（PebbleDB）可直接复用 -- 完成技术调研：Go HNSW 生态、持久化策略、性能预估 -- **设计文档初稿完成**（`design.md`） - - 推荐方案：基于 coder/hnsw 算法 + Pebble 持久化改造 - - 4 阶段实施路径 - - 性能目标：搜索 < 20ms, 插入 < 5ms - -## Next Step -等建军 review 设计文档并拍板。 - -## 讨论点（待建军决策） -1. Embedding 来源：本地模型 vs API？影响维度（384 vs 1536） -2. 删除策略：标记删除 + 定期重建 vs 即时重连？ -3. 是否需要过滤搜索（按文件类型等）？ - -## 依赖 -- 无 diff --git a/docs/tasks/HAY-004-hnsw/design.md b/docs/tasks/HAY-004-hnsw/design.md deleted file mode 100644 index 684149e0..00000000 --- a/docs/tasks/HAY-004-hnsw/design.md +++ /dev/null @@ -1,394 +0,0 @@ -# HAY-004 HNSW 向量搜索引擎设计文档 - -> 作者：Architect | 日期：2026-04-06 -> 状态：初稿，待讨论 - -## 0. 前提 - -`~/workspace/haystack-hnsw` 或任何 HNSW 初步版本**不存在**。本设计为从零方案。 - ---- - -## 1. 现状与切入点 - -### Haystack 现有架构 - -``` -internal/core/ -├── pebble/ # KV 抽象（DB / Batch interface） -├── storage/ # 存储门面（版本管理、KeyType 常量） -├── documents/ # 文档 CRUD（Document 结构体） -├── invertedindex/ # 倒排索引（关键词→文档ID） -├── symbols/ # 符号索引（函数名等） -├── idtable/ # 关键词→整数ID 映射 -└── workspace/ # 工作区生命周期 -``` - -**存储模式**：所有模块共享同一个 PebbleDB 实例，通过 byte 前缀分区隔离 key 空间。 -**初始化模式**：`Init(db pebble.DB, mpsc *queue.Mpsc)` + 包级全局变量。 -**当前 Document 结构**：无向量字段，纯文本+元数据。 - -### Key 空间分区（现有） - -``` -1-2: Workspace -10-13: Document -20-22: Inverted Index -28-29: ID Table -30-33: Symbols -``` - -**结论**：新增 `vectorindex/` 模块完全可融入现有架构，无需新开数据库。 - ---- - -## 2. 设计目标 - -| 目标 | 要求 | -|------|------| -| **功能** | 支持向量相似搜索（ANN），用于语义代码搜索 | -| **算法** | HNSW（Hierarchical Navigable Small World） | -| **实现** | 纯 Go，零 cgo 依赖 | -| **持久化** | Pebble，与现有存储统一 | -| **内存** | 懒加载，按需缓存，不全量加载 | -| **规模** | 单 workspace 万~十万级文档，384-1536 维向量 | -| **延迟** | 搜索 < 20ms @10 万向量（384 维） | -| **一致性** | 插入/删除增量持久化，crash-safe | - ---- - -## 3. 技术选型 - -### 3.1 算法实现：基于 coder/hnsw 改造 - -**不从零实现 HNSW 算法。** `github.com/coder/hnsw`（纯 Go、泛型、~600 行核心代码、Coder 出品、活跃维护）提供了经过验证的算法层。 - -| 方案 | 判断 | -|------|------| -| coder/hnsw 改造 | ✅ **推荐** — 算法成熟，只需替换存储层 | -| TFMV/hnsw（coder fork） | 🔶 备选 — 加了 metadata/faceted search，但引入不必要的复杂度 | -| hnswlib-go（cgo） | ❌ — cgo 依赖，与 haystack 零 cgo 目标冲突 | -| 从零实现 | ❌ — 无必要，~1500 行核心代码 + 边界 case 调试成本高 | - -**改造范围**： -- 提取核心算法（插入、搜索、删除、层级管理） -- 替换内存 map 存储为 Pebble-backed NodeStore 接口 -- 保留距离函数接口（支持 cosine / L2 / dot product） - -### 3.2 持久化：Pebble KV - -**不用 mmap、不用自定义二进制格式。** - -| 方案 | 优势 | 劣势 | 结论 | -|------|------|------|------| -| **PebbleDB** | 增量持久化、事务安全、与现有存储统一、支持懒加载 | LSM 读放大（可缓存缓解） | ✅ 首选 | -| mmap | 零拷贝读取、OS 页缓存 | 自管文件布局、扩容复杂、跨平台差异 | ❌ 过重 | -| 二进制格式 | 紧凑、快速全量加载 | 无增量更新、必须全量加载 | ❌ 不满足懒加载需求 | - ---- - -## 4. 模块设计 - -### 4.1 位置 - -``` -internal/core/vectorindex/ -├── hnsw.go # HNSW 算法核心（搜索、插入、删除） -├── store.go # NodeStore 接口 + Pebble 实现 -├── cache.go # LRU 缓存层 -├── distance.go # 距离函数（cosine, L2, dot） -├── types.go # Node, Vector, SearchResult 类型 -├── init.go # Init() + 生命周期管理 -└── hnsw_test.go # 测试 -``` - -### 4.2 核心接口 - -```go -// ============== 对外 API ============== - -// Init 初始化向量索引模块 -func Init(db pebble.DB) error - -// AddVector 插入或更新文档的向量 -func AddVector(tableId int, docId string, vector []float32) error - -// DeleteVector 删除文档的向量 -func DeleteVector(tableId int, docId string) error - -// Search 向量相似搜索，返回 top-k 最近邻 -func Search(tableId int, query []float32, k int) ([]SearchResult, error) - -// DeleteTable 删除 workspace 对应的整个向量索引 -func DeleteTable(tableId int) error - -// ============== 内部抽象 ============== - -// NodeStore 节点存储接口——隔离算法与持久化 -type NodeStore interface { - GetVector(id uint64) ([]float32, error) - PutNode(id uint64, level int, vector []float32) error - DeleteNode(id uint64) error - GetNeighbors(id uint64, layer int) ([]uint64, error) - SetNeighbors(id uint64, layer int, neighbors []uint64) error - GetEntryPoint() (uint64, int, error) // (nodeId, maxLayer, err) - SetEntryPoint(id uint64, maxLayer int) error - GetNodeLevel(id uint64) (int, error) - // ID 映射：docId ↔ nodeId - GetNodeId(docId string) (uint64, bool, error) - SetNodeMapping(docId string, nodeId uint64) error - DeleteNodeMapping(docId string) error -} -``` - -### 4.3 Pebble Key Schema - -新增 Key 前缀 **byte 40-45**，与现有 key 空间隔离： - -``` -Key Format Value -─────────────────────────────────────── ────────────────────── -40:{tableId}:meta:entry {nodeId}:{maxLayer} (entry point) -40:{tableId}:meta:count uint64 (node count) -41:{tableId}:vec:{nodeId} []float32 (raw vector bytes) -42:{tableId}:node:{nodeId}:level uint8 (node level) -43:{tableId}:node:{nodeId}:nb:{layer} []uint64 (neighbor IDs) -44:{tableId}:map:doc:{docId} uint64 (docId → nodeId) -44:{tableId}:map:node:{nodeId} string (nodeId → docId, 反查) -45:{tableId}:id_seq uint64 (auto-increment node ID) -``` - -**设计考虑**： -- `tableId` 对应 workspace，与倒排索引的 table 概念一致 -- 向量单独存（key 41），因为向量数据量大，读邻接表时不需要读向量 -- 邻接表按层分 key（key 43），支持按层读取，高层节点可独立缓存 -- docId ↔ nodeId 双向映射（key 44），因为对外接口用 docId，内部图用 uint64 nodeId - -### 4.4 缓存策略 - -``` -┌─────────────────────────────────────┐ -│ HNSW Algorithm │ -├─────────────────────────────────────┤ -│ LRU Cache Layer │ -│ ┌────────┐ ┌────────┐ ┌────────┐ │ -│ │Vectors │ │Neighbors│ │ Levels │ │ -│ │(热点) │ │ (全层) │ │ (全量) │ │ -│ └────────┘ └────────┘ └────────┘ │ -├─────────────────────────────────────┤ -│ Pebble NodeStore (持久化) │ -└─────────────────────────────────────┘ -``` - -**分层缓存**： -- **Level 缓存**：全量常驻内存（每个节点 1 byte，10 万节点 = 100 KB） -- **高层邻接表**：常驻内存（层级 ≥ 1 的节点约 n/M ≈ 几千个，几十 KB） -- **底层邻接表**：LRU 缓存，容量可配 -- **向量数据**：LRU 缓存，容量可配（这是内存大头） - -**缓存大小控制**（默认值，可配）： - -| 组件 | 默认容量 | 内存估算（10万/384维） | -|------|---------|---------------------| -| Level 缓存 | 全量 | ~100 KB | -| 高层邻接表 | 全量 | ~200 KB | -| 底层邻接表 LRU | 10,000 条 | ~2.5 MB | -| 向量 LRU | 10,000 条 | ~15 MB | -| **总计** | — | **~18 MB** | - -对比全量加载 ~176 MB（384 维）或 ~626 MB（1536 维），缓存模式节省 90%+ 内存。 - -### 4.5 HNSW 参数 - -| 参数 | 默认值 | 含义 | -|------|--------|------| -| M | 16 | 每层最大连接数 | -| M_max0 | 32 | 底层最大连接数（= 2M） | -| ef_construction | 200 | 构建时 beam width | -| ef_search | 64 | 搜索时 beam width（可运行时调整） | -| distance | cosine | 距离函数 | - -这些参数对万~十万级数据是标准选择。如果搜索精度不够，优先调大 ef_search（延迟换精度）。 - ---- - -## 5. 数据流 - -### 5.1 索引流程 - -``` -文件变更 → Scanner 检测 - → Tokenizer（倒排索引，已有） - → Embedder（新增：调用 embedding API 生成向量） - → vectorindex.AddVector(tableId, docId, vector) - ├── NodeStore.PutNode（写 Pebble） - ├── HNSW insert（搜索最近邻、建立连接） - ├── NodeStore.SetNeighbors（更新邻接表） - └── 缓存失效/更新 -``` - -### 5.2 搜索流程 - -``` -用户查询 → Embedder 生成 query 向量 - → vectorindex.Search(tableId, queryVec, k) - ├── 从 entry point 开始 - ├── 高层贪心下降（内存缓存命中） - ├── 底层 beam search（按需从 Pebble 加载） - └── 返回 top-k docIds + distances - → 与倒排索引结果融合排序（未来） -``` - -### 5.3 删除流程 - -``` -文件删除 → vectorindex.DeleteVector(tableId, docId) - ├── 标记删除（HNSW 中断开连接、重连邻居） - ├── NodeStore.DeleteNode（删 Pebble 数据） - └── 缓存失效 -``` - ---- - -## 6. 与现有模块的集成点 - -### 6.1 Document 模型扩展 - -**不修改 Document 结构体。** 向量数据量大（384×4=1.5KB / 1536×4=6KB per doc），不适合塞进 Document。向量独立存储在 vectorindex 的 key 空间中，通过 docId 关联。 - -### 6.2 Indexer（Scanner）集成 - -Scanner 当前流程：`扫描文件 → 生成 Document → 更新倒排索引`。 - -扩展点：在生成 Document 后，增加 embedding 步骤 → 调用 `vectorindex.AddVector()`。 - -**注意**：Embedding 生成（调用外部 API 或本地模型）是一个**异步、可能慢**的操作。建议： -- embedding 生成与倒排索引更新解耦 -- 单独的 embedding 队列 + worker -- 倒排索引先更新（保证文本搜索即时可用），向量索引异步跟进 - -### 6.3 Searcher 集成 - -新增搜索模式： -- 纯文本搜索（现有，不变） -- 纯向量搜索（新增） -- 混合搜索（未来，文本 + 向量分数融合） - -### 6.4 storage/types.go 扩展 - -```go -// 新增 Key 类型（byte 40-45） -const ( - KeyTypeVectorMeta byte = 40 - KeyTypeVectorData byte = 41 - KeyTypeVectorNodeLevel byte = 42 - KeyTypeVectorNeighbors byte = 43 - KeyTypeVectorMapping byte = 44 - KeyTypeVectorIdSeq byte = 45 -) -``` - ---- - -## 7. Embedding 生成（设计边界） - -Embedding 生成**不在本任务范围内**，但需要明确接口： - -```go -// Embedder 接口——vectorindex 不关心实现 -type Embedder interface { - Embed(text string) ([]float32, error) - Dimension() int -} -``` - -可能的实现： -- 远程 API（OpenAI、Cohere、本地 ollama） -- 本地模型（ONNX Runtime） - -本任务只关注：**给定向量，HNSW 如何存储、索引、搜索。** - ---- - -## 8. 实施步骤 - -### Step 1：NodeStore + Pebble 实现（基础层） -- 定义 NodeStore 接口 -- 实现 PebbleNodeStore -- Key schema 编解码 -- 单元测试：CRUD 操作正确性 -- **验证**：节点写入、读取、删除、ID 映射正确 - -### Step 2：HNSW 核心算法（移植 + 适配） -- 从 coder/hnsw 提取算法逻辑 -- 适配 NodeStore 接口（替换内存 map） -- 距离函数实现（cosine、L2） -- 单元测试：小规模插入+搜索的正确性 -- **验证**：100 个随机向量，搜索结果与暴力搜索一致 - -### Step 3：LRU 缓存层 -- 实现分层缓存（level / neighbors / vectors） -- 缓存失效策略（写入时更新、删除时清除） -- Benchmark：缓存命中率、搜索延迟 -- **验证**：缓存开/关搜索结果一致，缓存开时延迟显著降低 - -### Step 4：集成到 Haystack -- 新增 `Init()` 函数，注册到 storage 初始化流程 -- `storage/types.go` 新增 KeyType 常量 -- Indexer 集成（embedding 队列 + AddVector） -- Searcher 集成（向量搜索 API） -- 端到端测试 -- **验证**：文件索引后可通过向量搜索找到 - -### Step 5：性能验证 + 调优 -- Benchmark（1K / 10K / 100K 向量，384 / 1536 维） -- 搜索延迟、插入吞吐、内存占用 -- Recall@10 验证（与暴力搜索对比） -- 调整 HNSW 参数和缓存大小 -- **目标**：搜索 < 20ms @100K/384d，Recall@10 > 0.95 - ---- - -## 9. 内存占用估算 - -| 场景 | 全量加载 | 懒加载（默认缓存） | -|------|---------|-------------------| -| 1 万文档 × 384 维 | ~17 MB | ~5 MB | -| 10 万文档 × 384 维 | ~176 MB | ~18 MB | -| 10 万文档 × 1536 维 | ~626 MB | ~60 MB | - -懒加载对高维场景收益巨大。 - ---- - -## 10. 风险 - -| 风险 | 严重度 | 缓解 | -|------|--------|------| -| Pebble 读放大导致搜索延迟偏高 | 🟡 中 | LRU 缓存 + 高层常驻内存，热点路径缓存命中 | -| coder/hnsw 算法改造工作量超预期 | 🟡 中 | 算法层相对独立，改造主要是存储接口适配 | -| 并发安全（多个 workspace 并发索引+搜索） | 🟡 中 | 每个 tableId 独立锁，读写分离 | -| Embedding API 成为瓶颈（不在本任务范围但会影响端到端体验） | 🟢 低 | 异步队列解耦，不阻塞文本索引 | -| 向量维度不一致（不同 workspace 用不同 embedding 模型） | 🟡 中 | tableId 级别存储维度元数据，维度不匹配时报错 | - ---- - -## 11. 待讨论 - -1. **Embedding 模型选择**：哪个模型？什么维度？本地还是远程？（影响维度参数和延迟预期） -2. **混合搜索策略**：文本分数和向量分数如何融合？（RRF？加权？）先做纯向量搜索还是一步到位混合？ -3. **是否需要 per-workspace 开关**：有些 workspace 可能不需要向量搜索（节省 embedding 成本） -4. **VSCode 插件影响**：向量搜索需要新增 API endpoint，插件是否需要适配？ -5. **coder/hnsw 引入方式**：vendor 源码（方便魔改）还是 go module 依赖？建议 vendor 源码。 - ---- - -## 12. 总结 - -**核心方案**：`coder/hnsw 算法 + Pebble 持久化 + LRU 分层缓存` - -- 纯 Go，零新依赖（Pebble 已有） -- 与 haystack 现有架构模式一致（`internal/core/vectorindex/`） -- 懒加载控制内存（~18 MB vs ~176 MB @10 万/384 维） -- 增量持久化，crash-safe -- 5 步迭代，每步可独立验证 diff --git a/docs/tasks/HAY-004-hnsw/hnsw-deep-review.md b/docs/tasks/HAY-004-hnsw/hnsw-deep-review.md deleted file mode 100644 index f159dda4..00000000 --- a/docs/tasks/HAY-004-hnsw/hnsw-deep-review.md +++ /dev/null @@ -1,108 +0,0 @@ -# HAY-004 HNSW 深度审查报告 - -> 作者：Architect (via Copilot) | 日期：2026-04-06 -> 审查范围：`origin/user/jianjun/hnsw` 分支，hnsw + hnsw2 两个包 - -## 结论 - -**当前实现是原型/早期 MVP 阶段，核心算法有 3 个致命缺陷，功能上基本不可用。** - ---- - -## 🔴 P0 - 致命缺陷（系统完全无法正常工作） - -### 1. 持久化完全失效 -- **文件**: `hnsw/node_data.go:4-9`, `hnsw/storage.go:106-125` -- **问题**: `nodeData` 所有字段未导出（小写），`encoding/gob` 无法编码未导出字段，静默忽略 -- **影响**: 每次 SaveNode 写入空数据，重启后图完全为空 -- **修复**: 字段改为导出，或实现自定义序列化 - -### 2. 搜索提前终止 Bug -- **文件**: `hnsw/graph_pebble.go:454` -- **问题**: results 是最小堆但当最大堆用，`current.dist > (*results)[0].dist` 意为"候选比最近结果还远就停" -- **影响**: 搜索在找到第 k 个结果后立即停止，recall 接近 0 -- **修复**: results 应改为最大堆 - -### 3. 插入算法严重偏离论文 -- **文件**: `hnsw/graph_pebble.go:113-167` -- **问题**: 完全跳过论文 INSERT 的第 1 步（高层贪心导航），直接从新节点 level 开始 -- **影响**: 高层导航被跳过，搜索起点不是最优 - ---- - -## 🟠 P1 - 严重缺陷（算法正确性问题） - -### 4. Entry Point 不更新 -- **文件**: `hnsw/graph_pebble.go:86-168` -- **问题**: 新节点层级高于 entry point 时不更新 entry point -- **影响**: entry point 停留在低层节点，丧失层级跳跃优势 - -### 5. randomLevel() 概率分布不符合论文 -- **文件**: `hnsw/graph_pebble.go:527-549` -- **问题**: 用几何分布 Ml=0.25 + maxLevel 截断，论文是 `floor(-ln(uniform) * mL)`，mL=1/ln(M)≈0.36 -- **影响**: 层级分布偏低，高层节点过少 - -### 6. 没有 M0=2*M 的零层特殊处理 -- **文件**: `hnsw/graph_pebble.go:553-653` -- **问题**: 所有层使用相同 M，论文要求零层 M0=2*M -- **影响**: 零层连接密度不足，recall 降低 - -### 7. selectDiverseNeighbors 不是论文 Algorithm 4 -- **文件**: `hnsw/graph_pebble.go:773-825` -- **问题**: 用 maximin 多样性选择代替论文的启发式选择，且有 O(n²) I/O - -### 8. Search 返回结果可能 panic -- **文件**: `hnsw/graph_pebble.go:388` -- **问题**: `results[:k]` 没有边界检查，节点数 < k 时 panic - -### 9. Entry Point 删除后不一致 -- **文件**: `hnsw/graph_pebble.go:329-345` -- **问题**: 删除 entry point 时写入 id=0 而非删除 key，重启后指向不存在的节点 - ---- - -## 🟡 P2 - 中等问题（并发安全与性能） - -### 10. Storage 缓存 TOCTOU 竞争 -- **文件**: `hnsw/storage.go:128-157` - -### 11. 缓存存储值拷贝导致数据不一致 -- **文件**: `hnsw/lru_cache.go`, `hnsw/storage.go` - -### 12. BufferedStorage 无并发保护 -- **文件**: `hnsw/cached_storage.go` - -### 13. pruneNeighbors 与 ensureBidirectionalLinks 可能死循环 -- **文件**: `hnsw/graph_pebble.go:553-709` - ---- - -## 🔵 P3 - 次要问题 - -### 14. hnsw2 包未完成 -- 只有空壳，核心图算法未实现 - -### 15. 持久化格式无版本号和校验 - -### 16. 距离函数比较用 reflect - ---- - -## 测试覆盖评估 - -| 覆盖的场景 | 缺失的场景 | -|------------|------------| -| 基本 Add/Delete/Search | ❌ 无 recall@K 测试 | -| 邻居双向性验证 | ❌ 无并发插入/搜索测试 | -| LRU 缓存基本功能 | ❌ 无大规模 recall 基准测试 | -| 层级分布统计 | ❌ 无持久化后重启测试 | -| 高维数据基本测试 | ❌ 无 edge case（k > 图大小等）| - ---- - -## 建议 - -P0 缺陷叠加意味着当前实现功能上不可用。建议： -1. 参考论文原文重新实现核心 INSERT 和 SEARCH-LAYER -2. 或考虑使用成熟的第三方库（如 hnswlib 的 Go binding） -3. 如果自研，建议从 P0 开始逐一修复，每步加 recall@K 回归测试 diff --git a/docs/tasks/HAY-004-hnsw/perf-optimization-plan.md b/docs/tasks/HAY-004-hnsw/perf-optimization-plan.md deleted file mode 100644 index 125b3fe8..00000000 --- a/docs/tasks/HAY-004-hnsw/perf-optimization-plan.md +++ /dev/null @@ -1,306 +0,0 @@ -# HAY-004 HNSW 性能优化方案 - -> **状态**: Draft -> **创建**: 2026-04-15 -> **作者**: 飞马 (架构) -> **关联**: HAY-004 - ---- - -## 1. 现状分析 - -### 1.1 当前性能基线 - -| 场景 | 耗时 | 单条耗时 | -|------|------|----------| -| 10K×128d insert | 85.7s | ~8.5ms/op | -| Pebble batch (参考) | — | 4.5ms/op | -| 100K×384d insert (预估) | 15-20min | — | - -**目标**: 100K×384d insert < 2min（即 ~1.2ms/op） - -### 1.2 pprof 热点分布 - -| 函数 | CPU 占比 | 说明 | -|------|----------|------| -| `selectNeighborsHeuristic` | 46% | 邻居选择，含大量距离计算 | -| `CosineDistance` | 36% | 手写实现，无 SIMD 加速 | -| GC | ~24% | 临时对象分配压力大 | - -> 注: 三项有重叠（selectNeighborsHeuristic 内部调用 CosineDistance），总和 >100% 是正常的。 - -### 1.3 已完成优化 - -- **GetVector 零拷贝**: 消除 Pebble 读取时的内存拷贝，整体提升 ~33% -- **efConstruction 200→128**: 减少构建时搜索范围，牺牲少量 recall 换取速度 - -### 1.4 约束条件 - -- **纯 Go 实现**，不使用 CGo -- 不引入外部 C/C++ 依赖 -- 保持 recall@10 ≥ 0.95（不为速度大幅牺牲精度） - ---- - -## 2. 优化方案（按优先级） - -### P0: SIMD 距离计算 — viterin/vek - -**问题**: 手写的 `CosineDistance` 是标量实现，占 CPU 36%。 - -**方案**: 引入 [viterin/vek](https://github.com/viterin/vek) 库，用 `vek.CosineSimilarity` 替换手写实现。 - -**原理**: -- vek 利用 Go 编译器自动向量化 + unsafe 操作实现 SIMD 级别的向量运算 -- 纯 Go 代码，无 CGo 依赖，符合约束 -- 支持 AVX2/SSE 指令集（通过编译器优化自动生效） - -**改动范围**: -1. `go get github.com/viterin/vek` -2. 替换 `CosineDistance(a, b []float32) float32` 的实现体 -3. 注意: vek 返回的是 similarity（越大越近），需要转换为 distance（`1 - similarity`） - -**预期收益**: -- 距离计算提升 **3-5x** -- 距离计算占总 CPU 36%，整体 Insert 提升约 **30-40%** -- 8.5ms → ~5-6ms/op - -**复杂度**: 🟢 低（替换一个函数） - -**风险**: -- 需要验证 vek 的数值精度与手写实现一致 -- 需要 benchmark 验证实际加速比 - ---- - -### P1: selectNeighborsHeuristic 距离缓存 - -**问题**: `selectNeighborsHeuristic` 占 CPU 46%，其中大量时间花在重复计算候选节点之间的距离。 - -**分析**: -- HNSW 的 heuristic 邻居选择需要计算候选节点两两之间的距离 -- 同一个节点的距离在 insert 过程中被多次计算（不同层、不同邻居的 pruning） -- hnswlib 参考实现中维护了距离缓存 - -**方案**: 在单次 insert 操作的上下文中维护距离缓存 map。 - -**改动范围**: -1. 定义 `distanceCache` 结构（key: `(nodeID_a, nodeID_b)` → value: `float32`） -2. 在 `insert()` 入口创建缓存，通过参数传递到 `selectNeighborsHeuristic` -3. 距离计算前先查缓存，miss 时计算并写入 -4. key 做归一化：`min(a,b), max(a,b)` 确保对称 - -**预期收益**: -- 减少 **50%+** 的距离计算调用 -- 整体提升 **20-30%** -- 叠加 P0 后: ~5-6ms → ~3.5-4.5ms/op - -**复杂度**: 🟡 中（需要修改函数签名，传递缓存上下文） - -**风险**: -- 缓存 map 本身有内存开销，需要控制大小 -- 单次 insert 结束后缓存失效，不会无限增长 -- 需要 benchmark 验证缓存命中率 - ---- - -### P2: GC 压力优化 - -**问题**: GC 占 CPU ~24%，说明有大量临时对象分配。 - -**分析**: -- Priority queue 每次 insert 都新建 -- 候选列表 `[]Candidate` 动态增长 -- `interface{}` 类型导致装箱/拆箱 - -**方案**: 多管齐下减少 GC 压力。 - -**改动范围**: - -#### 2a. sync.Pool 池化 -```go -var pqPool = sync.Pool{ - New: func() interface{} { - return &PriorityQueue{items: make([]Item, 0, 128)} - }, -} -``` -- 池化 priority queue -- 池化候选列表 `[]Candidate` -- 使用后 Reset + Put 回池 - -#### 2b. 预分配 slice -- 候选列表按 `efConstruction` 预分配容量 -- 邻居列表按 `M` 预分配 -- 避免 `append` 触发的动态扩容和旧 slice 的 GC - -#### 2c. 减少 interface{} 使用 -- Priority queue 的 item 使用具体类型替代 `interface{}` -- 避免 float32/int 的装箱开销 - -**预期收益**: -- GC 占比从 24% 降到 **<10%** -- 整体提升 **15-20%** -- 叠加 P0+P1 后: ~3.5-4.5ms → ~2.5-3.5ms/op - -**复杂度**: 🟡 中（多处改动，但每处都不复杂） - -**风险**: -- sync.Pool 在高并发下效果好，单线程场景收益可能偏低 -- 需要仔细处理 Reset 逻辑，避免数据泄漏 - ---- - -### P3: 预计算向量范数 - -**问题**: `CosineDistance` 每次调用都要计算两个向量的 L2 范数，而同一个向量的范数在不同距离计算中被反复求解。 - -**方案**: 插入时预计算并持久化存储向量的 L2 范数。 - -**改动范围**: -1. 向量存储时额外存一个 `float32` 的范数值 -2. `CosineDistance` 改为接收预计算范数：`CosineDistanceWithNorm(a, b []float32, normA, normB float32)` -3. 新插入的向量在写入 Pebble 前计算范数 -4. 查询向量也预计算范数 - -**预期收益**: -- 距离计算再快 **30-40%**（省去两次 `sqrt(sum(x²))` 计算） -- 整体提升 **10-15%**（叠加 P0 的 SIMD 后，距离计算占比已下降） -- 叠加 P0+P1+P2 后: ~2.5-3.5ms → ~2-3ms/op - -**复杂度**: 🟢 低（存储多一个 float32，改一个函数签名） - -**风险**: -- 存储格式变更，需要考虑向后兼容或数据迁移 -- 如果使用 vek（P0），需要确认 vek 是否支持传入预计算范数（可能不支持，需要自己包装） - -**注意**: 如果 P0 使用了 vek 的 `CosineSimilarity`，vek 内部会自己算范数。此时 P3 的收益取决于能否绕过 vek 用自定义的 norm-aware 距离函数。需要评估是直接用 vek 还是用 vek 的底层向量运算（`vek.Dot`, `vek.Norm`）组合自己的距离函数。 - ---- - -### P4: 批量插入优化 - -**问题**: 当前逐条 insert，每条都走完整的 search → connect → prune 流程。 - -**方案**: 批量插入时先构建局部子图，再合并到主图。 - -**改动范围**: -1. `BatchInsert(vectors [][]float32)` 接口 -2. 批内向量先互相建图（跳过主图搜索） -3. 批内图构建完成后，找到与主图的连接点 -4. 合并两个图，重新 prune 边界节点的邻居 - -**预期收益**: -- 批量场景提升 **2-3x** -- 叠加 P0-P3 后: ~2-3ms → ~1-1.5ms/op（批量场景） - -**复杂度**: 🔴 高 - -**风险**: -- 实现复杂，容易引入 recall 下降 -- 批内子图质量取决于批大小和数据分布 -- 需要大量测试验证正确性 -- 合并逻辑是全新代码，没有现成参考 - ---- - -## 3. 综合预期 - -### 3.1 累积效果估算 - -| 阶段 | 单条耗时 | 100K×384d 预估 | 相对基线 | -|------|----------|----------------|----------| -| 基线 | 8.5ms | 15-20min | — | -| +P0 (SIMD) | ~5-6ms | 9-12min | -35% | -| +P1 (距离缓存) | ~3.5-4.5ms | 6-8min | -50% | -| +P2 (GC 优化) | ~2.5-3.5ms | 4-6min | -65% | -| +P3 (范数预计算) | ~2-3ms | 3-5min | -70% | -| +P4 (批量插入) | ~1-1.5ms | 1.5-2.5min | -82% | - -> ⚠️ 以上为估算值，384d 向量比 128d 更重（距离计算量 3x），实际数字可能有偏差。 -> 384d 的 SIMD 加速效果可能更显著（更长的向量 = SIMD 优势更大）。 - -### 3.2 达标分析 - -**目标**: 100K×384d < 2min - -- **P0-P3 完成后**: 预估 3-5min，**接近但可能未达标** -- **P0-P4 全部完成后**: 预估 1.5-2.5min，**基本达标或接近** -- 如果仍有差距，可考虑补充措施（见第 4 节） - ---- - -## 4. 补充措施（如 P0-P4 仍不够） - -如果 P0-P4 全部完成后仍未达标，可考虑以下降级/激进措施： - -### 4a. 降低 efConstruction -- 从 128 进一步降到 64 -- 预期: 构建速度提升 ~40%，recall@10 可能从 0.97 降到 0.93-0.95 -- 权衡: 需要跑 recall benchmark 确认可接受 - -### 4b. 向量量化 -- float32 → float16（半精度） -- 存储减半，距离计算带宽减半 -- 预期: 额外提升 30-50% -- 风险: 精度损失，需要评估对 recall 的影响 - -### 4c. 并行化 Insert -- 当前 insert 是单线程串行 -- HNSW 支持有限并行（不同层的搜索可以并行，但连接需要锁） -- 复杂度高，且 Go 的 goroutine 调度本身有开销 - ---- - -## 5. 实施计划 - -### 建议执行顺序 - -``` -P0 (SIMD) → P3 (范数预计算) → P1 (距离缓存) → P2 (GC) → P4 (批量) -``` - -**理由**: -- P0 和 P3 改动小、风险低、收益确定 → 先做 -- P1 依赖理解当前距离计算的调用模式 → P0 改完后更清晰 -- P2 是全局优化，P0-P3 做完后 profiling 数据更准确 -- P4 复杂度最高，留到最后，视情况决定是否做 - -### 每步验收标准 - -每完成一个优化项，必须： -1. **Benchmark**: 跑 `BenchmarkInsert10K` 对比前后 -2. **pprof**: 重新 profile，确认热点变化符合预期 -3. **Recall**: 跑 recall@10 测试，确认精度未显著下降 -4. **Test**: 所有现有测试通过 - ---- - -## 6. 开放问题 - -1. **vek 与范数预计算的兼容性**: P0 用 vek 后，P3 是否还有收益？需要看 vek 的 API 是否支持传入预计算范数。 -2. **384d 的实际 profile**: 当前 profile 基于 128d，384d 的热点分布可能不同（距离计算占比可能更高）。 -3. **并发 insert 是否在路线图中**: 如果未来需要并发 insert，P2 的 sync.Pool 和 P1 的缓存设计需要考虑线程安全。 -4. **Pebble 的 4.5ms/op 瓶颈**: 即使 HNSW 计算优化到 0，Pebble 写入本身 4.5ms 也是下限。需要确认这是否包含在 8.5ms 中，还是另算的。 - ---- - -## 附录 - -### A. 参考实现 -- [hnswlib (C++)](https://github.com/nmslib/hnswlib) — 距离缓存、SIMD 实现参考 -- [viterin/vek (Go)](https://github.com/viterin/vek) — 纯 Go SIMD 向量运算 -- [usearch (C++/Go bindings)](https://github.com/unum-cloud/usearch) — 高性能 ANN 参考 - -### B. Benchmark 命令 -```bash -# 基础 benchmark -go test -bench=BenchmarkInsert -benchtime=3s -count=5 ./pkg/index/hnsw/ - -# pprof -go test -bench=BenchmarkInsert -cpuprofile=cpu.prof ./pkg/index/hnsw/ -go tool pprof -http=:8080 cpu.prof - -# recall 测试 -go test -run=TestRecall -v ./pkg/index/hnsw/ -``` diff --git a/docs/tasks/HAY-004-hnsw/perf-research-detailed.md b/docs/tasks/HAY-004-hnsw/perf-research-detailed.md deleted file mode 100644 index 3aee3b88..00000000 --- a/docs/tasks/HAY-004-hnsw/perf-research-detailed.md +++ /dev/null @@ -1,144 +0,0 @@ -# HAY-004: HNSW 向量索引性能深度调研报告 - -**日期**: 2026-04-15 -**环境**: Linux 6.17.0, AMD EPYC 9V74 80-Core, Go 1.24.2 -**数据集**: 10K × 128d 随机向量, cosine distance, efConstruction=200, efSearch=128, M=16 - ---- - -## 摘要 - -| 方案 | 10K 插入时间 | 插入 µs/vec | 搜索 p50 | 搜索 p95 | 搜索 p99 | Recall@10 | CGO | -|------|------------|------------|---------|---------|---------|-----------|-----| -| **Haystack (MemStore)** | 48.07s | 3,339 | 967µs | 1,225µs | 1,252µs | 0.884 | ❌ | -| **USearch Go binding** | 4.68s | 468 | 201µs | 407µs | 843µs | 0.894 | ✅ | - -**USearch 比我们的实现快 ~7× 插入, ~5× 搜索，recall 略高。** - ---- - -## 1. Haystack 自研实现 (基准) - -### TestBenchmarkSearchLatency10K (10K×128d, MemStore, efC=200, efS=128) -``` -Insert time: 48.069s (4,807 µs/vec) -Search p50: 967.143µs -Search p95: 1.224983ms -Search p99: 1.251779ms -Recall@10: 0.8840 -``` - -### Go Benchmark (testing.B, 10K iterations) -``` -BenchmarkHNSWInsert-4 10000 3,338,735 ns/op 307,054 B/op 1,743 allocs/op -BenchmarkHNSWSearch-4 (1K index) 1000 343,877 ns/op 77,170 B/op 692 allocs/op -BenchmarkHNSWInsertPebble-4 10000 10,391,309 ns/op 356,020 B/op 2,741 allocs/op -BenchmarkHNSWInsertBatchPebble-4 10000 6,457,317 ns/op 389,400 B/op 2,669 allocs/op -``` - -**分析**: 每次插入 3.3ms + 1,743 次堆分配，说明 HNSW 图遍历中有大量小对象分配。这是纯 Go 实现的主要瓶颈。 - ---- - -## 2. USearch Go Binding - -### 编译过程 -1. `go get github.com/unum-cloud/usearch/golang` — 成功下载 -2. Go binding 通过 CGO 链接 `libusearch_c.so` -3. C 库需要从源码编译： - ```bash - git clone --depth 1 https://github.com/unum-cloud/usearch.git - cd usearch && git submodule update --init --recursive - cmake -B build -DCMAKE_BUILD_TYPE=Release -DUSEARCH_BUILD_LIB_C=ON - cmake --build build -j$(nproc) - # 产物: libusearch_c.so - ``` -4. 依赖: cmake, build-essential (gcc/g++), fp16 submodule -5. 编译成功，无额外系统依赖 - -### 10K×128d Benchmark 结果 -``` -Insert time: 4.675s (468 µs/vec) -Search p50: 200.624µs -Search p95: 407.086µs -Search p99: 842.755µs -Recall@10: 0.8940 -``` - -### CGO 依赖清单 -- `libusearch_c.so` (需自行编译或从 release 下载) -- 编译时: cmake, g++, git (for submodules) -- 运行时: 仅 libusearch_c.so (纯 C++ header-only library 编译产物) -- Go build flags: `CGO_LDFLAGS="-L/usr/local/lib -lusearch_c"` - ---- - -## 3. 其他方案评估 - -### Bithack/go-hnsw -- **状态**: ❌ 不可用 -- 最后更新 2017 年，依赖 `github.com/willf/bitset` 已改名，`go get` 解析失败 -- 不值得修复 - -### viterin/hnsw -- **状态**: ❌ 不存在 -- GitHub 上无此仓库 - -### chromem-go -- **状态**: ⚠️ 不适用 -- 这是一个嵌入式向量数据库，但内部使用**暴力搜索**（brute-force），不是 HNSW -- 10K 规模下搜索会很快，但不可扩展 - -### hnswlib Go binding -- **状态**: 无官方 Go binding -- hnswlib (C++) 是 USearch 的前身/竞品，Go 生态中没有维护良好的 binding - ---- - -## 4. 性能对比总结 - -### 插入性能 (10K×128d) -``` -Haystack MemStore: 48.07s (3,339 µs/vec) ████████████████████████████████████ 100% -Haystack Pebble: 103.91s (10,391 µs/vec) ████████████████████████████████████████████████ 311% -Haystack Pebble Batch: 64.57s (6,457 µs/vec) ████████████████████████████████████████ 193% -USearch: 4.68s (468 µs/vec) ███ 14% -``` - -### 搜索延迟 -``` - p50 p95 p99 -Haystack 10K: 967µs 1,225µs 1,252µs -USearch 10K: 201µs 407µs 843µs -改善: 4.8× 3.0× 1.5× -``` - -### 内存分配 -- Haystack: 1,743 allocs/insert, 692 allocs/search -- USearch: CGO 调用开销极小，C++ 侧无 Go GC 压力 - ---- - -## 5. 结论与推荐 - -### 推荐: 采用 USearch Go binding 替换自研 HNSW - -**理由（全部基于实测数据）**: - -1. **插入快 7.1×**: 468µs vs 3,339µs per vector -2. **搜索快 4.8×** (p50): 201µs vs 967µs -3. **Recall 略优**: 0.894 vs 0.884（相同参数下） -4. **零 GC 压力**: C++ 实现不产生 Go 堆分配 -5. **编译可行**: CGO 依赖仅 libusearch_c.so，编译流程已验证通过 -6. **活跃维护**: USearch 是 Unum Cloud 的核心产品，持续更新 - -### 风险与代价 -- **CGO 依赖**: 增加构建复杂度，需要 libusearch_c.so 随项目分发 -- **交叉编译**: CGO 使交叉编译更复杂（但 haystack 已有 pebble 的 CGO 依赖，增量成本低） -- **调试难度**: C++ 侧的问题更难排查 - -### 下一步 -1. 在 haystack 中创建 `USearchStore` 适配层，实现现有 `VectorIndex` 接口 -2. 将 libusearch_c.so 加入 CI 构建流程 -3. 跑 100K 规模 benchmark 确认大规模下的表现 -4. 评估 USearch 的持久化能力（save/load）是否满足需求 diff --git a/docs/tasks/HAY-004-hnsw/perf-research.md b/docs/tasks/HAY-004-hnsw/perf-research.md deleted file mode 100644 index f4f0985a..00000000 --- a/docs/tasks/HAY-004-hnsw/perf-research.md +++ /dev/null @@ -1,153 +0,0 @@ -# HNSW 性能基准调研报告 - -> 2026-04-15 | HAY-004 | 飞马 subagent - -## 背景 - -我们自研 Go HNSW（haystack 仓库），当前性能： -- **10K × 128d 插入**：85 秒（~117 QPS） -- **单条插入延迟**：p50 1.45ms，p95 3.49ms -- **Recall@10**：0.884 -- **瓶颈**：`selectNeighborsHeuristic` 占 47% CPU，距离计算无 SIMD，Pebble 持久化 - ---- - -## 1. 业界 HNSW 性能基准 - -### hnswlib (C++, nmslib) -| 规模 | 维度 | 插入 QPS (单线程) | 搜索 QPS | Recall | -|------|------|-------------------|----------|--------| -| 10K | 128d | ~50,000–80,000 | ~100,000+ | >0.99 | -| 1M | 128d | ~30,000–50,000 | ~50,000–80,000 | >0.95 | - -- 纯内存，C++ SIMD 优化（SSE/AVX） -- M=16, efConstruction=200 默认参数 -- **10K×128d 插入约 0.1–0.2 秒**（vs 我们的 85 秒，差距 **400–800x**） - -### USearch (C/C++, unum-cloud) -在 AWS c7g.metal (64 核 Graviton 3) 上，f32×256d： -| efConstruction | 插入 QPS | 搜索 QPS | Recall@1 | -|----------------|----------|----------|----------| -| 128 | 75,640 | 131,654 | 99.3% | -| 64 | 128,644 | 228,422 | 97.2% | - -- 支持 f16/i8 量化，i8 可达 115K insert QPS / 274K search QPS -- **有官方 Go binding**（CGO），接口简洁 -- 支持持久化（mmap） - -### Faiss (Meta, C++) -- HNSW 实现默认 M=32, efConstruction=40 -- 1M×128d 插入 ~30-60 秒（单线程），搜索 QPS 数万级 -- 重量级库，主要面向 Python/C++，Go 绑定不成熟 - -### Go 生态 -| 库 | 特点 | 性能 | -|----|------|------| -| **coder/hnsw** | 纯 Go，用 viterin/vek 做距离计算 | 轻量但无 SIMD，纯内存 | -| **viterin/hnsw** | 纯 Go | 类似 coder/hnsw | -| **chromem-go** | 嵌入式向量 DB，暴力搜索 | 无 ANN 索引，小规模可用 | - -纯 Go HNSW 实现普遍比 C++ 慢 **10–50x**，主要因为： -- Go 编译器不做 SIMD 自动向量化 -- GC 压力（大量小对象/slice） -- 无法内联热路径的汇编优化 - ---- - -## 2. 我们自研实现慢的原因分析 - -### 2.1 Go vs C++ 向量运算差距 -- C++ 编译器自动向量化 + 手写 SIMD intrinsics，128d float32 距离计算可用 AVX-256 一次处理 8 个 float -- Go 标准库无 SIMD，128d 距离计算比 C++ 慢 **5–15x** -- viterin/vek 有部分 SIMD 支持但覆盖有限 - -### 2.2 selectNeighborsHeuristic 占 47% CPU — **偏高但不异常** -- hnswlib 中该函数也是热路径，但通常占 20–30% -- 47% 偏高，可能原因： - - 候选集过大（efConstruction 设太高？） - - 内部排序/堆操作 Go 实现效率低 - - 每次调用重新分配 slice -- **优化方向**：复用 buffer、减小 efConstruction、优化堆实现 - -### 2.3 无 SIMD 的影响 -- 距离计算是 HNSW 最内层循环，每次插入调用数百到数千次 -- SIMD 对 128d float32 可加速 **4–8x**（AVX-256）到 **8–16x**（AVX-512） -- 这单项就能让整体插入快 **3–5x** - -### 2.4 Pebble 持久化 overhead -- 每次插入涉及 Pebble KV 读写，序列化/反序列化开销 -- 纯内存 HNSW（hnswlib）没有这个成本 -- 估算持久化带来 **2–5x** 额外开销（取决于 batch 策略） -- 但即使去掉持久化，纯 Go 内存实现仍会比 C++ 慢 10–30x - -### 综合估算 -| 因素 | 影响倍数 | -|------|----------| -| Go vs C++ (无 SIMD) | 10–15x | -| Pebble 持久化 | 2–5x | -| 算法实现细节（堆/alloc） | 2–3x | -| **总计** | **~40–200x** | - -我们实际差距 400–800x，说明还有算法层面的低效（可能是 efConstruction 过高或邻居选择逻辑未优化）。 - ---- - -## 3. 替代方案评估 - -### 方案 A：CGO 绑定 hnswlib -- **性能**：接近原生 C++，10K 插入 < 1 秒 -- **成本**：CGO 调用开销小（~100ns/call），但构建复杂度增加 -- **风险**：交叉编译困难，调试不便，内存管理跨语言 -- **评估**：✅ 可行但维护成本中等 - -### 方案 B：USearch Go binding（推荐 ✅） -- **性能**：与 hnswlib 持平或更优，有 SIMD + 量化支持 -- **Go binding**：官方维护，`github.com/unum-cloud/usearch/golang` -- **特性**： - - 嵌入式，无需外部服务 - - 支持 mmap 持久化（比 Pebble 更高效） - - 支持 f16/i8 量化减少内存 - - 活跃维护，MIT 协议 -- **成本**：CGO 依赖，但官方 binding 质量高 -- **评估**：⭐ **最佳选择** — 性能接近最优，嵌入式，官方 Go 支持 - -### 方案 C：外部服务（Qdrant/Milvus） -- **不适合**：haystack 要求嵌入式、低内存、单进程 -- 引入网络延迟 + 运维复杂度 -- **评估**：❌ 不符合需求 - -### 方案 D：继续优化自研 -- 可做的优化： - - 加 SIMD（Go 汇编或用 viterin/vek 增强版） - - 优化 selectNeighborsHeuristic（复用 buffer、调参数） - - Batch 写入减少 Pebble 开销 - - 降低 efConstruction -- **预期收益**：可能提升 5–10x，但仍比 C++ 慢 10–30x -- **评估**：⚠️ 投入大、天花板低 - -### 方案 E：其他嵌入式方案 -| 方案 | 说明 | -|------|------| -| sqlite-vec | SQLite 扩展，暴力搜索，小规模可用 | -| vectorlite | SQLite + hnswlib，有 Go 绑定 | -| Bleve | Go 全文搜索，向量搜索能力弱 | - ---- - -## 4. 结论与推荐 - -### 推荐：**方案 B — 用 USearch Go binding 替换自研 HNSW** - -**理由**： -1. **性能差距不可弥补**：纯 Go 实现的天花板（即使全优化）仍比 C++ 慢 10–30x，无法满足需求 -2. **USearch 完美匹配需求**：嵌入式、低内存（支持量化）、有 mmap 持久化、官方 Go binding -3. **投入产出比最高**：替换成本远低于持续优化自研实现 -4. **性能预期**：10K×128d 插入应在 **0.1–0.5 秒**（当前 85 秒），提升 **170–850x** - -**实施建议**： -1. 先做 PoC：用 USearch Go binding 跑 10K×128d 基准，验证实际性能 -2. 适配接口：将现有 HNSW 接口抽象为 interface，方便切换后端 -3. 持久化迁移：用 USearch 的 mmap 替换 Pebble 存储 -4. 保留 Pebble：仅用于其他 KV 数据，不存向量索引 - -**备选**：如果 USearch Go binding 有坑（CGO 问题等），退而选方案 A（CGO 绑定 hnswlib）。 diff --git a/docs/tasks/HAY-004f/review.md b/docs/tasks/HAY-004f/review.md deleted file mode 100644 index 0b5a967f..00000000 --- a/docs/tasks/HAY-004f/review.md +++ /dev/null @@ -1,141 +0,0 @@ -# HAY-004f 验收报告 - -**任务：** HNSW 性能验证 & E2E（参数化 benchmark + SIFT 真实数据验证） -**PR：** #49 -**验收日期：** 2026-04-17 -**验收人：** QA（烈马） - ---- - -## 环境 - -- 镜像：`haystack-test` -- 项目目录：`/workspace/haystack` -- 数据：SIFT-128（100K base vectors，sift_base.fvecs） - -## 1. 拉最新代码 - -``` -cd /workspace/haystack && git checkout main && git pull && git log --oneline -5 -``` - -**输出（关键）：** -``` -f372f94 test(vectorindex): parametric HNSW benchmarks - scale, efSearch, efConstruction, M comparison (#49) -db9ca70 perf(vectorindex): HNSW insert performance optimization - 7.5x speedup (#HAY-004f) (#48) -``` - -✅ PR #49 已合入 main - ---- - -## 2. 单元测试 - -**命令：** -```bash -docker run --rm -v /workspace/haystack:/app -w /app haystack-test go test -short -timeout 5m ./... -``` - -**结果：** -``` -ok github.com/codetrek/haystack/internal/client 1.042s -ok github.com/codetrek/haystack/internal/conf 0.005s -ok github.com/codetrek/haystack/internal/core/documents 1.153s -ok github.com/codetrek/haystack/internal/core/idtable 0.488s -ok github.com/codetrek/haystack/internal/core/invertedindex 1.864s -ok github.com/codetrek/haystack/internal/core/invertedindex/tokenizer 11.658s -ok github.com/codetrek/haystack/internal/core/pebble 0.759s -ok github.com/codetrek/haystack/internal/core/storage 0.068s -ok github.com/codetrek/haystack/internal/core/symbols 1.230s -ok github.com/codetrek/haystack/internal/core/vectorindex 6.524s -ok github.com/codetrek/haystack/internal/core/workspace 1.357s -ok github.com/codetrek/haystack/internal/core/workspace/internal 0.218s -ok github.com/codetrek/haystack/internal/server 1.267s -ok github.com/codetrek/haystack/internal/server/httpapi 2.608s -ok github.com/codetrek/haystack/internal/server/indexer 4.725s -ok github.com/codetrek/haystack/internal/server/mcptools 0.347s -ok github.com/codetrek/haystack/internal/server/searcher 10.229s -ok github.com/codetrek/haystack/internal/shared/running 0.023s -ok github.com/codetrek/haystack/internal/utils 0.006s -ok github.com/codetrek/haystack/internal/utils/fs 0.017s -ok github.com/codetrek/haystack/internal/utils/git 0.009s -ok github.com/codetrek/haystack/internal/utils/queue 0.104s -exit code: 0 -``` - -✅ 19 个包全部通过，0 个失败 - ---- - -## 3. SIFT Benchmark（真实数据） - -**命令：** -```bash -docker run --rm -v /workspace/haystack:/app -w /app haystack-test \ - go test -tags benchmark -v \ - -run "TestBenchmarkSIFT|TestPersistenceRecall" \ - -timeout 20m ./internal/core/vectorindex/... -``` - -**数据：** SIFT-128，100K base vectors，100 queries，暴力 ground truth（基于 100K 子集重算） - -### Insert 性能 - -``` -Insert 100000 SIFT vectors: 2m4.372851696s (1.24ms/op) -``` - -| 指标 | 实测值 | 标准 | 结论 | -|------|--------|------|------| -| Insert 100K 耗时 | **2min 4.4s** | <2min（飞马批准 2min 10s 可接受） | ✅ PASS | - -### Search 性能 & Recall - -``` -SIFT 100K efSearch=128: p50=675.973µs p95=881.454µs p99=924.991µs recall@10=0.9990 -SIFT 100K efSearch=200: p50=946.042µs p95=1.235611ms p99=1.324145ms recall@10=1.0000 -SIFT 100K efSearch=400: p50=1.731481ms p95=2.124096ms p99=2.267794ms recall@10=1.0000 -``` - -| 指标 | 实测值（efSearch=128） | 标准 | 结论 | -|------|------------------------|------|------| -| Search p99 | **0.925ms** | <20ms | ✅ PASS | -| Recall@10 | **0.9990** | >0.95 | ✅ PASS | - ---- - -## 4. 持久化 E2E - -**命令：**（与 benchmark 同一命令，包含 TestPersistenceRecall） - -``` -=== RUN TestPersistenceRecall - benchmark_test.go:503: Persistence recall@10 over 50 queries: 1.0000 ---- PASS: TestPersistenceRecall (0.63s) -``` - -验证方式：写入向量到 PebbleNodeStore → 重启（重新打开 Pebble DB） → 搜索结果与写入前一致 -Recall@10 over 50 queries: **1.0000** - -| 指标 | 实测值 | 标准 | 结论 | -|------|--------|------|------| -| 持久化 E2E | Recall@10 = 1.0000 | 重启后搜索结果一致 | ✅ PASS | - ---- - -## 验收标准总结 - -| # | 验收标准 | 实测值 | 结论 | -|---|----------|--------|------| -| 1 | 单测全部通过，覆盖率不下降 | 19 个包全部通过 | ✅ PASS | -| 2 | Insert 100K < 2min（批准 2min 10s） | 2min 4.4s | ✅ PASS | -| 3 | Search p99 < 20ms | 0.925ms | ✅ PASS | -| 4 | Recall@10 > 0.95（真实数据） | 0.9990 | ✅ PASS | -| 5 | 持久化 E2E：写入→重启→搜索一致 | Recall 1.0000 | ✅ PASS | - -## 整体结论 - -✅ **HAY-004f 验收通过** - -所有验收标准满足。HNSW 向量索引的性能优化（#48）和参数化 benchmark（#49）均达标。 -Insert 耗时 2min 4.4s 在飞马批准的 2min 10s 范围内，Search p99 远优于 20ms 标准（仅 0.925ms），Recall@10 = 0.999 高于 0.95 阈值。 diff --git a/docs/tasks/HAY-007-codereview-cc.md b/docs/tasks/HAY-007-codereview-cc.md deleted file mode 100644 index 134ed349..00000000 --- a/docs/tasks/HAY-007-codereview-cc.md +++ /dev/null @@ -1,73 +0,0 @@ -# HAY-007 Phase 1 Code Review — Spec vs Implementation - -> Reviewer: Claude Code -> Date: 2026-04-17 -> Scope: `git diff origin/main...HEAD` vs `docs/design/HAY-007-mmap-store.md` Phase 1 - ---- - -## Checklist: Phase 1 Requirements - -| # | Spec Requirement | Status | Notes | -|---|-----------------|--------|-------| -| 1 | vectors.dat mmap 读取 | PASS | | -| 2 | nodes.dat mmap 读取 | PASS | | -| 3 | graph_l0.dat mmap 读取 | PASS | | -| 4 | 自封装 syscall.Mmap (Linux/macOS/Windows) | PASS | `mmap_unix.go` + `mmap_windows.go` + `mmap.go` | -| 5 | GetVector | PASS | | -| 6 | GetVectorRef (初版=copy) | PASS | 文档注释正确说明 Phase 5 零拷贝计划 | -| 7 | GetNeighbors (L0 + upper) | PASS | | -| 8 | GetNorm | PASS | | -| 9 | 从 PebbleStore 导出数据到 mmap 文件格式 | **见 HIGH-1** | | -| 10 | meta.bin 原子写 (tmp→fsync→rename) | PASS | | -| 11 | 三平台 `CGO_ENABLED=0` 编译 | 未验证 | CI 层面事项，代码层面无 CGo 依赖 | - ---- - -## Findings - -### HIGH-1: 导出来源偏离 spec — MemStore 而非 PebbleStore - -**Spec 原文**: "从 PebbleStore 导出数据到 mmap 文件格式" -**实际实现**: `exportMemStoreToMmap()` — 从 MemNodeStore 导出，无任何 PebbleStore 相关代码。 - -**影响**: Phase 1 验收条件中的数据迁移路径与 spec 不一致。如果实际使用场景需要从 PebbleStore 迁移，则该功能缺失。如果 MemStore 导出是有意替代（例如 PebbleStore 已废弃），则 spec 应同步更新。 - -**建议**: 确认意图 — 是 spec 需要更新，还是需要补充 PebbleStore 导出实现。 - -### HIGH-2: MetaHeader 字段顺序与 spec 不一致 - -**Spec 定义**: -``` -EntryPoint uint64 // offset 位于 EntryLevel 之前 -EntryLevel uint32 -NextNodeId uint64 -``` - -**实际代码** (`mmap_format.go:26-39`): -```go -// uint32 group first, then uint64 group -MaxLevel uint32 -EntryLevel uint32 // ← 移到了 uint32 组 -NodeCount uint64 -TotalSlots uint64 -EntryPoint uint64 // ← 移到了 uint64 组 -NextNodeId uint64 -``` - -**影响**: 字段重新排列导致二进制布局与 spec 文档不一致。虽然代码通过 `var _ [64]byte = [unsafe.Sizeof(MetaHeader{})]byte{}` 保证了 64 字节大小，且代码内部自洽（读写都用同一个 struct），但： -- 其他语言/工具按 spec 解析 meta.bin 将得到错误数据 -- spec 文档和代码不同步，后续开发者会困惑 - -**建议**: 二选一 — (a) 更新 spec 文档匹配代码布局，并注明 padding 优化原因；(b) 恢复 spec 顺序并使用 explicit padding 字段保持 64 字节。推荐 (a)，代码的排列确实更优。 - ---- - -## Summary - -| Severity | Count | -|----------|-------| -| CRITICAL | 0 | -| HIGH | 2 | - -Phase 1 的核心只读路径实现完整且正确（GetVector / GetVectorRef / GetNeighbors / GetNorm / GetEntryPoint / GetNodeLevel）。mmap 封装跨平台实现到位，文件格式与测试覆盖充分。两个 HIGH 问题均为 spec-代码同步问题，不影响运行时正确性。 diff --git a/docs/tasks/HAY-007-phase2-review-cc.md b/docs/tasks/HAY-007-phase2-review-cc.md deleted file mode 100644 index 0e7c70f7..00000000 --- a/docs/tasks/HAY-007-phase2-review-cc.md +++ /dev/null @@ -1,72 +0,0 @@ -# HAY-007 Phase 2 Task Plan Review - -> 审核人：Claude Code -> 日期：2026-04-18 -> 对照：`docs/design/HAY-007-mmap-store.md` v2（Phase 2 部分） -> 对象：`docs/tasks/HAY-007-phase2-tasks.md` - ---- - -## CRITICAL-1：WAL Replay 恢复 meta 字段遗漏 - -**位置**：Task 2.1 步骤 6（Replay 方法） - -**问题**：Replay 回调仅提到恢复 `docToNode / nodeToDoc` 映射，但未提及恢复以下 meta 字段： - -- `NodeCount`（需统计 INSERT 记录数） -- `TotalSlots`（需追踪 max nodeId + 1） -- `NextNodeId`（需为 max nodeId + 1，否则重启后 ID 冲突） -- `EntryPoint` / `EntryLevel`（需从 SET_ENTRY 记录恢复） -- `MaxLevel`（需从 INSERT 记录的 level 字段推算） -- `GraphUpperNextSlot`（需从 INSERT level>0 记录推算） - -**影响**：非正常关闭后 WAL replay 无法恢复正确的 meta 状态。虽然 Phase 1 Close() 已有 `writeMetaHeader`，但若 Close() 未执行（crash），replay 是唯一恢复路径。如果 replay 不更新 meta，重启后 `NextNodeId` 可能从旧值开始分配导致 ID 覆盖、`NodeCount` 不准确、`EntryPoint` 丢失。 - -**建议**：Task 2.1 步骤 6 增加 replay 回调中 meta 字段恢复逻辑。可在 replay 完成后统一根据已扫描记录计算 meta，而非每条记录都更新。 - ---- - -## HIGH-1：Task 2.2 Close() 缺少 WAL flush 与 meta 写入的顺序说明 - -**位置**：Task 2.2 步骤 1（MmapStore 集成 WAL 字段） - -**问题**：Close() 描述为 "wal.Sync() + wal.Close()"，但未提及： -1. 需要先 msync 所有 mmap 区域（确保数据页落盘） -2. 再 writeMetaHeader（Phase 1 已有，但 Task 2.2 未提及保留此行为且需在 WAL sync 之后执行） -3. 顺序必须为：msync mmap → wal.Sync() → writeMetaHeader → wal.Close() → munmap → close files - -Phase 1 的 Close() 已调用 `writeMetaHeader`，但 Phase 2 修改 Close() 加入 WAL 后，需明确 WAL sync 与 meta 写入的先后顺序，否则可能出现 meta.WalCheckpointLSN 与实际 WAL 状态不一致。 - -**建议**：Task 2.2 步骤 1 明确 Close() 的完整步骤序列和顺序约束。 - ---- - -## HIGH-2：idmap.dat 加载时机未明确 - -**位置**：Task 2.2 步骤 6（SetNodeMapping） - -**问题**：Task 2.2 引入了 idmap.dat 的写入逻辑（追加写 + CRC），但未提及 Open() 时加载 idmap.dat 到内存 map 的步骤。Phase 1 是只读路径，Open() 中可能不含 idmap.dat 加载逻辑。 - -如果 Open() 不加载 idmap.dat，则 Close() → Open() 后 `docToNode`/`nodeToDoc` 为空，GetNodeId 失败。Task 2.2 验证用例第 10 条（"SetNodeMapping → Close → Open → GetNodeId 返回正确"）会暴露此问题，但实现步骤中遗漏了此工作。 - -**建议**：Task 2.2 步骤 1（修改 Open）中明确增加 idmap.dat 加载步骤：顺序读取所有 entry，CRC 校验，填充 docToNode/nodeToDoc。估计 ~30 行。 - ---- - -## HIGH-3：PutNode 写 WAL 的 docId 字段语义不清 - -**位置**：Task 2.2 步骤 2（PutNode） - -**问题**：PutNode 的 WAL INSERT 记录 `EncodeInsert(id, level, vec, norm, "")` 中 docId 为空字符串。但设计文档 §4.3 Insert 流程第 5 步将 idmap 写入作为 Insert 流程的一部分。 - -如果 WAL INSERT 记录不包含 docId，则 replay 时无法恢复 docToNode/nodeToDoc 映射——但 Task 2.1 步骤 6 却说 "replay INSERT 时回调中恢复 docToNode / nodeToDoc 映射"。两处矛盾。 - -有两种正确方案： -- (A) PutNode 接受 docId 参数，WAL INSERT 包含完整 docId，replay 可恢复映射 -- (B) WAL INSERT 不含 docId，映射恢复完全依赖 idmap.dat 加载（replay 不恢复映射） - -**建议**：明确选择方案并统一 Task 2.1 步骤 6 与 Task 2.2 步骤 2 的描述。推荐方案 B（映射从 idmap.dat 恢复），因为 idmap.dat 已有 CRC 校验且是映射的 source of truth。此时 Task 2.1 步骤 6 应删除 "恢复 docToNode / nodeToDoc 映射" 的描述。 - ---- - -**总结**：1 CRITICAL + 3 HIGH，均为遗漏或描述矛盾，无架构方向问题。修复后可执行。 diff --git a/docs/tasks/HAY-007-phase2-tasks.md b/docs/tasks/HAY-007-phase2-tasks.md deleted file mode 100644 index f8505eb5..00000000 --- a/docs/tasks/HAY-007-phase2-tasks.md +++ /dev/null @@ -1,348 +0,0 @@ -# HAY-007 Phase 2：写入路径 — 详细执行计划 - -> 日期：2026-04-18 -> 基于：`docs/design/HAY-007-mmap-store.md` v2 + `docs/tasks/HAY-007-task-breakdown.md` -> 前置：Phase 1 已合入 main（mmap 封装、文件格式、Open/Close、只读路径、导出） - ---- - -## Task 2.1：WAL 实现（append / CRC32 / LSN / replay） - -**目标**：实现 append-only WAL，支持 5 种记录类型、CRC32 校验、LSN 单调递增、顺序 replay。 - -**文件**：`internal/core/vectorindex/mmap_wal.go`（新建，~250 行） - -**步骤**： - -1. **定义 WAL 常量与记录类型**（~20 行） - - `WalRecordType` 枚举：`WalInsert=1, WalDelete=2, WalSetNeighbors=3, WalSetEntry=4, WalSetNorm=5` - - WAL record 磁盘布局：`LSN(8) + Length(4) + Type(1) + Payload(var) + CRC32(4)` - -2. **WAL 结构体**（~30 行） - ```go - type WAL struct { - file *os.File - lsn uint64 - mu sync.Mutex - buf *bufio.Writer // buffered writes for batch mode - } - ``` - - `OpenWAL(path string) (*WAL, error)` — 打开/创建 wal.bin，扫描确定当前 LSN - - `Close() error` — flush + close - -3. **Append 方法**（~50 行） - - `Append(typ WalRecordType, payload []byte) (uint64, error)` — 写 header + payload + CRC32，返回 LSN - - CRC32 范围覆盖 `LSN + Length + Type + Payload`（与设计文档一致） - - 非 batch 模式下 Append 后立即 fsync；batch 模式下只写 buffer - -4. **Flush / Sync 方法**（~20 行） - - `Flush() error` — flush buffer - - `Sync() error` — flush + fsync - -5. **Payload 编码辅助**（~60 行） - - `EncodeInsert(nodeId uint64, level int, vec []float32, norm float32, docId string) []byte` - - `EncodeDelete(nodeId uint64, docId string) []byte` - - `EncodeSetNeighbors(nodeId uint64, layer int, neighbors []uint64) []byte` - - `EncodeSetEntry(entryId uint64, maxLevel int) []byte` - - `EncodeSetNorm(nodeId uint64, norm float32) []byte` - - 对应的 `Decode*` 函数 - -6. **Replay 方法**（~70 行） - - `Replay(afterLSN uint64, fn func(lsn uint64, typ WalRecordType, payload []byte) error) error` - - 顺序扫描 wal.bin，跳过 `lsn <= afterLSN` - - CRC32 校验：不匹配或不完整记录 → 丢弃（截断到最后一条有效记录） - - replay 完成后，根据已扫描记录恢复 meta 字段： - - `NodeCount`：统计 INSERT 记录数 - - `NextNodeId`：max(已见 nodeId) + 1，避免重启后 ID 冲突 - - `TotalSlots`：max(已见 nodeId) + 1 - - `EntryPoint` / `EntryLevel`：从最后一条 SET_ENTRY 记录恢复 - - `MaxLevel`：从 INSERT 记录的 level 字段取最大值 - - `GraphUpperNextSlot`：从 INSERT level>0 记录推算 - - **注意**：docToNode / nodeToDoc 映射由 idmap.dat 加载恢复，WAL replay 不负责映射恢复 - -**预估行数**：~250 行 - -**验证方式**： -- 单元测试 `mmap_wal_test.go`（~150 行）： - - 写 N 条各类型记录 → Replay → LSN 连续、CRC 正确、payload 解码一致 - - 截断文件模拟不完整写入 → Replay 丢弃最后一条 → 前 N-1 条完整 - - 篡改中间记录的 1 字节 → CRC 校验失败 → Replay 在该记录处停止 - - `afterLSN` 过滤：Replay(afterLSN=5) 只回放 LSN>5 的记录 - ---- - -## Task 2.2：写方法实现（PutNode / SetNeighbors / SetNorm / SetNodeMapping / SetEntryPoint） - -**目标**：实现 `NodeStore` 接口的所有写方法，每个写操作先 WAL 再写 mmap。 - -**文件**：`internal/core/vectorindex/mmap_store_write.go`（新建，~220 行） - -**步骤**： - -1. **MmapStore 集成 WAL 字段**（修改 `mmap_store.go`，~15 行） - - 在 `MmapStore` struct 添加 `wal *WAL`、`batchMode bool`、`batchDepth int` - - `OpenMmapStore` 中 `OpenWAL(dir)` + 加载 idmap.dat 恢复 docToNode/nodeToDoc（顺序读取所有 entry，CRC 校验，填充内存 map）+ 调用 replay 恢复状态 - - `Close()` 完整步骤序列（顺序严格）： - 1. msync 所有 mmap 区域（确保数据页落盘） - 2. `wal.Sync()`（flush + fsync WAL） - 3. `writeMetaHeader()`（写入最终 meta，含 WalCheckpointLSN） - 4. `wal.Close()` - 5. munmap 所有映射 - 6. close 所有文件 - -2. **PutNode(id uint64, level int, vec []float32) error**（~50 行） - - 计算 norm = L2Norm(vec) - - WAL Append(INSERT, EncodeInsert(id, level, vec, norm, docId)) - - **注意**：docId 必须由调用方传入（不能为空），因 WAL replay 需要 docId 恢复 meta 中的 NodeCount 等字段。映射恢复虽依赖 idmap.dat，但 INSERT 记录的完整性要求包含 docId。PutNode 签名需增加 docId 参数：`PutNode(id uint64, level int, vec []float32, docId string) error` - - 如果 `id >= vecCapacity` → 调用 growFile（Task 2.4） - - 写 `vectors.dat[pageSize + id*slotSize]`：逐 float32 LittleEndian 写入 - - 写 `nodes.dat[pageSize + id*nodeSlotSize]`：Level=level, Flags=0, Norm=norm - - 如果 `level > 0`：分配 upper slot（`meta.GraphUpperNextSlot++`），写 `nodes.dat` UpperSlot 字段 - - 更新 `meta.TotalSlots`（如果 id >= TotalSlots）、`meta.NodeCount++` - - 非 batch 模式 → msync - -3. **SetNeighbors(id uint64, layer int, neighbors []uint64) error**（~50 行） - - WAL Append(SET_NEIGHBORS, EncodeSetNeighbors(id, layer, neighbors)) - - layer==0：写 `graphL0[pageSize + id*l0SlotSize]`：count(uint32) + neighbors([]uint64) - - layer>0：读 nodes.dat UpperSlot → 写 `graphUpper[pageSize + slot*upperSlotSz + (layer-1)*layerSize]` - - 非 batch 模式 → msync - -4. **SetNorm(id uint64, norm float32) error**（~20 行） - - WAL Append(SET_NORM, EncodeSetNorm(id, norm)) - - 写 `nodes.dat[pageSize + id*nodeSlotSize + 4]`：Norm(float32) - - 非 batch 模式 → msync - -5. **SetEntryPoint(id uint64, maxLayer int) error**（~15 行） - - WAL Append(SET_ENTRY, EncodeSetEntry(id, maxLayer)) - - 更新 `s.meta.EntryPoint = id`、`s.meta.EntryLevel = maxLayer`、`s.meta.MaxLevel = max(meta.MaxLevel, maxLayer)` - -6. **SetNodeMapping(docId string, nodeId uint64) error**（~40 行） - - 更新内存 map：`s.docToNode[docId] = nodeId`、`s.nodeToDoc[nodeId] = docId` - - 追加写 `idmap.dat`：`NodeId(8) + DocIdLen(2) + DocId(var) + CRC32(4)` - - CRC32 覆盖整条 entry（NodeId + DocIdLen + DocId） - - 需要 `s.idmapFile *os.File` 字段（在 Open 时打开） - -7. **DeleteNodeMapping(docId string) error**（~15 行） - - 从内存 map 删除对应条目 - - idmap.dat 暂不写删除标记（Phase 3 compact 时清理） - -8. **msync 辅助**（~15 行） - - `syncRegion(data []byte) error` — 仅在非 batch 模式下调用 `mmapSync` - - 需要在 mmap.go / mmap_unix.go / mmap_windows.go 中增加 `mmapSync` 函数 - -**文件变更明细**： -| 文件 | 变更 | 行数 | -|------|------|------| -| `mmap_store_write.go` | 新建，所有写方法 | ~200 | -| `mmap_store.go` | 添加 WAL/batch/idmap 字段，修改 Open/Close | ~20 | -| `mmap.go` | 添加 `mmapSync` 签名 | ~3 | -| `mmap_unix.go` | 添加 `mmapSync` 实现（`syscall.Msync`） | ~10 | -| `mmap_windows.go` | 添加 `mmapSync` 实现（`FlushViewOfFile`） | ~15 | - -**预估行数**：~250 行（含 msync 跨平台） - -**验证方式**： -- 单元测试 `mmap_store_write_test.go`（~200 行）： - - PutNode → GetVector 返回写入值 - - PutNode → GetNorm 返回正确 norm - - PutNode(level=2) → GetNodeLevel 返回 2，upper slot 已分配 - - SetNeighbors(id, 0, nbs) → GetNeighbors(id, 0) 返回一致 - - SetNeighbors(id, 2, nbs) → GetNeighbors(id, 2) 返回一致（需先 PutNode level>=2） - - SetNorm → GetNorm 返回新值 - - SetEntryPoint → GetEntryPoint 返回一致 - - SetNodeMapping("doc1", 42) → GetNodeId("doc1") 返回 (42, true) - - SetNodeMapping → Close → Open → GetNodeId 返回正确（idmap.dat 持久化验证） - - DeleteNodeMapping → GetNodeId 返回 (_, false) - ---- - -## Task 2.3：BatchableStore 接口实现 - -**目标**：实现 `BeginBatch / CommitBatch / DiscardBatch / BatchDepth`，batch 模式下延迟 sync。 - -**文件**：`mmap_store_write.go` 追加（~60 行） - -**步骤**： - -1. **BeginBatch()**（~5 行） - - `s.batchDepth++`、`s.batchMode = true` - -2. **CommitBatch(sync bool) error**（~30 行） - - `s.batchDepth--` - - 如果 `batchDepth == 0`： - - `s.batchMode = false` - - `s.wal.Flush()` — flush WAL buffer - - 如果 `sync == true`：`s.wal.Sync()` + msync 所有 mmap 区域 - - 返回 error - -3. **DiscardBatch()**（~10 行） - - `s.batchDepth = 0`、`s.batchMode = false` - - 注意：mmap 写入无法回滚（已写入页面），但 WAL 可以截断——初版不做 WAL 回滚，与设计文档一致 - -4. **BatchDepth() int**（~3 行） - -5. **syncAll() error**（~15 行） - - msync vectors + nodes + graphL0 + graphUpper - -**预估行数**：~60 行 - -**验证方式**： -- 单元测试（追加到 `mmap_store_write_test.go`，~60 行）： - - BeginBatch → 多次 PutNode/SetNeighbors → CommitBatch → 数据可读 - - 验证 batch 模式下 WAL 只在 CommitBatch 时 fsync（可通过 mock 或计数器验证） - - BatchDepth 嵌套：BeginBatch×2 → CommitBatch → depth=1 → CommitBatch → depth=0 - - DiscardBatch 重置 depth 为 0 - ---- - -## Task 2.4：文件增长（grow）+ 并发安全 - -**目标**：当写入超过当前 capacity 时自动 2x 扩展文件，保证并发读取安全。 - -**文件**：`internal/core/vectorindex/mmap_store_grow.go`（新建，~150 行） - -**步骤**： - -1. **通用 growFile 方法**（~60 行） - ```go - func (s *MmapStore) growFile(which fileType, requiredCap uint64) error - ``` - - `fileType` 枚举：`fileVectors, fileNodes, fileGraphL0, fileGraphUpper` - - 计算 newCap = 当前 cap × 2（循环直到 >= requiredCap） - - 获取对应文件的写锁（muVec / muNodes / muGraph） - - munmap 旧映射 - - ftruncate 扩展文件到 `pageSize + newCap * slotSize` - - 更新文件 header 中的 Capacity 字段 - - 重新 mmap（mmapAlloc，读写模式） - - 更新 MmapStore 中的 slice 引用和 capacity 字段 - - 释放写锁 - -2. **按文件类型分发**（~40 行） - - `growVectors(requiredCap)` — 锁 muVec - - `growNodes(requiredCap)` — 锁 muNodes - - `growGraphL0(requiredCap)` — 锁 muGraph - - `growGraphUpper(requiredCap)` — 锁 muGraph - - 各函数调用通用 growFile 逻辑，传入对应的 file/data/cap 指针 - -3. **ensureCapacity 辅助**（~20 行） - - `ensureVecCapacity(id uint64) error` — if id >= cap → growVectors(id+1) - - `ensureNodeCapacity(id uint64) error` — 同上 - - `ensureL0Capacity(id uint64) error` — 同上 - - PutNode 调用这些方法确保 slot 存在 - -4. **并发安全模型**（~30 行） - - 读操作已持 RLock（Phase 1 实现），grow 需要 Lock → 自然等待所有 reader 完成 - - grow 完成后释放 Lock，新 reader 获取 RLock 看到新的 mmap slice - - 注意：grow 期间同一文件的读操作阻塞，但不同文件的读操作不受影响 - -**预估行数**：~150 行 - -**验证方式**： -- 单元测试 `mmap_store_grow_test.go`（~120 行）： - - 初始 cap=1024 → 插入 id=1024 → 触发 grow → cap=2048 → GetVector(1024) 正确 - - 插入 id=5000 → 多次 grow → 最终 cap >= 5001 - - 验证 grow 后旧数据（id=0..1023）仍可正确读取 - - 并发测试：10 goroutine 循环 GetVector + 1 goroutine 连续 PutNode 超 capacity × 3 → 零 panic，所有读写正确 - - grow graphUpper：PutNode(level=3) 当 upper slot 满时 → auto grow → 正确 - ---- - -## Task 2.5：NextNodeId（简单自增） - -**目标**：实现 `NextNodeId()` 返回单调递增的 node ID，Phase 3 才加 freelist。 - -**文件**：`mmap_store_write.go` 追加（~15 行） - -**步骤**： - -1. **NextNodeId() (uint64, error)**（~10 行） - - `id := s.meta.NextNodeId` - - `s.meta.NextNodeId++` - - 返回 id - - 注意：无需锁保护——HNSW Insert 持 h.mu 互斥锁，NextNodeId 只在 Insert 中调用 - -2. **Open 时恢复**（~5 行） - - `s.meta.NextNodeId` 从 meta.bin 读取（已在 Phase 1 Open 中实现） - -**预估行数**：~15 行 - -**验证方式**： -- 单元测试（追加到 `mmap_store_write_test.go`，~20 行）： - - 新 store → NextNodeId() 连续返回 0, 1, 2, ... - - Close → Open → NextNodeId() 从上次的 next 值继续 - ---- - -## Task 2.6：50K Insert 集成 benchmark - -**目标**：验证 50K×128d 仅 level-0 路径的 Insert 性能 < 90s。 - -**文件**：`internal/core/vectorindex/mmap_store_bench_test.go`（新建，~100 行） - -**步骤**： - -1. **Benchmark 搭建**（~40 行） - - 创建临时目录 → OpenMmapStore(dim=128, M=16) - - 预生成 50K 随机 float32[128] 向量 - -2. **Level-0 only Insert 循环**（~30 行） - - BeginBatch - - 循环 50K 次： - - NextNodeId - - PutNode(id, level=0, vec) - - SetNodeMapping(fmt.Sprintf("doc-%d", id), id) - - SetNeighbors(id, 0, randomNeighbors) — 模拟 HNSW 连接（随机 M*2 个已有节点） - - CommitBatch(true) - - 记录耗时 - -3. **输出与断言**（~30 行） - - `b.ReportMetric(elapsed.Seconds(), "total_sec")` - - `b.ReportMetric(float64(50000)/elapsed.Seconds(), "inserts/sec")` - - 验证：随机抽样 100 个节点 GetVector 正确 - - 验证：GetNodeId 往返正确 - - **注意**：此 benchmark 不含 HNSW 搜索邻居的 GetVectorRef 调用，仅测写入路径吞吐。真实 E2E 性能在 Phase 5 Task 5.3 验证 - -**预估行数**：~100 行 - -**验证方式**： -- `go test -bench BenchmarkMmapStore50KInsert -benchtime 1x -timeout 180s` -- 目标：< 90s（仅 level-0 写入，无搜索开销，预期远低于 90s） -- 输出 inserts/sec 指标供后续对比 - ---- - -## 执行顺序与依赖关系 - -``` -2.1 WAL ──────────┐ - ├──→ 2.2 写方法 ──→ 2.3 Batch ──→ 2.6 Benchmark -2.4 grow ─────────┘ │ - │ -2.5 NextNodeId ─────────────┘ -``` - -- **2.1 WAL** 和 **2.4 grow** 可并行开发（无依赖） -- **2.2 写方法** 依赖 2.1（WAL）和 2.4（grow），是核心集成点 -- **2.5 NextNodeId** 逻辑独立但被 2.2 调用，可与 2.2 同步实现 -- **2.3 Batch** 依赖 2.2（需要写方法存在才能测试 batch 效果） -- **2.6 Benchmark** 依赖 2.2 + 2.3 + 2.5 全部完成 - ---- - -## 总变更汇总 - -| 文件 | 操作 | 预估行数 | -|------|------|----------| -| `mmap_wal.go` | 新建 | ~250 | -| `mmap_store_write.go` | 新建 | ~275（写方法 + Batch + NextNodeId） | -| `mmap_store_grow.go` | 新建 | ~150 | -| `mmap_store.go` | 修改（添加字段、修改 Open/Close） | ~25 | -| `mmap.go` | 修改（添加 mmapSync 签名） | ~3 | -| `mmap_unix.go` | 修改（添加 mmapSync） | ~10 | -| `mmap_windows.go` | 修改（添加 mmapSync） | ~15 | -| `mmap_wal_test.go` | 新建 | ~150 | -| `mmap_store_write_test.go` | 新建 | ~280 | -| `mmap_store_grow_test.go` | 新建 | ~120 | -| `mmap_store_bench_test.go` | 新建 | ~100 | -| **总计（生产代码）** | | **~728** | -| **总计（测试代码）** | | **~650** | diff --git a/docs/tasks/HAY-007-phase3-tasks.md b/docs/tasks/HAY-007-phase3-tasks.md deleted file mode 100644 index 7d84d742..00000000 --- a/docs/tasks/HAY-007-phase3-tasks.md +++ /dev/null @@ -1,207 +0,0 @@ -# HAY-007 Phase 3: Checkpoint & Recovery — Task Breakdown - -> 对应设计文档 Phase 4: 崩溃恢复 + Checkpoint -> 分支: `feat/hay-007-phase3-checkpoint` - ---- - -## 已有基础（Phase 2） - -| 能力 | 文件 | 状态 | -|------|------|------| -| WAL append (LSN + CRC32) | `mmap_wal.go:124` Append() | ✅ | -| WAL replay (afterLSN filter) | `mmap_wal.go:196` Replay() | ✅ | -| WAL scanLSN (启动扫描最大 LSN) | `mmap_wal.go:68` scanLSN() | ✅ | -| WAL truncate (CRC 不匹配截断) | `mmap_wal.go:112,250` | ✅ | -| replayWAL (5 种 record 全处理) | `mmap_store.go:394` | ✅ | -| meta.bin 原子写 (tmp→fsync→rename) | `mmap_format.go:110` writeMetaHeader() | ✅ | -| MetaHeader.WalCheckpointLSN 字段 | `mmap_format.go:38` | ✅ | -| Open 时 replay from WalCheckpointLSN | `mmap_store.go:144-150` | ✅ | -| Close 时 writeMetaHeader | `mmap_store.go:176` | ✅ | - ---- - -## Task 1: WAL Checkpoint 方法 - -**目标**: 实现 `(*MmapStore).Checkpoint()` — msync 所有 mmap → writeMetaHeader(含当前 walLSN) → truncate WAL - -**文件**: `mmap_store_write.go`（新增方法）, `mmap_wal.go`（新增 `Truncate(offset)` 或 `Reset()` 方法） - -**具体步骤**: -1. 在 `mmap_wal.go` 新增 `(*WAL).Reset()` — truncate 文件到 0 并 seek 到 0，重置 buf，**但保留 nextLSN 不变**（即 Reset 后 nextLSN = checkpointLSN + 1，确保 LSN 单调递增） -2. 在 `mmap_store_write.go` 新增 `(*MmapStore).Checkpoint() error`: - - msync 所有 4 个 mmap 区域（vectors, nodes, graphL0, graphUpper） - - `s.meta.WalCheckpointLSN = s.wal.LSN()` — 在 WAL 上新增 `LSN() uint64` 访问器 - - `writeMetaHeader(s.dir, &s.meta)` - - `s.wal.Reset()` — truncate WAL - - `s.idmap.Compact()` — 对 idmap 做 compaction，回收已删除条目的空间 -3. 在 `mmap_wal.go` 新增 `(*WAL).LSN() uint64` 返回当前 lastLSN - -**预估行数**: ~40 行 -**验证方式**: -- 单元测试：写 N 条记录 → Checkpoint() → 验证 meta.bin 中 WalCheckpointLSN = N → 验证 wal.bin 大小 = 0 -- 单元测试：Checkpoint 后继续写 → 再次 Replay 只看到新记录 -- 单元测试：Reset 后新写入的 LSN > checkpoint LSN（验证 LSN 不清零） - ---- - -## Task 2: Close 调用 Checkpoint - -**目标**: `Close()` 在 munmap 前执行完整 checkpoint，确保 clean shutdown 后 WAL 为空 - -**文件**: `mmap_store.go`（修改 Close） - -**具体步骤**: -1. 将 Close() 中现有的 msync + writeMetaHeader 替换为 `s.Checkpoint()` 调用 -2. Checkpoint 已包含 msync + writeMeta + WAL truncate，Close 后续只做 WAL.Close + idmap.Close + munmap -3. Open 中 replay 完成后自动调用 `s.Checkpoint()`，将 replay 恢复的状态持久化，避免每次 Open 都重放相同 WAL - -**预估行数**: ~10 行（净减少，逻辑合并） -**验证方式**: -- 现有测试全部通过（行为不变，只是多了 WAL truncate） -- 新测试：Open → 写 N 条 → Close → 检查 wal.bin 大小 = 0，meta.bin WalCheckpointLSN = N -- 新测试：Open → 写 N 条 → crash → 重新 Open → 验证 replay 后自动 checkpoint（WAL 被 truncate，meta 更新） - ---- - -## Task 3: 自动 Checkpoint（每 N 次操作） - -**目标**: 所有写路径（batch 和非 batch）在累计操作数超过阈值时自动 checkpoint - -**文件**: `mmap_store_write.go`（修改 CommitBatch 及所有非 batch 写方法）, `mmap_store.go`（新增 opsSinceCheckpoint 字段） - -**具体步骤**: -1. MmapStore 新增 `opsSinceCheckpoint uint64` 字段和 `checkpointInterval uint64`（默认 1000） -2. `MmapStoreOptions` 新增 `CheckpointInterval` 可配置字段 -3. WAL Append 时递增 opsSinceCheckpoint（这样所有通过 WAL 的写路径都自动计数） -4. CommitBatch(sync=true) 末尾：if opsSinceCheckpoint >= checkpointInterval → Checkpoint() -5. 非 batch 写方法（PutNode, SetNeighbors, DeleteNode 等直接写路径）末尾同理检查阈值并触发 checkpoint - -**预估行数**: ~25 行 -**验证方式**: -- 单元测试：设 interval=10，写 15 条，验证中间触发了 checkpoint（WAL 被 truncate 过） -- 单元测试：设 interval=1000，写 5 条，验证未触发 checkpoint（WAL 保留） - ---- - -## Task 4: 崩溃恢复集成测试 — 基本 replay - -**目标**: 验证 crash 后 Open 能正确恢复：写入 → 不调 Close（模拟 crash）→ 重新 Open → 数据完整 - -**文件**: `mmap_checkpoint_test.go`（新文件） - -**具体步骤**: -1. TestCrashRecovery_BasicReplay： - - Open → PutNode × N（带 BeginBatch/CommitBatch）→ 直接关闭 fd（不调 Close） - - 重新 Open → 验证所有 N 个节点可读且数据正确 -2. TestCrashRecovery_AfterCheckpoint： - - Open → 写 N 条 → Checkpoint() → 写 M 条 → crash - - 重新 Open → 验证 N+M 条数据完整 - -**预估行数**: ~80 行 -**验证方式**: `go test -run TestCrashRecovery -v` - ---- - -## Task 5: Crash Point 注入框架 - -**目标**: 提供 crash point hook，允许测试在特定写操作之间注入 panic 来模拟 5 类崩溃 - -**文件**: `mmap_store.go`（新增 hook 字段）, `mmap_checkpoint_test.go` - -**具体步骤**: -1. MmapStore 新增 `testing` 用 hook：`crashAfterMsync func()`, `crashAfterMeta func()`, `crashBeforeTruncate func()` -2. 在 Checkpoint() 流程中插入 hook 调用点（仅非 nil 时调用，零生产开销） -3. WAL Append 中同理：`crashAfterWALWrite func()` - -**预估行数**: ~30 行 -**验证方式**: 下一个 task 使用这些 hook - ---- - -## Task 6: 8 类 Crash Point 测试 - -**目标**: 覆盖设计文档要求的 5 类崩溃场景 + grow/SetNeighbors/DeleteNode 中途 crash - -**文件**: `mmap_checkpoint_test.go` - -**具体测试**: - -### 6a. WAL 写后 msync 前 -- 写 WAL record → crash → Open → WAL replay 恢复数据到 mmap - -### 6b. msync 后 meta.bin 写前 -- Checkpoint: msync 完成 → crash → Open → meta 仍是旧 checkpoint LSN → WAL replay 补回 -- 因为 mmap 已 msync，replay 是幂等的（覆盖相同数据），不会损坏 - -### 6c. meta.bin 写后 WAL truncate 前 -- Checkpoint: meta 已更新 → crash → Open → WalCheckpointLSN 已更新 → replay 跳过已 checkpoint 的记录 -- WAL 有残留旧记录但 LSN ≤ checkpoint 会被跳过 - -### 6d. Partial WAL record（写到一半的记录） -- 手动向 wal.bin 追加不完整字节 → Open → scanLSN/Replay truncate 不完整记录 → 数据完整 - -### 6e. Partial meta.bin write（meta 写坏） -- 将 meta.bin 截断为 32 字节 → Open 应 fallback 或报错 -- 或：meta.bin.tmp 存在但 rename 未完成 → 旧 meta.bin 仍有效 - -### 6f. grow 中途 crash -- 写入触发 grow（mmap 扩容）→ grow 完成前 crash → Open → 验证数据完整且 mmap 区域一致 -- 测试 grow 后 meta 未更新场景：旧 capacity 的 meta + WAL 中有超出旧 capacity 的记录 → replay 时需重新 grow - -### 6g. SetNeighbors 中途 crash -- 写 WAL SetNeighbors record → crash（msync 前）→ Open → WAL replay 恢复 neighbor 关系正确 - -### 6h. DeleteNode 中途 crash -- 写 WAL DeleteNode record → crash（msync 前）→ Open → WAL replay 恢复删除状态，节点不可见 - -**预估行数**: ~220 行 -**验证方式**: `go test -run TestCrashPoint -v`，每个子测试独立 pass - ---- - -## Task 7: kill -9 验收测试 - -**目标**: 端到端验收 — 真实进程 kill 后重启索引完整 - -**文件**: `mmap_checkpoint_test.go`（或独立 `mmap_kill_test.go`） - -**具体步骤**: -1. 使用 `os/exec` 启动子进程写入索引（helper binary 或 TestMain 模式） -2. 子进程写入 1000 个向量后 sleep -3. 父进程 kill -9 子进程 -4. 父进程 Open 同一目录 → 验证至少 checkpoint 过的数据完整（部分未 checkpoint 的通过 WAL replay 恢复） - -**预估行数**: ~60 行 -**验证方式**: `go test -run TestKill9Recovery -v -count=3`（多次运行验证稳定性） - ---- - -## 依赖关系 - -``` -Task 1 (Checkpoint 方法) - ↓ -Task 2 (Close 调 Checkpoint) Task 3 (自动 Checkpoint) - ↓ ↓ -Task 4 (基本 replay 测试) ←────────┘ - ↓ -Task 5 (crash hook 框架) - ↓ -Task 6 (5 类 crash 测试) - ↓ -Task 7 (kill -9 验收) -``` - -## 总预估 - -| Task | 行数 | 文件 | -|------|------|------| -| 1. Checkpoint 方法 | ~40 | mmap_wal.go, mmap_store_write.go | -| 2. Close 调 Checkpoint | ~10 | mmap_store.go | -| 3. 自动 Checkpoint | ~30 | mmap_store_write.go, mmap_store.go | -| 4. 基本 replay 测试 | ~80 | mmap_checkpoint_test.go | -| 5. Crash hook 框架 | ~30 | mmap_store.go, mmap_checkpoint_test.go | -| 6. 8 类 crash 测试 | ~220 | mmap_checkpoint_test.go | -| 7. kill -9 验收 | ~60 | mmap_kill_test.go | -| **总计** | **~470** | | diff --git a/docs/tasks/HAY-007-phase4-tasks.md b/docs/tasks/HAY-007-phase4-tasks.md deleted file mode 100644 index 117d3307..00000000 --- a/docs/tasks/HAY-007-phase4-tasks.md +++ /dev/null @@ -1,133 +0,0 @@ -# HAY-007 Phase 4: HNSW Integration + E2E - -> 对应设计文档 Phase 5（上层图 + 完整集成） -> 前置：Phase 1-3 已完成（只读/写入/WAL/Checkpoint/Crash Recovery） - -## 前置确认（已验证） - -- [x] MmapStore 完整实现 NodeStore 接口（14 方法） -- [x] MmapStore 完整实现 BatchableStore 接口（4 方法） -- [x] GetVectorRef 已实现（初版 copy，同 GetVector） -- [x] HNSW `NewHNSWIndex(store NodeStore, ...)` 可直接传入 MmapStore - ---- - -## Task 1: HNSW + MmapStore 基本集成测试 - -**文件**: `mmap_store_hnsw_test.go` - -- [x] 1.1 小规模 Insert → Search 测试（100 vectors, 128d） - - OpenMmapStore → NewHNSWIndex → Insert 100 docs → Search top-10 → 验证结果非空且有序 - - 参考 `hnsw_integration_test.go:49-68` 的 MemStore 模式 -- [x] 1.2 Insert → Delete → Search 测试 - - Insert 50 docs → Delete 10 → Search → 验证已删除 doc 不在结果中 -- [x] 1.3 Insert → Search → Delete → Re-insert → Search - - 验证 freelist slot 复用后搜索仍然正确 -- [x] 1.4 Upsert 测试 - - Insert doc → Upsert 同 docId 新向量 → Search → 验证返回新向量的结果 - -## Task 2: 持久化验证测试 - -**文件**: `mmap_store_hnsw_test.go` - -- [x] 2.1 写入 → Close → 重新打开 → Search - - Insert 200 docs → Close store → OpenMmapStore 同目录 → NewHNSWIndex → Search → recall 与关闭前一致 -- [x] 2.2 写入 → Close → 重新打开 → 继续 Insert → Search - - 验证持久化后可增量追加 -- [x] 2.3 写入 → Close → 重新打开 → Delete → Search - - 验证持久化后可正确删除 -- [x] 2.4 WAL replay → HNSW Search 完整 E2E - - Insert N docs → 模拟 crash（不调用 Close/Checkpoint）→ 重新 OpenMmapStore（触发 WAL replay）→ NewHNSWIndex → Search → 验证 recall 与 crash 前一致 - - 覆盖场景：crash 发生在 Insert 中途 / Delete 中途 / Checkpoint 中途 - -## Task 3: 50K SIFT-128 E2E Benchmark - -**文件**: `mmap_store_bench_test.go` - -- [x] 3.1 实现 SIFT-128 数据加载工具 - - 使用 SIFT-128 数据集（ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz） - - 实现 fvecs/ivecs 格式解析，取前 50K base vectors + 100 query vectors + ground truth - - 禁止使用合成数据：SIFT-128 提供真实分布和标准 ground truth，确保 recall 可比 -- [x] 3.2 MemStore 50K Insert benchmark（baseline） - - `BenchmarkHNSW_MemStore_50K_Insert` -- [x] 3.3 MmapStore 50K Insert benchmark - - `BenchmarkHNSW_MmapStore_50K_Insert` - - 目标 < 90s -- [x] 3.4 Search benchmark（MemStore vs MmapStore） - - 1000 次随机 Search top-10，记录 p50/p99 延迟 - - MmapStore 目标 p99 < 5ms - -## Task 4: Recall@10 验证 - -**文件**: `mmap_store_bench_test.go` 或 `mmap_store_hnsw_test.go` - -- [x] 4.1 实现 recall@K 计算工具 - - brute-force KNN 作为 ground truth → 与 HNSW 结果对比 -- [x] 4.2 MemStore recall@10 验证（baseline） - - 50K 128d 数据，100 query，recall@10 > 0.95 -- [x] 4.3 MmapStore recall@10 验证 - - 同数据同 query，recall@10 > 0.95 - - 验证 MmapStore recall 与 MemStore 一致（差异 < 0.01） - -## Task 5: graph_upper.dat 上层图验证 - -**文件**: `mmap_store_hnsw_test.go` - -- [x] 5.1 多层节点生成与上层图读写验证 - - 使用 efConstruction=200 + 固定 seed，Insert 足够数据（≥5000 vectors）确保生成 level>0 节点 - - 验证 graph_upper.dat 中上层图邻居列表的正确性：读出 level>0 节点的邻居，与 MemStore 基线对比 -- [x] 5.2 上层图持久化 reopen 验证 - - 写入含多层节点的 HNSW → Close → Reopen → Search → 验证 recall 与关闭前一致 - - 确认 entry point 和 maxLevel 正确恢复 -- [x] 5.3 上层图 grow 与 crash recovery - - 触发 graph_upper.dat grow（大量 Insert 使上层图扩容）→ 在 grow 过程中模拟 crash - - Replay WAL → 验证上层图结构完整，Search 正确 - - 验证 upper slot 分配在 crash 后无泄漏 - -## Task 6: MemStore → MmapStore 导出集成验证 - -**文件**: `mmap_store_hnsw_test.go` - -- [x] 6.1 MemStore 构建 HNSW → 导出 → MmapStore 打开 → Search - - 验证 ExportToMmapStore 后 recall 无损 -- [x] 6.2 导出后继续 Insert/Delete - - 验证导出的 MmapStore 可正常增删 - -## Task 7: 生产集成声明 - -> **本 Phase 范围限定为验证测试。** MmapStore 作为 HNSW 可选 backend 的生产集成（在 hnsw.go 或配置层暴露 MmapStore 选项、提供 `WithMmapBackend()` 等 API）将在独立 issue 中跟进。 -> 本 Phase 的验收标准确认 MmapStore 通过 NodeStore 接口与 HNSW 完全兼容后，即可开始生产集成工作。 - ---- - -## 验收标准 - -| 指标 | 目标 | -|------|------| -| 50K×128d MmapStore Insert | < 90s | -| Search p99 | < 5ms | -| Recall@10 (SIFT-like) | > 0.95 | -| MmapStore vs MemStore recall 差异 | < 0.01 | -| 持久化 reopen 后 search 正确 | pass | -| WAL replay 后 HNSW search 正确 | pass | -| graph_upper.dat 上层图读写正确 | pass | -| 上层图 grow crash recovery | pass | -| 所有现有测试 | pass | -| `CGO_ENABLED=0` 编译 | pass | -| CI 三平台验证（Linux amd64, macOS arm64, Windows amd64） | 全部 pass | - ---- - -## 完成状态 - -Phase 4 所有验证测试任务（Task 1-6）已完成并通过。 - -**已验证能力:** -- MmapStore 通过 NodeStore 接口与 HNSW 完全兼容（Insert/Search/Delete/Upsert） -- 持久化 close→reopen 搜索结果一致 -- WAL crash recovery 后搜索正确 -- Recall@10 > 0.95（MemStore vs MmapStore 差异 < 0.01） -- graph_upper.dat 多层节点正确生成、持久化、grow+crash recovery -- MemStore→MmapStore 导出无损 recall，支持后续 Insert/Delete - -**生产集成（Task 7 scope）:** 本 Phase 范围限定为验证测试。MmapStore 作为 HNSW backend 的生产 API 暴露（`WithMmapBackend()` 等）将在独立 issue 中跟进。 diff --git a/docs/tasks/HAY-007-review-cc-round1.md b/docs/tasks/HAY-007-review-cc-round1.md deleted file mode 100644 index f9f6d5f0..00000000 --- a/docs/tasks/HAY-007-review-cc-round1.md +++ /dev/null @@ -1,95 +0,0 @@ -# HAY-007 Task Breakdown Review — Round 1 - -> Reviewer: Claude Code (Opus 4) -> Date: 2026-04-17 -> 对照文档：`docs/design/HAY-007-mmap-store.md` v2 - ---- - -## CRITICAL - -### C1: Phase 2 缺少 WAL replay 实现任务 - -**设计文档 §4.5** 明确定义了恢复流程：从 `WalCheckpointLSN + 1` 开始 replay，逐条 CRC 校验，不完整记录丢弃。但 Phase 2 的 Task 2.1 只覆盖了 WAL append/CRC/LSN，**没有独立任务覆盖 replay 逻辑**。replay 被推迟到 Phase 4（Task 4.2），但 Phase 2 验证标准写道"WAL replay 正确"——矛盾。 - -**风险**：Phase 2 结束时 WAL 只能写不能读，无法验证写入正确性；Phase 4 才发现 WAL 格式有缺陷需要返工。 - -**建议**：将 WAL replay 基础逻辑（顺序扫描 + CRC 校验 + 回调）放入 Task 2.1，Phase 4 只做 Open 流程集成和 crash-point 注入。 - ---- - -### C2: Phase 1 缺少 graph_upper.dat 只读路径 - -**设计文档 §4.1** 定义了 `graph_upper.dat` 文件格式和 `GetNeighbors(id, layer>0)` 读路径。Task 1.4 列出了 `GetNeighbors(id, layer)` 并提到 "layer>0 读 graphUpper"，但 **Phase 1 没有任何任务创建/mmap graph_upper.dat 文件**。Task 1.3 的 `Open` 只说"初始化各文件"但未明确包含 graph_upper.dat。Task 5.1 才做 "graph_upper.dat 按需分配"。 - -**风险**：Phase 1 的 `GetNeighbors(id, layer>0)` 实现无法测试，Task 1.4 验证不完整。Phase 5 集成时才发现 upper graph 读路径有 bug。 - -**建议**：Task 1.3 显式列出 graph_upper.dat 的 Open/mmap，Task 1.4 增加 upper layer 读取测试。Task 5.1 聚焦"按需分配写入"而非读。 - ---- - -### C3: grow 与并发读的安全间隙未拆任务验证 - -**设计文档 §4.4** 描述 grow 流程：munmap → ftruncate → 重新 mmap。在 munmap 和重新 mmap 之间存在窗口，此时持有 RLock 的并发 reader 会访问已释放的 mmap 区域 → **SIGBUS/SIGSEGV**。 - -Task 2.4 验证写道"并发读不崩溃"，但没有独立任务设计并发安全方案。设计文档的 RWMutex 方案本身有缺陷：RLock holder 在 grow Lock() 等待期间仍在读旧 mmap。 - -**风险**：这是 mmap store 最高风险的技术点。仅靠 Task 2.4 一个 ~100 行任务无法同时实现 grow 逻辑 + 解决并发安全 + 验证正确性。 - -**建议**：(1) 拆出独立 Task 分析并发安全方案（如 double-buffer/epoch-based 切换），(2) Task 2.4 拆分为 grow 实现 + grow 并发压力测试两个子任务。 - ---- - -## HIGH - -### H1: idmap.dat 持久化与 WAL 的一致性依赖未明确 - -Task 3.4 实现 idmap.dat 持久化，Task 2.1 实现 WAL。但 **INSERT WAL 记录包含 docId**（设计文档 §4.1 WAL 类型），WAL replay 时需要恢复 idmap。两个任务分属不同 Phase，没有明确依赖关系。 - -**风险**：Phase 2 WAL replay INSERT 时无法恢复 docId→nodeId 映射（idmap 还没持久化），crash 恢复后 ID 映射丢失。 - -**建议**：Task 2.1 WAL replay 必须包含 INSERT 对应的内存 map 恢复逻辑（即使 idmap.dat compact 推迟到 Phase 3）。在 Task 2.1 依赖中显式标注。 - ---- - -### H2: Task 2.5 NextNodeId 依赖 freelist，但 freelist 在 Phase 3 - -Task 2.5 实现 `NextNodeId()`，设计文档 §4.2 写道 "freelist 有则 pop，否则 meta.NextNodeId++"。但 freelist 在 Phase 3 Task 3.1/3.2 才实现。 - -**风险**：不是功能 bug（Phase 2 没 Delete 所以 freelist 为空），但 Task 2.5 的代码必须预留 freelist 接口，否则 Phase 3 需要重写 NextNodeId。 - -**建议**：Task 2.5 说明中注明"初版只做 meta.NextNodeId++，Phase 3 补充 freelist 逻辑"，避免预留过度设计但也避免返工。 - ---- - -### H3: Phase 2 验收标准 "50K Insert < 90s" 缺少前置条件 - -Task 2.6 做 50K Insert benchmark，但 Phase 2 尚无 graph_upper.dat 写入（Phase 5 Task 5.1）。如果 HNSW Insert 需要写 upper layer neighbors，Phase 2 的 50K benchmark 会 panic 或 silently skip upper layers。 - -**风险**：Phase 2 benchmark 结果不反映真实性能；Phase 5 集成后性能退化超出 90s 目标。 - -**建议**：Task 2.6 明确说明 benchmark scope（仅 level-0，或 mock upper store），Phase 5 Task 5.3 才是完整 E2E benchmark。 - ---- - -### H4: Windows mmap truncate 问题（开放问题 #3）无任务覆盖 - -设计文档 §8 开放问题 #3 明确标注 "Windows 下 mmap 文件无法 truncate——需要先 munmap 再 truncate。grow 流程需测试"。但 Task 2.4（grow）和 Task 1.1（mmap 封装）均未提及 Windows 特殊处理。 - -**风险**：Phase 2 grow 实现在 Windows CI 上 panic，阻塞三平台交付。 - -**建议**：Task 1.1 增加 Windows grow 兼容性设计（munmap→truncate→mmap 顺序），Task 2.4 验证增加 Windows CI 显式测试。 - ---- - -### H5: SetNodeMapping 持久化时机不明 - -设计文档 §4.2 接口映射表：`SetNodeMapping` = "更新内存 map + 追加 idmap.dat（带 CRC）"。但 Task Breakdown 中 **Phase 2 没有任何任务覆盖 SetNodeMapping 的 idmap.dat 追加写**。Task 2.2 只列了 PutNode/SetNeighbors/SetNorm/SetNodeMapping，但 idmap.dat 写入在 Task 3.4。 - -**风险**：Phase 2 SetNodeMapping 只更新内存 map 不写盘，crash 后 ID 映射全丢。与 WAL replay 依赖同一问题（见 H1），但影响范围更大——正常 Close→Open 也丢映射。 - -**建议**：Task 2.2 或 Task 2.5 中增加 idmap.dat 追加写的基础实现（仅 append + CRC），Task 3.4 只做 compact + 损坏恢复。 - ---- - -*EOF — 3 CRITICAL, 5 HIGH* diff --git a/docs/tasks/HAY-007-review-cc-round2.md b/docs/tasks/HAY-007-review-cc-round2.md deleted file mode 100644 index 4d88822c..00000000 --- a/docs/tasks/HAY-007-review-cc-round2.md +++ /dev/null @@ -1,31 +0,0 @@ -# HAY-007 Task Breakdown Review — Round 2 - -> Reviewer: Claude Code (Opus 4) -> Date: 2026-04-17 -> 对照文档：`docs/design/HAY-007-mmap-store.md` v2 -> 上一轮：3 CRITICAL + 5 HIGH，已全部修复 - ---- - -## Round 1 修复验证 - -| ID | 问题 | 修复位置 | 状态 | -|----|------|----------|------| -| C1 | WAL replay 缺失 | Task 2.1 已包含 replay 基础逻辑（顺序扫描+CRC+不完整记录丢弃+回调），Phase 4 只做 Open 集成 | **PASS** | -| C2 | graph_upper.dat 只读路径缺失 | Task 1.3 显式包含 graph_upper.dat 创建/mmap，Task 1.4 增加 upper layer 读取测试 | **PASS** | -| C3 | grow 并发安全间隙 | Task 2.4 采用 atomic.Pointer epoch-based 方案，拆为 2.4a/2.4b 子任务含压力测试 | **PASS** | -| H1 | idmap 与 WAL replay 一致性 | Task 2.1 replay INSERT 时恢复内存 docId→nodeId 映射 | **PASS** | -| H2 | NextNodeId 依赖 freelist | Task 2.5 明确初版只做自增，Phase 3 补充 freelist | **PASS** | -| H3 | 50K benchmark 前置条件 | Task 2.6 明确仅 level-0 路径，Phase 5 Task 5.3 为完整 E2E | **PASS** | -| H4 | Windows mmap truncate | Task 1.1 已知限制段落说明 munmap→truncate→mmap 顺序，Task 2.4 处理 | **PASS** | -| H5 | SetNodeMapping 持久化时机 | Task 2.2 包含 idmap.dat 追加写（append+CRC），Task 3.4 只做 compact+损坏恢复 | **PASS** | - ---- - -## Round 2 新发现 - -**PASS** — 未发现新的 CRITICAL 或 HIGH 问题。 - ---- - -*EOF — 0 CRITICAL, 0 HIGH* diff --git a/docs/tasks/HAY-007-task-breakdown.md b/docs/tasks/HAY-007-task-breakdown.md deleted file mode 100644 index 82bd0146..00000000 --- a/docs/tasks/HAY-007-task-breakdown.md +++ /dev/null @@ -1,238 +0,0 @@ -# HAY-007: MmapStore Task Breakdown - -> 基于 `docs/design/HAY-007-mmap-store.md` v2 分解 -> 日期：2026-04-17 -> 修订：2026-04-17 — 根据 Round 1 Review 修复 C1-C3, H1-H5 - ---- - -## Phase 1：核心基础设施（只读路径 + mmap 封装） - -### Task 1.1：跨平台 mmap 封装 - -**目标**：自封装 `syscall.Mmap/Munmap`，支持 Linux/macOS/Windows，零 CGo。 - -**改动文件**： -- `internal/core/vectorindex/mmap_unix.go`（~40 行，build tag `//go:build !windows`） -- `internal/core/vectorindex/mmap_windows.go`（~50 行，build tag `//go:build windows`） -- `internal/core/vectorindex/mmap.go`（~30 行，公共类型/接口） - -**预估行数**：~120 行 - -**已知限制（H4）**：Windows 下 mmap 文件无法直接 truncate，grow 必须按 munmap→truncate→mmap 顺序执行。此为已知平台限制，不阻塞 Phase 1/2 交付；Task 2.4 grow 实现时在 Windows 路径中显式处理此顺序。 - -**验证方式**： -- 单元测试：创建临时文件 → mmap → 读写 → munmap → 验证内容 -- `CGO_ENABLED=0 go build ./...` 编译通过 -- `go test -run TestMmap` 通过 - ---- - -### Task 1.2：文件格式常量与 MetaHeader - -**目标**：定义所有文件 magic、header 结构体、序列化/反序列化工具函数。 - -**改动文件**： -- `internal/core/vectorindex/mmap_format.go`（~120 行） - - `MetaHeader` 结构体（64 bytes） - - `VectorsHeader`、`NodesHeader`、`GraphL0Header`、`GraphUpperHeader` 结构体 - - magic 常量 `"HNSW"`, `"VECS"`, `"NODE"`, `"GRL0"`, `"GRUP"`, `"IDMP"` - - `readMetaHeader()` / `writeMetaHeader()` （原子 rename） - - header 序列化辅助（`binary.LittleEndian`） - -**预估行数**：~120 行 - -**验证方式**： -- 单元测试：写 MetaHeader → 读回 → 字段一致 -- 测试 MetaHeader 精确 64 bytes（`unsafe.Sizeof`） -- 原子写测试：写入后读回正确 - ---- - -### Task 1.3：MmapStore 结构体与 Open/Close - -**目标**：实现 `MmapStore` 结构体，`Open(dir, dim, M)` 创建/打开索引目录，`Close()` 释放资源。 - -**改动文件**： -- `internal/core/vectorindex/mmap_store.go`（~180 行） - - `MmapStore` 结构体定义（所有字段） - - `OpenMmapStore(dir string, opts MmapStoreOptions) (*MmapStore, error)` - - 创建目录、初始化各文件（header + 初始容量 1024 slots） - - **显式包含 graph_upper.dat 的创建/mmap（C2 fix）** - - mmap 所有文件（vectors.dat, nodes.dat, graph_l0.dat, graph_upper.dat, meta.dat） - - `Close() error`：munmap all → flush meta - - 辅助：`initFile(path, magic, headerSize, slotSize, capacity)` - -**预估行数**：~180 行 - -**验证方式**： -- 单元测试：Open 空目录 → 文件全部创建（含 graph_upper.dat）→ Close → 再次 Open → meta 一致 -- Close 后 re-open 不报错 -- 文件大小 = headerSize + capacity × slotSize - ---- - -### Task 1.4：只读方法实现（GetVector / GetVectorRef / GetNeighbors / GetNorm / GetNodeLevel / GetEntryPoint） - -**目标**：实现 `NodeStore` 接口中的所有只读方法。 - -**改动文件**： -- `internal/core/vectorindex/mmap_store.go`（追加 ~120 行） - - `GetVector(id)` — copy from mmap slice - - `GetVectorRef(id)` — 初版同 GetVector（copy） - - `GetNeighbors(id, layer)` — layer=0 读 graphL0，**layer>0 读 graph_upper.dat（C2 fix：依赖 Task 1.3 的 graph_upper.dat mmap）** - - `GetNorm(id)` — 读 nodes.dat slot - - `GetNodeLevel(id)` — 读 nodes.dat slot - - `GetEntryPoint()` — 从 meta 返回 - -**预估行数**：~120 行 - -**验证方式**： -- 单元测试：手动写入 mmap 区域的已知数据 → 调用读方法 → 验证返回值 -- **Upper layer 读取测试：写入 graph_upper.dat 已知数据 → GetNeighbors(id, layer>0) 返回正确（C2 fix）** -- Benchmark：50K 随机 GetVector 调用 < 1μs/op（hot path） -- 边界测试：id=0、id=capacity-1、id 超范围返回 error - ---- - -### Task 1.5：从现有数据导出到 mmap 文件格式（测试辅助） - -**目标**：实现 `ExportFromMemStore(ms *MemNodeStore, dir string)` 工具函数，将 MemStore 数据导出为 mmap 文件格式，供读路径测试和 benchmark 使用。 - -**改动文件**： -- `internal/core/vectorindex/mmap_export_test.go`（~80 行，仅测试代码） - - 遍历 MemStore → 写入各 mmap 文件 - - 用于生成测试数据 - -**预估行数**：~80 行 - -**验证方式**： -- 集成测试：MemStore 插入 1K 向量 → 导出 → MmapStore Open → 逐个 GetVector 对比 -- Recall 对比：同一数据集 MemStore vs MmapStore 搜索结果一致 - ---- - -### Phase 1 总验收 - -- [ ] `CGO_ENABLED=0 GOOS=linux go build ./...` 通过 -- [ ] `CGO_ENABLED=0 GOOS=darwin go build ./...` 通过 -- [ ] `CGO_ENABLED=0 GOOS=windows go build ./...` 通过 -- [ ] 50K 随机向量读 benchmark < 1μs（hot） -- [ ] 所有单元测试通过 -- [ ] `go vet ./...` 和 `staticcheck ./...` 无新增告警 - ---- - -## Phase 2：写入路径 + WAL + Batch（后续） - -### Task 2.1：WAL 实现（append / CRC32 / LSN / **replay**） -- 文件：`mmap_wal.go`（~250 行） -- **WAL replay 基础逻辑包含在此任务中（C1 fix）**：顺序扫描 + CRC 校验 + 不完整记录丢弃 + 回调接口 -- replay INSERT 记录时恢复内存中的 docId→nodeId 映射（H1 fix：即使 idmap.dat compact 推迟到 Phase 3，replay 必须恢复内存 map） -- 验证：写 N 条记录 → 读回 → LSN 连续、CRC 正确、payload 一致 -- 验证：模拟写入后不 Close → replay → 数据完整恢复（含 ID 映射） -- Phase 4 Task 4.2 聚焦 Open 流程集成和 crash-point 注入，不再重复 replay 基础逻辑 - -### Task 2.2：写方法实现（PutNode / SetNeighbors / SetNorm / SetNodeMapping） -- 文件：`mmap_store.go` 追加（~170 行） -- **SetNodeMapping 包含 idmap.dat 追加写（append + CRC）（H1/H5 fix）**：确保正常 Close→Open 不丢 ID 映射；Phase 3 Task 3.4 只做 compact + 损坏恢复 -- 验证：PutNode → GetVector 返回写入值、SetNeighbors → GetNeighbors 返回一致 -- 验证：SetNodeMapping → Close → Open → GetNodeId 返回正确映射 - -### Task 2.3：BatchableStore 接口实现 -- 文件：`mmap_store.go` 追加（~60 行） -- 验证：batch 模式下多次写操作只触发 1 次 sync - -### Task 2.4：文件增长（grow）+ 并发安全 -- 文件：`mmap_store.go` 追加（~150 行） -- **并发安全方案（C3 fix）**：grow 期间采用 copy-on-read 策略——grow 前获取写锁，将旧 mmap slice 引用保存为 stale snapshot；grow 完成后释放写锁。并发 reader 通过 atomic.Pointer 获取当前 mmap slice，grow 切换为新 slice 后旧 reader 自然排空（epoch-based）。独立子任务： - - 2.4a：grow 基本实现（munmap → ftruncate → mmap，atomic slice 切换） - - 2.4b：grow 并发压力测试（多 goroutine 持续读 + 触发 grow，验证无 SIGBUS/panic） -- 验证：插入超过初始 capacity → 自动 grow → 数据完整 -- 验证：并发压力测试 10 goroutine 读 + 1 goroutine 连续 grow × 100 次，零 panic - -### Task 2.5：NextNodeId + ID 映射写入 -- 文件：`mmap_store.go` 追加（~80 行） -- **初版策略（H2 fix）**：只做 `meta.NextNodeId++` 简单自增分配，不预留 freelist 接口。Phase 3 Task 3.1/3.2 实现 freelist 后再修改 NextNodeId 加入 freelist pop 逻辑。 -- 验证：NextNodeId 单调递增、SetNodeMapping → GetNodeId 往返一致 - -### Task 2.6：50K Insert 集成 benchmark -- 文件：`mmap_store_bench_test.go`（~100 行） -- **Benchmark scope（H3 fix）**：Phase 2 尚无 graph_upper.dat 写入（Phase 5 Task 5.1），因此本 benchmark 仅验证 level-0 写入路径。结果为部分验证，不代表完整 E2E 性能。Phase 5 Task 5.3 为完整 E2E benchmark。 -- 验证：50K×128d Insert < 90s（仅 level-0 路径） - -**Phase 2 预估总行数**：~750 行 - ---- - -## Phase 3：Delete + Upsert + Freelist（后续） - -### Task 3.1：DeleteNode（墓碑 + freelist） -- 文件：`mmap_store.go` 追加（~120 行） -- 验证：Delete 后 GetNodeLevel 返回 error、freelist 包含回收 ID - -### Task 3.2：Freelist 启动扫描重建 -- 文件：`mmap_store.go` Open 流程追加（~60 行） -- 验证：插入 → 删除部分 → Close → Open → freelist 正确重建 < 1ms - -### Task 3.3：Upsert 实现 -- 文件：`mmap_store.go` 追加（~50 行） -- 验证：同 docId Upsert → slot 复用、无孤儿节点 - -### Task 3.4：idmap.dat compact + 损坏恢复 -- 文件：`mmap_idmap.go`（~170 行） -- 基础 idmap.dat 追加写已在 Phase 2 Task 2.2 实现（H1/H5 fix）；本任务聚焦 compact（合并重复 entry 缩减文件）和损坏 entry 跳过恢复 -- 验证：大量写入 → compact → 文件缩小、映射完整、损坏 entry 被跳过 - -**Phase 3 预估总行数**：~400 行 - ---- - -## Phase 4：崩溃恢复 + Checkpoint（后续） - -### Task 4.1：WAL checkpoint 机制 -- 文件：`mmap_wal.go` 追加（~80 行） -- 验证：checkpoint 后 WAL 截断、meta.WalCheckpointLSN 更新 - -### Task 4.2：崩溃恢复流程（Open 集成 WAL replay） -- 文件：`mmap_store.go` Open 流程追加（~100 行） -- WAL replay 基础逻辑已在 Phase 2 Task 2.1 实现（C1 fix）；本任务将 replay 集成到 Open 流程（从 WalCheckpointLSN+1 开始） -- 验证：写入 N 条 → 模拟 crash（不 Close）→ Open → 数据完整 - -### Task 4.3：Crash point 注入测试（5 类场景） -- 文件：`mmap_crash_test.go`（~120 行） -- 验证：Insert/SetNeighbors/Checkpoint/Grow/Delete 中途 panic → 恢复正确 - -**Phase 4 预估总行数**：~300 行 - ---- - -## Phase 5：上层图 + 完整集成（后续） - -### Task 5.1：graph_upper.dat 按需分配写入 -- 文件：`mmap_store.go` 追加（~100 行） -- graph_upper.dat 的 Open/mmap 和只读路径已在 Phase 1 Task 1.3/1.4 实现（C2 fix）；本任务聚焦写入：level>0 节点的 upper slot 按需分配 -- 验证：level>0 节点 upper slot 分配正确、level=0 节点不占 upper slot - -### Task 5.2：HNSW 集成（替换 PebbleStore） -- 文件：`hnsw.go` / `hnsw_test.go` 修改（~80 行） -- 验证：全部现有 HNSW 测试通过 - -### Task 5.3：E2E benchmark + Recall 验证 -- 文件：`mmap_store_e2e_test.go`（~120 行） -- 验证：SIFT 50K recall@10 > 0.95、搜索 p99 < 5ms - -**Phase 5 预估总行数**：~300 行 - ---- - -## 总计 - -| Phase | 行数 | 状态 | -|-------|------|------| -| Phase 1：核心基础设施 | ~620 行 | **当前** | -| Phase 2：写入 + WAL + Batch | ~750 行 | 后续 | -| Phase 3：Delete + Upsert | ~400 行 | 后续 | -| Phase 4：崩溃恢复 | ~300 行 | 后续 | -| Phase 5：上层图 + 集成 | ~300 行 | 后续 | -| **总计** | **~2370 行** | | diff --git a/docs/tasks/idtable-pebble-backend-tasks.md b/docs/tasks/idtable-pebble-backend-tasks.md deleted file mode 100644 index ebe2ae13..00000000 --- a/docs/tasks/idtable-pebble-backend-tasks.md +++ /dev/null @@ -1,47 +0,0 @@ -# idtable → shared-store restore — Task Breakdown v2 - -> Status: **Draft — awaiting review (SDD step 4).** Supersedes v1 tasks. -> Spec: `docs/design/idtable-pebble-backend.md` v2 (Approved 2026-06-24). -> Goal: restore the #97-predecessor shared-store idtable; re-wire BOTH consumers; no StorageVersion bump. - -### T0 — Clean the branch of the v1 (wrong) commits -Drop `refactor(idtable): pebble backend` (94627f1) and `feat(storage): bump StorageVersion` (f2d26e8). Keep `docs(agents)` + the spec/tasks docs (now v2). Result base: AGENTS.md + v2 docs on top of #103; idtable.go/storage.go back to the #103 state. -**Verify**: `git log` shows no v1 code/storage commit; `storage.go` `StorageVersion == "1.5"`; tree builds. - -### T1 — Restore `idtable.go` to the shared-store allocator -Base = `873ef7f~1:core/idtable/idtable.go` (`New(store kv.Store, opts)`, `DefaultKeyType{NextId,Key}` 28/29 configurable, LRU + `store.NewBatch` + ticker). Then: -- **Add** the post-#97 API consumers use: `Lookup()` (non-allocating `store.Get`); `CrashRelease()` with **shared-store semantics** (stop ticker + discard pending + mark closed; do NOT close the caller's store). -- Reconcile #103: the 28/29 consts are `DefaultKeyType*` again (active defaults, not `LegacyKeyType*` reserved). -- No `Open(path)`, no pebblekv/os import, no invented prefixes/cache. -**Verify**: `go build ./core/idtable/`. - -### T2 — idtable tests back to `New(store, …)` -Tests construct a real `kv.Store` (pebblekv in a temp dir) and call `New`. Restore the pre-#97 test shapes; keep `Lookup`/`CrashRelease`/`Commit`/ticker/closed-guard tests. -**Verify**: `go test ./core/idtable/` + `go test -race -run 'Close|Commit|Periodic' ./core/idtable/`. - -### T3 — Re-wire indexer (`internal/server/server.go`) -`idtable.Open(idtablePath)` → `idtable.New(db, idtable.Options{})` against the shared `data` store (default 28/29). Drop `idtablePath`. -**Verify**: `go build ./internal/server/`; `go test ./internal/server/`. - -### T4 — Re-wire vectorstore (`core/vectorstore/store.go` + tests) -Restore `Options.KV kv.Store`, the `idtableKeyTypeNextId=40`/`idtableKeyTypeKey=41` consts, `idtable.New(opts.KV, {40,41})`, and the id→docId reverse-map helper; drop `idtable.Open(opts.Dir/idtable.db)`. Keep the bbolt control plane. Update the ~20 vectorstore test files to pass `opts.KV` (a pebblekv store). -**Verify**: `go test ./core/vectorstore/`; re-check recovery tests under `-race` (R3: `CrashRelease` must not close the caller's store). - -### T5 — storage collision registry -`internal/core/storage/storage_test.go`: keep the idtable 28/29 entries; rename the symbol ref `idtable.LegacyKeyType*` → `idtable.DefaultKeyType*` (now active defaults). Re-add the `idtable` import. -**Verify**: `go test ./internal/core/storage/`. - -### T6 — Full verification -`go build ./...`; `go vet ./...`; `go test ./core/idtable/... ./core/vectorstore/... ./internal/server/... ./internal/core/storage/...` + idtable consumers (documents/indexer/searcher); go-cov gate on affected packages. - -## Acceptance -1. idtable is a shared-store allocator (`New(store, opts)`); **no** `Open(path)`, no separate idtable file created anywhere. -2. indexer uses the shared `data` store @ 28/29; vectorstore uses `opts.KV` @ 40/41. -3. `StorageVersion` is still **1.5** (no bump). -4. `Lookup` kept; `CrashRelease` shared-store semantics (doesn't close caller's store). -5. build + vet + all affected tests green; go-cov gate holds. - -## Branch / commit plan -- T0 drops the two v1 code commits (rebase --onto / reset, keep AGENTS.md + v2 docs). -- New commits: `docs(idtable): restore shared-store design (v2 spec/tasks)` / `refactor(idtable): restore shared kv.Store allocator` (T1+T2) / `refactor(vectorstore,server): wire idtable over the shared store` (T3+T4+T5). -- Update PR #104 (force-push) or open fresh — decide at commit time. diff --git a/go.work b/go.work index 185081c8..15966c54 100644 --- a/go.work +++ b/go.work @@ -2,6 +2,6 @@ go 1.23.0 toolchain go1.24.2 -use . +use ./packages/server -use ./core +use ./packages/core diff --git a/go.work.sum b/go.work.sum index 322bba63..73ba71b6 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,51 +1,71 @@ cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= -github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= +github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.12.0/go.mod h1:SvzfYNNBshAVbZ8wzNc/UPK3w1vf0dKDUP41ucAIf7g= -github.com/cockroachdb/fifo v0.0.0-20240816210425-c5d0cb0b6fc0/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= -github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo= -github.com/cockroachdb/redact v1.1.6/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/tokenbucket v0.0.0-20250429170803-42689b6311bb/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/getsentry/sentry-go v0.33.0/go.mod h1:C55omcY9ChRQIUcVcGcs+Zdy4ZpQGvNJ7JYHIoSWOtE= +github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/guptarohit/asciigraph v0.5.5 h1:ccFnUF8xYIOUPPY3tmdvRyHqmn1MYI9iv1pLKX+/ZkQ= +github.com/guptarohit/asciigraph v0.5.5/go.mod h1:dYl5wwK4gNsnFf9Zp+l06rFiDZ5YtXM6x7SRWZ3KGag= +github.com/hydrogen18/memlistener v1.0.0 h1:JR7eDj8HD6eXrc5fWLbSUnfcQFL06PYvCc0DKQnWfaU= github.com/hydrogen18/memlistener v1.0.0/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -54,15 +74,24 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= +go.etcd.io/gofail v0.2.0 h1:p19drv16FKK345a09a1iubchlw/vmRuksmRzgBIGjcA= +go.etcd.io/gofail v0.2.0/go.mod h1:nL3ILMGfkXTekKI3clMBNazKnjUZjYLKmBHzsVAnC1o= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5 h1:ObuXPmIgI4ZMyQLIz48cJYgSyWdjUXc2SZAdyJMwEAU= +golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5/go.mod h1:UBKtEnL8aqnd+0JHqZ+2qoMDwtuy6cYhhKNoHLBiTQc= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -70,20 +99,25 @@ golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/packages/core/.gitignore b/packages/core/.gitignore new file mode 100644 index 00000000..4c338875 --- /dev/null +++ b/packages/core/.gitignore @@ -0,0 +1 @@ +.coverage/ diff --git a/core/.go-cov.toml b/packages/core/.go-cov.toml similarity index 100% rename from core/.go-cov.toml rename to packages/core/.go-cov.toml diff --git a/core/README.md b/packages/core/README.md similarity index 100% rename from core/README.md rename to packages/core/README.md diff --git a/core/collection/catalog.go b/packages/core/collection/catalog.go similarity index 100% rename from core/collection/catalog.go rename to packages/core/collection/catalog.go diff --git a/core/collection/catalog_test.go b/packages/core/collection/catalog_test.go similarity index 100% rename from core/collection/catalog_test.go rename to packages/core/collection/catalog_test.go diff --git a/core/collection/codec.go b/packages/core/collection/codec.go similarity index 100% rename from core/collection/codec.go rename to packages/core/collection/codec.go diff --git a/core/collection/codec_extra_test.go b/packages/core/collection/codec_extra_test.go similarity index 100% rename from core/collection/codec_extra_test.go rename to packages/core/collection/codec_extra_test.go diff --git a/core/collection/fail_test.go b/packages/core/collection/fail_test.go similarity index 100% rename from core/collection/fail_test.go rename to packages/core/collection/fail_test.go diff --git a/core/docs/README.md b/packages/core/docs/README.md similarity index 100% rename from core/docs/README.md rename to packages/core/docs/README.md diff --git a/core/docs/vectorstore/architecture.md b/packages/core/docs/vectorstore/architecture.md similarity index 100% rename from core/docs/vectorstore/architecture.md rename to packages/core/docs/vectorstore/architecture.md diff --git a/core/docs/vectorstore/durability.md b/packages/core/docs/vectorstore/durability.md similarity index 100% rename from core/docs/vectorstore/durability.md rename to packages/core/docs/vectorstore/durability.md diff --git a/core/docs/vectorstore/indexing.md b/packages/core/docs/vectorstore/indexing.md similarity index 100% rename from core/docs/vectorstore/indexing.md rename to packages/core/docs/vectorstore/indexing.md diff --git a/core/docs/vectorstore/reclamation.md b/packages/core/docs/vectorstore/reclamation.md similarity index 100% rename from core/docs/vectorstore/reclamation.md rename to packages/core/docs/vectorstore/reclamation.md diff --git a/core/docs/vectorstore/storage.md b/packages/core/docs/vectorstore/storage.md similarity index 100% rename from core/docs/vectorstore/storage.md rename to packages/core/docs/vectorstore/storage.md diff --git a/core/documents/README.md b/packages/core/documents/README.md similarity index 100% rename from core/documents/README.md rename to packages/core/documents/README.md diff --git a/core/documents/batch_write.go b/packages/core/documents/batch_write.go similarity index 100% rename from core/documents/batch_write.go rename to packages/core/documents/batch_write.go diff --git a/core/documents/codec.go b/packages/core/documents/codec.go similarity index 100% rename from core/documents/codec.go rename to packages/core/documents/codec.go diff --git a/core/documents/codec_additional_test.go b/packages/core/documents/codec_additional_test.go similarity index 100% rename from core/documents/codec_additional_test.go rename to packages/core/documents/codec_additional_test.go diff --git a/core/documents/codec_bench_test.go b/packages/core/documents/codec_bench_test.go similarity index 100% rename from core/documents/codec_bench_test.go rename to packages/core/documents/codec_bench_test.go diff --git a/core/documents/codec_test.go b/packages/core/documents/codec_test.go similarity index 100% rename from core/documents/codec_test.go rename to packages/core/documents/codec_test.go diff --git a/core/documents/content.go b/packages/core/documents/content.go similarity index 100% rename from core/documents/content.go rename to packages/core/documents/content.go diff --git a/core/documents/content_test.go b/packages/core/documents/content_test.go similarity index 100% rename from core/documents/content_test.go rename to packages/core/documents/content_test.go diff --git a/core/documents/count_test.go b/packages/core/documents/count_test.go similarity index 100% rename from core/documents/count_test.go rename to packages/core/documents/count_test.go diff --git a/core/documents/document.go b/packages/core/documents/document.go similarity index 100% rename from core/documents/document.go rename to packages/core/documents/document.go diff --git a/core/documents/document_internal.go b/packages/core/documents/document_internal.go similarity index 100% rename from core/documents/document_internal.go rename to packages/core/documents/document_internal.go diff --git a/core/documents/document_test.go b/packages/core/documents/document_test.go similarity index 100% rename from core/documents/document_test.go rename to packages/core/documents/document_test.go diff --git a/core/documents/seams_test.go b/packages/core/documents/seams_test.go similarity index 100% rename from core/documents/seams_test.go rename to packages/core/documents/seams_test.go diff --git a/core/documents/search.go b/packages/core/documents/search.go similarity index 100% rename from core/documents/search.go rename to packages/core/documents/search.go diff --git a/core/documents/search_test.go b/packages/core/documents/search_test.go similarity index 100% rename from core/documents/search_test.go rename to packages/core/documents/search_test.go diff --git a/core/documents/storage.go b/packages/core/documents/storage.go similarity index 100% rename from core/documents/storage.go rename to packages/core/documents/storage.go diff --git a/core/documents/storage_test.go b/packages/core/documents/storage_test.go similarity index 100% rename from core/documents/storage_test.go rename to packages/core/documents/storage_test.go diff --git a/core/documents/test_helper_test.go b/packages/core/documents/test_helper_test.go similarity index 100% rename from core/documents/test_helper_test.go rename to packages/core/documents/test_helper_test.go diff --git a/core/engine/engine.go b/packages/core/engine/engine.go similarity index 100% rename from core/engine/engine.go rename to packages/core/engine/engine.go diff --git a/core/engine/engine_test.go b/packages/core/engine/engine_test.go similarity index 100% rename from core/engine/engine_test.go rename to packages/core/engine/engine_test.go diff --git a/core/engine/integration_test.go b/packages/core/engine/integration_test.go similarity index 100% rename from core/engine/integration_test.go rename to packages/core/engine/integration_test.go diff --git a/core/engine/readme_example_test.go b/packages/core/engine/readme_example_test.go similarity index 100% rename from core/engine/readme_example_test.go rename to packages/core/engine/readme_example_test.go diff --git a/core/engine/scan.go b/packages/core/engine/scan.go similarity index 100% rename from core/engine/scan.go rename to packages/core/engine/scan.go diff --git a/core/engine/scan_test.go b/packages/core/engine/scan_test.go similarity index 100% rename from core/engine/scan_test.go rename to packages/core/engine/scan_test.go diff --git a/core/go.mod b/packages/core/go.mod similarity index 100% rename from core/go.mod rename to packages/core/go.mod diff --git a/core/go.sum b/packages/core/go.sum similarity index 100% rename from core/go.sum rename to packages/core/go.sum diff --git a/core/idtable/closed_commit_test.go b/packages/core/idtable/closed_commit_test.go similarity index 100% rename from core/idtable/closed_commit_test.go rename to packages/core/idtable/closed_commit_test.go diff --git a/core/idtable/commit_public_test.go b/packages/core/idtable/commit_public_test.go similarity index 100% rename from core/idtable/commit_public_test.go rename to packages/core/idtable/commit_public_test.go diff --git a/core/idtable/commit_test.go b/packages/core/idtable/commit_test.go similarity index 100% rename from core/idtable/commit_test.go rename to packages/core/idtable/commit_test.go diff --git a/core/idtable/encode_id_test.go b/packages/core/idtable/encode_id_test.go similarity index 100% rename from core/idtable/encode_id_test.go rename to packages/core/idtable/encode_id_test.go diff --git a/core/idtable/idtable.go b/packages/core/idtable/idtable.go similarity index 100% rename from core/idtable/idtable.go rename to packages/core/idtable/idtable.go diff --git a/core/idtable/idtable_extra_test.go b/packages/core/idtable/idtable_extra_test.go similarity index 100% rename from core/idtable/idtable_extra_test.go rename to packages/core/idtable/idtable_extra_test.go diff --git a/core/idtable/idtable_test.go b/packages/core/idtable/idtable_test.go similarity index 100% rename from core/idtable/idtable_test.go rename to packages/core/idtable/idtable_test.go diff --git a/core/idtable/lookup_crashrelease_test.go b/packages/core/idtable/lookup_crashrelease_test.go similarity index 100% rename from core/idtable/lookup_crashrelease_test.go rename to packages/core/idtable/lookup_crashrelease_test.go diff --git a/core/idtable/lru_cache.go b/packages/core/idtable/lru_cache.go similarity index 100% rename from core/idtable/lru_cache.go rename to packages/core/idtable/lru_cache.go diff --git a/core/idtable/lru_cache_test.go b/packages/core/idtable/lru_cache_test.go similarity index 100% rename from core/idtable/lru_cache_test.go rename to packages/core/idtable/lru_cache_test.go diff --git a/core/invertedindex/README.md b/packages/core/invertedindex/README.md similarity index 100% rename from core/invertedindex/README.md rename to packages/core/invertedindex/README.md diff --git a/core/invertedindex/batch_close_test.go b/packages/core/invertedindex/batch_close_test.go similarity index 100% rename from core/invertedindex/batch_close_test.go rename to packages/core/invertedindex/batch_close_test.go diff --git a/core/invertedindex/batch_write.go b/packages/core/invertedindex/batch_write.go similarity index 100% rename from core/invertedindex/batch_write.go rename to packages/core/invertedindex/batch_write.go diff --git a/core/invertedindex/codec.go b/packages/core/invertedindex/codec.go similarity index 100% rename from core/invertedindex/codec.go rename to packages/core/invertedindex/codec.go diff --git a/core/invertedindex/codec_bench_test.go b/packages/core/invertedindex/codec_bench_test.go similarity index 100% rename from core/invertedindex/codec_bench_test.go rename to packages/core/invertedindex/codec_bench_test.go diff --git a/core/invertedindex/codec_delimiter_test.go b/packages/core/invertedindex/codec_delimiter_test.go similarity index 100% rename from core/invertedindex/codec_delimiter_test.go rename to packages/core/invertedindex/codec_delimiter_test.go diff --git a/core/invertedindex/codec_test.go b/packages/core/invertedindex/codec_test.go similarity index 100% rename from core/invertedindex/codec_test.go rename to packages/core/invertedindex/codec_test.go diff --git a/core/invertedindex/doccount_test.go b/packages/core/invertedindex/doccount_test.go similarity index 100% rename from core/invertedindex/doccount_test.go rename to packages/core/invertedindex/doccount_test.go diff --git a/core/invertedindex/forward_test.go b/packages/core/invertedindex/forward_test.go similarity index 100% rename from core/invertedindex/forward_test.go rename to packages/core/invertedindex/forward_test.go diff --git a/core/invertedindex/index_test.go b/packages/core/invertedindex/index_test.go similarity index 100% rename from core/invertedindex/index_test.go rename to packages/core/invertedindex/index_test.go diff --git a/core/invertedindex/invertedindex.go b/packages/core/invertedindex/invertedindex.go similarity index 100% rename from core/invertedindex/invertedindex.go rename to packages/core/invertedindex/invertedindex.go diff --git a/core/invertedindex/invertedindex_internal.go b/packages/core/invertedindex/invertedindex_internal.go similarity index 100% rename from core/invertedindex/invertedindex_internal.go rename to packages/core/invertedindex/invertedindex_internal.go diff --git a/core/invertedindex/key_robustness_test.go b/packages/core/invertedindex/key_robustness_test.go similarity index 100% rename from core/invertedindex/key_robustness_test.go rename to packages/core/invertedindex/key_robustness_test.go diff --git a/core/invertedindex/keywords_merger.go b/packages/core/invertedindex/keywords_merger.go similarity index 100% rename from core/invertedindex/keywords_merger.go rename to packages/core/invertedindex/keywords_merger.go diff --git a/core/invertedindex/keywords_merger_test.go b/packages/core/invertedindex/keywords_merger_test.go similarity index 100% rename from core/invertedindex/keywords_merger_test.go rename to packages/core/invertedindex/keywords_merger_test.go diff --git a/core/invertedindex/options_test.go b/packages/core/invertedindex/options_test.go similarity index 100% rename from core/invertedindex/options_test.go rename to packages/core/invertedindex/options_test.go diff --git a/core/invertedindex/pending_bound_test.go b/packages/core/invertedindex/pending_bound_test.go similarity index 100% rename from core/invertedindex/pending_bound_test.go rename to packages/core/invertedindex/pending_bound_test.go diff --git a/core/invertedindex/pending_writes.go b/packages/core/invertedindex/pending_writes.go similarity index 100% rename from core/invertedindex/pending_writes.go rename to packages/core/invertedindex/pending_writes.go diff --git a/core/invertedindex/search_bench_test.go b/packages/core/invertedindex/search_bench_test.go similarity index 100% rename from core/invertedindex/search_bench_test.go rename to packages/core/invertedindex/search_bench_test.go diff --git a/core/invertedindex/storage.go b/packages/core/invertedindex/storage.go similarity index 100% rename from core/invertedindex/storage.go rename to packages/core/invertedindex/storage.go diff --git a/core/invertedindex/table.go b/packages/core/invertedindex/table.go similarity index 100% rename from core/invertedindex/table.go rename to packages/core/invertedindex/table.go diff --git a/core/invertedindex/test_helper_test.go b/packages/core/invertedindex/test_helper_test.go similarity index 100% rename from core/invertedindex/test_helper_test.go rename to packages/core/invertedindex/test_helper_test.go diff --git a/core/invertedstore/README.md b/packages/core/invertedstore/README.md similarity index 100% rename from core/invertedstore/README.md rename to packages/core/invertedstore/README.md diff --git a/core/invertedstore/apply_fastpath_test.go b/packages/core/invertedstore/apply_fastpath_test.go similarity index 100% rename from core/invertedstore/apply_fastpath_test.go rename to packages/core/invertedstore/apply_fastpath_test.go diff --git a/core/invertedstore/backpressure.go b/packages/core/invertedstore/backpressure.go similarity index 100% rename from core/invertedstore/backpressure.go rename to packages/core/invertedstore/backpressure.go diff --git a/core/invertedstore/backpressure_test.go b/packages/core/invertedstore/backpressure_test.go similarity index 100% rename from core/invertedstore/backpressure_test.go rename to packages/core/invertedstore/backpressure_test.go diff --git a/core/invertedstore/block_index_test.go b/packages/core/invertedstore/block_index_test.go similarity index 100% rename from core/invertedstore/block_index_test.go rename to packages/core/invertedstore/block_index_test.go diff --git a/core/invertedstore/codec.go b/packages/core/invertedstore/codec.go similarity index 100% rename from core/invertedstore/codec.go rename to packages/core/invertedstore/codec.go diff --git a/core/invertedstore/codec_test.go b/packages/core/invertedstore/codec_test.go similarity index 100% rename from core/invertedstore/codec_test.go rename to packages/core/invertedstore/codec_test.go diff --git a/core/invertedstore/concurrency.go b/packages/core/invertedstore/concurrency.go similarity index 100% rename from core/invertedstore/concurrency.go rename to packages/core/invertedstore/concurrency.go diff --git a/core/invertedstore/concurrency_test.go b/packages/core/invertedstore/concurrency_test.go similarity index 100% rename from core/invertedstore/concurrency_test.go rename to packages/core/invertedstore/concurrency_test.go diff --git a/core/invertedstore/crash_recovery_test.go b/packages/core/invertedstore/crash_recovery_test.go similarity index 100% rename from core/invertedstore/crash_recovery_test.go rename to packages/core/invertedstore/crash_recovery_test.go diff --git a/core/invertedstore/dictcache.go b/packages/core/invertedstore/dictcache.go similarity index 100% rename from core/invertedstore/dictcache.go rename to packages/core/invertedstore/dictcache.go diff --git a/core/invertedstore/dictcache_test.go b/packages/core/invertedstore/dictcache_test.go similarity index 100% rename from core/invertedstore/dictcache_test.go rename to packages/core/invertedstore/dictcache_test.go diff --git a/core/invertedstore/differential_test.go b/packages/core/invertedstore/differential_test.go similarity index 100% rename from core/invertedstore/differential_test.go rename to packages/core/invertedstore/differential_test.go diff --git a/core/invertedstore/drain_merge_test.go b/packages/core/invertedstore/drain_merge_test.go similarity index 100% rename from core/invertedstore/drain_merge_test.go rename to packages/core/invertedstore/drain_merge_test.go diff --git a/core/invertedstore/export_test.go b/packages/core/invertedstore/export_test.go similarity index 100% rename from core/invertedstore/export_test.go rename to packages/core/invertedstore/export_test.go diff --git a/core/invertedstore/foreach_forward_test.go b/packages/core/invertedstore/foreach_forward_test.go similarity index 100% rename from core/invertedstore/foreach_forward_test.go rename to packages/core/invertedstore/foreach_forward_test.go diff --git a/core/invertedstore/forward_skip_test.go b/packages/core/invertedstore/forward_skip_test.go similarity index 100% rename from core/invertedstore/forward_skip_test.go rename to packages/core/invertedstore/forward_skip_test.go diff --git a/core/invertedstore/head.go b/packages/core/invertedstore/head.go similarity index 100% rename from core/invertedstore/head.go rename to packages/core/invertedstore/head.go diff --git a/core/invertedstore/head_lazy_dels_test.go b/packages/core/invertedstore/head_lazy_dels_test.go similarity index 100% rename from core/invertedstore/head_lazy_dels_test.go rename to packages/core/invertedstore/head_lazy_dels_test.go diff --git a/core/invertedstore/keys.go b/packages/core/invertedstore/keys.go similarity index 100% rename from core/invertedstore/keys.go rename to packages/core/invertedstore/keys.go diff --git a/core/invertedstore/keys_test.go b/packages/core/invertedstore/keys_test.go similarity index 100% rename from core/invertedstore/keys_test.go rename to packages/core/invertedstore/keys_test.go diff --git a/core/invertedstore/live_count_test.go b/packages/core/invertedstore/live_count_test.go similarity index 100% rename from core/invertedstore/live_count_test.go rename to packages/core/invertedstore/live_count_test.go diff --git a/core/invertedstore/live_recompute_test.go b/packages/core/invertedstore/live_recompute_test.go similarity index 100% rename from core/invertedstore/live_recompute_test.go rename to packages/core/invertedstore/live_recompute_test.go diff --git a/core/invertedstore/manifest.go b/packages/core/invertedstore/manifest.go similarity index 100% rename from core/invertedstore/manifest.go rename to packages/core/invertedstore/manifest.go diff --git a/core/invertedstore/manifest_test.go b/packages/core/invertedstore/manifest_test.go similarity index 100% rename from core/invertedstore/manifest_test.go rename to packages/core/invertedstore/manifest_test.go diff --git a/core/invertedstore/merge.go b/packages/core/invertedstore/merge.go similarity index 100% rename from core/invertedstore/merge.go rename to packages/core/invertedstore/merge.go diff --git a/core/invertedstore/merge_highcardinality_test.go b/packages/core/invertedstore/merge_highcardinality_test.go similarity index 100% rename from core/invertedstore/merge_highcardinality_test.go rename to packages/core/invertedstore/merge_highcardinality_test.go diff --git a/core/invertedstore/merge_offworker_test.go b/packages/core/invertedstore/merge_offworker_test.go similarity index 100% rename from core/invertedstore/merge_offworker_test.go rename to packages/core/invertedstore/merge_offworker_test.go diff --git a/core/invertedstore/merge_robustness_test.go b/packages/core/invertedstore/merge_robustness_test.go similarity index 100% rename from core/invertedstore/merge_robustness_test.go rename to packages/core/invertedstore/merge_robustness_test.go diff --git a/core/invertedstore/merge_test.go b/packages/core/invertedstore/merge_test.go similarity index 100% rename from core/invertedstore/merge_test.go rename to packages/core/invertedstore/merge_test.go diff --git a/core/invertedstore/orphan_reclaim_test.go b/packages/core/invertedstore/orphan_reclaim_test.go similarity index 100% rename from core/invertedstore/orphan_reclaim_test.go rename to packages/core/invertedstore/orphan_reclaim_test.go diff --git a/core/invertedstore/orphan_sweep_test.go b/packages/core/invertedstore/orphan_sweep_test.go similarity index 100% rename from core/invertedstore/orphan_sweep_test.go rename to packages/core/invertedstore/orphan_sweep_test.go diff --git a/core/invertedstore/reconcile.go b/packages/core/invertedstore/reconcile.go similarity index 100% rename from core/invertedstore/reconcile.go rename to packages/core/invertedstore/reconcile.go diff --git a/core/invertedstore/reconcile_test.go b/packages/core/invertedstore/reconcile_test.go similarity index 100% rename from core/invertedstore/reconcile_test.go rename to packages/core/invertedstore/reconcile_test.go diff --git a/core/invertedstore/rename_reseat_test.go b/packages/core/invertedstore/rename_reseat_test.go similarity index 100% rename from core/invertedstore/rename_reseat_test.go rename to packages/core/invertedstore/rename_reseat_test.go diff --git a/core/invertedstore/resolve_ops_test.go b/packages/core/invertedstore/resolve_ops_test.go similarity index 100% rename from core/invertedstore/resolve_ops_test.go rename to packages/core/invertedstore/resolve_ops_test.go diff --git a/core/invertedstore/search.go b/packages/core/invertedstore/search.go similarity index 100% rename from core/invertedstore/search.go rename to packages/core/invertedstore/search.go diff --git a/core/invertedstore/search_test.go b/packages/core/invertedstore/search_test.go similarity index 100% rename from core/invertedstore/search_test.go rename to packages/core/invertedstore/search_test.go diff --git a/core/invertedstore/segment.go b/packages/core/invertedstore/segment.go similarity index 100% rename from core/invertedstore/segment.go rename to packages/core/invertedstore/segment.go diff --git a/core/invertedstore/segment_inline_dict_test.go b/packages/core/invertedstore/segment_inline_dict_test.go similarity index 100% rename from core/invertedstore/segment_inline_dict_test.go rename to packages/core/invertedstore/segment_inline_dict_test.go diff --git a/core/invertedstore/segment_test.go b/packages/core/invertedstore/segment_test.go similarity index 100% rename from core/invertedstore/segment_test.go rename to packages/core/invertedstore/segment_test.go diff --git a/core/invertedstore/segmeta_postings_test.go b/packages/core/invertedstore/segmeta_postings_test.go similarity index 100% rename from core/invertedstore/segmeta_postings_test.go rename to packages/core/invertedstore/segmeta_postings_test.go diff --git a/core/invertedstore/segmeta_sort_test.go b/packages/core/invertedstore/segmeta_sort_test.go similarity index 100% rename from core/invertedstore/segmeta_sort_test.go rename to packages/core/invertedstore/segmeta_sort_test.go diff --git a/core/invertedstore/spill_offworker_test.go b/packages/core/invertedstore/spill_offworker_test.go similarity index 100% rename from core/invertedstore/spill_offworker_test.go rename to packages/core/invertedstore/spill_offworker_test.go diff --git a/core/invertedstore/spilling.go b/packages/core/invertedstore/spilling.go similarity index 100% rename from core/invertedstore/spilling.go rename to packages/core/invertedstore/spilling.go diff --git a/core/invertedstore/spilling_read_test.go b/packages/core/invertedstore/spilling_read_test.go similarity index 100% rename from core/invertedstore/spilling_read_test.go rename to packages/core/invertedstore/spilling_read_test.go diff --git a/core/invertedstore/store.go b/packages/core/invertedstore/store.go similarity index 100% rename from core/invertedstore/store.go rename to packages/core/invertedstore/store.go diff --git a/core/invertedstore/store_test.go b/packages/core/invertedstore/store_test.go similarity index 100% rename from core/invertedstore/store_test.go rename to packages/core/invertedstore/store_test.go diff --git a/core/invertedstore/trigger_test.go b/packages/core/invertedstore/trigger_test.go similarity index 100% rename from core/invertedstore/trigger_test.go rename to packages/core/invertedstore/trigger_test.go diff --git a/core/invertedstore/update.go b/packages/core/invertedstore/update.go similarity index 100% rename from core/invertedstore/update.go rename to packages/core/invertedstore/update.go diff --git a/core/invertedstore/update_test.go b/packages/core/invertedstore/update_test.go similarity index 100% rename from core/invertedstore/update_test.go rename to packages/core/invertedstore/update_test.go diff --git a/core/kv/kv.go b/packages/core/kv/kv.go similarity index 100% rename from core/kv/kv.go rename to packages/core/kv/kv.go diff --git a/core/kv/pebblekv/batch.go b/packages/core/kv/pebblekv/batch.go similarity index 100% rename from core/kv/pebblekv/batch.go rename to packages/core/kv/pebblekv/batch.go diff --git a/core/kv/pebblekv/db.go b/packages/core/kv/pebblekv/db.go similarity index 100% rename from core/kv/pebblekv/db.go rename to packages/core/kv/pebblekv/db.go diff --git a/core/kv/pebblekv/pebble_test.go b/packages/core/kv/pebblekv/pebble_test.go similarity index 100% rename from core/kv/pebblekv/pebble_test.go rename to packages/core/kv/pebblekv/pebble_test.go diff --git a/core/queue/mpsc.go b/packages/core/queue/mpsc.go similarity index 100% rename from core/queue/mpsc.go rename to packages/core/queue/mpsc.go diff --git a/core/queue/mpsc_test.go b/packages/core/queue/mpsc_test.go similarity index 100% rename from core/queue/mpsc_test.go rename to packages/core/queue/mpsc_test.go diff --git a/core/third_party/vellum/.github/workflows/tests.yml b/packages/core/third_party/vellum/.github/workflows/tests.yml similarity index 100% rename from core/third_party/vellum/.github/workflows/tests.yml rename to packages/core/third_party/vellum/.github/workflows/tests.yml diff --git a/core/third_party/vellum/LICENSE b/packages/core/third_party/vellum/LICENSE similarity index 100% rename from core/third_party/vellum/LICENSE rename to packages/core/third_party/vellum/LICENSE diff --git a/core/third_party/vellum/README.md b/packages/core/third_party/vellum/README.md similarity index 100% rename from core/third_party/vellum/README.md rename to packages/core/third_party/vellum/README.md diff --git a/core/third_party/vellum/automaton.go b/packages/core/third_party/vellum/automaton.go similarity index 100% rename from core/third_party/vellum/automaton.go rename to packages/core/third_party/vellum/automaton.go diff --git a/core/third_party/vellum/builder.go b/packages/core/third_party/vellum/builder.go similarity index 100% rename from core/third_party/vellum/builder.go rename to packages/core/third_party/vellum/builder.go diff --git a/core/third_party/vellum/common.go b/packages/core/third_party/vellum/common.go similarity index 100% rename from core/third_party/vellum/common.go rename to packages/core/third_party/vellum/common.go diff --git a/core/third_party/vellum/decoder_v1.go b/packages/core/third_party/vellum/decoder_v1.go similarity index 100% rename from core/third_party/vellum/decoder_v1.go rename to packages/core/third_party/vellum/decoder_v1.go diff --git a/core/third_party/vellum/encoder_v1.go b/packages/core/third_party/vellum/encoder_v1.go similarity index 100% rename from core/third_party/vellum/encoder_v1.go rename to packages/core/third_party/vellum/encoder_v1.go diff --git a/core/third_party/vellum/encoding.go b/packages/core/third_party/vellum/encoding.go similarity index 100% rename from core/third_party/vellum/encoding.go rename to packages/core/third_party/vellum/encoding.go diff --git a/core/third_party/vellum/fst.go b/packages/core/third_party/vellum/fst.go similarity index 100% rename from core/third_party/vellum/fst.go rename to packages/core/third_party/vellum/fst.go diff --git a/core/third_party/vellum/fst_iterator.go b/packages/core/third_party/vellum/fst_iterator.go similarity index 100% rename from core/third_party/vellum/fst_iterator.go rename to packages/core/third_party/vellum/fst_iterator.go diff --git a/core/third_party/vellum/fst_reuse.go b/packages/core/third_party/vellum/fst_reuse.go similarity index 100% rename from core/third_party/vellum/fst_reuse.go rename to packages/core/third_party/vellum/fst_reuse.go diff --git a/core/third_party/vellum/go.mod b/packages/core/third_party/vellum/go.mod similarity index 100% rename from core/third_party/vellum/go.mod rename to packages/core/third_party/vellum/go.mod diff --git a/core/third_party/vellum/go.sum b/packages/core/third_party/vellum/go.sum similarity index 100% rename from core/third_party/vellum/go.sum rename to packages/core/third_party/vellum/go.sum diff --git a/core/third_party/vellum/merge_iterator.go b/packages/core/third_party/vellum/merge_iterator.go similarity index 100% rename from core/third_party/vellum/merge_iterator.go rename to packages/core/third_party/vellum/merge_iterator.go diff --git a/core/third_party/vellum/pack.go b/packages/core/third_party/vellum/pack.go similarity index 100% rename from core/third_party/vellum/pack.go rename to packages/core/third_party/vellum/pack.go diff --git a/core/third_party/vellum/registry.go b/packages/core/third_party/vellum/registry.go similarity index 100% rename from core/third_party/vellum/registry.go rename to packages/core/third_party/vellum/registry.go diff --git a/core/third_party/vellum/transducer.go b/packages/core/third_party/vellum/transducer.go similarity index 100% rename from core/third_party/vellum/transducer.go rename to packages/core/third_party/vellum/transducer.go diff --git a/core/third_party/vellum/vellum.go b/packages/core/third_party/vellum/vellum.go similarity index 100% rename from core/third_party/vellum/vellum.go rename to packages/core/third_party/vellum/vellum.go diff --git a/core/third_party/vellum/vellum_mmap.go b/packages/core/third_party/vellum/vellum_mmap.go similarity index 100% rename from core/third_party/vellum/vellum_mmap.go rename to packages/core/third_party/vellum/vellum_mmap.go diff --git a/core/third_party/vellum/vellum_nommap.go b/packages/core/third_party/vellum/vellum_nommap.go similarity index 100% rename from core/third_party/vellum/vellum_nommap.go rename to packages/core/third_party/vellum/vellum_nommap.go diff --git a/core/third_party/vellum/writer.go b/packages/core/third_party/vellum/writer.go similarity index 100% rename from core/third_party/vellum/writer.go rename to packages/core/third_party/vellum/writer.go diff --git a/core/tokenizer/BENCHMARK.md b/packages/core/tokenizer/BENCHMARK.md similarity index 100% rename from core/tokenizer/BENCHMARK.md rename to packages/core/tokenizer/BENCHMARK.md diff --git a/core/tokenizer/ascii_scan.go b/packages/core/tokenizer/ascii_scan.go similarity index 100% rename from core/tokenizer/ascii_scan.go rename to packages/core/tokenizer/ascii_scan.go diff --git a/core/tokenizer/ascii_scan_test.go b/packages/core/tokenizer/ascii_scan_test.go similarity index 100% rename from core/tokenizer/ascii_scan_test.go rename to packages/core/tokenizer/ascii_scan_test.go diff --git a/core/tokenizer/ascii_tokenizer.go b/packages/core/tokenizer/ascii_tokenizer.go similarity index 100% rename from core/tokenizer/ascii_tokenizer.go rename to packages/core/tokenizer/ascii_tokenizer.go diff --git a/core/tokenizer/benchmark_test.go b/packages/core/tokenizer/benchmark_test.go similarity index 100% rename from core/tokenizer/benchmark_test.go rename to packages/core/tokenizer/benchmark_test.go diff --git a/core/tokenizer/camel_snake_split.go b/packages/core/tokenizer/camel_snake_split.go similarity index 100% rename from core/tokenizer/camel_snake_split.go rename to packages/core/tokenizer/camel_snake_split.go diff --git a/core/tokenizer/cjk_detect.go b/packages/core/tokenizer/cjk_detect.go similarity index 100% rename from core/tokenizer/cjk_detect.go rename to packages/core/tokenizer/cjk_detect.go diff --git a/core/tokenizer/cjk_detect_test.go b/packages/core/tokenizer/cjk_detect_test.go similarity index 100% rename from core/tokenizer/cjk_detect_test.go rename to packages/core/tokenizer/cjk_detect_test.go diff --git a/core/tokenizer/cjk_fidelity_test.go b/packages/core/tokenizer/cjk_fidelity_test.go similarity index 100% rename from core/tokenizer/cjk_fidelity_test.go rename to packages/core/tokenizer/cjk_fidelity_test.go diff --git a/core/tokenizer/cjk_normalize_test.go b/packages/core/tokenizer/cjk_normalize_test.go similarity index 100% rename from core/tokenizer/cjk_normalize_test.go rename to packages/core/tokenizer/cjk_normalize_test.go diff --git a/core/tokenizer/cjk_tokenizer.go b/packages/core/tokenizer/cjk_tokenizer.go similarity index 100% rename from core/tokenizer/cjk_tokenizer.go rename to packages/core/tokenizer/cjk_tokenizer.go diff --git a/core/tokenizer/cjk_tokenizer_test.go b/packages/core/tokenizer/cjk_tokenizer_test.go similarity index 100% rename from core/tokenizer/cjk_tokenizer_test.go rename to packages/core/tokenizer/cjk_tokenizer_test.go diff --git a/core/tokenizer/fstcjk/README.md b/packages/core/tokenizer/fstcjk/README.md similarity index 100% rename from core/tokenizer/fstcjk/README.md rename to packages/core/tokenizer/fstcjk/README.md diff --git a/core/tokenizer/fstcjk/build.go b/packages/core/tokenizer/fstcjk/build.go similarity index 100% rename from core/tokenizer/fstcjk/build.go rename to packages/core/tokenizer/fstcjk/build.go diff --git a/core/tokenizer/fstcjk/dict.fst b/packages/core/tokenizer/fstcjk/dict.fst similarity index 100% rename from core/tokenizer/fstcjk/dict.fst rename to packages/core/tokenizer/fstcjk/dict.fst diff --git a/core/tokenizer/fstcjk/dict.totalfreq b/packages/core/tokenizer/fstcjk/dict.totalfreq similarity index 100% rename from core/tokenizer/fstcjk/dict.totalfreq rename to packages/core/tokenizer/fstcjk/dict.totalfreq diff --git a/core/tokenizer/fstcjk/doc.go b/packages/core/tokenizer/fstcjk/doc.go similarity index 100% rename from core/tokenizer/fstcjk/doc.go rename to packages/core/tokenizer/fstcjk/doc.go diff --git a/core/tokenizer/fstcjk/fidelity_test.go b/packages/core/tokenizer/fstcjk/fidelity_test.go similarity index 100% rename from core/tokenizer/fstcjk/fidelity_test.go rename to packages/core/tokenizer/fstcjk/fidelity_test.go diff --git a/core/tokenizer/fstcjk/gen_test.go b/packages/core/tokenizer/fstcjk/gen_test.go similarity index 100% rename from core/tokenizer/fstcjk/gen_test.go rename to packages/core/tokenizer/fstcjk/gen_test.go diff --git a/core/tokenizer/fstcjk/loader_test.go b/packages/core/tokenizer/fstcjk/loader_test.go similarity index 100% rename from core/tokenizer/fstcjk/loader_test.go rename to packages/core/tokenizer/fstcjk/loader_test.go diff --git a/core/tokenizer/fstcjk/search_fidelity_test.go b/packages/core/tokenizer/fstcjk/search_fidelity_test.go similarity index 100% rename from core/tokenizer/fstcjk/search_fidelity_test.go rename to packages/core/tokenizer/fstcjk/search_fidelity_test.go diff --git a/core/tokenizer/fstcjk/segmenter.go b/packages/core/tokenizer/fstcjk/segmenter.go similarity index 100% rename from core/tokenizer/fstcjk/segmenter.go rename to packages/core/tokenizer/fstcjk/segmenter.go diff --git a/core/tokenizer/fstcjk/smoke_test.go b/packages/core/tokenizer/fstcjk/smoke_test.go similarity index 100% rename from core/tokenizer/fstcjk/smoke_test.go rename to packages/core/tokenizer/fstcjk/smoke_test.go diff --git a/core/tokenizer/fstcjk/util.go b/packages/core/tokenizer/fstcjk/util.go similarity index 100% rename from core/tokenizer/fstcjk/util.go rename to packages/core/tokenizer/fstcjk/util.go diff --git a/core/tokenizer/mixed_tokenizer.go b/packages/core/tokenizer/mixed_tokenizer.go similarity index 100% rename from core/tokenizer/mixed_tokenizer.go rename to packages/core/tokenizer/mixed_tokenizer.go diff --git a/core/tokenizer/mixed_tokenizer_test.go b/packages/core/tokenizer/mixed_tokenizer_test.go similarity index 100% rename from core/tokenizer/mixed_tokenizer_test.go rename to packages/core/tokenizer/mixed_tokenizer_test.go diff --git a/core/tokenizer/parser_camel_snake_split_test.go b/packages/core/tokenizer/parser_camel_snake_split_test.go similarity index 100% rename from core/tokenizer/parser_camel_snake_split_test.go rename to packages/core/tokenizer/parser_camel_snake_split_test.go diff --git a/core/tokenizer/real_corpus_test.go b/packages/core/tokenizer/real_corpus_test.go similarity index 100% rename from core/tokenizer/real_corpus_test.go rename to packages/core/tokenizer/real_corpus_test.go diff --git a/core/tokenizer/stopwords.go b/packages/core/tokenizer/stopwords.go similarity index 100% rename from core/tokenizer/stopwords.go rename to packages/core/tokenizer/stopwords.go diff --git a/core/tokenizer/stopwords_test.go b/packages/core/tokenizer/stopwords_test.go similarity index 100% rename from core/tokenizer/stopwords_test.go rename to packages/core/tokenizer/stopwords_test.go diff --git a/core/tokenizer/suffix_golden_test.go b/packages/core/tokenizer/suffix_golden_test.go similarity index 100% rename from core/tokenizer/suffix_golden_test.go rename to packages/core/tokenizer/suffix_golden_test.go diff --git a/core/tokenizer/suffix_tokenizer.go b/packages/core/tokenizer/suffix_tokenizer.go similarity index 100% rename from core/tokenizer/suffix_tokenizer.go rename to packages/core/tokenizer/suffix_tokenizer.go diff --git a/core/tokenizer/suffix_tokenizer_test.go b/packages/core/tokenizer/suffix_tokenizer_test.go similarity index 100% rename from core/tokenizer/suffix_tokenizer_test.go rename to packages/core/tokenizer/suffix_tokenizer_test.go diff --git a/core/tokenizer/tokenize.go b/packages/core/tokenizer/tokenize.go similarity index 100% rename from core/tokenizer/tokenize.go rename to packages/core/tokenizer/tokenize.go diff --git a/core/tokenizer/tokenize_for_index_test.go b/packages/core/tokenizer/tokenize_for_index_test.go similarity index 100% rename from core/tokenizer/tokenize_for_index_test.go rename to packages/core/tokenizer/tokenize_for_index_test.go diff --git a/core/tokenizer/tokenize_for_search_test.go b/packages/core/tokenizer/tokenize_for_search_test.go similarity index 100% rename from core/tokenizer/tokenize_for_search_test.go rename to packages/core/tokenizer/tokenize_for_search_test.go diff --git a/core/tokenizer/tokenizer.go b/packages/core/tokenizer/tokenizer.go similarity index 100% rename from core/tokenizer/tokenizer.go rename to packages/core/tokenizer/tokenizer.go diff --git a/core/utils/git/ignore.go b/packages/core/utils/git/ignore.go similarity index 100% rename from core/utils/git/ignore.go rename to packages/core/utils/git/ignore.go diff --git a/core/utils/git/ignore_bench_test.go b/packages/core/utils/git/ignore_bench_test.go similarity index 100% rename from core/utils/git/ignore_bench_test.go rename to packages/core/utils/git/ignore_bench_test.go diff --git a/core/utils/git/ignore_relbase_test.go b/packages/core/utils/git/ignore_relbase_test.go similarity index 100% rename from core/utils/git/ignore_relbase_test.go rename to packages/core/utils/git/ignore_relbase_test.go diff --git a/core/utils/git/ignore_test.go b/packages/core/utils/git/ignore_test.go similarity index 100% rename from core/utils/git/ignore_test.go rename to packages/core/utils/git/ignore_test.go diff --git a/core/utils/git/pattern_match.go b/packages/core/utils/git/pattern_match.go similarity index 100% rename from core/utils/git/pattern_match.go rename to packages/core/utils/git/pattern_match.go diff --git a/core/utils/git/pattern_match_test.go b/packages/core/utils/git/pattern_match_test.go similarity index 100% rename from core/utils/git/pattern_match_test.go rename to packages/core/utils/git/pattern_match_test.go diff --git a/core/vectorstore/attr.go b/packages/core/vectorstore/attr.go similarity index 100% rename from core/vectorstore/attr.go rename to packages/core/vectorstore/attr.go diff --git a/core/vectorstore/attr_test.go b/packages/core/vectorstore/attr_test.go similarity index 100% rename from core/vectorstore/attr_test.go rename to packages/core/vectorstore/attr_test.go diff --git a/core/vectorstore/attrfile.go b/packages/core/vectorstore/attrfile.go similarity index 100% rename from core/vectorstore/attrfile.go rename to packages/core/vectorstore/attrfile.go diff --git a/core/vectorstore/attrfile_test.go b/packages/core/vectorstore/attrfile_test.go similarity index 100% rename from core/vectorstore/attrfile_test.go rename to packages/core/vectorstore/attrfile_test.go diff --git a/core/vectorstore/autoseal_test.go b/packages/core/vectorstore/autoseal_test.go similarity index 100% rename from core/vectorstore/autoseal_test.go rename to packages/core/vectorstore/autoseal_test.go diff --git a/core/vectorstore/batch.go b/packages/core/vectorstore/batch.go similarity index 100% rename from core/vectorstore/batch.go rename to packages/core/vectorstore/batch.go diff --git a/core/vectorstore/batch_test.go b/packages/core/vectorstore/batch_test.go similarity index 100% rename from core/vectorstore/batch_test.go rename to packages/core/vectorstore/batch_test.go diff --git a/core/vectorstore/bench_test.go b/packages/core/vectorstore/bench_test.go similarity index 100% rename from core/vectorstore/bench_test.go rename to packages/core/vectorstore/bench_test.go diff --git a/core/vectorstore/bitmap.go b/packages/core/vectorstore/bitmap.go similarity index 100% rename from core/vectorstore/bitmap.go rename to packages/core/vectorstore/bitmap.go diff --git a/core/vectorstore/bitmap_test.go b/packages/core/vectorstore/bitmap_test.go similarity index 100% rename from core/vectorstore/bitmap_test.go rename to packages/core/vectorstore/bitmap_test.go diff --git a/core/vectorstore/builder.go b/packages/core/vectorstore/builder.go similarity index 100% rename from core/vectorstore/builder.go rename to packages/core/vectorstore/builder.go diff --git a/core/vectorstore/builder_async_test.go b/packages/core/vectorstore/builder_async_test.go similarity index 100% rename from core/vectorstore/builder_async_test.go rename to packages/core/vectorstore/builder_async_test.go diff --git a/core/vectorstore/builder_test.go b/packages/core/vectorstore/builder_test.go similarity index 100% rename from core/vectorstore/builder_test.go rename to packages/core/vectorstore/builder_test.go diff --git a/core/vectorstore/controlstore.go b/packages/core/vectorstore/controlstore.go similarity index 100% rename from core/vectorstore/controlstore.go rename to packages/core/vectorstore/controlstore.go diff --git a/core/vectorstore/controlstore_test.go b/packages/core/vectorstore/controlstore_test.go similarity index 100% rename from core/vectorstore/controlstore_test.go rename to packages/core/vectorstore/controlstore_test.go diff --git a/core/vectorstore/coverage_faults_test.go b/packages/core/vectorstore/coverage_faults_test.go similarity index 100% rename from core/vectorstore/coverage_faults_test.go rename to packages/core/vectorstore/coverage_faults_test.go diff --git a/core/vectorstore/coverage_phase4_test.go b/packages/core/vectorstore/coverage_phase4_test.go similarity index 100% rename from core/vectorstore/coverage_phase4_test.go rename to packages/core/vectorstore/coverage_phase4_test.go diff --git a/core/vectorstore/doc.go b/packages/core/vectorstore/doc.go similarity index 100% rename from core/vectorstore/doc.go rename to packages/core/vectorstore/doc.go diff --git a/core/vectorstore/dot_amd64.go b/packages/core/vectorstore/dot_amd64.go similarity index 100% rename from core/vectorstore/dot_amd64.go rename to packages/core/vectorstore/dot_amd64.go diff --git a/core/vectorstore/dot_amd64.s b/packages/core/vectorstore/dot_amd64.s similarity index 100% rename from core/vectorstore/dot_amd64.s rename to packages/core/vectorstore/dot_amd64.s diff --git a/core/vectorstore/dot_amd64_test.go b/packages/core/vectorstore/dot_amd64_test.go similarity index 100% rename from core/vectorstore/dot_amd64_test.go rename to packages/core/vectorstore/dot_amd64_test.go diff --git a/core/vectorstore/dot_arm64.go b/packages/core/vectorstore/dot_arm64.go similarity index 100% rename from core/vectorstore/dot_arm64.go rename to packages/core/vectorstore/dot_arm64.go diff --git a/core/vectorstore/dot_arm64.s b/packages/core/vectorstore/dot_arm64.s similarity index 100% rename from core/vectorstore/dot_arm64.s rename to packages/core/vectorstore/dot_arm64.s diff --git a/core/vectorstore/drop_graph_files_test.go b/packages/core/vectorstore/drop_graph_files_test.go similarity index 100% rename from core/vectorstore/drop_graph_files_test.go rename to packages/core/vectorstore/drop_graph_files_test.go diff --git a/core/vectorstore/faulthelpers_test.go b/packages/core/vectorstore/faulthelpers_test.go similarity index 100% rename from core/vectorstore/faulthelpers_test.go rename to packages/core/vectorstore/faulthelpers_test.go diff --git a/core/vectorstore/graph_delete_test.go b/packages/core/vectorstore/graph_delete_test.go similarity index 100% rename from core/vectorstore/graph_delete_test.go rename to packages/core/vectorstore/graph_delete_test.go diff --git a/core/vectorstore/graphbatch.go b/packages/core/vectorstore/graphbatch.go similarity index 100% rename from core/vectorstore/graphbatch.go rename to packages/core/vectorstore/graphbatch.go diff --git a/core/vectorstore/graphfile.go b/packages/core/vectorstore/graphfile.go similarity index 100% rename from core/vectorstore/graphfile.go rename to packages/core/vectorstore/graphfile.go diff --git a/core/vectorstore/graphfile_format.go b/packages/core/vectorstore/graphfile_format.go similarity index 100% rename from core/vectorstore/graphfile_format.go rename to packages/core/vectorstore/graphfile_format.go diff --git a/core/vectorstore/graphfile_test.go b/packages/core/vectorstore/graphfile_test.go similarity index 100% rename from core/vectorstore/graphfile_test.go rename to packages/core/vectorstore/graphfile_test.go diff --git a/core/vectorstore/graphmem_test.go b/packages/core/vectorstore/graphmem_test.go similarity index 100% rename from core/vectorstore/graphmem_test.go rename to packages/core/vectorstore/graphmem_test.go diff --git a/core/vectorstore/graphstore.go b/packages/core/vectorstore/graphstore.go similarity index 100% rename from core/vectorstore/graphstore.go rename to packages/core/vectorstore/graphstore.go diff --git a/core/vectorstore/graphstore_test.go b/packages/core/vectorstore/graphstore_test.go similarity index 100% rename from core/vectorstore/graphstore_test.go rename to packages/core/vectorstore/graphstore_test.go diff --git a/core/vectorstore/hnsw.go b/packages/core/vectorstore/hnsw.go similarity index 100% rename from core/vectorstore/hnsw.go rename to packages/core/vectorstore/hnsw.go diff --git a/core/vectorstore/hnsw_test.go b/packages/core/vectorstore/hnsw_test.go similarity index 100% rename from core/vectorstore/hnsw_test.go rename to packages/core/vectorstore/hnsw_test.go diff --git a/core/vectorstore/manifest.go b/packages/core/vectorstore/manifest.go similarity index 100% rename from core/vectorstore/manifest.go rename to packages/core/vectorstore/manifest.go diff --git a/core/vectorstore/manifest_test.go b/packages/core/vectorstore/manifest_test.go similarity index 100% rename from core/vectorstore/manifest_test.go rename to packages/core/vectorstore/manifest_test.go diff --git a/core/vectorstore/memgraphstore_test.go b/packages/core/vectorstore/memgraphstore_test.go similarity index 100% rename from core/vectorstore/memgraphstore_test.go rename to packages/core/vectorstore/memgraphstore_test.go diff --git a/core/vectorstore/merge.go b/packages/core/vectorstore/merge.go similarity index 100% rename from core/vectorstore/merge.go rename to packages/core/vectorstore/merge.go diff --git a/core/vectorstore/merge_abort_test.go b/packages/core/vectorstore/merge_abort_test.go similarity index 100% rename from core/vectorstore/merge_abort_test.go rename to packages/core/vectorstore/merge_abort_test.go diff --git a/core/vectorstore/merge_buildtrigger_test.go b/packages/core/vectorstore/merge_buildtrigger_test.go similarity index 100% rename from core/vectorstore/merge_buildtrigger_test.go rename to packages/core/vectorstore/merge_buildtrigger_test.go diff --git a/core/vectorstore/merge_concurrent_race_test.go b/packages/core/vectorstore/merge_concurrent_race_test.go similarity index 100% rename from core/vectorstore/merge_concurrent_race_test.go rename to packages/core/vectorstore/merge_concurrent_race_test.go diff --git a/core/vectorstore/merge_concurrent_test.go b/packages/core/vectorstore/merge_concurrent_test.go similarity index 100% rename from core/vectorstore/merge_concurrent_test.go rename to packages/core/vectorstore/merge_concurrent_test.go diff --git a/core/vectorstore/merge_corrupt_payload_test.go b/packages/core/vectorstore/merge_corrupt_payload_test.go similarity index 100% rename from core/vectorstore/merge_corrupt_payload_test.go rename to packages/core/vectorstore/merge_corrupt_payload_test.go diff --git a/core/vectorstore/merge_countcap_test.go b/packages/core/vectorstore/merge_countcap_test.go similarity index 100% rename from core/vectorstore/merge_countcap_test.go rename to packages/core/vectorstore/merge_countcap_test.go diff --git a/core/vectorstore/merge_crash_test.go b/packages/core/vectorstore/merge_crash_test.go similarity index 100% rename from core/vectorstore/merge_crash_test.go rename to packages/core/vectorstore/merge_crash_test.go diff --git a/core/vectorstore/merge_repend_race_test.go b/packages/core/vectorstore/merge_repend_race_test.go similarity index 100% rename from core/vectorstore/merge_repend_race_test.go rename to packages/core/vectorstore/merge_repend_race_test.go diff --git a/core/vectorstore/merge_test.go b/packages/core/vectorstore/merge_test.go similarity index 100% rename from core/vectorstore/merge_test.go rename to packages/core/vectorstore/merge_test.go diff --git a/core/vectorstore/merge_trigger_test.go b/packages/core/vectorstore/merge_trigger_test.go similarity index 100% rename from core/vectorstore/merge_trigger_test.go rename to packages/core/vectorstore/merge_trigger_test.go diff --git a/core/vectorstore/mergepolicy.go b/packages/core/vectorstore/mergepolicy.go similarity index 100% rename from core/vectorstore/mergepolicy.go rename to packages/core/vectorstore/mergepolicy.go diff --git a/core/vectorstore/mergepolicy_test.go b/packages/core/vectorstore/mergepolicy_test.go similarity index 100% rename from core/vectorstore/mergepolicy_test.go rename to packages/core/vectorstore/mergepolicy_test.go diff --git a/core/vectorstore/metric.go b/packages/core/vectorstore/metric.go similarity index 100% rename from core/vectorstore/metric.go rename to packages/core/vectorstore/metric.go diff --git a/core/vectorstore/metric_distance_amd64.go b/packages/core/vectorstore/metric_distance_amd64.go similarity index 100% rename from core/vectorstore/metric_distance_amd64.go rename to packages/core/vectorstore/metric_distance_amd64.go diff --git a/core/vectorstore/metric_distance_arm64.go b/packages/core/vectorstore/metric_distance_arm64.go similarity index 100% rename from core/vectorstore/metric_distance_arm64.go rename to packages/core/vectorstore/metric_distance_arm64.go diff --git a/core/vectorstore/metric_persist_test.go b/packages/core/vectorstore/metric_persist_test.go similarity index 100% rename from core/vectorstore/metric_persist_test.go rename to packages/core/vectorstore/metric_persist_test.go diff --git a/core/vectorstore/metric_test.go b/packages/core/vectorstore/metric_test.go similarity index 100% rename from core/vectorstore/metric_test.go rename to packages/core/vectorstore/metric_test.go diff --git a/core/vectorstore/mmap.go b/packages/core/vectorstore/mmap.go similarity index 100% rename from core/vectorstore/mmap.go rename to packages/core/vectorstore/mmap.go diff --git a/core/vectorstore/mmap_test.go b/packages/core/vectorstore/mmap_test.go similarity index 100% rename from core/vectorstore/mmap_test.go rename to packages/core/vectorstore/mmap_test.go diff --git a/core/vectorstore/mmap_unix.go b/packages/core/vectorstore/mmap_unix.go similarity index 100% rename from core/vectorstore/mmap_unix.go rename to packages/core/vectorstore/mmap_unix.go diff --git a/core/vectorstore/mmap_windows.go b/packages/core/vectorstore/mmap_windows.go similarity index 100% rename from core/vectorstore/mmap_windows.go rename to packages/core/vectorstore/mmap_windows.go diff --git a/core/vectorstore/nodestore.go b/packages/core/vectorstore/nodestore.go similarity index 100% rename from core/vectorstore/nodestore.go rename to packages/core/vectorstore/nodestore.go diff --git a/core/vectorstore/orphan_firstseal_test.go b/packages/core/vectorstore/orphan_firstseal_test.go similarity index 100% rename from core/vectorstore/orphan_firstseal_test.go rename to packages/core/vectorstore/orphan_firstseal_test.go diff --git a/core/vectorstore/orphan_test.go b/packages/core/vectorstore/orphan_test.go similarity index 100% rename from core/vectorstore/orphan_test.go rename to packages/core/vectorstore/orphan_test.go diff --git a/core/vectorstore/osfile.go b/packages/core/vectorstore/osfile.go similarity index 100% rename from core/vectorstore/osfile.go rename to packages/core/vectorstore/osfile.go diff --git a/core/vectorstore/payload.go b/packages/core/vectorstore/payload.go similarity index 100% rename from core/vectorstore/payload.go rename to packages/core/vectorstore/payload.go diff --git a/core/vectorstore/payload_test.go b/packages/core/vectorstore/payload_test.go similarity index 100% rename from core/vectorstore/payload_test.go rename to packages/core/vectorstore/payload_test.go diff --git a/core/vectorstore/phase2_coverage_test.go b/packages/core/vectorstore/phase2_coverage_test.go similarity index 100% rename from core/vectorstore/phase2_coverage_test.go rename to packages/core/vectorstore/phase2_coverage_test.go diff --git a/core/vectorstore/predicate.go b/packages/core/vectorstore/predicate.go similarity index 100% rename from core/vectorstore/predicate.go rename to packages/core/vectorstore/predicate.go diff --git a/core/vectorstore/predicate_test.go b/packages/core/vectorstore/predicate_test.go similarity index 100% rename from core/vectorstore/predicate_test.go rename to packages/core/vectorstore/predicate_test.go diff --git a/core/vectorstore/recover_reopen_resume_race_test.go b/packages/core/vectorstore/recover_reopen_resume_race_test.go similarity index 100% rename from core/vectorstore/recover_reopen_resume_race_test.go rename to packages/core/vectorstore/recover_reopen_resume_race_test.go diff --git a/core/vectorstore/recover_stray_graph_test.go b/packages/core/vectorstore/recover_stray_graph_test.go similarity index 100% rename from core/vectorstore/recover_stray_graph_test.go rename to packages/core/vectorstore/recover_stray_graph_test.go diff --git a/core/vectorstore/recovery_branches_test.go b/packages/core/vectorstore/recovery_branches_test.go similarity index 100% rename from core/vectorstore/recovery_branches_test.go rename to packages/core/vectorstore/recovery_branches_test.go diff --git a/core/vectorstore/recovery_test.go b/packages/core/vectorstore/recovery_test.go similarity index 100% rename from core/vectorstore/recovery_test.go rename to packages/core/vectorstore/recovery_test.go diff --git a/core/vectorstore/reindex.go b/packages/core/vectorstore/reindex.go similarity index 100% rename from core/vectorstore/reindex.go rename to packages/core/vectorstore/reindex.go diff --git a/core/vectorstore/reindex_test.go b/packages/core/vectorstore/reindex_test.go similarity index 100% rename from core/vectorstore/reindex_test.go rename to packages/core/vectorstore/reindex_test.go diff --git a/core/vectorstore/result.go b/packages/core/vectorstore/result.go similarity index 100% rename from core/vectorstore/result.go rename to packages/core/vectorstore/result.go diff --git a/core/vectorstore/result_test.go b/packages/core/vectorstore/result_test.go similarity index 100% rename from core/vectorstore/result_test.go rename to packages/core/vectorstore/result_test.go diff --git a/core/vectorstore/seal.go b/packages/core/vectorstore/seal.go similarity index 100% rename from core/vectorstore/seal.go rename to packages/core/vectorstore/seal.go diff --git a/core/vectorstore/seal_idtable_durability_test.go b/packages/core/vectorstore/seal_idtable_durability_test.go similarity index 100% rename from core/vectorstore/seal_idtable_durability_test.go rename to packages/core/vectorstore/seal_idtable_durability_test.go diff --git a/core/vectorstore/seal_test.go b/packages/core/vectorstore/seal_test.go similarity index 100% rename from core/vectorstore/seal_test.go rename to packages/core/vectorstore/seal_test.go diff --git a/core/vectorstore/seal_tombstone_test.go b/packages/core/vectorstore/seal_tombstone_test.go similarity index 100% rename from core/vectorstore/seal_tombstone_test.go rename to packages/core/vectorstore/seal_tombstone_test.go diff --git a/core/vectorstore/sealed.go b/packages/core/vectorstore/sealed.go similarity index 100% rename from core/vectorstore/sealed.go rename to packages/core/vectorstore/sealed.go diff --git a/core/vectorstore/sealed_getvectorref_test.go b/packages/core/vectorstore/sealed_getvectorref_test.go similarity index 100% rename from core/vectorstore/sealed_getvectorref_test.go rename to packages/core/vectorstore/sealed_getvectorref_test.go diff --git a/core/vectorstore/segfile_format.go b/packages/core/vectorstore/segfile_format.go similarity index 100% rename from core/vectorstore/segfile_format.go rename to packages/core/vectorstore/segfile_format.go diff --git a/core/vectorstore/segfile_format_test.go b/packages/core/vectorstore/segfile_format_test.go similarity index 100% rename from core/vectorstore/segfile_format_test.go rename to packages/core/vectorstore/segfile_format_test.go diff --git a/core/vectorstore/segment.go b/packages/core/vectorstore/segment.go similarity index 100% rename from core/vectorstore/segment.go rename to packages/core/vectorstore/segment.go diff --git a/core/vectorstore/segment_test.go b/packages/core/vectorstore/segment_test.go similarity index 100% rename from core/vectorstore/segment_test.go rename to packages/core/vectorstore/segment_test.go diff --git a/core/vectorstore/store.go b/packages/core/vectorstore/store.go similarity index 100% rename from core/vectorstore/store.go rename to packages/core/vectorstore/store.go diff --git a/core/vectorstore/store_attr_recovery_test.go b/packages/core/vectorstore/store_attr_recovery_test.go similarity index 100% rename from core/vectorstore/store_attr_recovery_test.go rename to packages/core/vectorstore/store_attr_recovery_test.go diff --git a/core/vectorstore/store_attr_test.go b/packages/core/vectorstore/store_attr_test.go similarity index 100% rename from core/vectorstore/store_attr_test.go rename to packages/core/vectorstore/store_attr_test.go diff --git a/core/vectorstore/store_filter_test.go b/packages/core/vectorstore/store_filter_test.go similarity index 100% rename from core/vectorstore/store_filter_test.go rename to packages/core/vectorstore/store_filter_test.go diff --git a/core/vectorstore/store_head_attr_test.go b/packages/core/vectorstore/store_head_attr_test.go similarity index 100% rename from core/vectorstore/store_head_attr_test.go rename to packages/core/vectorstore/store_head_attr_test.go diff --git a/core/vectorstore/store_heavy_delete_test.go b/packages/core/vectorstore/store_heavy_delete_test.go similarity index 100% rename from core/vectorstore/store_heavy_delete_test.go rename to packages/core/vectorstore/store_heavy_delete_test.go diff --git a/core/vectorstore/store_payload_test.go b/packages/core/vectorstore/store_payload_test.go similarity index 100% rename from core/vectorstore/store_payload_test.go rename to packages/core/vectorstore/store_payload_test.go diff --git a/core/vectorstore/store_search_test.go b/packages/core/vectorstore/store_search_test.go similarity index 100% rename from core/vectorstore/store_search_test.go rename to packages/core/vectorstore/store_search_test.go diff --git a/core/vectorstore/store_segset_test.go b/packages/core/vectorstore/store_segset_test.go similarity index 100% rename from core/vectorstore/store_segset_test.go rename to packages/core/vectorstore/store_segset_test.go diff --git a/core/vectorstore/store_test.go b/packages/core/vectorstore/store_test.go similarity index 100% rename from core/vectorstore/store_test.go rename to packages/core/vectorstore/store_test.go diff --git a/core/vectorstore/storehelpers_test.go b/packages/core/vectorstore/storehelpers_test.go similarity index 100% rename from core/vectorstore/storehelpers_test.go rename to packages/core/vectorstore/storehelpers_test.go diff --git a/core/vectorstore/validate.go b/packages/core/vectorstore/validate.go similarity index 100% rename from core/vectorstore/validate.go rename to packages/core/vectorstore/validate.go diff --git a/core/vectorstore/validate_test.go b/packages/core/vectorstore/validate_test.go similarity index 100% rename from core/vectorstore/validate_test.go rename to packages/core/vectorstore/validate_test.go diff --git a/core/vectorstore/vindex.go b/packages/core/vectorstore/vindex.go similarity index 100% rename from core/vectorstore/vindex.go rename to packages/core/vectorstore/vindex.go diff --git a/core/vectorstore/vindex_metric_recover_test.go b/packages/core/vectorstore/vindex_metric_recover_test.go similarity index 100% rename from core/vectorstore/vindex_metric_recover_test.go rename to packages/core/vectorstore/vindex_metric_recover_test.go diff --git a/core/vectorstore/vindex_metric_test.go b/packages/core/vectorstore/vindex_metric_test.go similarity index 100% rename from core/vectorstore/vindex_metric_test.go rename to packages/core/vectorstore/vindex_metric_test.go diff --git a/core/vectorstore/vindex_test.go b/packages/core/vectorstore/vindex_test.go similarity index 100% rename from core/vectorstore/vindex_test.go rename to packages/core/vectorstore/vindex_test.go diff --git a/.go-cov.toml b/packages/server/.go-cov.toml similarity index 75% rename from .go-cov.toml rename to packages/server/.go-cov.toml index 15529ee3..07c809d8 100644 --- a/.go-cov.toml +++ b/packages/server/.go-cov.toml @@ -1,10 +1,9 @@ -project = "haystack" +project = "server" module_prefix = "github.com/codetrek/haystack/" [exclude] packages = [ - "haystack/scripts/", - "haystack/cmd", + "server/cmd/haystack", ] files = [ "internal/testutil/", diff --git a/packages/server/Dockerfile.test b/packages/server/Dockerfile.test new file mode 100644 index 00000000..ec1698e8 --- /dev/null +++ b/packages/server/Dockerfile.test @@ -0,0 +1,10 @@ +FROM golang:1.24 + +WORKDIR /app + +# The whole repo — including go.work and the workspace vendor/ directory — is +# mounted read-only at /app at runtime (see `make test-safe`). Tests therefore run +# fully offline (--network=none) against the vendored dependencies; no COPY / +# `go mod download` is needed. golang:1.24 satisfies the go1.24.2 toolchain +# directive, so nothing is downloaded at test time either. +CMD ["bash", "-c", "ulimit -u 256 && go test ./packages/server/... ./packages/core/... -count=1 -timeout 5m"] diff --git a/cmd/haystack/main.go b/packages/server/cmd/haystack/main.go similarity index 79% rename from cmd/haystack/main.go rename to packages/server/cmd/haystack/main.go index 591f9cac..c796db7b 100644 --- a/cmd/haystack/main.go +++ b/packages/server/cmd/haystack/main.go @@ -7,10 +7,10 @@ import ( _ "net/http/pprof" "path/filepath" - "github.com/codetrek/haystack/internal/client" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/server" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/client" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/server" + "github.com/codetrek/haystack/server/internal/shared/running" ) var version = "dev" diff --git a/config.example.yaml b/packages/server/config.example.yaml similarity index 100% rename from config.example.yaml rename to packages/server/config.example.yaml diff --git a/go.mod b/packages/server/go.mod similarity index 96% rename from go.mod rename to packages/server/go.mod index c6ee5b79..400815e0 100644 --- a/go.mod +++ b/packages/server/go.mod @@ -1,4 +1,4 @@ -module github.com/codetrek/haystack +module github.com/codetrek/haystack/server go 1.23.0 diff --git a/go.sum b/packages/server/go.sum similarity index 100% rename from go.sum rename to packages/server/go.sum diff --git a/internal/client/client.go b/packages/server/internal/client/client.go similarity index 96% rename from internal/client/client.go rename to packages/server/internal/client/client.go index 225760b5..ce2db4c2 100644 --- a/internal/client/client.go +++ b/packages/server/internal/client/client.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/running" ) func Run() { diff --git a/internal/client/client_test.go b/packages/server/internal/client/client_test.go similarity index 100% rename from internal/client/client_test.go rename to packages/server/internal/client/client_test.go diff --git a/internal/client/common.go b/packages/server/internal/client/common.go similarity index 93% rename from internal/client/common.go rename to packages/server/internal/client/common.go index 337dae97..ba639e44 100644 --- a/internal/client/common.go +++ b/packages/server/internal/client/common.go @@ -10,8 +10,8 @@ import ( "net/http" "time" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/shared/types" ) type result struct { diff --git a/internal/client/common_test.go b/packages/server/internal/client/common_test.go similarity index 97% rename from internal/client/common_test.go rename to packages/server/internal/client/common_test.go index 03a0b99a..8823862f 100644 --- a/internal/client/common_test.go +++ b/packages/server/internal/client/common_test.go @@ -10,8 +10,8 @@ import ( "strings" "testing" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/shared/types" ) // startMockServer creates a test HTTP server and configures conf to point to it. diff --git a/internal/client/handle_search.go b/packages/server/internal/client/handle_search.go similarity index 97% rename from internal/client/handle_search.go rename to packages/server/internal/client/handle_search.go index ff1477a8..6de17bba 100644 --- a/internal/client/handle_search.go +++ b/packages/server/internal/client/handle_search.go @@ -6,9 +6,9 @@ import ( "fmt" "strings" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" ) func handleSearch(args []string) { diff --git a/internal/client/handle_search_test.go b/packages/server/internal/client/handle_search_test.go similarity index 99% rename from internal/client/handle_search_test.go rename to packages/server/internal/client/handle_search_test.go index 521903c4..b2d21a21 100644 --- a/internal/client/handle_search_test.go +++ b/packages/server/internal/client/handle_search_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/types" ) func TestHandleSearch_Help(t *testing.T) { diff --git a/internal/client/handle_server.go b/packages/server/internal/client/handle_server.go similarity index 93% rename from internal/client/handle_server.go rename to packages/server/internal/client/handle_server.go index e2a58c4f..5f325683 100644 --- a/internal/client/handle_server.go +++ b/packages/server/internal/client/handle_server.go @@ -5,10 +5,10 @@ import ( "fmt" "time" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/server" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/server" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" ) func handleServer(args []string) { diff --git a/internal/client/handle_server_coverage_test.go b/packages/server/internal/client/handle_server_coverage_test.go similarity index 98% rename from internal/client/handle_server_coverage_test.go rename to packages/server/internal/client/handle_server_coverage_test.go index c53f2d68..73310ea3 100644 --- a/internal/client/handle_server_coverage_test.go +++ b/packages/server/internal/client/handle_server_coverage_test.go @@ -7,8 +7,8 @@ import ( "strings" "testing" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" "github.com/gofrs/flock" ) diff --git a/internal/client/handle_server_test.go b/packages/server/internal/client/handle_server_test.go similarity index 98% rename from internal/client/handle_server_test.go rename to packages/server/internal/client/handle_server_test.go index 215de242..53cd39bd 100644 --- a/internal/client/handle_server_test.go +++ b/packages/server/internal/client/handle_server_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/types" ) func TestHandleServer_Help(t *testing.T) { diff --git a/internal/client/handle_symbols.go b/packages/server/internal/client/handle_symbols.go similarity index 93% rename from internal/client/handle_symbols.go rename to packages/server/internal/client/handle_symbols.go index a647c226..7dd3d1e1 100644 --- a/internal/client/handle_symbols.go +++ b/packages/server/internal/client/handle_symbols.go @@ -6,9 +6,9 @@ import ( "fmt" "strings" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" ) func sendSearchSymbolsRequest(req types.SearchSymbolsRequest) (*types.SymbolsContentResults, error) { diff --git a/internal/client/handle_symbols_test.go b/packages/server/internal/client/handle_symbols_test.go similarity index 98% rename from internal/client/handle_symbols_test.go rename to packages/server/internal/client/handle_symbols_test.go index 70ed1981..07ca97db 100644 --- a/internal/client/handle_symbols_test.go +++ b/packages/server/internal/client/handle_symbols_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/types" ) func TestHandleSymbols_Help(t *testing.T) { diff --git a/internal/client/handle_workspace.go b/packages/server/internal/client/handle_workspace.go similarity index 97% rename from internal/client/handle_workspace.go rename to packages/server/internal/client/handle_workspace.go index d8b571ab..179e42e3 100644 --- a/internal/client/handle_workspace.go +++ b/packages/server/internal/client/handle_workspace.go @@ -6,8 +6,8 @@ import ( "path/filepath" "strconv" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" ) func handleWorkspace(args []string) { diff --git a/internal/client/handle_workspace_test.go b/packages/server/internal/client/handle_workspace_test.go similarity index 99% rename from internal/client/handle_workspace_test.go rename to packages/server/internal/client/handle_workspace_test.go index a955a472..03607192 100644 --- a/internal/client/handle_workspace_test.go +++ b/packages/server/internal/client/handle_workspace_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/types" ) func TestHandleWorkspace_Help(t *testing.T) { diff --git a/internal/client/help.go b/packages/server/internal/client/help.go similarity index 100% rename from internal/client/help.go rename to packages/server/internal/client/help.go diff --git a/internal/client/help_test.go b/packages/server/internal/client/help_test.go similarity index 100% rename from internal/client/help_test.go rename to packages/server/internal/client/help_test.go diff --git a/internal/conf/conf.go b/packages/server/internal/conf/conf.go similarity index 97% rename from internal/conf/conf.go rename to packages/server/internal/conf/conf.go index eeeb9977..84d74179 100644 --- a/internal/conf/conf.go +++ b/packages/server/internal/conf/conf.go @@ -6,9 +6,9 @@ import ( "path/filepath" "runtime" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" - fsutils "github.com/codetrek/haystack/internal/utils/fs" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" + fsutils "github.com/codetrek/haystack/server/internal/utils/fs" "gopkg.in/yaml.v3" ) diff --git a/internal/conf/conf_test.go b/packages/server/internal/conf/conf_test.go similarity index 100% rename from internal/conf/conf_test.go rename to packages/server/internal/conf/conf_test.go diff --git a/internal/core/storage/storage.go b/packages/server/internal/core/storage/storage.go similarity index 100% rename from internal/core/storage/storage.go rename to packages/server/internal/core/storage/storage.go diff --git a/internal/core/storage/storage_test.go b/packages/server/internal/core/storage/storage_test.go similarity index 100% rename from internal/core/storage/storage_test.go rename to packages/server/internal/core/storage/storage_test.go diff --git a/internal/core/storage/types.go b/packages/server/internal/core/storage/types.go similarity index 100% rename from internal/core/storage/types.go rename to packages/server/internal/core/storage/types.go diff --git a/internal/core/symbols/codec.go b/packages/server/internal/core/symbols/codec.go similarity index 96% rename from internal/core/symbols/codec.go rename to packages/server/internal/core/symbols/codec.go index f63a911b..6fb5ed40 100644 --- a/internal/core/symbols/codec.go +++ b/packages/server/internal/core/symbols/codec.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - "github.com/codetrek/haystack/internal/core/storage" + "github.com/codetrek/haystack/server/internal/core/storage" ) const ( diff --git a/internal/core/symbols/codec_test.go b/packages/server/internal/core/symbols/codec_test.go similarity index 99% rename from internal/core/symbols/codec_test.go rename to packages/server/internal/core/symbols/codec_test.go index 85715b64..527e8e1f 100644 --- a/internal/core/symbols/codec_test.go +++ b/packages/server/internal/core/symbols/codec_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/codetrek/haystack/internal/core/storage" + "github.com/codetrek/haystack/server/internal/core/storage" "github.com/stretchr/testify/assert" ) diff --git a/internal/core/symbols/database.go b/packages/server/internal/core/symbols/database.go similarity index 97% rename from internal/core/symbols/database.go rename to packages/server/internal/core/symbols/database.go index dbbdf9b3..402fe7e8 100644 --- a/internal/core/symbols/database.go +++ b/packages/server/internal/core/symbols/database.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/codetrek/haystack/internal/conf" + "github.com/codetrek/haystack/server/internal/conf" ) type SymbolUniversalTable struct { diff --git a/internal/core/symbols/function.go b/packages/server/internal/core/symbols/function.go similarity index 99% rename from internal/core/symbols/function.go rename to packages/server/internal/core/symbols/function.go index f9a1d90c..e1d80b9d 100644 --- a/internal/core/symbols/function.go +++ b/packages/server/internal/core/symbols/function.go @@ -9,7 +9,7 @@ import ( "github.com/codetrek/haystack/core/idtable" "github.com/codetrek/haystack/core/kv" "github.com/codetrek/haystack/core/tokenizer" - "github.com/codetrek/haystack/internal/conf" + "github.com/codetrek/haystack/server/internal/conf" ) type Function struct { diff --git a/internal/core/symbols/storage.go b/packages/server/internal/core/symbols/storage.go similarity index 100% rename from internal/core/symbols/storage.go rename to packages/server/internal/core/symbols/storage.go diff --git a/internal/core/symbols/symbols_test.go b/packages/server/internal/core/symbols/symbols_test.go similarity index 99% rename from internal/core/symbols/symbols_test.go rename to packages/server/internal/core/symbols/symbols_test.go index 422cb852..3bb4d10f 100644 --- a/internal/core/symbols/symbols_test.go +++ b/packages/server/internal/core/symbols/symbols_test.go @@ -7,8 +7,8 @@ import ( "time" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/storage" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/storage" "github.com/stretchr/testify/assert" ) diff --git a/internal/core/symbols/test_helper_test.go b/packages/server/internal/core/symbols/test_helper_test.go similarity index 96% rename from internal/core/symbols/test_helper_test.go rename to packages/server/internal/core/symbols/test_helper_test.go index de04aab8..0e57ad3a 100644 --- a/internal/core/symbols/test_helper_test.go +++ b/packages/server/internal/core/symbols/test_helper_test.go @@ -8,8 +8,8 @@ import ( "github.com/codetrek/haystack/core/invertedindex" "github.com/codetrek/haystack/core/kv/pebblekv" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/testutil" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/testutil" ) // testEnv holds all resources created during test setup so they can diff --git a/internal/core/workspace/init.go b/packages/server/internal/core/workspace/init.go similarity index 95% rename from internal/core/workspace/init.go rename to packages/server/internal/core/workspace/init.go index c5cd784b..7bab5173 100644 --- a/internal/core/workspace/init.go +++ b/packages/server/internal/core/workspace/init.go @@ -5,7 +5,7 @@ import ( "sync" "github.com/codetrek/haystack/core/collection" - "github.com/codetrek/haystack/internal/utils" + "github.com/codetrek/haystack/server/internal/utils" ) var ( diff --git a/internal/core/workspace/init_test.go b/packages/server/internal/core/workspace/init_test.go similarity index 98% rename from internal/core/workspace/init_test.go rename to packages/server/internal/core/workspace/init_test.go index a6b3c4b7..0f19e092 100644 --- a/internal/core/workspace/init_test.go +++ b/packages/server/internal/core/workspace/init_test.go @@ -14,9 +14,9 @@ import ( "github.com/codetrek/haystack/core/invertedindex" "github.com/codetrek/haystack/core/kv" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/storage" - "github.com/codetrek/haystack/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/storage" + "github.com/codetrek/haystack/server/internal/core/symbols" ) // setupCatalog is a test helper: runs migration, creates collection.Catalog + documents.Store. diff --git a/internal/core/workspace/manage.go b/packages/server/internal/core/workspace/manage.go similarity index 96% rename from internal/core/workspace/manage.go rename to packages/server/internal/core/workspace/manage.go index d9f792f8..289da7d7 100644 --- a/internal/core/workspace/manage.go +++ b/packages/server/internal/core/workspace/manage.go @@ -8,9 +8,9 @@ import ( "sort" "time" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/shared/types" - "github.com/codetrek/haystack/internal/utils" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/shared/types" + "github.com/codetrek/haystack/server/internal/utils" ) func GetAllPaths() []string { diff --git a/internal/core/workspace/serialize_test.go b/packages/server/internal/core/workspace/serialize_test.go similarity index 95% rename from internal/core/workspace/serialize_test.go rename to packages/server/internal/core/workspace/serialize_test.go index 6c4e2a45..fcb37089 100644 --- a/internal/core/workspace/serialize_test.go +++ b/packages/server/internal/core/workspace/serialize_test.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/types" ) // wsOnDisk is the legacy JSON shape emitted by Serialize. It has no production diff --git a/internal/core/workspace/workspace.go b/packages/server/internal/core/workspace/workspace.go similarity index 98% rename from internal/core/workspace/workspace.go rename to packages/server/internal/core/workspace/workspace.go index d915d986..e6e3e9ec 100644 --- a/internal/core/workspace/workspace.go +++ b/packages/server/internal/core/workspace/workspace.go @@ -8,8 +8,8 @@ import ( "github.com/codetrek/haystack/core/collection" "github.com/codetrek/haystack/core/documents" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/shared/types" ) type IndexingState int diff --git a/internal/core/workspace/workspace_indexing_test.go b/packages/server/internal/core/workspace/workspace_indexing_test.go similarity index 100% rename from internal/core/workspace/workspace_indexing_test.go rename to packages/server/internal/core/workspace/workspace_indexing_test.go diff --git a/internal/core/workspace/workspace_test.go b/packages/server/internal/core/workspace/workspace_test.go similarity index 98% rename from internal/core/workspace/workspace_test.go rename to packages/server/internal/core/workspace/workspace_test.go index 52f2f5df..b6a6e1cb 100644 --- a/internal/core/workspace/workspace_test.go +++ b/packages/server/internal/core/workspace/workspace_test.go @@ -12,9 +12,9 @@ import ( "github.com/codetrek/haystack/core/collection" "github.com/codetrek/haystack/core/documents" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/storage" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/storage" + "github.com/codetrek/haystack/server/internal/shared/types" ) // newTestStore creates a transient documents.Store for use in workspace unit diff --git a/internal/server/README.md b/packages/server/internal/server/README.md similarity index 100% rename from internal/server/README.md rename to packages/server/internal/server/README.md diff --git a/internal/server/coverage_test.go b/packages/server/internal/server/coverage_test.go similarity index 97% rename from internal/server/coverage_test.go rename to packages/server/internal/server/coverage_test.go index 3c851e2f..fe7aec89 100644 --- a/internal/server/coverage_test.go +++ b/packages/server/internal/server/coverage_test.go @@ -12,8 +12,8 @@ import ( "github.com/codetrek/haystack/core/invertedindex" "github.com/codetrek/haystack/core/kv" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/shared/running" ) // TestInitLog_Stdout verifies initLog configures stdout logging. diff --git a/internal/server/httpapi/document.go b/packages/server/internal/server/httpapi/document.go similarity index 90% rename from internal/server/httpapi/document.go rename to packages/server/internal/server/httpapi/document.go index d2fb7ed5..874cd955 100644 --- a/internal/server/httpapi/document.go +++ b/packages/server/internal/server/httpapi/document.go @@ -5,9 +5,9 @@ import ( "log" "net/http" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/shared/types" ) func handleUpdateDocument(w http.ResponseWriter, r *http.Request) { diff --git a/internal/server/httpapi/handler_validation_test.go b/packages/server/internal/server/httpapi/handler_validation_test.go similarity index 99% rename from internal/server/httpapi/handler_validation_test.go rename to packages/server/internal/server/httpapi/handler_validation_test.go index 3d9aa809..7fcf52cd 100644 --- a/internal/server/httpapi/handler_validation_test.go +++ b/packages/server/internal/server/httpapi/handler_validation_test.go @@ -7,7 +7,7 @@ import ( "net/http/httptest" "testing" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/types" "github.com/stretchr/testify/assert" ) diff --git a/internal/server/httpapi/handlers_test.go b/packages/server/internal/server/httpapi/handlers_test.go similarity index 99% rename from internal/server/httpapi/handlers_test.go rename to packages/server/internal/server/httpapi/handlers_test.go index eb0ada16..662e8b0b 100644 --- a/internal/server/httpapi/handlers_test.go +++ b/packages/server/internal/server/httpapi/handlers_test.go @@ -18,14 +18,14 @@ import ( "github.com/codetrek/haystack/core/idtable" "github.com/codetrek/haystack/core/invertedindex" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/storage" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/server/searcher" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/storage" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/server/searcher" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" mcpGoServer "github.com/mark3labs/mcp-go/server" "github.com/stretchr/testify/assert" ) diff --git a/internal/server/httpapi/mcp.go b/packages/server/internal/server/httpapi/mcp.go similarity index 96% rename from internal/server/httpapi/mcp.go rename to packages/server/internal/server/httpapi/mcp.go index 325be067..13c2bd60 100644 --- a/internal/server/httpapi/mcp.go +++ b/packages/server/internal/server/httpapi/mcp.go @@ -6,9 +6,9 @@ import ( "net/http" "time" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/server/mcptools" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/server/mcptools" + "github.com/codetrek/haystack/server/internal/shared/running" "github.com/mark3labs/mcp-go/mcp" "github.com/mark3labs/mcp-go/server" diff --git a/internal/server/httpapi/search.go b/packages/server/internal/server/httpapi/search.go similarity index 96% rename from internal/server/httpapi/search.go rename to packages/server/internal/server/httpapi/search.go index eceedc29..c2809156 100644 --- a/internal/server/httpapi/search.go +++ b/packages/server/internal/server/httpapi/search.go @@ -8,11 +8,11 @@ import ( "path/filepath" "time" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/searcher" - "github.com/codetrek/haystack/internal/shared/types" - "github.com/codetrek/haystack/internal/utils" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/searcher" + "github.com/codetrek/haystack/server/internal/shared/types" + "github.com/codetrek/haystack/server/internal/utils" ) // handleSearchContent handles the search content endpoint diff --git a/internal/server/httpapi/server.go b/packages/server/internal/server/httpapi/server.go similarity index 98% rename from internal/server/httpapi/server.go rename to packages/server/internal/server/httpapi/server.go index e4e79520..8ae0b845 100644 --- a/internal/server/httpapi/server.go +++ b/packages/server/internal/server/httpapi/server.go @@ -10,7 +10,7 @@ import ( "sync/atomic" "time" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/running" ) // StartServer initializes and starts the HTTP server diff --git a/internal/server/httpapi/server_cntl.go b/packages/server/internal/server/httpapi/server_cntl.go similarity index 93% rename from internal/server/httpapi/server_cntl.go rename to packages/server/internal/server/httpapi/server_cntl.go index 8fdd437f..ed527b9f 100644 --- a/internal/server/httpapi/server_cntl.go +++ b/packages/server/internal/server/httpapi/server_cntl.go @@ -6,9 +6,9 @@ import ( "net/http" "os" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" ) // handleHealth handles the health check endpoint diff --git a/internal/server/httpapi/workspace.go b/packages/server/internal/server/httpapi/workspace.go similarity index 97% rename from internal/server/httpapi/workspace.go rename to packages/server/internal/server/httpapi/workspace.go index c9106992..c398d364 100644 --- a/internal/server/httpapi/workspace.go +++ b/packages/server/internal/server/httpapi/workspace.go @@ -7,9 +7,9 @@ import ( "net/http" "path/filepath" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/shared/types" ) func handleCreateWorkspace(w http.ResponseWriter, r *http.Request) { diff --git a/internal/server/indexer/README.md b/packages/server/internal/server/indexer/README.md similarity index 100% rename from internal/server/indexer/README.md rename to packages/server/internal/server/indexer/README.md diff --git a/internal/server/indexer/indexer.go b/packages/server/internal/server/indexer/indexer.go similarity index 95% rename from internal/server/indexer/indexer.go rename to packages/server/internal/server/indexer/indexer.go index 1968bd0c..5cf60731 100644 --- a/internal/server/indexer/indexer.go +++ b/packages/server/internal/server/indexer/indexer.go @@ -8,10 +8,10 @@ import ( "sync" "github.com/codetrek/haystack/core/documents" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" ) var ( diff --git a/internal/server/indexer/indexer_test.go b/packages/server/internal/server/indexer/indexer_test.go similarity index 98% rename from internal/server/indexer/indexer_test.go rename to packages/server/internal/server/indexer/indexer_test.go index b9c8c3e2..b4d4ee19 100644 --- a/internal/server/indexer/indexer_test.go +++ b/packages/server/internal/server/indexer/indexer_test.go @@ -8,11 +8,11 @@ import ( "time" "github.com/codetrek/haystack/core/documents" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" ) // --------------------------------------------------------------------------- diff --git a/internal/server/indexer/indexer_utils_test.go b/packages/server/internal/server/indexer/indexer_utils_test.go similarity index 100% rename from internal/server/indexer/indexer_utils_test.go rename to packages/server/internal/server/indexer/indexer_utils_test.go diff --git a/internal/server/indexer/parser.go b/packages/server/internal/server/indexer/parser.go similarity index 97% rename from internal/server/indexer/parser.go rename to packages/server/internal/server/indexer/parser.go index 6bc55390..92f5195a 100644 --- a/internal/server/indexer/parser.go +++ b/packages/server/internal/server/indexer/parser.go @@ -9,8 +9,8 @@ import ( "time" "github.com/codetrek/haystack/core/documents" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/workspace" ) // ParseFile represents a file to be parsed diff --git a/internal/server/indexer/parser_test.go b/packages/server/internal/server/indexer/parser_test.go similarity index 98% rename from internal/server/indexer/parser_test.go rename to packages/server/internal/server/indexer/parser_test.go index edb06878..0a50f217 100644 --- a/internal/server/indexer/parser_test.go +++ b/packages/server/internal/server/indexer/parser_test.go @@ -11,11 +11,11 @@ import ( "github.com/codetrek/haystack/core/documents" "github.com/codetrek/haystack/core/idtable" "github.com/codetrek/haystack/core/invertedindex" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/testutil" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/testutil" ) // setupTestEnv initialises the subsystems required for parsing tests. diff --git a/internal/server/indexer/scanner.go b/packages/server/internal/server/indexer/scanner.go similarity index 94% rename from internal/server/indexer/scanner.go rename to packages/server/internal/server/indexer/scanner.go index 19203414..977e628c 100644 --- a/internal/server/indexer/scanner.go +++ b/packages/server/internal/server/indexer/scanner.go @@ -8,11 +8,11 @@ import ( "time" gitutils "github.com/codetrek/haystack/core/utils/git" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" - "github.com/codetrek/haystack/internal/utils" - fsutils "github.com/codetrek/haystack/internal/utils/fs" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" + "github.com/codetrek/haystack/server/internal/utils" + fsutils "github.com/codetrek/haystack/server/internal/utils/fs" ) // buildExcludeFilter returns a keep-filter: Match==true => keep, Match==false => exclude. diff --git a/internal/server/indexer/scanner_test.go b/packages/server/internal/server/indexer/scanner_test.go similarity index 98% rename from internal/server/indexer/scanner_test.go rename to packages/server/internal/server/indexer/scanner_test.go index d7a406fb..83c993d9 100644 --- a/internal/server/indexer/scanner_test.go +++ b/packages/server/internal/server/indexer/scanner_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/shared/types" ) // --------------------------------------------------------------------------- diff --git a/internal/server/indexer/symbol_parser.go b/packages/server/internal/server/indexer/symbol_parser.go similarity index 97% rename from internal/server/indexer/symbol_parser.go rename to packages/server/internal/server/indexer/symbol_parser.go index 33b27209..eb640768 100644 --- a/internal/server/indexer/symbol_parser.go +++ b/packages/server/internal/server/indexer/symbol_parser.go @@ -14,10 +14,10 @@ import ( "sync" "time" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/shared/running" ) const ( diff --git a/internal/server/indexer/symbol_parser_test.go b/packages/server/internal/server/indexer/symbol_parser_test.go similarity index 99% rename from internal/server/indexer/symbol_parser_test.go rename to packages/server/internal/server/indexer/symbol_parser_test.go index 808f1a71..8826bec8 100644 --- a/internal/server/indexer/symbol_parser_test.go +++ b/packages/server/internal/server/indexer/symbol_parser_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/workspace" ) func TestGetLangFromFilename_JavaScript(t *testing.T) { diff --git a/internal/server/indexer/testing.go b/packages/server/internal/server/indexer/testing.go similarity index 100% rename from internal/server/indexer/testing.go rename to packages/server/internal/server/indexer/testing.go diff --git a/internal/server/indexer/utils.go b/packages/server/internal/server/indexer/utils.go similarity index 100% rename from internal/server/indexer/utils.go rename to packages/server/internal/server/indexer/utils.go diff --git a/internal/server/indexer/utils_test.go b/packages/server/internal/server/indexer/utils_test.go similarity index 100% rename from internal/server/indexer/utils_test.go rename to packages/server/internal/server/indexer/utils_test.go diff --git a/internal/server/indexer/writer.go b/packages/server/internal/server/indexer/writer.go similarity index 97% rename from internal/server/indexer/writer.go rename to packages/server/internal/server/indexer/writer.go index 0f666c73..b55e401b 100644 --- a/internal/server/indexer/writer.go +++ b/packages/server/internal/server/indexer/writer.go @@ -6,7 +6,7 @@ import ( "time" "github.com/codetrek/haystack/core/documents" - "github.com/codetrek/haystack/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/core/workspace" ) type WriteDoc struct { diff --git a/internal/server/indexer/writer_test.go b/packages/server/internal/server/indexer/writer_test.go similarity index 99% rename from internal/server/indexer/writer_test.go rename to packages/server/internal/server/indexer/writer_test.go index e6a1ff4d..a30c3556 100644 --- a/internal/server/indexer/writer_test.go +++ b/packages/server/internal/server/indexer/writer_test.go @@ -6,7 +6,7 @@ import ( "time" "github.com/codetrek/haystack/core/documents" - "github.com/codetrek/haystack/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/core/workspace" ) // --------------------------------------------------------------------------- diff --git a/internal/server/log.go b/packages/server/internal/server/log.go similarity index 96% rename from internal/server/log.go rename to packages/server/internal/server/log.go index 19e1fc6a..59f09014 100644 --- a/internal/server/log.go +++ b/packages/server/internal/server/log.go @@ -5,7 +5,7 @@ import ( "os" "path/filepath" - "github.com/codetrek/haystack/internal/conf" + "github.com/codetrek/haystack/server/internal/conf" "gopkg.in/natefinch/lumberjack.v2" ) diff --git a/internal/server/mcptools/mcptools_test.go b/packages/server/internal/server/mcptools/mcptools_test.go similarity index 96% rename from internal/server/mcptools/mcptools_test.go rename to packages/server/internal/server/mcptools/mcptools_test.go index dbe708d4..cdad9c01 100644 --- a/internal/server/mcptools/mcptools_test.go +++ b/packages/server/internal/server/mcptools/mcptools_test.go @@ -14,13 +14,13 @@ import ( "github.com/codetrek/haystack/core/idtable" "github.com/codetrek/haystack/core/invertedindex" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/storage" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/server/searcher" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/storage" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/server/searcher" + "github.com/codetrek/haystack/server/internal/shared/running" "github.com/mark3labs/mcp-go/mcp" "github.com/stretchr/testify/assert" diff --git a/internal/server/mcptools/search_content.go b/packages/server/internal/server/mcptools/search_content.go similarity index 90% rename from internal/server/mcptools/search_content.go rename to packages/server/internal/server/mcptools/search_content.go index 079c4b1d..ce450e8d 100644 --- a/internal/server/mcptools/search_content.go +++ b/packages/server/internal/server/mcptools/search_content.go @@ -9,11 +9,11 @@ import ( "strings" "time" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/searcher" - "github.com/codetrek/haystack/internal/shared/types" - "github.com/codetrek/haystack/internal/utils" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/searcher" + "github.com/codetrek/haystack/server/internal/shared/types" + "github.com/codetrek/haystack/server/internal/utils" "github.com/mark3labs/mcp-go/mcp" ) diff --git a/internal/server/mcptools/search_files.go b/packages/server/internal/server/mcptools/search_files.go similarity index 86% rename from internal/server/mcptools/search_files.go rename to packages/server/internal/server/mcptools/search_files.go index ed260d79..8f8d65af 100644 --- a/internal/server/mcptools/search_files.go +++ b/packages/server/internal/server/mcptools/search_files.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/searcher" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/searcher" + "github.com/codetrek/haystack/server/internal/shared/types" "github.com/mark3labs/mcp-go/mcp" ) diff --git a/internal/server/mcptools/utils.go b/packages/server/internal/server/mcptools/utils.go similarity index 87% rename from internal/server/mcptools/utils.go rename to packages/server/internal/server/mcptools/utils.go index f994c197..3597d1cd 100644 --- a/internal/server/mcptools/utils.go +++ b/packages/server/internal/server/mcptools/utils.go @@ -4,8 +4,8 @@ import ( "fmt" "path/filepath" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/utils" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/utils" ) func parseAndValidateSearchArgs(arguments map[string]any) (string, string, int, error) { diff --git a/internal/server/run_error_test.go b/packages/server/internal/server/run_error_test.go similarity index 98% rename from internal/server/run_error_test.go rename to packages/server/internal/server/run_error_test.go index 88cc9028..42420f20 100644 --- a/internal/server/run_error_test.go +++ b/packages/server/internal/server/run_error_test.go @@ -11,7 +11,7 @@ import ( "github.com/codetrek/haystack/core/invertedindex" "github.com/codetrek/haystack/core/kv" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" + "github.com/codetrek/haystack/server/internal/conf" ) var errFake = errors.New("fake init error") diff --git a/internal/server/searcher/README.md b/packages/server/internal/server/searcher/README.md similarity index 100% rename from internal/server/searcher/README.md rename to packages/server/internal/server/searcher/README.md diff --git a/internal/server/searcher/search.md b/packages/server/internal/server/searcher/search.md similarity index 100% rename from internal/server/searcher/search.md rename to packages/server/internal/server/searcher/search.md diff --git a/internal/server/searcher/searcher.go b/packages/server/internal/server/searcher/searcher.go similarity index 97% rename from internal/server/searcher/searcher.go rename to packages/server/internal/server/searcher/searcher.go index 8f2dcfd2..e33379f3 100644 --- a/internal/server/searcher/searcher.go +++ b/packages/server/internal/server/searcher/searcher.go @@ -15,12 +15,12 @@ import ( "github.com/codetrek/haystack/core/engine" "github.com/codetrek/haystack/core/idtable" "github.com/codetrek/haystack/core/invertedindex" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" - "github.com/codetrek/haystack/internal/utils" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" + "github.com/codetrek/haystack/server/internal/utils" "github.com/lithammer/fuzzysearch/fuzzy" ) diff --git a/internal/server/searcher/searcher_additional_test.go b/packages/server/internal/server/searcher/searcher_additional_test.go similarity index 100% rename from internal/server/searcher/searcher_additional_test.go rename to packages/server/internal/server/searcher/searcher_additional_test.go diff --git a/internal/server/searcher/searcher_boost_test.go b/packages/server/internal/server/searcher/searcher_boost_test.go similarity index 99% rename from internal/server/searcher/searcher_boost_test.go rename to packages/server/internal/server/searcher/searcher_boost_test.go index 70220065..f2d0c14a 100644 --- a/internal/server/searcher/searcher_boost_test.go +++ b/packages/server/internal/server/searcher/searcher_boost_test.go @@ -7,7 +7,7 @@ import ( "github.com/codetrek/haystack/core/engine" "github.com/codetrek/haystack/core/invertedindex" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/shared/types" "github.com/stretchr/testify/assert" ) diff --git a/internal/server/searcher/searcher_cjk_test.go b/packages/server/internal/server/searcher/searcher_cjk_test.go similarity index 100% rename from internal/server/searcher/searcher_cjk_test.go rename to packages/server/internal/server/searcher/searcher_cjk_test.go diff --git a/internal/server/searcher/searcher_coverage_test.go b/packages/server/internal/server/searcher/searcher_coverage_test.go similarity index 99% rename from internal/server/searcher/searcher_coverage_test.go rename to packages/server/internal/server/searcher/searcher_coverage_test.go index 44528eb1..b19c0954 100644 --- a/internal/server/searcher/searcher_coverage_test.go +++ b/packages/server/internal/server/searcher/searcher_coverage_test.go @@ -17,13 +17,13 @@ import ( "github.com/codetrek/haystack/core/engine" "github.com/codetrek/haystack/core/idtable" "github.com/codetrek/haystack/core/invertedindex" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" - "github.com/codetrek/haystack/internal/testutil" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" + "github.com/codetrek/haystack/server/internal/testutil" "github.com/stretchr/testify/assert" ) diff --git a/internal/server/searcher/symbols_searcher.go b/packages/server/internal/server/searcher/symbols_searcher.go similarity index 95% rename from internal/server/searcher/symbols_searcher.go rename to packages/server/internal/server/searcher/symbols_searcher.go index f60b6bdb..5e3fef66 100644 --- a/internal/server/searcher/symbols_searcher.go +++ b/packages/server/internal/server/searcher/symbols_searcher.go @@ -10,10 +10,10 @@ import ( "github.com/codetrek/haystack/core/documents" "github.com/codetrek/haystack/core/idtable" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/shared/types" "github.com/AntoineAugusti/wordsegmentation" "github.com/AntoineAugusti/wordsegmentation/corpus" diff --git a/internal/server/server.go b/packages/server/internal/server/server.go similarity index 91% rename from internal/server/server.go rename to packages/server/internal/server/server.go index 8099a8b4..833c56a3 100644 --- a/internal/server/server.go +++ b/packages/server/internal/server/server.go @@ -13,14 +13,14 @@ import ( "github.com/codetrek/haystack/core/invertedindex" "github.com/codetrek/haystack/core/kv" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/storage" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/httpapi" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/server/searcher" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/storage" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/httpapi" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/server/searcher" + "github.com/codetrek/haystack/server/internal/shared/running" ) // newInvertedIndex is the constructor seam; tests override it to capture the diff --git a/internal/server/server_maxpending_test.go b/packages/server/internal/server/server_maxpending_test.go similarity index 100% rename from internal/server/server_maxpending_test.go rename to packages/server/internal/server/server_maxpending_test.go diff --git a/internal/server/server_test.go b/packages/server/internal/server/server_test.go similarity index 98% rename from internal/server/server_test.go rename to packages/server/internal/server/server_test.go index 84253aec..be32eb14 100644 --- a/internal/server/server_test.go +++ b/packages/server/internal/server/server_test.go @@ -20,15 +20,15 @@ import ( "github.com/codetrek/haystack/core/idtable" "github.com/codetrek/haystack/core/invertedindex" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/storage" - "github.com/codetrek/haystack/internal/core/symbols" - "github.com/codetrek/haystack/internal/core/workspace" - "github.com/codetrek/haystack/internal/server/httpapi" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/server/searcher" - "github.com/codetrek/haystack/internal/shared/running" - "github.com/codetrek/haystack/internal/shared/types" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/storage" + "github.com/codetrek/haystack/server/internal/core/symbols" + "github.com/codetrek/haystack/server/internal/core/workspace" + "github.com/codetrek/haystack/server/internal/server/httpapi" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/server/searcher" + "github.com/codetrek/haystack/server/internal/shared/running" + "github.com/codetrek/haystack/server/internal/shared/types" ) const ( diff --git a/internal/server/zz_server_run_test.go b/packages/server/internal/server/zz_server_run_test.go similarity index 93% rename from internal/server/zz_server_run_test.go rename to packages/server/internal/server/zz_server_run_test.go index 6fbeebd9..b6234143 100644 --- a/internal/server/zz_server_run_test.go +++ b/packages/server/internal/server/zz_server_run_test.go @@ -11,9 +11,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/server/indexer" - "github.com/codetrek/haystack/internal/shared/running" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/server/indexer" + "github.com/codetrek/haystack/server/internal/shared/running" ) // TestZZ_Run exercises the full Run() happy path. This file is named diff --git a/internal/shared/running/running_extra_test.go b/packages/server/internal/shared/running/running_extra_test.go similarity index 100% rename from internal/shared/running/running_extra_test.go rename to packages/server/internal/shared/running/running_extra_test.go diff --git a/internal/shared/running/running_test.go b/packages/server/internal/shared/running/running_test.go similarity index 100% rename from internal/shared/running/running_test.go rename to packages/server/internal/shared/running/running_test.go diff --git a/internal/shared/running/runtime.go b/packages/server/internal/shared/running/runtime.go similarity index 97% rename from internal/shared/running/runtime.go rename to packages/server/internal/shared/running/runtime.go index f235db06..43b9dd74 100644 --- a/internal/shared/running/runtime.go +++ b/packages/server/internal/shared/running/runtime.go @@ -7,7 +7,7 @@ import ( "path/filepath" "sync" - "github.com/codetrek/haystack/internal/utils" + "github.com/codetrek/haystack/server/internal/utils" ) var ( diff --git a/internal/shared/running/server.go b/packages/server/internal/shared/running/server.go similarity index 100% rename from internal/shared/running/server.go rename to packages/server/internal/shared/running/server.go diff --git a/internal/shared/running/shutdown.go b/packages/server/internal/shared/running/shutdown.go similarity index 100% rename from internal/shared/running/shutdown.go rename to packages/server/internal/shared/running/shutdown.go diff --git a/internal/shared/types/common.go b/packages/server/internal/shared/types/common.go similarity index 100% rename from internal/shared/types/common.go rename to packages/server/internal/shared/types/common.go diff --git a/internal/shared/types/document.go b/packages/server/internal/shared/types/document.go similarity index 100% rename from internal/shared/types/document.go rename to packages/server/internal/shared/types/document.go diff --git a/internal/shared/types/search.go b/packages/server/internal/shared/types/search.go similarity index 100% rename from internal/shared/types/search.go rename to packages/server/internal/shared/types/search.go diff --git a/internal/shared/types/workspace.go b/packages/server/internal/shared/types/workspace.go similarity index 100% rename from internal/shared/types/workspace.go rename to packages/server/internal/shared/types/workspace.go diff --git a/internal/testutil/testutil.go b/packages/server/internal/testutil/testutil.go similarity index 94% rename from internal/testutil/testutil.go rename to packages/server/internal/testutil/testutil.go index 51c090a4..7262afc4 100644 --- a/internal/testutil/testutil.go +++ b/packages/server/internal/testutil/testutil.go @@ -13,8 +13,8 @@ import ( "github.com/codetrek/haystack/core/kv" "github.com/codetrek/haystack/core/queue" - "github.com/codetrek/haystack/internal/conf" - "github.com/codetrek/haystack/internal/core/storage" + "github.com/codetrek/haystack/server/internal/conf" + "github.com/codetrek/haystack/server/internal/core/storage" ) // Env holds the resources created by SetupEnv so that callers can access diff --git a/internal/utils/fs/file.go b/packages/server/internal/utils/fs/file.go similarity index 100% rename from internal/utils/fs/file.go rename to packages/server/internal/utils/fs/file.go diff --git a/internal/utils/fs/file_test.go b/packages/server/internal/utils/fs/file_test.go similarity index 100% rename from internal/utils/fs/file_test.go rename to packages/server/internal/utils/fs/file_test.go diff --git a/internal/utils/fs/fs_test.go b/packages/server/internal/utils/fs/fs_test.go similarity index 100% rename from internal/utils/fs/fs_test.go rename to packages/server/internal/utils/fs/fs_test.go diff --git a/internal/utils/fs/list_files.go b/packages/server/internal/utils/fs/list_files.go similarity index 100% rename from internal/utils/fs/list_files.go rename to packages/server/internal/utils/fs/list_files.go diff --git a/internal/utils/fs/list_files_test.go b/packages/server/internal/utils/fs/list_files_test.go similarity index 100% rename from internal/utils/fs/list_files_test.go rename to packages/server/internal/utils/fs/list_files_test.go diff --git a/internal/utils/fs/read_file.go b/packages/server/internal/utils/fs/read_file.go similarity index 100% rename from internal/utils/fs/read_file.go rename to packages/server/internal/utils/fs/read_file.go diff --git a/internal/utils/md5.go b/packages/server/internal/utils/md5.go similarity index 100% rename from internal/utils/md5.go rename to packages/server/internal/utils/md5.go diff --git a/internal/utils/md5_test.go b/packages/server/internal/utils/md5_test.go similarity index 100% rename from internal/utils/md5_test.go rename to packages/server/internal/utils/md5_test.go diff --git a/internal/utils/path.go b/packages/server/internal/utils/path.go similarity index 100% rename from internal/utils/path.go rename to packages/server/internal/utils/path.go diff --git a/internal/utils/path_test.go b/packages/server/internal/utils/path_test.go similarity index 100% rename from internal/utils/path_test.go rename to packages/server/internal/utils/path_test.go diff --git a/internal/utils/simple_filter.go b/packages/server/internal/utils/simple_filter.go similarity index 100% rename from internal/utils/simple_filter.go rename to packages/server/internal/utils/simple_filter.go diff --git a/internal/utils/simple_filter_extended_test.go b/packages/server/internal/utils/simple_filter_extended_test.go similarity index 100% rename from internal/utils/simple_filter_extended_test.go rename to packages/server/internal/utils/simple_filter_extended_test.go diff --git a/internal/utils/simple_filter_integration_test.go b/packages/server/internal/utils/simple_filter_integration_test.go similarity index 100% rename from internal/utils/simple_filter_integration_test.go rename to packages/server/internal/utils/simple_filter_integration_test.go diff --git a/internal/utils/simple_filter_test.go b/packages/server/internal/utils/simple_filter_test.go similarity index 100% rename from internal/utils/simple_filter_test.go rename to packages/server/internal/utils/simple_filter_test.go diff --git a/internal/utils/utils_test.go b/packages/server/internal/utils/utils_test.go similarity index 100% rename from internal/utils/utils_test.go rename to packages/server/internal/utils/utils_test.go diff --git a/scripts/coverage.cmd b/scripts/coverage.cmd index eeaacaaf..30ab1959 100644 --- a/scripts/coverage.cmd +++ b/scripts/coverage.cmd @@ -1,7 +1,7 @@ @echo off setlocal -cd /d "%~dp0.." +cd /d "%~dp0../packages/server" go run github.com/codetreker/go-cov/cmd/go-cov@v0.1.2 %* set EXITCODE=%ERRORLEVEL% diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 0ef9c6f1..3261e59c 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -4,5 +4,5 @@ set -e -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/../packages/server" go run github.com/codetreker/go-cov/cmd/go-cov@v0.1.2 "$@" diff --git a/vendor/modules.txt b/vendor/modules.txt index 36d9dd97..9541cdd5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -19,7 +19,7 @@ github.com/bits-and-blooms/bitset # github.com/blevesearch/mmap-go v1.0.4 ## explicit; go 1.13 github.com/blevesearch/mmap-go -# github.com/blevesearch/vellum v1.0.10 => ./core/third_party/vellum +# github.com/blevesearch/vellum v1.0.10 => ./packages/core/third_party/vellum ## explicit; go 1.18 github.com/blevesearch/vellum # github.com/cespare/xxhash/v2 v2.3.0 @@ -297,4 +297,4 @@ gopkg.in/natefinch/lumberjack.v2 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 -# github.com/blevesearch/vellum => ./core/third_party/vellum +# github.com/blevesearch/vellum => ./packages/core/third_party/vellum