refactor: reorganize the repo into packages/ (core + server)

.gitattributes

@@ -1,4 +1,4 @@
-deps/** filter=lfs diff=lfs merge=lfs -text
+deps/** filter=lfs diff=lfs merge=lfs -text
 
 # Embedded prebuilt FST dictionary (go:embed build asset)
-core/tokenizer/fstcjk/dict.fst filter=lfs diff=lfs merge=lfs -text
+packages/core/tokenizer/fstcjk/dict.fst filter=lfs diff=lfs merge=lfs -text

.github/copilot-instructions.md

@@ -43,14 +43,4 @@ Haystack is a local code search indexer tool designed to create and query search
 - Pebble DB (storage)
 - Git integration
 
-### Project Documentation Structure
-The project contains multiple README.md files throughout the directory structure, providing specific documentation for different components:
-- Root README.md: Overall project description, installation and setup
-- internal/server/README.md: Server component implementation details
-- internal/core/invertedindex/README.md: Documentation for the storage subsystem and Pebble DB usage
-- internal/server/indexer/README.md: Explanation of the indexing mechanism
-- internal/server/searcher/README.md: Details on search functionality and query syntax
-
-These README.md files serve as component-specific documentation and should be consulted when working on the respective parts of the codebase.
-
 This context should help you provide more relevant and aligned suggestions when working with the Haystack codebase.

.github/workflows/ci.yml

@@ -43,20 +43,20 @@ jobs:
         run: go version
 
       - name: Build
-        working-directory: core
+        working-directory: packages/core
         run: go build ./...
 
       - name: Coverage gate (Linux)
         # go-cov runs core's full suite and enforces the same bar as the
         # core module.
         if: matrix.os == 'ubuntu-latest'
-        working-directory: core
+        working-directory: packages/core
         run: go run github.com/codetreker/go-cov/cmd/go-cov@v0.1.2
 
       - name: Test (macOS/Windows)
         if: matrix.os != 'ubuntu-latest'
         shell: bash
-        working-directory: core
+        working-directory: packages/core
         run: |
           # -v prints each case's execution time. Show only result lines, and
           # drop the 0.00s cases (noise) — keep all FAILs and the full log on
@@ -110,7 +110,7 @@ jobs:
         run: ./.github/workflows/scripts/check-fmt.sh
 
       - name: Build
-        run: go build ./...
+        run: go build ./packages/server/... ./packages/core/...
 
       - name: Test and Coverage
         run: ./.github/workflows/scripts/test_and_coverage.sh

.github/workflows/fidelity.yml

@@ -9,16 +9,16 @@ name: FST Fidelity
 on:
   pull_request:
     paths:
-      - 'core/tokenizer/fstcjk/**'
-      - 'core/tokenizer/cjk_fidelity_test.go'
-      - 'core/go.mod'
+      - 'packages/core/tokenizer/fstcjk/**'
+      - 'packages/core/tokenizer/cjk_fidelity_test.go'
+      - 'packages/core/go.mod'
       - '.github/workflows/fidelity.yml'
   push:
     branches: [main]
     paths:
-      - 'core/tokenizer/fstcjk/**'
-      - 'core/tokenizer/cjk_fidelity_test.go'
-      - 'core/go.mod'
+      - 'packages/core/tokenizer/fstcjk/**'
+      - 'packages/core/tokenizer/cjk_fidelity_test.go'
+      - 'packages/core/go.mod'
 
 jobs:
   fidelity:
@@ -34,11 +34,11 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.24.2' # keep in sync with core/go.mod toolchain
+          go-version: '1.24.2' # keep in sync with packages/core/go.mod toolchain
           cache: true
 
       - name: Exhaustive FST-vs-gse byte-fidelity
-        working-directory: core
+        working-directory: packages/core
         env:
           HAYSTACK_FIDELITY: '1' # enables the live-gse golden suite
           HAYSTACK_FIDELITY_FULL: '1' # full fuzz counts (~60k inputs)

.github/workflows/release.yml

@@ -22,12 +22,12 @@ jobs:
           git lfs pull
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: '1.23'
+          go-version: '1.24.2' # keep in sync with the go.work / packages/*/go.mod toolchain
 
       - name: Run tests
-        run: go test ./...
+        run: go test ./packages/server/... ./packages/core/...
 
       - name: Run build.go
         run: go run build.go

.github/workflows/scripts/test_and_coverage.sh

@@ -4,4 +4,6 @@
 
 set -e
 
+# The App CI job gates the SERVER module (core has its own job).
+cd "$(dirname "$0")/../../../packages/server"
 go run github.com/codetreker/go-cov/cmd/go-cov@v0.1.2

.gitignore

@@ -17,12 +17,10 @@ node_modules/
 /haystack
 
 # Generated test fixtures (large binary files)
-core/vectorindex/testdata/*.bin
 *.out
 cover.html
 scripts/lib/coverage/coverage
-core/vectorindex/testdata/sift/
-/cmd/bench50k*
+packages/server/cmd/bench50k*
 .tmp/
 tmp/
 .bf/

.vscode/launch.json

@@ -9,7 +9,7 @@
       "type": "go",
       "request": "launch",
       "mode": "auto",
-      "program": "${workspaceFolder}/cmd/haystack/",
+      "program": "${workspaceFolder}/packages/server/cmd/haystack/",
       "env": {},
       "args": [
         "server",

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build test coverage fmt clean gen-testdata test-docker-build test-docker-ensure test-safe test-safe-race
+.PHONY: build test coverage fmt clean test-docker-build test-docker-ensure test-safe test-safe-race
 
 APP_NAME=haystack
 BUILD_DIR=build
@@ -20,12 +20,12 @@ APP_BIN=$(BUILD_DIR)/$(APP_NAME)$(EXE_EXT)
 build:
 	@$(MKDIR_P)
 	@echo "Building $(APP_NAME)..."
-	@go build -o $(APP_BIN) ./cmd/haystack/
+	@go build -o $(APP_BIN) ./packages/server/cmd/haystack/
 	@echo "Build complete: $(APP_BIN)"
 
 test:
 	@echo "Running tests..."
-	@go test ./... -count=1
+	@go test ./packages/core/... ./packages/server/... -count=1
 
 coverage:
 	@echo "Running tests with coverage..."
@@ -40,14 +40,10 @@ clean:
 	@echo "Cleaning..."
 	@$(RM_RF)
 
-gen-testdata:
-	@echo "Generating test fixtures (this takes ~15 min)..."
-	@cd core && go run -tags tools ./cmd/gen-testdata/
-
 DOCKER_TEST_IMAGE=haystack-test
 
 test-docker-build:
-	@docker build -f Dockerfile.test -t $(DOCKER_TEST_IMAGE) .
+	@docker build -f ./packages/server/Dockerfile.test -t $(DOCKER_TEST_IMAGE) .
 
 test-docker-ensure:
 	@if [ -z "$$(docker images -q $(DOCKER_TEST_IMAGE) 2>/dev/null)" ]; then \
@@ -67,4 +63,4 @@ test-safe-race: test-docker-ensure
 	@echo "Running tests with race detector in Docker (isolated)..."
 	@docker run --rm --cpus=2 --memory=4g --pids-limit=256 --network=none \
 		-v $$(pwd):/app:ro \
-		$(DOCKER_TEST_IMAGE) bash -c "ulimit -u 256 && go test -race ./... -count=1 -timeout 5m"
+		$(DOCKER_TEST_IMAGE) bash -c "ulimit -u 256 && go test -race ./packages/server/... ./packages/core/... -count=1 -timeout 5m"

README.md

@@ -31,7 +31,7 @@ The Search Indexer is structured with the following components:
 
 ### Prerequisites
 
-- Go 1.23+
+- Go 1.24+
 - Git
 
 ### Installation
@@ -40,36 +40,35 @@ Clone the repository:
 
 ```bash
 git clone https://github.com/codetrek/haystack.git
-cd haystack/src
+cd haystack
 ```
 
-Install dependencies:
-
-```bash
-go mod download
-```
+The repository is a Go **workspace** (`go.work`) spanning two modules —
+`packages/core` (the search/index library) and `packages/server` (the server + CLI
+app). Dependencies are vendored under `vendor/`, so there is no separate
+dependency-install step.
 
 ### Configuration
 
 Copy the example configuration and modify as needed:
 
 ```bash
-cp config.example.yaml config.local.yaml
+cp packages/server/config.example.yaml config.local.yaml
 ```
 
 Edit `config.local.yaml` to configure your indexing preferences.
 
 ### Running the Server
 
 ```bash
-go run ./ server run
+go run ./packages/server/cmd/haystack server run
 ```
 
 ### Using the Client
 
 ```bash
 # Example query command
-go run ./ search "your search query"
+go run ./packages/server/cmd/haystack search "your search query"
 ```
 
 ## Development
@@ -79,13 +78,13 @@ go run ./ search "your search query"
 Run the test suite with:
 
 ```bash
-go test ./...
+make test            # or: go test ./packages/core/... ./packages/server/...
 ```
 
 ### Building
 
 ```bash
-go build ./
+make build           # or: go build -o build/haystack ./packages/server/cmd/haystack/
 ```
 
 ## Contributing

build.go

@@ -50,7 +50,7 @@ func main() {
 			"-ldflags", ldflags,
 			"-gcflags=all=-l",
 			"-o", binPath,
-			"./cmd/haystack/",
+			"./packages/server/cmd/haystack/",
 		}
 
 		cmd := exec.Command("go", args...)