diff --git a/.github/workflows/kb-gate-test.yml b/.github/workflows/kb-gate-test.yml index 2ba201b..d365e8d 100644 --- a/.github/workflows/kb-gate-test.yml +++ b/.github/workflows/kb-gate-test.yml @@ -23,7 +23,7 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 # Twice, deliberately: the barrier is scoped by a monotonic turn counter # rather than by wall-clock time, and a regression to timestamps would show diff --git a/.github/workflows/sync-blocks.yml b/.github/workflows/sync-blocks.yml index ea9befe..077465b 100644 --- a/.github/workflows/sync-blocks.yml +++ b/.github/workflows/sync-blocks.yml @@ -25,7 +25,7 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Verify SYNC blocks exist and agree across both skills + SYNC-BLOCKS.md run: | diff --git a/.github/workflows/sync-memories-test.yml b/.github/workflows/sync-memories-test.yml new file mode 100644 index 0000000..fbf35da --- /dev/null +++ b/.github/workflows/sync-memories-test.yml @@ -0,0 +1,30 @@ +name: sync-memories-test + +on: + pull_request: + paths: + - "hooks/sync-memories.sh" + - "tests/sync-memories-test.sh" + - ".github/workflows/sync-memories-test.yml" + push: + branches: [main] + paths: + - "hooks/sync-memories.sh" + - "tests/sync-memories-test.sh" + +permissions: + contents: read + +concurrency: + group: sync-memories-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + # qmd is stubbed, so no model or index is needed. + - name: Run the sync-memories suite + run: bash tests/sync-memories-test.sh diff --git a/.gitignore b/.gitignore index 0fbf0b2..afeab08 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ # Thumbnails ._* +.maestri + # Custom folder icons Icon @@ -15,4 +17,4 @@ Icon .TemporaryItems .Trashes .VolumeIcon.icns -.com.apple.timemachine.donotpresent \ No newline at end of file +.com.apple.timemachine.donotpresent diff --git a/CLAUDE.md b/CLAUDE.md index a768b46..9921176 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,9 +12,10 @@ The consequence that matters most: **nothing here executes from the repo.** Edit | Task | Command | |---|---| -| Run the hook test suite | `bash tests/kb-gate-test.sh` | +| Run the KB gate suite | `bash tests/kb-gate-test.sh` | +| Run the indexer suite | `bash tests/sync-memories-test.sh` (stubs qmd; no model needed) | | Verify the SYNC blocks agree | the snippet below (full version at the bottom of `SYNC-BLOCKS.md`) | -| Check the manifest parses | `python3 -c "import yaml;yaml.safe_load(open('techpack.yaml'))"` | +| Check the manifest | `mcs pack validate` — validates structure and component references, not just YAML syntax. A raw parse needs a Python that has PyYAML, which is not guaranteed to be `/usr/bin/python3` | | Install a change locally | `mcs sync --global`, or `mcs sync` inside a project | | Check installed health | `mcs doctor` | @@ -38,22 +39,38 @@ Two harness details are load-bearing rather than incidental: - It runs from a temp dir **outside any git repo**, because the hook resolves its project root with `git rev-parse --show-toplevel` first. Run from the checkout, the harness would write state into the working tree and read your real session files. - The fixture project must contain `.claude/memories/`, or every `PreToolUse` call takes the `no_memories_dir` skip and nothing is gated. The denial count asserted at the end is what turns that into a loud failure instead of a green run that asserted nothing. +`tests/sync-memories-test.sh` shares the outside-a-git-repo rule and stubs `qmd` on `PATH`, so it needs no model. Its own trap: a first run reindexes under any version of the hook, because the config file does not exist yet. The assertions that mean anything are the **second** runs, and they only discriminate because the stub creates `$INDEX_PATH` — a staleness gate guarded on that file falls through without it. + ## Invariants that span files **Placeholders are baked at sync time, not read at runtime.** `prompts:` in `techpack.yaml` declares `KB_GATE_MODE`; `hooks/kb-gate.sh` carries `MODE="__KB_GATE_MODE__"`, substituted during install. Changing the mode means re-running `mcs sync` — there is no runtime setting. The test suite injects modes the same way (`sed s/__KB_GATE_MODE__/$m/`). **One dispatcher, four hook events.** `hooks/kb-gate.sh` is registered four times in `techpack.yaml` (UserPromptSubmit, PostToolUse, PreToolUse, SubagentStart) and branches on `hook_event_name`. Matchers are broad on purpose; which agent types count as "discovery" is decided in exactly one place, `GATED_AGENTS`. `hooks/sync-memories.sh` is likewise registered twice, on SessionStart and UserPromptSubmit. -**That dispatcher deliberately omits `set -e` and `set -u`**, unlike `sync-memories.sh` which uses `set -uo pipefail`. Its file header explains why and lists rules that are load-bearing: fail open, never `exit 2`, never call `docs-mcp-server` (too slow for `PreToolUse`), log every evaluation. Read that header before editing it. +**That dispatcher deliberately omits `set -e` and `set -u`**, unlike `sync-memories.sh` which uses `set -uo pipefail`. Its file header explains why and lists rules that are load-bearing: fail open, never `exit 2`, never call `qmd` (it loads an embedding model; far too slow for `PreToolUse`), log every evaluation. Read that header before editing it. + +**Project-root derivation must be byte-identical everywhere it appears.** Both hooks and every shell string in `techpack.yaml` — the `memory-loop` launcher and each of its doctor checks — resolve git toplevel → `CLAUDE_PROJECT_DIR` → `$PWD`. The hooks must agree on which project they are looking at; the manifest must additionally agree with `sync-memories.sh` on `.claude/.kb-index/`, because one writes the index the others read. `commandExists` runs every check in its own process with no way to share a value, so the duplication is forced; keeping the copies textually identical is the only thing that makes a future edit obviously propagatable. Do not abbreviate one of them, and do not "simplify" the ladder to `$PWD` — a shortened copy that dropped the `CLAUDE_PROJECT_DIR` rung shipped once already, and made those checks silently pass without checking anything in non-git projects. + +**The indexing hook and a doctor check are coupled through a file.** `sync-memories.sh` writes `.claude/.kb-index/memory-loop.log` when a run fails or leaves documents unembedded, and deletes it on success; the "Memory indexing completed" check reports the file's existence. Move or rename it on one side and the check passes forever without testing anything. + +**The index is reached by `--index`, never by a project-local `.qmd/`.** Two reasons, and the second is the dangerous one. A user may keep their own `.qmd/` at the project root for their own code, which this pack must not touch. And a project-local `.qmd/index.yml` falls under qmd's trust gate, which covers a non-default `models.embed` — for a non-interactive caller the gate does not prompt or fail, it *skips*, silently substituting a much weaker default model. Named indexes are never gated. `QMD_CONFIG_DIR` and `INDEX_PATH` are what move a named index back under the project directory. + +**Reranking and query expansion are disabled by pointing their model slots at the embedding model.** The MCP `query` tool hard-defaults `rerank: true` with no server-side way to turn it off, and a *missing* model is downloaded mid-query with no progress output. An embedding model has no ranking head, so qmd fails to build a ranking context, warns, and falls back to RRF — measured at MRR 0.792 against 0.800 for an explicit `rerank: false`, and it buys back zero R@5 versus a real reranker. Those numbers come from a 20-query fixture over this project's own memories, kept outside the repo — nothing here reproduces them, so treat them as recorded measurements rather than something CI re-checks. The fallback is not a *strict* no-op: on a later fixture, a document RRF ranked 9th was dropped entirely when `rerank: true` was passed. Top ranks were unaffected, which is why this is a safe structural disable, but do not describe it as "reranking is off and nothing changes". + +To measure any of this, `qmd bench -c memories` is usable as shipped. Its fixture `query` field accepts the structured multi-line form (`intent:`/`lex:`/`vec:`), and a fixture written that way is passed through **unexpanded** — only a bare query string goes down the expansion path. So its `hybrid` row measures the pack's real configuration, not a degraded one. This depends on qmd's graceful-degradation path rather than a documented switch, which is why `@tobilu/qmd` is pinned to an exact version and why one doctor check issues a *default-argument* query: that check is what would catch the behaviour changing under an upgrade. + +**The search call shape is stated in three places, deliberately.** "Typed `lex`+`vec` lines, `rerank: false`, `limit: 6`" appears in `templates/continuous-learning.md` (the only thing that reaches a user's `CLAUDE.md`), `skills/continuous-learning/SKILL.md`, and the `SubagentStart` briefing in `hooks/kb-gate.sh`. No single mechanism reaches all three consumers, so this is three copies rather than one source — change one and check the other two. It matters because the unguided path is measurably worse, not just slower. + +**The index's `global_context` is deliberately not a fourth copy.** qmd serves that one string two ways: as the MCP server's `instructions`, once per connection, and as the `context` field of *every* search result. Guidance placed there is therefore paid for per result — at 342 characters it was 38% of a six-result response — while the only consumer it uniquely reaches is a client with no installed `CLAUDE.md` section, which cannot happen because the template is `isRequired`. So it carries identity only ("this is a project memory KB, not external documentation") and the guidance lives in the three copies that are not echoed. Resist putting the call shape back into it. -**Project-root and library derivation must match across two hooks.** `sync-memories.sh` and `resolve_paths()` in `kb-gate.sh` both resolve git toplevel → `CLAUDE_PROJECT_DIR` → `$PWD`, and derive the library name as the root directory's basename. kb-gate quotes that name back to Claude, and it has to be the one sync-memories indexed. Both files carry "keep in sync" comments. +**The same three copies carry "retrieve before relying on a result", and the reason is only recorded here.** qmd's MCP snippet is at most five lines and 300 characters (`extractSnippet` in `store.js`, called with a hardcoded `300` from `mcp/server.js`), and it is anchored by literal substring matching of the first `lex` sub-query. When those terms are not in the matched text it falls back to the top of the chunk — in practice the file's first three lines, which for a memory is its title and `Applies to:`. Measured on a 527-document corpus, 28% of results came back title-only. So a search result is a lead, and answering from it is guessing; `get`/`multi_get` is the step that makes the answer real. The instruction is phrased as an absolute in all three copies on purpose — stating the failure condition invites the reader to decide a snippet looks complete this time. Do not "simplify" it back into step 2's old wording (`Read matching memories`), which worked only because the previous backend returned a whole chunk. **Three text blocks must stay byte-identical across three files.** `capture-rules`, `strip-the-anchors`, and `applies-to` appear in both `SKILL.md`s and in `SYNC-BLOCKS.md`, enforced by `.github/workflows/sync-blocks.yml`. Two rules when touching them: - Blocks are verdict-neutral. Each skill adds its own verb *outside* the fence — capture says "skip", audit says "DROP". Never move an action verb inside the locked block. - Never write a real tag name in prose. The drift check's `awk` range grabs the first matching opener, so a literal mention would shadow the canonical block and make it invisible to the verifier. `SYNC-BLOCKS.md` uses a placeholder form for exactly this reason. -**Templates are installed as marked sections inside someone's `CLAUDE.md`, not as files.** The `templates:` block in `techpack.yaml` maps `templates/continuous-learning.md` to a section fenced by ``. On a global sync it lands in `~/.claude/CLAUDE.md`; on a project sync, in that project's `CLAUDE.local.md`. Edit the template here and re-sync, because editing inside the markers drifts and is overwritten. `__PROJECT_DIR_NAME__` is substituted at install time, like the hook's mode. +**Templates are installed as marked sections inside someone's `CLAUDE.md`, not as files.** The `templates:` block in `techpack.yaml` maps `templates/continuous-learning.md` to a section fenced by ``. On a global sync it lands in `~/.claude/CLAUDE.md`; on a project sync, in that project's `CLAUDE.local.md`. Edit the template here and re-sync, because editing inside the markers drifts and is overwritten. The template has no placeholders of its own; only `hooks/kb-gate.sh` carries one, `KB_GATE_MODE`. **Installed artifacts are content-hash verified.** `mcs doctor` compares hashes of installed files, so hand-editing an installed copy registers as drift and the next `mcs sync` restores the packaged version. This is why a skill can never write to its own files: anything saved that way is destroyed on the next sync. diff --git a/README.md b/README.md index ea1f929..2a35556 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ mcs sync --global # 3. install globally (~/.claude) mcs doctor # 4. verify everything is healthy ``` -**Prerequisites:** macOS, [Claude Code](https://docs.anthropic.com/en/docs/claude-code), and [Ollama](https://ollama.com) (local embeddings runtime). `mcs` installs the rest (Node, `gh`, `jq`) automatically. +**Prerequisites:** macOS, [Claude Code](https://docs.anthropic.com/en/docs/claude-code), and Node 22 or newer. `mcs` installs the rest (Node, `gh`, `jq`, [qmd](https://github.com/tobi/qmd)) automatically, and downloads a ~610 MB embedding model once on first sync — shared by every project, with no daemon left running afterwards. Global is the recommended scope — this pack has no per-project config, so installing once makes memory available in every project automatically. To scope it to a single repo instead, run `mcs sync` from inside that repo. @@ -32,11 +32,11 @@ flowchart LR B --> C[Work session] C --> D[Capture learnings & decisions] D --> E[(.claude/memories/)] - E --> F["Ollama embeddings
semantic index"] + E --> F["local embeddings
semantic index"] F -. re-index on session start / change .-> B ``` -1. **Session starts** — a hook re-indexes `.claude/memories/` into a local vector store (Ollama `nomic-embed-text`), in the background. +1. **Session starts** — a hook re-indexes `.claude/memories/` into a local vector store in the background. Embeddings are computed in-process by `qmd`, so there is no service to start and nothing left running between sessions. 2. **Before any task** — Claude is instructed to search the knowledge base first, surfacing relevant past learnings and decisions. 3. **Before delegating** — sub-agents can't see the parent's KB results, so they'd rediscover everything from scratch. A gate hook closes that gap from both ends: it requires the findings to be pasted into the sub-agent's prompt, and tells any discovery agent to search the KB itself if they weren't. Configurable per project, from a reminder up to a hard block. 4. **During work** — a prompt-submit hook reminds Claude to notice when the current interaction produces knowledge worth saving. @@ -66,7 +66,7 @@ To change the answer later, re-run `mcs sync` — the mode is baked into the ins | Component | What it does | |---|---| -| **docs-mcp-server** (MCP) | Read-only semantic search over `.claude/memories/`, backed by local Ollama embeddings | +| **memory-loop** (MCP) | Semantic search over `.claude/memories/`, backed by a local embedding model | | **continuous-learning** (skill) | Extracts learnings and decisions from a session into structured memory files | | **memory-audit** (skill) | Reviews existing memories and flags stale or duplicate entries to keep the KB lean | | **sync-memories.sh** (hook) | Indexes/re-indexes memories on session start and when they change mid-session | @@ -81,6 +81,36 @@ Memories come in two flavors, both stored as version-controlled, human-readable --- +## Upgrading from the Ollama version + +Earlier releases indexed memories through `docs-mcp-server` backed by an Ollama daemon. `mcs sync` +converges on its own: the old MCP server is deregistered for you, and the new index is built on the +next session start. Nothing below is required. + +What `mcs` cannot clean up is what the old release installed through plain shell commands. If you +want the disk space back, and **only if nothing else on your machine uses these**: + +```bash +npm uninstall -g @arabold/docs-mcp-server +rm -rf ~/Library/Application\ Support/docs-mcp-server # see the warning below +ollama rm nomic-embed-text +``` + +Three things to check before running any of them: + +- **`docs-mcp-server` indexes external documentation too.** If you ever pointed it at a library's + docs, that is what you would be uninstalling. +- **Its store is shared across every library you indexed with it**, not just your memories. Deleting + the directory deletes all of them. Run `docs-mcp-server list` first and see what is in there. +- **Ollama may be serving something else.** `ollama list` shows what it holds; if + `nomic-embed-text` is the only entry and nothing else here needs the runtime, you can also remove + the app and its data — `/Applications/Ollama.app` and `~/.ollama`. Removing the app clears the + macOS Login Item it registers. + +This pack no longer installs or manages any of them. + +--- + ## Directory structure ``` @@ -88,7 +118,7 @@ memory/ ├── techpack.yaml # Manifest — defines all components ├── config/settings.json # Disables built-in auto-memory ├── hooks/ -│ ├── sync-memories.sh # Ollama health + memory indexing/reindexing +│ ├── sync-memories.sh # Memory indexing/reindexing │ ├── continuous-learning-activator.sh # Knowledge extraction reminder │ └── kb-gate.sh # Keeps KB lookups ahead of delegated discovery ├── skills/ diff --git a/SYNC-BLOCKS.md b/SYNC-BLOCKS.md index f014594..3c3166d 100644 --- a/SYNC-BLOCKS.md +++ b/SYNC-BLOCKS.md @@ -51,7 +51,7 @@ Locations: ```markdown -**The `Applies to:` field.** Place `**Applies to:**` on the line immediately after the `# Title` heading of every memory; it declares which project(s) the memory targets. Use the **git repo name** — the last path segment of `git remote get-url origin`, with `.git` stripped (e.g. `git@github.com:org/repo.git` → `repo`; `https://github.com/owner/my-app.git` → `my-app`). Fall back to the working directory's basename only when the repo has no remote configured. Use the repo name — not the directory basename — because folder names vary across clones while the repo name is stable. This is also why `Applies to:` may differ from the `library:` parameter used for `search_docs`, which is folder-based and set automatically by the indexing hook. +**The `Applies to:` field.** Place `**Applies to:**` on the line immediately after the `# Title` heading of every memory; it declares which project(s) the memory targets. Use the **git repo name** — the last path segment of `git remote get-url origin`, with `.git` stripped (e.g. `git@github.com:org/repo.git` → `repo`; `https://github.com/owner/my-app.git` → `my-app`). Fall back to the working directory's basename only when the repo has no remote configured. Use the repo name — not the directory basename — because folder names vary across clones while the repo name is stable. This is also why `Applies to:` may differ from the set of memories the search index actually covers, which is folder-based and set automatically by the indexing hook. When a memory genuinely applies to multiple projects, list them comma-separated (e.g. `**Applies to:** web-dashboard, ios-app, api-backend`); the content must stay true in every listed project. When a memory is only partially relevant to one listed project, split it into separate memories instead of mixing. diff --git a/hooks/continuous-learning-activator.sh b/hooks/continuous-learning-activator.sh index 65c2ecd..35e6a7b 100755 --- a/hooks/continuous-learning-activator.sh +++ b/hooks/continuous-learning-activator.sh @@ -4,7 +4,7 @@ cat << 'EOF' MANDATORY MEMORY PROTOCOL If this starts a new sub-task or phase (tests, refactor, deploy, etc.) -→ search the KB via search_docs for relevant patterns first. +→ search the KB via mcp__memory-loop__query for relevant patterns first. If you hit an unexpected error or are about to debug/diagnose → search the KB FIRST, before reasoning from scratch. diff --git a/hooks/kb-gate.sh b/hooks/kb-gate.sh index 68e46ec..6e2574f 100644 --- a/hooks/kb-gate.sh +++ b/hooks/kb-gate.sh @@ -4,7 +4,7 @@ # # One dispatcher registered on four events, branching on `hook_event_name`: # UserPromptSubmit → stamp the turn boundary, and run housekeeping -# PostToolUse → record a successful search_docs query +# PostToolUse → record a successful KB search # PreToolUse → judge a sub-agent spawn, and warn or block # SubagentStart → tell a discovery sub-agent the KB exists # @@ -14,7 +14,7 @@ # - NEVER exit 2. That blocks unconditionally, turning a script bug into a hard # gate with no escape. This is also why `set -u`/`set -e` are omitted: an # unset variable should fall through to a silent allow, not abort mid-branch. -# - NEVER call docs-mcp-server here. That CLI takes seconds; PreToolUse runs +# - NEVER call qmd here. That CLI loads an embedding model; PreToolUse runs # before every sub-agent spawn. File stats only. # - Log every evaluation. Comparing log line count against the spawn count in a # session transcript is the only way to catch a matcher that matches nothing. @@ -102,11 +102,10 @@ EOF # worth more than the fork. resolve_paths() { [ -n "$project_root" ] && return 0 + # keep in sync with hooks/sync-memories.sh — that hook indexes the memories + # directory this one gates on, and both must land on the same project. project_root=$(git rev-parse --show-toplevel 2>/dev/null) [ -n "$project_root" ] || project_root="${CLAUDE_PROJECT_DIR:-$PWD}" - # keep in sync with hooks/sync-memories.sh project root + library derivation — - # the library name quoted back to Claude has to be the one that was indexed. - library="${project_root##*/}" MEMORIES_DIR="$project_root/.claude/memories" LOG_FILE="$project_root/.claude/.kb-gate.log" STATE_DIR="$project_root/.claude/.kb-gate" @@ -219,7 +218,18 @@ PostToolUse) # A KB search landed. Store the query text (not just the turn) so the barrier # can become topical later without a state migration. The number of lines # matching the current turn is also what stamps a denial's progress marker. - query=$(jq -r '.tool_input.query // ""' <<<"$payload" 2>/dev/null) + # The search tool takes *either* a single `query` or a list of typed + # `searches`, never both. Reading only `query` would miss every hybrid + # lex+vec call — the form this pack steers Claude toward — and the barrier + # would then deny forever with nothing to show why. + # Only the two shapes that actually search: a bare `query`, or the joined + # text of typed `searches`. `intent` is context and never searches on its + # own, so accepting it would let a request that retrieved nothing satisfy + # the barrier. Built as a list rather than a `//` chain: `//` only falls + # through on null, so an empty join would stop it, and `null | map` raises. + query=$(jq -r '[.tool_input.query, + ((.tool_input.searches // []) | map(.query) | join(" "))] + | map(select(. != null and . != "")) | first // ""' <<<"$payload" 2>/dev/null) [ -n "$query" ] || exit 0 ensure_state || exit 0 printf '%s\t%s\n' "$(current_turn)" "$query" >>"$QUERIES_FILE" 2>/dev/null || true @@ -238,20 +248,26 @@ SubagentStart) # Cost is bounded on purpose: skip the search entirely when the prompt already # carries findings, and spend at most one query otherwise. SubagentStart input # does not include the prompt, so the agent has to make that call itself. - jq -nc --arg e "$event" --arg m "$KB_MARKER" --arg lib "$library" '{ + jq -nc --arg e "$event" --arg m "$KB_MARKER" '{ hookSpecificOutput: { hookEventName: $e, additionalContext: ( "This project keeps a knowledge base of past learnings, decisions, and\n" + "debugging discoveries in .claude/memories/, searchable with\n" + - "mcp__docs-mcp-server__search_docs using library=\"" + $lib + "\".\n\n" + + "mcp__memory-loop__query.\n\n" + "- If your prompt contains a \"" + $m + "\" block, treat it as established\n" + " ground truth. Verify it against the code, but do NOT search the KB and\n" + " do NOT re-derive it.\n" + "- Otherwise, if you are about to read or grep more than a couple of files,\n" + - " issue ONE search_docs query for the topic of your task first — one search\n" + - " is far cheaper than a blind file sweep. Unlike the main thread, do not try\n" + - " keyword variations: if nothing relevant comes back, move on to the code.\n" + + " issue ONE mcp__memory-loop__query for the topic of your task first — a\n" + + " lex line of terms you expect verbatim plus a vec line phrased as a\n" + + " question, and an intent saying what you want and what to avoid, with\n" + + " rerank:false and limit:6. One search is far cheaper than a blind\n" + + " file sweep. Unlike the main thread, do not try keyword variations: if\n" + + " nothing relevant comes back, move on to the code.\n" + + "- Snippets are leads, not evidence — they are capped at 300 characters and\n" + + " often show only a title. mcp__memory-loop__get a document before relying\n" + + " on what it says.\n" + "- Report back anything the KB got wrong or left out." ) } @@ -317,7 +333,7 @@ PreToolUse) # Name whichever half failed, so the message is actionable. missing="" - [ "$fresh_query" = false ] && missing="no search_docs call has run since this turn began" + [ "$fresh_query" = false ] && missing="no KB search has run since this turn began" [ "$had_kb_block" = false ] && missing="${missing:+$missing, and }the prompt has no \"$KB_MARKER\" block" @@ -377,7 +393,7 @@ EOF # Phrased as a missing prerequisite plus an explicit retry instruction. A # bare denial reads as "the user declined" and makes Claude abandon the # path instead of satisfying the requirement. - jq -nc --arg e "$event" --arg r "Prerequisite missing: $missing. Call search_docs(library=\"$library\", query=\"\") first, then re-issue this exact call with a \"$KB_MARKER\" block (1-5 bullets of findings, or \"$KB_MARKER none relevant.\") at the top of the prompt. Spawning several agents at once: write the findings to a scratchpad file and open each prompt with \"$KB_MARKER see \" instead of repeating them. Sub-agents cannot see your KB results — unpasted context is rediscovered from scratch." \ + jq -nc --arg e "$event" --arg r "Prerequisite missing: $missing. Search the KB with mcp__memory-loop__query first, then re-issue this exact call with a \"$KB_MARKER\" block (1-5 bullets of findings, or \"$KB_MARKER none relevant.\") at the top of the prompt. Spawning several agents at once: write the findings to a scratchpad file and open each prompt with \"$KB_MARKER see \" instead of repeating them. Sub-agents cannot see your KB results — unpasted context is rediscovered from scratch." \ '{hookSpecificOutput:{hookEventName:$e,permissionDecision:"deny",permissionDecisionReason:$r}}' exit 0 fi @@ -385,7 +401,7 @@ EOF # warn: advise without blocking. No permissionDecision field — returning # "allow" would bypass the normal permission flow. log_decision false warn - jq -nc --arg e "$event" --arg m "KB protocol: $missing. Sub-agents cannot see your KB results, so this agent will rediscover from scratch. Before the next spawn, search the KB (library=\"$library\") and open the prompt with a \"$KB_MARKER\" block." \ + jq -nc --arg e "$event" --arg m "KB protocol: $missing. Sub-agents cannot see your KB results, so this agent will rediscover from scratch. Before the next spawn, search the KB and open the prompt with a \"$KB_MARKER\" block." \ '{hookSpecificOutput:{hookEventName:$e,additionalContext:$m}}' ;; esac diff --git a/hooks/sync-memories.sh b/hooks/sync-memories.sh index a406be4..7a74c8f 100755 --- a/hooks/sync-memories.sh +++ b/hooks/sync-memories.sh @@ -1,7 +1,15 @@ #!/bin/bash -# Hook: index .claude/memories/ into docs-mcp-server for semantic search. +# Hook: index .claude/memories/ into this project's search index. # Runs on SessionStart and UserPromptSubmit (async). Never fails the hook. +# +# Reindexes unconditionally; do not add a staleness check. A `find -newer` gate +# was removed because `find` does not descend a symlinked root, and +# .claude/memories is a symlink whenever the shared-memories pack is installed: +# it saw zero files, so the index froze for days with no failure log. Cost of +# dropping it, on a 527-document index: update 0.185s, embed 0.105s, status +# 0.134s, against 0.056s saved. `qmd update` resolves the collection path itself, +# so a directory and a symlink behave the same. set -uo pipefail @@ -9,57 +17,147 @@ set -uo pipefail cat >/dev/null 2>&1 || true # Resolve the project root. Prefer the git repo toplevel — it's the stable -# anchor so launching from any subdirectory of a repo maps to the same library. +# anchor so launching from any subdirectory of a repo maps to the same index. # In a non-git folder git exits non-zero (empty output); fall back to where # Claude was launched (CLAUDE_PROJECT_DIR, else $PWD). # -# keep in sync with hooks/kb-gate.sh resolve_paths() — that hook quotes the -# library name back to Claude, and it has to be the one indexed here. +# keep in sync with the memory-loop MCP launcher in techpack.yaml — that command +# opens the index this hook writes, so both have to agree on where it lives. project_root=$(git rev-parse --show-toplevel 2>/dev/null) [ -n "$project_root" ] || project_root="${CLAUDE_PROJECT_DIR:-$PWD}" MEMORIES_DIR="$project_root/.claude/memories" -TIMESTAMP_FILE="$project_root/.claude/.memories-last-indexed" +INDEX_DIR="$project_root/.claude/.kb-index" +CONFIG="$INDEX_DIR/memory-loop.yml" -# Exit early if no memories directory -[ -d "$MEMORIES_DIR" ] || exit 0 +# One model fills all three of qmd's slots. Embed is the only one that does real +# work; rerank and generate are pointed here deliberately. An embedding model has +# no ranking head, so qmd's reranker fails to build a context, warns, and falls +# back to RRF scores. That matters because the MCP `query` tool defaults +# `rerank: true` and a *missing* model would instead be downloaded mid-query — +# hundreds of MB, inside a tool call, with no progress output. Measured cost of +# the fallback: MRR 0.792 against 0.800 for an explicit rerank:false. +EMBED_MODEL="hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf" -# Exit early if Ollama is not running -curl -s --max-time 2 http://localhost:11434/api/tags >/dev/null 2>&1 || exit 0 - -# --- Staleness check --- -# If timestamp file exists and nothing changed, nothing to do. -# If timestamp file doesn't exist, this is the first run — do a full index. -if [ -f "$TIMESTAMP_FILE" ]; then - newest=$(find "$MEMORIES_DIR" -name "*.md" -newer "$TIMESTAMP_FILE" -print -quit 2>/dev/null) - # Also check if the directory itself was modified (file added/removed) - dir_changed="" - [ "$MEMORIES_DIR" -nt "$TIMESTAMP_FILE" ] && dir_changed="yes" - [ -n "$newest" ] || [ -n "$dir_changed" ] || exit 0 -fi +# Exit early if qmd is not installed +command -v qmd >/dev/null 2>&1 || exit 0 -# --- Index --- -repo_name=$(basename "$project_root") +# --- Index configuration --- +# A named index (`--index`) rather than a project-local .qmd/ directory, for two +# reasons. It leaves any .qmd/ the user keeps for their own code alone; and a +# checked-in .qmd/index.yml falls under qmd's trust gate, which skips a +# non-default embedding model for non-interactive callers *without an error* — +# retrieval would silently fall back to a weaker model. Named indexes are never +# gated. QMD_CONFIG_DIR and INDEX_PATH move that named index back under the +# project so it stays inspectable and deletable. +export QMD_CONFIG_DIR="$INDEX_DIR" +export INDEX_PATH="$INDEX_DIR/memory-loop.sqlite" -export OPENAI_API_KEY=ollama -export OPENAI_API_BASE=http://localhost:11434/v1 -export DOCS_MCP_SCRAPER_SECURITY_FILE_ACCESS_MODE=unrestricted -export DOCS_MCP_SCRAPER_SECURITY_FILE_ACCESS_INCLUDE_HIDDEN=true -export DOCS_MCP_SCRAPER_SECURITY_FILE_ACCESS_FOLLOW_SYMLINKS=true -export DOCS_MCP_EMBEDDING_MODEL="openai:nomic-embed-text" +mkdir -p "$INDEX_DIR" 2>/dev/null || exit 0 + +# Written wholesale rather than patched: the file has exactly one collection and +# one model, so there is nothing to preserve and no YAML parser to depend on. +# Rewritten whenever it drifts, which also repairs a config edited by hand. +# +# The collection path is written as-is, unresolved. .claude/memories may be a +# plain directory or a symlink into a shared checkout (the shared-memories pack +# makes it one), and both index correctly — qmd opens the collection root +# directly and resolves both sides of its containment check. Storing the literal +# path is what keeps the two interchangeable: `qmd collection add` would record +# the symlink's *target*, which goes stale the moment the link is re-pointed or +# the pack is added to a project that already had a real directory. +# Single-quoted YAML scalar with internal quotes doubled: a project path may +# legitimately contain "#" or ": ", either of which silently changes what the +# parser sees when written bare. +mem_yaml="'${MEMORIES_DIR//\'/\'\'}'" -# Check if library already indexed with the same source URL -existing_url=$(docs-mcp-server list 2>/dev/null \ - | jq -r --arg name "$repo_name" '.[] | select(.name == $name) | .versions[0].sourceUrl // empty') +# Unquoted heredoc — it has to interpolate the two variables below, so backticks +# and $ in the context text are executed by the shell and vanish from the config. +cat >"$CONFIG.new" </dev/null 2>&1 +# Identity only — no search guidance. qmd serves global_context two ways from one +# knob: the MCP server's `instructions`, once, and the `context` field of *every* +# search result. Guidance here is paid for per result and is the only copy an +# install already has via the CLAUDE.md template, which is isRequired. At 342 +# characters it was 38% of a six-result response; at ~120 it is 13%. +# +# Replaced only when the content actually differs, so a no-op run leaves the +# mtime alone. Comparing whole files rather than probing for one line is what +# lets any later edit here — the context text especially — reach an install that +# already has a config. +if ! cmp -s "$CONFIG.new" "$CONFIG" 2>/dev/null; then + mv -f "$CONFIG.new" "$CONFIG" || exit 0 else - docs-mcp-server scrape "$repo_name" \ - "file://$MEMORIES_DIR" \ - --silent >/dev/null 2>&1 + rm -f "$CONFIG.new" fi -# Mark indexing time for subsequent staleness checks -touch "$TIMESTAMP_FILE" +# The config is written even when no memory has been captured yet, because the +# MCP server reads its collection list once at startup: a server that booted +# against a config with no collections cannot see the first memory until the next +# session, however promptly this hook indexes it. Registering the collection up +# front costs nothing — qmd accepts a collection whose directory does not exist +# and reports zero documents. +[ -d "$MEMORIES_DIR" ] || exit 0 + +# --- Reindex --- +# Both hook registrations are async, so two runs can overlap on a slow first +# index. qmd's own embed lock reports contention as success — it prints that +# another embed is running and exits 0 — so without a mutex here the losing run +# would clear the failure log having embedded nothing, hiding the pending work +# behind a clean bill of health. +# +# mkdir is the atomic test-and-set. A run killed by the hook timeout cannot +# release the lock, so a lock older than the timeout is treated as abandoned. +LOCK="$INDEX_DIR/.reindex.lock" +if ! mkdir "$LOCK" 2>/dev/null; then + [ -n "$(find "$LOCK" -maxdepth 0 -mmin +5 2>/dev/null)" ] || exit 0 + rmdir "$LOCK" 2>/dev/null + mkdir "$LOCK" 2>/dev/null || exit 0 +fi +trap 'rmdir "$LOCK" 2>/dev/null' EXIT + +# Removed on success, so its presence is itself the signal that indexing is broken +# and it holds the reason. A doctor check reports it. Lives beside the index, so +# removing that directory clears it too. +ERROR_LOG="$INDEX_DIR/memory-loop.log" + +# `update` rescans the collection for added, changed and removed files; `embed` +# vectorises whatever came back without one. Both are incremental — a no-op pass +# is under a tenth of a second. +qmd --index memory-loop update >"$ERROR_LOG" 2>&1 || exit 0 +qmd --index memory-loop embed >>"$ERROR_LOG" 2>&1 || exit 0 + +# Exit code is not enough: qmd reports embed-lock contention as success, so a run +# can leave documents unembedded and still exit 0. +# +# `status`, not `update`: `update`'s "needing vectors" count is computed without +# the embed model, so it reports every hash as pending on a fully embedded index. +pending=$(qmd --index memory-loop status 2>/dev/null | + sed -n 's/.*Pending: *\([0-9][0-9]*\).*/\1/p' | head -1) +if [ "${pending:-0}" -ne 0 ]; then + printf '%s %s documents still need embedding after this run.\n' \ + "$(date '+%Y-%m-%dT%H:%M:%S')" "$pending" >>"$ERROR_LOG" + exit 0 +fi + +# Nothing pending: clear the failure log so its presence stays meaningful. +rm -f "$ERROR_LOG" + +# Editing a memory strands its old vectors; one audit left 275 orphaned chunks, +# 47% of the file. Unconditional for the same reason the staleness check is gone: +# 0.116s is not worth a condition. Also clears qmd's LLM cache, empty here since +# nothing populates it while rerank and expansion are off. Best-effort — a failed +# cleanup is a disk problem, not an indexing one. +qmd --index memory-loop cleanup >/dev/null 2>&1 || true diff --git a/skills/continuous-learning/SKILL.md b/skills/continuous-learning/SKILL.md index 24deebb..e3ee056 100644 --- a/skills/continuous-learning/SKILL.md +++ b/skills/continuous-learning/SKILL.md @@ -6,7 +6,7 @@ description: > .claude/memories/ for codebase knowledge, CLAUDE.local.md for environment/tool/instance config, or skip for public documentation. Also use when the user asks to "run a retrospective", "extract learnings", or "save what we learned" from the current session. -allowed-tools: Write, Read, Glob, Edit, Bash, WebSearch, mcp__docs-mcp-server__search_docs, mcp__docs-mcp-server__list_libraries +allowed-tools: Write, Read, Glob, Edit, Bash, WebSearch, mcp__memory-loop__query, mcp__memory-loop__get --- # Continuous Learning Skill @@ -96,18 +96,24 @@ If any rule fails, rewrite the memory to satisfy it (e.g. anonymize an actor, re ### Step 2: Search Existing Knowledge -**Always search docs-mcp-server first** (semantic search across documentation and project memories): +**Always search the memory index first** (semantic search across this project's memories): ``` -mcp__docs-mcp-server__search_docs(library: "", query: "") +mcp__memory-loop__query(searches: [{type: "lex", query: ""}, + {type: "vec", query: ""}], + intent: "", rerank: false, limit: 6) ``` -**Fall back to file listing** if search_docs returns no results or the project library is not yet indexed: +**Fall back to file listing** if the search returns no results or the index is not yet built: ``` Glob(pattern: ".claude/memories/*.md") ``` +**Read the candidates with `mcp__memory-loop__get` before deciding.** Result snippets are capped +at 300 characters and often show only a document's title, so a merge-or-skip call made from them +is a guess — and the cost of guessing wrong is a duplicate memory or a lost refinement. + Decide what to do, in this order of preference: 1. **Knowledge is already captured.** Skip. @@ -142,7 +148,7 @@ Read [references/templates.md](references/templates.md) for template structures. Fill in `Applies to:` directly under the title heading of every memory. -**The `Applies to:` field.** Place `**Applies to:**` on the line immediately after the `# Title` heading of every memory; it declares which project(s) the memory targets. Use the **git repo name** — the last path segment of `git remote get-url origin`, with `.git` stripped (e.g. `git@github.com:org/repo.git` → `repo`; `https://github.com/owner/my-app.git` → `my-app`). Fall back to the working directory's basename only when the repo has no remote configured. Use the repo name — not the directory basename — because folder names vary across clones while the repo name is stable. This is also why `Applies to:` may differ from the `library:` parameter used for `search_docs`, which is folder-based and set automatically by the indexing hook. +**The `Applies to:` field.** Place `**Applies to:**` on the line immediately after the `# Title` heading of every memory; it declares which project(s) the memory targets. Use the **git repo name** — the last path segment of `git remote get-url origin`, with `.git` stripped (e.g. `git@github.com:org/repo.git` → `repo`; `https://github.com/owner/my-app.git` → `my-app`). Fall back to the working directory's basename only when the repo has no remote configured. Use the repo name — not the directory basename — because folder names vary across clones while the repo name is stable. This is also why `Applies to:` may differ from the set of memories the search index actually covers, which is folder-based and set automatically by the indexing hook. When a memory genuinely applies to multiple projects, list them comma-separated (e.g. `**Applies to:** web-dashboard, ios-app, api-backend`); the content must stay true in every listed project. When a memory is only partially relevant to one listed project, split it into separate memories instead of mixing. diff --git a/skills/memory-audit/SKILL.md b/skills/memory-audit/SKILL.md index e43f7f8..24eeda4 100644 --- a/skills/memory-audit/SKILL.md +++ b/skills/memory-audit/SKILL.md @@ -5,7 +5,7 @@ description: > Use this skill when the user says "audit memories", "review memories", "clean up memories", "memory audit", "check my memories", or wants to prune, deduplicate, or assess the quality of their stored learnings and decisions. This is a manual-only skill — never trigger automatically. -allowed-tools: Read, Glob, Grep, Edit, Bash, Write, mcp__docs-mcp-server__search_docs, mcp__docs-mcp-server__list_libraries, AskUserQuestion +allowed-tools: Read, Glob, Grep, Edit, Bash, Write, mcp__memory-loop__query, mcp__memory-loop__get, AskUserQuestion --- # Memory Audit Skill @@ -36,7 +36,7 @@ The audit enforces these rules through the criteria below — see Group A. ## The `Applies to:` field -**The `Applies to:` field.** Place `**Applies to:**` on the line immediately after the `# Title` heading of every memory; it declares which project(s) the memory targets. Use the **git repo name** — the last path segment of `git remote get-url origin`, with `.git` stripped (e.g. `git@github.com:org/repo.git` → `repo`; `https://github.com/owner/my-app.git` → `my-app`). Fall back to the working directory's basename only when the repo has no remote configured. Use the repo name — not the directory basename — because folder names vary across clones while the repo name is stable. This is also why `Applies to:` may differ from the `library:` parameter used for `search_docs`, which is folder-based and set automatically by the indexing hook. +**The `Applies to:` field.** Place `**Applies to:**` on the line immediately after the `# Title` heading of every memory; it declares which project(s) the memory targets. Use the **git repo name** — the last path segment of `git remote get-url origin`, with `.git` stripped (e.g. `git@github.com:org/repo.git` → `repo`; `https://github.com/owner/my-app.git` → `my-app`). Fall back to the working directory's basename only when the repo has no remote configured. Use the repo name — not the directory basename — because folder names vary across clones while the repo name is stable. This is also why `Applies to:` may differ from the set of memories the search index actually covers, which is folder-based and set automatically by the indexing hook. When a memory genuinely applies to multiple projects, list them comma-separated (e.g. `**Applies to:** web-dashboard, ios-app, api-backend`); the content must stay true in every listed project. When a memory is only partially relevant to one listed project, split it into separate memories instead of mixing. diff --git a/techpack.yaml b/techpack.yaml index 17d4502..5cb23eb 100644 --- a/techpack.yaml +++ b/techpack.yaml @@ -43,96 +43,159 @@ components: description: Lightweight JSON processor brew: jq - - id: ollama - displayName: Ollama - description: Local LLM runtime (compatible with all Apple Silicon) + # ── Retrieval engine ──────────────────────────────────────────────────── + # Pinned so no package manager runs inside a hook, and because the rerank and + # generate neutralisation relies on undocumented fallback behaviour a version + # bump could change. See CLAUDE.md. + - id: qmd-install + displayName: qmd + description: Local search engine over markdown — no daemon, bundled SQLite + sqlite-vec + dependencies: [node] type: configuration - shell: "curl -fsSL https://ollama.com/install.sh | sh" - shellInteractive: true + shell: "npm install -g @tobilu/qmd@2.8.3" doctorChecks: + # ONE check, deliberately: `isAlreadyInstalled` takes the first *passing* + # check as proof the component is installed, so any second check that can + # pass while the pin is unsatisfied silently disables the install. + # Asserts the version and that the binary loads (`--version` alone never + # loads the native modules); a qmd that runs proves Node is new enough. + # Anchored, so 2.8.30 and 12.8.3 do not pass. Keep the literal in step + # with `shell:` above. + # + # The load probe runs against a throwaway index. A bare `qmd status` adopts + # whatever config it finds by walking up from the working directory, so in a + # project where the user keeps their own .qmd/ it writes model keys into + # their config and creates their index.sqlite — the pack must not touch it. + # `--index` is what skips that discovery; QMD_CONFIG_DIR alone does not, + # because a discovered .qmd/ outranks it. - type: commandExists - name: "Ollama installed" - section: AI Models - command: ollama - args: ["--version"] + name: "qmd 2.8.3 installed and loading" + section: MCP Servers + command: /bin/sh + args: + - "-c" + - | + PATH="$PATH:/opt/homebrew/bin:/usr/local/bin" + qmd --version 2>/dev/null | grep -qE '^qmd 2\.8\.3( |$)' || exit 1 + d=$(mktemp -d) || exit 1 + QMD_CONFIG_DIR="$d" INDEX_PATH="$d/probe.sqlite" \ + qmd --index probe status >/dev/null 2>&1 + s=$?; rm -rf "$d"; exit $s + fixCommand: "npm install -g @tobilu/qmd@2.8.3" - - id: ollama-service - displayName: Ollama service - description: Ensure Ollama is running + # One model serves all three of qmd's slots, so this downloads a single file. + # It runs at sync time on purpose: a first-run download inside the async + # indexing hook would blow its timeout and leave the index silently empty. + - id: qmd-model + displayName: Embedding model + description: Downloads Qwen3-Embedding-0.6B once (~610 MB, shared across all projects) + dependencies: [qmd-install] type: configuration - dependencies: [ollama] - shell: "open /Applications/Ollama.app --args hidden" + shell: | + d=$(mktemp -d) + m="hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf" + QMD_CONFIG_DIR="$d" INDEX_PATH="$d/pull.sqlite" \ + QMD_EMBED_MODEL="$m" QMD_GENERATE_MODEL="$m" QMD_RERANK_MODEL="$m" \ + qmd --index pull pull --progress + s=$?; rm -rf "$d"; exit $s doctorChecks: - type: commandExists - name: "Ollama service running" + name: "Qwen3-Embedding-0.6B downloaded" section: AI Models - command: curl - args: ["-sf", "http://localhost:11434/api/tags"] - fixCommand: "open /Applications/Ollama.app --args hidden" + command: /bin/sh + args: ["-c", 'test -f "${XDG_CACHE_HOME:-$HOME/.cache}/qmd/models/hf_Qwen_Qwen3-Embedding-0.6B-Q8_0.gguf"'] - - id: ollama-nomic-embed - displayName: nomic-embed-text model - description: Embedding model for docs-mcp-server - type: configuration - dependencies: [ollama-service] - shell: "ollama pull nomic-embed-text" + # ── MCP Server ────────────────────────────────────────────────────────── + # One static command line serves every project, so the index path is computed + # at launch — Claude spawns MCP servers with the project as their working + # directory. Using `--index` rather than a project-local .qmd/ is an invariant; + # see CLAUDE.md. + # + # `mkdir -p` because qmd cannot open an index under a directory that does not + # exist, and the server would fail to connect in every project that has not + # captured a memory yet — which is every new project. The CLAUDE.md section is + # installed globally and tells Claude to search before each task, so the tool + # has to exist everywhere that instruction does. An empty index is honest: the + # server reports "0 markdown documents" in its own instructions. + # + # keep in sync with hooks/sync-memories.sh — it writes the index this opens. + - id: memory-loop + description: Semantic search over this project's memories, backed by local embeddings + isRequired: true + dependencies: [node, qmd-install, qmd-model] + mcp: + command: /bin/sh + args: + - "-c" + - | + r=$(git rev-parse --show-toplevel 2>/dev/null) + [ -n "$r" ] || r="${CLAUDE_PROJECT_DIR:-$PWD}" + d="$r/.claude/.kb-index" + PATH="$PATH:/opt/homebrew/bin:/usr/local/bin" + mkdir -p "$d" + export QMD_CONFIG_DIR="$d" INDEX_PATH="$d/memory-loop.sqlite" + exec qmd --index memory-loop mcp + # All three pass quietly when nothing is indexed yet, and each repeats the + # launcher's project-root ladder verbatim — every check is its own process, so + # the duplication is forced. Keep the copies identical. The two that invoke qmd + # also harden PATH, because `mcs doctor` runs them through Process, which has + # no login shell. doctorChecks: + # Tier 2, and the only thing that would catch a silent model downgrade: + # a weaker embedding model produces no error, just worse results. - type: commandExists - name: "nomic-embed-text model" + name: "Memory index uses the expected embedding model" section: AI Models - command: curl - args: ["-sf", "http://localhost:11434/api/show", "-d", "{\"name\":\"nomic-embed-text\"}"] - # Probes the exact endpoint docs-mcp-server calls. /api/show only reads the - # manifest — it does not load the model. This check forces a full load+embed - # round-trip, so a runner crash (e.g. Ollama/ggml incompatibility) surfaces - # as HTTP 500 and fails the check. + command: /bin/sh + args: + - "-c" + - | + r=$(git rev-parse --show-toplevel 2>/dev/null) + [ -n "$r" ] || r="${CLAUDE_PROJECT_DIR:-$PWD}" + d="$r/.claude/.kb-index" + PATH="$PATH:/opt/homebrew/bin:/usr/local/bin" + command -v qmd >/dev/null 2>&1 || exit 0 + [ -f "$d/memory-loop.yml" ] || exit 0 + export QMD_CONFIG_DIR="$d" INDEX_PATH="$d/memory-loop.sqlite" + qmd --index memory-loop status 2>/dev/null | grep -q Qwen3-Embedding-0.6B + # The indexing hook removes its log on success, so the file existing at all + # means the last run failed or left documents unembedded — and the reason is + # inside it. Without this, a broken index is only visible as searches quietly + # returning nothing. - type: commandExists - name: "nomic-embed-text embedding endpoint" + name: "Memory indexing completed" section: AI Models - command: curl + command: /bin/sh args: - - "-sf" - - "-X" - - "POST" - - "-H" - - "Content-Type: application/json" - - "-d" - - '{"model":"nomic-embed-text","input":"ping"}' - - "http://localhost:11434/v1/embeddings" - - # ── MCP Servers ───────────────────────────────────────────────────────── - # Install the binary once (pinned) so Claude launches a prebuilt binary. - # `npx @latest` re-downloaded on every cache eviction, blowing Claude's 30s - # MCP startup limit → "MCP error -32000: Connection closed". - - id: docs-mcp-server-install - displayName: docs-mcp-server binary - description: Installs the docs-mcp-server MCP binary once so Claude launches a prebuilt binary - dependencies: [node] - type: configuration - shell: "npm install -g @arabold/docs-mcp-server@2.4.3" - doctorChecks: - # `list` loads the better-sqlite3 native addon (a broken/ABI-mismatched - # build fails here); `--version` wouldn't — it never loads it. + - "-c" + - | + r=$(git rev-parse --show-toplevel 2>/dev/null) + [ -n "$r" ] || r="${CLAUDE_PROJECT_DIR:-$PWD}" + [ ! -f "$r/.claude/.kb-index/memory-loop.log" ] + # Queries with *default* arguments — reranking on — because that is the path + # this pack neutralises. If an upgrade made that fatal rather than a warning, + # this check is where it surfaces. The alarm is the timeout `commandExists` + # does not provide. - type: commandExists - name: "docs-mcp-server binary loads" + name: "Memory search returns results" section: MCP Servers - command: docs-mcp-server - args: ["list"] - fixCommand: "npm install -g @arabold/docs-mcp-server@2.4.3" - - - id: docs-mcp-server - description: Semantic search over memories using local Ollama embeddings - isRequired: true - dependencies: [node, ollama, docs-mcp-server-install] - mcp: - command: docs-mcp-server - args: - - "--read-only" - - "--telemetry=false" - env: - OPENAI_API_KEY: "ollama" - OPENAI_API_BASE: "http://localhost:11434/v1" - DOCS_MCP_EMBEDDING_MODEL: "openai:nomic-embed-text" + command: /bin/sh + args: + - "-c" + - | + r=$(git rev-parse --show-toplevel 2>/dev/null) + [ -n "$r" ] || r="${CLAUDE_PROJECT_DIR:-$PWD}" + d="$r/.claude/.kb-index" + PATH="$PATH:/opt/homebrew/bin:/usr/local/bin" + command -v qmd >/dev/null 2>&1 || exit 0 + [ -f "$d/memory-loop.sqlite" ] || exit 0 + export QMD_CONFIG_DIR="$d" INDEX_PATH="$d/memory-loop.sqlite" + v=$(qmd --index memory-loop status 2>/dev/null | + sed -n 's/.*Vectors: *\([0-9][0-9]*\).*/\1/p' | head -1) + [ "${v:-0}" -gt 0 ] || exit 0 + perl -e 'alarm 30; exec @ARGV' \ + qmd --index memory-loop query "vec: past decisions and learnings" \ + -n 1 --format json 2>/dev/null | grep -q docid # ── Skills ────────────────────────────────────────────────────────────── - id: skill-continuous-learning @@ -162,8 +225,8 @@ components: - id: hook-sync-memories displayName: Sync memories hook - description: Checks Ollama health and syncs docs-mcp-server library on session start - dependencies: [ollama, docs-mcp-server, jq] + description: Indexes memories into the project's search index on session start + dependencies: [qmd-install, jq] hookEvent: SessionStart hookAsync: true hookTimeout: 120 @@ -174,8 +237,8 @@ components: - id: hook-reindex-memories displayName: Reindex memories hook - description: Reindexes docs-mcp-server library when memories have changed mid-session - dependencies: [ollama, docs-mcp-server, jq] + description: Reindexes memories when they have changed mid-session + dependencies: [qmd-install, jq] hookEvent: UserPromptSubmit hookAsync: true hookTimeout: 120 @@ -213,7 +276,7 @@ components: description: Records each KB search so delegated discovery can be checked against it dependencies: [jq] hookEvent: PostToolUse - hookMatcher: "mcp__docs-mcp-server__search_docs" + hookMatcher: "mcp__memory-loop__query" hook: source: hooks/kb-gate.sh destination: kb-gate.sh @@ -267,6 +330,7 @@ components: description: Ignores memory files from version control gitignore: - ".claude/.memories-last-indexed" + - ".claude/.kb-index/" - ".claude/.kb-gate/" - ".claude/.kb-gate.log" @@ -284,6 +348,4 @@ ignore: # --------------------------------------------------------------------------- templates: - sectionIdentifier: continuous-learning - placeholders: - - __PROJECT_DIR_NAME__ contentFile: templates/continuous-learning.md diff --git a/templates/continuous-learning.md b/templates/continuous-learning.md index ce3e592..53663d3 100644 --- a/templates/continuous-learning.md +++ b/templates/continuous-learning.md @@ -2,8 +2,22 @@ Before writing code, planning, or exploring — **always search the knowledge base first**: -1. **Search the KB** — use the `docs-mcp-server` tools (`search_docs`) and set the `library` parameter to the name of the current project folder. The library name always matches the root directory name of this project. This server indexes `.claude/memories/` — it contains past learnings, debugging discoveries, and architectural decisions from previous sessions, not external documentation. Try multiple keyword variations if needed. -2. **Read matching memories** — review any relevant results for full context (architecture decisions, gotchas, patterns from past sessions). +1. **Search the KB** — use `mcp__memory-loop__query`. It searches this project's own `.claude/memories/` — past learnings, debugging discoveries, and architectural decisions from previous sessions, not external documentation. Always pair the two line types — they answer different questions and neither is sufficient alone: + + - `lex` takes **keywords you expect verbatim** in the target memory: identifiers, error names, `"quoted phrases"`, `-negation`. It is the only thing that finds a rare exact token, and it also selects the snippet you get back. + - `vec` takes **prose**: the question as you would ask a colleague. It is what finds a memory that describes your symptom in different words. + - `intent` states what you are looking for **and what to avoid**. + + ``` + mcp__memory-loop__query(searches: [{type: "lex", query: ""}, + {type: "vec", query: ""}], + intent: "", + rerank: false, limit: 6) + ``` + + Results carry a score of `1/rank`, not a confidence — a poor match still scores 1.00 at the top. If nothing fits, re-query with different terms; raising `limit` only appends a tail and never reorders the results above it. + +2. **Retrieve before relying on a result** — a result carries a snippet, which is a lead, not evidence. Fetch what you intend to use with `mcp__memory-loop__multi_get` (or `get` for one document) and read it. Never quote, summarise, or act on a memory you have only seen as a snippet. Only after completing these steps should you proceed with discovery and implementation. @@ -21,7 +35,7 @@ Past sessions often contain decisions and patterns that prevent unnecessary iter ### Delegation barrier -`search_docs` and any sub-agent spawn (the `Agent` / `Task` tool) are **not** independent calls — +`mcp__memory-loop__query` and any sub-agent spawn (the `Agent` / `Task` tool) are **not** independent calls — the KB result is an *input* to the sub-agent's prompt. Never place them in the same message, and never spawn a sub-agent before you have read the KB results. diff --git a/tests/kb-gate-test.sh b/tests/kb-gate-test.sh index 6fb7b0a..be48d37 100755 --- a/tests/kb-gate-test.sh +++ b/tests/kb-gate-test.sh @@ -91,6 +91,20 @@ j_search() { '{hook_event_name:"PostToolUse",session_id:$s,tool_input:{query:$q}}' } +# A hybrid call carries `searches`, not `query`, so a recorder that reads only +# `query` records nothing and the barrier denies forever. `intent` is deliberately +# not a third shape: it never searches on its own, so it must not create a record. +j_search_intent() { + jq -nc --arg s "$sid" --arg i "${1:-intent only}" \ + '{hook_event_name:"PostToolUse",session_id:$s,tool_input:{intent:$i}}' +} +j_search_typed() { + jq -nc --arg s "$sid" --arg a "${1:-lex terms}" --arg b "${2:-vec phrasing}" \ + '{hook_event_name:"PostToolUse",session_id:$s, + tool_input:{searches:[{type:"lex",query:$a},{type:"vec",query:$b}]}}' +} + + # j_spawn [prompt] [agent_type] [agent_id] j_spawn() { jq -nc --arg s "$sid" --arg p "${1:-find the thing}" \ @@ -290,6 +304,33 @@ assert_silent "no stray temp files" "$(find "$state" -name '*.tmp' 2>/dev/null)" assert_num "retained tail stays well clear of the cap" \ "$((LOG_KEEP_LINES * 200))" -lt "$((LOG_MAX_BYTES / 2))" +group "both searching shapes are recorded" + +# Without this the gate fails silently rather than loudly: nothing is written, +# no error is raised, and every subsequent spawn is denied for a search that +# did happen. +queries() { tail -1 "$state/$sid.queries" 2>/dev/null; } + +new_session +run "$mode" "$proj" "$(j_search "plain query field")" >/dev/null +assert_contains "a single-string query" "$(queries)" "plain query field" + +new_session +run "$mode" "$proj" "$(j_search_typed "counter state" "why not wall clock")" >/dev/null +assert_contains "typed lex+vec searches are joined" "$(queries)" "counter state why not wall clock" + +new_session +run "$mode" "$proj" "$(j_search_typed "hybrid" "hybrid")" >/dev/null +assert_allow "a typed search satisfies the barrier" "$(spawn "$KB — from a typed search")" + +# `intent` is context, not a search: qmd rejects a call carrying neither `query` +# nor `searches`, so treating it as evidence would let a request that retrieved +# nothing open the barrier. +new_session +run "$mode" "$proj" "$(j_search_intent "narrowing the topic")" >/dev/null +assert_silent "an intent-only call records nothing" "$(queries)" +assert_deny "and does not satisfy the barrier" "$(spawn "$KB — but no search ran")" + # ========================================================================== printf '\n%s\n' "-----------------------------------------" diff --git a/tests/sync-memories-test.sh b/tests/sync-memories-test.sh new file mode 100755 index 0000000..8b0d491 --- /dev/null +++ b/tests/sync-memories-test.sh @@ -0,0 +1,203 @@ +#!/bin/bash +# +# Tests for hooks/sync-memories.sh. +# +# qmd is stubbed on PATH, so the suite asserts what the hook decides to do rather +# than what qmd does with it — no model, no index, fast in CI. +# +# Three setup details are load-bearing: +# - Everything runs from a temp dir OUTSIDE any git repo. The hook resolves its +# project root with `git rev-parse --show-toplevel` first. +# - The stub creates $INDEX_PATH on `update`. The staleness gate this suite +# guards against was itself guarded by `[ -f "$INDEX_PATH" ]`, so against a +# stub that never creates the file the old hook reindexes anyway and the +# symlink case passes for the wrong reason. +# - The discriminating assertion is the SECOND run. A first run always +# reindexes, old code included, because the config file does not exist yet. + +set -uo pipefail + +repo_root=$(cd "$(dirname "$0")/.." && pwd) +src="$repo_root/hooks/sync-memories.sh" +[ -f "$src" ] || { + echo "FATAL: $src not found" + exit 1 +} + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +pass=0 +fail=0 +calls=0 +last_exit=0 + +ok() { + pass=$((pass + 1)) + printf ' ok %s\n' "$1" +} + +bad() { + fail=$((fail + 1)) + printf ' FAIL %s\n %s\n' "$1" "$2" +} + +group() { printf '\n== %s\n' "$1"; } + +# --- qmd stub ------------------------------------------------------------- + +stub="$work/bin" +mkdir -p "$stub" +cat >"$stub/qmd" <<'STUB' +#!/bin/bash +printf '%s\n' "$*" >>"$STUB_LOG" +case " $* " in +*" update "*) + [ "${STUB_MODE:-ok}" = update_fails ] && { + echo "update exploded" >&2 + exit 1 + } + [ -n "${INDEX_PATH:-}" ] && : >>"$INDEX_PATH" + echo "All collections updated." + ;; +*" embed "*) + echo "All content hashes already have embeddings." + ;; +*" status "*) + echo " Total: 2 files indexed" + [ "${STUB_MODE:-ok}" = pending ] && + echo " Pending: 3 need embedding (run 'qmd embed')" + ;; +*" cleanup "*) + [ "${STUB_MODE:-ok}" = cleanup_fails ] && exit 1 + ;; +esac +exit 0 +STUB +chmod +x "$stub/qmd" + +# --- fixtures ------------------------------------------------------------- + +# A real directory, and a symlink into a sibling checkout — the shape the +# shared-memories pack installs. +plain="$work/plain" +mkdir -p "$plain/.claude/memories" +printf '# One\n' >"$plain/.claude/memories/one.md" + +link="$work/link" +mkdir -p "$link/.claude/.memories-repo/memories" +printf '# Two\n' >"$link/.claude/.memories-repo/memories/two.md" +ln -s ".memories-repo/memories" "$link/.claude/memories" + +nomem="$work/nomem" +mkdir -p "$nomem/.claude" + +# --- invocation ----------------------------------------------------------- + +# run [stub-mode] +run() { + : >"$work/stub.log" + ( + cd "$1" || exit 1 + printf '{}' | env -u CLAUDE_PROJECT_DIR \ + PATH="$stub:$PATH" STUB_LOG="$work/stub.log" STUB_MODE="${2:-ok}" \ + bash "$src" + ) + last_exit=$? + calls=$((calls + $(grep -c . "$work/stub.log"))) +} + +stub_log() { cat "$work/stub.log"; } + +# --- assertions ----------------------------------------------------------- + +assert_contains() { #