From 915ea334ca1ba90833fd231c09d3fee850341faa Mon Sep 17 00:00:00 2001 From: Codex_Lin_Lay Date: Mon, 7 Sep 2026 22:28:14 +0900 Subject: [PATCH] feat(search): add document path-prefix filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit doc検索の母集合を既存リポジトリ内のdirectory prefixで隔離する。dense・sparse・scanへ同じ境界を候補生成前に適用し、入力契約・不成立probe・bridge schema・英日文書を同期した。 --- README.ja.md | 3 +- README.md | 3 +- docs/0-requirements.ja.md | 16 +++--- docs/0-requirements.md | 16 +++--- docs/installation.ja.md | 13 ++++- docs/installation.md | 13 ++++- mcp-server/server/tools.js | 9 +++- mcp-server/test/search-tool-schema.test.js | 10 ++++ migrations/0007_doc_path_prefix.sql | 4 ++ src/fts.test.ts | 57 +++++++++++++++++--- src/fts.ts | 53 +++++++++++++++--- src/fts.workers.test.ts | 62 ++++++++++++++++++++++ src/mcp-stateless-contract.test.ts | 1 + src/mcp.ts | 28 +++++++++- src/path-prefix.test.ts | 49 +++++++++++++++++ src/path-prefix.ts | 44 +++++++++++++++ src/scan.ts | 15 ++++-- src/scan.workers.test.ts | 44 ++++++++++++++- src/store.ts | 11 +++- 19 files changed, 415 insertions(+), 36 deletions(-) create mode 100644 migrations/0007_doc_path_prefix.sql create mode 100644 src/path-prefix.test.ts create mode 100644 src/path-prefix.ts diff --git a/README.ja.md b/README.ja.md index a3b321c..cad4b08 100644 --- a/README.ja.md +++ b/README.ja.md @@ -108,7 +108,7 @@ GitHub の issue / pull request / release / documentation / **GitHub Wiki page** 3. **doc 本文取得** — `include_content: true` を指定すると、`type="doc"` 結果の本文が GitHub contents API 経由で取得され、該当行の `content` フィールドに inline されます。API fan-out を抑えるため先頭の数件に絞られます。従来の `get_doc_content` を置き換えます。 4. **保存済み本文の取得** — `vector_ids`(先行する結果が持つ `vector_id`)を渡します。索引済みの全 type がその行の本文を返します——doc だけでなく issue / PR / comment / review / release / diff も対象です。`search` であたりを付けたあと本文を読むための `gh` / grep の一往復が不要になります。D1 から返すので GitHub API は呼びません。返る文字列が何であって何でないかは下記「保存済み本文の取得」を参照してください。 -structured filter (`repo` / `state` / `labels` / `milestone` / `assignee` / `type`) は、保存済み本文の取得を除くすべてのモードで有効です。保存済み本文の取得では行をサーバが選ぶのではなく呼び出し側が名指しするため、filter は適用しません。 +structured filter (`repo` / `path_prefix` / `state` / `labels` / `milestone` / `assignee` / `type`) は、保存済み本文の取得を除くすべてのモードで有効です。`path_prefix` だけは意図的に狭く、`type: "doc"` と組み合わせて repository-relative directory を ranking 前に選びます。保存済み本文の取得では行をサーバが選ぶのではなく呼び出し側が名指しするため、filter は適用しません。 search モードは「1件もマッチしなかったフィルタ」を `filters_unmatched` に載せます (常に存在し、すべて成立していれば `[]`)。`repo` はフルスラッグ `owner/repo` の完全一致なので、短いリポジトリ名を渡すと母集合が空になり、本当にヒットゼロだった場合と同じ形のレスポンスが返ります。このフィールドがその2つを区別します。効くのは多段のエージェンティック検索で、ゼロが正常な中間結果として読まれてしまい、フィルタ不成立が表に出ないまま終わる場面です。 @@ -120,6 +120,7 @@ bot (`sender.login` が `[bot]` で終わる) と trim 後 10 文字未満の bo |------|----|------| | `query` | string (省略可) | 自然言語クエリ。省略または空文字で scan モード。 | | `repo` | string | repository で絞り込み。フルスラッグ (`owner/repo`) の完全一致。短いリポジトリ名は1件もマッチせず、search モードはそれをレスポンスの `filters_unmatched` に `"repo"` として報告します。 | +| `path_prefix` | string | repository-relative directory prefix で doc を絞り込み。`type: "doc"`、末尾 `/`、UTF-8で64 byte以内が必須。 | | `state` | `"open"` / `"closed"` / `"all"` | state で絞り込み (既定 `all`)。 | | `labels` | string[] | label 名で AND 絞り込み。 | | `milestone` | string | milestone title で絞り込み。 | diff --git a/README.md b/README.md index 36a1157..3bbcf7a 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Four modes are selected by the parameter set: 3. **Doc / wiki content fetch** — set `include_content: true`. For result rows whose `type` is `"doc"`, the raw file content is fetched from the GitHub contents API; for `type: "wiki_doc"` rows, the raw markup is fetched from `raw.githubusercontent.com/wiki/`. Both are inlined as a `content` field. Capped at the first few rows of each type to bound API fan-out. This subsumes the previous `get_doc_content` tool. 4. **Stored-content fetch** — pass `vector_ids` (the `vector_id` values carried by earlier results). Every indexed type returns the body text the index already holds for that exact row — issues, PRs, comments, reviews, releases and diffs included, not just docs — so locating something with `search` and then reading it no longer costs a round trip through `gh` or grep. Served from D1: no GitHub API call is made. See [Stored-content fetch](#stored-content-fetch) below for what the returned text is and is not. -Structured filters (`repo`, `state`, `labels`, `milestone`, `assignee`, `type`) apply in every mode except stored-content fetch, where the rows are named rather than selected. +Structured filters (`repo`, `path_prefix`, `state`, `labels`, `milestone`, `assignee`, `type`) apply in every mode except stored-content fetch, where the rows are named rather than selected. `path_prefix` is intentionally narrower: it is valid only with `type: "doc"` and selects one repository-relative directory before ranking. Search mode reports filters that matched nothing at all in `filters_unmatched` (always present, `[]` when every filter matched something). `repo` is an exact match on the full `owner/repo` slug, so a bare repository name selects an empty population and returns a response shaped exactly like a genuine zero-hit search — this field is what separates the two. It matters most in multi-step agentic search, where a zero reads as a normal intermediate result and the mis-specified filter would otherwise never surface. @@ -120,6 +120,7 @@ Bot-authored comments (`sender.login` ending in `[bot]`) and comments shorter th |------|------|-------------| | `query` | string (optional) | Natural-language query. Omit or empty = scan mode. | | `repo` | string | Filter by repository — full slug (`owner/repo`), exact match. A bare repository name matches nothing; search mode reports that as `"repo"` in the response's `filters_unmatched`. | +| `path_prefix` | string | Filter docs by repository-relative directory prefix. Requires `type: "doc"`, a trailing `/`, and at most 64 UTF-8 bytes. | | `state` | `"open"` \| `"closed"` \| `"all"` | Filter by state (default `all`). | | `labels` | string[] | Filter by label names (AND). | | `milestone` | string | Filter by milestone title. | diff --git a/docs/0-requirements.ja.md b/docs/0-requirements.ja.md index 877d174..15207c2 100644 --- a/docs/0-requirements.ja.md +++ b/docs/0-requirements.ja.md @@ -288,8 +288,10 @@ Vectorize は hybrid retrieval の dense 側を担う。次の metadata を伴 Metadata index(10/10 枠使用): -- Pre-filter 対応: repo, type, state, milestone -- 将来の pre-filter 用に格納: label_0, label_1, label_2, label_3, assignee_0, assignee_1 +- Pre-filter 対応: repo, type, state, milestone, doc_path +- 将来の pre-filter 用に格納: label_0, label_1, label_2, label_3, assignee_0 + +`assignee_1` は vector metadata に保存し続け、現行 post-filter からも利用できるが、metadata index は持たない。platform の10枠上限に対する優先順位として、未使用だった将来用 `assignee_1` pre-filter 枠を `doc_path` へ振り替え、directory 単位の文書検索が ranking 前に dense 候補母集合を絞れるようにする。 Vectorize の metadata filter はフィールド間で AND のみサポートし、OR は非対応。`label_0 = "bug" OR label_1 = "bug"` のようなクエリは表現できない。そのため labels / assignees は overfetch + post-filter で recall を改善している。Vectorize が OR または `$in`-across-fields をサポートした時点で、個別フィールドは即座に pre-filter 化可能。 @@ -390,7 +392,7 @@ retrieval layer は hybrid search(dense + sparse)+ cross-encoder rerank + st 想定フロー: 1. query の embedding を Workers AI BGE-M3 で生成 -2. structured params から Vectorize filter(dense 側)と D1 SQL WHERE(sparse 側)を同時構築(repo, state, type, milestone は pre-filter) +2. structured params から Vectorize filter(dense 側)と D1 SQL WHERE(sparse 側)を同時構築(repo, state, type, milestone と文書 `path_prefix` は両側で pre-filter) 3. 内部 topK を常にオーバーフェッチ(requestedTopK × 5, max 50)。条件なしなのは、8 の entity 集約がどの経路でも複数行を 1 件に畳むため、rerank 無効時でも候補プールが top_k を上回っていなければ要求件数を満たせないからである。reranker は最大 50 件まで処理 4. dense (Vectorize.query) と sparse (D1 FTS5 MATCH + BM25) を並列実行 5. 両 ranker の結果を Reciprocal Rank Fusion(RRF、k=60)で合成 @@ -527,13 +529,15 @@ Returns: - 同一実体の他の行を吸収した結果には `same_entity`(Entity Aggregation 参照)。`top_k` は行数ではなく実体数で数える - top-level metadata: `fusion`、`dense_candidates`、`sparse_candidates`、`rerank_requested`、`rerank_applied`、`filters_unmatched` -**フィルタ不成立(`filters_unmatched`).** `repo` はフルスラッグ(`owner/repo`)の完全一致である——dense 側は Vectorize metadata の `$eq`、sparse 側は `d.repo = ?`。短いリポジトリ名を渡すと1行にもマッチせず、返るレスポンスは「本当にヒットが無かった」場合と同じ形になる。`filters_unmatched` がこの2つを分ける: search mode では常に存在し、`[]` は適用した全フィルタが空でない母集合を選べたこと(つまり `count: 0` は真にヒットゼロ)を意味し、名前が載っていればそのフィルタの母集合が空、すなわち誤っているのはクエリではなくフィルタの値である。 +**フィルタ不成立(`filters_unmatched`).** `repo` はフルスラッグ(`owner/repo`)の完全一致である——dense 側は Vectorize metadata の `$eq`、sparse 側は `d.repo = ?`。`path_prefix` は repository-relative path が指定 directory prefix で始まる doc 行を選ぶ。短いリポジトリ名や、doc 行を1件も選ばない path prefix を渡すと、返るレスポンスは「本当にヒットが無かった」場合と同じ形になる。`filters_unmatched` がこの2つを分ける: search mode では常に存在し、`[]` は適用した全フィルタが空でない母集合を選べたこと(つまり `count: 0` は真にヒットゼロ)を意味し、名前が載っていればそのフィルタの母集合が空、すなわち誤っているのはクエリではなくフィルタの値である。 区別にフィールドを割く理由は、エージェンティックな多段検索が silent zero のコストを反転させるからである。単発検索ならゼロは呼び出し側が見に行く行き止まりだが、検索ループの中では「この角度には何も無かった」という正常な中間結果として消費されて次へ回る。フィルタ不成立が表に出ないまま、クエリ予算を1回分、偽陰性に使って終わる。 -判定は存在確認クエリ(`SELECT 1 FROM search_docs WHERE repo = ? LIMIT 1`)で、候補集合が空のときだけ走る——候補が1件でもあればフィルタが成立した証拠なので、追加の読みが hot path に乗ることはない。プローブ自体が失敗した場合は「観測していない不成立」を主張せず、何も報告しない。プローブ対象は `repo` のみ: もっともらしく見える誤値(フルスラッグに対する短いリポジトリ名)が存在するのはこのフィルタだからである。短い名前からフルスラッグへの自動解決は意図的に非スコープ——複数リポジトリにマッチする名前の曖昧解決を設計する必要がある。 +判定は候補集合が空のときだけ存在確認クエリを走らせる——候補が1件でもあればフィルタが成立した証拠なので、追加の読みが hot path に乗ることはない。まず repository を確認し、成立した場合だけその repository 内の doc path を確認する。誤った repository に対して、別の repository では妥当かもしれない path まで誤りと報告しないためである。プローブ自体が失敗した場合は「観測していない不成立」を主張せず、何も報告しない。短い名前からフルスラッグへの自動解決は意図的に非スコープ——複数リポジトリにマッチする名前の曖昧解決を設計する必要がある。 + +**scan mode(query 空).** Vectorize / FTS5 / reranker を経由せず、structured store の recency endpoint から集約する。`since` / `until` と文書 `path_prefix` は store 側へ push down されるので、窓に対象行があれば、その窓がどれだけ古くても、prefix 外に新しい文書が何件あっても返る。`since` 省略時の既定は `until` の 7 日前(`until` も省略時は現在の 7 日前)。`until` だけ指定した問い合わせが「下限が上限より新しい空窓」に潰れないための既定である。 -**scan mode(query 空).** Vectorize / FTS5 / reranker を経由せず、structured store の recency endpoint から集約する。`since` / `until` は store 側へ push down されるので、窓に行があれば、その窓がどれだけ古くても返る。`since` 省略時の既定は `until` の 7 日前(`until` も省略時は現在の 7 日前)。`until` だけ指定した問い合わせが「下限が上限より新しい空窓」に潰れないための既定である。 +**文書 path prefix.** `path_prefix` は `type: "doc"` と組み合わせた場合だけ有効。repository-relative directory を表し、末尾 `/` が必須で、先頭 `/`、backslash、NUL、空 segment、`.` / `..` segment を含めず、Vectorize が string metadata の先頭64 byteだけを索引する制約に合わせUTF-8で64 byte以内とする。dense / sparse は `doc_path` に同じ半開 lexical range を適用し、scan は range を doc store query へ押し下げ、返った行を `startsWith` でも確認する。fetch mode は `vector_ids` が行を直接名指しするため、従来どおり無視する。 scan mode は top-level に `truncated` を追加する。窓が応答に載せた以上の行を持つとき true になる(endpoint が cap 一杯まで返した、または merge 後の件数が `top_k` を超えた)。これが「該当なし」と「読み切れていない」を呼び出し側に区別させる: 返った最古の行の時刻を次の `until` にして遡ればよい。両者を区別できない欠損調査ツールは、存在しない欠損を報告し実在する取り込みを見落とす——#178 の再検証で 1 日に 2 度踏んだ誤りがこれである。 diff --git a/docs/0-requirements.md b/docs/0-requirements.md index 68d1c24..f771acf 100644 --- a/docs/0-requirements.md +++ b/docs/0-requirements.md @@ -294,8 +294,10 @@ Vectorize is the dense side of hybrid retrieval. It stores semantic embeddings a Metadata indexes (10/10 slots used): -- Pre-filter capable: repo, type, state, milestone -- Stored for future pre-filter: label_0, label_1, label_2, label_3, assignee_0, assignee_1 +- Pre-filter capable: repo, type, state, milestone, doc_path +- Stored for future pre-filter: label_0, label_1, label_2, label_3, assignee_0 + +`assignee_1` remains stored in vector metadata and remains available to the current post-filter, but has no metadata index. The ten-index platform ceiling makes the index allocation a priority choice: `doc_path` replaced the unused future `assignee_1` pre-filter slot so directory-scoped document retrieval can narrow the dense candidate population before ranking. Vectorize metadata filters support AND between fields only, not OR. A query like `label_0 = "bug" OR label_1 = "bug"` cannot be expressed. Labels and assignees therefore remain post-filtered via overfetch strategy. When Vectorize adds OR or `$in`-across-fields support, the expanded fields are immediately usable for pre-filtering. @@ -396,7 +398,7 @@ The retrieval layer supports a 3-tier pipeline: hybrid search (dense + sparse), Expected retrieval behavior: 1. Generate an embedding for the query via Workers AI BGE-M3. -2. Build Vectorize metadata filter (dense side) and D1 SQL WHERE clause (sparse side) from the same structured params (repo, state, type, milestone are pre-filtered on both sides). +2. Build Vectorize metadata filter (dense side) and D1 SQL WHERE clause (sparse side) from the same structured params (repo, state, type, milestone, and document `path_prefix` are pre-filtered on both sides). 3. Overfetch internally on both sides (requestedTopK × 5, max 50). Unconditional: entity aggregation (step 8) collapses several rows into one result on every path, so the candidate pool must exceed top_k even when the reranker is off. The reranker processes at most 50 candidates per call. 4. Query Vectorize (dense) and D1 FTS5 (sparse, BM25) in parallel. 5. Combine the two rankers via Reciprocal Rank Fusion (RRF, k=60). @@ -533,13 +535,15 @@ Returns: - `same_entity` on results that absorbed other rows of the same entity (see Entity Aggregation); `top_k` counts entities, not rows - top-level metadata: `fusion`, `dense_candidates`, `sparse_candidates`, `rerank_requested`, `rerank_applied`, `filters_unmatched` -**Unmatched filters (`filters_unmatched`).** `repo` takes the full slug (`owner/repo`) and matches exactly — on the dense side as a Vectorize metadata `$eq`, on the sparse side as `d.repo = ?`. A bare repository name therefore matches no row, and the response that comes back is shaped exactly like a genuine zero-hit search. `filters_unmatched` separates the two: it is always present in search mode, `[]` means every applied filter selected a non-empty population (so `count: 0` really is "no hits"), and a listed name means that filter's population is empty — the value is wrong, not the query. +**Unmatched filters (`filters_unmatched`).** `repo` takes the full slug (`owner/repo`) and matches exactly — on the dense side as a Vectorize metadata `$eq`, on the sparse side as `d.repo = ?`. `path_prefix` selects doc rows whose repository-relative path begins with the named directory prefix. A bare repository name or a path prefix that selects no doc row therefore produces a response shaped exactly like a genuine zero-hit search. `filters_unmatched` separates the two: it is always present in search mode, `[]` means every applied filter selected a non-empty population (so `count: 0` really is "no hits"), and a listed name means that filter's population is empty — the value is wrong, not the query. The distinction is worth a field because agentic multi-step search inverts the cost of a silent zero. In a single search a zero is a dead end the caller inspects; in a search loop it is a normal intermediate result ("nothing down this angle") that the caller consumes and moves past, so the mis-specified filter never surfaces and one query out of the budget is spent on a false negative. -The check is an existence probe (`SELECT 1 FROM search_docs WHERE repo = ? LIMIT 1`) run only when the candidate set is empty — a non-empty candidate set already proves the filter matched, so the extra read stays off the hot path. A failed probe reports nothing rather than asserting a mismatch it did not observe. Only `repo` is probed: it is the filter with a plausible-looking wrong value. Resolving a short name to a full slug is deliberately out of scope — that needs an ambiguity design for a name matching several repositories. +The checks are existence probes run only when the candidate set is empty — a non-empty candidate set already proves the filters matched, so the extra reads stay off the hot path. The repository probe runs first. When it succeeds, the path probe checks doc rows inside that repository; a bad repository therefore reports `repo` without falsely blaming the path that may be valid elsewhere. A failed probe reports nothing rather than asserting a mismatch it did not observe. Resolving a short repository name to a full slug is deliberately out of scope — that needs an ambiguity design for a name matching several repositories. + +**Scan mode (empty query).** Vectorize / FTS5 / reranker are skipped and the result set is aggregated from the structured store's recency endpoints. `since` / `until` and document `path_prefix` are pushed down to the store, so a window returns matching rows whenever it holds them, however far back it sits and however many newer documents exist outside the prefix. `since` defaults to 7 days before `until` (before now when `until` is omitted), so an `until`-only query does not degenerate into an empty window above its own ceiling. -**Scan mode (empty query).** Vectorize / FTS5 / reranker are skipped and the result set is aggregated from the structured store's recency endpoints. `since` / `until` are pushed down to the store, so a window returns rows whenever it holds rows, however far back it sits. `since` defaults to 7 days before `until` (before now when `until` is omitted), so an `until`-only query does not degenerate into an empty window above its own ceiling. +**Document path prefix.** `path_prefix` is valid only with `type: "doc"`. It names a repository-relative directory, must end in `/`, must not begin with `/`, contain backslashes, NUL, empty segments, or `.` / `..` segments, and must fit in 64 UTF-8 bytes because Vectorize indexes only that prefix of string metadata. Dense and sparse retrieval use the same half-open lexical range over `doc_path`; scan mode pushes the range into the doc store query and verifies `startsWith` on returned rows. Fetch mode continues to ignore it because `vector_ids` names rows directly. Scan mode adds one top-level field, `truncated`, which is true when the window holds more rows than the response carries — either an endpoint filled its row cap, or the merged set was longer than `top_k`. This is what tells a caller that zero results means "no such rows" rather than "the read stopped short": walk backwards by re-issuing the scan with `until` set to the oldest row returned. A gap-hunting tool that cannot separate those two answers reports absent rows that exist and misses rows that do not, which is how #178 was mis-diagnosed twice in one day. diff --git a/docs/installation.ja.md b/docs/installation.ja.md index 7ddb27f..c0fe6ee 100644 --- a/docs/installation.ja.md +++ b/docs/installation.ja.md @@ -47,15 +47,17 @@ wrangler vectorize create-metadata-index github-rag-issues --type string --prope wrangler vectorize create-metadata-index github-rag-issues --type string --property-name type wrangler vectorize create-metadata-index github-rag-issues --type string --property-name state wrangler vectorize create-metadata-index github-rag-issues --type string --property-name milestone +wrangler vectorize create-metadata-index github-rag-issues --type string --property-name doc_path # label/assignee 展開フィールド (将来の Vectorize OR フィルター対応に備えて格納) wrangler vectorize create-metadata-index github-rag-issues --type string --property-name label_0 wrangler vectorize create-metadata-index github-rag-issues --type string --property-name label_1 wrangler vectorize create-metadata-index github-rag-issues --type string --property-name label_2 wrangler vectorize create-metadata-index github-rag-issues --type string --property-name label_3 wrangler vectorize create-metadata-index github-rag-issues --type string --property-name assignee_0 -wrangler vectorize create-metadata-index github-rag-issues --type string --property-name assignee_1 ``` +Vectorize の10 property上限をすべて使う。`assignee_1` は metadata に保存し、現行 post-filter でも使い続けるが、metadata index は意図的に持たない。`path_prefix` は ranking 前に dense 候補を絞る必要があるため、`doc_path` がその枠を使う。 + ### 3.3 KV namespace ```bash @@ -180,6 +182,15 @@ wrangler deploy vector 作成後に metadata index を追加した場合、stored hash を reset して次回 cron で全件 re-embed させる。 +既存 deployment が `assignee_1` を index 済みなら、`path_prefix` 対応の deploy 前に未使用の将来用 filter 枠を置き換える。 + +```bash +wrangler vectorize delete-metadata-index github-rag-issues --property-name assignee_1 +wrangler vectorize create-metadata-index github-rag-issues --type string --property-name doc_path +``` + +`doc_path` index 作成前に upsert 済みの vector は path filter の対象にならない。既存 doc に `path_prefix` を使う repository ごとに reset する。metadata index 作成後に追加する新しい固定コーパスには過去分の再 index は不要。 + Admin endpoint: ```text diff --git a/docs/installation.md b/docs/installation.md index a7c90a0..aa450c8 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -47,15 +47,17 @@ wrangler vectorize create-metadata-index github-rag-issues --type string --prope wrangler vectorize create-metadata-index github-rag-issues --type string --property-name type wrangler vectorize create-metadata-index github-rag-issues --type string --property-name state wrangler vectorize create-metadata-index github-rag-issues --type string --property-name milestone +wrangler vectorize create-metadata-index github-rag-issues --type string --property-name doc_path # Expanded label/assignee fields (stored for future Vectorize OR-filter support) wrangler vectorize create-metadata-index github-rag-issues --type string --property-name label_0 wrangler vectorize create-metadata-index github-rag-issues --type string --property-name label_1 wrangler vectorize create-metadata-index github-rag-issues --type string --property-name label_2 wrangler vectorize create-metadata-index github-rag-issues --type string --property-name label_3 wrangler vectorize create-metadata-index github-rag-issues --type string --property-name assignee_0 -wrangler vectorize create-metadata-index github-rag-issues --type string --property-name assignee_1 ``` +The ten-property Vectorize limit is fully allocated. `assignee_1` is still stored as metadata and used by the current post-filter, but it deliberately has no metadata index; `doc_path` uses that slot because path-prefix filtering must narrow dense candidates before ranking. + ### 3.3 KV namespace ```bash @@ -180,6 +182,15 @@ Recommended verification flow: If metadata indexes were created after vectors already existed, reset stored hashes so the next cron run re-embeds everything. +For an existing deployment that still indexes `assignee_1`, replace that unused future-filter slot before deploying `path_prefix` support: + +```bash +wrangler vectorize delete-metadata-index github-rag-issues --property-name assignee_1 +wrangler vectorize create-metadata-index github-rag-issues --type string --property-name doc_path +``` + +Vectors upserted before `doc_path` was indexed are not path-filterable. Reset each existing repository whose old docs must support `path_prefix`; a new fixed corpus added after the metadata index exists needs no historical re-index. + Admin endpoint: ```text diff --git a/mcp-server/server/tools.js b/mcp-server/server/tools.js index e95b554..fa954d1 100644 --- a/mcp-server/server/tools.js +++ b/mcp-server/server/tools.js @@ -25,7 +25,7 @@ export const TOOLS = [ "(3) doc content fetch — include_content: true inlines raw content on top doc and wiki_doc results; " + "(4) stored-content fetch — vector_ids reads back the body text the index holds for the named rows, " + "for every type and with no GitHub API call, truncated at the 8000-character ingest ceiling. " + - "Structured filters (repo, state, labels, milestone, assignee, type) apply across modes 1-3 " + + "Structured filters (repo, path_prefix, state, labels, milestone, assignee, type) apply across modes 1-3 " + "(mode 4 names its rows, so nothing is filtered there); " + "type: \"wiki_doc\" narrows to GitHub Wiki pages only; repo takes the full slug (owner/repo) and matches " + "exactly, so a bare repository name selects nothing. In search mode the response carries " + @@ -60,6 +60,13 @@ export const TOOLS = [ "A bare repository name (\"my-repo\") matches nothing and yields an empty result set; " + "search mode flags that case as \"repo\" in the response's filters_unmatched.", }, + path_prefix: { + type: "string", + description: + "Filter repository docs by a repository-relative directory prefix before ranking. " + + "Requires type=\"doc\", a trailing /, no leading /, backslash, NUL, empty, . or .. path segments, " + + "and at most 64 UTF-8 bytes. Search mode reports an unmatched value as \"path_prefix\" in filters_unmatched.", + }, state: { type: "string", enum: ["open", "closed", "all"], diff --git a/mcp-server/test/search-tool-schema.test.js b/mcp-server/test/search-tool-schema.test.js index becbff9..1e03de6 100644 --- a/mcp-server/test/search-tool-schema.test.js +++ b/mcp-server/test/search-tool-schema.test.js @@ -78,3 +78,13 @@ test("repo description states the full-slug exact match and the unmatched-filter assert.match(repoParam.description, /exact match/i); assert.match(repoParam.description, /filters_unmatched/); }); + +test("path_prefix is exposed as a doc-only directory filter", () => { + const param = search?.inputSchema?.properties?.path_prefix; + assert.ok(param, "path_prefix param is present in the mirrored schema"); + assert.equal(param.type, "string"); + assert.match(param.description, /type="doc"/); + assert.match(param.description, /(trailing \/|end with \/)/i); + assert.match(param.description, /64 UTF-8 bytes/); + assert.match(param.description, /filters_unmatched/); +}); diff --git a/migrations/0007_doc_path_prefix.sql b/migrations/0007_doc_path_prefix.sql new file mode 100644 index 0000000..63ca34c --- /dev/null +++ b/migrations/0007_doc_path_prefix.sql @@ -0,0 +1,4 @@ +-- Search-mode path_prefix probes and sparse filtering use this range together: +-- exact repo + doc type + half-open doc_path prefix. +CREATE INDEX IF NOT EXISTS idx_search_docs_repo_type_doc_path + ON search_docs (repo, type, doc_path); diff --git a/src/fts.test.ts b/src/fts.test.ts index 180912b..e1d0779 100644 --- a/src/fts.test.ts +++ b/src/fts.test.ts @@ -122,19 +122,30 @@ describe("fts: reciprocalRankFusion", () => { // differ, so the test pins the decision, not the SQL (the SQL is exercised // against a real D1 in fts.workers.test.ts). describe("fts: detectUnmatchedFilters (#219)", () => { - /** Minimal D1 stand-in: records probes, answers from a fixed repo set. */ - function fakeDb(indexedRepos: string[], opts: { throws?: boolean } = {}) { + /** Minimal D1 stand-in: records probes, answers from fixed repo/doc sets. */ + function fakeDb( + indexedRepos: string[], + opts: { throws?: boolean; indexedDocs?: Array<{ repo: string; path: string }> } = {}, + ) { const probes: string[] = []; const db = { probes, - prepare() { + prepare(sql: string) { return { - bind(repo: string) { - probes.push(repo); + bind(...args: string[]) { + probes.push(args.join("|")); return { first: async () => { if (opts.throws) throw new Error("D1_ERROR: unreachable"); - return indexedRepos.includes(repo) ? { present: 1 } : null; + if (sql.includes("type = 'doc'")) { + const [lower, upper, repo] = args; + const found = (opts.indexedDocs ?? []).some( + (doc) => + (!repo || doc.repo === repo) && doc.path >= lower && doc.path < upper, + ); + return found ? { present: 1 } : null; + } + return indexedRepos.includes(args[0]) ? { present: 1 } : null; }, }; }, @@ -170,6 +181,40 @@ describe("fts: detectUnmatchedFilters (#219)", () => { expect(db.probes).toEqual([]); }); + it("flags path_prefix only after the selected repository is known to exist", async () => { + const repo = "Liplus-Project/neuron-graph-rag"; + const db = fakeDb([repo], { + indexedDocs: [{ repo, path: "benchmarks/parity-v4/one.md" }], + }); + expect( + await detectUnmatchedFilters( + db, + { repo, pathPrefix: "benchmarks/missing/" }, + 0, + ), + ).toEqual(["path_prefix"]); + expect(db.probes).toHaveLength(2); + }); + + it("does not blame path_prefix when the repository itself is unmatched", async () => { + const db = fakeDb([], { + indexedDocs: [ + { + repo: "Liplus-Project/neuron-graph-rag", + path: "benchmarks/parity-v4/one.md", + }, + ], + }); + expect( + await detectUnmatchedFilters( + db, + { repo: "wrong/repo", pathPrefix: "benchmarks/parity-v4/" }, + 0, + ), + ).toEqual(["repo"]); + expect(db.probes).toEqual(["wrong/repo"]); + }); + it("reports nothing when the probe itself fails (never assert an unobserved mismatch)", async () => { const db = fakeDb([], { throws: true }); expect(await detectUnmatchedFilters(db, { repo: "owner/repo" }, 0)).toEqual([]); diff --git a/src/fts.ts b/src/fts.ts index e80ee0c..82ae600 100644 --- a/src/fts.ts +++ b/src/fts.ts @@ -27,6 +27,7 @@ */ import { segmentForFts } from "./segment.js"; +import { pathPrefixRange } from "./path-prefix.js"; import type { DiffFileStatus, VectorMetadata } from "./types.js"; /** Which FTS5 virtual table a row is indexed in. */ @@ -272,6 +273,25 @@ export async function repoHasIndexedRows( return row != null; } +/** True when a repository contains at least one indexed doc under the prefix. */ +export async function docPathPrefixHasIndexedRows( + db: D1Database, + pathPrefix: string, + repo?: string, +): Promise { + const { lower, upper } = pathPrefixRange(pathPrefix); + const repoSql = repo ? " AND repo = ?" : ""; + const params = repo ? [lower, upper, repo] : [lower, upper]; + const row = await db + .prepare( + `SELECT 1 AS present FROM search_docs ` + + `WHERE type = 'doc' AND doc_path >= ? AND doc_path < ?${repoSql} LIMIT 1`, + ) + .bind(...params) + .first<{ present: number }>(); + return row != null; +} + /** * Names of the applied filters whose selected population is empty (issue #219). * @@ -287,28 +307,43 @@ export async function repoHasIndexedRows( * - a probe failure is not a finding. An unreachable D1 yields an empty list * (the safer direction: never assert a mismatch that was not observed). * - * `repo` is the only filter probed. It is the one where a plausible-looking wrong - * value exists — the bare repository name against the required `owner/repo` slug. - * `state` / `type` are enum-constrained, and `milestone` / `assignee` do not have - * a comparable near-miss form. + * `repo` is probed first. `pathPrefix` is then probed inside that repository only + * when the repository exists, so a bad repo does not falsely blame a path that may + * be valid elsewhere. `state` / `type` are enum-constrained, and the remaining + * filters do not have a comparable near-miss form. */ export async function detectUnmatchedFilters( db: D1Database, - filters: { repo?: string }, + filters: { repo?: string; pathPrefix?: string }, candidateCount: number, ): Promise { const unmatched: string[] = []; if (candidateCount > 0) return unmatched; + let repoMatched = true; if (filters.repo) { try { - if (!(await repoHasIndexedRows(db, filters.repo))) unmatched.push("repo"); + repoMatched = await repoHasIndexedRows(db, filters.repo); + if (!repoMatched) unmatched.push("repo"); } catch (err) { + repoMatched = false; console.error( "detectUnmatchedFilters: repo probe failed:", err instanceof Error ? err.message : String(err), ); } } + if (filters.pathPrefix && repoMatched) { + try { + if (!(await docPathPrefixHasIndexedRows(db, filters.pathPrefix, filters.repo))) { + unmatched.push("path_prefix"); + } + } catch (err) { + console.error( + "detectUnmatchedFilters: path_prefix probe failed:", + err instanceof Error ? err.message : String(err), + ); + } + } return unmatched; } @@ -340,6 +375,7 @@ export interface FtsFilter { type?: VectorMetadata["type"]; state?: "open" | "closed" | "published" | "active"; milestone?: string; + pathPrefix?: string; } /** @@ -395,6 +431,11 @@ export async function queryFts( whereClauses.push("d.milestone = ?"); params.push(filter.milestone); } + if (filter?.pathPrefix) { + const { lower, upper } = pathPrefixRange(filter.pathPrefix); + whereClauses.push("d.doc_path >= ? AND d.doc_path < ?"); + params.push(lower, upper); + } const whereSql = whereClauses.length > 0 ? ` AND ${whereClauses.join(" AND ")}` : ""; diff --git a/src/fts.workers.test.ts b/src/fts.workers.test.ts index 045a793..ca432de 100644 --- a/src/fts.workers.test.ts +++ b/src/fts.workers.test.ts @@ -4,6 +4,7 @@ import { upsertFtsRow, queryFts, repoHasIndexedRows, + docPathPrefixHasIndexedRows, deleteFtsRow, backfillNatSegments, tokenizerKindForType, @@ -588,6 +589,37 @@ describe("fts D1: structured filters", () => { const unfiltered = await queryFts(env.DB_FTS, "marker", 10, { repo }); expect(unfiltered.map((h) => h.vectorId).sort()).toEqual(["d:wt-doc", "w:wt-wiki"]); }); + + it("pre-filters doc candidates by repository-relative path prefix", async () => { + const repo = "t/doc-path-prefix"; + await upsertFtsRow( + env.DB_FTS, + mkRow({ + vectorId: "d:path-in", + type: "doc", + repo, + docPath: "benchmarks/parity-v4/inside.md", + content: "shared parity marker", + }), + ); + await upsertFtsRow( + env.DB_FTS, + mkRow({ + vectorId: "d:path-out", + type: "doc", + repo, + docPath: "docs/outside.md", + content: "shared parity marker", + }), + ); + + const hits = await queryFts(env.DB_FTS, "marker", 10, { + repo, + type: "doc", + pathPrefix: "benchmarks/parity-v4/", + }); + expect(hits.map((h) => h.vectorId)).toEqual(["d:path-in"]); + }); }); // Issue #219: an unmatched `repo` filter and a genuine zero-hit search produce @@ -626,6 +658,36 @@ describe("fts D1: repoHasIndexedRows (filters_unmatched probe)", () => { }); }); +describe("fts D1: docPathPrefixHasIndexedRows", () => { + it("checks the prefix inside the selected repository", async () => { + const repo = "t/path-probe"; + await upsertFtsRow( + env.DB_FTS, + mkRow({ + vectorId: "d:path-probe", + type: "doc", + repo, + docPath: "benchmarks/parity-v4/one.md", + content: "probe body", + }), + ); + + expect( + await docPathPrefixHasIndexedRows(env.DB_FTS, "benchmarks/parity-v4/", repo), + ).toBe(true); + expect(await docPathPrefixHasIndexedRows(env.DB_FTS, "benchmarks/missing/", repo)).toBe( + false, + ); + expect( + await docPathPrefixHasIndexedRows( + env.DB_FTS, + "benchmarks/parity-v4/", + "t/other-repo", + ), + ).toBe(false); + }); +}); + describe("fts D1: queryFts edge cases", () => { it("returns [] for an empty / whitespace query (no MATCH)", async () => { const repo = "t/empty"; diff --git a/src/mcp-stateless-contract.test.ts b/src/mcp-stateless-contract.test.ts index 9c2a59b..3272d5b 100644 --- a/src/mcp-stateless-contract.test.ts +++ b/src/mcp-stateless-contract.test.ts @@ -130,6 +130,7 @@ describe("worker <-> bridge stateless contract", () => { "include_content", "labels", "milestone", + "path_prefix", "query", "repo", "rerank", diff --git a/src/mcp.ts b/src/mcp.ts index 1a1e191..59ad7d1 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -59,6 +59,7 @@ import { FETCH_MAX_VECTOR_IDS, } from "./fetch.js"; import { entityKey, groupByEntity } from "./aggregate.js"; +import { pathPrefixRange, validateDocPathPrefix } from "./path-prefix.js"; const GITHUB_API = "https://api.github.com"; const USER_AGENT = "github-rag-mcp/0.1.0"; @@ -315,7 +316,7 @@ export function createRagMcpServer(env: Env): McpServer { "diff body without a second round trip to GitHub. Served from D1, so it makes no GitHub API call; what it " + "returns is the indexed copy of the body, truncated at " + `${FETCH_CONTENT_MAX_CHARS} characters, not the live source.\n` + - "Optional metadata filters (repo, state, labels, milestone, assignee, type) apply across modes 1-3 " + + "Optional metadata filters (repo, path_prefix, state, labels, milestone, assignee, type) apply across modes 1-3 " + "(mode 4 names its rows, so nothing is filtered there); " + "repo takes the full slug (owner/repo) and matches exactly, so a bare repository name selects nothing. " + "In search mode the response carries filters_unmatched: any filter listed there matched no row in the " + @@ -356,6 +357,14 @@ export function createRagMcpServer(env: Env): McpServer { "A bare repository name (\"my-repo\") matches nothing and yields an empty result set; " + "search mode flags that case as \"repo\" in the response's filters_unmatched.", ), + path_prefix: z + .string() + .optional() + .describe( + "Filter repository docs by a repository-relative directory prefix before ranking. " + + "Requires type=\"doc\", a trailing /, no leading /, backslash, NUL, empty, . or .. path segments, " + + "and at most 64 UTF-8 bytes. Search mode reports an unmatched value as \"path_prefix\" in filters_unmatched.", + ), state: z .enum(["open", "closed", "all"]) .optional() @@ -505,6 +514,7 @@ export function createRagMcpServer(env: Env): McpServer { async ({ query, repo, + path_prefix, state, labels, milestone, @@ -554,6 +564,14 @@ export function createRagMcpServer(env: Env): McpServer { } } + const pathPrefixError = validateDocPathPrefix(path_prefix, type); + if (pathPrefixError) { + return { + content: [{ type: "text" as const, text: pathPrefixError }], + isError: true, + }; + } + const requestedTopK = top_k ?? 10; const fusionMode = fusion ?? "rrf"; const rerankEnabled = rerank ?? true; @@ -571,6 +589,7 @@ export function createRagMcpServer(env: Env): McpServer { if (isScanMode) { const scan = await runScan(getStore(env), { repo, + pathPrefix: path_prefix, state, labels, milestone, @@ -650,6 +669,10 @@ export function createRagMcpServer(env: Env): McpServer { if (state && state !== "all") filter["state"] = { $eq: state }; if (type && type !== "all") filter["type"] = { $eq: type }; if (milestone) filter["milestone"] = { $eq: milestone }; + if (path_prefix) { + const { lower, upper } = pathPrefixRange(path_prefix); + filter["doc_path"] = { $gte: lower, $lt: upper }; + } const vectorizeFilter: VectorizeVectorMetadataFilter | undefined = Object.keys(filter).length > 0 ? filter : undefined; @@ -683,6 +706,7 @@ export function createRagMcpServer(env: Env): McpServer { ftsFilter.type = type as FtsFilter["type"]; } if (milestone) ftsFilter.milestone = milestone; + if (path_prefix) ftsFilter.pathPrefix = path_prefix; try { return await queryFts(env.DB_FTS, trimmedQuery, internalTopK, ftsFilter); } catch (err) { @@ -721,7 +745,7 @@ export function createRagMcpServer(env: Env): McpServer { // several repositories, which is a heavier change (issue #219 non-scope). const filtersUnmatched = await detectUnmatchedFilters( env.DB_FTS, - { repo }, + { repo, pathPrefix: path_prefix }, denseResult.hits.length + sparseHits.length, ); diff --git a/src/path-prefix.test.ts b/src/path-prefix.test.ts new file mode 100644 index 0000000..fa96ce1 --- /dev/null +++ b/src/path-prefix.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { + MAX_PATH_PREFIX_UTF8_BYTES, + pathPrefixRange, + validateDocPathPrefix, +} from "./path-prefix.js"; + +describe("document path-prefix contract", () => { + it("accepts a repository-relative directory only for doc searches", () => { + expect(validateDocPathPrefix("benchmarks/parity-v4/", "doc")).toBeNull(); + expect(validateDocPathPrefix(undefined, "all")).toBeNull(); + expect(validateDocPathPrefix("benchmarks/", "all")).toMatch(/type="doc"/); + expect(validateDocPathPrefix("benchmarks/", undefined)).toMatch(/type="doc"/); + }); + + it.each([ + ["", /empty/], + ["/benchmarks/", /repository-relative/], + ["benchmarks", /end with/], + ["benchmarks\\parity/", /separators/], + ["benchmarks//parity/", /empty/], + ["benchmarks/./parity/", /\. or \.\./], + ["benchmarks/../parity/", /\. or \.\./], + ["benchmarks/\0parity/", /NUL/], + ])("rejects invalid prefix %j", (prefix, message) => { + expect(validateDocPathPrefix(prefix, "doc")).toMatch(message); + }); + + it("measures the platform limit in UTF-8 bytes, not JavaScript characters", () => { + const exact = `${"a".repeat(MAX_PATH_PREFIX_UTF8_BYTES - 1)}/`; + const over = `${"あ".repeat(22)}/`; + expect(validateDocPathPrefix(exact, "doc")).toBeNull(); + expect(validateDocPathPrefix(over, "doc")).toMatch(/64 UTF-8 bytes/); + }); + + it("builds a half-open range that includes every suffix but excludes siblings", () => { + const { lower, upper } = pathPrefixRange("benchmarks/parity/"); + expect(lower).toBe("benchmarks/parity/"); + expect(upper).toBe("benchmarks/parity0"); + for (const path of [ + "benchmarks/parity/a.md", + "benchmarks/parity/あ.md", + "benchmarks/parity/😀.md", + ]) { + expect(path >= lower && path < upper).toBe(true); + } + expect("benchmarks/parity-other/a.md" >= lower && "benchmarks/parity-other/a.md" < upper).toBe(false); + }); +}); diff --git a/src/path-prefix.ts b/src/path-prefix.ts new file mode 100644 index 0000000..eb43b71 --- /dev/null +++ b/src/path-prefix.ts @@ -0,0 +1,44 @@ +/** Maximum string prefix Vectorize keeps in one metadata index entry. */ +export const MAX_PATH_PREFIX_UTF8_BYTES = 64; + +/** + * Validate the intentionally narrow document-directory filter contract. + * Fetch mode skips this check because named rows ignore every metadata filter. + */ +export function validateDocPathPrefix( + pathPrefix: string | undefined, + type: string | undefined, +): string | null { + if (pathPrefix === undefined) return null; + if (type !== "doc") return 'path_prefix requires type="doc"'; + if (pathPrefix.length === 0) return "path_prefix must not be empty"; + if (pathPrefix.startsWith("/")) { + return "path_prefix must be repository-relative and must not start with /"; + } + if (!pathPrefix.endsWith("/")) return "path_prefix must end with /"; + if (pathPrefix.includes("\\")) return "path_prefix must use / separators"; + if (pathPrefix.includes("\0")) return "path_prefix must not contain NUL"; + + const segments = pathPrefix.slice(0, -1).split("/"); + if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + return "path_prefix must not contain empty, . or .. path segments"; + } + + if (new TextEncoder().encode(pathPrefix).length > MAX_PATH_PREFIX_UTF8_BYTES) { + return `path_prefix must be at most ${MAX_PATH_PREFIX_UTF8_BYTES} UTF-8 bytes`; + } + return null; +} + +/** + * Half-open lexical range containing exactly the strings that start with a + * validated directory prefix. The contract requires a trailing slash, so its + * immediate successor (`/` -> `0`) is a safe upper bound for every suffix, + * including supplementary-plane Unicode characters. + */ +export function pathPrefixRange(pathPrefix: string): { lower: string; upper: string } { + return { + lower: pathPrefix, + upper: `${pathPrefix.slice(0, -1)}0`, + }; +} diff --git a/src/scan.ts b/src/scan.ts index aa2859e..8dc1623 100644 --- a/src/scan.ts +++ b/src/scan.ts @@ -79,6 +79,7 @@ export interface ScanStore { export interface ScanParams { repo?: string; + pathPrefix?: string; state?: string; labels?: string[]; milestone?: string; @@ -141,12 +142,13 @@ export async function runScan( // starving the final page. const storeLimit = Math.min(topK * 5, STORE_ROW_CAP); - const buildParams = (): URLSearchParams => { + const buildParams = (pathPrefix?: string): URLSearchParams => { const p = new URLSearchParams(); p.set("since", since); if (until) p.set("until", until); p.set("limit", String(storeLimit)); if (params.repo) p.set("repo", params.repo); + if (pathPrefix) p.set("path_prefix", pathPrefix); return p; }; @@ -156,10 +158,11 @@ export async function runScan( const collect = async ( path: string, map: (record: T) => ScanRow | null, + pathPrefix?: string, ): Promise => { try { const res = await store.fetch( - new Request(`http://store/${path}?${buildParams().toString()}`), + new Request(`http://store/${path}?${buildParams(pathPrefix).toString()}`), ); if (!res.ok) return; const records = (await res.json()) as T[]; @@ -230,7 +233,7 @@ export async function runScan( updated_at: d.updatedAt, created_at: d.updatedAt, doc_path: d.path, - })); + }), params.pathPrefix); } if (wantType("wiki_doc")) { @@ -346,6 +349,12 @@ export async function runScan( if (labels && labels.length > 0) { filtered = filtered.filter((r) => labels.every((l) => r.labels.includes(l))); } + if (params.pathPrefix) { + const prefix = params.pathPrefix; + filtered = filtered.filter( + (r) => r.type === "doc" && r.doc_path?.startsWith(prefix) === true, + ); + } // Time sort. "created_desc" sorts by created_at; "updated_desc" (the scan // default) sorts by updated_at. "relevance" has no meaning here and falls diff --git a/src/scan.workers.test.ts b/src/scan.workers.test.ts index 7a8143f..5714c21 100644 --- a/src/scan.workers.test.ts +++ b/src/scan.workers.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from "vitest"; import { env, runInDurableObject } from "cloudflare:test"; import type { IssueStore } from "./store.js"; -import type { DiffRecord } from "./types.js"; +import type { DiffRecord, DocRecord } from "./types.js"; import { runScan } from "./scan.js"; /** @@ -218,3 +218,45 @@ describe("scan mode: default window", () => { ]); }); }); + +describe("scan mode: document path prefix", () => { + it("pushes the prefix below the recency cap instead of starving matching docs", async () => { + const stub = env.ISSUE_STORE.get(env.ISSUE_STORE.idFromName("scan-doc-path-prefix")); + await runInDurableObject(stub, (s: IssueStore) => { + for (let i = 0; i < 150; i++) { + const outside: DocRecord = { + repo: REPO, + path: `docs/outside-${i}.md`, + blobSha: `outside-${i}`, + updatedAt: "2026-08-02T00:00:00Z", + }; + s.upsertDoc(outside); + } + for (let i = 0; i < 3; i++) { + const inside: DocRecord = { + repo: REPO, + path: `benchmarks/parity-v4/inside-${i}.md`, + blobSha: `inside-${i}`, + updatedAt: "2026-08-01T00:00:00Z", + }; + s.upsertDoc(inside); + } + }); + + const out = await runScan(stub, { + repo: REPO, + pathPrefix: "benchmarks/parity-v4/", + type: "doc", + topK: 10, + sort: "updated_desc", + since: "2026-08-01T00:00:00Z", + until: "2026-08-03T00:00:00Z", + }); + + expect(out.rows).toHaveLength(3); + expect(out.rows.every((row) => row.doc_path?.startsWith("benchmarks/parity-v4/"))).toBe( + true, + ); + expect(out.truncated).toBe(false); + }); +}); diff --git a/src/store.ts b/src/store.ts index 6a75421..e850bdc 100644 --- a/src/store.ts +++ b/src/store.ts @@ -18,6 +18,7 @@ import type { PRReviewCommentRecord, PollWatermark, } from "./types.js"; +import { pathPrefixRange } from "./path-prefix.js"; /** * Options shared by every `getRecent*` reader: a half-open time window @@ -31,6 +32,7 @@ export type RecentWindowOpts = { until?: string; limit?: number; repo?: string; + pathPrefix?: string; }; /** Row shape returned by SQLite for the issues table */ @@ -542,6 +544,7 @@ export class IssueStore implements DurableObject { table: string, timeCol: string, opts?: RecentWindowOpts, + pathColumn?: string, ): { query: string; params: (string | number)[] } { const limit = opts?.limit ?? 20; const since = @@ -553,6 +556,11 @@ export class IssueStore implements DurableObject { conditions.push(`repo = ?`); params.push(opts.repo); } + if (opts?.pathPrefix && pathColumn) { + const { lower, upper } = pathPrefixRange(opts.pathPrefix); + conditions.push(`${pathColumn} >= ? AND ${pathColumn} < ?`); + params.push(lower, upper); + } conditions.push(`${timeCol} >= ?`); params.push(since); if (opts?.until) { @@ -682,7 +690,7 @@ export class IssueStore implements DurableObject { } getRecentDocs(opts?: RecentWindowOpts): DocRecord[] { - const { query, params } = this.recentWindowQuery("docs", "updated_at", opts); + const { query, params } = this.recentWindowQuery("docs", "updated_at", opts, "path"); const cursor = this.sql.exec(query, ...params); return [...cursor].map(rowToDocRecord); } @@ -1101,6 +1109,7 @@ export class IssueStore implements DurableObject { until: url.searchParams.get("until") ?? undefined, limit: limit ? parseInt(limit, 10) : undefined, repo: url.searchParams.get("repo") ?? undefined, + pathPrefix: url.searchParams.get("path_prefix") ?? undefined, }; };