diff --git a/.mex/patterns/safe-graph-snapshot-evolution.md b/.mex/patterns/safe-graph-snapshot-evolution.md index f166f59a..0ad064c8 100644 --- a/.mex/patterns/safe-graph-snapshot-evolution.md +++ b/.mex/patterns/safe-graph-snapshot-evolution.md @@ -12,7 +12,7 @@ edges: condition: "when changing the graph data plane or its consumers" - target: "context/conventions.md" condition: "when verifying a graph implementation change" -last_updated: 2026-09-03 +last_updated: 2026-09-07 mex: id: mx_01M1M0CJP81C590FCKTSN5HA3Q type: pattern @@ -102,6 +102,32 @@ only to explicit maintenance workflows. cannot detect facts extracted from B; extraction must be bound to A. - Graph diagnostics and remediation commands must be truthful. Do not recommend a command for a state it cannot safely repair. +- A bounded limit is not one kind of thing. A **per-file** ceiling means one + file cannot be parsed and every other file still can, so it must skip and + report; a **corpus-wide** ceiling means the run has no honest partial answer + and must abort. Conflating them lets one pathological file take a whole + repository's graph with it. Classify the breach, never the fact of one. +- Do the skipping at the single discovery seam. Indexing, publication + verification, sync's corpus comparison and freshness inspection must all agree + about which files exist, or a file skipped by one and expected by another + makes the index permanently unable to read `fresh`. +- Never infer which limit was breached by comparing byte values. Two limits can + hold the same number, and then the comparison silently reports the wrong one + forever. Pass the limit's name. +- A containment guard has two separable decisions: whether to decline, and what + declining does. Read paths already treated an out-of-root source as a soft + miss while config paths threw on the identical condition — an inconsistency + inside one class that cost whole repositories their graph. An existence probe + is not a read request: the honest answer for a path we will not read is + "absent", recorded as a diagnostic so the degradation is visible. +- Report a declined out-of-root path by dependency specifier, not absolutely. + The compiler resolves a bare specifier by walking every ancestor directory, so + one unresolvable name yields a dozen near-identical entries carrying absolute + paths from outside the repository into product output. +- Any per-repository discovery input — a configured ignore list included — must + enter the corpus policy hash, or changing it leaves a stale index silently + describing files that are no longer in the corpus. Hash to the existing + constant when nothing is configured, so existing indexes stay valid. - Wall-clock status timings vary by machine and process-start overhead. Keep the benchmark non-gating, record its environment, and protect correctness with deterministic race, non-mutation, and bounded-work tests. diff --git a/README.es.md b/README.es.md index 035a5ff1..94d25682 100644 --- a/README.es.md +++ b/README.es.md @@ -424,5 +424,6 @@ MEX mantiene la memoria del equipo en archivos del repositorio y proporciona flu - Consulta la [guía de entorno de ejecución y compatibilidad](https://github.com/mex-memory/mex/blob/v0.8.0/COMPATIBILITY.md) y la [política de seguridad](https://github.com/mex-memory/mex/blob/v0.8.0/SECURITY.md). - Revisa la [matriz de compatibilidad del Code Graph](https://github.com/mex-memory/mex/blob/v0.8.0/docs/code-graph-support.md). - Consulta el [modelo de extractores y las relaciones compatibles](https://github.com/mex-memory/mex/blob/v0.8.0/docs/extractors.md). +- Lee los [resultados del benchmark de recuperación del Code Graph](https://github.com/mex-memory/mex/blob/v0.8.0/evaluate/RESULTS.md), incluida la comparación evaluada a ciegas frente a una búsqueda de archivos convencional. - Inspecciona la CLI localmente con `mex capabilities --json` y `mex commands`. - Únete a la [comunidad de MEX en Discord](https://discord.gg/FEdNsQ4Qt4) o visita [mexmemory.com](https://mexmemory.com). diff --git a/README.md b/README.md index d010d043..15f060e2 100644 --- a/README.md +++ b/README.md @@ -385,5 +385,6 @@ MEX keeps team memory in repository files and provides local retrieval and revie - Check the [runtime and compatibility guide](https://github.com/mex-memory/mex/blob/v0.8.0/COMPATIBILITY.md) and [security policy](https://github.com/mex-memory/mex/blob/v0.8.0/SECURITY.md). - Review the [Code Graph support matrix](https://github.com/mex-memory/mex/blob/v0.8.0/docs/code-graph-support.md). - See the [extractor model and supported relationships](https://github.com/mex-memory/mex/blob/v0.8.0/docs/extractors.md). +- Read the [graph retrieval benchmark results](https://github.com/mex-memory/mex/blob/v0.8.0/evaluate/RESULTS.md), including the blind-graded comparison against an ordinary file-search baseline. - Inspect the CLI locally with `mex capabilities --json` and `mex commands`. - Join the [MEX community on Discord](https://discord.gg/FEdNsQ4Qt4) or visit [mexmemory.com](https://mexmemory.com). diff --git a/README.pt-BR.md b/README.pt-BR.md index f546bac9..1865685c 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -424,5 +424,6 @@ O MEX mantém a memória da equipe em arquivos do repositório e oferece fluxos - Consulte o [guia de ambiente de execução e compatibilidade](https://github.com/mex-memory/mex/blob/v0.8.0/COMPATIBILITY.md) e a [política de segurança](https://github.com/mex-memory/mex/blob/v0.8.0/SECURITY.md). - Veja a [matriz de suporte do Code Graph](https://github.com/mex-memory/mex/blob/v0.8.0/docs/code-graph-support.md). - Consulte o [modelo dos extratores e as relações suportadas](https://github.com/mex-memory/mex/blob/v0.8.0/docs/extractors.md). +- Leia os [resultados do benchmark de recuperação do Code Graph](https://github.com/mex-memory/mex/blob/v0.8.0/evaluate/RESULTS.md), incluindo a comparação avaliada às cegas com uma busca de arquivos comum. - Examine a CLI localmente com `mex capabilities --json` e `mex commands`. - Entre na [comunidade do MEX no Discord](https://discord.gg/FEdNsQ4Qt4) ou visite [mexmemory.com](https://mexmemory.com). diff --git a/README.zh-CN.md b/README.zh-CN.md index 92c34f1f..9bf454cf 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -424,5 +424,6 @@ MEX 将团队记忆保存在仓库文件中，并提供本地检索和审阅工 - 查看[运行时与兼容性指南](https://github.com/mex-memory/mex/blob/v0.8.0/COMPATIBILITY.md)和[安全政策](https://github.com/mex-memory/mex/blob/v0.8.0/SECURITY.md)。 - 查阅 [Code Graph 支持矩阵](https://github.com/mex-memory/mex/blob/v0.8.0/docs/code-graph-support.md)。 - 了解[提取器模型和支持的关系](https://github.com/mex-memory/mex/blob/v0.8.0/docs/extractors.md)。 +- 阅读 [Code Graph 检索基准结果](https://github.com/mex-memory/mex/blob/v0.8.0/evaluate/RESULTS.md)，其中包含与普通文件搜索基线的盲评对比。 - 在本地使用 `mex capabilities --json` 和 `mex commands` 检查 CLI。 - 加入 [Discord 上的 MEX 社区](https://discord.gg/FEdNsQ4Qt4)，或访问 [mexmemory.com](https://mexmemory.com)。 diff --git a/docs/code-graph-support.md b/docs/code-graph-support.md index a0235a83..ff590738 100644 --- a/docs/code-graph-support.md +++ b/docs/code-graph-support.md @@ -127,6 +127,100 @@ legacy checks running” case in Unsupported source-language files are also skipped. A missing extractor does not make the rest of setup or drift checking fail. +### Files the corpus policy will not index + +The graph applies a bounded per-file size ceiling (2 MB) so one pathological +file cannot exhaust memory. A file over that ceiling is **skipped, not fatal**: +the rest of the repository is indexed normally, and the skipped files are +reported by name, size and limit in `mex graph` output and in the `skipped` +array of its `--json` result. + +Corpus-*wide* ceilings still abort the run. They describe the whole build and +there is no honest partial answer to "this repository is too large to index +within the bounded policy". + +### Config inputs outside the project + +mex never reads a file outside the repository root, and a TypeScript config +routinely points at one: `"extends": "some-package/tsconfig"` resolves through +`node_modules`, which any hoisted pnpm/yarn layout — or a monorepo sub-package +indexed on its own — places above the indexed root. + +Such an input is **declined, not fatal**. The build finishes, the affected +project's type resolution is less complete than its config asks for, and the +declined inputs are reported by dependency specifier (never by absolute path) +in `mex graph` output and in the `declinedInputs` array of its `--json` result. +The same applies to a `tsconfig` `include` or project `reference` that points +above the root. + +The containment guard itself is unchanged: nothing outside the root is read, +and nothing outside the root enters the graph's provenance. + +### Excluding paths from the graph + +`node_modules`, `.git`, `dist`, `build`, `.mex`, `coverage`, `.next` and `out` +are always excluded. A repository can exclude more by listing globs under +`graph.ignore` in `.mex/config.json`: + +```json +{ + "graph": { + "ignore": ["vendor/**", "**/*.generated.ts"] + } +} +``` + +The list is **additive**: configured globs are appended to the built-in ones +and cannot un-ignore them, so `node_modules` and `.mex` stay excluded whatever +the configuration says. Globs are repository-relative; absolute paths and +upward traversal are ignored, and the list is bounded. A missing or malformed +config simply contributes no extra globs rather than failing a build. + +Changing this list changes which files the graph describes, so it changes the +build manifest and the next `mex graph status` will report the index as stale +until it is rebuilt. + +## Unresolved references + +Extraction records every reference it sees. The resolver then binds what it +can to a declaration and emits an edge; what it cannot bind stays recorded as +an unresolved reference. Those records are the graph being honest about its own +blind spots: a name a file referenced, that the resolver could not decide the +meaning of. + +They matter for `who-calls`. A dynamically generated method has real call sites +and no literal declaration, so no node resolves and the structural answer is +"not found" — accurate, and useless as a next step. When `who-calls` cannot +resolve its target, it now looks the name up among the recorded unresolved +references and reports those call sites: + +```bash +mex graph query who-calls mark_failed +``` + +```json +{"type":"unresolved-reference","relation":"who-calls","target":"mark_failed", + "name":"mark_failed","referenceKind":"calls","resolution":"unresolved", + "file":"app/models/job.rb","line":42,"col":8,"fromNode":"function:…", + "receiver":"job"} +``` + +Three properties of that output are deliberate: + +- **It is not a `result` record.** An unresolved reference is not a resolved + graph fact and an agent must not be able to confuse the two, so it carries + its own record type. +- **It is capped and charged to the same output budget** as every other + response. Common names accumulate hundreds of unresolved references, and an + uncapped fallback on a hot name would flood the caller. The `summary` reports + the total that matched alongside what was returned. +- **The response is a normal one**, with `meta` and `summary`, and a `summary` + whose `status` is `partial` and `evidenceStrength` is `weak`. + +A name with no declaration *and* no recorded reference still abstains with +`TARGET_NOT_FOUND`. `where-defined` and `what-calls` are unchanged: they +either resolve the requested declaration exactly or abstain. + ## Known limitations - **Ambiguous references stay unresolved.** The base resolver prefers a @@ -139,10 +233,9 @@ not make the rest of setup or drift checking fail. reflection, dependency injection, monkey-patching, or computed calls. - **Generated code is path-filtered, not identified semantically.** Common output trees such as `node_modules`, `dist`, `build`, `.next`, `out`, - `coverage`, and `.mex` are excluded by the source globs in - [`engine-impl.ts`](../src/graph/engine-impl.ts) and - [`runtime.ts`](../src/graph/runtime.ts). Generated files outside those paths - may still be indexed. + `coverage`, and `.mex` are excluded by the corpus policy in + [`corpus-policy.ts`](../src/graph/corpus-policy.ts). Generated files outside + those paths may still be indexed; add a `graph.ignore` glob to exclude them. - **Framework behavior is opt-in and narrow.** Express route-to-handler binding is the only framework fixture in v0.7.0. Other frameworks remain unsupported until their language extractor and resolver work merges. diff --git a/evaluate/README.md b/evaluate/README.md index c1685e7a..91d0c942 100644 --- a/evaluate/README.md +++ b/evaluate/README.md @@ -13,6 +13,10 @@ The older compactness and scripted-agent scripts remain available as historical `npm run eval:legacy` and `npm run eval:e2e`. They are not evidence that natural-language graph retrieval works. +Measured results from these harnesses are written up in +[`RESULTS.md`](RESULTS.md), including the blind-graded headless comparison +against an ordinary file-search baseline. + ## Deterministic graph evaluation Build the CLI, then run the native MEX suite: diff --git a/src/graph/__tests__/compiler-containment.test.ts b/src/graph/__tests__/compiler-containment.test.ts new file mode 100644 index 00000000..b9877c26 --- /dev/null +++ b/src/graph/__tests__/compiler-containment.test.ts @@ -0,0 +1,102 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildTypeScriptExtraction } from "../extraction/compiler.js"; +import { createGraphEngine } from "../engine-impl.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +/** A project nested one level down, so `..` is a real directory outside it. */ +function nestedProject(): { workspace: string; root: string } { + const workspace = mkdtempSync(join(tmpdir(), "mex-graph-containment-")); + roots.push(workspace); + const root = join(workspace, "package"); + mkdirSync(root, { recursive: true }); + return { workspace, root }; +} + +function write(root: string, path: string, source: string): void { + const absolute = join(root, path); + mkdirSync(dirname(absolute), { recursive: true }); + writeFileSync(absolute, source, "utf8"); +} + +describe("compiler input containment declines rather than aborting", () => { + it("finishes a build whose tsconfig extends a package hoisted above the root", async () => { + const { workspace, root } = nestedProject(); + // The hoisted layout every pnpm/yarn workspace produces: the package the + // config extends resolves above the indexed root. + write(workspace, "node_modules/shared-config/tsconfig.json", + JSON.stringify({ compilerOptions: { strict: true } })); + write(workspace, "node_modules/shared-config/package.json", + JSON.stringify({ name: "shared-config", version: "1.0.0" })); + write(root, "tsconfig.json", JSON.stringify({ + extends: "shared-config/tsconfig.json", + compilerOptions: { target: "ES2022" }, + })); + write(root, "package.json", JSON.stringify({ name: "package", version: "1.0.0" })); + write(root, "src/service.ts", "export function containedPrimary(): number { return 1; }\n"); + const engine = createGraphEngine({ rootDir: root }); + + try { + const result = await engine.build(); + + expect(result.filesIndexed).toBe(1); + // The config was declined, not read — and reported by dependency + // specifier rather than by absolute path. + expect(result.declinedInputs).toContainEqual(expect.objectContaining({ + filePath: "node_modules/shared-config/tsconfig.json", + reason: "outside-project-corpus", + })); + for (const input of result.declinedInputs ?? []) { + expect(input.filePath.startsWith("node_modules/")).toBe(true); + } + expect(engine.searchNodes("containedPrimary").length).toBeGreaterThan(0); + } finally { + engine.close(); + } + }, 60_000); + + it("declines an include and a project reference above the root without throwing", () => { + const { workspace, root } = nestedProject(); + write(workspace, "sibling/tsconfig.json", JSON.stringify({ compilerOptions: {} })); + write(workspace, "sibling/src/other.ts", "export const other = 1;\n"); + write(root, "tsconfig.json", JSON.stringify({ + compilerOptions: { composite: true }, + include: ["src/**/*", "../sibling/src/**/*"], + references: [{ path: "../sibling" }], + })); + write(root, "src/service.ts", "export function referencedPrimary(): number { return 1; }\n"); + + const result = buildTypeScriptExtraction(root, ["src/service.ts"], { + stagedInputs: [ + { filePath: "tsconfig.json", source: JSON.stringify({ + compilerOptions: { composite: true }, + include: ["src/**/*", "../sibling/src/**/*"], + references: [{ path: "../sibling" }], + }) }, + { filePath: "src/service.ts", + source: "export function referencedPrimary(): number { return 1; }\n" }, + ], + }); + + expect(result.files.map((file) => file.filePath)).toEqual(["src/service.ts"]); + // The referenced project above the root is declined by path rather than + // traversed. (The `..` include is not declined here: TypeScript invokes + // readDirectory with the config's own directory and applies the include + // patterns inside its matcher, so that guard is reached only when a config + // names an out-of-root directory as the matcher root.) + expect(result.declinedInputs.map((input) => input.filePath)) + .toEqual(["../sibling/tsconfig.json"]); + expect(result.declinedInputs[0]?.reason).toBe("outside-project-corpus"); + // Nothing outside the root reached the graph's provenance. + for (const input of result.semanticInputs) { + expect(input.filePath.startsWith("..")).toBe(false); + } + }); +}); diff --git a/src/graph/__tests__/corpus-policy.test.ts b/src/graph/__tests__/corpus-policy.test.ts index f873e14a..4744d8d7 100644 --- a/src/graph/__tests__/corpus-policy.test.ts +++ b/src/graph/__tests__/corpus-policy.test.ts @@ -4,8 +4,15 @@ import { dirname, join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { GRAPH_CORPUS_GLOB_OPTIONS, + GRAPH_CORPUS_IGNORE_GLOBS, GRAPH_CORPUS_LIMITS, + GRAPH_CORPUS_POLICY_HASH, + GRAPH_IGNORE_CONFIG_LIMITS, GraphCorpusLimitError, + graphCorpusIgnoreGlobs, + graphCorpusPolicyHash, + isPerFileCorpusLimitError, + readConfiguredGraphIgnoreGlobs, addGraphCompilerSourceBytes, addGraphCorpusBytes, addGraphSemanticInput, @@ -88,3 +95,127 @@ describe("graph corpus policy", () => { expect(coveredProbeLedger.semanticPaths.size).toBe(0); }); }); + +describe("per-file corpus limits", () => { + it("separates a single-file ceiling from a corpus-wide one", () => { + const perFile = new GraphCorpusLimitError("maxSourceFileBytes", 34_380_944); + const corpusWide = new GraphCorpusLimitError("maxSourceBytes"); + + expect(perFile.perFile).toBe(true); + expect(isPerFileCorpusLimitError(perFile)).toBe(true); + expect(corpusWide.perFile).toBe(false); + expect(isPerFileCorpusLimitError(corpusWide)).toBe(false); + expect(isPerFileCorpusLimitError(new Error("unrelated"))).toBe(false); + }); + + it("names the observed size and the limit on the first line", () => { + // The source and config per-file ceilings are numerically identical, so + // the limit cannot be recovered by comparing byte values. + expect(GRAPH_CORPUS_LIMITS.maxSourceFileBytes) + .toBe(GRAPH_CORPUS_LIMITS.maxConfigFileBytes); + expect(new GraphCorpusLimitError("maxSourceFileBytes", 34_380_944).message) + .toBe("The graph corpus exceeded the configured maxSourceFileBytes safety bound: " + + "34380944 bytes against a 2097152-byte limit."); + expect(() => addGraphCorpusBytes(0, GRAPH_CORPUS_LIMITS.maxSourceFileBytes + 1, "source")) + .toThrow(/maxSourceFileBytes safety bound: 2097153 bytes/u); + }); +}); + +describe("configured graph ignore globs", () => { + function withConfig(body: unknown): string { + const root = mkdtempSync(join(tmpdir(), "mex-graph-ignore-config-")); + roots.push(root); + mkdirSync(join(root, ".mex"), { recursive: true }); + writeFileSync( + join(root, ".mex", "config.json"), + typeof body === "string" ? body : JSON.stringify(body), + "utf8", + ); + return root; + } + + it("appends configured globs to the frozen defaults, never replacing them", () => { + const root = withConfig({ graph: { ignore: ["generated/**", "**/*.gen.ts"] } }); + + expect(readConfiguredGraphIgnoreGlobs(root)).toEqual(["**/*.gen.ts", "generated/**"]); + expect(graphCorpusIgnoreGlobs(root)).toEqual([ + ...GRAPH_CORPUS_IGNORE_GLOBS, + "**/*.gen.ts", + "generated/**", + ]); + }); + + it("cannot un-ignore a default, whatever the configuration says", () => { + const root = withConfig({ graph: { ignore: ["!**/node_modules/**", "!**/.mex/**"] } }); + + // Negations are appended like any other glob and cancel *earlier* patterns + // only in glob semantics we never rely on; the defaults still stand. + for (const glob of GRAPH_CORPUS_IGNORE_GLOBS) { + expect(graphCorpusIgnoreGlobs(root)).toContain(glob); + } + }); + + it("yields no extra globs rather than failing on a malformed or hostile config", () => { + expect(readConfiguredGraphIgnoreGlobs(withConfig("{ not json"))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs(withConfig([1, 2, 3]))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs(withConfig({ graph: "nope" }))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs(withConfig({ graph: { ignore: "nope" } }))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs(withConfig({ + graph: { ignore: [42, "", " "] }, + }))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs( + mkdtempSync(join(tmpdir(), "mex-graph-ignore-missing-")), + )).toEqual([]); + }); + + it("rejects every escaping glob identically on every platform", () => { + // `.mex/config.json` is tracked and travels with the repository, and these + // globs feed the corpus policy hash. A platform-dependent verdict — which + // `path.isAbsolute` gives, since `C:/x` is absolute only on Windows — would + // give one repository two manifest hashes and make its index read as stale + // purely from being opened on another machine. + const escaping = [ + "/absolute/**", + "//server/share/**", + "\\\\server\\share\\**", + "C:/absolute/**", + "c:/absolute/**", + "C:\\absolute\\**", + "C:relative/**", + "../escape/**", + "..\\escape\\**", + "vendor/../../escape/**", + "..", + ]; + const root = withConfig({ graph: { ignore: escaping } }); + + expect(readConfiguredGraphIgnoreGlobs(root)).toEqual([]); + expect(graphCorpusPolicyHash(root)).toBe(GRAPH_CORPUS_POLICY_HASH); + }); + + it("keeps ordinary globs that merely contain dots", () => { + const root = withConfig({ graph: { ignore: ["a..b/**", "**/*.min.js", "./local/**"] } }); + + expect(readConfiguredGraphIgnoreGlobs(root)) + .toEqual(["**/*.min.js", "./local/**", "a..b/**"]); + }); + + it("bounds the configured list", () => { + const tooMany = Array.from({ length: 500 }, (_, index) => `dir${index}/**`); + const tooLong = "x".repeat(GRAPH_IGNORE_CONFIG_LIMITS.maxGlobLength + 1); + const root = withConfig({ graph: { ignore: [...tooMany, tooLong] } }); + + const globs = readConfiguredGraphIgnoreGlobs(root); + expect(globs.length).toBe(GRAPH_IGNORE_CONFIG_LIMITS.maxGlobs); + expect(globs).not.toContain(tooLong); + }); + + it("keeps the discovery identity of a repository that configures nothing", () => { + const unconfigured = mkdtempSync(join(tmpdir(), "mex-graph-ignore-none-")); + roots.push(unconfigured); + const configured = withConfig({ graph: { ignore: ["generated/**"] } }); + + expect(graphCorpusPolicyHash(unconfigured)).toBe(GRAPH_CORPUS_POLICY_HASH); + expect(graphCorpusPolicyHash(configured)).not.toBe(GRAPH_CORPUS_POLICY_HASH); + }); +}); diff --git a/src/graph/__tests__/corpus-skip.test.ts b/src/graph/__tests__/corpus-skip.test.ts new file mode 100644 index 00000000..cae626fc --- /dev/null +++ b/src/graph/__tests__/corpus-skip.test.ts @@ -0,0 +1,94 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { GRAPH_CORPUS_LIMITS } from "../corpus-policy.js"; +import { createGraphEngine } from "../engine-impl.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function temporaryRoot(): string { + const root = mkdtempSync(join(tmpdir(), "mex-graph-corpus-skip-")); + roots.push(root); + return root; +} + +function write(root: string, path: string, source: string): void { + const absolute = join(root, path); + mkdirSync(dirname(absolute), { recursive: true }); + writeFileSync(absolute, source, "utf8"); +} + +/** A syntactically valid module far above the per-file ceiling. */ +function oversizedModule(): string { + const filler = `// ${"x".repeat(120)}\n`; + return `export const oversized = true;\n` + + filler.repeat(Math.ceil(GRAPH_CORPUS_LIMITS.maxSourceFileBytes / filler.length) + 1); +} + +describe("oversized files do not abort a graph build", () => { + it("indexes every other file and reports the one it skipped", async () => { + const root = temporaryRoot(); + write(root, "src/service.ts", "export function servicePrimary(): number { return 1; }\n"); + write(root, "src/other.ts", "export function otherPrimary(): number { return 2; }\n"); + write(root, "src/generated.ts", oversizedModule()); + const engine = createGraphEngine({ rootDir: root }); + + try { + const result = await engine.build(); + + expect(result.filesIndexed).toBe(2); + expect(result.skipped).toEqual([{ + filePath: "src/generated.ts", + reason: "corpus-limit", + limit: "maxSourceFileBytes", + limitBytes: GRAPH_CORPUS_LIMITS.maxSourceFileBytes, + observedBytes: expect.any(Number), + message: expect.stringContaining("maxSourceFileBytes safety bound:"), + }]); + expect(engine.getIndexedFiles?.().map((file) => file.path)) + .toEqual(["src/other.ts", "src/service.ts"]); + } finally { + engine.close(); + } + }, 60_000); + + it("does not treat a skipped file as a lost source on sync", async () => { + const root = temporaryRoot(); + write(root, "src/service.ts", "export function syncedPrimary(): number { return 1; }\n"); + write(root, "src/generated.ts", oversizedModule()); + const engine = createGraphEngine({ rootDir: root }); + + try { + await engine.build(); + // The skipped file is absent from the staged corpus by design, so naming + // it as changed must not read as a source that disappeared mid-build. + const result = await engine.sync(["src/generated.ts"]); + + expect(result.skipped?.map((file) => file.filePath)).toEqual(["src/generated.ts"]); + } finally { + engine.close(); + } + }, 60_000); + + it("honours an additive ignore glob configured by the repository", async () => { + const root = temporaryRoot(); + write(root, "src/service.ts", "export function keptPrimary(): number { return 1; }\n"); + write(root, "vendor/bundle.ts", "export function droppedPrimary(): number { return 2; }\n"); + write(root, ".mex/config.json", JSON.stringify({ graph: { ignore: ["vendor/**"] } })); + const engine = createGraphEngine({ rootDir: root }); + + try { + const result = await engine.build(); + + expect(result.filesIndexed).toBe(1); + expect(engine.getIndexedFiles?.().map((file) => file.path)).toEqual(["src/service.ts"]); + } finally { + engine.close(); + } + }, 60_000); +}); diff --git a/src/graph/__tests__/snapshot.test.ts b/src/graph/__tests__/snapshot.test.ts index 61d9ab00..0e005f70 100644 --- a/src/graph/__tests__/snapshot.test.ts +++ b/src/graph/__tests__/snapshot.test.ts @@ -369,7 +369,12 @@ describe("graph snapshot provenance", () => { engine.close(); }); - it("rejects a direct external tsconfig extension and preserves prior provenance", async () => { + // This asserted that an external `extends` aborted the build. Declining a + // config input no longer costs a repository its entire graph — a tsconfig + // extending a hoisted package is ordinary in any pnpm/yarn workspace. What + // this test actually protects, and still asserts, is that the external file + // is never read and never becomes graph provenance. + it("declines a direct external tsconfig extension without reading it", async () => { const root = temporaryRoot("mex-graph-config-direct-escape-"); const externalRoot = temporaryRoot("mex-graph-config-direct-external-"); const sourcePath = join(root, "src", "stable.ts"); @@ -388,14 +393,20 @@ describe("graph snapshot provenance", () => { extends: externalConfig, include: ["src/**/*.ts"], })); - await expect(engine.sync(["tsconfig.json"])).rejects.toMatchObject({ - name: "GraphSourceStagingError", - failures: [expect.objectContaining({ - filePath: ".", - code: "GRAPH_SOURCE_PATH_ESCAPE", - })], - }); - expect(metadata(dbPath, GRAPH_SNAPSHOT_METADATA_KEY)).toBe(successfulSnapshot); + const result = await engine.sync(["tsconfig.json"]); + + expect(result.declinedInputs).toContainEqual(expect.objectContaining({ + reason: "outside-project-corpus", + })); + // The external file is not read, so it is in no snapshot and no compiler + // option it declares can reach the graph. + const snapshot = parseGraphSnapshot(metadata(dbPath, GRAPH_SNAPSHOT_METADATA_KEY)); + expect(snapshot).not.toBeNull(); + expect(successfulSnapshot).not.toBeNull(); + for (const input of snapshot!.semanticInputs) { + expect(input.path.startsWith("..")).toBe(false); + expect(input.path).not.toContain("attacker"); + } engine.close(); }); diff --git a/src/graph/__tests__/status.test.ts b/src/graph/__tests__/status.test.ts index 38bdd925..d56236ee 100644 --- a/src/graph/__tests__/status.test.ts +++ b/src/graph/__tests__/status.test.ts @@ -155,21 +155,39 @@ describe("inspectGraphStatus", () => { expect(treeState(root)).toEqual(before); }); - it("preserves a path-free corpus-limit diagnostic when changed-path output is disabled", async () => { - const root = temporaryRoot("mex-graph-status-corpus-limit-"); + // One oversized file is a skipped file, not a corpus breach. This test used + // to assert the opposite — that a single oversized source raised the + // path-free GRAPH_SOURCE_CORPUS_LIMIT_EXCEEDED — which is the same defect + // that let one 34 MB generated file abort a 3,259-file build. The invariant + // that test actually guarded (a path-free corpus diagnostic survives when + // per-path diagnostics are suppressed) is covered by the config-corpus case + // immediately below, which still trips a genuine corpus-wide ceiling. + it("skips one oversized source file, as a bounded per-path diagnostic", async () => { + const root = temporaryRoot("mex-graph-status-oversized-file-"); + source(root, "src/service.ts", "export const service = true;\n"); source( root, "src/oversized.py", "x".repeat(GRAPH_CORPUS_LIMITS.maxSourceFileBytes + 1), ); - const status = await inspect(root, 0); + const suppressed = await inspect(root, 0); + const reported = await inspect(root, 10); - expect(status.diagnostics).toContainEqual({ - code: "GRAPH_SOURCE_CORPUS_LIMIT_EXCEEDED", + // The skip names a path, so it is bounded like every other path diagnostic. + expect(suppressed.diagnostics).not.toContainEqual( + expect.objectContaining({ code: "GRAPH_SOURCE_FILE_SKIPPED" }), + ); + expect(suppressed.diagnostics).not.toContainEqual( + expect.objectContaining({ code: "GRAPH_SOURCE_CORPUS_LIMIT_EXCEEDED" }), + ); + expect(reported.diagnostics).toContainEqual(expect.objectContaining({ + code: "GRAPH_SOURCE_FILE_SKIPPED", severity: "warning", - message: "The supported source corpus exceeds MEX's bounded inspection policy.", - }); + path: "src/oversized.py", + })); + // The rest of the repository is still observed: the walk did not stop. + expect(reported.changes.added).toEqual(["src/service.ts"]); }); it("classifies a bounded manifest refusal separately from generic inspection failure", async () => { diff --git a/src/graph/__tests__/unresolved-fallback.test.ts b/src/graph/__tests__/unresolved-fallback.test.ts new file mode 100644 index 00000000..e33b320d --- /dev/null +++ b/src/graph/__tests__/unresolved-fallback.test.ts @@ -0,0 +1,134 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runGraphQuery } from "../cli-agent.js"; +import { openSqlite } from "../db/sqlite.js"; +import { createGraphEngine } from "../engine-impl.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function write(root: string, path: string, source: string): void { + const absolute = join(root, path); + mkdirSync(dirname(absolute), { recursive: true }); + writeFileSync(absolute, source, "utf8"); +} + +/** + * A repository whose call sites reference a name no declaration provides — + * the shape a dynamically generated method leaves behind. + */ +async function repositoryWithDynamicCalls(): Promise<{ root: string; dbPath: string }> { + const root = mkdtempSync(join(tmpdir(), "mex-graph-unresolved-")); + roots.push(root); + const dbPath = join(root, "graph.db"); + // `any` receivers leave a real call site with no declaration to bind it to, + // exactly as a dynamically generated method does. + write(root, "src/first.ts", [ + "export function firstCaller(subject: any): void {", + " subject.markFailed();", + "}", + "", + ].join("\n")); + write(root, "src/second.ts", [ + "export function secondCaller(subject: any): void {", + " subject.markFailed();", + "}", + "", + ].join("\n")); + const engine = createGraphEngine({ rootDir: root, dbPath }); + try { + await engine.build(); + } finally { + engine.close(); + } + return { root, dbPath }; +} + +function query(root: string, dbPath: string, relation: string, target: string, options = {}) { + const output: string[] = []; + const db = openSqlite(`file:${dbPath.replaceAll("\\", "/")}?mode=ro&immutable=1`); + const engine = createGraphEngine({ rootDir: root, dbPath, readOnly: true }); + try { + runGraphQuery(relation, target, root, { + open: () => ({ graph: engine, db, close: () => {} }), + write: (line) => output.push(line), + }, options); + } finally { + engine.close(); + db.close(); + } + return output.map((line) => JSON.parse(line) as Record); +} + +describe("who-calls falls back to recorded unresolved references", () => { + it("reports call sites for a name with no indexed declaration", async () => { + const { root, dbPath } = await repositoryWithDynamicCalls(); + + const rows = query(root, dbPath, "who-calls", "markFailed"); + + // The protocol stays uniform: meta first, summary last, like every other + // response — not the bare error record this path used to emit. + expect(rows[0]).toMatchObject({ type: "meta" }); + expect(rows.at(-1)).toMatchObject({ type: "summary" }); + + const unresolved = rows.filter((row) => row.type === "unresolved-reference"); + expect(unresolved.length).toBeGreaterThan(0); + // Never `type: "result"`: an agent must not confuse an unbound name for a + // resolved graph fact. + expect(rows.filter((row) => row.type === "result")).toEqual([]); + for (const row of unresolved) { + expect(row).toMatchObject({ + relation: "who-calls", + target: "markFailed", + name: "markFailed", + }); + expect(typeof row.file).toBe("string"); + expect(typeof row.line).toBe("number"); + expect(typeof row.col).toBe("number"); + expect(typeof row.fromNode).toBe("string"); + } + expect(unresolved.map((row) => row.file)).toEqual(["src/first.ts", "src/second.ts"]); + + const summary = rows.at(-1)!; + expect(summary).toMatchObject({ status: "partial", evidenceStrength: "weak", returnedNodes: 0 }); + expect((summary.warnings as string[]).join(" ")).toContain("markFailed"); + expect((summary.suggestedNextCommands as string[])[0]).toContain("mex graph get"); + }, 60_000); + + it("caps the fallback and declares the truncation", async () => { + const { root, dbPath } = await repositoryWithDynamicCalls(); + + const rows = query(root, dbPath, "who-calls", "markFailed", { maxNodes: 1 }); + + expect(rows.filter((row) => row.type === "unresolved-reference")).toHaveLength(1); + const summary = rows.at(-1)!; + expect(summary.truncated).toBe(true); + // The cap bounds the answer, not the count of what exists. + expect(summary.matchedNodes).toBeGreaterThan(1); + }, 60_000); + + it("still abstains when the name appears nowhere at all", async () => { + const { root, dbPath } = await repositoryWithDynamicCalls(); + + const rows = query(root, dbPath, "who-calls", "definitelyNotAName"); + + expect(rows).toEqual([ + { type: "error", code: "TARGET_NOT_FOUND", target: "definitelyNotAName" }, + ]); + }, 60_000); + + it("leaves where-defined and what-calls abstention unchanged", async () => { + const { root, dbPath } = await repositoryWithDynamicCalls(); + + for (const relation of ["where-defined", "what-calls"]) { + expect(query(root, dbPath, relation, "markFailed")).toEqual([ + { type: "error", code: "TARGET_NOT_FOUND", target: "markFailed" }, + ]); + } + }, 60_000); +}); diff --git a/src/graph/cli-agent.ts b/src/graph/cli-agent.ts index aa8dd22d..57b957e1 100644 --- a/src/graph/cli-agent.ts +++ b/src/graph/cli-agent.ts @@ -162,6 +162,8 @@ export function runGraphQuery( return withAgentGraphSession(rootDir, deps, output, "fresh", (session, write) => { const nodes = resolveSymbol(session.graph, target); if (nodes.length === 0) { + if (relation === "who-calls" + && emitUnresolvedCallers(session, write, target, opts)) return; writeJson(write, { type: "error", code: "TARGET_NOT_FOUND", target }); return; } @@ -2079,6 +2081,121 @@ function transitiveCallers(graph: GraphEngine, root: GraphNode, maxDepth: number * underlying store. Older schemas are rejected by the immutable reader and * require an explicit rebuild before this query can run. */ +/** + * A call site the resolver captured but could not bind to a declaration. + * + * These are not graph facts. They record that some file referenced this name + * and the resolver could not decide what it meant. + */ +interface UnresolvedCallSite { + reference_name: string; + reference_kind: string; + status: string; + file_path: string; + line: number; + col: number; + from_node_id: string; + receiver: string | null; + qualifier: string | null; +} + +/** + * `who-calls` fallback for a name with call sites but no declaration. + * + * A dynamically generated method has real callers and no literal definition, + * so no node resolves and the honest structural answer is "not found" — which + * left an agent with no next step, while the call sites sat in + * `unresolved_refs` with no reader anywhere in the CLI. Recovering them meant + * hand-written SQL against internal schema. + * + * Returns false when there is nothing to offer, so the caller still abstains + * with TARGET_NOT_FOUND rather than emitting an empty alternative answer. + */ +function emitUnresolvedCallers( + session: AgentGraphSession, + write: (line: string) => void, + target: string, + opts: AgentOptions, +): boolean { + const matched = unresolvedCallSiteCount(session.db, target); + if (matched === 0) return false; + + // `limit` bounds the answer, not the work: hot names are common + // (`forwardRef`, `json`, `error`) and an uncapped fallback would flood the + // agent with hundreds of rows for one query. + const rows = unresolvedCallSites(session.db, target, opts.maxNodes); + const anticipated = rows.length > 0 + ? [`mex graph get ${rows[0]!.from_node_id} --detail source`] : []; + const ctx = beginResponse("graph query who-calls", opts, undefined, anticipated); + const records: Rec[] = []; + let truncated = matched > rows.length; + for (const row of rows) { + // A distinct record type, never `type: "result"`. An agent must not be + // able to mistake an unbound name for a resolved graph edge. + const record: Rec = { + type: "unresolved-reference", + relation: "who-calls", + target, + name: row.reference_name, + referenceKind: row.reference_kind, + resolution: row.status, + file: row.file_path, + line: row.line, + col: row.col, + fromNode: row.from_node_id, + ...(row.receiver === null ? {} : { receiver: row.receiver }), + ...(row.qualifier === null ? {} : { qualifier: row.qualifier }), + }; + if (!ctx.ledger.tryAdd(record)) { truncated = true; break; } + records.push(record); + } + + emitAll(write, ctx.meta, records); + write(JSON.stringify(summaryRecord(ctx, { + matchedNodes: matched, + // No node was returned: these are call sites, not declarations. + returnedNodes: 0, + returnedEdges: 0, + truncated, + status: "partial", + evidenceStrength: "weak", + suggestedNextCommands: records.length > 0 + ? [`mex graph get ${records[0]!.fromNode as string} --detail source`] : [], + warnings: [ + `No declaration named "${target}" is indexed. ` + + `${matched} unresolved reference(s) to that name were recorded during ` + + "extraction and are reported instead of resolved callers; they may be " + + "dynamically generated, defined outside the indexed corpus, or ambiguous.", + ], + }))); + return true; +} + +/** Total matching call sites, so the summary can report what it capped. */ +function unresolvedCallSiteCount(db: SqliteDatabase, name: string): number { + const row = db.prepare( + `SELECT COUNT(*) AS total FROM unresolved_refs + WHERE reference_name = ? AND status <> 'resolved'`, + ).get(name) as { total?: unknown } | undefined; + return typeof row?.total === "number" ? row.total : 0; +} + +/** Exact-name lookup, deterministically ordered and bounded in SQL. */ +function unresolvedCallSites( + db: SqliteDatabase, + name: string, + limit: number, +): UnresolvedCallSite[] { + return db.prepare( + `SELECT reference_name, reference_kind, status, file_path, line, col, + from_node_id, receiver, qualifier + FROM unresolved_refs + WHERE reference_name = ? AND status <> 'resolved' + ORDER BY file_path, line, col, from_node_id + LIMIT ?`, + ).all(name, Math.max(0, limit)) as UnresolvedCallSite[]; +} + function groundedFiles(db: SqliteDatabase, nodeIds: string[]): Array<{ scaffold_file: string; node_id: string }> { if (nodeIds.length === 0) return []; const placeholders = nodeIds.map(() => "?").join(","); diff --git a/src/graph/cli-graph.ts b/src/graph/cli-graph.ts index 132cc015..04d9cb64 100644 --- a/src/graph/cli-graph.ts +++ b/src/graph/cli-graph.ts @@ -1,4 +1,6 @@ -import type { GraphSourceChanges, GraphStatus } from "../team/contracts/graph.js"; +import type { + GraphRefreshResult, GraphSourceChanges, GraphStatus, +} from "../team/contracts/graph.js"; import { GraphMaintenanceError, repairGraph, @@ -54,6 +56,10 @@ export async function runGraph(options: GraphCommandOptions = {}): Promise partial: result.status.parseHealth.partial, failed: result.status.parseHealth.failed, }, + ...(result.skipped && result.skipped.length > 0 ? { skipped: result.skipped } : {}), + ...(result.declinedInputs && result.declinedInputs.length > 0 + ? { declinedInputs: result.declinedInputs } + : {}), }, null, 2)); return; } @@ -61,8 +67,47 @@ export async function runGraph(options: GraphCommandOptions = {}): Promise `Code graph built: ${result.nodesCreated} nodes, ${result.edgesCreated} edges ` + `across ${result.filesIndexed} files in ${result.durationMs}ms → .mex/graph.db`, ); + printSkippedSources(result.skipped); + printDeclinedInputs(result.declinedInputs); } +/** + * Name the files that are deliberately missing from the graph. + * + * Silence here is what made an oversized file look like a graph bug: a symbol + * was simply absent with nothing to explain it. + */ +function printSkippedSources(skipped: GraphRefreshResult["skipped"]): void { + if (!skipped || skipped.length === 0) return; + const shown = skipped.slice(0, MAX_SKIPPED_PATHS_SHOWN); + console.log(`Skipped ${skipped.length} file(s) the bounded corpus policy will not index:`); + for (const file of shown) console.log(` ${file.filePath} — ${file.message}`); + const omitted = skipped.length - shown.length; + if (omitted > 0) console.log(` …and ${omitted} more (use --json for the full list)`); + console.log("Add a glob to \"graph.ignore\" in .mex/config.json to exclude a path deliberately."); +} + +/** + * Name the config inputs the graph refused to read. + * + * Every source file is still in the graph; what is reduced is type resolution + * for the affected project, which is otherwise invisible. + */ +function printDeclinedInputs(declined: GraphRefreshResult["declinedInputs"]): void { + if (!declined || declined.length === 0) return; + const shown = declined.slice(0, MAX_SKIPPED_PATHS_SHOWN); + console.log( + `Declined ${declined.length} TypeScript config input(s) outside the project; ` + + "type resolution for the affected projects is less complete:", + ); + for (const input of shown) console.log(` ${input.filePath}`); + const omitted = declined.length - shown.length; + if (omitted > 0) console.log(` …and ${omitted} more (use --json for the full list)`); +} + +/** Human output stays bounded; `--json` carries the complete list. */ +const MAX_SKIPPED_PATHS_SHOWN = 10; + function printStatus(status: GraphStatus): void { const branch = status.currentRepo.branch ?? "detached/no branch"; const head = status.currentRepo.head?.slice(0, 12) ?? "no HEAD"; @@ -103,6 +148,8 @@ function printMaintenance(verb: "refreshed" | "rebuilt", result: GraphMaintenanc `Code graph ${verb}: ${result.nodesCreated} nodes, ${result.edgesCreated} edges ` + `across ${result.filesIndexed} files in ${result.durationMs}ms; status ${result.status.status}.`, ); + printSkippedSources(result.skipped); + printDeclinedInputs(result.declinedInputs); if (result.recoveryPath) { console.log(`Previous index retained for local recovery: ${result.recoveryPath}`); } diff --git a/src/graph/corpus-policy.ts b/src/graph/corpus-policy.ts index 43838417..5754e1b9 100644 --- a/src/graph/corpus-policy.ts +++ b/src/graph/corpus-policy.ts @@ -1,4 +1,6 @@ import { createHash } from "node:crypto"; +import { readFileSync, statSync } from "node:fs"; +import { resolve } from "node:path"; import { globIterateSync, type GlobOptions } from "glob"; import { SUPPORTED_SOURCE_GLOB } from "./extraction/grammars.js"; @@ -14,6 +16,93 @@ export const GRAPH_CORPUS_IGNORE_GLOBS = Object.freeze([ "**/out/**", ] as const); +/** + * Hard bounds on the additive ignore list a repository may configure. + * + * The list is read from an untrusted working-tree file on every corpus walk, + * so it is bounded the same way every other corpus input is. + */ +export const GRAPH_IGNORE_CONFIG_LIMITS = Object.freeze({ + maxGlobs: 64, + maxGlobLength: 256, + maxConfigBytes: 256 * 1024, +} as const); + +/** + * Additional ignore globs a repository may configure, from + * `.mex/config.json` -> `graph.ignore`. + * + * **Additive only.** The returned list is always appended to + * {@link GRAPH_CORPUS_IGNORE_GLOBS}, never substituted for it, so no + * configuration can un-ignore `node_modules`, `.git` or `.mex`. The frozen + * defaults are the floor; this raises it. + * + * Read defensively and never thrown from: a missing, unreadable, oversized or + * malformed config yields no extra globs rather than failing a build. A + * repository that cannot be indexed because its own config file is malformed + * would trade one hard abort for another. + */ +export function readConfiguredGraphIgnoreGlobs(root: string): string[] { + let raw: string; + try { + const configPath = resolve(root, ".mex", "config.json"); + const stats = statSync(configPath); + if (!stats.isFile() || stats.size > GRAPH_IGNORE_CONFIG_LIMITS.maxConfigBytes) return []; + raw = readFileSync(configPath, "utf8"); + } catch { + return []; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return []; + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return []; + const graph = (parsed as Record).graph; + if (typeof graph !== "object" || graph === null || Array.isArray(graph)) return []; + const ignore = (graph as Record).ignore; + if (!Array.isArray(ignore)) return []; + const globs = new Set(); + for (const entry of ignore) { + if (typeof entry !== "string") continue; + const trimmed = entry.trim(); + if (!trimmed || trimmed.length > GRAPH_IGNORE_CONFIG_LIMITS.maxGlobLength) continue; + const glob = trimmed.split("\\").join("/"); + if (!isRepositoryRelativeGlob(glob)) continue; + globs.add(glob); + if (globs.size >= GRAPH_IGNORE_CONFIG_LIMITS.maxGlobs) break; + } + return [...globs].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); +} + +/** + * Accept only globs that name something inside the repository. + * + * Absolute paths and upward traversal describe files outside the corpus, which + * discovery already refuses to walk. The check is deliberately **not** + * `path.isAbsolute`: that is platform-dependent, and `C:/build/**` is absolute + * on Windows but an ordinary relative path on Linux. `.mex/config.json` is + * tracked and travels with the repository, and these globs feed the corpus + * policy hash — so a platform-dependent verdict would give one repository two + * different manifest hashes and make its index read as stale purely from being + * opened on another machine. + * + * Expects a glob already normalized to forward slashes. + */ +function isRepositoryRelativeGlob(glob: string): boolean { + // Leading "/" covers POSIX-absolute and "//server/share" UNC alike. + if (glob.startsWith("/")) return false; + // Windows drive-absolute ("C:/x") and drive-relative ("C:x") forms. + if (/^[A-Za-z]:/u.test(glob)) return false; + return !glob.split("/").includes(".."); +} + +/** The complete ignore list for one repository: frozen defaults, then config. */ +export function graphCorpusIgnoreGlobs(root: string): string[] { + return [...GRAPH_CORPUS_IGNORE_GLOBS, ...readConfiguredGraphIgnoreGlobs(root)]; +} + export const GRAPH_CONFIG_GLOBS = Object.freeze([ "package.json", "**/package.json", @@ -63,13 +152,48 @@ export const GRAPH_CORPUS_LIMITS = Object.freeze({ maxDiagnostics: 100, } as const); +export type GraphCorpusLimitName = keyof typeof GRAPH_CORPUS_LIMITS; + +/** + * Limits that describe **one file** rather than the whole corpus. + * + * The distinction has teeth. A corpus-wide ceiling says the repository is too + * big for a bounded run and there is no honest partial answer, so it aborts. + * A per-file ceiling says one file is too big to parse — every other file in + * the repository is still perfectly indexable, so the file is skipped and + * reported instead of taking the build down with it. + */ +export const GRAPH_PER_FILE_CORPUS_LIMITS: ReadonlySet = new Set([ + "maxSourceFileBytes", + "maxConfigFileBytes", +] as const); + export class GraphCorpusLimitError extends Error { readonly code = "GRAPH_CORPUS_LIMIT_EXCEEDED"; - constructor(readonly limit: keyof typeof GRAPH_CORPUS_LIMITS) { - super(`The graph corpus exceeded the configured ${limit} safety bound.`); + constructor( + readonly limit: GraphCorpusLimitName, + /** Observed size, when the breach was measured against a single file. */ + readonly observedBytes?: number, + ) { + super( + observedBytes === undefined + ? `The graph corpus exceeded the configured ${limit} safety bound.` + : `The graph corpus exceeded the configured ${limit} safety bound: ` + + `${observedBytes} bytes against a ${GRAPH_CORPUS_LIMITS[limit]}-byte limit.`, + ); this.name = "GraphCorpusLimitError"; } + + /** True when only this one file is affected and the run may continue. */ + get perFile(): boolean { + return GRAPH_PER_FILE_CORPUS_LIMITS.has(this.limit); + } +} + +/** Narrow an unknown error to a per-file corpus-limit breach. */ +export function isPerFileCorpusLimitError(error: unknown): error is GraphCorpusLimitError { + return error instanceof GraphCorpusLimitError && error.perFile; } /** Lazily consume glob results and stop before an unbounded path array forms. */ @@ -100,7 +224,10 @@ export function addGraphCorpusBytes( ? GRAPH_CORPUS_LIMITS.maxSourceBytes : GRAPH_CORPUS_LIMITS.maxConfigBytes; if (!Number.isSafeInteger(fileBytes) || fileBytes < 0 || fileBytes > maxFile) { - throw new GraphCorpusLimitError(kind === "source" ? "maxSourceFileBytes" : "maxConfigFileBytes"); + throw new GraphCorpusLimitError( + kind === "source" ? "maxSourceFileBytes" : "maxConfigFileBytes", + Number.isSafeInteger(fileBytes) ? fileBytes : undefined, + ); } const next = total + fileBytes; if (!Number.isSafeInteger(next) || next > maxTotal) { @@ -113,7 +240,10 @@ export function addGraphCorpusBytes( export function addGraphCompilerSourceBytes(total: number, fileBytes: number): number { if (!Number.isSafeInteger(fileBytes) || fileBytes < 0 || fileBytes > GRAPH_CORPUS_LIMITS.maxSourceFileBytes) { - throw new GraphCorpusLimitError("maxSourceFileBytes"); + throw new GraphCorpusLimitError( + "maxSourceFileBytes", + Number.isSafeInteger(fileBytes) ? fileBytes : undefined, + ); } const next = total + fileBytes; if (!Number.isSafeInteger(next) || next > GRAPH_CORPUS_LIMITS.maxCompilerSourceBytes) { @@ -179,3 +309,21 @@ export const GRAPH_CORPUS_POLICY_HASH = createHash("sha256").update(JSON.stringi globOptions: GRAPH_CORPUS_GLOB_OPTIONS, limits: GRAPH_CORPUS_LIMITS, })).digest("hex"); + +/** + * Discovery identity for one repository: the frozen policy, plus whatever the + * repository additionally chose to ignore. + * + * A repository with no configured globs hashes to exactly + * {@link GRAPH_CORPUS_POLICY_HASH}, so every index built before this existed + * stays valid. Adding or removing a configured glob changes the corpus, so it + * must change the manifest and force an explicit rebuild. + */ +export function graphCorpusPolicyHash(root: string): string { + const configured = readConfiguredGraphIgnoreGlobs(root); + if (configured.length === 0) return GRAPH_CORPUS_POLICY_HASH; + return createHash("sha256").update(JSON.stringify({ + base: GRAPH_CORPUS_POLICY_HASH, + configuredIgnoreGlobs: configured, + })).digest("hex"); +} diff --git a/src/graph/db/store.ts b/src/graph/db/store.ts index 358778d8..ca5136c7 100644 --- a/src/graph/db/store.ts +++ b/src/graph/db/store.ts @@ -98,22 +98,6 @@ export interface NodeAliasRecord { confidence: number; } -/** Reference edge kinds — everything except the intra-file `contains` edge. - * `sync` wipes these and rebuilds them from `unresolved_refs`. */ -export const REFERENCE_EDGE_KINDS: EdgeKind[] = [ - "calls", - "imports", - "exports", - "extends", - "implements", - "references", - "type_of", - "returns", - "instantiates", - "overrides", - "decorates", -]; - interface NodeRow { id: string; kind: string; @@ -465,17 +449,6 @@ export class GraphStore { ); } - updateReferenceResolution(ref: UnresolvedRefRecord): void { - const key = ref.refKey ?? referenceKey(ref); - this.db.prepare( - `UPDATE unresolved_refs SET status = ?, target_id = ?, confidence = ?, resolver = ?, candidates = ? - WHERE ref_key = ?`, - ).run( - ref.status ?? "unresolved", ref.targetId ?? null, ref.confidence ?? null, - ref.resolver ?? null, ref.candidates ? JSON.stringify(ref.candidates) : null, key, - ); - } - insertImportBinding(binding: ImportBindingRecord): void { this.db.prepare( `INSERT INTO import_bindings ( @@ -638,12 +611,6 @@ export class GraphStore { this.db.exec("DELETE FROM files"); } - /** Wipe every reference edge (keeping intra-file `contains`), so `sync` can - * rebuild them from `unresolved_refs` with no duplicates. */ - clearReferenceEdges(): void { - this.db.prepare("DELETE FROM edges WHERE kind != 'contains'").run(); - } - // --- Reads ---------------------------------------------------------------- getNodeById(id: string): GraphNode | null { diff --git a/src/graph/engine-impl.ts b/src/graph/engine-impl.ts index 1585b16d..ced0fe3d 100644 --- a/src/graph/engine-impl.ts +++ b/src/graph/engine-impl.ts @@ -15,13 +15,15 @@ import { import { tmpdir } from "node:os"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { toPosix } from "../paths.js"; -import type { BuildResult, GraphEngine, NodeSearchOptions } from "./engine.js"; +import type { + BuildResult, DeclinedCompilerInput, GraphEngine, NodeSearchOptions, SkippedSourceFile, +} from "./engine.js"; import { GRAPH_CONFIG_GLOBS, GRAPH_CORPUS_GLOB_OPTIONS, - GRAPH_CORPUS_IGNORE_GLOBS, + graphCorpusIgnoreGlobs, GRAPH_CORPUS_LIMITS, - GRAPH_CORPUS_POLICY_HASH, + graphCorpusPolicyHash, GRAPH_SUPPORTED_SOURCE_GLOB, GraphCorpusLimitError, addGraphCompilerSourceBytes, @@ -29,6 +31,7 @@ import { addGraphSemanticInput, createGraphSemanticInputLedger, discoverBoundedGraphPaths, + isPerFileCorpusLimitError, } from "./corpus-policy.js"; import { DB_SCHEMA_VERSION, markGraphReady, openGraphDatabase } from "./db/database.js"; import { @@ -234,6 +237,14 @@ interface DiscoveredFile { modifiedAt: number; } +type GraphSkippedSourceFile = SkippedSourceFile; + +/** What one source walk found: the corpus, and what it declined to read. */ +interface DiscoveredCorpus { + files: DiscoveredFile[]; + skipped: GraphSkippedSourceFile[]; +} + interface StagedFile { discovered: DiscoveredFile; record: FileRecord; @@ -246,6 +257,10 @@ interface StagedFile { interface StagedCorpus { files: StagedFile[]; + /** Files discovered but not indexed, carried through to the build result. */ + skipped: GraphSkippedSourceFile[]; + /** Config inputs the containment policy declined during compiler extraction. */ + declinedInputs: DeclinedCompilerInput[]; compiler: Pick; semanticInputs: CompilerSemanticInput[]; fingerprints: Array<{ nodeId: string; fingerprint: Fingerprint }>; @@ -343,7 +358,7 @@ class GraphEngineImpl implements GraphEngine { const store = this.getStore(true); const manifestChanged = store.getMetadata("manifest_hash") !== manifest.manifestHash; if (changedSources.length === 0 && !manifestChanged) { - const currentCorpus = discoverSourceFiles(this.rootDir, this.sourceFileAccess); + const currentCorpus = discoverSourceFiles(this.rootDir, this.sourceFileAccess).files; const snapshot = parseGraphSnapshot(store.getMetadata(GRAPH_SNAPSHOT_METADATA_KEY)); if (snapshot?.manifestHash === manifest.manifestHash && snapshot.indexedBranch === gitBeforeStaging.branch @@ -364,8 +379,13 @@ class GraphEngineImpl implements GraphEngine { ); try { const stagedByPath = new Map(staged.files.map((file) => [file.record.path, file])); + // A file the corpus policy deliberately skipped is absent from the + // staged corpus by design, so it is not evidence of a lost source. + const skippedPaths = new Set(staged.skipped.map((file) => file.filePath)); const unstagedExisting = changedSources.filter((file) => ( - !deletedChangedSources.has(file) && !stagedByPath.has(file) + !deletedChangedSources.has(file) + && !stagedByPath.has(file) + && !skippedPaths.has(file) )); if (unstagedExisting.length > 0) { throw new GraphSourceStagingError(unstagedExisting.map((filePath) => ({ @@ -466,7 +486,15 @@ class GraphEngineImpl implements GraphEngine { for (const file of staged.files) for (const node of file.nodes) store.insertNode(node); for (const file of staged.files) { for (const edge of file.edges) if (store.insertEdge(edge)) edgeCount++; - for (const reference of file.references) store.insertUnresolvedRef(reference); + // A resolved reference is an edge. Keeping the row too duplicated one + // for every reference the resolver bound — 27-73% of this table on + // every repository measured, all of them already in `edges`. Resolution + // happens in memory during staging, so nothing downstream reads these + // rows back to rebuild anything. + for (const reference of file.references) { + if (reference.status === "resolved") continue; + store.insertUnresolvedRef(reference); + } for (const binding of file.imports) store.insertImportBinding(binding); } @@ -490,6 +518,10 @@ class GraphEngineImpl implements GraphEngine { nodesCreated: expectedNodes, edgesCreated: edgeCount, health: healthCounts(staged.files), + ...(staged.skipped.length > 0 ? { skipped: [...staged.skipped] } : {}), + ...(staged.declinedInputs.length > 0 + ? { declinedInputs: [...staged.declinedInputs] } + : {}), }; } @@ -552,7 +584,7 @@ async function stageCorpus( ): Promise { const sourceSpool = new GraphSourceSpool(); try { - const discovered = discoverSourceFiles(root, sourceFileAccess, sourceSpool); + const { files: discovered, skipped } = discoverSourceFiles(root, sourceFileAccess, sourceSpool); const configSources = discoverGraphConfigSources(root); const stagedConfigHash = configHashForSources(configSources); if (stagedConfigHash !== manifest.configHash) { @@ -658,6 +690,8 @@ async function stageCorpus( } return { files, + skipped, + declinedInputs: [...compiler.declinedInputs], compiler: { compilerVersion: compiler.compilerVersion, semanticInputs: [...compiler.semanticInputs], @@ -1266,13 +1300,31 @@ function inspectChangedSources( return deleted; } +/** + * Walk the repository's supported sources. + * + * A file the bounded corpus policy will not read is **skipped, not fatal.** + * One oversized generated file used to abort an entire multi-thousand-file + * build: the per-file limit error was pushed as a staging failure, the loop + * `break`s so no later file was even attempted, and the accumulated failures + * were thrown. Every other file in the repository is still perfectly + * indexable, so a per-file ceiling now records the file and continues. + * + * Corpus-wide ceilings (`maxSourceBytes`, `maxSourceFiles`) still abort. They + * describe the whole run and there is no honest partial answer to them. + * + * The skip happens here, at the single discovery seam every consumer shares, + * so the staged corpus, `verifyPublicationInputs`, `sync`'s corpus comparison + * and the freshness inspector all agree about which files exist. + */ function discoverSourceFiles( root: string, sourceFileAccess: GraphSourceFileAccess, sourceSpool?: GraphSourceSpool, -): DiscoveredFile[] { +): DiscoveredCorpus { let matches: string[]; const files: DiscoveredFile[] = []; + const skipped: GraphSkippedSourceFile[] = []; const failures: GraphSourceStagingFailure[] = []; let canonicalRoot: string; try { @@ -1280,7 +1332,7 @@ function discoverSourceFiles( matches = discoverBoundedGraphPaths(GRAPH_SUPPORTED_SOURCE_GLOB, { ...GRAPH_CORPUS_GLOB_OPTIONS, cwd: root, - ignore: [...GRAPH_CORPUS_IGNORE_GLOBS], + ignore: graphCorpusIgnoreGlobs(root), }, GRAPH_CORPUS_LIMITS.maxSourceFiles).map(toPosix); } catch (error) { throw new GraphSourceStagingError([sourceStagingFailure(".", "discover", error)]); @@ -1312,8 +1364,11 @@ function discoverSourceFiles( sourceSpool?.stage(relPath, file.source); files.push({ relPath, contentHash, size: file.size, modifiedAt: file.modifiedAt }); } catch (error) { - failures.push(sourceStagingFailure(relPath, "read", error)); - if (error instanceof GraphCorpusLimitError) break; + if (isPerFileCorpusLimitError(error)) skipped.push(skippedSourceFile(relPath, error)); + else { + failures.push(sourceStagingFailure(relPath, "read", error)); + if (error instanceof GraphCorpusLimitError) break; + } } continue; } @@ -1348,12 +1403,29 @@ function discoverSourceFiles( }); sourceSpool?.stage(relPath, source); } catch (error) { - failures.push(sourceStagingFailure(relPath, "read", error)); - if (error instanceof GraphCorpusLimitError) break; + if (isPerFileCorpusLimitError(error)) skipped.push(skippedSourceFile(relPath, error)); + else { + failures.push(sourceStagingFailure(relPath, "read", error)); + if (error instanceof GraphCorpusLimitError) break; + } } } if (failures.length > 0) throw new GraphSourceStagingError(failures); - return files; + return { files, skipped }; +} + +function skippedSourceFile( + filePath: string, + error: GraphCorpusLimitError, +): GraphSkippedSourceFile { + return { + filePath, + reason: "corpus-limit", + limit: error.limit, + limitBytes: GRAPH_CORPUS_LIMITS[error.limit], + observedBytes: error.observedBytes, + message: error.message, + }; } function sourceCorpusMatchesFileRecords( @@ -1401,6 +1473,10 @@ function readStableUtf8File( canonicalPath: string, sourcePath = canonicalPath, maxBytes = GRAPH_CORPUS_LIMITS.maxSourceFileBytes, + // Named explicitly rather than inferred from `maxBytes`. The source and + // config per-file ceilings are numerically identical, so comparing the value + // reported every oversized *source* file as a config-limit breach. + limitName: "maxSourceFileBytes" | "maxConfigFileBytes" = "maxSourceFileBytes", ): { source: string; size: number; @@ -1419,9 +1495,8 @@ function readStableUtf8File( } if (!Number.isSafeInteger(opened.size) || opened.size < 0 || opened.size > maxBytes) { throw new GraphCorpusLimitError( - maxBytes === GRAPH_CORPUS_LIMITS.maxConfigFileBytes - ? "maxConfigFileBytes" - : "maxSourceFileBytes", + limitName, + Number.isSafeInteger(opened.size) ? opened.size : undefined, ); } const source = readFileSync(fd, "utf8"); @@ -1588,7 +1663,7 @@ function verifyPublicationInputs( sourceFileAccess: GraphSourceFileAccess, internal: GraphEngineInternalHooks = {}, ): GraphGitProvenance { - const currentFiles = discoverSourceFiles(root, sourceFileAccess); + const currentFiles = discoverSourceFiles(root, sourceFileAccess).files; const currentManifest = graphManifest(root); const failures: GraphSourceStagingFailure[] = []; @@ -1686,7 +1761,7 @@ export function graphManifest(root: string): GraphManifest { compiler: TYPESCRIPT_COMPILER_VERSION, extractor: CORPUS_EXTRACTOR_VERSION, resolver: RESOLVER_VERSION, - corpusPolicyHash: GRAPH_CORPUS_POLICY_HASH, + corpusPolicyHash: graphCorpusPolicyHash(root), grammarHash, configHash, })); @@ -1701,7 +1776,7 @@ function discoverGraphConfigSources(root: string): Map { configPaths = discoverBoundedGraphPaths([...GRAPH_CONFIG_GLOBS], { ...GRAPH_CORPUS_GLOB_OPTIONS, cwd: root, - ignore: [...GRAPH_CORPUS_IGNORE_GLOBS], + ignore: graphCorpusIgnoreGlobs(root), }, GRAPH_CORPUS_LIMITS.maxConfigFiles).map(toPosix); } catch (error) { throw new GraphSourceStagingError([sourceStagingFailure(".", "discover", error)]); @@ -1714,6 +1789,7 @@ function discoverGraphConfigSources(root: string): Map { canonicalPath, resolve(root, path), GRAPH_CORPUS_LIMITS.maxConfigFileBytes, + "maxConfigFileBytes", ); configBytes = addGraphCorpusBytes( configBytes, diff --git a/src/graph/engine.ts b/src/graph/engine.ts index 51bf9470..92929592 100644 --- a/src/graph/engine.ts +++ b/src/graph/engine.ts @@ -24,6 +24,24 @@ import { NotImplementedError } from "./errors.js"; // Value types // ---------------------------------------------------------------------------- +/** + * One repository file the bounded corpus policy declined to index. + * + * A skipped file is a reported outcome, not a failure: the rest of the + * repository indexed normally. It is surfaced so a user learns why a symbol is + * missing from the graph instead of concluding the graph is wrong. + */ +export interface SkippedSourceFile { + filePath: string; + reason: "corpus-limit"; + /** The bounded-policy limit the file breached. */ + limit: string; + limitBytes: number; + /** Observed size, when it was measured. */ + observedBytes?: number; + message: string; +} + /** Summary of a build/sync pass — for the `mex graph` CLI. */ export interface BuildResult { filesIndexed: number; @@ -35,6 +53,26 @@ export interface BuildResult { partial: number; failed: number; }; + /** + * Files discovered but deliberately not indexed. Sits beside `health` + * rather than inside it: `health` counts files that *are* in the graph and + * how well they parsed, and a skipped file is in none of those buckets. + */ + skipped?: SkippedSourceFile[]; + /** + * Compiler config inputs outside the project corpus that were declined. + * + * Distinct from `skipped`: no source file is missing from the graph, but the + * affected project's type resolution is less complete than it looks. + */ + declinedInputs?: DeclinedCompilerInput[]; +} + +/** A compiler input the containment policy declined to read. */ +export interface DeclinedCompilerInput { + filePath: string; + reason: "outside-project-corpus"; + message: string; } /** Options for {@link GraphEngine.searchNodes}. */ diff --git a/src/graph/extraction/compiler.ts b/src/graph/extraction/compiler.ts index 705a89a8..a6724eb8 100644 --- a/src/graph/extraction/compiler.ts +++ b/src/graph/extraction/compiler.ts @@ -175,6 +175,19 @@ export interface DiscoveredTypeScriptProject { diagnostics: CompilerDiagnosticSummary[]; } +/** + * A compiler input the containment policy declined to read. + * + * Reported rather than fatal. The resulting type graph is less complete for + * the affected project, and a user has to be told that rather than left to + * infer it from missing edges. + */ +export interface DeclinedCompilerInput { + filePath: string; + reason: "outside-project-corpus"; + message: string; +} + export interface CompilerExtractionResult { compilerVersion: string; extractorVersion: string; @@ -182,6 +195,8 @@ export interface CompilerExtractionResult { files: CompilerFileExtraction[]; /** Repository inputs whose exact bytes influenced config parsing or compiler facts. */ semanticInputs: CompilerSemanticInput[]; + /** Config inputs outside the project corpus that were declined, not read. */ + declinedInputs: DeclinedCompilerInput[]; } export interface CompilerStagedInput { @@ -636,9 +651,35 @@ export function buildTypeScriptExtraction( projects: projectSummaries, files, semanticInputs: inputs.semanticInputs(), + declinedInputs: inputs.declinedConfigInputs(), }; } +/** Bounded like every other diagnostic channel; the list is deduplicated. */ +const MAX_DECLINED_CONFIG_INPUTS = 100; + +/** + * Name a declined path by what the user can act on. + * + * TypeScript resolves a bare `extends` specifier by walking every ancestor + * directory, so one unresolvable `extends "astro/tsconfigs/strict"` produces a + * decline at `/node_modules/...`, `/node_modules/...` and so on + * up to the filesystem root. Reporting each rung is noise, and it puts absolute + * paths from outside the repository — a user's home directory among them — into + * build output. + * + * Keying on the dependency specifier collapses those rungs to the one fact + * worth reporting: this package is not resolvable inside the project. Paths + * with no `node_modules` segment are reported relative to the root instead. + */ +function reportableDeclinedPath(root: string, fileName: string): string { + const absolute = normalizedAbsolute(absoluteCandidate(root, fileName)); + const segments = absolute.split("/"); + const lastDependencyRoot = segments.lastIndexOf("node_modules"); + if (lastDependencyRoot >= 0) return segments.slice(lastDependencyRoot).join("/"); + return normalizeRelative(relative(root, absolute)) || absolute; +} + /** Stable identity input shared with migration/invariant tests. */ export function canonicalCompilerIdentity(input: { filePath: string; @@ -726,6 +767,7 @@ class CompilerInputLedger { private readonly knownDirectories = new Set(); private readonly directoryFiles = new Map>(); private readonly directoryChildren = new Map>(); + private readonly declinedConfigs = new Map(); private candidates: string[] = []; constructor(root: string, private readonly options: CompilerExtractionOptions) { @@ -787,26 +829,76 @@ class CompilerInputLedger { return source; } + /** + * Read one config input, or decline it. + * + * A config path outside the project root used to throw, which took the + * entire build down: a `tsconfig.json` extending a package hoisted above the + * root — any pnpm/yarn hoisted layout, or a monorepo sub-package indexed on + * its own — made the repository un-indexable. `readFile` had always treated + * the identical condition as a soft miss for *source* files; the two halves + * of this class simply disagreed. + * + * The containment guard is unchanged and still declines. Declining now + * returns "no such config", which is the honest answer for a file we will + * not read, and is recorded so the degradation is reported rather than + * silent. The type graph is less complete; the build finishes. + */ readConfigFile(fileName: string): string | undefined { const absolute = absoluteCandidate(this.root, fileName); - if (!withinRoot(this.root, absolute)) { - throw compilerInputContainmentError(`TypeScript config escapes the project root: ${fileName}`); - } - if (!this.isProjectInput(absolute) && !isAllowedCompilerDependency(absolute)) { - throw compilerInputContainmentError(`TypeScript config is outside the supported project corpus: ${fileName}`); - } + if (!this.permitsConfigInput(absolute, fileName)) return undefined; return this.readFile(absolute); } + /** + * Answer TypeScript's existence probe for a config path. + * + * This is a probe, not a read request. The honest answer for a path we have + * declined to read is `false`. + */ configFileExists(fileName: string): boolean { const absolute = absoluteCandidate(this.root, fileName); + if (!this.permitsConfigInput(absolute, fileName)) return false; + return this.fileExists(absolute); + } + + /** Record and decline a config input the containment policy will not read. */ + private permitsConfigInput(absolute: string, fileName: string): boolean { if (!withinRoot(this.root, absolute)) { - throw compilerInputContainmentError(`TypeScript config escapes the project root: ${fileName}`); + this.declineConfigInput(fileName, "escapes the project root"); + return false; } if (!this.isProjectInput(absolute) && !isAllowedCompilerDependency(absolute)) { - throw compilerInputContainmentError(`TypeScript config is outside the supported project corpus: ${fileName}`); + this.declineConfigInput(fileName, "is outside the supported project corpus"); + return false; } - return this.fileExists(absolute); + return true; + } + + private declineConfigInput(fileName: string, reason: string): void { + const reported = reportableDeclinedPath(this.root, fileName); + if (this.declinedConfigs.has(reported)) return; + if (this.declinedConfigs.size >= MAX_DECLINED_CONFIG_INPUTS) return; + this.declinedConfigs.set( + reported, + `TypeScript config ${reason} and was not read: ${reported}`, + ); + } + + /** Record a referenced project outside the root, which we do not traverse. */ + declineProjectReference(referencePath: string): void { + this.declineConfigInput(referencePath, "project reference escapes the project root"); + } + + /** Config inputs the containment policy declined, in deterministic order. */ + declinedConfigInputs(): DeclinedCompilerInput[] { + return [...this.declinedConfigs.entries()] + .sort(([left], [right]) => compareCodePoints(left, right)) + .map(([filePath, message]) => ({ + filePath, + reason: "outside-project-corpus" as const, + message, + })); } fileExists(fileName: string): boolean { @@ -852,8 +944,13 @@ class CompilerInputLedger { includes: readonly string[], depth?: number, ): string[] { + // Treated exactly like a declined config read. A tsconfig `include` that + // points above the project root describes files we will not walk, and the + // honest answer for a directory we decline is "no matching files" — not an + // exception that costs the whole repository its graph. if (!withinRoot(this.root, absoluteCandidate(this.root, rootDir))) { - throw compilerInputContainmentError(`TypeScript config include escapes the project root: ${rootDir}`); + this.declineConfigInput(rootDir, "include escapes the project root"); + return []; } // TypeScript's matcher is intentionally used with a virtual directory tree // built only from the immutable candidate list. This preserves exact @@ -1014,8 +1111,13 @@ function parseProjects( diagnostics.push(...parsed.errors); for (const reference of parsed.projectReferences ?? []) { const referencePath = normalizedAbsolute(ts.resolveProjectReferencePath(reference)); + // Same decision as a declined config read: a referenced project above + // the root is not traversed, and is reported rather than fatal. A + // sub-package of a monorepo indexed on its own routinely references its + // siblings, and that must not make it un-indexable. if (!withinRoot(root, referencePath)) { - throw compilerInputContainmentError(`TypeScript project reference escapes the project root: ${referencePath}`); + inputs?.declineProjectReference(referencePath); + continue; } if (!seen.has(referencePath)) queue.push(referencePath); } @@ -2220,10 +2322,6 @@ function compareCodePoints(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; } -function compilerInputContainmentError(message: string): Error & { code: string } { - return Object.assign(new Error(message), { code: "GRAPH_SOURCE_PATH_ESCAPE" }); -} - function isAllowedCompilerDependency(absolutePath: string): boolean { return normalizeRelative(absolutePath).split("/").includes("node_modules"); } diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index 07a6859f..0c0ecb47 100644 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -1312,6 +1312,10 @@ function maintenanceResult( filesIndexed: build.filesIndexed, nodesCreated: build.nodesCreated, edgesCreated: build.edgesCreated, + ...(build.skipped && build.skipped.length > 0 ? { skipped: build.skipped } : {}), + ...(build.declinedInputs && build.declinedInputs.length > 0 + ? { declinedInputs: build.declinedInputs } + : {}), ...(published.recoveryPath ? { recoveryPath: published.recoveryPath } : {}), }; } diff --git a/src/graph/runtime.ts b/src/graph/runtime.ts index 73c3eb2e..36d9bb33 100644 --- a/src/graph/runtime.ts +++ b/src/graph/runtime.ts @@ -8,7 +8,7 @@ import { createGraphEngine } from "./engine-impl.js"; import type { GraphEngine } from "./engine.js"; import { GRAPH_CORPUS_GLOB_OPTIONS, - GRAPH_CORPUS_IGNORE_GLOBS, + graphCorpusIgnoreGlobs, GRAPH_SUPPORTED_SOURCE_GLOB, } from "./corpus-policy.js"; import { openGraphDatabase } from "./db/database.js"; @@ -247,7 +247,7 @@ export function findChangedSourceFiles(projectRoot: string, db: SqliteDatabase): const current = globSync(GRAPH_SUPPORTED_SOURCE_GLOB, { ...GRAPH_CORPUS_GLOB_OPTIONS, cwd: projectRoot, - ignore: [...GRAPH_CORPUS_IGNORE_GLOBS], + ignore: graphCorpusIgnoreGlobs(projectRoot), }) .map((path) => path.replaceAll("\\", "/")); const changed: string[] = []; diff --git a/src/graph/schema.sql b/src/graph/schema.sql index f419ddd1..62be854f 100644 --- a/src/graph/schema.sql +++ b/src/graph/schema.sql @@ -122,8 +122,20 @@ CREATE TABLE IF NOT EXISTS files ( extractor_version TEXT NOT NULL DEFAULT 'unknown' ); --- Unresolved references: parked during single-file extraction, resolved after a --- full index pass (two-phase extract -> resolve). Kept so cross-file edges work. +-- Unresolved references: parked during single-file extraction, then resolved +-- after a full index pass (two-phase extract -> resolve). +-- +-- Only references the resolver could NOT bind are stored. A bound reference +-- becomes an edge, and every row this table used to keep with status +-- 'resolved' duplicated one — measured at 100% on every repository, and 27-73% +-- of the table. `edges` is a superset (it also holds structural `contains` +-- edges with no reference row), so nothing is recoverable from here that is +-- not already there. +-- +-- What remains is the graph being honest about its own blind spots: a name +-- some file referenced that the resolver could not decide the meaning of. That +-- is what `mex graph query who-calls` falls back to when a name has call sites +-- but no indexed declaration. CREATE TABLE IF NOT EXISTS unresolved_refs ( id INTEGER PRIMARY KEY AUTOINCREMENT, ref_key TEXT NOT NULL UNIQUE, @@ -253,11 +265,17 @@ ON edges(source, target, kind, IFNULL(line, -1), IFNULL(col, -1)); CREATE INDEX IF NOT EXISTS idx_files_language ON files(language); CREATE INDEX IF NOT EXISTS idx_files_modified_at ON files(modified_at); -CREATE INDEX IF NOT EXISTS idx_unresolved_from_node ON unresolved_refs(from_node_id); +-- A narrow (from_node_id) index is intentionally omitted: it is a strict +-- prefix of the composite below, which SQLite uses for every lookup the narrow +-- one served, the ON DELETE CASCADE probe included. Verified with EXPLAIN +-- QUERY PLAN on a real store; no plan degrades to a scan. CREATE INDEX IF NOT EXISTS idx_unresolved_name ON unresolved_refs(reference_name); CREATE INDEX IF NOT EXISTS idx_unresolved_file_path ON unresolved_refs(file_path); CREATE INDEX IF NOT EXISTS idx_unresolved_from_name ON unresolved_refs(from_node_id, reference_name); -CREATE INDEX IF NOT EXISTS idx_unresolved_status ON unresolved_refs(status); +-- Partial: a resolved reference is an edge and is not stored here. The +-- predicate keeps the index honest if a legacy store still carries such rows. +CREATE INDEX IF NOT EXISTS idx_unresolved_status +ON unresolved_refs(status) WHERE status <> 'resolved'; CREATE INDEX IF NOT EXISTS idx_import_bindings_file ON import_bindings(file_path); CREATE INDEX IF NOT EXISTS idx_import_bindings_local ON import_bindings(file_path, local_name); CREATE INDEX IF NOT EXISTS idx_aliases_canonical ON node_aliases(canonical_node_id); diff --git a/src/graph/status.ts b/src/graph/status.ts index a350e510..c4825da5 100644 --- a/src/graph/status.ts +++ b/src/graph/status.ts @@ -25,8 +25,9 @@ import { openSqlite, type SqliteDatabase } from "./db/sqlite.js"; import { BANDS, K } from "./config.js"; import { GRAPH_CORPUS_GLOB_OPTIONS, - GRAPH_CORPUS_IGNORE_GLOBS, + graphCorpusIgnoreGlobs, GRAPH_CORPUS_LIMITS, + isPerFileCorpusLimitError, GRAPH_SUPPORTED_SOURCE_GLOB, GraphCorpusLimitError, addGraphCorpusBytes, @@ -114,7 +115,6 @@ const REQUIRED_SCHEMA_OBJECTS = { "idx_source_chunks_file", "idx_unresolved_file_path", "idx_unresolved_from_name", - "idx_unresolved_from_node", "idx_unresolved_name", "idx_unresolved_status", ], @@ -1702,7 +1702,7 @@ function inspectLiveSources( matches = discoverBoundedGraphPaths(GRAPH_SUPPORTED_SOURCE_GLOB, { ...GRAPH_CORPUS_GLOB_OPTIONS, cwd: projectRoot, - ignore: [...GRAPH_CORPUS_IGNORE_GLOBS], + ignore: graphCorpusIgnoreGlobs(projectRoot), }, GRAPH_CORPUS_LIMITS.maxSourceFiles) .map(toPosix) .filter(isSupportedSourceFile) @@ -1738,6 +1738,19 @@ function inspectLiveSources( ); hashes.set(path, sha256(content)); } catch (error) { + // A file the bounded policy will not read for its own size is skipped by + // indexing too, so the corpus observation stays complete and the walk + // continues. Only a corpus-wide breach makes the observation partial. + if (isPerFileCorpusLimitError(error)) { + discoveredPaths.delete(path); + reportPathDiagnostic({ + code: "GRAPH_SOURCE_FILE_SKIPPED", + severity: "warning", + message: `Supported source file ${path} is not indexed: ${errorMessage(error)}`, + path, + }); + continue; + } complete = false; const code = errorCode(error); const outsideProject = code === "GRAPH_CONTAINED_FILE_OUTSIDE_PROJECT"; diff --git a/src/team/contracts/graph.ts b/src/team/contracts/graph.ts index e03272f6..4db83cc1 100644 --- a/src/team/contracts/graph.ts +++ b/src/team/contracts/graph.ts @@ -76,6 +76,21 @@ export interface GraphMaintenanceOptions { onProgress?: (progress: GraphMaintenanceProgress) => void; } +/** + * One repository file the bounded corpus policy declined to index. + * + * Reported rather than fatal: the rest of the repository indexed normally, + * and this is how a user learns why a symbol is absent from the graph. + */ +export interface GraphSkippedSource { + filePath: RepoRelativePath; + reason: "corpus-limit"; + limit: string; + limitBytes: number; + observedBytes?: number; + message: string; +} + /** Port mutations return only successful results; failures throw typed errors. */ export interface GraphRefreshResult extends IndexJobResult { state: "succeeded"; @@ -83,6 +98,22 @@ export interface GraphRefreshResult extends IndexJobResult { filesIndexed: number; nodesCreated: number; edgesCreated: number; + /** Optional and additive; absent when every discovered file was indexed. */ + skipped?: readonly GraphSkippedSource[]; + /** Compiler config inputs outside the project corpus that were declined. */ + declinedInputs?: readonly GraphDeclinedInput[]; +} + +/** + * A compiler config input the containment policy declined to read. + * + * No source file is missing from the graph, but the affected project's type + * resolution is less complete than it looks, so it is reported. + */ +export interface GraphDeclinedInput { + filePath: string; + reason: "outside-project-corpus"; + message: string; } export interface GraphPage extends Page {