From db7e189f8a5c4101677ca64f315fc35a046fd110 Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Mon, 7 Sep 2026 20:09:22 +0530 Subject: [PATCH 1/9] fix(graph): skip a file the corpus policy will not read instead of aborting A single oversized file could abort an entire graph build. The per-file limit error was pushed as a staging failure, the discovery loop then broke so no later file was attempted, and the accumulated failures were thrown. On a 3,259-file TypeScript monorepo one 34 MB generated source made the whole repository un-indexable, with no way to exclude it short of editing the tree. A per-file ceiling (maxSourceFileBytes, maxConfigFileBytes) now records the file and continues; corpus-wide ceilings still abort, because they describe the whole run and have no honest partial answer. The skip happens at the single discovery seam every consumer shares, so the staged corpus, publication verification, sync's corpus comparison and the freshness inspector all agree about which files exist. Build results carry the skipped files so a user learns why a symbol is missing from the graph. Two supporting fixes: * The limit name in the error was inferred by comparing the byte ceiling against maxConfigFileBytes. The source and config per-file limits are numerically identical, so every oversized source file was reported as a config-limit breach. The name is now passed in, and the message states the observed size and the limit on its first line. * Freshness inspection had the same break-on-limit shape and would stop walking the corpus at the first oversized file. It now skips that file with a bounded per-path diagnostic and keeps the observation complete, matching what indexing does. The containment and bounded-work guards are unchanged. This changes only what happens when a guard declines, never whether it declines. --- src/graph/__tests__/status.test.ts | 32 +++++-- src/graph/corpus-policy.ts | 135 ++++++++++++++++++++++++++++- src/graph/engine-impl.ts | 100 +++++++++++++++++---- src/graph/engine.ts | 24 +++++ src/graph/runtime.ts | 4 +- src/graph/status.ts | 18 +++- 6 files changed, 279 insertions(+), 34 deletions(-) diff --git a/src/graph/__tests__/status.test.ts b/src/graph/__tests__/status.test.ts index 38bdd925..d56236ee 100644 --- a/src/graph/__tests__/status.test.ts +++ b/src/graph/__tests__/status.test.ts @@ -155,21 +155,39 @@ describe("inspectGraphStatus", () => { expect(treeState(root)).toEqual(before); }); - it("preserves a path-free corpus-limit diagnostic when changed-path output is disabled", async () => { - const root = temporaryRoot("mex-graph-status-corpus-limit-"); + // One oversized file is a skipped file, not a corpus breach. This test used + // to assert the opposite — that a single oversized source raised the + // path-free GRAPH_SOURCE_CORPUS_LIMIT_EXCEEDED — which is the same defect + // that let one 34 MB generated file abort a 3,259-file build. The invariant + // that test actually guarded (a path-free corpus diagnostic survives when + // per-path diagnostics are suppressed) is covered by the config-corpus case + // immediately below, which still trips a genuine corpus-wide ceiling. + it("skips one oversized source file, as a bounded per-path diagnostic", async () => { + const root = temporaryRoot("mex-graph-status-oversized-file-"); + source(root, "src/service.ts", "export const service = true;\n"); source( root, "src/oversized.py", "x".repeat(GRAPH_CORPUS_LIMITS.maxSourceFileBytes + 1), ); - const status = await inspect(root, 0); + const suppressed = await inspect(root, 0); + const reported = await inspect(root, 10); - expect(status.diagnostics).toContainEqual({ - code: "GRAPH_SOURCE_CORPUS_LIMIT_EXCEEDED", + // The skip names a path, so it is bounded like every other path diagnostic. + expect(suppressed.diagnostics).not.toContainEqual( + expect.objectContaining({ code: "GRAPH_SOURCE_FILE_SKIPPED" }), + ); + expect(suppressed.diagnostics).not.toContainEqual( + expect.objectContaining({ code: "GRAPH_SOURCE_CORPUS_LIMIT_EXCEEDED" }), + ); + expect(reported.diagnostics).toContainEqual(expect.objectContaining({ + code: "GRAPH_SOURCE_FILE_SKIPPED", severity: "warning", - message: "The supported source corpus exceeds MEX's bounded inspection policy.", - }); + path: "src/oversized.py", + })); + // The rest of the repository is still observed: the walk did not stop. + expect(reported.changes.added).toEqual(["src/service.ts"]); }); it("classifies a bounded manifest refusal separately from generic inspection failure", async () => { diff --git a/src/graph/corpus-policy.ts b/src/graph/corpus-policy.ts index 43838417..3577d4d8 100644 --- a/src/graph/corpus-policy.ts +++ b/src/graph/corpus-policy.ts @@ -1,4 +1,6 @@ import { createHash } from "node:crypto"; +import { readFileSync, statSync } from "node:fs"; +import { isAbsolute, resolve } from "node:path"; import { globIterateSync, type GlobOptions } from "glob"; import { SUPPORTED_SOURCE_GLOB } from "./extraction/grammars.js"; @@ -14,6 +16,72 @@ export const GRAPH_CORPUS_IGNORE_GLOBS = Object.freeze([ "**/out/**", ] as const); +/** + * Hard bounds on the additive ignore list a repository may configure. + * + * The list is read from an untrusted working-tree file on every corpus walk, + * so it is bounded the same way every other corpus input is. + */ +export const GRAPH_IGNORE_CONFIG_LIMITS = Object.freeze({ + maxGlobs: 64, + maxGlobLength: 256, + maxConfigBytes: 256 * 1024, +} as const); + +/** + * Additional ignore globs a repository may configure, from + * `.mex/config.json` -> `graph.ignore`. + * + * **Additive only.** The returned list is always appended to + * {@link GRAPH_CORPUS_IGNORE_GLOBS}, never substituted for it, so no + * configuration can un-ignore `node_modules`, `.git` or `.mex`. The frozen + * defaults are the floor; this raises it. + * + * Read defensively and never thrown from: a missing, unreadable, oversized or + * malformed config yields no extra globs rather than failing a build. A + * repository that cannot be indexed because its own config file is malformed + * would trade one hard abort for another. + */ +export function readConfiguredGraphIgnoreGlobs(root: string): string[] { + let raw: string; + try { + const configPath = resolve(root, ".mex", "config.json"); + const stats = statSync(configPath); + if (!stats.isFile() || stats.size > GRAPH_IGNORE_CONFIG_LIMITS.maxConfigBytes) return []; + raw = readFileSync(configPath, "utf8"); + } catch { + return []; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return []; + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return []; + const graph = (parsed as Record).graph; + if (typeof graph !== "object" || graph === null || Array.isArray(graph)) return []; + const ignore = (graph as Record).ignore; + if (!Array.isArray(ignore)) return []; + const globs = new Set(); + for (const entry of ignore) { + if (typeof entry !== "string") continue; + const glob = entry.trim(); + if (!glob || glob.length > GRAPH_IGNORE_CONFIG_LIMITS.maxGlobLength) continue; + // Absolute paths and upward traversal describe files outside the corpus, + // which discovery already refuses to walk. Accept only repo-relative globs. + if (isAbsolute(glob) || glob.startsWith("../") || glob.startsWith("..\\")) continue; + globs.add(glob.split("\\").join("/")); + if (globs.size >= GRAPH_IGNORE_CONFIG_LIMITS.maxGlobs) break; + } + return [...globs].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); +} + +/** The complete ignore list for one repository: frozen defaults, then config. */ +export function graphCorpusIgnoreGlobs(root: string): string[] { + return [...GRAPH_CORPUS_IGNORE_GLOBS, ...readConfiguredGraphIgnoreGlobs(root)]; +} + export const GRAPH_CONFIG_GLOBS = Object.freeze([ "package.json", "**/package.json", @@ -63,13 +131,48 @@ export const GRAPH_CORPUS_LIMITS = Object.freeze({ maxDiagnostics: 100, } as const); +export type GraphCorpusLimitName = keyof typeof GRAPH_CORPUS_LIMITS; + +/** + * Limits that describe **one file** rather than the whole corpus. + * + * The distinction has teeth. A corpus-wide ceiling says the repository is too + * big for a bounded run and there is no honest partial answer, so it aborts. + * A per-file ceiling says one file is too big to parse — every other file in + * the repository is still perfectly indexable, so the file is skipped and + * reported instead of taking the build down with it. + */ +export const GRAPH_PER_FILE_CORPUS_LIMITS: ReadonlySet = new Set([ + "maxSourceFileBytes", + "maxConfigFileBytes", +] as const); + export class GraphCorpusLimitError extends Error { readonly code = "GRAPH_CORPUS_LIMIT_EXCEEDED"; - constructor(readonly limit: keyof typeof GRAPH_CORPUS_LIMITS) { - super(`The graph corpus exceeded the configured ${limit} safety bound.`); + constructor( + readonly limit: GraphCorpusLimitName, + /** Observed size, when the breach was measured against a single file. */ + readonly observedBytes?: number, + ) { + super( + observedBytes === undefined + ? `The graph corpus exceeded the configured ${limit} safety bound.` + : `The graph corpus exceeded the configured ${limit} safety bound: ` + + `${observedBytes} bytes against a ${GRAPH_CORPUS_LIMITS[limit]}-byte limit.`, + ); this.name = "GraphCorpusLimitError"; } + + /** True when only this one file is affected and the run may continue. */ + get perFile(): boolean { + return GRAPH_PER_FILE_CORPUS_LIMITS.has(this.limit); + } +} + +/** Narrow an unknown error to a per-file corpus-limit breach. */ +export function isPerFileCorpusLimitError(error: unknown): error is GraphCorpusLimitError { + return error instanceof GraphCorpusLimitError && error.perFile; } /** Lazily consume glob results and stop before an unbounded path array forms. */ @@ -100,7 +203,10 @@ export function addGraphCorpusBytes( ? GRAPH_CORPUS_LIMITS.maxSourceBytes : GRAPH_CORPUS_LIMITS.maxConfigBytes; if (!Number.isSafeInteger(fileBytes) || fileBytes < 0 || fileBytes > maxFile) { - throw new GraphCorpusLimitError(kind === "source" ? "maxSourceFileBytes" : "maxConfigFileBytes"); + throw new GraphCorpusLimitError( + kind === "source" ? "maxSourceFileBytes" : "maxConfigFileBytes", + Number.isSafeInteger(fileBytes) ? fileBytes : undefined, + ); } const next = total + fileBytes; if (!Number.isSafeInteger(next) || next > maxTotal) { @@ -113,7 +219,10 @@ export function addGraphCorpusBytes( export function addGraphCompilerSourceBytes(total: number, fileBytes: number): number { if (!Number.isSafeInteger(fileBytes) || fileBytes < 0 || fileBytes > GRAPH_CORPUS_LIMITS.maxSourceFileBytes) { - throw new GraphCorpusLimitError("maxSourceFileBytes"); + throw new GraphCorpusLimitError( + "maxSourceFileBytes", + Number.isSafeInteger(fileBytes) ? fileBytes : undefined, + ); } const next = total + fileBytes; if (!Number.isSafeInteger(next) || next > GRAPH_CORPUS_LIMITS.maxCompilerSourceBytes) { @@ -179,3 +288,21 @@ export const GRAPH_CORPUS_POLICY_HASH = createHash("sha256").update(JSON.stringi globOptions: GRAPH_CORPUS_GLOB_OPTIONS, limits: GRAPH_CORPUS_LIMITS, })).digest("hex"); + +/** + * Discovery identity for one repository: the frozen policy, plus whatever the + * repository additionally chose to ignore. + * + * A repository with no configured globs hashes to exactly + * {@link GRAPH_CORPUS_POLICY_HASH}, so every index built before this existed + * stays valid. Adding or removing a configured glob changes the corpus, so it + * must change the manifest and force an explicit rebuild. + */ +export function graphCorpusPolicyHash(root: string): string { + const configured = readConfiguredGraphIgnoreGlobs(root); + if (configured.length === 0) return GRAPH_CORPUS_POLICY_HASH; + return createHash("sha256").update(JSON.stringify({ + base: GRAPH_CORPUS_POLICY_HASH, + configuredIgnoreGlobs: configured, + })).digest("hex"); +} diff --git a/src/graph/engine-impl.ts b/src/graph/engine-impl.ts index 1585b16d..598ad274 100644 --- a/src/graph/engine-impl.ts +++ b/src/graph/engine-impl.ts @@ -15,13 +15,15 @@ import { import { tmpdir } from "node:os"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { toPosix } from "../paths.js"; -import type { BuildResult, GraphEngine, NodeSearchOptions } from "./engine.js"; +import type { + BuildResult, GraphEngine, NodeSearchOptions, SkippedSourceFile, +} from "./engine.js"; import { GRAPH_CONFIG_GLOBS, GRAPH_CORPUS_GLOB_OPTIONS, - GRAPH_CORPUS_IGNORE_GLOBS, + graphCorpusIgnoreGlobs, GRAPH_CORPUS_LIMITS, - GRAPH_CORPUS_POLICY_HASH, + graphCorpusPolicyHash, GRAPH_SUPPORTED_SOURCE_GLOB, GraphCorpusLimitError, addGraphCompilerSourceBytes, @@ -29,6 +31,7 @@ import { addGraphSemanticInput, createGraphSemanticInputLedger, discoverBoundedGraphPaths, + isPerFileCorpusLimitError, } from "./corpus-policy.js"; import { DB_SCHEMA_VERSION, markGraphReady, openGraphDatabase } from "./db/database.js"; import { @@ -234,6 +237,14 @@ interface DiscoveredFile { modifiedAt: number; } +type GraphSkippedSourceFile = SkippedSourceFile; + +/** What one source walk found: the corpus, and what it declined to read. */ +interface DiscoveredCorpus { + files: DiscoveredFile[]; + skipped: GraphSkippedSourceFile[]; +} + interface StagedFile { discovered: DiscoveredFile; record: FileRecord; @@ -246,6 +257,8 @@ interface StagedFile { interface StagedCorpus { files: StagedFile[]; + /** Files discovered but not indexed, carried through to the build result. */ + skipped: GraphSkippedSourceFile[]; compiler: Pick; semanticInputs: CompilerSemanticInput[]; fingerprints: Array<{ nodeId: string; fingerprint: Fingerprint }>; @@ -343,7 +356,7 @@ class GraphEngineImpl implements GraphEngine { const store = this.getStore(true); const manifestChanged = store.getMetadata("manifest_hash") !== manifest.manifestHash; if (changedSources.length === 0 && !manifestChanged) { - const currentCorpus = discoverSourceFiles(this.rootDir, this.sourceFileAccess); + const currentCorpus = discoverSourceFiles(this.rootDir, this.sourceFileAccess).files; const snapshot = parseGraphSnapshot(store.getMetadata(GRAPH_SNAPSHOT_METADATA_KEY)); if (snapshot?.manifestHash === manifest.manifestHash && snapshot.indexedBranch === gitBeforeStaging.branch @@ -364,8 +377,13 @@ class GraphEngineImpl implements GraphEngine { ); try { const stagedByPath = new Map(staged.files.map((file) => [file.record.path, file])); + // A file the corpus policy deliberately skipped is absent from the + // staged corpus by design, so it is not evidence of a lost source. + const skippedPaths = new Set(staged.skipped.map((file) => file.filePath)); const unstagedExisting = changedSources.filter((file) => ( - !deletedChangedSources.has(file) && !stagedByPath.has(file) + !deletedChangedSources.has(file) + && !stagedByPath.has(file) + && !skippedPaths.has(file) )); if (unstagedExisting.length > 0) { throw new GraphSourceStagingError(unstagedExisting.map((filePath) => ({ @@ -490,6 +508,7 @@ class GraphEngineImpl implements GraphEngine { nodesCreated: expectedNodes, edgesCreated: edgeCount, health: healthCounts(staged.files), + ...(staged.skipped.length > 0 ? { skipped: [...staged.skipped] } : {}), }; } @@ -552,7 +571,7 @@ async function stageCorpus( ): Promise { const sourceSpool = new GraphSourceSpool(); try { - const discovered = discoverSourceFiles(root, sourceFileAccess, sourceSpool); + const { files: discovered, skipped } = discoverSourceFiles(root, sourceFileAccess, sourceSpool); const configSources = discoverGraphConfigSources(root); const stagedConfigHash = configHashForSources(configSources); if (stagedConfigHash !== manifest.configHash) { @@ -658,6 +677,7 @@ async function stageCorpus( } return { files, + skipped, compiler: { compilerVersion: compiler.compilerVersion, semanticInputs: [...compiler.semanticInputs], @@ -1266,13 +1286,31 @@ function inspectChangedSources( return deleted; } +/** + * Walk the repository's supported sources. + * + * A file the bounded corpus policy will not read is **skipped, not fatal.** + * One oversized generated file used to abort an entire multi-thousand-file + * build: the per-file limit error was pushed as a staging failure, the loop + * `break`s so no later file was even attempted, and the accumulated failures + * were thrown. Every other file in the repository is still perfectly + * indexable, so a per-file ceiling now records the file and continues. + * + * Corpus-wide ceilings (`maxSourceBytes`, `maxSourceFiles`) still abort. They + * describe the whole run and there is no honest partial answer to them. + * + * The skip happens here, at the single discovery seam every consumer shares, + * so the staged corpus, `verifyPublicationInputs`, `sync`'s corpus comparison + * and the freshness inspector all agree about which files exist. + */ function discoverSourceFiles( root: string, sourceFileAccess: GraphSourceFileAccess, sourceSpool?: GraphSourceSpool, -): DiscoveredFile[] { +): DiscoveredCorpus { let matches: string[]; const files: DiscoveredFile[] = []; + const skipped: GraphSkippedSourceFile[] = []; const failures: GraphSourceStagingFailure[] = []; let canonicalRoot: string; try { @@ -1280,7 +1318,7 @@ function discoverSourceFiles( matches = discoverBoundedGraphPaths(GRAPH_SUPPORTED_SOURCE_GLOB, { ...GRAPH_CORPUS_GLOB_OPTIONS, cwd: root, - ignore: [...GRAPH_CORPUS_IGNORE_GLOBS], + ignore: graphCorpusIgnoreGlobs(root), }, GRAPH_CORPUS_LIMITS.maxSourceFiles).map(toPosix); } catch (error) { throw new GraphSourceStagingError([sourceStagingFailure(".", "discover", error)]); @@ -1312,8 +1350,11 @@ function discoverSourceFiles( sourceSpool?.stage(relPath, file.source); files.push({ relPath, contentHash, size: file.size, modifiedAt: file.modifiedAt }); } catch (error) { - failures.push(sourceStagingFailure(relPath, "read", error)); - if (error instanceof GraphCorpusLimitError) break; + if (isPerFileCorpusLimitError(error)) skipped.push(skippedSourceFile(relPath, error)); + else { + failures.push(sourceStagingFailure(relPath, "read", error)); + if (error instanceof GraphCorpusLimitError) break; + } } continue; } @@ -1348,12 +1389,29 @@ function discoverSourceFiles( }); sourceSpool?.stage(relPath, source); } catch (error) { - failures.push(sourceStagingFailure(relPath, "read", error)); - if (error instanceof GraphCorpusLimitError) break; + if (isPerFileCorpusLimitError(error)) skipped.push(skippedSourceFile(relPath, error)); + else { + failures.push(sourceStagingFailure(relPath, "read", error)); + if (error instanceof GraphCorpusLimitError) break; + } } } if (failures.length > 0) throw new GraphSourceStagingError(failures); - return files; + return { files, skipped }; +} + +function skippedSourceFile( + filePath: string, + error: GraphCorpusLimitError, +): GraphSkippedSourceFile { + return { + filePath, + reason: "corpus-limit", + limit: error.limit, + limitBytes: GRAPH_CORPUS_LIMITS[error.limit], + observedBytes: error.observedBytes, + message: error.message, + }; } function sourceCorpusMatchesFileRecords( @@ -1401,6 +1459,10 @@ function readStableUtf8File( canonicalPath: string, sourcePath = canonicalPath, maxBytes = GRAPH_CORPUS_LIMITS.maxSourceFileBytes, + // Named explicitly rather than inferred from `maxBytes`. The source and + // config per-file ceilings are numerically identical, so comparing the value + // reported every oversized *source* file as a config-limit breach. + limitName: "maxSourceFileBytes" | "maxConfigFileBytes" = "maxSourceFileBytes", ): { source: string; size: number; @@ -1419,9 +1481,8 @@ function readStableUtf8File( } if (!Number.isSafeInteger(opened.size) || opened.size < 0 || opened.size > maxBytes) { throw new GraphCorpusLimitError( - maxBytes === GRAPH_CORPUS_LIMITS.maxConfigFileBytes - ? "maxConfigFileBytes" - : "maxSourceFileBytes", + limitName, + Number.isSafeInteger(opened.size) ? opened.size : undefined, ); } const source = readFileSync(fd, "utf8"); @@ -1588,7 +1649,7 @@ function verifyPublicationInputs( sourceFileAccess: GraphSourceFileAccess, internal: GraphEngineInternalHooks = {}, ): GraphGitProvenance { - const currentFiles = discoverSourceFiles(root, sourceFileAccess); + const currentFiles = discoverSourceFiles(root, sourceFileAccess).files; const currentManifest = graphManifest(root); const failures: GraphSourceStagingFailure[] = []; @@ -1686,7 +1747,7 @@ export function graphManifest(root: string): GraphManifest { compiler: TYPESCRIPT_COMPILER_VERSION, extractor: CORPUS_EXTRACTOR_VERSION, resolver: RESOLVER_VERSION, - corpusPolicyHash: GRAPH_CORPUS_POLICY_HASH, + corpusPolicyHash: graphCorpusPolicyHash(root), grammarHash, configHash, })); @@ -1701,7 +1762,7 @@ function discoverGraphConfigSources(root: string): Map { configPaths = discoverBoundedGraphPaths([...GRAPH_CONFIG_GLOBS], { ...GRAPH_CORPUS_GLOB_OPTIONS, cwd: root, - ignore: [...GRAPH_CORPUS_IGNORE_GLOBS], + ignore: graphCorpusIgnoreGlobs(root), }, GRAPH_CORPUS_LIMITS.maxConfigFiles).map(toPosix); } catch (error) { throw new GraphSourceStagingError([sourceStagingFailure(".", "discover", error)]); @@ -1714,6 +1775,7 @@ function discoverGraphConfigSources(root: string): Map { canonicalPath, resolve(root, path), GRAPH_CORPUS_LIMITS.maxConfigFileBytes, + "maxConfigFileBytes", ); configBytes = addGraphCorpusBytes( configBytes, diff --git a/src/graph/engine.ts b/src/graph/engine.ts index 51bf9470..32e9594f 100644 --- a/src/graph/engine.ts +++ b/src/graph/engine.ts @@ -24,6 +24,24 @@ import { NotImplementedError } from "./errors.js"; // Value types // ---------------------------------------------------------------------------- +/** + * One repository file the bounded corpus policy declined to index. + * + * A skipped file is a reported outcome, not a failure: the rest of the + * repository indexed normally. It is surfaced so a user learns why a symbol is + * missing from the graph instead of concluding the graph is wrong. + */ +export interface SkippedSourceFile { + filePath: string; + reason: "corpus-limit"; + /** The bounded-policy limit the file breached. */ + limit: string; + limitBytes: number; + /** Observed size, when it was measured. */ + observedBytes?: number; + message: string; +} + /** Summary of a build/sync pass — for the `mex graph` CLI. */ export interface BuildResult { filesIndexed: number; @@ -35,6 +53,12 @@ export interface BuildResult { partial: number; failed: number; }; + /** + * Files discovered but deliberately not indexed. Sits beside `health` + * rather than inside it: `health` counts files that *are* in the graph and + * how well they parsed, and a skipped file is in none of those buckets. + */ + skipped?: SkippedSourceFile[]; } /** Options for {@link GraphEngine.searchNodes}. */ diff --git a/src/graph/runtime.ts b/src/graph/runtime.ts index 73c3eb2e..36d9bb33 100644 --- a/src/graph/runtime.ts +++ b/src/graph/runtime.ts @@ -8,7 +8,7 @@ import { createGraphEngine } from "./engine-impl.js"; import type { GraphEngine } from "./engine.js"; import { GRAPH_CORPUS_GLOB_OPTIONS, - GRAPH_CORPUS_IGNORE_GLOBS, + graphCorpusIgnoreGlobs, GRAPH_SUPPORTED_SOURCE_GLOB, } from "./corpus-policy.js"; import { openGraphDatabase } from "./db/database.js"; @@ -247,7 +247,7 @@ export function findChangedSourceFiles(projectRoot: string, db: SqliteDatabase): const current = globSync(GRAPH_SUPPORTED_SOURCE_GLOB, { ...GRAPH_CORPUS_GLOB_OPTIONS, cwd: projectRoot, - ignore: [...GRAPH_CORPUS_IGNORE_GLOBS], + ignore: graphCorpusIgnoreGlobs(projectRoot), }) .map((path) => path.replaceAll("\\", "/")); const changed: string[] = []; diff --git a/src/graph/status.ts b/src/graph/status.ts index a350e510..929d1af8 100644 --- a/src/graph/status.ts +++ b/src/graph/status.ts @@ -25,8 +25,9 @@ import { openSqlite, type SqliteDatabase } from "./db/sqlite.js"; import { BANDS, K } from "./config.js"; import { GRAPH_CORPUS_GLOB_OPTIONS, - GRAPH_CORPUS_IGNORE_GLOBS, + graphCorpusIgnoreGlobs, GRAPH_CORPUS_LIMITS, + isPerFileCorpusLimitError, GRAPH_SUPPORTED_SOURCE_GLOB, GraphCorpusLimitError, addGraphCorpusBytes, @@ -1702,7 +1703,7 @@ function inspectLiveSources( matches = discoverBoundedGraphPaths(GRAPH_SUPPORTED_SOURCE_GLOB, { ...GRAPH_CORPUS_GLOB_OPTIONS, cwd: projectRoot, - ignore: [...GRAPH_CORPUS_IGNORE_GLOBS], + ignore: graphCorpusIgnoreGlobs(projectRoot), }, GRAPH_CORPUS_LIMITS.maxSourceFiles) .map(toPosix) .filter(isSupportedSourceFile) @@ -1738,6 +1739,19 @@ function inspectLiveSources( ); hashes.set(path, sha256(content)); } catch (error) { + // A file the bounded policy will not read for its own size is skipped by + // indexing too, so the corpus observation stays complete and the walk + // continues. Only a corpus-wide breach makes the observation partial. + if (isPerFileCorpusLimitError(error)) { + discoveredPaths.delete(path); + reportPathDiagnostic({ + code: "GRAPH_SOURCE_FILE_SKIPPED", + severity: "warning", + message: `Supported source file ${path} is not indexed: ${errorMessage(error)}`, + path, + }); + continue; + } complete = false; const code = errorCode(error); const outsideProject = code === "GRAPH_CONTAINED_FILE_OUTSIDE_PROJECT"; From 411731a7fb2bef1a86ac096f3f193de410c9bbe6 Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Mon, 7 Sep 2026 20:15:24 +0530 Subject: [PATCH 2/9] feat(graph): let a repository add its own corpus ignore globs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eight built-in ignore globs were frozen with no user configuration anywhere, so a repository containing one file the graph could not index had no way to exclude it short of editing its own source tree. Globs listed under "graph.ignore" in .mex/config.json are now appended to the built-in list. Additive only: a configuration cannot un-ignore node_modules or .mex, because the defaults are the floor rather than a default value to replace. The list is read defensively and bounded — a missing, oversized or malformed config contributes no extra globs rather than trading one hard abort for another. Configured globs enter the corpus policy hash, so changing them invalidates the index and forces an explicit rebuild. A repository that configures nothing hashes exactly as before, leaving existing indexes valid. All four corpus walks — indexing, config discovery, freshness inspection and the changed-file scan — now resolve the same per-repository list. Skipped files are reported by mex graph, with the size, the limit, and the config key that would exclude the path deliberately. --- docs/code-graph-support.md | 43 +++++++++- src/graph/__tests__/corpus-policy.test.ts | 99 +++++++++++++++++++++++ src/graph/__tests__/corpus-skip.test.ts | 94 +++++++++++++++++++++ src/graph/cli-graph.ts | 26 +++++- src/graph/maintenance.ts | 1 + src/team/contracts/graph.ts | 17 ++++ 6 files changed, 275 insertions(+), 5 deletions(-) create mode 100644 src/graph/__tests__/corpus-skip.test.ts diff --git a/docs/code-graph-support.md b/docs/code-graph-support.md index a0235a83..d34bed8f 100644 --- a/docs/code-graph-support.md +++ b/docs/code-graph-support.md @@ -127,6 +127,42 @@ legacy checks running” case in Unsupported source-language files are also skipped. A missing extractor does not make the rest of setup or drift checking fail. +### Files the corpus policy will not index + +The graph applies a bounded per-file size ceiling (2 MB) so one pathological +file cannot exhaust memory. A file over that ceiling is **skipped, not fatal**: +the rest of the repository is indexed normally, and the skipped files are +reported by name, size and limit in `mex graph` output and in the `skipped` +array of its `--json` result. + +Corpus-*wide* ceilings still abort the run. They describe the whole build and +there is no honest partial answer to "this repository is too large to index +within the bounded policy". + +### Excluding paths from the graph + +`node_modules`, `.git`, `dist`, `build`, `.mex`, `coverage`, `.next` and `out` +are always excluded. A repository can exclude more by listing globs under +`graph.ignore` in `.mex/config.json`: + +```json +{ + "graph": { + "ignore": ["vendor/**", "**/*.generated.ts"] + } +} +``` + +The list is **additive**: configured globs are appended to the built-in ones +and cannot un-ignore them, so `node_modules` and `.mex` stay excluded whatever +the configuration says. Globs are repository-relative; absolute paths and +upward traversal are ignored, and the list is bounded. A missing or malformed +config simply contributes no extra globs rather than failing a build. + +Changing this list changes which files the graph describes, so it changes the +build manifest and the next `mex graph status` will report the index as stale +until it is rebuilt. + ## Known limitations - **Ambiguous references stay unresolved.** The base resolver prefers a @@ -139,10 +175,9 @@ not make the rest of setup or drift checking fail. reflection, dependency injection, monkey-patching, or computed calls. - **Generated code is path-filtered, not identified semantically.** Common output trees such as `node_modules`, `dist`, `build`, `.next`, `out`, - `coverage`, and `.mex` are excluded by the source globs in - [`engine-impl.ts`](../src/graph/engine-impl.ts) and - [`runtime.ts`](../src/graph/runtime.ts). Generated files outside those paths - may still be indexed. + `coverage`, and `.mex` are excluded by the corpus policy in + [`corpus-policy.ts`](../src/graph/corpus-policy.ts). Generated files outside + those paths may still be indexed; add a `graph.ignore` glob to exclude them. - **Framework behavior is opt-in and narrow.** Express route-to-handler binding is the only framework fixture in v0.7.0. Other frameworks remain unsupported until their language extractor and resolver work merges. diff --git a/src/graph/__tests__/corpus-policy.test.ts b/src/graph/__tests__/corpus-policy.test.ts index f873e14a..f0a53bf5 100644 --- a/src/graph/__tests__/corpus-policy.test.ts +++ b/src/graph/__tests__/corpus-policy.test.ts @@ -4,8 +4,15 @@ import { dirname, join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { GRAPH_CORPUS_GLOB_OPTIONS, + GRAPH_CORPUS_IGNORE_GLOBS, GRAPH_CORPUS_LIMITS, + GRAPH_CORPUS_POLICY_HASH, + GRAPH_IGNORE_CONFIG_LIMITS, GraphCorpusLimitError, + graphCorpusIgnoreGlobs, + graphCorpusPolicyHash, + isPerFileCorpusLimitError, + readConfiguredGraphIgnoreGlobs, addGraphCompilerSourceBytes, addGraphCorpusBytes, addGraphSemanticInput, @@ -88,3 +95,95 @@ describe("graph corpus policy", () => { expect(coveredProbeLedger.semanticPaths.size).toBe(0); }); }); + +describe("per-file corpus limits", () => { + it("separates a single-file ceiling from a corpus-wide one", () => { + const perFile = new GraphCorpusLimitError("maxSourceFileBytes", 34_380_944); + const corpusWide = new GraphCorpusLimitError("maxSourceBytes"); + + expect(perFile.perFile).toBe(true); + expect(isPerFileCorpusLimitError(perFile)).toBe(true); + expect(corpusWide.perFile).toBe(false); + expect(isPerFileCorpusLimitError(corpusWide)).toBe(false); + expect(isPerFileCorpusLimitError(new Error("unrelated"))).toBe(false); + }); + + it("names the observed size and the limit on the first line", () => { + // The source and config per-file ceilings are numerically identical, so + // the limit cannot be recovered by comparing byte values. + expect(GRAPH_CORPUS_LIMITS.maxSourceFileBytes) + .toBe(GRAPH_CORPUS_LIMITS.maxConfigFileBytes); + expect(new GraphCorpusLimitError("maxSourceFileBytes", 34_380_944).message) + .toBe("The graph corpus exceeded the configured maxSourceFileBytes safety bound: " + + "34380944 bytes against a 2097152-byte limit."); + expect(() => addGraphCorpusBytes(0, GRAPH_CORPUS_LIMITS.maxSourceFileBytes + 1, "source")) + .toThrow(/maxSourceFileBytes safety bound: 2097153 bytes/u); + }); +}); + +describe("configured graph ignore globs", () => { + function withConfig(body: unknown): string { + const root = mkdtempSync(join(tmpdir(), "mex-graph-ignore-config-")); + roots.push(root); + mkdirSync(join(root, ".mex"), { recursive: true }); + writeFileSync( + join(root, ".mex", "config.json"), + typeof body === "string" ? body : JSON.stringify(body), + "utf8", + ); + return root; + } + + it("appends configured globs to the frozen defaults, never replacing them", () => { + const root = withConfig({ graph: { ignore: ["generated/**", "**/*.gen.ts"] } }); + + expect(readConfiguredGraphIgnoreGlobs(root)).toEqual(["**/*.gen.ts", "generated/**"]); + expect(graphCorpusIgnoreGlobs(root)).toEqual([ + ...GRAPH_CORPUS_IGNORE_GLOBS, + "**/*.gen.ts", + "generated/**", + ]); + }); + + it("cannot un-ignore a default, whatever the configuration says", () => { + const root = withConfig({ graph: { ignore: ["!**/node_modules/**", "!**/.mex/**"] } }); + + // Negations are appended like any other glob and cancel *earlier* patterns + // only in glob semantics we never rely on; the defaults still stand. + for (const glob of GRAPH_CORPUS_IGNORE_GLOBS) { + expect(graphCorpusIgnoreGlobs(root)).toContain(glob); + } + }); + + it("yields no extra globs rather than failing on a malformed or hostile config", () => { + expect(readConfiguredGraphIgnoreGlobs(withConfig("{ not json"))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs(withConfig([1, 2, 3]))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs(withConfig({ graph: "nope" }))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs(withConfig({ graph: { ignore: "nope" } }))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs(withConfig({ + graph: { ignore: [42, "", " ", "../escape/**", "C:/absolute/**", "/absolute/**"] }, + }))).toEqual([]); + expect(readConfiguredGraphIgnoreGlobs( + mkdtempSync(join(tmpdir(), "mex-graph-ignore-missing-")), + )).toEqual([]); + }); + + it("bounds the configured list", () => { + const tooMany = Array.from({ length: 500 }, (_, index) => `dir${index}/**`); + const tooLong = "x".repeat(GRAPH_IGNORE_CONFIG_LIMITS.maxGlobLength + 1); + const root = withConfig({ graph: { ignore: [...tooMany, tooLong] } }); + + const globs = readConfiguredGraphIgnoreGlobs(root); + expect(globs.length).toBe(GRAPH_IGNORE_CONFIG_LIMITS.maxGlobs); + expect(globs).not.toContain(tooLong); + }); + + it("keeps the discovery identity of a repository that configures nothing", () => { + const unconfigured = mkdtempSync(join(tmpdir(), "mex-graph-ignore-none-")); + roots.push(unconfigured); + const configured = withConfig({ graph: { ignore: ["generated/**"] } }); + + expect(graphCorpusPolicyHash(unconfigured)).toBe(GRAPH_CORPUS_POLICY_HASH); + expect(graphCorpusPolicyHash(configured)).not.toBe(GRAPH_CORPUS_POLICY_HASH); + }); +}); diff --git a/src/graph/__tests__/corpus-skip.test.ts b/src/graph/__tests__/corpus-skip.test.ts new file mode 100644 index 00000000..cae626fc --- /dev/null +++ b/src/graph/__tests__/corpus-skip.test.ts @@ -0,0 +1,94 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { GRAPH_CORPUS_LIMITS } from "../corpus-policy.js"; +import { createGraphEngine } from "../engine-impl.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function temporaryRoot(): string { + const root = mkdtempSync(join(tmpdir(), "mex-graph-corpus-skip-")); + roots.push(root); + return root; +} + +function write(root: string, path: string, source: string): void { + const absolute = join(root, path); + mkdirSync(dirname(absolute), { recursive: true }); + writeFileSync(absolute, source, "utf8"); +} + +/** A syntactically valid module far above the per-file ceiling. */ +function oversizedModule(): string { + const filler = `// ${"x".repeat(120)}\n`; + return `export const oversized = true;\n` + + filler.repeat(Math.ceil(GRAPH_CORPUS_LIMITS.maxSourceFileBytes / filler.length) + 1); +} + +describe("oversized files do not abort a graph build", () => { + it("indexes every other file and reports the one it skipped", async () => { + const root = temporaryRoot(); + write(root, "src/service.ts", "export function servicePrimary(): number { return 1; }\n"); + write(root, "src/other.ts", "export function otherPrimary(): number { return 2; }\n"); + write(root, "src/generated.ts", oversizedModule()); + const engine = createGraphEngine({ rootDir: root }); + + try { + const result = await engine.build(); + + expect(result.filesIndexed).toBe(2); + expect(result.skipped).toEqual([{ + filePath: "src/generated.ts", + reason: "corpus-limit", + limit: "maxSourceFileBytes", + limitBytes: GRAPH_CORPUS_LIMITS.maxSourceFileBytes, + observedBytes: expect.any(Number), + message: expect.stringContaining("maxSourceFileBytes safety bound:"), + }]); + expect(engine.getIndexedFiles?.().map((file) => file.path)) + .toEqual(["src/other.ts", "src/service.ts"]); + } finally { + engine.close(); + } + }, 60_000); + + it("does not treat a skipped file as a lost source on sync", async () => { + const root = temporaryRoot(); + write(root, "src/service.ts", "export function syncedPrimary(): number { return 1; }\n"); + write(root, "src/generated.ts", oversizedModule()); + const engine = createGraphEngine({ rootDir: root }); + + try { + await engine.build(); + // The skipped file is absent from the staged corpus by design, so naming + // it as changed must not read as a source that disappeared mid-build. + const result = await engine.sync(["src/generated.ts"]); + + expect(result.skipped?.map((file) => file.filePath)).toEqual(["src/generated.ts"]); + } finally { + engine.close(); + } + }, 60_000); + + it("honours an additive ignore glob configured by the repository", async () => { + const root = temporaryRoot(); + write(root, "src/service.ts", "export function keptPrimary(): number { return 1; }\n"); + write(root, "vendor/bundle.ts", "export function droppedPrimary(): number { return 2; }\n"); + write(root, ".mex/config.json", JSON.stringify({ graph: { ignore: ["vendor/**"] } })); + const engine = createGraphEngine({ rootDir: root }); + + try { + const result = await engine.build(); + + expect(result.filesIndexed).toBe(1); + expect(engine.getIndexedFiles?.().map((file) => file.path)).toEqual(["src/service.ts"]); + } finally { + engine.close(); + } + }, 60_000); +}); diff --git a/src/graph/cli-graph.ts b/src/graph/cli-graph.ts index 132cc015..1c9bcb11 100644 --- a/src/graph/cli-graph.ts +++ b/src/graph/cli-graph.ts @@ -1,4 +1,6 @@ -import type { GraphSourceChanges, GraphStatus } from "../team/contracts/graph.js"; +import type { + GraphRefreshResult, GraphSourceChanges, GraphStatus, +} from "../team/contracts/graph.js"; import { GraphMaintenanceError, repairGraph, @@ -54,6 +56,7 @@ export async function runGraph(options: GraphCommandOptions = {}): Promise partial: result.status.parseHealth.partial, failed: result.status.parseHealth.failed, }, + ...(result.skipped && result.skipped.length > 0 ? { skipped: result.skipped } : {}), }, null, 2)); return; } @@ -61,8 +64,28 @@ export async function runGraph(options: GraphCommandOptions = {}): Promise `Code graph built: ${result.nodesCreated} nodes, ${result.edgesCreated} edges ` + `across ${result.filesIndexed} files in ${result.durationMs}ms → .mex/graph.db`, ); + printSkippedSources(result.skipped); } +/** + * Name the files that are deliberately missing from the graph. + * + * Silence here is what made an oversized file look like a graph bug: a symbol + * was simply absent with nothing to explain it. + */ +function printSkippedSources(skipped: GraphRefreshResult["skipped"]): void { + if (!skipped || skipped.length === 0) return; + const shown = skipped.slice(0, MAX_SKIPPED_PATHS_SHOWN); + console.log(`Skipped ${skipped.length} file(s) the bounded corpus policy will not index:`); + for (const file of shown) console.log(` ${file.filePath} — ${file.message}`); + const omitted = skipped.length - shown.length; + if (omitted > 0) console.log(` …and ${omitted} more (use --json for the full list)`); + console.log("Add a glob to \"graph.ignore\" in .mex/config.json to exclude a path deliberately."); +} + +/** Human output stays bounded; `--json` carries the complete list. */ +const MAX_SKIPPED_PATHS_SHOWN = 10; + function printStatus(status: GraphStatus): void { const branch = status.currentRepo.branch ?? "detached/no branch"; const head = status.currentRepo.head?.slice(0, 12) ?? "no HEAD"; @@ -103,6 +126,7 @@ function printMaintenance(verb: "refreshed" | "rebuilt", result: GraphMaintenanc `Code graph ${verb}: ${result.nodesCreated} nodes, ${result.edgesCreated} edges ` + `across ${result.filesIndexed} files in ${result.durationMs}ms; status ${result.status.status}.`, ); + printSkippedSources(result.skipped); if (result.recoveryPath) { console.log(`Previous index retained for local recovery: ${result.recoveryPath}`); } diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index 07a6859f..558c7ee2 100644 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -1312,6 +1312,7 @@ function maintenanceResult( filesIndexed: build.filesIndexed, nodesCreated: build.nodesCreated, edgesCreated: build.edgesCreated, + ...(build.skipped && build.skipped.length > 0 ? { skipped: build.skipped } : {}), ...(published.recoveryPath ? { recoveryPath: published.recoveryPath } : {}), }; } diff --git a/src/team/contracts/graph.ts b/src/team/contracts/graph.ts index e03272f6..42ba4364 100644 --- a/src/team/contracts/graph.ts +++ b/src/team/contracts/graph.ts @@ -76,6 +76,21 @@ export interface GraphMaintenanceOptions { onProgress?: (progress: GraphMaintenanceProgress) => void; } +/** + * One repository file the bounded corpus policy declined to index. + * + * Reported rather than fatal: the rest of the repository indexed normally, + * and this is how a user learns why a symbol is absent from the graph. + */ +export interface GraphSkippedSource { + filePath: RepoRelativePath; + reason: "corpus-limit"; + limit: string; + limitBytes: number; + observedBytes?: number; + message: string; +} + /** Port mutations return only successful results; failures throw typed errors. */ export interface GraphRefreshResult extends IndexJobResult { state: "succeeded"; @@ -83,6 +98,8 @@ export interface GraphRefreshResult extends IndexJobResult { filesIndexed: number; nodesCreated: number; edgesCreated: number; + /** Optional and additive; absent when every discovered file was indexed. */ + skipped?: readonly GraphSkippedSource[]; } export interface GraphPage extends Page { From b1d1a13f017edc5cfe9378cd65735cbc5f4827d2 Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Mon, 7 Sep 2026 20:44:55 +0530 Subject: [PATCH 3/9] fix(graph): decline a config input outside the root instead of aborting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One `tsconfig.json` extending a package hoisted above the project root made a 517-file repository completely un-indexable. `readFile` had always treated an out-of-root source path as a soft miss; `readConfigFile` and `configFileExists` threw on the identical condition. The two halves of one class simply disagreed, and the thrown error surfaced as a staging failure against filePath "." — naming no useful path at all. `configFileExists` is a probe. TypeScript is asking whether a file is there, and the honest answer for a path we will not read is `false`, not an exception. `readConfigFile` returns undefined for the same reason. The two sibling guards are treated the same way, because they are the same defect: a tsconfig `include` above the root now matches no files, and a project reference above the root is not traversed. A monorepo sub-package routinely references its siblings and must not become un-indexable for it. Every decline is recorded and reported, so a repository whose type graph is less complete than its config asks for is told. Paths are reported by dependency specifier rather than absolutely: TypeScript resolves a bare `extends` by walking every ancestor directory, so one unresolvable specifier otherwise produced a dozen near-identical entries carrying absolute paths from outside the repository, including a user's home directory. The containment guard is unchanged and still declines. Nothing outside the project root is read, and nothing outside it enters graph provenance — now asserted directly rather than implied by an abort. --- docs/code-graph-support.md | 17 +++ .../__tests__/compiler-containment.test.ts | 102 ++++++++++++++ src/graph/__tests__/snapshot.test.ts | 29 ++-- src/graph/cli-graph.ts | 23 ++++ src/graph/engine-impl.ts | 8 +- src/graph/engine.ts | 14 ++ src/graph/extraction/compiler.ts | 128 ++++++++++++++++-- src/graph/maintenance.ts | 3 + src/team/contracts/graph.ts | 14 ++ 9 files changed, 313 insertions(+), 25 deletions(-) create mode 100644 src/graph/__tests__/compiler-containment.test.ts diff --git a/docs/code-graph-support.md b/docs/code-graph-support.md index d34bed8f..4eee5ff3 100644 --- a/docs/code-graph-support.md +++ b/docs/code-graph-support.md @@ -139,6 +139,23 @@ Corpus-*wide* ceilings still abort the run. They describe the whole build and there is no honest partial answer to "this repository is too large to index within the bounded policy". +### Config inputs outside the project + +mex never reads a file outside the repository root, and a TypeScript config +routinely points at one: `"extends": "some-package/tsconfig"` resolves through +`node_modules`, which any hoisted pnpm/yarn layout — or a monorepo sub-package +indexed on its own — places above the indexed root. + +Such an input is **declined, not fatal**. The build finishes, the affected +project's type resolution is less complete than its config asks for, and the +declined inputs are reported by dependency specifier (never by absolute path) +in `mex graph` output and in the `declinedInputs` array of its `--json` result. +The same applies to a `tsconfig` `include` or project `reference` that points +above the root. + +The containment guard itself is unchanged: nothing outside the root is read, +and nothing outside the root enters the graph's provenance. + ### Excluding paths from the graph `node_modules`, `.git`, `dist`, `build`, `.mex`, `coverage`, `.next` and `out` diff --git a/src/graph/__tests__/compiler-containment.test.ts b/src/graph/__tests__/compiler-containment.test.ts new file mode 100644 index 00000000..b9877c26 --- /dev/null +++ b/src/graph/__tests__/compiler-containment.test.ts @@ -0,0 +1,102 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildTypeScriptExtraction } from "../extraction/compiler.js"; +import { createGraphEngine } from "../engine-impl.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +/** A project nested one level down, so `..` is a real directory outside it. */ +function nestedProject(): { workspace: string; root: string } { + const workspace = mkdtempSync(join(tmpdir(), "mex-graph-containment-")); + roots.push(workspace); + const root = join(workspace, "package"); + mkdirSync(root, { recursive: true }); + return { workspace, root }; +} + +function write(root: string, path: string, source: string): void { + const absolute = join(root, path); + mkdirSync(dirname(absolute), { recursive: true }); + writeFileSync(absolute, source, "utf8"); +} + +describe("compiler input containment declines rather than aborting", () => { + it("finishes a build whose tsconfig extends a package hoisted above the root", async () => { + const { workspace, root } = nestedProject(); + // The hoisted layout every pnpm/yarn workspace produces: the package the + // config extends resolves above the indexed root. + write(workspace, "node_modules/shared-config/tsconfig.json", + JSON.stringify({ compilerOptions: { strict: true } })); + write(workspace, "node_modules/shared-config/package.json", + JSON.stringify({ name: "shared-config", version: "1.0.0" })); + write(root, "tsconfig.json", JSON.stringify({ + extends: "shared-config/tsconfig.json", + compilerOptions: { target: "ES2022" }, + })); + write(root, "package.json", JSON.stringify({ name: "package", version: "1.0.0" })); + write(root, "src/service.ts", "export function containedPrimary(): number { return 1; }\n"); + const engine = createGraphEngine({ rootDir: root }); + + try { + const result = await engine.build(); + + expect(result.filesIndexed).toBe(1); + // The config was declined, not read — and reported by dependency + // specifier rather than by absolute path. + expect(result.declinedInputs).toContainEqual(expect.objectContaining({ + filePath: "node_modules/shared-config/tsconfig.json", + reason: "outside-project-corpus", + })); + for (const input of result.declinedInputs ?? []) { + expect(input.filePath.startsWith("node_modules/")).toBe(true); + } + expect(engine.searchNodes("containedPrimary").length).toBeGreaterThan(0); + } finally { + engine.close(); + } + }, 60_000); + + it("declines an include and a project reference above the root without throwing", () => { + const { workspace, root } = nestedProject(); + write(workspace, "sibling/tsconfig.json", JSON.stringify({ compilerOptions: {} })); + write(workspace, "sibling/src/other.ts", "export const other = 1;\n"); + write(root, "tsconfig.json", JSON.stringify({ + compilerOptions: { composite: true }, + include: ["src/**/*", "../sibling/src/**/*"], + references: [{ path: "../sibling" }], + })); + write(root, "src/service.ts", "export function referencedPrimary(): number { return 1; }\n"); + + const result = buildTypeScriptExtraction(root, ["src/service.ts"], { + stagedInputs: [ + { filePath: "tsconfig.json", source: JSON.stringify({ + compilerOptions: { composite: true }, + include: ["src/**/*", "../sibling/src/**/*"], + references: [{ path: "../sibling" }], + }) }, + { filePath: "src/service.ts", + source: "export function referencedPrimary(): number { return 1; }\n" }, + ], + }); + + expect(result.files.map((file) => file.filePath)).toEqual(["src/service.ts"]); + // The referenced project above the root is declined by path rather than + // traversed. (The `..` include is not declined here: TypeScript invokes + // readDirectory with the config's own directory and applies the include + // patterns inside its matcher, so that guard is reached only when a config + // names an out-of-root directory as the matcher root.) + expect(result.declinedInputs.map((input) => input.filePath)) + .toEqual(["../sibling/tsconfig.json"]); + expect(result.declinedInputs[0]?.reason).toBe("outside-project-corpus"); + // Nothing outside the root reached the graph's provenance. + for (const input of result.semanticInputs) { + expect(input.filePath.startsWith("..")).toBe(false); + } + }); +}); diff --git a/src/graph/__tests__/snapshot.test.ts b/src/graph/__tests__/snapshot.test.ts index 61d9ab00..5db65970 100644 --- a/src/graph/__tests__/snapshot.test.ts +++ b/src/graph/__tests__/snapshot.test.ts @@ -369,7 +369,12 @@ describe("graph snapshot provenance", () => { engine.close(); }); - it("rejects a direct external tsconfig extension and preserves prior provenance", async () => { + // This asserted that an external `extends` aborted the build. Declining a + // config input no longer costs a repository its entire graph — a tsconfig + // extending a hoisted package is ordinary in any pnpm/yarn workspace. What + // this test actually protects, and still asserts, is that the external file + // is never read and never becomes graph provenance. + it("declines a direct external tsconfig extension without reading it", async () => { const root = temporaryRoot("mex-graph-config-direct-escape-"); const externalRoot = temporaryRoot("mex-graph-config-direct-external-"); const sourcePath = join(root, "src", "stable.ts"); @@ -388,14 +393,20 @@ describe("graph snapshot provenance", () => { extends: externalConfig, include: ["src/**/*.ts"], })); - await expect(engine.sync(["tsconfig.json"])).rejects.toMatchObject({ - name: "GraphSourceStagingError", - failures: [expect.objectContaining({ - filePath: ".", - code: "GRAPH_SOURCE_PATH_ESCAPE", - })], - }); - expect(metadata(dbPath, GRAPH_SNAPSHOT_METADATA_KEY)).toBe(successfulSnapshot); + const result = await engine.sync(["tsconfig.json"]); + + expect(result.declinedInputs).toContainEqual(expect.objectContaining({ + reason: "outside-project-corpus", + })); + // The external file is not read, so it is in no snapshot and no compiler + // option it declares can reach the graph. + const snapshot = parseGraphSnapshot(metadata(dbPath, GRAPH_SNAPSHOT_METADATA_KEY)); + expect(snapshot).not.toBeNull(); + expect(successfulSnapshot).not.toBeNull(); + for (const input of snapshot!.semanticInputs) { + expect(input.filePath.startsWith("..")).toBe(false); + expect(input.filePath).not.toContain("attacker"); + } engine.close(); }); diff --git a/src/graph/cli-graph.ts b/src/graph/cli-graph.ts index 1c9bcb11..04d9cb64 100644 --- a/src/graph/cli-graph.ts +++ b/src/graph/cli-graph.ts @@ -57,6 +57,9 @@ export async function runGraph(options: GraphCommandOptions = {}): Promise failed: result.status.parseHealth.failed, }, ...(result.skipped && result.skipped.length > 0 ? { skipped: result.skipped } : {}), + ...(result.declinedInputs && result.declinedInputs.length > 0 + ? { declinedInputs: result.declinedInputs } + : {}), }, null, 2)); return; } @@ -65,6 +68,7 @@ export async function runGraph(options: GraphCommandOptions = {}): Promise + `across ${result.filesIndexed} files in ${result.durationMs}ms → .mex/graph.db`, ); printSkippedSources(result.skipped); + printDeclinedInputs(result.declinedInputs); } /** @@ -83,6 +87,24 @@ function printSkippedSources(skipped: GraphRefreshResult["skipped"]): void { console.log("Add a glob to \"graph.ignore\" in .mex/config.json to exclude a path deliberately."); } +/** + * Name the config inputs the graph refused to read. + * + * Every source file is still in the graph; what is reduced is type resolution + * for the affected project, which is otherwise invisible. + */ +function printDeclinedInputs(declined: GraphRefreshResult["declinedInputs"]): void { + if (!declined || declined.length === 0) return; + const shown = declined.slice(0, MAX_SKIPPED_PATHS_SHOWN); + console.log( + `Declined ${declined.length} TypeScript config input(s) outside the project; ` + + "type resolution for the affected projects is less complete:", + ); + for (const input of shown) console.log(` ${input.filePath}`); + const omitted = declined.length - shown.length; + if (omitted > 0) console.log(` …and ${omitted} more (use --json for the full list)`); +} + /** Human output stays bounded; `--json` carries the complete list. */ const MAX_SKIPPED_PATHS_SHOWN = 10; @@ -127,6 +149,7 @@ function printMaintenance(verb: "refreshed" | "rebuilt", result: GraphMaintenanc + `across ${result.filesIndexed} files in ${result.durationMs}ms; status ${result.status.status}.`, ); printSkippedSources(result.skipped); + printDeclinedInputs(result.declinedInputs); if (result.recoveryPath) { console.log(`Previous index retained for local recovery: ${result.recoveryPath}`); } diff --git a/src/graph/engine-impl.ts b/src/graph/engine-impl.ts index 598ad274..f16c58e1 100644 --- a/src/graph/engine-impl.ts +++ b/src/graph/engine-impl.ts @@ -16,7 +16,7 @@ import { tmpdir } from "node:os"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { toPosix } from "../paths.js"; import type { - BuildResult, GraphEngine, NodeSearchOptions, SkippedSourceFile, + BuildResult, DeclinedCompilerInput, GraphEngine, NodeSearchOptions, SkippedSourceFile, } from "./engine.js"; import { GRAPH_CONFIG_GLOBS, @@ -259,6 +259,8 @@ interface StagedCorpus { files: StagedFile[]; /** Files discovered but not indexed, carried through to the build result. */ skipped: GraphSkippedSourceFile[]; + /** Config inputs the containment policy declined during compiler extraction. */ + declinedInputs: DeclinedCompilerInput[]; compiler: Pick; semanticInputs: CompilerSemanticInput[]; fingerprints: Array<{ nodeId: string; fingerprint: Fingerprint }>; @@ -509,6 +511,9 @@ class GraphEngineImpl implements GraphEngine { edgesCreated: edgeCount, health: healthCounts(staged.files), ...(staged.skipped.length > 0 ? { skipped: [...staged.skipped] } : {}), + ...(staged.declinedInputs.length > 0 + ? { declinedInputs: [...staged.declinedInputs] } + : {}), }; } @@ -678,6 +683,7 @@ async function stageCorpus( return { files, skipped, + declinedInputs: [...compiler.declinedInputs], compiler: { compilerVersion: compiler.compilerVersion, semanticInputs: [...compiler.semanticInputs], diff --git a/src/graph/engine.ts b/src/graph/engine.ts index 32e9594f..92929592 100644 --- a/src/graph/engine.ts +++ b/src/graph/engine.ts @@ -59,6 +59,20 @@ export interface BuildResult { * how well they parsed, and a skipped file is in none of those buckets. */ skipped?: SkippedSourceFile[]; + /** + * Compiler config inputs outside the project corpus that were declined. + * + * Distinct from `skipped`: no source file is missing from the graph, but the + * affected project's type resolution is less complete than it looks. + */ + declinedInputs?: DeclinedCompilerInput[]; +} + +/** A compiler input the containment policy declined to read. */ +export interface DeclinedCompilerInput { + filePath: string; + reason: "outside-project-corpus"; + message: string; } /** Options for {@link GraphEngine.searchNodes}. */ diff --git a/src/graph/extraction/compiler.ts b/src/graph/extraction/compiler.ts index 705a89a8..a6724eb8 100644 --- a/src/graph/extraction/compiler.ts +++ b/src/graph/extraction/compiler.ts @@ -175,6 +175,19 @@ export interface DiscoveredTypeScriptProject { diagnostics: CompilerDiagnosticSummary[]; } +/** + * A compiler input the containment policy declined to read. + * + * Reported rather than fatal. The resulting type graph is less complete for + * the affected project, and a user has to be told that rather than left to + * infer it from missing edges. + */ +export interface DeclinedCompilerInput { + filePath: string; + reason: "outside-project-corpus"; + message: string; +} + export interface CompilerExtractionResult { compilerVersion: string; extractorVersion: string; @@ -182,6 +195,8 @@ export interface CompilerExtractionResult { files: CompilerFileExtraction[]; /** Repository inputs whose exact bytes influenced config parsing or compiler facts. */ semanticInputs: CompilerSemanticInput[]; + /** Config inputs outside the project corpus that were declined, not read. */ + declinedInputs: DeclinedCompilerInput[]; } export interface CompilerStagedInput { @@ -636,9 +651,35 @@ export function buildTypeScriptExtraction( projects: projectSummaries, files, semanticInputs: inputs.semanticInputs(), + declinedInputs: inputs.declinedConfigInputs(), }; } +/** Bounded like every other diagnostic channel; the list is deduplicated. */ +const MAX_DECLINED_CONFIG_INPUTS = 100; + +/** + * Name a declined path by what the user can act on. + * + * TypeScript resolves a bare `extends` specifier by walking every ancestor + * directory, so one unresolvable `extends "astro/tsconfigs/strict"` produces a + * decline at `/node_modules/...`, `/node_modules/...` and so on + * up to the filesystem root. Reporting each rung is noise, and it puts absolute + * paths from outside the repository — a user's home directory among them — into + * build output. + * + * Keying on the dependency specifier collapses those rungs to the one fact + * worth reporting: this package is not resolvable inside the project. Paths + * with no `node_modules` segment are reported relative to the root instead. + */ +function reportableDeclinedPath(root: string, fileName: string): string { + const absolute = normalizedAbsolute(absoluteCandidate(root, fileName)); + const segments = absolute.split("/"); + const lastDependencyRoot = segments.lastIndexOf("node_modules"); + if (lastDependencyRoot >= 0) return segments.slice(lastDependencyRoot).join("/"); + return normalizeRelative(relative(root, absolute)) || absolute; +} + /** Stable identity input shared with migration/invariant tests. */ export function canonicalCompilerIdentity(input: { filePath: string; @@ -726,6 +767,7 @@ class CompilerInputLedger { private readonly knownDirectories = new Set(); private readonly directoryFiles = new Map>(); private readonly directoryChildren = new Map>(); + private readonly declinedConfigs = new Map(); private candidates: string[] = []; constructor(root: string, private readonly options: CompilerExtractionOptions) { @@ -787,26 +829,76 @@ class CompilerInputLedger { return source; } + /** + * Read one config input, or decline it. + * + * A config path outside the project root used to throw, which took the + * entire build down: a `tsconfig.json` extending a package hoisted above the + * root — any pnpm/yarn hoisted layout, or a monorepo sub-package indexed on + * its own — made the repository un-indexable. `readFile` had always treated + * the identical condition as a soft miss for *source* files; the two halves + * of this class simply disagreed. + * + * The containment guard is unchanged and still declines. Declining now + * returns "no such config", which is the honest answer for a file we will + * not read, and is recorded so the degradation is reported rather than + * silent. The type graph is less complete; the build finishes. + */ readConfigFile(fileName: string): string | undefined { const absolute = absoluteCandidate(this.root, fileName); - if (!withinRoot(this.root, absolute)) { - throw compilerInputContainmentError(`TypeScript config escapes the project root: ${fileName}`); - } - if (!this.isProjectInput(absolute) && !isAllowedCompilerDependency(absolute)) { - throw compilerInputContainmentError(`TypeScript config is outside the supported project corpus: ${fileName}`); - } + if (!this.permitsConfigInput(absolute, fileName)) return undefined; return this.readFile(absolute); } + /** + * Answer TypeScript's existence probe for a config path. + * + * This is a probe, not a read request. The honest answer for a path we have + * declined to read is `false`. + */ configFileExists(fileName: string): boolean { const absolute = absoluteCandidate(this.root, fileName); + if (!this.permitsConfigInput(absolute, fileName)) return false; + return this.fileExists(absolute); + } + + /** Record and decline a config input the containment policy will not read. */ + private permitsConfigInput(absolute: string, fileName: string): boolean { if (!withinRoot(this.root, absolute)) { - throw compilerInputContainmentError(`TypeScript config escapes the project root: ${fileName}`); + this.declineConfigInput(fileName, "escapes the project root"); + return false; } if (!this.isProjectInput(absolute) && !isAllowedCompilerDependency(absolute)) { - throw compilerInputContainmentError(`TypeScript config is outside the supported project corpus: ${fileName}`); + this.declineConfigInput(fileName, "is outside the supported project corpus"); + return false; } - return this.fileExists(absolute); + return true; + } + + private declineConfigInput(fileName: string, reason: string): void { + const reported = reportableDeclinedPath(this.root, fileName); + if (this.declinedConfigs.has(reported)) return; + if (this.declinedConfigs.size >= MAX_DECLINED_CONFIG_INPUTS) return; + this.declinedConfigs.set( + reported, + `TypeScript config ${reason} and was not read: ${reported}`, + ); + } + + /** Record a referenced project outside the root, which we do not traverse. */ + declineProjectReference(referencePath: string): void { + this.declineConfigInput(referencePath, "project reference escapes the project root"); + } + + /** Config inputs the containment policy declined, in deterministic order. */ + declinedConfigInputs(): DeclinedCompilerInput[] { + return [...this.declinedConfigs.entries()] + .sort(([left], [right]) => compareCodePoints(left, right)) + .map(([filePath, message]) => ({ + filePath, + reason: "outside-project-corpus" as const, + message, + })); } fileExists(fileName: string): boolean { @@ -852,8 +944,13 @@ class CompilerInputLedger { includes: readonly string[], depth?: number, ): string[] { + // Treated exactly like a declined config read. A tsconfig `include` that + // points above the project root describes files we will not walk, and the + // honest answer for a directory we decline is "no matching files" — not an + // exception that costs the whole repository its graph. if (!withinRoot(this.root, absoluteCandidate(this.root, rootDir))) { - throw compilerInputContainmentError(`TypeScript config include escapes the project root: ${rootDir}`); + this.declineConfigInput(rootDir, "include escapes the project root"); + return []; } // TypeScript's matcher is intentionally used with a virtual directory tree // built only from the immutable candidate list. This preserves exact @@ -1014,8 +1111,13 @@ function parseProjects( diagnostics.push(...parsed.errors); for (const reference of parsed.projectReferences ?? []) { const referencePath = normalizedAbsolute(ts.resolveProjectReferencePath(reference)); + // Same decision as a declined config read: a referenced project above + // the root is not traversed, and is reported rather than fatal. A + // sub-package of a monorepo indexed on its own routinely references its + // siblings, and that must not make it un-indexable. if (!withinRoot(root, referencePath)) { - throw compilerInputContainmentError(`TypeScript project reference escapes the project root: ${referencePath}`); + inputs?.declineProjectReference(referencePath); + continue; } if (!seen.has(referencePath)) queue.push(referencePath); } @@ -2220,10 +2322,6 @@ function compareCodePoints(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; } -function compilerInputContainmentError(message: string): Error & { code: string } { - return Object.assign(new Error(message), { code: "GRAPH_SOURCE_PATH_ESCAPE" }); -} - function isAllowedCompilerDependency(absolutePath: string): boolean { return normalizeRelative(absolutePath).split("/").includes("node_modules"); } diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index 558c7ee2..0c0ecb47 100644 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -1313,6 +1313,9 @@ function maintenanceResult( nodesCreated: build.nodesCreated, edgesCreated: build.edgesCreated, ...(build.skipped && build.skipped.length > 0 ? { skipped: build.skipped } : {}), + ...(build.declinedInputs && build.declinedInputs.length > 0 + ? { declinedInputs: build.declinedInputs } + : {}), ...(published.recoveryPath ? { recoveryPath: published.recoveryPath } : {}), }; } diff --git a/src/team/contracts/graph.ts b/src/team/contracts/graph.ts index 42ba4364..4db83cc1 100644 --- a/src/team/contracts/graph.ts +++ b/src/team/contracts/graph.ts @@ -100,6 +100,20 @@ export interface GraphRefreshResult extends IndexJobResult { edgesCreated: number; /** Optional and additive; absent when every discovered file was indexed. */ skipped?: readonly GraphSkippedSource[]; + /** Compiler config inputs outside the project corpus that were declined. */ + declinedInputs?: readonly GraphDeclinedInput[]; +} + +/** + * A compiler config input the containment policy declined to read. + * + * No source file is missing from the graph, but the affected project's type + * resolution is less complete than it looks, so it is reported. + */ +export interface GraphDeclinedInput { + filePath: string; + reason: "outside-project-corpus"; + message: string; } export interface GraphPage extends Page { From 1440a01382794dc1c89bdb186698d7b07c24ae24 Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Mon, 7 Sep 2026 20:53:43 +0530 Subject: [PATCH 4/9] feat(graph): give who-calls a fallback to recorded unresolved references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported in #115. A dynamically generated method has real call sites and no literal declaration, so no node resolves and who-calls answered TARGET_NOT_FOUND with no next step. The call sites were not lost — they were in unresolved_refs, which had no CLI or documentation surface anywhere. Recovering them required hand-written SQL against internal schema. who-calls now looks its target up among the recorded unresolved references before giving up, and reports those call sites with file, line, column and the node they were seen in. They are emitted as a distinct `unresolved-reference` record, never as `type: "result"`. An unresolved reference is not a resolved graph fact and an agent must not be able to confuse the two. They are charged to the same budget ledger as everything else and capped by maxNodes — common names accumulate hundreds of references and an uncapped fallback on a hot name would flood the caller — with the summary reporting the total that matched alongside what was returned. This path also emits proper meta and summary records, where it previously emitted a single bare error with neither, inconsistent with the rest of the protocol. status is "partial" and evidenceStrength is "weak", because that is what this evidence is. Scope is deliberately narrow. A name with neither a declaration nor a recorded reference still abstains with TARGET_NOT_FOUND, and where-defined, what-calls and impact are unchanged: they either resolve the requested declaration exactly or abstain. resolveSymbol still refuses fuzzy matching. This adds a second question before giving up, not a looser answer to the first. --- README.md | 2 + docs/code-graph-support.md | 41 ++++++ src/graph/__tests__/snapshot.test.ts | 4 +- .../__tests__/unresolved-fallback.test.ts | 134 ++++++++++++++++++ src/graph/cli-agent.ts | 117 +++++++++++++++ 5 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 src/graph/__tests__/unresolved-fallback.test.ts diff --git a/README.md b/README.md index d010d043..1cd7ae4a 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,8 @@ mex graph get mex impact requireSession ``` +When `who-calls` finds no declaration for a name that call sites nonetheless reference — a dynamically generated method, or a symbol defined outside the indexed corpus — it reports the recorded call sites as `unresolved-reference` records instead of a bare not-found. They are labelled as unresolved rather than returned as graph facts, and they are capped like every other response. See [Code graph support](docs/code-graph-support.md#unresolved-references). + MEX indexes TypeScript/TSX, JavaScript/JSX, Python, and Rust. Module variants such as `.mts`, `.cts`, `.mjs`, and `.cjs` have partial coverage, and Express is the only framework-specific resolver documented for 0.8. Exact `query`, `get`, and `impact` reads—and Hub Code—require a provably fresh Graph; `scope` can instead return bounded live-text evidence for stale or unindexed files, clearly marked `text-only`. ### Grounding and drift diff --git a/docs/code-graph-support.md b/docs/code-graph-support.md index 4eee5ff3..ff590738 100644 --- a/docs/code-graph-support.md +++ b/docs/code-graph-support.md @@ -180,6 +180,47 @@ Changing this list changes which files the graph describes, so it changes the build manifest and the next `mex graph status` will report the index as stale until it is rebuilt. +## Unresolved references + +Extraction records every reference it sees. The resolver then binds what it +can to a declaration and emits an edge; what it cannot bind stays recorded as +an unresolved reference. Those records are the graph being honest about its own +blind spots: a name a file referenced, that the resolver could not decide the +meaning of. + +They matter for `who-calls`. A dynamically generated method has real call sites +and no literal declaration, so no node resolves and the structural answer is +"not found" — accurate, and useless as a next step. When `who-calls` cannot +resolve its target, it now looks the name up among the recorded unresolved +references and reports those call sites: + +```bash +mex graph query who-calls mark_failed +``` + +```json +{"type":"unresolved-reference","relation":"who-calls","target":"mark_failed", + "name":"mark_failed","referenceKind":"calls","resolution":"unresolved", + "file":"app/models/job.rb","line":42,"col":8,"fromNode":"function:…", + "receiver":"job"} +``` + +Three properties of that output are deliberate: + +- **It is not a `result` record.** An unresolved reference is not a resolved + graph fact and an agent must not be able to confuse the two, so it carries + its own record type. +- **It is capped and charged to the same output budget** as every other + response. Common names accumulate hundreds of unresolved references, and an + uncapped fallback on a hot name would flood the caller. The `summary` reports + the total that matched alongside what was returned. +- **The response is a normal one**, with `meta` and `summary`, and a `summary` + whose `status` is `partial` and `evidenceStrength` is `weak`. + +A name with no declaration *and* no recorded reference still abstains with +`TARGET_NOT_FOUND`. `where-defined` and `what-calls` are unchanged: they +either resolve the requested declaration exactly or abstain. + ## Known limitations - **Ambiguous references stay unresolved.** The base resolver prefers a diff --git a/src/graph/__tests__/snapshot.test.ts b/src/graph/__tests__/snapshot.test.ts index 5db65970..0e005f70 100644 --- a/src/graph/__tests__/snapshot.test.ts +++ b/src/graph/__tests__/snapshot.test.ts @@ -404,8 +404,8 @@ describe("graph snapshot provenance", () => { expect(snapshot).not.toBeNull(); expect(successfulSnapshot).not.toBeNull(); for (const input of snapshot!.semanticInputs) { - expect(input.filePath.startsWith("..")).toBe(false); - expect(input.filePath).not.toContain("attacker"); + expect(input.path.startsWith("..")).toBe(false); + expect(input.path).not.toContain("attacker"); } engine.close(); }); diff --git a/src/graph/__tests__/unresolved-fallback.test.ts b/src/graph/__tests__/unresolved-fallback.test.ts new file mode 100644 index 00000000..e33b320d --- /dev/null +++ b/src/graph/__tests__/unresolved-fallback.test.ts @@ -0,0 +1,134 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { runGraphQuery } from "../cli-agent.js"; +import { openSqlite } from "../db/sqlite.js"; +import { createGraphEngine } from "../engine-impl.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function write(root: string, path: string, source: string): void { + const absolute = join(root, path); + mkdirSync(dirname(absolute), { recursive: true }); + writeFileSync(absolute, source, "utf8"); +} + +/** + * A repository whose call sites reference a name no declaration provides — + * the shape a dynamically generated method leaves behind. + */ +async function repositoryWithDynamicCalls(): Promise<{ root: string; dbPath: string }> { + const root = mkdtempSync(join(tmpdir(), "mex-graph-unresolved-")); + roots.push(root); + const dbPath = join(root, "graph.db"); + // `any` receivers leave a real call site with no declaration to bind it to, + // exactly as a dynamically generated method does. + write(root, "src/first.ts", [ + "export function firstCaller(subject: any): void {", + " subject.markFailed();", + "}", + "", + ].join("\n")); + write(root, "src/second.ts", [ + "export function secondCaller(subject: any): void {", + " subject.markFailed();", + "}", + "", + ].join("\n")); + const engine = createGraphEngine({ rootDir: root, dbPath }); + try { + await engine.build(); + } finally { + engine.close(); + } + return { root, dbPath }; +} + +function query(root: string, dbPath: string, relation: string, target: string, options = {}) { + const output: string[] = []; + const db = openSqlite(`file:${dbPath.replaceAll("\\", "/")}?mode=ro&immutable=1`); + const engine = createGraphEngine({ rootDir: root, dbPath, readOnly: true }); + try { + runGraphQuery(relation, target, root, { + open: () => ({ graph: engine, db, close: () => {} }), + write: (line) => output.push(line), + }, options); + } finally { + engine.close(); + db.close(); + } + return output.map((line) => JSON.parse(line) as Record); +} + +describe("who-calls falls back to recorded unresolved references", () => { + it("reports call sites for a name with no indexed declaration", async () => { + const { root, dbPath } = await repositoryWithDynamicCalls(); + + const rows = query(root, dbPath, "who-calls", "markFailed"); + + // The protocol stays uniform: meta first, summary last, like every other + // response — not the bare error record this path used to emit. + expect(rows[0]).toMatchObject({ type: "meta" }); + expect(rows.at(-1)).toMatchObject({ type: "summary" }); + + const unresolved = rows.filter((row) => row.type === "unresolved-reference"); + expect(unresolved.length).toBeGreaterThan(0); + // Never `type: "result"`: an agent must not confuse an unbound name for a + // resolved graph fact. + expect(rows.filter((row) => row.type === "result")).toEqual([]); + for (const row of unresolved) { + expect(row).toMatchObject({ + relation: "who-calls", + target: "markFailed", + name: "markFailed", + }); + expect(typeof row.file).toBe("string"); + expect(typeof row.line).toBe("number"); + expect(typeof row.col).toBe("number"); + expect(typeof row.fromNode).toBe("string"); + } + expect(unresolved.map((row) => row.file)).toEqual(["src/first.ts", "src/second.ts"]); + + const summary = rows.at(-1)!; + expect(summary).toMatchObject({ status: "partial", evidenceStrength: "weak", returnedNodes: 0 }); + expect((summary.warnings as string[]).join(" ")).toContain("markFailed"); + expect((summary.suggestedNextCommands as string[])[0]).toContain("mex graph get"); + }, 60_000); + + it("caps the fallback and declares the truncation", async () => { + const { root, dbPath } = await repositoryWithDynamicCalls(); + + const rows = query(root, dbPath, "who-calls", "markFailed", { maxNodes: 1 }); + + expect(rows.filter((row) => row.type === "unresolved-reference")).toHaveLength(1); + const summary = rows.at(-1)!; + expect(summary.truncated).toBe(true); + // The cap bounds the answer, not the count of what exists. + expect(summary.matchedNodes).toBeGreaterThan(1); + }, 60_000); + + it("still abstains when the name appears nowhere at all", async () => { + const { root, dbPath } = await repositoryWithDynamicCalls(); + + const rows = query(root, dbPath, "who-calls", "definitelyNotAName"); + + expect(rows).toEqual([ + { type: "error", code: "TARGET_NOT_FOUND", target: "definitelyNotAName" }, + ]); + }, 60_000); + + it("leaves where-defined and what-calls abstention unchanged", async () => { + const { root, dbPath } = await repositoryWithDynamicCalls(); + + for (const relation of ["where-defined", "what-calls"]) { + expect(query(root, dbPath, relation, "markFailed")).toEqual([ + { type: "error", code: "TARGET_NOT_FOUND", target: "markFailed" }, + ]); + } + }, 60_000); +}); diff --git a/src/graph/cli-agent.ts b/src/graph/cli-agent.ts index aa8dd22d..57b957e1 100644 --- a/src/graph/cli-agent.ts +++ b/src/graph/cli-agent.ts @@ -162,6 +162,8 @@ export function runGraphQuery( return withAgentGraphSession(rootDir, deps, output, "fresh", (session, write) => { const nodes = resolveSymbol(session.graph, target); if (nodes.length === 0) { + if (relation === "who-calls" + && emitUnresolvedCallers(session, write, target, opts)) return; writeJson(write, { type: "error", code: "TARGET_NOT_FOUND", target }); return; } @@ -2079,6 +2081,121 @@ function transitiveCallers(graph: GraphEngine, root: GraphNode, maxDepth: number * underlying store. Older schemas are rejected by the immutable reader and * require an explicit rebuild before this query can run. */ +/** + * A call site the resolver captured but could not bind to a declaration. + * + * These are not graph facts. They record that some file referenced this name + * and the resolver could not decide what it meant. + */ +interface UnresolvedCallSite { + reference_name: string; + reference_kind: string; + status: string; + file_path: string; + line: number; + col: number; + from_node_id: string; + receiver: string | null; + qualifier: string | null; +} + +/** + * `who-calls` fallback for a name with call sites but no declaration. + * + * A dynamically generated method has real callers and no literal definition, + * so no node resolves and the honest structural answer is "not found" — which + * left an agent with no next step, while the call sites sat in + * `unresolved_refs` with no reader anywhere in the CLI. Recovering them meant + * hand-written SQL against internal schema. + * + * Returns false when there is nothing to offer, so the caller still abstains + * with TARGET_NOT_FOUND rather than emitting an empty alternative answer. + */ +function emitUnresolvedCallers( + session: AgentGraphSession, + write: (line: string) => void, + target: string, + opts: AgentOptions, +): boolean { + const matched = unresolvedCallSiteCount(session.db, target); + if (matched === 0) return false; + + // `limit` bounds the answer, not the work: hot names are common + // (`forwardRef`, `json`, `error`) and an uncapped fallback would flood the + // agent with hundreds of rows for one query. + const rows = unresolvedCallSites(session.db, target, opts.maxNodes); + const anticipated = rows.length > 0 + ? [`mex graph get ${rows[0]!.from_node_id} --detail source`] : []; + const ctx = beginResponse("graph query who-calls", opts, undefined, anticipated); + const records: Rec[] = []; + let truncated = matched > rows.length; + for (const row of rows) { + // A distinct record type, never `type: "result"`. An agent must not be + // able to mistake an unbound name for a resolved graph edge. + const record: Rec = { + type: "unresolved-reference", + relation: "who-calls", + target, + name: row.reference_name, + referenceKind: row.reference_kind, + resolution: row.status, + file: row.file_path, + line: row.line, + col: row.col, + fromNode: row.from_node_id, + ...(row.receiver === null ? {} : { receiver: row.receiver }), + ...(row.qualifier === null ? {} : { qualifier: row.qualifier }), + }; + if (!ctx.ledger.tryAdd(record)) { truncated = true; break; } + records.push(record); + } + + emitAll(write, ctx.meta, records); + write(JSON.stringify(summaryRecord(ctx, { + matchedNodes: matched, + // No node was returned: these are call sites, not declarations. + returnedNodes: 0, + returnedEdges: 0, + truncated, + status: "partial", + evidenceStrength: "weak", + suggestedNextCommands: records.length > 0 + ? [`mex graph get ${records[0]!.fromNode as string} --detail source`] : [], + warnings: [ + `No declaration named "${target}" is indexed. ` + + `${matched} unresolved reference(s) to that name were recorded during ` + + "extraction and are reported instead of resolved callers; they may be " + + "dynamically generated, defined outside the indexed corpus, or ambiguous.", + ], + }))); + return true; +} + +/** Total matching call sites, so the summary can report what it capped. */ +function unresolvedCallSiteCount(db: SqliteDatabase, name: string): number { + const row = db.prepare( + `SELECT COUNT(*) AS total FROM unresolved_refs + WHERE reference_name = ? AND status <> 'resolved'`, + ).get(name) as { total?: unknown } | undefined; + return typeof row?.total === "number" ? row.total : 0; +} + +/** Exact-name lookup, deterministically ordered and bounded in SQL. */ +function unresolvedCallSites( + db: SqliteDatabase, + name: string, + limit: number, +): UnresolvedCallSite[] { + return db.prepare( + `SELECT reference_name, reference_kind, status, file_path, line, col, + from_node_id, receiver, qualifier + FROM unresolved_refs + WHERE reference_name = ? AND status <> 'resolved' + ORDER BY file_path, line, col, from_node_id + LIMIT ?`, + ).all(name, Math.max(0, limit)) as UnresolvedCallSite[]; +} + function groundedFiles(db: SqliteDatabase, nodeIds: string[]): Array<{ scaffold_file: string; node_id: string }> { if (nodeIds.length === 0) return []; const placeholders = nodeIds.map(() => "?").join(","); From 6dc28fe33549456ab511fcf9a4ebe79c6e089a73 Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Mon, 7 Sep 2026 21:01:52 +0530 Subject: [PATCH 5/9] perf(graph): stop storing references the resolver already turned into edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit References #140. Once schema v3 shrank everything else, unresolved_refs became the largest object in the store — 32-42% of it across every repository measured, with its secondary indexes costing about as much as the table. Three changes, in increasing order of risk. Drop the narrow (from_node_id) index. It is a strict prefix of the (from_node_id, reference_name) composite, and EXPLAIN QUERY PLAN on a real store confirms SQLite serves every lookup it used to from the composite — the ON DELETE CASCADE probe included, still as a covering search rather than a scan. Make the status index partial. Stop writing rows for references that resolved. Every such row duplicated an edge: measured at 100% on fresh builds and existing stores alike, and 27-73% of the table. `edges` is a superset — it also holds structural `contains` edges with no reference row — so nothing is recoverable from here that is not already there. Resolution happens in memory during staging, and publication clears and rewrites the whole derived graph, so no read path consumed these rows. Three store methods described a path that does not run: an incremental sync that wipes reference edges and rebuilds them from unresolved_refs. None of them had a caller anywhere in the tree. They are removed rather than left as a comment contradicting what the code now does. Measured on two repositories, per-object via dbstat: 99-file TS/TSX/Python/JS 12.34 MB -> 10.29 MB (-16.6%) 366-file Python-heavy 65.07 MB -> 56.44 MB (-13.3%) with the table and its indexes down about 40% on both. No schema version bump: the changes are index definitions and which rows are written, both compatible with an existing store, which reclaims the space on its next rebuild. --- src/graph/db/store.ts | 33 --------------------------------- src/graph/engine-impl.ts | 10 +++++++++- src/graph/schema.sql | 26 ++++++++++++++++++++++---- src/graph/status.ts | 1 - 4 files changed, 31 insertions(+), 39 deletions(-) diff --git a/src/graph/db/store.ts b/src/graph/db/store.ts index 358778d8..ca5136c7 100644 --- a/src/graph/db/store.ts +++ b/src/graph/db/store.ts @@ -98,22 +98,6 @@ export interface NodeAliasRecord { confidence: number; } -/** Reference edge kinds — everything except the intra-file `contains` edge. - * `sync` wipes these and rebuilds them from `unresolved_refs`. */ -export const REFERENCE_EDGE_KINDS: EdgeKind[] = [ - "calls", - "imports", - "exports", - "extends", - "implements", - "references", - "type_of", - "returns", - "instantiates", - "overrides", - "decorates", -]; - interface NodeRow { id: string; kind: string; @@ -465,17 +449,6 @@ export class GraphStore { ); } - updateReferenceResolution(ref: UnresolvedRefRecord): void { - const key = ref.refKey ?? referenceKey(ref); - this.db.prepare( - `UPDATE unresolved_refs SET status = ?, target_id = ?, confidence = ?, resolver = ?, candidates = ? - WHERE ref_key = ?`, - ).run( - ref.status ?? "unresolved", ref.targetId ?? null, ref.confidence ?? null, - ref.resolver ?? null, ref.candidates ? JSON.stringify(ref.candidates) : null, key, - ); - } - insertImportBinding(binding: ImportBindingRecord): void { this.db.prepare( `INSERT INTO import_bindings ( @@ -638,12 +611,6 @@ export class GraphStore { this.db.exec("DELETE FROM files"); } - /** Wipe every reference edge (keeping intra-file `contains`), so `sync` can - * rebuild them from `unresolved_refs` with no duplicates. */ - clearReferenceEdges(): void { - this.db.prepare("DELETE FROM edges WHERE kind != 'contains'").run(); - } - // --- Reads ---------------------------------------------------------------- getNodeById(id: string): GraphNode | null { diff --git a/src/graph/engine-impl.ts b/src/graph/engine-impl.ts index f16c58e1..ced0fe3d 100644 --- a/src/graph/engine-impl.ts +++ b/src/graph/engine-impl.ts @@ -486,7 +486,15 @@ class GraphEngineImpl implements GraphEngine { for (const file of staged.files) for (const node of file.nodes) store.insertNode(node); for (const file of staged.files) { for (const edge of file.edges) if (store.insertEdge(edge)) edgeCount++; - for (const reference of file.references) store.insertUnresolvedRef(reference); + // A resolved reference is an edge. Keeping the row too duplicated one + // for every reference the resolver bound — 27-73% of this table on + // every repository measured, all of them already in `edges`. Resolution + // happens in memory during staging, so nothing downstream reads these + // rows back to rebuild anything. + for (const reference of file.references) { + if (reference.status === "resolved") continue; + store.insertUnresolvedRef(reference); + } for (const binding of file.imports) store.insertImportBinding(binding); } diff --git a/src/graph/schema.sql b/src/graph/schema.sql index f419ddd1..62be854f 100644 --- a/src/graph/schema.sql +++ b/src/graph/schema.sql @@ -122,8 +122,20 @@ CREATE TABLE IF NOT EXISTS files ( extractor_version TEXT NOT NULL DEFAULT 'unknown' ); --- Unresolved references: parked during single-file extraction, resolved after a --- full index pass (two-phase extract -> resolve). Kept so cross-file edges work. +-- Unresolved references: parked during single-file extraction, then resolved +-- after a full index pass (two-phase extract -> resolve). +-- +-- Only references the resolver could NOT bind are stored. A bound reference +-- becomes an edge, and every row this table used to keep with status +-- 'resolved' duplicated one — measured at 100% on every repository, and 27-73% +-- of the table. `edges` is a superset (it also holds structural `contains` +-- edges with no reference row), so nothing is recoverable from here that is +-- not already there. +-- +-- What remains is the graph being honest about its own blind spots: a name +-- some file referenced that the resolver could not decide the meaning of. That +-- is what `mex graph query who-calls` falls back to when a name has call sites +-- but no indexed declaration. CREATE TABLE IF NOT EXISTS unresolved_refs ( id INTEGER PRIMARY KEY AUTOINCREMENT, ref_key TEXT NOT NULL UNIQUE, @@ -253,11 +265,17 @@ ON edges(source, target, kind, IFNULL(line, -1), IFNULL(col, -1)); CREATE INDEX IF NOT EXISTS idx_files_language ON files(language); CREATE INDEX IF NOT EXISTS idx_files_modified_at ON files(modified_at); -CREATE INDEX IF NOT EXISTS idx_unresolved_from_node ON unresolved_refs(from_node_id); +-- A narrow (from_node_id) index is intentionally omitted: it is a strict +-- prefix of the composite below, which SQLite uses for every lookup the narrow +-- one served, the ON DELETE CASCADE probe included. Verified with EXPLAIN +-- QUERY PLAN on a real store; no plan degrades to a scan. CREATE INDEX IF NOT EXISTS idx_unresolved_name ON unresolved_refs(reference_name); CREATE INDEX IF NOT EXISTS idx_unresolved_file_path ON unresolved_refs(file_path); CREATE INDEX IF NOT EXISTS idx_unresolved_from_name ON unresolved_refs(from_node_id, reference_name); -CREATE INDEX IF NOT EXISTS idx_unresolved_status ON unresolved_refs(status); +-- Partial: a resolved reference is an edge and is not stored here. The +-- predicate keeps the index honest if a legacy store still carries such rows. +CREATE INDEX IF NOT EXISTS idx_unresolved_status +ON unresolved_refs(status) WHERE status <> 'resolved'; CREATE INDEX IF NOT EXISTS idx_import_bindings_file ON import_bindings(file_path); CREATE INDEX IF NOT EXISTS idx_import_bindings_local ON import_bindings(file_path, local_name); CREATE INDEX IF NOT EXISTS idx_aliases_canonical ON node_aliases(canonical_node_id); diff --git a/src/graph/status.ts b/src/graph/status.ts index 929d1af8..c4825da5 100644 --- a/src/graph/status.ts +++ b/src/graph/status.ts @@ -115,7 +115,6 @@ const REQUIRED_SCHEMA_OBJECTS = { "idx_source_chunks_file", "idx_unresolved_file_path", "idx_unresolved_from_name", - "idx_unresolved_from_node", "idx_unresolved_name", "idx_unresolved_status", ], From 181fa6f53c61da2515cb1095dfd5962c0c6206b1 Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Mon, 7 Sep 2026 21:02:10 +0530 Subject: [PATCH 6/9] docs: link the graph retrieval benchmark results from the READMEs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit evaluate/RESULTS.md carries the blind-graded comparison against a file-search baseline and was linked from nowhere — not the top-level README, not evaluate/README.md. It is the evidence answering the benchmark half of #115, so it should be reachable from both. --- README.md | 1 + evaluate/README.md | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index 1cd7ae4a..cc0b99d6 100644 --- a/README.md +++ b/README.md @@ -387,5 +387,6 @@ MEX keeps team memory in repository files and provides local retrieval and revie - Check the [runtime and compatibility guide](https://github.com/mex-memory/mex/blob/v0.8.0/COMPATIBILITY.md) and [security policy](https://github.com/mex-memory/mex/blob/v0.8.0/SECURITY.md). - Review the [Code Graph support matrix](https://github.com/mex-memory/mex/blob/v0.8.0/docs/code-graph-support.md). - See the [extractor model and supported relationships](https://github.com/mex-memory/mex/blob/v0.8.0/docs/extractors.md). +- Read the [graph retrieval benchmark results](https://github.com/mex-memory/mex/blob/v0.8.0/evaluate/RESULTS.md), including the blind-graded comparison against an ordinary file-search baseline. - Inspect the CLI locally with `mex capabilities --json` and `mex commands`. - Join the [MEX community on Discord](https://discord.gg/FEdNsQ4Qt4) or visit [mexmemory.com](https://mexmemory.com). diff --git a/evaluate/README.md b/evaluate/README.md index c1685e7a..91d0c942 100644 --- a/evaluate/README.md +++ b/evaluate/README.md @@ -13,6 +13,10 @@ The older compactness and scripted-agent scripts remain available as historical `npm run eval:legacy` and `npm run eval:e2e`. They are not evidence that natural-language graph retrieval works. +Measured results from these harnesses are written up in +[`RESULTS.md`](RESULTS.md), including the blind-graded headless comparison +against an ordinary file-search baseline. + ## Deterministic graph evaluation Build the CLI, then run the native MEX suite: From b60c3b28bbce67c6c1cc4036aa77cfbd62d2b568 Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Mon, 7 Sep 2026 22:04:18 +0530 Subject: [PATCH 7/9] docs(scaffold): record the bounded-limit and containment-decline gotchas Both hard aborts fixed on this branch came from the same two mistakes: treating a per-file ceiling as a corpus-wide one, and letting a containment guard's refusal be an exception rather than an answer. Record them where the next graph indexing change will read them. --- .../patterns/safe-graph-snapshot-evolution.md | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.mex/patterns/safe-graph-snapshot-evolution.md b/.mex/patterns/safe-graph-snapshot-evolution.md index f166f59a..0ad064c8 100644 --- a/.mex/patterns/safe-graph-snapshot-evolution.md +++ b/.mex/patterns/safe-graph-snapshot-evolution.md @@ -12,7 +12,7 @@ edges: condition: "when changing the graph data plane or its consumers" - target: "context/conventions.md" condition: "when verifying a graph implementation change" -last_updated: 2026-09-03 +last_updated: 2026-09-07 mex: id: mx_01M1M0CJP81C590FCKTSN5HA3Q type: pattern @@ -102,6 +102,32 @@ only to explicit maintenance workflows. cannot detect facts extracted from B; extraction must be bound to A. - Graph diagnostics and remediation commands must be truthful. Do not recommend a command for a state it cannot safely repair. +- A bounded limit is not one kind of thing. A **per-file** ceiling means one + file cannot be parsed and every other file still can, so it must skip and + report; a **corpus-wide** ceiling means the run has no honest partial answer + and must abort. Conflating them lets one pathological file take a whole + repository's graph with it. Classify the breach, never the fact of one. +- Do the skipping at the single discovery seam. Indexing, publication + verification, sync's corpus comparison and freshness inspection must all agree + about which files exist, or a file skipped by one and expected by another + makes the index permanently unable to read `fresh`. +- Never infer which limit was breached by comparing byte values. Two limits can + hold the same number, and then the comparison silently reports the wrong one + forever. Pass the limit's name. +- A containment guard has two separable decisions: whether to decline, and what + declining does. Read paths already treated an out-of-root source as a soft + miss while config paths threw on the identical condition — an inconsistency + inside one class that cost whole repositories their graph. An existence probe + is not a read request: the honest answer for a path we will not read is + "absent", recorded as a diagnostic so the degradation is visible. +- Report a declined out-of-root path by dependency specifier, not absolutely. + The compiler resolves a bare specifier by walking every ancestor directory, so + one unresolvable name yields a dozen near-identical entries carrying absolute + paths from outside the repository into product output. +- Any per-repository discovery input — a configured ignore list included — must + enter the corpus policy hash, or changing it leaves a stale index silently + describing files that are no longer in the corpus. Hash to the existing + constant when nothing is configured, so existing indexes stay valid. - Wall-clock status timings vary by machine and process-start overhead. Keep the benchmark non-gating, record its environment, and protect correctness with deterministic race, non-mutation, and bounded-work tests. From a546f209b0c41db4d28b0613c569e4f33e2fc519 Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Mon, 7 Sep 2026 23:04:04 +0530 Subject: [PATCH 8/9] docs: link the benchmark results from the translated READMEs The Explore further list in the Spanish, Portuguese and Chinese READMEs mirrors the English one, so the benchmark link belongs in all four rather than only the English entry point. Also drops the who-calls fallback paragraph from the English README. The behaviour stays documented in docs/code-graph-support.md, which is where the detail belongs. --- README.es.md | 1 + README.md | 2 -- README.pt-BR.md | 1 + README.zh-CN.md | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.es.md b/README.es.md index 035a5ff1..94d25682 100644 --- a/README.es.md +++ b/README.es.md @@ -424,5 +424,6 @@ MEX mantiene la memoria del equipo en archivos del repositorio y proporciona flu - Consulta la [guía de entorno de ejecución y compatibilidad](https://github.com/mex-memory/mex/blob/v0.8.0/COMPATIBILITY.md) y la [política de seguridad](https://github.com/mex-memory/mex/blob/v0.8.0/SECURITY.md). - Revisa la [matriz de compatibilidad del Code Graph](https://github.com/mex-memory/mex/blob/v0.8.0/docs/code-graph-support.md). - Consulta el [modelo de extractores y las relaciones compatibles](https://github.com/mex-memory/mex/blob/v0.8.0/docs/extractors.md). +- Lee los [resultados del benchmark de recuperación del Code Graph](https://github.com/mex-memory/mex/blob/v0.8.0/evaluate/RESULTS.md), incluida la comparación evaluada a ciegas frente a una búsqueda de archivos convencional. - Inspecciona la CLI localmente con `mex capabilities --json` y `mex commands`. - Únete a la [comunidad de MEX en Discord](https://discord.gg/FEdNsQ4Qt4) o visita [mexmemory.com](https://mexmemory.com). diff --git a/README.md b/README.md index cc0b99d6..15f060e2 100644 --- a/README.md +++ b/README.md @@ -207,8 +207,6 @@ mex graph get mex impact requireSession ``` -When `who-calls` finds no declaration for a name that call sites nonetheless reference — a dynamically generated method, or a symbol defined outside the indexed corpus — it reports the recorded call sites as `unresolved-reference` records instead of a bare not-found. They are labelled as unresolved rather than returned as graph facts, and they are capped like every other response. See [Code graph support](docs/code-graph-support.md#unresolved-references). - MEX indexes TypeScript/TSX, JavaScript/JSX, Python, and Rust. Module variants such as `.mts`, `.cts`, `.mjs`, and `.cjs` have partial coverage, and Express is the only framework-specific resolver documented for 0.8. Exact `query`, `get`, and `impact` reads—and Hub Code—require a provably fresh Graph; `scope` can instead return bounded live-text evidence for stale or unindexed files, clearly marked `text-only`. ### Grounding and drift diff --git a/README.pt-BR.md b/README.pt-BR.md index f546bac9..1865685c 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -424,5 +424,6 @@ O MEX mantém a memória da equipe em arquivos do repositório e oferece fluxos - Consulte o [guia de ambiente de execução e compatibilidade](https://github.com/mex-memory/mex/blob/v0.8.0/COMPATIBILITY.md) e a [política de segurança](https://github.com/mex-memory/mex/blob/v0.8.0/SECURITY.md). - Veja a [matriz de suporte do Code Graph](https://github.com/mex-memory/mex/blob/v0.8.0/docs/code-graph-support.md). - Consulte o [modelo dos extratores e as relações suportadas](https://github.com/mex-memory/mex/blob/v0.8.0/docs/extractors.md). +- Leia os [resultados do benchmark de recuperação do Code Graph](https://github.com/mex-memory/mex/blob/v0.8.0/evaluate/RESULTS.md), incluindo a comparação avaliada às cegas com uma busca de arquivos comum. - Examine a CLI localmente com `mex capabilities --json` e `mex commands`. - Entre na [comunidade do MEX no Discord](https://discord.gg/FEdNsQ4Qt4) ou visite [mexmemory.com](https://mexmemory.com). diff --git a/README.zh-CN.md b/README.zh-CN.md index 92c34f1f..9bf454cf 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -424,5 +424,6 @@ MEX 将团队记忆保存在仓库文件中，并提供本地检索和审阅工 - 查看[运行时与兼容性指南](https://github.com/mex-memory/mex/blob/v0.8.0/COMPATIBILITY.md)和[安全政策](https://github.com/mex-memory/mex/blob/v0.8.0/SECURITY.md)。 - 查阅 [Code Graph 支持矩阵](https://github.com/mex-memory/mex/blob/v0.8.0/docs/code-graph-support.md)。 - 了解[提取器模型和支持的关系](https://github.com/mex-memory/mex/blob/v0.8.0/docs/extractors.md)。 +- 阅读 [Code Graph 检索基准结果](https://github.com/mex-memory/mex/blob/v0.8.0/evaluate/RESULTS.md)，其中包含与普通文件搜索基线的盲评对比。 - 在本地使用 `mex capabilities --json` 和 `mex commands` 检查 CLI。 - 加入 [Discord 上的 MEX 社区](https://discord.gg/FEdNsQ4Qt4)，或访问 [mexmemory.com](https://mexmemory.com)。 From 7ffebf9d7b410e1a4ec18f1729efdfdc0520023a Mon Sep 17 00:00:00 2001 From: Yashasvi Date: Tue, 8 Sep 2026 00:18:48 +0530 Subject: [PATCH 9/9] fix(graph): judge configured ignore globs the same way on every platform MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught this on Linux: the filter used path.isAbsolute, so "C:/build/**" was rejected on Windows and accepted on Linux. That is not only a test discrepancy. .mex/config.json is tracked and travels with the repository, and these globs feed the corpus policy hash — so one repository would hash to two different manifests depending on the machine that opened it, and the index would read as stale purely from moving between platforms. The check is now explicit and platform-free: reject a leading "/" (POSIX absolute and "//server/share" UNC alike), a Windows drive prefix in either its absolute or drive-relative form, and any ".." path segment. Upward traversal is now caught anywhere in the glob rather than only at the front, so "vendor/../../escape/**" no longer slips through. Globs that merely contain dots, such as "a..b/**" and "**/*.min.js", are unaffected. --- src/graph/__tests__/corpus-policy.test.ts | 34 +++++++++++++++++++++- src/graph/corpus-policy.ts | 35 ++++++++++++++++++----- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/graph/__tests__/corpus-policy.test.ts b/src/graph/__tests__/corpus-policy.test.ts index f0a53bf5..4744d8d7 100644 --- a/src/graph/__tests__/corpus-policy.test.ts +++ b/src/graph/__tests__/corpus-policy.test.ts @@ -161,13 +161,45 @@ describe("configured graph ignore globs", () => { expect(readConfiguredGraphIgnoreGlobs(withConfig({ graph: "nope" }))).toEqual([]); expect(readConfiguredGraphIgnoreGlobs(withConfig({ graph: { ignore: "nope" } }))).toEqual([]); expect(readConfiguredGraphIgnoreGlobs(withConfig({ - graph: { ignore: [42, "", " ", "../escape/**", "C:/absolute/**", "/absolute/**"] }, + graph: { ignore: [42, "", " "] }, }))).toEqual([]); expect(readConfiguredGraphIgnoreGlobs( mkdtempSync(join(tmpdir(), "mex-graph-ignore-missing-")), )).toEqual([]); }); + it("rejects every escaping glob identically on every platform", () => { + // `.mex/config.json` is tracked and travels with the repository, and these + // globs feed the corpus policy hash. A platform-dependent verdict — which + // `path.isAbsolute` gives, since `C:/x` is absolute only on Windows — would + // give one repository two manifest hashes and make its index read as stale + // purely from being opened on another machine. + const escaping = [ + "/absolute/**", + "//server/share/**", + "\\\\server\\share\\**", + "C:/absolute/**", + "c:/absolute/**", + "C:\\absolute\\**", + "C:relative/**", + "../escape/**", + "..\\escape\\**", + "vendor/../../escape/**", + "..", + ]; + const root = withConfig({ graph: { ignore: escaping } }); + + expect(readConfiguredGraphIgnoreGlobs(root)).toEqual([]); + expect(graphCorpusPolicyHash(root)).toBe(GRAPH_CORPUS_POLICY_HASH); + }); + + it("keeps ordinary globs that merely contain dots", () => { + const root = withConfig({ graph: { ignore: ["a..b/**", "**/*.min.js", "./local/**"] } }); + + expect(readConfiguredGraphIgnoreGlobs(root)) + .toEqual(["**/*.min.js", "./local/**", "a..b/**"]); + }); + it("bounds the configured list", () => { const tooMany = Array.from({ length: 500 }, (_, index) => `dir${index}/**`); const tooLong = "x".repeat(GRAPH_IGNORE_CONFIG_LIMITS.maxGlobLength + 1); diff --git a/src/graph/corpus-policy.ts b/src/graph/corpus-policy.ts index 3577d4d8..5754e1b9 100644 --- a/src/graph/corpus-policy.ts +++ b/src/graph/corpus-policy.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { readFileSync, statSync } from "node:fs"; -import { isAbsolute, resolve } from "node:path"; +import { resolve } from "node:path"; import { globIterateSync, type GlobOptions } from "glob"; import { SUPPORTED_SOURCE_GLOB } from "./extraction/grammars.js"; @@ -66,17 +66,38 @@ export function readConfiguredGraphIgnoreGlobs(root: string): string[] { const globs = new Set(); for (const entry of ignore) { if (typeof entry !== "string") continue; - const glob = entry.trim(); - if (!glob || glob.length > GRAPH_IGNORE_CONFIG_LIMITS.maxGlobLength) continue; - // Absolute paths and upward traversal describe files outside the corpus, - // which discovery already refuses to walk. Accept only repo-relative globs. - if (isAbsolute(glob) || glob.startsWith("../") || glob.startsWith("..\\")) continue; - globs.add(glob.split("\\").join("/")); + const trimmed = entry.trim(); + if (!trimmed || trimmed.length > GRAPH_IGNORE_CONFIG_LIMITS.maxGlobLength) continue; + const glob = trimmed.split("\\").join("/"); + if (!isRepositoryRelativeGlob(glob)) continue; + globs.add(glob); if (globs.size >= GRAPH_IGNORE_CONFIG_LIMITS.maxGlobs) break; } return [...globs].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)); } +/** + * Accept only globs that name something inside the repository. + * + * Absolute paths and upward traversal describe files outside the corpus, which + * discovery already refuses to walk. The check is deliberately **not** + * `path.isAbsolute`: that is platform-dependent, and `C:/build/**` is absolute + * on Windows but an ordinary relative path on Linux. `.mex/config.json` is + * tracked and travels with the repository, and these globs feed the corpus + * policy hash — so a platform-dependent verdict would give one repository two + * different manifest hashes and make its index read as stale purely from being + * opened on another machine. + * + * Expects a glob already normalized to forward slashes. + */ +function isRepositoryRelativeGlob(glob: string): boolean { + // Leading "/" covers POSIX-absolute and "//server/share" UNC alike. + if (glob.startsWith("/")) return false; + // Windows drive-absolute ("C:/x") and drive-relative ("C:x") forms. + if (/^[A-Za-z]:/u.test(glob)) return false; + return !glob.split("/").includes(".."); +} + /** The complete ignore list for one repository: frozen defaults, then config. */ export function graphCorpusIgnoreGlobs(root: string): string[] { return [...GRAPH_CORPUS_IGNORE_GLOBS, ...readConfiguredGraphIgnoreGlobs(root)];