From 31d9f9593c9f8dc17ba864e5ac5000e045b7aba1 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Tue, 11 Aug 2026 01:12:23 +0800 Subject: [PATCH 01/11] =?UTF-8?q?Fix:=20=E6=B7=BB=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E6=9C=AA=E6=8F=90=E5=8F=96=E8=AE=B0=E5=BF=86=E8=A1=A5=E5=81=BF?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 可以重新尝试提取之前未提取的记忆节点，在正式提取前会让用户确认。 --- index.ts | 1 + package.json | 3 +- src/cli-extract.ts | 288 +++++++++++++++++++++++++++ src/cli.ts | 55 ++++++ src/store/store.ts | 31 +++ test/cli-extract.test.ts | 417 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 794 insertions(+), 1 deletion(-) create mode 100644 src/cli-extract.ts create mode 100644 test/cli-extract.test.ts diff --git a/index.ts b/index.ts index 8397aa5..2df04fd 100755 --- a/index.ts +++ b/index.ts @@ -258,6 +258,7 @@ const graphMemoryProPlugin = { pluginId: "graph-memory-pro", pluginConfig: raw as Record | undefined, resolveConfigPath: (p: string) => api.resolvePath?.(p) ?? p, + defaultModel: readDefaultModel(api.config), }), { commands: ["graph-memory"] }, ); diff --git a/package.json b/package.json index 4ea628f..2c0112d 100755 --- a/package.json +++ b/package.json @@ -24,8 +24,9 @@ "test:watch": "vitest --passWithNoTests" }, "dependencies": { + "@sinclair/typebox": "^0.34.48", "neo4j-driver": "^5.27.0", - "@sinclair/typebox": "^0.34.48" + "opencode-ai": "^1.18.16" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/src/cli-extract.ts b/src/cli-extract.ts new file mode 100644 index 0000000..cfe9d18 --- /dev/null +++ b/src/cli-extract.ts @@ -0,0 +1,288 @@ +/** + * graph-memory-pro CLI — `openclaw graph-memory extract` + * + * 对未被提取的会话消息做批量图谱提取，补齐因 compact 未触发、提取失败或 + * 进程退出而残留的 GmMessage。流程镜像 index.ts 的 compact() 路径： + * getUnextracted → extractor.extract → upsertNode + syncEmbed → upsertEdge → markExtracted + * + * 命令在 cli-metadata 模式下运行（register() 早早 return），所以这里必须自行 + * 完成 Neo4j driver / schema / LLM / embedder / Extractor / Recaller 的初始化。 + */ + +import readline from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; + +import type { Driver } from "neo4j-driver"; +import type { GmConfig } from "./types.ts"; +import { getDriver, initSchema, closeDriver } from "./store/db.ts"; +import { + listUnextractedSessions, + getUnextracted, + markExtracted, + upsertNode, + upsertEdge, + findByName, + getBySession, + type UnextractedSessionInfo, +} from "./store/store.ts"; +import { createCompleteFn, resolveProvider } from "./engine/llm.ts"; +import { createEmbedFn } from "./engine/embed.ts"; +import { Recaller } from "./recaller/recall.ts"; +import { Extractor } from "./extractor/extract.ts"; + +const AFFIRMATIVE = new Set(["y", "yes", "yeah", "yep", "ok", "okay", "true", "1", "confirm"]); + +export function isAffirmative(answer: string): boolean { + return AFFIRMATIVE.has(answer.trim().toLowerCase()); +} + +export interface BackfillExtractOptions { + yes?: boolean; + limit?: number; + session?: string; + dryRun?: boolean; +} + +export interface BackfillExtractParams { + cfg: GmConfig; + effectiveModel: string; + options: BackfillExtractOptions; + log?: (msg: string) => void; + prompt?: (question: string) => Promise; +} + +export interface BackfillExtractResult { + sessionsTotal: number; + sessionsProcessed: number; + sessionsSkipped: number; + nodesCreated: number; + edgesCreated: number; + batches: number; + durationMs: number; +} + +const DEFAULT_BATCH_LIMIT_MULTIPLIER = 3; + +function defaultLog(msg: string): void { + console.log(msg); +} + +function formatSessionLine(info: UnextractedSessionInfo, index: number): string { + const created = info.minCreatedAt > 0 + ? new Date(info.minCreatedAt).toISOString().replace("T", " ").slice(0, 19) + : "?"; + return ` ${String(index + 1).padStart(3, " ")}. sid=${info.sessionId.slice(0, 12)}… msgs=${info.messageCount} maxTurn=${info.maxTurn} since=${created}`; +} + +export async function runBackfillExtraction( + params: BackfillExtractParams, +): Promise { + const start = Date.now(); + const log = params.log ?? defaultLog; + const opts = params.options; + const cfg = params.cfg; + + const result: BackfillExtractResult = { + sessionsTotal: 0, + sessionsProcessed: 0, + sessionsSkipped: 0, + nodesCreated: 0, + edgesCreated: 0, + batches: 0, + durationMs: 0, + }; + + if (!cfg.neo4j?.uri) { + throw new Error( + "[graph-memory-pro] extract 需要 neo4j.uri 配置。请在 graph-memory-pro 插件配置中设置 neo4j.uri / neo4j.user / neo4j.password。", + ); + } + + if (!params.effectiveModel) { + throw new Error( + "[graph-memory-pro] extract 需要一个 LLM model。请在 config.llm.model 或 agents.defaults.model 中设置。", + ); + } + + const providerInfo = resolveProvider(cfg.llm); + if (providerInfo.provider === "anthropic" && !cfg.llm?.apiKey) { + throw new Error("[graph-memory-pro] llm.provider=anthropic 但未配 llm.apiKey，无法提取。"); + } + if (providerInfo.provider === "openai" && (!cfg.llm?.apiKey || !cfg.llm?.baseURL)) { + throw new Error("[graph-memory-pro] llm.provider=openai 需要 llm.apiKey + llm.baseURL，无法提取。"); + } + if (providerInfo.provider === "oauth" && !cfg.llm?.oauthPath) { + throw new Error( + "[graph-memory-pro] llm.provider=oauth 但未配 llm.oauthPath。请先运行 `openclaw graph-memory auth login`。", + ); + } + + const driver: Driver = getDriver(cfg.neo4j); + + try { + log("[graph-memory-pro] 正在初始化 Neo4j schema..."); + await initSchema(driver, cfg.embedding); + + log("[graph-memory-pro] 正在初始化 LLM 与 embedder..."); + const llm = createCompleteFn(params.effectiveModel, cfg.llm); + const extractor = new Extractor(llm); + const recaller = new Recaller(driver, cfg); + const embedFn = await createEmbedFn(cfg.embedding); + if (embedFn) { + recaller.setEmbedFn(embedFn); + log("[graph-memory-pro] embedding 已就绪，新节点将同步向量。"); + } else { + log("[graph-memory-pro] 未配置 embedding，跳过向量同步（dual-path recall 会降级为文本搜索）。"); + } + + let sessions = await listUnextractedSessions(driver); + if (opts.session) { + sessions = sessions.filter(s => s.sessionId === opts.session); + if (!sessions.length) { + log(`[graph-memory-pro] --session=${opts.session} 没有匹配到含未提取消息的会话。`); + result.durationMs = Date.now() - start; + return result; + } + } + result.sessionsTotal = sessions.length; + + if (sessions.length === 0) { + log("[graph-memory-pro] 没有需要提取的会话。"); + result.durationMs = Date.now() - start; + return result; + } + + const totalMessages = sessions.reduce((s, info) => s + info.messageCount, 0); + log(`[graph-memory-pro] 发现 ${sessions.length} 个会话共 ${totalMessages} 条未提取消息：`); + sessions.forEach((info, i) => log(formatSessionLine(info, i))); + + if (opts.dryRun) { + log("[graph-memory-pro] --dry-run 模式，未执行提取。"); + result.sessionsSkipped = sessions.length; + result.durationMs = Date.now() - start; + return result; + } + + if (!opts.yes) { + const prompt = params.prompt ?? ((q: string) => defaultPrompt(q)); + const answer = await prompt(`\n将对以上 ${sessions.length} 个会话发起 LLM 提取，继续？[y/N] `); + if (!isAffirmative(answer)) { + log("[graph-memory-pro] 已取消。"); + result.sessionsSkipped = sessions.length; + result.durationMs = Date.now() - start; + return result; + } + } + + const batchLimit = opts.limit && opts.limit > 0 + ? opts.limit + : Math.max(1, cfg.compactTurnCount) * DEFAULT_BATCH_LIMIT_MULTIPLIER; + + log(`\n[graph-memory-pro] 开始提取（每批最多 ${batchLimit} 条消息）...`); + + for (const info of sessions) { + log(`\n[graph-memory-pro] 会话 ${info.sessionId.slice(0, 12)}… (${info.messageCount} 条消息)`); + try { + const processed = await extractSessionLoop(driver, extractor, recaller, info.sessionId, batchLimit, log); + result.nodesCreated += processed.nodes; + result.edgesCreated += processed.edges; + result.batches += processed.batches; + result.sessionsProcessed += 1; + log(` -> 完成：${processed.nodes} 节点 / ${processed.edges} 边 / ${processed.batches} 批`); + } catch (err) { + result.sessionsSkipped += 1; + log(` -> 失败：${err instanceof Error ? err.message : String(err)}`); + } + } + + result.durationMs = Date.now() - start; + + log( + `\n[graph-memory-pro] 提取完成：${result.sessionsProcessed}/${result.sessionsTotal} 会话，` + + `${result.nodesCreated} 节点，${result.edgesCreated} 边，${result.batches} 批，` + + `用时 ${(result.durationMs / 1000).toFixed(1)}s`, + ); + return result; + } finally { + await closeDriver(); + } +} + +interface SessionExtractStats { + nodes: number; + edges: number; + batches: number; +} + +async function extractSessionLoop( + driver: Driver, + extractor: Extractor, + recaller: Recaller, + sessionId: string, + batchLimit: number, + log: (msg: string) => void, +): Promise { + const stats: SessionExtractStats = { nodes: 0, edges: 0, batches: 0 }; + const hardBatchCeiling = 50; + let exhausted = false; + + for (let i = 0; i < hardBatchCeiling; i++) { + const msgs = await getUnextracted(driver, sessionId, batchLimit); + if (!msgs.length) break; + + stats.batches += 1; + const existing = (await getBySession(driver, sessionId)).map(n => n.name); + const extraction = await extractor.extract({ messages: msgs, existingNames: existing }); + + const nameToId = new Map(); + for (const nc of extraction.nodes) { + const { node } = await upsertNode(driver, { + type: nc.type, name: nc.name, + description: nc.description, content: nc.content, + }, sessionId); + nameToId.set(node.name, node.id); + stats.nodes += 1; + void recaller.syncEmbed(node).catch(() => {}); + } + + for (const ec of extraction.edges) { + const fromNode = await findByName(driver, ec.from); + const toNode = await findByName(driver, ec.to); + const fromId = nameToId.get(ec.from) ?? fromNode?.id; + const toId = nameToId.get(ec.to) ?? toNode?.id; + if (fromId && toId) { + await upsertEdge(driver, { + fromId, toId, type: ec.type, + instruction: ec.instruction, condition: ec.condition, sessionId, + }); + stats.edges += 1; + } + } + + const maxTurn = msgs.reduce((m, msg) => Math.max(m, msg.turn_index ?? 0), 0); + await markExtracted(driver, sessionId, maxTurn); + log(` batch ${stats.batches}: ${msgs.length} 消息 -> ${extraction.nodes.length} 节点 / ${extraction.edges.length} 边（累计 ${stats.nodes}/${stats.edges}）`); + + if (msgs.length < batchLimit) break; + if (i === hardBatchCeiling - 1) exhausted = true; + } + + if (exhausted) { + log(` 警告：达到批数上限 ${hardBatchCeiling}，会话 ${sessionId.slice(0, 12)}… 仍有未提取消息，请再次运行。`); + } + + return stats; +} + +async function defaultPrompt(question: string): Promise { + if (!process.stdin.isTTY && process.env.GRAPH_MEMORY_EXTRACT_CONFIRM === undefined) { + return ""; + } + const rl = readline.createInterface({ input, output }); + try { + const answer = await rl.question(question); + return answer; + } finally { + rl.close(); + } +} diff --git a/src/cli.ts b/src/cli.ts index f4c5a25..a7faeaf 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -25,6 +25,8 @@ import { type OAuthProviderId, } from "./engine/oauth.ts"; import type { ReasoningEffort } from "./engine/llm.ts"; +import { runBackfillExtraction } from "./cli-extract.ts"; +import { DEFAULT_CONFIG, type GmConfig } from "./types.ts"; // ─── 最小 Commander 鸭子类型（避免引入 commander 依赖） ─────────── // host 运行时注入真正的 commander.Command 实例，结构兼容此接口即可。 @@ -45,6 +47,7 @@ export interface GraphMemoryCliDeps { pluginId?: string; pluginConfig?: Record | undefined; resolveConfigPath?: (input: string) => string; + defaultModel?: string; oauthTestHooks?: { openUrl?: (url: string) => void | Promise; authorizeUrl?: (url: string) => void | Promise; @@ -375,5 +378,57 @@ export function createGraphMemoryCli(deps: GraphMemoryCliDeps) { throw new Error(`[graph-memory-pro] OAuth login failed: ${message}`); } }); + + root + .command("extract") + .description( + "扫描 Neo4j 中未提取的会话消息，按 compact 流程批量补提知识图谱，并同步节点 embedding", + ) + .option("--yes", "跳过确认提示，直接执行提取", false) + .option("--dry-run", "只列出待提取会话，不调用 LLM", false) + .option("--limit ", "每个会话每批最多提取的消息条数（默认 compactTurnCount * 3）", undefined) + .option("--session ", "仅提取指定 sessionId（默认全部含未提取消息的会话）", undefined) + .option("--model ", "本次提取使用的 LLM 模型（覆盖配置中的 llm.model / agents.defaults.model）", undefined) + .action(async (options: Record) => { + try { + const rawCfg = isPlainObject(deps.pluginConfig) + ? (deps.pluginConfig as Record) + : {}; + const cfg: GmConfig = { + ...DEFAULT_CONFIG, + ...(rawCfg as Partial), + }; + if (isPlainObject(rawCfg.neo4j)) { + cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...(rawCfg.neo4j as any) }; + } + + const cfgLlm = isPlainObject(rawCfg.llm) ? (rawCfg.llm as any) : undefined; + const flagModel = typeof options.model === "string" && options.model.trim() + ? options.model.trim() + : undefined; + const effectiveModel = flagModel ?? cfgLlm?.model ?? deps.defaultModel ?? ""; + + const limitFlag = typeof options.limit === "string" + ? Number.parseInt(options.limit, 10) + : (typeof options.limit === "number" ? options.limit : undefined); + + await runBackfillExtraction({ + cfg, + effectiveModel, + options: { + yes: options.yes === true, + dryRun: options.dryRun === true, + session: typeof options.session === "string" ? options.session : undefined, + limit: limitFlag !== undefined && Number.isFinite(limitFlag) && limitFlag > 0 + ? Math.floor(limitFlag) + : undefined, + }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error("[graph-memory-pro] extract 失败：", message); + throw new Error(`[graph-memory-pro] extract failed: ${message}`); + } + }); }; } diff --git a/src/store/store.ts b/src/store/store.ts index e08787c..992d259 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -888,6 +888,37 @@ export async function getUnextracted(driver: Driver, sid: string, limit: number) } } +export interface UnextractedSessionInfo { + sessionId: string; + messageCount: number; + maxTurn: number; + minCreatedAt: number; +} + +export async function listUnextractedSessions(driver: Driver): Promise { + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (m:GmMessage {extracted: false}) + WITH m.sessionId AS sid, + count(*) AS msgCount, + max(m.turnIndex) AS maxTurn, + min(coalesce(m.createdAt, 0)) AS minCreated + WHERE sid IS NOT NULL + RETURN sid, msgCount, maxTurn, minCreated + ORDER BY minCreated ASC, sid ASC + `); + return result.records.map(r => ({ + sessionId: r.get("sid"), + messageCount: toInt(r.get("msgCount")), + maxTurn: toInt(r.get("maxTurn")), + minCreatedAt: toInt(r.get("minCreated")), + })); + } finally { + await session.close(); + } +} + export async function markExtracted(driver: Driver, sid: string, upToTurn: number): Promise { const session = getSession(driver); try { diff --git a/test/cli-extract.test.ts b/test/cli-extract.test.ts new file mode 100644 index 0000000..902016b --- /dev/null +++ b/test/cli-extract.test.ts @@ -0,0 +1,417 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + listUnextractedSessions: vi.fn(async () => [] as any[]), + getUnextracted: vi.fn(async (_d: any, _sid: any, _limit: any) => [] as any[]), + markExtracted: vi.fn(async () => {}), + upsertNode: vi.fn(async (_driver: any, c: any) => ({ + node: { + id: `n-${c.name}`, + type: c.type, + name: c.name, + description: c.description ?? "", + content: c.content, + status: "active", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: 0, + updatedAt: 0, + }, + isNew: true, + })), + upsertEdge: vi.fn(async () => {}), + findByName: vi.fn(async () => null), + getBySession: vi.fn(async () => [] as any[]), + extract: vi.fn(async () => ({ nodes: [] as any[], edges: [] as any[] })), + initSchema: vi.fn(async () => {}), + closeDriver: vi.fn(async () => {}), +})); + +vi.mock("../src/store/db.ts", () => ({ + getDriver: () => ({}), + initSchema: mocks.initSchema, + getSession: () => ({ close: async () => {} }), + closeDriver: mocks.closeDriver, +})); + +vi.mock("../src/store/store.ts", () => ({ + listUnextractedSessions: mocks.listUnextractedSessions, + getUnextracted: mocks.getUnextracted, + markExtracted: mocks.markExtracted, + upsertNode: mocks.upsertNode, + upsertEdge: mocks.upsertEdge, + findByName: mocks.findByName, + getBySession: mocks.getBySession, +})); + +vi.mock("../src/engine/llm.ts", () => ({ + createCompleteFn: () => async () => "", + resolveProvider: () => ({ provider: "openai", inferred: false }), +})); + +vi.mock("../src/engine/embed.ts", () => ({ + createEmbedFn: async () => null, +})); + +vi.mock("../src/recaller/recall.ts", () => ({ + Recaller: class { + setEmbedFn(): void {} + async syncEmbed(): Promise {} + }, +})); + +vi.mock("../src/extractor/extract.ts", () => ({ + Extractor: class { + async extract() { + return mocks.extract(); + } + }, +})); + +import { isAffirmative, runBackfillExtraction } from "../src/cli-extract.ts"; +import { DEFAULT_CONFIG } from "../src/types.ts"; + +function makeCfg(overrides: Record = {}) { + return { + ...DEFAULT_CONFIG, + neo4j: { uri: "bolt://localhost:7687", user: "neo4j", password: "x" }, + llm: { provider: "openai", apiKey: "k", baseURL: "https://api.openai.com/v1", model: "gpt-test" }, + ...overrides, + } as any; +} + +const SAMPLE_SESSION = { + sessionId: "sid-abc-1234567890", + messageCount: 5, + maxTurn: 5, + minCreatedAt: 1700000000000, +}; + +describe("isAffirmative", () => { + it.each([ + ["y", true], + ["Y", true], + ["yes", true], + ["YES", true], + [" yes ", true], + ["yeah", true], + ["ok", true], + ["confirm", true], + ["1", true], + ["true", true], + ["n", false], + ["no", false], + ["", false], + ["maybe", false], + ["nope", false], + ["0", false], + ])("isAffirmative(%j) -> %s", (input, expected) => { + expect(isAffirmative(input)).toBe(expected); + }); +}); + +describe("runBackfillExtraction", () => { + beforeEach(() => { + mocks.listUnextractedSessions.mockReset(); + mocks.getUnextracted.mockReset(); + mocks.markExtracted.mockReset(); + mocks.upsertNode.mockReset(); + mocks.upsertEdge.mockReset(); + mocks.findByName.mockReset(); + mocks.getBySession.mockReset(); + mocks.extract.mockReset(); + mocks.initSchema.mockReset(); + mocks.closeDriver.mockReset(); + + mocks.initSchema.mockResolvedValue(undefined); + mocks.closeDriver.mockResolvedValue(undefined); + mocks.getBySession.mockResolvedValue([]); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + mocks.upsertNode.mockImplementation(async (_d: any, c: any) => ({ + node: { + id: `n-${c.name}`, + type: c.type, + name: c.name, + description: c.description ?? "", + content: c.content, + status: "active", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: 0, + updatedAt: 0, + }, + isNew: true, + })); + mocks.upsertEdge.mockResolvedValue(undefined); + mocks.findByName.mockResolvedValue(null); + mocks.markExtracted.mockResolvedValue(undefined); + }); + + it("returns sessionsTotal=0 and skips everything when no unextracted sessions", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: {}, + log, + }); + + expect(result.sessionsTotal).toBe(0); + expect(result.sessionsProcessed).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalledWith(expect.stringContaining("没有需要提取的会话")); + }); + + it("requires an LLM model and throws a clear error when missing", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + await expect( + runBackfillExtraction({ cfg: makeCfg(), effectiveModel: "", options: {}, log: vi.fn() }), + ).rejects.toThrow(/LLM model/); + expect(mocks.closeDriver).not.toHaveBeenCalled(); + }); + + it("requires neo4j.uri and throws a clear error when missing", async () => { + mocks.listUnextractedSessions.mockResolvedValue([]); + await expect( + runBackfillExtraction({ + cfg: { ...makeCfg(), neo4j: { uri: "", user: "", password: "" } } as any, + effectiveModel: "gpt-test", + options: {}, + log: vi.fn(), + }), + ).rejects.toThrow(/neo4j\.uri/); + }); + + it("aborts when the user declines the confirmation prompt", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const prompt = vi.fn().mockResolvedValue("n"); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: {}, + log, + prompt, + }); + + expect(prompt).toHaveBeenCalledTimes(1); + expect(result.sessionsProcessed).toBe(0); + expect(result.sessionsSkipped).toBe(1); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.markExtracted).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("does not prompt when --yes is set and runs extraction", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted.mockResolvedValueOnce([ + { role: "user", content: "hello", turn_index: 1 }, + { role: "assistant", content: "hi", turn_index: 2 }, + ]).mockResolvedValueOnce([]); + mocks.extract.mockResolvedValueOnce({ + nodes: [ + { type: "TASK", name: "t1", description: "d", content: "c" }, + { type: "SKILL", name: "s1", description: "d", content: "c" }, + ], + edges: [ + { from: "t1", to: "s1", type: "USED_SKILL", instruction: "i" }, + ], + }); + const prompt = vi.fn(); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + prompt, + }); + + expect(prompt).not.toHaveBeenCalled(); + expect(result.sessionsProcessed).toBe(1); + expect(result.nodesCreated).toBe(2); + expect(result.edgesCreated).toBe(1); + expect(result.batches).toBe(1); + expect(mocks.extract).toHaveBeenCalledTimes(1); + expect(mocks.markExtracted).toHaveBeenCalledWith(expect.anything(), "sid-abc-1234567890", 2); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("filters sessions to the one specified by --session", async () => { + mocks.listUnextractedSessions.mockResolvedValue([ + { ...SAMPLE_SESSION, sessionId: "aaa" }, + { ...SAMPLE_SESSION, sessionId: "bbb" }, + ]); + mocks.getUnextracted.mockResolvedValue([]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true, session: "bbb" }, + log, + }); + + expect(result.sessionsTotal).toBe(1); + expect(result.sessionsProcessed).toBe(1); + expect(mocks.getUnextracted).toHaveBeenCalledWith(expect.anything(), "bbb", expect.any(Number)); + }); + + it("exits cleanly when --session matches no sessions", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { session: "does-not-exist" }, + log, + }); + + expect(result.sessionsTotal).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("lists sessions but does not extract under --dry-run", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const prompt = vi.fn(); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { dryRun: true }, + log, + prompt, + }); + + expect(prompt).not.toHaveBeenCalled(); + expect(result.sessionsSkipped).toBe(1); + expect(result.sessionsProcessed).toBe(0); + expect(mocks.extract).not.toHaveBeenCalled(); + expect(log).toHaveBeenCalledWith(expect.stringContaining("--dry-run")); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("loops multiple batches until getUnextracted returns empty", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted + .mockResolvedValueOnce([ + { role: "user", content: "m1", turn_index: 1 }, + ]) + .mockResolvedValueOnce([ + { role: "user", content: "m2", turn_index: 2 }, + ]) + .mockResolvedValueOnce([]); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true, limit: 1 }, + log, + }); + + expect(result.batches).toBe(2); + expect(mocks.markExtracted).toHaveBeenCalledTimes(2); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("records a session as skipped when getUnextracted throws", async () => { + mocks.listUnextractedSessions.mockResolvedValue([ + { ...SAMPLE_SESSION, sessionId: "good" }, + { ...SAMPLE_SESSION, sessionId: "bad" }, + ]); + const callCount = new Map(); + mocks.getUnextracted.mockImplementation(async (_d: any, sid: string) => { + if (sid === "bad") throw new Error("boom"); + const n = (callCount.get(sid) ?? 0) + 1; + callCount.set(sid, n); + if (n === 1) return [{ role: "user", content: "x", turn_index: 1 }]; + return []; + }); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + }); + + expect(result.sessionsProcessed).toBe(1); + expect(result.sessionsSkipped).toBe(1); + expect(result.sessionsTotal).toBe(2); + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("calls closeDriver even when listUnextractedSessions throws (try/finally)", async () => { + mocks.listUnextractedSessions.mockRejectedValue(new Error("neo4j down")); + const log = vi.fn(); + + await expect( + runBackfillExtraction({ + cfg: makeCfg(), + effectiveModel: "gpt-test", + options: { yes: true }, + log, + }), + ).rejects.toThrow("neo4j down"); + + expect(mocks.closeDriver).toHaveBeenCalledTimes(1); + }); + + it("does not warn about batch ceiling when session completes before the ceiling", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + const fullBatches = 3; + mocks.getUnextracted.mockImplementation(async () => { + const call = mocks.getUnextracted.mock.calls.length; + if (call < fullBatches) return Array.from({ length: 5 }, (_, i) => ({ role: "user", content: `m${i}`, turn_index: call * 5 + i })); + return [{ role: "user", content: "last", turn_index: fullBatches * 5 }]; + }); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg({ compactTurnCount: 1 }), + effectiveModel: "gpt-test", + options: { yes: true, limit: 5 }, + log, + }); + + expect(result.batches).toBe(fullBatches); + expect(result.sessionsProcessed).toBe(1); + const warningCalls = log.mock.calls.filter(c => typeof c[0] === "string" && c[0].includes("达到批数上限")); + expect(warningCalls).toHaveLength(0); + }); + + it("warns about batch ceiling when the session genuinely has more messages than the ceiling allows", async () => { + mocks.listUnextractedSessions.mockResolvedValue([SAMPLE_SESSION]); + mocks.getUnextracted.mockResolvedValue(Array.from({ length: 5 }, (_, i) => ({ role: "user", content: `m${i}`, turn_index: i }))); + mocks.extract.mockResolvedValue({ nodes: [], edges: [] }); + const log = vi.fn(); + + const result = await runBackfillExtraction({ + cfg: makeCfg({ compactTurnCount: 1 }), + effectiveModel: "gpt-test", + options: { yes: true, limit: 5 }, + log, + }); + + expect(result.batches).toBe(50); + const warningCalls = log.mock.calls.filter(c => typeof c[0] === "string" && c[0].includes("达到批数上限")); + expect(warningCalls).toHaveLength(1); + }); +}); From 3df36142bfb538374539453e15dfc3e9f9cb778e Mon Sep 17 00:00:00 2001 From: TriDefender Date: Tue, 11 Aug 2026 11:22:06 +0800 Subject: [PATCH 02/11] =?UTF-8?q?Feat:=20=E6=B7=BB=E5=8A=A0=E9=81=97?= =?UTF-8?q?=E5=BF=98=E6=9B=B2=E7=BA=BF=E9=97=A8=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加模仿艾宾浩斯遗忘曲线的门控机制，长期不用的节点会被deprecate掉（非硬性删除，可被重新激活）避免过时噪声影响检索结果 --- README.md | 25 +++- docs/decay.md | 175 ++++++++++++++++++++++++ index.ts | 13 +- openclaw.plugin.json | 22 ++++ src/graph/decay.ts | 263 ++++++++++++++++++++++++++++++++++++ src/graph/maintenance.ts | 8 +- src/store/store.ts | 13 +- src/types.ts | 63 +++++++++ test/decay.test.ts | 278 +++++++++++++++++++++++++++++++++++++++ 9 files changed, 853 insertions(+), 7 deletions(-) create mode 100644 docs/decay.md create mode 100644 src/graph/decay.ts create mode 100644 test/decay.test.ts diff --git a/README.md b/README.md index fbf1c23..79bdc64 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,29 @@ Anthropic direct (Claude) — drop `baseURL`, switch `provider`: `embedding` is optional. When present, `dimensions` must match the Neo4j vector index dimension. For a fresh database, the plugin creates matching indexes during startup. If you change dimensions later, recreate the vector indexes or the Neo4j database. +### Memory decay (forgetting curve) + +Each maintenance cycle scores every active node with a three-factor weighted model (recency + frequency + intrinsic) and bidirectionally transitions nodes across three tiers: `core` / `working` / `peripheral`. Nodes never get `status=deprecated` from decay — only manual deprecate / merge does that. Decay only adjusts `tier`, so all active nodes remain searchable. + +The full formula, field mapping from the reference implementation, default-value rationale, and tuning guide live in **[`docs/decay.md`](docs/decay.md)**. + +Minimal config (all fields optional, defaults shown): + +```json +"decay": { "enabled": true } +``` + +Common overrides — for fuller control see `docs/decay.md` §4: + +```json +"decay": { + "enabled": true, + "recencyHalfLifeDays": 30, + "peripheralCompositeThreshold": 0.15, + "workingAccessThreshold": 3 +} +``` + ### OAuth login (experimental) ```bash @@ -127,7 +150,7 @@ conversation messages -> GmMessage nodes -> LLM triple extraction -> embeddings -> vector recall + community expansion + GDS PPR -> XML context injection -session end -> dedup -> global PageRank -> communities -> summaries +session end -> decay (forgetting curve) -> dedup -> global PageRank -> communities -> summaries ``` ## Verify diff --git a/docs/decay.md b/docs/decay.md new file mode 100644 index 0000000..ba57d5f --- /dev/null +++ b/docs/decay.md @@ -0,0 +1,175 @@ +# Memory Decay — 柔性评分模型 + +graph-memory-pro 的衰减机制采用**三因子加权评分 + tier 双向转换**，参考 [memory-lancedb-pro](https://github.com/CortexReach/memory-lancedb-pro) 的设计并映射到本仓库的图模型信号。 + +- **decay 不动 `status`**——只调整 `tier`（`core` / `working` / `peripheral`）。`status=deprecated` 仅由手动弃用（`gm_update mode=deprecate` / merge）触发。 +- 每次 `gm_maintain` 或 `session_end` 维护的第 0 步执行：扫描所有 active 节点 → 评分 → tier 转换 → 写回 `decayScore` / `tier` / `decayComputedAt`。 +- 评分结果可通过 `gm_stats` / CRUD API 查看；外层搜索目前**不读 decayScore 排序**（已由 PageRank + tier 隐含分层）。 + +--- + +## 1. 评分公式 + +``` +composite = wR · recency + wF · frequency + wI · intrinsic +``` + +三个权重默认 `0.4 / 0.3 / 0.3`，**推荐**和为 1。运行时若和≠1 会自动按比例归一化（`wR' = wR / (wR+wF+wI)`），保证 `composite ∈ [0,1]`，避免用户覆盖单个权重导致评分越界。归一化在 `scoreNode()` 内进行，原始 `cfg.*Weight` 值不被修改。 + +### 1.1 Recency（时间衰减，权重 0.4） + +Weibull 拉伸指数： + +``` +recency = exp( −λ · daysSinceLastAccess^β ) + +λ = ln(2) / effectiveHL +effectiveHL = recencyHalfLifeDays · exp( importanceModulation · importance ) +``` + +- **半衰期调制**：重要记忆（高 `importance`）的 `effectiveHL` 更大 → 衰减更慢。对应艾宾浩斯曲线"重要事件保留更久"。 +- **tier-β**：曲线形状随 tier 变化，反馈式调整衰减速度： + + | tier | β | 效果 | + |---|---|---| + | `core` | 0.8 | 尾部衰减缓（核心知识保得久） | + | `working` | 1.0 | 标准指数衰减 | + | `peripheral` | 1.3 | 加速衰减（边缘知识更快被遗忘） | + +### 1.2 Frequency（访问频率，权重 0.3） + +``` +frequency = base · ( 0.5 + 0.5 · recentnessBonus ) + +base = 1 − exp( −validatedCount / 5 ) +recentnessBonus = exp( −avgAccessGapDays / 30 ) # 仅当 validatedCount > 1 +avgAccessGapDays = ( lastAccessedAt − createdAt ) / ( validatedCount − 1 ) +``` + +- 用 `validatedCount`（LLM 重新提取的次数）替代 lancedb-pro 的 `accessCount`（manual recall 触发的次数）。前者是更强的"重新确认"信号。 +- `validatedCount ≤ 1` 时跳过 `recentnessBonus`，只返回 `base`（无法算平均间隔）。 + +### 1.3 Intrinsic（内在价值，权重 0.3） + +``` +intrinsic = importance · confidence + +importance = pagerank / maxPagerank # 每次扫描时按当前批次归一化到 [0,1] +confidence = 1 − 1 / ( 1 + validatedCount ) # 饱和函数，收敛到 1 +``` + +--- + +## 2. 字段映射（lancedb-pro → graph-memory-pro） + +| lancedb-pro 字段 | 本仓库替代 | 说明 | +|---|---|---| +| `accessCount` | `validatedCount` | LLM 重新提取次数（强信号，原为 manual recall 触发） | +| `lastAccessedAt` | `lastAccessedAt` | 由 `upsertNode` 在任意写入路径刷新（重新提取、`gm_record`、`gm_update`、CRUD POST）。`mergeNodes` 故意不刷新（合并 ≠ 用户重新激活） | +| `importance` | `pagerank / maxPagerank` | 图结构重要性，每次扫描归一化 | +| `confidence` | `1 − 1/(1+validatedCount)` | 饱和置信度 | +| `tier` | `tier`（新增字段） | 与 `status` 正交 | + +--- + +## 3. Tier 双向转换 + +| 转换 | 条件 | +|---|---| +| **core → working** | `composite < peripheralCompositeThreshold` **AND** `count < workingAccessThreshold` | +| **working → peripheral** | `composite < peripheralCompositeThreshold` **OR**（`ageDays > peripheralAgeDays` **AND** `count < workingAccessThreshold`） | +| **peripheral → working** | `count >= workingAccessThreshold` **AND** `composite >= workingCompositeThreshold` | +| **working → core** | `count >= coreAccessThreshold` **AND** `composite >= coreCompositeThreshold` **AND** `importance >= coreImportanceThreshold` | + +- 新节点默认 `tier = "working"`。 +- 节点保持 `status = active` 不变；tier 变化时仅更新 `updatedAt`，不改变搜索过滤行为。 +- 不存在的"core→peripheral"和"peripheral→core"由两次相邻转换实现（经过 working）。 + +--- + +## 4. 默认值与调参指南 + +### 4.1 默认配置 + +```json +{ + "decay": { + "enabled": true, + "recencyHalfLifeDays": 30, + "recencyWeight": 0.4, + "importanceModulation": 1.5, + "frequencyWeight": 0.3, + "intrinsicWeight": 0.3, + "betaCore": 0.8, + "betaWorking": 1.0, + "betaPeripheral": 1.3, + "coreAccessThreshold": 10, + "coreCompositeThreshold": 0.7, + "coreImportanceThreshold": 0.8, + "peripheralCompositeThreshold": 0.15, + "peripheralAgeDays": 60, + "workingAccessThreshold": 3, + "workingCompositeThreshold": 0.4 + } +} +``` + +### 4.2 数值来源 + +| 参数 | 默认值 | 来源 | +|---|---|---| +| `recencyHalfLifeDays` | 30 | 艾宾浩斯曲线 ~25% 保留率拐点；同时与 lancedb-pro 的 `recencyHalfLifeDays` + `ACCESS_DECAY_HALF_LIFE_DAYS` 一致 | +| `importanceModulation` | 1.5 | lancedb-pro：`effectiveHL = 30 · exp(1.5 · importance)`，importance=1 时半衰期延长到 ~134 天 | +| `betaCore/Working/Peripheral` | 0.8 / 1.0 / 1.3 | lancedb-pro Weibull 形状参数 | +| 7 个 tier 转换阈值 | — | lancedb-pro `tier-manager` 默认值 | +| `recencyWeight / frequencyWeight / intrinsicWeight` | 0.4 / 0.3 / 0.3 | lancedb-pro 三因子权重，和为 1 | +| `validatedCount` 分母 | 5 | lancedb-pro 的 `1 − exp(−count/5)` 基础频率项（未改） | + +### 4.3 常见调参场景 + +| 想要的效果 | 调整方向 | +|---|---| +| 记忆整体保留更久 | 调高 `recencyHalfLifeDays`（如 60）或调低 `peripheralCompositeThreshold`（更难降级） | +| 更激进遗忘 | 调低 `recencyHalfLifeDays`（如 14）或调高 `peripheralCompositeThreshold` | +| 重要知识显著保得久 | 调高 `importanceModulation`（半衰期调制更强） | +| 核心知识不易降级 | 调低 `betaCore`（更缓的尾部）或调高 `coreCompositeThreshold`（更难升 core，留在 working 也保得久） | +| 单次曝光更易遗忘 | 调高 `workingAccessThreshold`（promote 到 working 需要更多确认） | +| 永久禁用衰减 | `"enabled": false` | + +### 4.4 与原布尔阈值方案的对照（向后兼容） + +旧版本（`maxAgeDays` + `minCalls`）的布尔规则已被这套柔性评分取代。原默认值 `maxAgeDays=30, minCalls=2` 在新模型下大致对应于： + +- 一个 `validatedCount=1`、`tier=working`、低 pagerank 的节点，约 30 天后 `recency` 跌破 0.15 → `composite` 跌破 `peripheralCompositeThreshold` → demote 到 `peripheral`。 +- 关键差别：新模型**不会 deprecate**，只是降到 `peripheral` tier，搜索过滤仍包含它（只是 decayScore 较低）。 + +--- + +## 5. 数据库字段 + +| 字段 | 类型 | 写入者 | 说明 | +|---|---|---|---| +| `tier` | string | `applyDecay` / `upsertNode`（创建时初始化为 `working`） | `core` / `working` / `peripheral` | +| `lastAccessedAt` | int (epoch ms) | `upsertNode`（重新提取时） | decay 评分的时间基准 | +| `decayScore` | float (0~1) | `applyDecay` | 最近一次评分结果 | +| `decayComputedAt` | int (epoch ms) | `applyDecay` | 评分时间戳 | + +旧节点缺这些字段时： +- `tier` 缺失 → 评分按 `working` 处理；首次 `applyDecay` 时自动写入 `working` +- `lastAccessedAt` 缺失 → 回退到 `updatedAt` / `createdAt` +- `decayScore` / `decayComputedAt` 缺失 → 在首次 `applyDecay` 前为 undefined，不影响评分 + +**Backfill 时机**：新字段在第一次 `applyDecay` 运行时为每个 active 节点批量写入。如果部署初始用 `decay.enabled=false`，字段会一直缺失直到切换为 `true` 后的第一次维护周期。在切换前的窗口期，对 raw DB 直接做 `tier` 过滤查询会返回 null/missing 而非 `"working"`——目前搜索路径不读 `tier`，但自定义查询需要留意。 + +--- + +## 6. 实现位置 + +| 文件 | 内容 | +|---|---| +| `src/graph/decay.ts` | 评分函数 + tier 决策 + `applyDecay()` 批处理 | +| `src/types.ts` | `DecayConfig` 接口、`NodeTier` 类型、`GmNode` 新字段、`DEFAULT_CONFIG.decay` | +| `src/store/store.ts` | `toNode` 字段映射、`upsertNode` 初始化 `tier` / `lastAccessedAt` | +| `src/graph/maintenance.ts` | 调用入口（step 0） | +| `test/decay.test.ts` | 评分函数 + tier 决策纯函数单元测试 | +| `openclaw.plugin.json` | 用户可见的配置 schema | diff --git a/index.ts b/index.ts index 2df04fd..4799841 100755 --- a/index.ts +++ b/index.ts @@ -272,6 +272,7 @@ const graphMemoryProPlugin = { const cfg: GmConfig = { ...DEFAULT_CONFIG, ...raw }; if (raw.neo4j) cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...raw.neo4j }; + if (raw.decay) cfg.decay = { ...DEFAULT_CONFIG.decay, ...raw.decay }; const providerModel = readDefaultModel(api.config); @@ -1121,13 +1122,21 @@ const graphMemoryProPlugin = { (_ctx: any) => ({ name: "gm_maintain", label: "Graph Memory Maintenance", - description: "手动触发图维护：去重、PageRank、社区检测。", + description: "手动触发图维护：衰减评分 + tier 转换、去重、PageRank、社区检测。", parameters: Type.Object({}), async execute() { const embedFn = (recaller as any).embed ?? undefined; const result = await runMaintenance(driver, cfg, llm, embedFn); + const t = result.decay.tierTransitions; + const totalTransitions = t.coreToWorking + t.workingToPeripheral + t.peripheralToWorking + t.workingToCore; const text = [ `🔧 图维护完成（${result.durationMs}ms）`, + result.decay.enabled + ? `衰减：扫描 ${result.decay.scanned} 个节点，tier 转换 ${totalTransitions} 次` + + (totalTransitions > 0 + ? `（core→working ${t.coreToWorking}，working→peripheral ${t.workingToPeripheral}，peripheral→working ${t.peripheralToWorking}，working→core ${t.workingToCore}）` + : "") + : `衰减：已禁用`, `去重：${result.dedup.pairs.length} 对相似，合并 ${result.dedup.merged} 对`, ...(result.dedup.pairs.length > 0 ? result.dedup.pairs.slice(0, 5).map(p => ` "${p.nameA}" ≈ "${p.nameB}" (${(p.similarity * 100).toFixed(1)}%)`) @@ -1137,7 +1146,7 @@ const graphMemoryProPlugin = { `PageRank Top 5：`, ...result.pagerank.topK.slice(0, 5).map((n, i) => ` ${i + 1}. ${n.name} (${n.score.toFixed(4)})`), ].join("\n"); - return { content: [{ type: "text", text }], details: { durationMs: result.durationMs, dedupMerged: result.dedup.merged, communities: result.community.count } }; + return { content: [{ type: "text", text }], details: { durationMs: result.durationMs, decayTransitions: totalTransitions, dedupMerged: result.dedup.merged, communities: result.community.count } }; }, }), { name: "gm_maintain" }, diff --git a/openclaw.plugin.json b/openclaw.plugin.json index f54c9d0..1a6b474 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -29,6 +29,28 @@ "dedupThreshold": { "type": "number", "default": 0.90 }, "pagerankDamping": { "type": "number", "default": 0.85 }, "pagerankIterations": { "type": "number", "default": 20 }, + "decay": { + "type": "object", + "description": "柔性衰减：三因子加权评分（recency+frequency+intrinsic）+ tier 双向转换（core/working/peripheral）。完整公式与调参指南见 docs/decay.md。recencyWeight + frequencyWeight + intrinsicWeight 推荐和为 1（运行时会自动归一化）。", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否启用自动衰减。关闭后 tier 永久保持初始 working 状态。" }, + "recencyHalfLifeDays": { "type": "number", "default": 30, "description": "Recency 半衰期（天）。effectiveHL = halfLife * exp(importanceModulation * importance)。" }, + "recencyWeight": { "type": "number", "default": 0.4, "description": "Recency 在 composite 中的权重。三个权重推荐和为 1。" }, + "importanceModulation": { "type": "number", "default": 1.5, "description": "半衰期调制系数；越大则高 importance 节点衰减越慢。" }, + "frequencyWeight": { "type": "number", "default": 0.3, "description": "Frequency 在 composite 中的权重。三个权重推荐和为 1。" }, + "intrinsicWeight": { "type": "number", "default": 0.3, "description": "Intrinsic（importance × confidence）在 composite 中的权重。三个权重推荐和为 1。" }, + "betaCore": { "type": "number", "default": 0.8, "description": "core tier 的 Weibull 形状参数；<1 = 缓衰。" }, + "betaWorking": { "type": "number", "default": 1.0, "description": "working tier 的 Weibull 形状参数；=1 = 标准指数衰减。" }, + "betaPeripheral": { "type": "number", "default": 1.3, "description": "peripheral tier 的 Weibull 形状参数；>1 = 加速衰减。" }, + "coreAccessThreshold": { "type": "number", "default": 10, "description": "working→core 所需的最低 validatedCount。" }, + "coreCompositeThreshold": { "type": "number", "default": 0.7, "description": "working→core 所需的最低 composite 分数。" }, + "coreImportanceThreshold": { "type": "number", "default": 0.8, "description": "working→core 所需的最低归一化 importance。" }, + "peripheralCompositeThreshold": { "type": "number", "default": 0.15, "description": "composite 低于此值触发 demote（core→working 或 working→peripheral）。" }, + "peripheralAgeDays": { "type": "number", "default": 60, "description": "working→peripheral 的年龄阈值（同时 validatedCount < workingAccessThreshold 才触发）。" }, + "workingAccessThreshold": { "type": "number", "default": 3, "description": "demote（count 不足时）/ promote（count 充足时）的 access 次数分界。" }, + "workingCompositeThreshold": { "type": "number", "default": 0.4, "description": "peripheral→working 所需的最低 composite 分数。" } + } + }, "llm": { "type": "object", "properties": { diff --git a/src/graph/decay.ts b/src/graph/decay.ts new file mode 100644 index 0000000..6de5c07 --- /dev/null +++ b/src/graph/decay.ts @@ -0,0 +1,263 @@ +/** + * graph-memory-pro — 柔性衰减（三因子加权评分 + tier 双向转换） + * + * 完整公式、字段映射、默认值来源、调参指南见 docs/decay.md。 + * 评分 / tier 决策 / applyDecay 的入口均在本文件。 + * + * 调用时机：runMaintenance 的第 0 步（去重/PageRank/社区之前）。 + * decay 不动 status，只动 tier。 + */ + +import type { Driver } from "neo4j-driver"; +import type { GmConfig, DecayConfig, GmNode, NodeTier } from "../types.ts"; +import { getSession } from "../store/db.ts"; +import { allActiveNodes } from "../store/store.ts"; + +const MS_PER_DAY = 86_400_000; + +export interface CompositeScore { + composite: number; + recency: number; + frequency: number; + intrinsic: number; +} + +export interface TierTransition { + coreToWorking: number; + workingToPeripheral: number; + peripheralToWorking: number; + workingToCore: number; +} + +export interface DecayResult { + enabled: boolean; + scanned: number; + tierTransitions: TierTransition; + durationMs: number; +} + +// ─── 归一化辅助（纯函数，便于单元测试） ────────────────────── + +/** importance ∈ [0,1]：当前批次的 pagerank 归一化值。 */ +export function normalizeImportance(pagerank: number, maxPagerank: number): number { + if (maxPagerank <= 0) return 0; + return Math.min(1, Math.max(0, pagerank / maxPagerank)); +} + +/** confidence ∈ [0,1)：validatedCount 越高越可信，饱和收敛到 1。 */ +export function computeConfidence(validatedCount: number): number { + const c = Math.max(0, validatedCount); + return 1 - 1 / (1 + c); +} + +// ─── 三因子评分（纯函数） ──────────────────────────────────── + +/** β 随 tier 变化：core 缓衰、peripheral 促衰。 */ +export function computeBeta(tier: NodeTier, cfg: DecayConfig): number { + switch (tier) { + case "core": return cfg.betaCore; + case "working": return cfg.betaWorking; + case "peripheral": return cfg.betaPeripheral; + } +} + +/** + * Recency 分量：Weibull 拉伸指数衰减。 + * tier 决定 β；importance 调制半衰期（高重要性 → 慢衰减）。 + */ +export function scoreRecency( + node: Pick, + importance: number, + now: number, + cfg: DecayConfig, +): number { + const lastActive = node.lastAccessedAt > 0 + ? node.lastAccessedAt + : (node.updatedAt > 0 ? node.updatedAt : node.createdAt); + const daysSince = Math.max(0, (now - lastActive) / MS_PER_DAY); + + const effectiveHL = cfg.recencyHalfLifeDays * Math.exp(cfg.importanceModulation * importance); + const lambda = Math.LN2 / effectiveHL; + const beta = computeBeta(node.tier ?? "working", cfg); + + return Math.exp(-lambda * Math.pow(daysSince, beta)); +} + +/** + * Frequency 分量：基础饱和项 × 平均访问间隔新鲜度。 + * validatedCount ≤ 1 时只返回基础项（无法算平均间隔）。 + */ +export function scoreFrequency( + node: Pick, +): number { + const count = Math.max(0, node.validatedCount); + const base = 1 - Math.exp(-count / 5); + if (count <= 1) return base; + + const lastActive = node.lastAccessedAt > 0 + ? node.lastAccessedAt + : (node.updatedAt > 0 ? node.updatedAt : node.createdAt); + const accessSpanDays = Math.max(1, (lastActive - node.createdAt) / MS_PER_DAY); + const avgGapDays = accessSpanDays / Math.max(count - 1, 1); + const recentnessBonus = Math.exp(-avgGapDays / 30); + + return base * (0.5 + 0.5 * recentnessBonus); +} + +/** Intrinsic 分量：importance × confidence。 */ +export function scoreIntrinsic(importance: number, confidence: number): number { + return importance * confidence; +} + +/** 三因子加权汇总。权重和在运行时归一化到 1，避免用户配置偏差导致 composite > 1。 */ +export function scoreNode( + node: Pick, + maxPagerank: number, + now: number, + cfg: DecayConfig, +): CompositeScore { + const importance = normalizeImportance(node.pagerank, maxPagerank); + const confidence = computeConfidence(node.validatedCount); + const recency = scoreRecency(node, importance, now, cfg); + const frequency = scoreFrequency(node); + const intrinsic = scoreIntrinsic(importance, confidence); + + const wSum = cfg.recencyWeight + cfg.frequencyWeight + cfg.intrinsicWeight; + const safeSum = wSum > 0 ? wSum : 1; + const wR = cfg.recencyWeight / safeSum; + const wF = cfg.frequencyWeight / safeSum; + const wI = cfg.intrinsicWeight / safeSum; + + const composite = wR * recency + wF * frequency + wI * intrinsic; + + return { composite, recency, frequency, intrinsic }; +} + +// ─── Tier 转换决策（纯函数） ───────────────────────────────── + +/** + * 决定节点的下一个 tier。返回 null 表示保持不变。 + * importance 已归一化（调用方须先 normalizeImportance）。 + */ +export function decideTierTransition( + node: Pick, + score: CompositeScore, + importance: number, + cfg: DecayConfig, + now: number = Date.now(), +): NodeTier | null { + const current = node.tier ?? "working"; + const count = node.validatedCount; + const ageDays = Math.max(0, (now - node.createdAt) / MS_PER_DAY); + const composite = score.composite; + + if (current === "core" + && composite < cfg.peripheralCompositeThreshold + && count < cfg.workingAccessThreshold) { + return "working"; + } + + if (current === "working") { + if (composite < cfg.peripheralCompositeThreshold) return "peripheral"; + if (ageDays > cfg.peripheralAgeDays && count < cfg.workingAccessThreshold) { + return "peripheral"; + } + } + + if (current === "peripheral" + && count >= cfg.workingAccessThreshold + && composite >= cfg.workingCompositeThreshold) { + return "working"; + } + + if (current === "working" + && count >= cfg.coreAccessThreshold + && composite >= cfg.coreCompositeThreshold + && importance >= cfg.coreImportanceThreshold) { + return "core"; + } + + return null; +} + +// ─── 应用层：扫描 + 评分 + 转换 ────────────────────────────── + +const EMPTY_TRANSITIONS: TierTransition = { + coreToWorking: 0, + workingToPeripheral: 0, + peripheralToWorking: 0, + workingToCore: 0, +}; + +function bumpTransition(transitions: TierTransition, from: NodeTier, to: NodeTier): void { + if (from === "core" && to === "working") transitions.coreToWorking++; + else if (from === "working" && to === "peripheral") transitions.workingToPeripheral++; + else if (from === "peripheral" && to === "working") transitions.peripheralToWorking++; + else if (from === "working" && to === "core") transitions.workingToCore++; +} + +/** + * 扫描所有 active 节点：评分 + tier 转换 + 写回 decayScore / tier。 + * 不动 status（status=deprecated 仅由手动弃用触发）。 + */ +export async function applyDecay(driver: Driver, cfg: Pick): Promise { + const start = Date.now(); + const d = cfg.decay; + if (!d?.enabled) { + return { enabled: false, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, durationMs: 0 }; + } + + const nodes = await allActiveNodes(driver); + if (nodes.length === 0) { + return { enabled: true, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, durationMs: 0 }; + } + + const maxPagerank = Math.max(...nodes.map(n => n.pagerank), 0.0001); + + const updates: Array<{ id: string; tier: NodeTier; composite: number; tierChanged: boolean }> = []; + const transitions: TierTransition = { ...EMPTY_TRANSITIONS }; + + for (const node of nodes) { + const score = scoreNode(node, maxPagerank, start, d); + const importance = normalizeImportance(node.pagerank, maxPagerank); + const currentTier = node.tier ?? "working"; + const nextTier = decideTierTransition(node, score, importance, d, start); + const finalTier = nextTier ?? currentTier; + const tierChanged = nextTier !== null; + + if (tierChanged) bumpTransition(transitions, currentTier, finalTier); + + updates.push({ + id: node.id, + tier: finalTier, + composite: score.composite, + tierChanged, + }); + } + + if (updates.length > 0) { + const session = getSession(driver); + try { + await session.run( + `UNWIND $updates AS u + MATCH (n:Task|Skill|Event {id: u.id}) + SET n.tier = u.tier, + n.decayScore = u.composite, + n.decayComputedAt = $now, + n.updatedAt = CASE WHEN u.tierChanged THEN $now ELSE n.updatedAt END`, + { updates, now: start }, + ); + } finally { + await session.close(); + } + } + + return { + enabled: true, + scanned: nodes.length, + tierTransitions: transitions, + durationMs: Date.now() - start, + }; +} diff --git a/src/graph/maintenance.ts b/src/graph/maintenance.ts index 64cd4fa..a2e68b8 100755 --- a/src/graph/maintenance.ts +++ b/src/graph/maintenance.ts @@ -2,7 +2,7 @@ * graph-memory-pro — 图谱维护 * * 调用时机：session_end（finalize 之后） - * 执行顺序：去重 → 全局 PageRank → 社区检测 → 社区描述 + * 执行顺序：衰减 → 去重 → 全局 PageRank → 社区检测 → 社区描述 */ import type { Driver } from "neo4j-driver"; @@ -12,8 +12,10 @@ import type { EmbedFn } from "../engine/embed.ts"; import { computeGlobalPageRank, type GlobalPageRankResult } from "./pagerank.ts"; import { detectCommunities, summarizeCommunities, type CommunityResult } from "./community.ts"; import { dedup, type DedupResult } from "./dedup.ts"; +import { applyDecay, type DecayResult } from "./decay.ts"; export interface MaintenanceResult { + decay: DecayResult; dedup: DedupResult; pagerank: GlobalPageRankResult; community: CommunityResult; @@ -26,6 +28,9 @@ export async function runMaintenance( ): Promise { const start = Date.now(); + // 0. 衰减（柔性评分 + tier 转换）—— 先于其他步骤，让后续基于最新 tier 集合运算 + const decayResult = await applyDecay(driver, cfg); + // 1. 去重 const dedupResult = await dedup(driver, cfg); @@ -44,6 +49,7 @@ export async function runMaintenance( } return { + decay: decayResult, dedup: dedupResult, pagerank: pagerankResult, community: communityResult, diff --git a/src/store/store.ts b/src/store/store.ts index 992d259..d85541c 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -8,7 +8,7 @@ import type { Driver } from "neo4j-driver"; import neo4j from "neo4j-driver"; import { createHash } from "crypto"; -import type { GmNode, GmEdge, EdgeType, NodeType } from "../types.ts"; +import type { GmNode, GmEdge, EdgeType, NodeType, NodeTier } from "../types.ts"; import { NODE_TYPE_TO_LABEL, isValidEdgeDirection } from "../types.ts"; import { getSession } from "./db.ts"; @@ -32,6 +32,8 @@ function toNode(r: any): GmNode { description: n.description ?? "", content: n.content, status: n.status, + tier: (n.tier === "core" || n.tier === "working" || n.tier === "peripheral" + ? n.tier : "working") as NodeTier, validatedCount: toInt(n.validatedCount ?? n.validated_count ?? 1), sourceSessions: typeof n.sourceSessions === "string" ? JSON.parse(n.sourceSessions) @@ -40,6 +42,9 @@ function toNode(r: any): GmNode { pagerank: toFloat(n.pagerank ?? 0), createdAt: toInt(n.createdAt ?? n.created_at ?? 0), updatedAt: toInt(n.updatedAt ?? n.updated_at ?? 0), + lastAccessedAt: toInt(n.lastAccessedAt ?? n.last_accessed_at ?? n.updatedAt ?? n.updated_at ?? n.createdAt ?? 0), + decayScore: typeof n.decayScore === "number" ? n.decayScore : undefined, + decayComputedAt: n.decayComputedAt ? toInt(n.decayComputedAt) : undefined, }; } @@ -164,6 +169,7 @@ export async function upsertNode( THEN n.sourceSessions + $sessionId ELSE n.sourceSessions END, + n.lastAccessedAt = $now, n.updatedAt = $now RETURN n `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); @@ -180,9 +186,10 @@ export async function upsertNode( CREATE (n:MemoryNode:${label} { id: $id, name: $name, type: $type, description: $description, content: $content, - status: 'active', validatedCount: 1, + status: 'active', tier: 'working', validatedCount: 1, sourceSessions: $sessions, communityId: null, - pagerank: 0.0, createdAt: $now, updatedAt: $now + pagerank: 0.0, createdAt: $now, updatedAt: $now, + lastAccessedAt: $now }) RETURN n `, { diff --git a/src/types.ts b/src/types.ts index 4ae637d..47f1596 100755 --- a/src/types.ts +++ b/src/types.ts @@ -10,6 +10,13 @@ export type NodeType = "TASK" | "SKILL" | "EVENT"; export type NodeStatus = "active" | "deprecated"; +/** + * 记忆分层 tier（与 NodeStatus 正交）。 + * decay 评分模型据此双向转换：core↔working↔peripheral。 + * 节点仍保持 status=active，仅 tier 变化；status=deprecated 只由手动弃用触发。 + */ +export type NodeTier = "core" | "working" | "peripheral"; + /** Neo4j label 映射：TASK->Task, SKILL->Skill, EVENT->Event */ export const NODE_TYPE_TO_LABEL: Record = { TASK: "Task", @@ -24,12 +31,24 @@ export interface GmNode { description: string; content: string; status: NodeStatus; + tier: NodeTier; validatedCount: number; sourceSessions: string[]; communityId: string | null; pagerank: number; createdAt: number; updatedAt: number; + /** + * 最近一次"相关性活动"时间戳（epoch ms），由 upsertNode 在任意写入路径刷新 + * （重新提取、gm_record、gm_update、CRUD POST）。是衰减判定的基准。 + * 与 updatedAt 的区别：updatedAt 在 deprecate/merge 时也会变，不能代表相关性； + * 而 mergeNodes 故意不更新 lastAccessedAt（合并 ≠ 用户重新激活）。 + */ + lastAccessedAt: number; + /** 最近一次 decay 评分（0~1，越大越鲜活/重要）。仅 applyDecay 写入。 */ + decayScore?: number; + /** decayScore 的计算时间戳（epoch ms）。 */ + decayComputedAt?: number; } // ─── 边 ─────────────────────────────────────────────────────── @@ -137,6 +156,30 @@ export interface Neo4jConfig { password: string; } +// ─── 衰减（柔性评分模型）配置 ───────────────────────────────── +// +// 完整公式、字段映射、默认值来源、调参指南见 docs/decay.md。 +// 评分和 tier 转换逻辑实现在 src/graph/decay.ts。 + +export interface DecayConfig { + enabled: boolean; + recencyHalfLifeDays: number; + recencyWeight: number; + importanceModulation: number; + frequencyWeight: number; + intrinsicWeight: number; + betaCore: number; + betaWorking: number; + betaPeripheral: number; + coreAccessThreshold: number; + coreCompositeThreshold: number; + coreImportanceThreshold: number; + peripheralCompositeThreshold: number; + peripheralAgeDays: number; + workingAccessThreshold: number; + workingCompositeThreshold: number; +} + // ─── 插件配置 ───────────────────────────────────────────────── export interface GmConfig { @@ -163,6 +206,8 @@ export interface GmConfig { dedupThreshold: number; pagerankDamping: number; pagerankIterations: number; + /** 遗忘曲线衰减配置；未提供时使用 DEFAULT_CONFIG.decay。 */ + decay?: DecayConfig; } export const DEFAULT_CONFIG: GmConfig = { @@ -178,4 +223,22 @@ export const DEFAULT_CONFIG: GmConfig = { dedupThreshold: 0.90, pagerankDamping: 0.85, pagerankIterations: 20, + decay: { + enabled: true, + recencyHalfLifeDays: 30, + recencyWeight: 0.4, + importanceModulation: 1.5, + frequencyWeight: 0.3, + intrinsicWeight: 0.3, + betaCore: 0.8, + betaWorking: 1.0, + betaPeripheral: 1.3, + coreAccessThreshold: 10, + coreCompositeThreshold: 0.7, + coreImportanceThreshold: 0.8, + peripheralCompositeThreshold: 0.15, + peripheralAgeDays: 60, + workingAccessThreshold: 3, + workingCompositeThreshold: 0.4, + }, }; diff --git a/test/decay.test.ts b/test/decay.test.ts new file mode 100644 index 0000000..c6ab7a6 --- /dev/null +++ b/test/decay.test.ts @@ -0,0 +1,278 @@ +import { describe, it, expect } from "vitest"; +import { + normalizeImportance, + computeConfidence, + computeBeta, + scoreRecency, + scoreFrequency, + scoreIntrinsic, + scoreNode, + decideTierTransition, +} from "../src/graph/decay.ts"; +import { DEFAULT_CONFIG, type DecayConfig, type GmNode } from "../src/types.ts"; + +const cfg: DecayConfig = { ...DEFAULT_CONFIG.decay! }; +const NOW = Date.UTC(2026, 0, 15, 0, 0, 0); +const MS_PER_DAY = 86_400_000; + +function makeNode(overrides: Partial = {}): GmNode { + return { + id: "test-id", + type: "SKILL", + name: "test", + description: "", + content: "", + status: "active", + tier: "working", + validatedCount: 1, + sourceSessions: [], + communityId: null, + pagerank: 0, + createdAt: NOW - 10 * MS_PER_DAY, + updatedAt: NOW - 10 * MS_PER_DAY, + lastAccessedAt: NOW - 10 * MS_PER_DAY, + ...overrides, + }; +} + +describe("normalizeImportance", () => { + it("pagerank=0 时返回 0（即使 maxPagerank>0）", () => { + expect(normalizeImportance(0, 1.0)).toBe(0); + }); + + it("maxPagerank≤0 时返回 0（避免除零）", () => { + expect(normalizeImportance(5, 0)).toBe(0); + expect(normalizeImportance(5, -1)).toBe(0); + }); + + it("pagerank = maxPagerank 时返回 1", () => { + expect(normalizeImportance(0.5, 0.5)).toBe(1); + }); + + it("截断到 [0,1]", () => { + expect(normalizeImportance(2.0, 1.0)).toBe(1); + expect(normalizeImportance(-1, 1.0)).toBe(0); + }); +}); + +describe("computeConfidence", () => { + it("count=0 时 confidence=0", () => { + expect(computeConfidence(0)).toBe(0); + }); + + it("count=1 时 confidence=0.5", () => { + expect(computeConfidence(1)).toBeCloseTo(0.5, 6); + }); + + it("count 增大时饱和收敛到 1（永不达到）", () => { + expect(computeConfidence(10)).toBeLessThan(1); + expect(computeConfidence(100)).toBeLessThan(1); + expect(computeConfidence(100)).toBeGreaterThan(computeConfidence(10)); + }); + + it("负数按 0 处理", () => { + expect(computeConfidence(-5)).toBe(0); + }); +}); + +describe("computeBeta", () => { + it("core < working < peripheral（缓衰 → 促衰）", () => { + expect(computeBeta("core", cfg)).toBe(0.8); + expect(computeBeta("working", cfg)).toBe(1.0); + expect(computeBeta("peripheral", cfg)).toBe(1.3); + }); +}); + +describe("scoreRecency", () => { + it("刚刚访问（daysSince=0）→ 1.0", () => { + const node = makeNode({ lastAccessedAt: NOW }); + expect(scoreRecency(node, 0, NOW, cfg)).toBeCloseTo(1, 6); + }); + + it("importance=0 + working tier + 30 天 → recency ≈ 0.5（半衰期）", () => { + const node = makeNode({ tier: "working", lastAccessedAt: NOW - 30 * MS_PER_DAY }); + expect(scoreRecency(node, 0, NOW, cfg)).toBeCloseTo(0.5, 2); + }); + + it("高 importance 拉长 effectiveHL（衰减更慢）", () => { + const node = makeNode({ tier: "working", lastAccessedAt: NOW - 30 * MS_PER_DAY }); + const highImp = scoreRecency(node, 1.0, NOW, cfg); + const zeroImp = scoreRecency(node, 0, NOW, cfg); + expect(highImp).toBeGreaterThan(zeroImp); + expect(highImp).toBeGreaterThan(0.5); + }); + + it("tier=peripheral 比 tier=working 衰减更快", () => { + const days = 10; + const w = scoreRecency(makeNode({ tier: "working", lastAccessedAt: NOW - days * MS_PER_DAY }), 0, NOW, cfg); + const p = scoreRecency(makeNode({ tier: "peripheral", lastAccessedAt: NOW - days * MS_PER_DAY }), 0, NOW, cfg); + expect(p).toBeLessThan(w); + }); + + it("lastAccessedAt 缺失时回退到 updatedAt", () => { + const viaFallback = makeNode({ lastAccessedAt: 0, updatedAt: NOW - 5 * MS_PER_DAY }); + const direct = makeNode({ lastAccessedAt: NOW - 5 * MS_PER_DAY }); + expect(scoreRecency(viaFallback, 0, NOW, cfg)) + .toBeCloseTo(scoreRecency(direct, 0, NOW, cfg), 6); + }); +}); + +describe("scoreFrequency", () => { + it("count=0 时 base=0", () => { + expect(scoreFrequency(makeNode({ validatedCount: 0 }))).toBe(0); + }); + + it("count=1 时只返回 base（无 recentnessBonus）", () => { + const expected = 1 - Math.exp(-1 / 5); + expect(scoreFrequency(makeNode({ validatedCount: 1 }))).toBeCloseTo(expected, 6); + }); + + it("count > 1 时 base × (0.5 + 0.5*recentnessBonus)，结果 ≤ base", () => { + const node = makeNode({ + validatedCount: 3, + createdAt: NOW - 30 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + const base = 1 - Math.exp(-3 / 5); + const score = scoreFrequency(node); + expect(score).toBeLessThanOrEqual(base); + expect(score).toBeGreaterThan(0); + }); + + it("访问越紧凑（avgGapDays 越小）recentnessBonus 越大", () => { + const tight = makeNode({ + validatedCount: 5, + createdAt: NOW - 4 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + const sparse = makeNode({ + validatedCount: 5, + createdAt: NOW - 100 * MS_PER_DAY, + lastAccessedAt: NOW, + }); + expect(scoreFrequency(tight)).toBeGreaterThan(scoreFrequency(sparse)); + }); +}); + +describe("scoreIntrinsic", () => { + it("= importance × confidence", () => { + expect(scoreIntrinsic(0.5, 0.5)).toBeCloseTo(0.25, 6); + expect(scoreIntrinsic(1, 1)).toBe(1); + expect(scoreIntrinsic(0, 0.5)).toBe(0); + }); +}); + +describe("scoreNode", () => { + it("权重和为 1 时 composite 落在 [0,1]", () => { + const node = makeNode({ pagerank: 0.5, validatedCount: 5, lastAccessedAt: NOW }); + const r = scoreNode(node, 1.0, NOW, cfg); + expect(r.composite).toBeGreaterThanOrEqual(0); + expect(r.composite).toBeLessThanOrEqual(1); + }); + + it("新鲜高 PR 节点 composite 显著高于陈旧低 PR 节点", () => { + const fresh = makeNode({ pagerank: 1.0, validatedCount: 1, lastAccessedAt: NOW }); + const stale = makeNode({ + pagerank: 0, + validatedCount: 1, + lastAccessedAt: NOW - 90 * MS_PER_DAY, + }); + expect(scoreNode(fresh, 1.0, NOW, cfg).composite) + .toBeGreaterThan(scoreNode(stale, 1.0, NOW, cfg).composite); + }); + + it("权重和≠1 时自动归一化，composite 仍落在 [0,1]", () => { + const skewedCfg: DecayConfig = { + ...cfg, + recencyWeight: 0.5, + frequencyWeight: 0.5, + intrinsicWeight: 0.5, // 和=1.5 + }; + const node = makeNode({ + pagerank: 1.0, + validatedCount: 10, + lastAccessedAt: NOW, + updatedAt: NOW, + createdAt: NOW, + }); + const r = scoreNode(node, 1.0, NOW, skewedCfg); + expect(r.composite).toBeLessThanOrEqual(1); + expect(r.composite).toBeGreaterThanOrEqual(0); + }); + + it("权重和为 0 时回退到等权重，不抛错", () => { + const zeroCfg: DecayConfig = { + ...cfg, + recencyWeight: 0, + frequencyWeight: 0, + intrinsicWeight: 0, + }; + const node = makeNode({ pagerank: 0.5, validatedCount: 1, lastAccessedAt: NOW }); + const r = scoreNode(node, 1.0, NOW, zeroCfg); + expect(Number.isFinite(r.composite)).toBe(true); + }); +}); + +describe("decideTierTransition", () => { + const scoreLow = { composite: 0.1, recency: 0, frequency: 0, intrinsic: 0 }; + const scoreHigh = { composite: 0.9, recency: 0.9, frequency: 0.9, intrinsic: 0.9 }; + const scoreMid = { composite: 0.5, recency: 0.5, frequency: 0.5, intrinsic: 0 }; + + it("core + composite 低 + count 低 → working", () => { + const node = makeNode({ tier: "core", validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("working"); + }); + + it("core + composite 高 → 保持 core", () => { + const node = makeNode({ tier: "core", validatedCount: 20 }); + expect(decideTierTransition(node, scoreHigh, 0.9, cfg, NOW)).toBeNull(); + }); + + it("working + composite < pct → peripheral", () => { + const node = makeNode({ tier: "working", validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("peripheral"); + }); + + it("working + 陈旧（age > peripheralAgeDays）+ count 低 → peripheral", () => { + const node = makeNode({ + tier: "working", + validatedCount: 1, + createdAt: NOW - (cfg.peripheralAgeDays + 1) * MS_PER_DAY, + }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBe("peripheral"); + }); + + it("working + 陈旧但 count 充足 → 保持 working", () => { + const node = makeNode({ + tier: "working", + validatedCount: 5, + createdAt: NOW - (cfg.peripheralAgeDays + 1) * MS_PER_DAY, + }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBeNull(); + }); + + it("peripheral + count 充足 + composite 高 → working", () => { + const node = makeNode({ tier: "peripheral", validatedCount: 5 }); + expect(decideTierTransition(node, scoreMid, 0, cfg, NOW)).toBe("working"); + }); + + it("peripheral + count 不足 → 保持 peripheral", () => { + const node = makeNode({ tier: "peripheral", validatedCount: 1 }); + expect(decideTierTransition(node, scoreHigh, 0, cfg, NOW)).toBeNull(); + }); + + it("working + count + composite + importance 都高 → core", () => { + const node = makeNode({ tier: "working", validatedCount: 15 }); + expect(decideTierTransition(node, scoreHigh, 0.9, cfg, NOW)).toBe("core"); + }); + + it("working + count + composite 高但 importance 不足 → 保持 working", () => { + const node = makeNode({ tier: "working", validatedCount: 15 }); + expect(decideTierTransition(node, scoreHigh, 0.5, cfg, NOW)).toBeNull(); + }); + + it("tier undefined 按 working 处理", () => { + const node = makeNode({ tier: undefined as unknown as GmNode["tier"], validatedCount: 1 }); + expect(decideTierTransition(node, scoreLow, 0, cfg, NOW)).toBe("peripheral"); + }); +}); From 2f7fa58c71c617e5c5e51db925b93930480330c6 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Tue, 11 Aug 2026 13:56:57 +0800 Subject: [PATCH 03/11] Fix: Fixed tests --- src/graph/decay.ts | 17 +++++++++++------ src/types.ts | 6 ++++-- test/assemble-context.test.ts | 2 ++ test/integration.assemble.test.ts | 2 ++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/graph/decay.ts b/src/graph/decay.ts index 6de5c07..9859c88 100644 --- a/src/graph/decay.ts +++ b/src/graph/decay.ts @@ -52,6 +52,15 @@ export function computeConfidence(validatedCount: number): number { // ─── 三因子评分（纯函数） ──────────────────────────────────── +/** 选 lastAccessedAt → updatedAt → createdAt 中第一个 > 0 的，用于回退旧节点缺字段。 */ +function pickLastActive(node: Pick): number { + const la = node.lastAccessedAt ?? 0; + const up = node.updatedAt ?? 0; + if (la > 0) return la; + if (up > 0) return up; + return node.createdAt ?? 0; +} + /** β 随 tier 变化：core 缓衰、peripheral 促衰。 */ export function computeBeta(tier: NodeTier, cfg: DecayConfig): number { switch (tier) { @@ -71,9 +80,7 @@ export function scoreRecency( now: number, cfg: DecayConfig, ): number { - const lastActive = node.lastAccessedAt > 0 - ? node.lastAccessedAt - : (node.updatedAt > 0 ? node.updatedAt : node.createdAt); + const lastActive = pickLastActive(node); const daysSince = Math.max(0, (now - lastActive) / MS_PER_DAY); const effectiveHL = cfg.recencyHalfLifeDays * Math.exp(cfg.importanceModulation * importance); @@ -94,9 +101,7 @@ export function scoreFrequency( const base = 1 - Math.exp(-count / 5); if (count <= 1) return base; - const lastActive = node.lastAccessedAt > 0 - ? node.lastAccessedAt - : (node.updatedAt > 0 ? node.updatedAt : node.createdAt); + const lastActive = pickLastActive(node); const accessSpanDays = Math.max(1, (lastActive - node.createdAt) / MS_PER_DAY); const avgGapDays = accessSpanDays / Math.max(count - 1, 1); const recentnessBonus = Math.exp(-avgGapDays / 30); diff --git a/src/types.ts b/src/types.ts index 47f1596..4fd1097 100755 --- a/src/types.ts +++ b/src/types.ts @@ -31,7 +31,8 @@ export interface GmNode { description: string; content: string; status: NodeStatus; - tier: NodeTier; + /** 与 NodeStatus 正交的衰减分层；旧节点/新节点缺省时按 working 处理。 */ + tier?: NodeTier; validatedCount: number; sourceSessions: string[]; communityId: string | null; @@ -43,8 +44,9 @@ export interface GmNode { * （重新提取、gm_record、gm_update、CRUD POST）。是衰减判定的基准。 * 与 updatedAt 的区别：updatedAt 在 deprecate/merge 时也会变，不能代表相关性； * 而 mergeNodes 故意不更新 lastAccessedAt（合并 ≠ 用户重新激活）。 + * 缺省时回退到 updatedAt / createdAt。 */ - lastAccessedAt: number; + lastAccessedAt?: number; /** 最近一次 decay 评分（0~1，越大越鲜活/重要）。仅 applyDecay 写入。 */ decayScore?: number; /** decayScore 的计算时间戳（epoch ms）。 */ diff --git a/test/assemble-context.test.ts b/test/assemble-context.test.ts index e025e90..0c2340c 100644 --- a/test/assemble-context.test.ts +++ b/test/assemble-context.test.ts @@ -11,12 +11,14 @@ function makeNode(overrides: Partial): GmNode { description: "description", content: "content", status: "active", + tier: "working", validatedCount: 1, sourceSessions: ["test"], communityId: null, pagerank: 0, createdAt: now, updatedAt: now, + lastAccessedAt: now, ...overrides, }; } diff --git a/test/integration.assemble.test.ts b/test/integration.assemble.test.ts index 594458e..705985b 100644 --- a/test/integration.assemble.test.ts +++ b/test/integration.assemble.test.ts @@ -24,12 +24,14 @@ function makeNode(over: Partial): GmNode { description: over.description ?? "desc", content: over.content ?? "content body", status: over.status ?? "active", + tier: over.tier ?? "working", validatedCount: over.validatedCount ?? 1, sourceSessions: over.sourceSessions ?? ["s1"], communityId: over.communityId ?? null, pagerank: over.pagerank ?? 0, createdAt: over.createdAt ?? Date.now(), updatedAt: over.updatedAt ?? Date.now(), + lastAccessedAt: over.lastAccessedAt ?? Date.now(), }; } From dcfad1d2b2e8739eb7db877d2aafafccf746dfc8 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Wed, 12 Aug 2026 16:23:34 +0800 Subject: [PATCH 04/11] =?UTF-8?q?Fix:=20=E7=A1=AE=E4=BF=9D=E8=B7=A8?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E9=87=8D=E5=90=AF=E5=8F=AF=E4=BB=A5=E7=BB=A7?= =?UTF-8?q?=E6=89=BF=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.ts | 135 +++++++++++++++++++++++++------------- setup-graph-memory-pro.sh | 1 + src/store/store.ts | 19 ++++++ 3 files changed, 110 insertions(+), 45 deletions(-) diff --git a/index.ts b/index.ts index 4b1aa7b..cd88dde 100755 --- a/index.ts +++ b/index.ts @@ -9,7 +9,7 @@ import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { Type } from "@sinclair/typebox"; import { getDriver, initSchema, getSession } from "./src/store/db.ts"; import { - saveMessage, getUnextracted, + saveMessage, getUnextracted, getMaxTurnIndex, markExtracted, isTurnExtracted, upsertNode, upsertEdge, findByName, updateNode, deleteNode, deprecateNodeAndDisconnect, @@ -352,13 +352,30 @@ const graphMemoryProPlugin = { /** * 每轮结束后直接从原始消息提取知识图谱 * 一轮 = 用户发一条消息 → agent 不管调了多少工具 → 最终回复用户 + * + * compact() 与本函数对同一 session 存在 TOCTOU 竞争：两条路径都先 + * isTurnExtracted/getUnextracted → 调 LLM → 最后 markExtracted，中间窗口 + * 允许另一条路径重复提取同一批消息（重复 LLM 调用 + validatedCount 双递增）。 + * 用 per-session async 互斥锁串行化两条路径的提取体。 */ + const extractLocks = new Map>(); + function withExtractLock(sessionId: string, fn: () => Promise): Promise { + const prev = extractLocks.get(sessionId) ?? Promise.resolve(); + const chain = prev.catch(() => {}); + const result = chain.then(() => fn()); + // 链上只保留"上一轮是否结束"的状态，丢弃返回值并吞掉错误， + // 否则一次失败会永久污染链 → 后续 acquire 直接 reject。 + extractLocks.set(sessionId, result.then(() => undefined, () => undefined)); + return result; + } + async function extractTurnKnowledge(sessionId: string, turnNum: number, rawMessages: any[]): Promise { - try { - if (await isTurnExtracted(driver, sessionId, turnNum)) { - api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); - return; - } + return withExtractLock(sessionId, async () => { + try { + if (await isTurnExtracted(driver, sessionId, turnNum)) { + api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); + return; + } const existing = (await getBySession(driver, sessionId)).map(n => n.name); const result = await extractor.extract({ messages: rawMessages, @@ -401,10 +418,12 @@ const graphMemoryProPlugin = { } catch (err) { api.logger.error(`[graph-memory-pro] turn ${turnNum} extract failed: ${err}`); } + }); } // ── Session 运行时状态 ────────────────────────────────── const msgSeq = new Map(); + const msgSeqLoaders = new Map>(); const recalled = new Map(); const sessionIdsByKey = new Map(); const pendingSubagentRecall = new Map(); @@ -421,6 +440,24 @@ const graphMemoryProPlugin = { } async function ingestMessage(sessionId: string, message: any): Promise { + if (!msgSeq.has(sessionId)) { + // 插件重启后内存 Map 会丢，必须从 DB 恢复 MAX(turnIndex)，否则下一条消息 + // turnIndex=1 → MERGE 命中旧行 → ON CREATE 被跳过 → 新消息静默丢失。 + // in-flight Promise 去重，避免并发 ingest 同时查询 + 互相覆盖 seq。 + let loader = msgSeqLoaders.get(sessionId); + if (!loader) { + loader = getMaxTurnIndex(driver, sessionId).then(max => { + msgSeq.set(sessionId, max); + msgSeqLoaders.delete(sessionId); + return max; + }).catch(err => { + msgSeqLoaders.delete(sessionId); + throw err; + }); + msgSeqLoaders.set(sessionId, loader); + } + await loader; + } const seq = (msgSeq.get(sessionId) ?? 0) + 1; msgSeq.set(sessionId, seq); await saveMessage(driver, sessionId, seq, message.role ?? "unknown", message); @@ -548,51 +585,53 @@ const graphMemoryProPlugin = { async compact({ sessionId, sessionKey, currentTokenCount }: { sessionId: string; sessionKey?: string; sessionFile: string; tokenBudget?: number; force?: boolean; currentTokenCount?: number }) { bindSessionIdentity(sessionId, sessionKey); - const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); + return withExtractLock(sessionId, async () => { + const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); - if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; + if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; - try { - const existing = (await getBySession(driver, sessionId)).map(n => n.name); - const result = await extractor.extract({ messages: msgs, existingNames: existing }); - - const nameToId = new Map(); - for (const nc of result.nodes) { - const { node } = await upsertNode(driver, { - type: nc.type, name: nc.name, - description: nc.description, content: nc.content, - }, sessionId); - nameToId.set(node.name, node.id); - recaller.syncEmbed(node).catch(() => {}); - } + try { + const existing = (await getBySession(driver, sessionId)).map(n => n.name); + const result = await extractor.extract({ messages: msgs, existingNames: existing }); + + const nameToId = new Map(); + for (const nc of result.nodes) { + const { node } = await upsertNode(driver, { + type: nc.type, name: nc.name, + description: nc.description, content: nc.content, + }, sessionId); + nameToId.set(node.name, node.id); + recaller.syncEmbed(node).catch(() => {}); + } - for (const ec of result.edges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - const fromId = nameToId.get(ec.from) ?? fromNode?.id; - const toId = nameToId.get(ec.to) ?? toNode?.id; - if (fromId && toId) { - await upsertEdge(driver, { - fromId, toId, type: ec.type, - instruction: ec.instruction, condition: ec.condition, sessionId, - }); + for (const ec of result.edges) { + const fromNode = await findByName(driver, ec.from); + const toNode = await findByName(driver, ec.to); + const fromId = nameToId.get(ec.from) ?? fromNode?.id; + const toId = nameToId.get(ec.to) ?? toNode?.id; + if (fromId && toId) { + await upsertEdge(driver, { + fromId, toId, type: ec.type, + instruction: ec.instruction, condition: ec.condition, sessionId, + }); + } } - } - const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); - await markExtracted(driver, sessionId, maxTurn); + const maxTurn = Math.max(...msgs.map((m: any) => m.turn_index)); + await markExtracted(driver, sessionId, maxTurn); - return { - ok: true, compacted: true, - result: { - summary: `extracted ${result.nodes.length} nodes, ${result.edges.length} edges`, - tokensBefore: currentTokenCount ?? 0, - }, - }; - } catch (err) { - api.logger.error(`[graph-memory-pro] compact failed: ${err}`); - return { ok: false, compacted: false, reason: String(err) }; - } + return { + ok: true, compacted: true, + result: { + summary: `extracted ${result.nodes.length} nodes, ${result.edges.length} edges`, + tokensBefore: currentTokenCount ?? 0, + }, + }; + } catch (err) { + api.logger.error(`[graph-memory-pro] compact failed: ${err}`); + return { ok: false, compacted: false, reason: String(err) }; + } + }); }, async afterTurn({ sessionId, sessionKey, messages, prePromptMessageCount, isHeartbeat }: { @@ -649,6 +688,8 @@ const graphMemoryProPlugin = { if (childSessionId) { recalled.delete(childSessionId); msgSeq.delete(childSessionId); + msgSeqLoaders.delete(childSessionId); + extractLocks.delete(childSessionId); ingestedSinceTurn.delete(childSessionId); } sessionIdsByKey.delete(childSessionKey); @@ -657,6 +698,8 @@ const graphMemoryProPlugin = { async dispose() { msgSeq.clear(); + msgSeqLoaders.clear(); + extractLocks.clear(); recalled.clear(); sessionIdsByKey.clear(); pendingSubagentRecall.clear(); @@ -733,6 +776,8 @@ const graphMemoryProPlugin = { api.logger.error(`[graph-memory-pro] session_end error: ${err}`); } finally { msgSeq.delete(sid); + msgSeqLoaders.delete(sid); + extractLocks.delete(sid); recalled.delete(sid); ingestedSinceTurn.delete(sid); if (sessionKey && sessionIdsByKey.get(sessionKey) === sid) { diff --git a/setup-graph-memory-pro.sh b/setup-graph-memory-pro.sh index 6307a09..1e9baf8 100644 --- a/setup-graph-memory-pro.sh +++ b/setup-graph-memory-pro.sh @@ -79,6 +79,7 @@ NEO4J_USER="neo4j" NEO4J_URI="" # 留空 → 根据是否自建 Neo4j 自动决定 PLUGIN_REF="" INTERACTIVE=true +PC="" # 嵌入式 provider 选择（1-7）；交互模式由 read 赋值，非交互留空 AUTOSTART_METHODS=() # configure_autostart 写入；卸载与完成提示读取 while [[ $# -gt 0 ]]; do case "$1" in diff --git a/src/store/store.ts b/src/store/store.ts index d85541c..8e9eb7b 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -859,6 +859,9 @@ export async function saveMessage( m.content = $content, m.extracted = false, m.createdAt = $now + ON MATCH SET + m.role = $role, + m.content = $content `, { id: uid("m"), sid, @@ -872,6 +875,22 @@ export async function saveMessage( } } +/** 该会话当前最大 turnIndex（无消息返回 0）。用于插件重启后恢复内存 msgSeq； + * 否则 turnIndex 从 1 重计 → MERGE 命中旧行 → ON CREATE 被跳过 → 新消息被静默丢弃。 */ +export async function getMaxTurnIndex(driver: Driver, sid: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + `MATCH (m:GmMessage {sessionId: $sid}) + RETURN coalesce(max(m.turnIndex), 0) AS maxTurn`, + { sid }, + ); + return toInt(result.records[0].get("maxTurn")); + } finally { + await session.close(); + } +} + export async function getUnextracted(driver: Driver, sid: string, limit: number): Promise { const session = getSession(driver); try { From 8ccfb793d383fb5eb5331c1d04a4419af226eba8 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Sat, 15 Aug 2026 19:47:29 +0000 Subject: [PATCH 05/11] =?UTF-8?q?=E7=A7=BB=E6=A4=8D=E4=B8=BB=E5=B9=B2?= =?UTF-8?q?=E6=94=B9=E5=8A=A81fdec04=EF=BC=8C=E9=81=BF=E5=85=8D=E5=85=A8?= =?UTF-8?q?=E9=87=8F=E6=9B=B4=E6=96=B0communities?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移植了https://github.com/adoresever/graph-memory/commit/1fdec04a8d49ffee1a4585c20c7162ee3e204370 src/store/store.ts - CommunitySummary 接口新增 memberSignature: string | null - upsertCommunitySummary() 增加 memberSignature 参数，MERGE 时写入（沿用 embedding 的 CASE 保留模式：传 null 不清空旧值） - getCommunitySummary() / getAllCommunitySummaries() 返回 memberSignature - 新增 getCommunitySummaryBySignature()：按签名查社区（ORDER BY updatedAt DESC LIMIT 1），带回 embedding 供复用 src/graph/community.ts - 新增导出 buildCommunityMemberSignature()：成员 ID 排序后 sha1（与上游逐字等价） - summarizeCommunities() 循环内两层短路： 1. 签名未变且摘要非空 → 跳过，不调 LLM 2. 其他社区存在相同签名 + 非空摘要 → 复用其 summary + embedding，不调 LLM 3. 否则走原有 LLM 生成路径，upsert 时写入签名 --- src/graph/community.ts | 29 ++++++++++++++++++++++++- src/store/db.ts | 1 + src/store/store.ts | 35 ++++++++++++++++++++++++++++++- test/community-signature.test.ts | 36 ++++++++++++++++++++++++++++++++ test/integration.graph.test.ts | 34 ++++++++++++++++++++++++++++-- 5 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 test/community-signature.test.ts diff --git a/src/graph/community.ts b/src/graph/community.ts index d5f5dd5..71b021a 100755 --- a/src/graph/community.ts +++ b/src/graph/community.ts @@ -6,12 +6,15 @@ * 保留 summarizeCommunities()（需要 LLM） */ +import { createHash } from "node:crypto"; import type { Driver } from "neo4j-driver"; import { getSession } from "../store/db.ts"; import { clearCommunities, updateCommunities, upsertCommunitySummary, + getCommunitySummary, + getCommunitySummaryBySignature, pruneCommunitySummaries, } from "../store/store.ts"; import { getExistingActiveRelTypes, projectActiveGraph } from "./projection.ts"; @@ -142,6 +145,10 @@ const COMMUNITY_SUMMARY_SYS = `你是知识图谱社区摘要引擎。根据社 - 不要使用"社区"这个词 - 不要加引号或标点以外的格式`; +export function buildCommunityMemberSignature(memberIds: string[]): string { + return createHash("sha1").update([...memberIds].sort().join(",")).digest("hex"); +} + export async function summarizeCommunities( driver: Driver, communities: Map, @@ -154,6 +161,26 @@ export async function summarizeCommunities( for (const [communityId, memberIds] of communities) { if (memberIds.length === 0) continue; + const memberSignature = buildCommunityMemberSignature(memberIds); + + const current = await getCommunitySummary(driver, communityId); + if (current?.memberSignature === memberSignature && current.summary.trim()) { + continue; + } + + const reusable = await getCommunitySummaryBySignature(driver, memberSignature); + if (reusable?.summary.trim()) { + await upsertCommunitySummary( + driver, + communityId, + reusable.summary, + memberIds.length, + reusable.embedding, + memberSignature, + ); + continue; + } + const session = getSession(driver); let members: any[]; try { @@ -204,7 +231,7 @@ export async function summarizeCommunities( } catch {} } - await upsertCommunitySummary(driver, communityId, cleaned, memberIds.length, embedding); + await upsertCommunitySummary(driver, communityId, cleaned, memberIds.length, embedding, memberSignature); generated++; } catch (err) { console.log(` [WARN] community summary failed for ${communityId}: ${err}`); diff --git a/src/store/db.ts b/src/store/db.ts index 620441c..d5d7315 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -75,6 +75,7 @@ export async function initSchema(driver: Driver, embedding?: EmbeddingConfig): P // Community await session.run("CREATE CONSTRAINT community_id IF NOT EXISTS FOR (c:Community) REQUIRE c.id IS UNIQUE"); + await session.run("CREATE INDEX community_member_signature IF NOT EXISTS FOR (c:Community) ON (c.memberSignature)"); // Message (temporary extraction buffer) await session.run("CREATE CONSTRAINT gm_msg_id IF NOT EXISTS FOR (m:GmMessage) REQUIRE m.id IS UNIQUE"); diff --git a/src/store/store.ts b/src/store/store.ts index 8e9eb7b..b5d6f6a 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -1029,12 +1029,15 @@ export interface CommunitySummary { id: string; summary: string; nodeCount: number; + /** 成员 ID 排序后的 sha1 — 用于识别"成员构成未变"的社区（复用摘要） */ + memberSignature: string | null; createdAt: number; updatedAt: number; } export async function upsertCommunitySummary( - driver: Driver, id: string, summary: string, nodeCount: number, embedding?: number[], + driver: Driver, id: string, summary: string, nodeCount: number, + embedding?: number[], memberSignature?: string, ): Promise { const session = getSession(driver); try { @@ -1044,18 +1047,21 @@ export async function upsertCommunitySummary( c.summary = $summary, c.nodeCount = $nodeCount, c.embedding = $embedding, + c.memberSignature = $memberSignature, c.createdAt = $now, c.updatedAt = $now ON MATCH SET c.summary = $summary, c.nodeCount = $nodeCount, c.embedding = CASE WHEN $embedding IS NOT NULL THEN $embedding ELSE c.embedding END, + c.memberSignature = CASE WHEN $memberSignature IS NOT NULL THEN $memberSignature ELSE c.memberSignature END, c.updatedAt = $now `, { id, summary, nodeCount, embedding: embedding ?? null, + memberSignature: memberSignature ?? null, now: Date.now(), }); } finally { @@ -1076,6 +1082,32 @@ export async function getCommunitySummary(driver: Driver, id: string): Promise { + const session = getSession(driver); + try { + const result = await session.run( + "MATCH (c:Community {memberSignature: $memberSignature}) RETURN c ORDER BY c.updatedAt DESC LIMIT 1", + { memberSignature }, + ); + if (result.records.length === 0) return null; + const c = result.records[0].get("c").properties; + return { + id: c.id, + summary: c.summary, + nodeCount: toInt(c.nodeCount), + memberSignature: c.memberSignature ?? null, + embedding: Array.isArray(c.embedding) ? (c.embedding as number[]) : undefined, createdAt: toInt(c.createdAt), updatedAt: toInt(c.updatedAt), }; @@ -1096,6 +1128,7 @@ export async function getAllCommunitySummaries(driver: Driver): Promise { + it("成员顺序不影响签名（排序后哈希）", () => { + expect(buildCommunityMemberSignature(["a", "b", "c"])) + .toBe(buildCommunityMemberSignature(["c", "a", "b"])); + }); + + it("相同成员恒生成相同签名", () => { + expect(buildCommunityMemberSignature(["x", "y"])) + .toBe(buildCommunityMemberSignature(["x", "y"])); + }); + + it("成员构成不同则签名不同", () => { + expect(buildCommunityMemberSignature(["a", "b"])) + .not.toBe(buildCommunityMemberSignature(["a", "c"])); + expect(buildCommunityMemberSignature(["a", "b"])) + .not.toBe(buildCommunityMemberSignature(["a", "b", "c"])); + }); + + it("输出为 40 位小写 hex（sha1）", () => { + expect(buildCommunityMemberSignature(["a"])).toMatch(/^[0-9a-f]{40}$/); + }); + + it("不修改入参数组", () => { + const input = ["b", "a"]; + buildCommunityMemberSignature(input); + expect(input).toEqual(["b", "a"]); + }); +}); diff --git a/test/integration.graph.test.ts b/test/integration.graph.test.ts index 9dc2e90..db43d45 100644 --- a/test/integration.graph.test.ts +++ b/test/integration.graph.test.ts @@ -12,12 +12,14 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import type { Driver } from "neo4j-driver"; import { getDriver, initSchema, closeDriver, getSession } from "../src/store/db.ts"; import { - upsertNode, upsertEdge, saveVector, findById, deprecate, + upsertNode, upsertEdge, saveVector, findById, deprecate, getCommunitySummary, } from "../src/store/store.ts"; import { personalizedPageRank, computeGlobalPageRank, } from "../src/graph/pagerank.ts"; -import { detectCommunities, getCommunityPeers } from "../src/graph/community.ts"; +import { + detectCommunities, getCommunityPeers, summarizeCommunities, buildCommunityMemberSignature, +} from "../src/graph/community.ts"; import { detectDuplicates, dedup } from "../src/graph/dedup.ts"; import { runMaintenance } from "../src/graph/maintenance.ts"; import { DEFAULT_CONFIG, type GmConfig } from "../src/types.ts"; @@ -187,6 +189,34 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { } }); + it("summarizeCommunities：社区成员未变时复用摘要，不重调 LLM", async () => { + const memberIds = [nodeIds["gmpsrc-deploy"], nodeIds["gmpsrc-compose"]]; + const communities = new Map([["c-reuse-test", memberIds]]); + let llmCalls = 0; + const llm = async () => { + llmCalls += 1; + return "容器部署与编排技能"; + }; + + const first = await summarizeCommunities(driver, communities, llm); + const second = await summarizeCommunities(driver, communities, llm); + + expect(first).toBe(1); + expect(second).toBe(0); + expect(llmCalls).toBe(1); + + const summary = await getCommunitySummary(driver, "c-reuse-test"); + expect(summary?.summary).toBe("容器部署与编排技能"); + expect(summary?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); + + const cleanup = getSession(driver); + try { + await cleanup.run("MATCH (c:Community {id: 'c-reuse-test'}) DELETE c"); + } finally { + await cleanup.close(); + } + }); + it("detectDuplicates：gmpsrc-* 无 embedding，函数不抛错", async () => { let passed = false; await expectDimSafe(async () => { From 492923cbd81da541f7e2e7fcf5b50102a14c263f Mon Sep 17 00:00:00 2001 From: TriDefender Date: Sat, 15 Aug 2026 19:58:54 +0000 Subject: [PATCH 06/11] Update integration.graph.test.ts --- test/integration.graph.test.ts | 36 ++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/test/integration.graph.test.ts b/test/integration.graph.test.ts index db43d45..8b52d10 100644 --- a/test/integration.graph.test.ts +++ b/test/integration.graph.test.ts @@ -191,15 +191,27 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { it("summarizeCommunities：社区成员未变时复用摘要，不重调 LLM", async () => { const memberIds = [nodeIds["gmpsrc-deploy"], nodeIds["gmpsrc-compose"]]; - const communities = new Map([["c-reuse-test", memberIds]]); + + // 生产不变量：detectCommunities 会先给成员节点写入 communityId， + // pruneCommunitySummaries 只保留仍被 active 成员引用的社区 — 不先 SET 会被 prune 删掉 + const prepare = getSession(driver); + try { + await prepare.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = $cid", + { ids: memberIds, cid: "c-reuse-test" }, + ); + } finally { + await prepare.close(); + } + let llmCalls = 0; const llm = async () => { llmCalls += 1; return "容器部署与编排技能"; }; - const first = await summarizeCommunities(driver, communities, llm); - const second = await summarizeCommunities(driver, communities, llm); + const first = await summarizeCommunities(driver, new Map([["c-reuse-test", memberIds]]), llm); + const second = await summarizeCommunities(driver, new Map([["c-reuse-test", memberIds]]), llm); expect(first).toBe(1); expect(second).toBe(0); @@ -209,9 +221,25 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { expect(summary?.summary).toBe("容器部署与编排技能"); expect(summary?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); + // detectCommunities 每轮按成员数重编号（c-1..c-N），ID 变但成员相同 → 按签名跨社区复用 + const third = await summarizeCommunities( + driver, new Map([["c-reuse-renumbered", memberIds]]), llm, + ); + expect(third).toBe(0); + expect(llmCalls).toBe(1); + const renumbered = await getCommunitySummary(driver, "c-reuse-renumbered"); + expect(renumbered?.summary).toBe("容器部署与编排技能"); + expect(renumbered?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); + const cleanup = getSession(driver); try { - await cleanup.run("MATCH (c:Community {id: 'c-reuse-test'}) DELETE c"); + await cleanup.run( + "MATCH (c:Community) WHERE c.id IN ['c-reuse-test', 'c-reuse-renumbered'] DELETE c", + ); + await cleanup.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = null", + { ids: memberIds }, + ); } finally { await cleanup.close(); } From 499cb767485aeaf2fe71d8732024ef4cd650cf4b Mon Sep 17 00:00:00 2001 From: TriDefender Date: Sat, 15 Aug 2026 21:24:07 +0000 Subject: [PATCH 07/11] Added finegrain memory control on cron This commit aims to tackle more finegrained memory control on cron sessions: Does a repeating cron session generates 'false popularity' on specific memory nodes? Now it enables you to: - disable memory extraction for cron runs - disable entirely graph functions, so it will not inject context information (it may change workflows, use this with caution since models may lack crucial information. you're advised to test the workflow before hand) - skip session end actions on cron --- README.md | 24 +++++ README_CN.md | 22 ++++ index.ts | 51 ++++++++- openclaw.plugin.json | 9 ++ src/types.ts | 27 +++++ test/session-identity.test.ts | 197 +++++++++++++++++++++++++++++++++- 6 files changed, 324 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 79bdc64..5ecb143 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,30 @@ Common overrides — for fuller control see `docs/decay.md` §4: } ``` +### Cron sessions + +Sessions created by OpenClaw scheduled tasks can be configured independently of normal sessions. The host places the cron marker on the **sessionKey** (`sessionId` is a random UUID); real shapes are `cron:`, `agent::cron:`, or `agent::cron::run:`: + +```json +"cron": { + "enabled": true, + "extract": true, + "finalizeAndMaintain": true +} +``` + +| Option | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | Enable graph functionality inside cron sessions (recall injection + message buffering). When `false`, cron sessions skip automatic recall and message persistence; the `gm_*` tools remain available for explicit calls (manual escape hatch). | +| `extract` | `false` | Trigger knowledge extraction (LLM triples) in cron sessions via `afterTurn` / `compact`. When `false`, messages are still buffered and can be backfilled later with `openclaw graph-memory extract`. | +| `finalizeAndMaintain` | `false` | Run finalize (EVENT→SKILL promotion) and graph maintenance (decay / PageRank / communities) when a cron session ends. Disable when frequent cron runs make end-of-session global maintenance too costly. | + +All three options default to **`false`**: cron sessions skip the graph entirely (no recall, no buffering, no extraction, no maintenance) unless explicitly enabled. `enabled: false` is the master switch — even with `extract` / `finalizeAndMaintain` set to `true`, nothing runs. Non-cron sessions are never affected by these options. + +All three sub-options are optional; omitted fields keep the default `false` (e.g. with `"cron": { "enabled": true }` only recall and message buffering are enabled — extraction and end-of-session maintenance stay off). + +Caveat: when a cron job sets an explicit custom `sessionKey`, the host does not append the `cron` segment — such sessions cannot be detected and are treated as normal sessions. + ### OAuth login (experimental) ```bash diff --git a/README_CN.md b/README_CN.md index eabc330..2851ca7 100644 --- a/README_CN.md +++ b/README_CN.md @@ -81,6 +81,28 @@ bash setup-graph-memory-pro.sh --uninstall `embedding` 可选。设置时，`dimensions` 必须与 Neo4j 向量索引维度一致。新数据库会在插件启动时按配置创建索引；更换维度后需要重建向量索引或 Neo4j 数据库。 +### cron 会话行为控制 + +OpenClaw 定时任务创建的会话可以独立配置图谱行为。host 把 cron 标记放在 **sessionKey** 上（`sessionId` 是随机 UUID），实际形状为 `cron:`、`agent::cron:` 或 `agent::cron::run:`： + +```json +"cron": { + "enabled": true, + "extract": true, + "finalizeAndMaintain": true +} +``` + +| 选项 | 默认 | 说明 | +| --- | --- | --- | +| `enabled` | `true` | 是否在 cron 会话内启用图谱功能（召回注入 + 消息入库）。关闭后 cron 会话不自动召回、不自动入库；`gm_*` 工具仍可手动调用（作为显式逃生通道）。 | +| `extract` | `true` | 是否在 cron 会话内触发知识提取（afterTurn / compact 的 LLM 三元组提取）。关闭后消息仍入库缓冲，之后可用 `openclaw graph-memory extract` 手动回填。 | +| `finalizeAndMaintain` | `true` | cron 会话结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay / PageRank / 社区检测）。定时任务频繁时可关闭，避免每次会话结束都跑全局维护。 | + +三个选项**默认全部开启**：cron 会话默认使用图谱，需按需显式关闭。`enabled=true` 是总开关：即使 `extract`/`finalizeAndMaintain` 设为 `false` 也不生效。非 cron 会话不受这些选项影响。 + +注意：若 cron 任务显式设置了自定义 `sessionKey`，host 不再附加 `cron` 段，此类会话无法被识别，将按普通会话处理。 + ### OAuth 登录（实验性） ```bash diff --git a/index.ts b/index.ts index cd88dde..9973e7b 100755 --- a/index.ts +++ b/index.ts @@ -24,7 +24,7 @@ import { Extractor } from "./src/extractor/extract.ts"; import { assembleContext } from "./src/format/assemble.ts"; import { sanitizeToolUseResultPairing } from "./src/format/transcript-repair.ts"; import { runMaintenance } from "./src/graph/maintenance.ts"; -import { DEFAULT_CONFIG, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; +import { DEFAULT_CONFIG, DEFAULT_CRON_CONFIG, isCronSessionKey, type GmConfig, type RecallResult, type EdgeType } from "./src/types.ts"; import { registerCrudRoutes } from "./src/routes/crud.ts"; import { createGraphMemoryCli } from "./src/cli.ts"; @@ -279,6 +279,8 @@ const graphMemoryProPlugin = { const cfg: GmConfig = { ...DEFAULT_CONFIG, ...raw }; if (raw.neo4j) cfg.neo4j = { ...DEFAULT_CONFIG.neo4j, ...raw.neo4j }; if (raw.decay) cfg.decay = { ...DEFAULT_CONFIG.decay, ...raw.decay }; + if (raw.cron) cfg.cron = { ...DEFAULT_CONFIG.cron, ...raw.cron }; + const cronCfg = cfg.cron ?? DEFAULT_CRON_CONFIG; const providerModel = readDefaultModel(api.config); @@ -467,6 +469,9 @@ const graphMemoryProPlugin = { api.on("before_agent_start", async (event: any, ctx: any) => { try { + // cron session 关闭图谱功能时不召回（cron 标记在 sessionKey 上，sessionId 是随机 UUID） + if (isCronSessionKey(typeof ctx?.sessionKey === "string" ? ctx.sessionKey : null) && !cronCfg.enabled) return; + const rawPrompt = typeof event?.prompt === "string" ? event.prompt : ""; const prompt = cleanPrompt(rawPrompt); if (!prompt) return; @@ -506,6 +511,10 @@ const graphMemoryProPlugin = { async ingest({ sessionId, sessionKey, message, isHeartbeat }: { sessionId: string; sessionKey?: string; message: any; isHeartbeat?: boolean }) { if (isHeartbeat) return { ingested: false }; bindSessionIdentity(sessionId, sessionKey); + // cron session 关闭图谱功能：消息不入库 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + return { ingested: false }; + } await ingestMessage(sessionId, message); ingestedSinceTurn.set(sessionId, (ingestedSinceTurn.get(sessionId) ?? 0) + 1); return { ingested: true }; @@ -517,6 +526,21 @@ const graphMemoryProPlugin = { bindSessionIdentity(sessionId, sessionKey); const budget = tokenBudget ?? 128_000; + // cron session 关闭图谱功能：仅做消息裁剪与配对修复，不注入图谱上下文 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + const prepared = prepareAssemblyMessages(messages); + if (prepared.dropped > 0) { + api.logger.info( + `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + + `dropped ${prepared.dropped} older msgs, graph skipped (cron session)`, + ); + } + return { + messages: prepared.messages, + estimatedTokens: prepared.tokens, + }; + } + const activeNodes = await getBySession(driver, sessionId); const activeEdges: any[] = []; for (const n of activeNodes) { @@ -585,6 +609,13 @@ const graphMemoryProPlugin = { async compact({ sessionId, sessionKey, currentTokenCount }: { sessionId: string; sessionKey?: string; sessionFile: string; tokenBudget?: number; force?: boolean; currentTokenCount?: number }) { bindSessionIdentity(sessionId, sessionKey); + // cron session 关闭图谱功能或知识提取：不触发 LLM 提取 + if (isCronSessionKey(sessionKey) && !(cronCfg.enabled && cronCfg.extract)) { + return { + ok: true, compacted: false, + reason: cronCfg.enabled ? "cron session extraction disabled" : "cron session graph disabled", + }; + } return withExtractLock(sessionId, async () => { const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); @@ -648,6 +679,12 @@ const graphMemoryProPlugin = { return; } + // cron session 关闭图谱功能：跳过入库回填与知识提取 + if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { + ingestedSinceTurn.delete(sessionId); + return; + } + // Official OpenClaw delivers ingest() and afterTurn() as separate // lifecycle phases. Older downstream builds incorrectly call only // afterTurn(). Persist just the missing suffix so neither host loses @@ -668,6 +705,12 @@ const graphMemoryProPlugin = { api.logger.info(`[graph-memory-pro] afterTurn sid=${sessionId.slice(0, 8)} turn=${turnNum} rawMsgs=${newMessages.length}`); + // cron session 关闭知识提取：消息仅入库缓冲，可稍后用 `graph-memory extract` 手动回填 + if (isCronSessionKey(sessionKey) && !cronCfg.extract) { + api.logger.info("[graph-memory-pro] cron session: extraction skipped (cron.extract=false)"); + return; + } + // 直接用原始消息提取知识图谱（异步，不阻塞） extractTurnKnowledge(sessionId, turnNum, newMessages).catch(err => { api.logger.error(`[graph-memory-pro] extract failed: ${err}`); @@ -722,6 +765,12 @@ const graphMemoryProPlugin = { : typeof ctx?.sessionKey === "string" ? ctx.sessionKey : undefined; try { + // cron session：图谱功能关闭或明确禁用时，跳过 finalize 与图维护（finally 清理仍执行） + if (isCronSessionKey(sessionKey) && !(cronCfg.enabled && cronCfg.finalizeAndMaintain)) { + api.logger.info(`[graph-memory-pro] cron session ${sid.slice(0, 12)}…: finalize + maintenance skipped (cron config)`); + return; + } + const nodes = await getBySession(driver, sid); if (nodes.length) { // 获取图谱摘要 diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 1a6b474..8022351 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -51,6 +51,15 @@ "workingCompositeThreshold": { "type": "number", "default": 0.4, "description": "peripheral→working 所需的最低 composite 分数。" } } }, + "cron": { + "type": "object", + "description": "cron 定时会话的图谱行为开关（host 将 cron 标记放在 sessionKey 上，形如 agent::cron:）。默认全部启用，与普通会话一致。注意：cron 任务若显式设置自定义 sessionKey，则无法被识别为 cron 会话。", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否在 cron session 内启用图谱功能（召回注入 + 消息入库）。关闭后 cron 会话不自动召回、不自动入库；gm_* 工具仍可手动调用。" }, + "extract": { "type": "boolean", "default": true, "description": "是否在 cron session 内触发知识提取（afterTurn / compact 的 LLM 三元组提取）。关闭后消息仍入库缓冲，可用 openclaw graph-memory extract 手动回填。" }, + "finalizeAndMaintain": { "type": "boolean", "default": true, "description": "cron session 结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay/PageRank/社区检测）。频繁的 cron 任务可关闭以避免每次结束都跑全局维护。" } + } + }, "llm": { "type": "object", "properties": { diff --git a/src/types.ts b/src/types.ts index 4fd1097..ce30fca 100755 --- a/src/types.ts +++ b/src/types.ts @@ -182,6 +182,31 @@ export interface DecayConfig { workingCompositeThreshold: number; } +// ─── cron 会话（定时任务）的图谱行为配置 ───────────────────── + +/** + * 判断是否为 cron 定时会话。host 把 cron 标记放在 sessionKey 上（sessionId 是随机 UUID）， + * 实际形状：cron: / agent::cron: / agent::cron::run:。 + * 按段匹配（split(":") 后包含 "cron"），避免误匹配 "cron-daily" 这类自定义段。 + * 注意：cron 任务若显式设置了自定义 sessionKey，host 不再附加 cron 段，此类会话无法识别（见 README）。 + * cron session 的图谱行为（召回/消息入库、知识提取、结束维护）可由 `cron` 配置独立开关；非 cron session 不受影响。 + */ +export function isCronSessionKey(sessionKey: string | undefined | null): boolean { + return typeof sessionKey === "string" && sessionKey.split(":").includes("cron"); +} + +export interface CronConfig { + enabled: boolean; + extract: boolean; + finalizeAndMaintain: boolean; +} + +export const DEFAULT_CRON_CONFIG: CronConfig = { + enabled: true, + extract: true, + finalizeAndMaintain: true, +}; + // ─── 插件配置 ───────────────────────────────────────────────── export interface GmConfig { @@ -210,6 +235,7 @@ export interface GmConfig { pagerankIterations: number; /** 遗忘曲线衰减配置；未提供时使用 DEFAULT_CONFIG.decay。 */ decay?: DecayConfig; + cron?: CronConfig; } export const DEFAULT_CONFIG: GmConfig = { @@ -243,4 +269,5 @@ export const DEFAULT_CONFIG: GmConfig = { workingAccessThreshold: 3, workingCompositeThreshold: 0.4, }, + cron: DEFAULT_CRON_CONFIG, }; diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index c2ec70c..1078531 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -1,7 +1,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { isCronSessionKey } from "../src/types.ts"; const mocks = vi.hoisted(() => ({ getBySession: vi.fn(async () => []), + saveMessage: vi.fn(async () => {}), + getMaxTurnIndex: vi.fn(async () => 0), + getUnextracted: vi.fn(async () => []), + isTurnExtracted: vi.fn(async () => false), recall: vi.fn(async () => ({ nodes: [{ id: "recalled-node" }], edges: [], @@ -25,10 +30,11 @@ vi.mock("../src/store/db.ts", () => ({ })); vi.mock("../src/store/store.ts", () => ({ - saveMessage: async () => {}, - getUnextracted: async () => [], + saveMessage: mocks.saveMessage, + getUnextracted: mocks.getUnextracted, + getMaxTurnIndex: mocks.getMaxTurnIndex, markExtracted: async () => {}, - isTurnExtracted: async () => false, + isTurnExtracted: mocks.isTurnExtracted, upsertNode: async () => ({ node: {}, isNew: false }), upsertEdge: async () => {}, findByName: async () => null, @@ -81,11 +87,28 @@ import graphMemoryProPlugin from "../index.ts"; type HookHandler = (event: Record, context: Record) => Promise; type EngineHarness = { readonly bootstrap: (params: { readonly sessionId: string; readonly sessionKey?: string }) => Promise; + readonly ingest: (params: { + readonly sessionId: string; + readonly sessionKey?: string; + readonly message: unknown; + readonly isHeartbeat?: boolean; + }) => Promise<{ readonly ingested: boolean }>; readonly assemble: (params: { readonly sessionId: string; readonly sessionKey?: string; readonly messages: readonly unknown[]; }) => Promise; + readonly compact: (params: { readonly sessionId: string; readonly sessionKey?: string }) => Promise<{ + readonly ok: boolean; + readonly compacted: boolean; + readonly reason?: string; + }>; + readonly afterTurn: (params: { + readonly sessionId: string; + readonly sessionKey?: string; + readonly messages: readonly unknown[]; + readonly prePromptMessageCount: number; + }) => Promise; readonly prepareSubagentSpawn: (params: { readonly parentSessionKey: string; readonly childSessionKey: string; @@ -93,7 +116,7 @@ type EngineHarness = { }) => Promise<{ readonly rollback: () => void }>; }; -function registerPlugin(): { readonly hooks: Map; readonly engine: EngineHarness } { +function registerPlugin(pluginConfig: Record = {}): { readonly hooks: Map; readonly engine: EngineHarness } { const hooks = new Map(); let engine: EngineHarness | undefined; graphMemoryProPlugin.register({ @@ -104,7 +127,7 @@ function registerPlugin(): { readonly hooks: Map; readonly error: () => {}, }, config: {}, - pluginConfig: {}, + pluginConfig, resolvePath: (path: string) => path, on: (event: string, handler: HookHandler) => { hooks.set(event, handler); }, registerContextEngine: (_id: string, factory: () => EngineHarness) => { engine = factory(); }, @@ -165,3 +188,167 @@ describe("session identity", () => { ); }); }); + +describe("cron session gating (cron 配置)", () => { + beforeEach(() => { + mocks.getBySession.mockClear(); + mocks.saveMessage.mockClear(); + mocks.getMaxTurnIndex.mockClear(); + mocks.getUnextracted.mockClear(); + mocks.isTurnExtracted.mockClear(); + mocks.recall.mockClear(); + mocks.assembleContext.mockClear(); + mocks.runMaintenance.mockClear(); + }); + + // host 契约：sessionId 是随机 transcript UUID，cron 标记在 sessionKey 上 + const CRON_KEY = "agent:agent-1:cron:daily-report"; + const CRON_SID = "0f1e2d3c-4b5a-6978-8976-543210fedcba"; + + it("isCronSessionKey 按 sessionKey 段匹配 cron 标记", () => { + expect(isCronSessionKey("cron:job-1")).toBe(true); + expect(isCronSessionKey("agent:agent-1:cron:daily")).toBe(true); + expect(isCronSessionKey("agent:agent-1:cron:daily:run:r1")).toBe(true); + expect(isCronSessionKey("agent:main")).toBe(false); + expect(isCronSessionKey("agent:cron-daily:main")).toBe(false); + expect(isCronSessionKey("scheduled-cron:x")).toBe(false); + expect(isCronSessionKey("")).toBe(false); + expect(isCronSessionKey(undefined)).toBe(false); + expect(isCronSessionKey(null)).toBe(false); + }); + + it("默认配置（全 false）下 cron session_end 跳过 finalize 与图维护", async () => { + const handler = registerPlugin().hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).not.toHaveBeenCalled(); + expect(mocks.runMaintenance).not.toHaveBeenCalled(); + }); + + it("默认配置下 cron session 不入库（默认关闭）", async () => { + const { engine } = registerPlugin(); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: false }); + expect(mocks.saveMessage).not.toHaveBeenCalled(); + }); + + it("finalizeAndMaintain=false 时 cron session 跳过 finalize 与图维护", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: false } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).not.toHaveBeenCalled(); + expect(mocks.runMaintenance).not.toHaveBeenCalled(); + }); + + it("finalizeAndMaintain=true 时 cron session 执行 finalize 与图维护", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: true } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); + + expect(mocks.getBySession).toHaveBeenCalledWith({}, CRON_SID); + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); + }); + + it("finalizeAndMaintain=true 不影响普通会话的既有行为", async () => { + const handler = registerPlugin({ cron: { enabled: true, finalizeAndMaintain: false } }).hooks.get("session_end"); + if (!handler) throw new Error("session_end hook was not registered"); + + await handler({ sessionId: "normal-session", sessionKey: "agent:main" }, {}); + + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); + }); + + it("enabled=false 时 cron session 不召回、不入库、不注入图谱上下文", async () => { + const { hooks, engine } = registerPlugin({ cron: { enabled: false } }); + + const beforeAgentStart = hooks.get("before_agent_start"); + if (!beforeAgentStart) throw new Error("before_agent_start hook was not registered"); + await beforeAgentStart({ prompt: "daily digest" }, { sessionKey: CRON_KEY }); + expect(mocks.recall).not.toHaveBeenCalled(); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: false }); + expect(mocks.saveMessage).not.toHaveBeenCalled(); + + await engine.assemble({ sessionId: CRON_SID, sessionKey: CRON_KEY, messages: [] }); + expect(mocks.assembleContext).not.toHaveBeenCalled(); + }); + + it("enabled=false 总开关：cron afterTurn 跳过入库回填，compact 跳过提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: false } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + expect(mocks.saveMessage).not.toHaveBeenCalled(); + expect(mocks.isTurnExtracted).not.toHaveBeenCalled(); + + const res = await engine.compact({ sessionId: CRON_SID, sessionKey: CRON_KEY }); + expect(res).toEqual({ ok: true, compacted: false, reason: "cron session graph disabled" }); + expect(mocks.getUnextracted).not.toHaveBeenCalled(); + }); + + it("enabled=true 时 cron session 正常入库", async () => { + const { engine } = registerPlugin({ cron: { enabled: true } }); + + await engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); + + it("extract=false 时 cron session 消息仍入库缓冲但不触发提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: false } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + expect(mocks.isTurnExtracted).not.toHaveBeenCalled(); + }); + + it("extract=true 时 cron session 触发提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: true } }); + + await engine.afterTurn({ + sessionId: CRON_SID, + sessionKey: CRON_KEY, + messages: [{ role: "user", content: "hi" }], + prePromptMessageCount: 0, + }); + // afterTurn 内的 extractTurnKnowledge 是 fire-and-forget，flush 微任务后再断言 + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(mocks.isTurnExtracted).toHaveBeenCalledTimes(1); + }); + + it("extract=false 时 cron session compact 直接跳过提取", async () => { + const { engine } = registerPlugin({ cron: { enabled: true, extract: false } }); + + const res = await engine.compact({ sessionId: CRON_SID, sessionKey: CRON_KEY }); + + expect(res).toEqual({ ok: true, compacted: false, reason: "cron session extraction disabled" }); + expect(mocks.getUnextracted).not.toHaveBeenCalled(); + }); + + it("cron 任务设置自定义 sessionKey（无 cron 段）时按普通会话处理", async () => { + const { engine } = registerPlugin({ cron: { enabled: false } }); + + await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: "agent:my-custom-key", message: { role: "user", content: "hi" } })) + .resolves.toEqual({ ingested: true }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); +}); From 02c4bb21168e2dc8b55de273de3ae23ad3e73199 Mon Sep 17 00:00:00 2001 From: TriDefender Date: Sat, 15 Aug 2026 21:39:14 +0000 Subject: [PATCH 08/11] Fix: fixed incoherent tests --- README.md | 10 +++++----- README_CN.md | 2 +- test/session-identity.test.ts | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 5ecb143..cd95ab5 100644 --- a/README.md +++ b/README.md @@ -145,13 +145,13 @@ Sessions created by OpenClaw scheduled tasks can be configured independently of | Option | Default | Description | | --- | --- | --- | -| `enabled` | `false` | Enable graph functionality inside cron sessions (recall injection + message buffering). When `false`, cron sessions skip automatic recall and message persistence; the `gm_*` tools remain available for explicit calls (manual escape hatch). | -| `extract` | `false` | Trigger knowledge extraction (LLM triples) in cron sessions via `afterTurn` / `compact`. When `false`, messages are still buffered and can be backfilled later with `openclaw graph-memory extract`. | -| `finalizeAndMaintain` | `false` | Run finalize (EVENT→SKILL promotion) and graph maintenance (decay / PageRank / communities) when a cron session ends. Disable when frequent cron runs make end-of-session global maintenance too costly. | +| `enabled` | `true` | Enable graph functionality inside cron sessions (recall injection + message buffering). When `false`, cron sessions skip automatic recall and message persistence; the `gm_*` tools remain available for explicit calls (manual escape hatch). | +| `extract` | `true` | Trigger knowledge extraction (LLM triples) in cron sessions via `afterTurn` / `compact`. When `false`, messages are still buffered and can be backfilled later with `openclaw graph-memory extract`. | +| `finalizeAndMaintain` | `true` | Run finalize (EVENT→SKILL promotion) and graph maintenance (decay / PageRank / communities) when a cron session ends. Disable when frequent cron runs make end-of-session global maintenance too costly. | -All three options default to **`false`**: cron sessions skip the graph entirely (no recall, no buffering, no extraction, no maintenance) unless explicitly enabled. `enabled: false` is the master switch — even with `extract` / `finalizeAndMaintain` set to `true`, nothing runs. Non-cron sessions are never affected by these options. +All three options default to **`true`**: cron sessions behave like normal sessions (recall, buffering, extraction, and end-of-session maintenance all enabled) unless explicitly disabled. `enabled: false` is the master switch — even with `extract` / `finalizeAndMaintain` set to `true`, nothing runs. Non-cron sessions are never affected by these options. -All three sub-options are optional; omitted fields keep the default `false` (e.g. with `"cron": { "enabled": true }` only recall and message buffering are enabled — extraction and end-of-session maintenance stay off). +All three sub-options are optional; omitted fields keep the default `true` (e.g. with `"cron": { "extract": false }` only extraction is disabled — recall, buffering, and end-of-session maintenance stay on). Caveat: when a cron job sets an explicit custom `sessionKey`, the host does not append the `cron` segment — such sessions cannot be detected and are treated as normal sessions. diff --git a/README_CN.md b/README_CN.md index 2851ca7..a67305b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -99,7 +99,7 @@ OpenClaw 定时任务创建的会话可以独立配置图谱行为。host 把 cr | `extract` | `true` | 是否在 cron 会话内触发知识提取（afterTurn / compact 的 LLM 三元组提取）。关闭后消息仍入库缓冲，之后可用 `openclaw graph-memory extract` 手动回填。 | | `finalizeAndMaintain` | `true` | cron 会话结束时是否执行 finalize（EVENT→SKILL 晋升）和图维护（decay / PageRank / 社区检测）。定时任务频繁时可关闭，避免每次会话结束都跑全局维护。 | -三个选项**默认全部开启**：cron 会话默认使用图谱，需按需显式关闭。`enabled=true` 是总开关：即使 `extract`/`finalizeAndMaintain` 设为 `false` 也不生效。非 cron 会话不受这些选项影响。 +三个选项**默认全部开启**：cron 会话默认使用图谱，需按需显式关闭。`enabled=false` 是总开关：即使 `extract`/`finalizeAndMaintain` 设为 `true` 也不生效。非 cron 会话不受这些选项影响。三个子项均可省略，未写的字段取默认值 `true`。 注意：若 cron 任务显式设置了自定义 `sessionKey`，host 不再附加 `cron` 段，此类会话无法被识别，将按普通会话处理。 diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 1078531..8b069bf 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -217,22 +217,22 @@ describe("cron session gating (cron 配置)", () => { expect(isCronSessionKey(null)).toBe(false); }); - it("默认配置（全 false）下 cron session_end 跳过 finalize 与图维护", async () => { + it("默认配置（全 true）下 cron session_end 仍执行 finalize 与图维护（向后兼容）", async () => { const handler = registerPlugin().hooks.get("session_end"); if (!handler) throw new Error("session_end hook was not registered"); await handler({ sessionId: CRON_SID, sessionKey: CRON_KEY }, {}); - expect(mocks.getBySession).not.toHaveBeenCalled(); - expect(mocks.runMaintenance).not.toHaveBeenCalled(); + expect(mocks.getBySession).toHaveBeenCalledWith({}, CRON_SID); + expect(mocks.runMaintenance).toHaveBeenCalledTimes(1); }); - it("默认配置下 cron session 不入库（默认关闭）", async () => { + it("默认配置下 cron session 正常入库", async () => { const { engine } = registerPlugin(); await expect(engine.ingest({ sessionId: CRON_SID, sessionKey: CRON_KEY, message: { role: "user", content: "hi" } })) - .resolves.toEqual({ ingested: false }); - expect(mocks.saveMessage).not.toHaveBeenCalled(); + .resolves.toEqual({ ingested: true }); + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); }); it("finalizeAndMaintain=false 时 cron session 跳过 finalize 与图维护", async () => { From d1a760432f479956b9af4ac79773c5afad9d4f9d Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:44:58 +0000 Subject: [PATCH 09/11] =?UTF-8?q?Fix:=20=E4=BF=AE=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=BA=9B=E9=98=BB=E5=A1=9E=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 修掉了原来多会话并发结束会并发跑两次全局维护的竞态 - 防止插件启动时检测不到embeddings就整个生命周期回退到FTS --- index.ts | 278 ++++++++++++++++++++++++++++------ src/recaller/recall.ts | 6 + src/store/db.ts | 10 +- src/store/gate.ts | 60 ++++++++ test/neo4j-gate.test.ts | 68 +++++++++ test/session-identity.test.ts | 2 + 6 files changed, 370 insertions(+), 54 deletions(-) create mode 100644 src/store/gate.ts create mode 100644 test/neo4j-gate.test.ts diff --git a/index.ts b/index.ts index 9973e7b..6c05142 100755 --- a/index.ts +++ b/index.ts @@ -8,6 +8,7 @@ import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import { Type } from "@sinclair/typebox"; import { getDriver, initSchema, getSession } from "./src/store/db.ts"; +import { Neo4jGate } from "./src/store/gate.ts"; import { saveMessage, getUnextracted, getMaxTurnIndex, markExtracted, isTurnExtracted, @@ -328,10 +329,16 @@ const graphMemoryProPlugin = { // ── 初始化 Neo4j ──────────────────────────────────────── const driver = getDriver(cfg.neo4j); + // Neo4j 熔断门控：掉线时快速降级（跳图谱注入 / 缓冲消息），避免每轮吃满 driver 超时 + const neo4jGate = new Neo4jGate(); + // Schema 初始化（异步，不阻塞启动） initSchema(driver, cfg.embedding) .then(() => api.logger.info("[graph-memory-pro] Neo4j schema initialized")) - .catch(err => api.logger.error(`[graph-memory-pro] schema init failed: ${err}`)); + .catch(err => { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] schema init failed: ${err}`); + }); const llm = createCompleteFn(effectiveModel, cfg.llm); const recaller = new Recaller(driver, cfg); @@ -372,6 +379,11 @@ const graphMemoryProPlugin = { } async function extractTurnKnowledge(sessionId: string, turnNum: number, rawMessages: any[]): Promise { + // 熔断开启时跳过本轮提取：消息保持未标记，恢复后由 compact / extract 补提取 + if (!neo4jGate.isAvailable()) { + api.logger.info(`[graph-memory-pro] turn ${turnNum}: extraction skipped (neo4j circuit open)`); + return; + } return withExtractLock(sessionId, async () => { try { if (await isTurnExtracted(driver, sessionId, turnNum)) { @@ -465,6 +477,155 @@ const graphMemoryProPlugin = { await saveMessage(driver, sessionId, seq, message.role ?? "unknown", message); } + // ── 消息持久化：门控 + 内存缓冲（Neo4j 掉线时兜底） ──── + + interface BufferedMessage { sessionId: string; seq: number; role: string; message: any } + const messageBuffer: BufferedMessage[] = []; + const MESSAGE_BUFFER_CAP = 2000; + let bufferFlushInFlight = false; + + /** + * 缓冲一条消息：seq 在缓冲时预分配（不走 getMaxTurnIndex —— 那也是 DB 查询）， + * 保证恢复后落库顺序与到达顺序一致。 + */ + function bufferMessage(sessionId: string, message: any): void { + const seq = (msgSeq.get(sessionId) ?? 0) + 1; + msgSeq.set(sessionId, seq); + if (messageBuffer.length >= MESSAGE_BUFFER_CAP) { + messageBuffer.shift(); + api.logger.warn("[graph-memory-pro] message buffer full, dropping oldest buffered message"); + } + messageBuffer.push({ sessionId, seq, role: message?.role ?? "unknown", message }); + } + + /** 恢复后把缓冲消息刷回 Neo4j（single-flight；失败保留余量稍后重试）。 */ + async function flushMessageBuffer(): Promise { + if (bufferFlushInFlight || !messageBuffer.length || !neo4jGate.isAvailable()) return; + bufferFlushInFlight = true; + let flushed = 0; + try { + while (messageBuffer.length) { + const next = messageBuffer[0]; + try { + await saveMessage(driver, next.sessionId, next.seq, next.role, next.message); + messageBuffer.shift(); + flushed += 1; + } catch (err) { + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] buffered message flush failed, will retry later: ${err}`); + break; + } + } + if (flushed > 0) { + api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); + } + } finally { + bufferFlushInFlight = false; + } + } + + /** + * ingest / afterTurn 共用的落库入口： + * 可用 → 直接写；不可用或写失败 → 缓冲并吞掉错误（不向 host 抛）， + * 恢复后由 flushMessageBuffer 补写。返回的 ingested=true 语义为"引擎已接管该消息"。 + */ + async function persistMessage(sessionId: string, message: any): Promise { + if (!neo4jGate.isAvailable()) { + bufferMessage(sessionId, message); + return; + } + try { + await ingestMessage(sessionId, message); + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + bufferMessage(sessionId, message); + api.logger.warn(`[graph-memory-pro] neo4j write failed, message buffered (${messageBuffer.length} pending): ${err}`); + } + } + + // ── recall 超时预算：慢查询不拖回合，回退缓存/降级 ────── + + const RECALL_BUDGET_MS = 5_000; + + /** + * 给 Promise 加等待上限。不取消底层操作（Neo4j 查询会在后台自然完成、 + * 连接归还连接池），只是放弃等待 —— 慢 != 死。 + */ + function withBudget(p: Promise, ms: number, label: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); + p.then( + v => { clearTimeout(timer); resolve(v); }, + e => { clearTimeout(timer); reject(e); }, + ); + }); + } + + // ── 图维护：后台单飞 + trailing rerun（A1） ───────────── + // session_end 不再 await 维护链（衰减→去重→PR→社区→LLM 摘要可能耗时数分钟）； + // 全局单飞修掉多会话并发跑维护的竞态；运行期间的再次请求只标记 rerun， + // 当前一轮结束后最多补跑一次（覆盖"最后一个结束的会话"）。 + + let maintenanceRun: Promise | null = null; + let maintenanceRerunRequested = false; + + function scheduleMaintenance(): void { + if (!neo4jGate.isAvailable()) { + api.logger.info("[graph-memory-pro] maintenance skipped: neo4j unavailable (circuit open)"); + return; + } + if (maintenanceRun) { + maintenanceRerunRequested = true; + api.logger.info("[graph-memory-pro] maintenance already running, rerun queued"); + return; + } + maintenanceRun = (async () => { + try { + do { + maintenanceRerunRequested = false; + const embedFn = recaller.embedFn ?? undefined; + const result = await runMaintenance(driver, cfg, llm, embedFn); + neo4jGate.recordSuccess(); + api.logger.info( + `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + + `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + + `summaries=${result.communitySummaries}, ` + + `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, + ); + } while (maintenanceRerunRequested && neo4jGate.isAvailable()); + } catch (err) { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] maintenance failed: ${err}`); + } finally { + maintenanceRun = null; + maintenanceRerunRequested = false; + } + })(); + } + + // ── embedding 会话级 re-probe ────────────────────────── + // 启动 probe 失败会让插件停在文本搜索模式直到重启；这里在每个会话开始时 + // 重试一次（single-flight），临时性故障恢复后自动回到向量召回。 + + const embeddingConfigured = !!(cfg.embedding && (cfg.embedding.apiKey || cfg.embedding.baseURL)); + let embedProbeInFlight = false; + + function ensureEmbeddingReady(): void { + if (!embeddingConfigured || recaller.hasEmbedFn() || embedProbeInFlight) return; + embedProbeInFlight = true; + createEmbedFn(cfg.embedding) + .then(fn => { + if (fn) { + recaller.setEmbedFn(fn); + api.logger.info("[graph-memory-pro] embedding re-probe succeeded — vector search re-enabled"); + } + }) + .catch(() => {}) + .finally(() => { embedProbeInFlight = false; }); + } + // ── before_agent_start：召回 ──────────────────────────── api.on("before_agent_start", async (event: any, ctx: any) => { @@ -476,10 +637,12 @@ const graphMemoryProPlugin = { const prompt = cleanPrompt(rawPrompt); if (!prompt) return; if (prompt.includes("/new or /reset") || prompt.includes("new session was started")) return; + // 熔断开启时跳过召回 —— assemble 也会走降级路径（仅转录文本） + if (!neo4jGate.isAvailable()) return; api.logger.info(`[graph-memory-pro] recall query: "${prompt.slice(0, 80)}"`); - const res = await recaller.recall(prompt); + const res = await withBudget(recaller.recall(prompt), RECALL_BUDGET_MS, "[graph-memory-pro] recall"); if (res.nodes.length) { const sessionId = typeof ctx?.sessionId === "string" ? ctx.sessionId : undefined; const sessionKey = typeof ctx?.sessionKey === "string" ? ctx.sessionKey : undefined; @@ -505,6 +668,8 @@ const graphMemoryProPlugin = { async bootstrap({ sessionId, sessionKey }: { sessionId: string; sessionKey?: string }) { bindSessionIdentity(sessionId, sessionKey); + // 每个会话开始时尝试恢复 embedding（启动 probe 失败后的会话级 re-probe） + ensureEmbeddingReady(); return { bootstrapped: true }; }, @@ -515,7 +680,7 @@ const graphMemoryProPlugin = { if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { return { ingested: false }; } - await ingestMessage(sessionId, message); + await persistMessage(sessionId, message); ingestedSinceTurn.set(sessionId, (ingestedSinceTurn.get(sessionId) ?? 0) + 1); return { ingested: true }; }, @@ -541,20 +706,13 @@ const graphMemoryProPlugin = { }; } - const activeNodes = await getBySession(driver, sessionId); - const activeEdges: any[] = []; - for (const n of activeNodes) { - activeEdges.push(...await edgesFrom(driver, n.id)); - activeEdges.push(...await edgesTo(driver, n.id)); - } - // prompt-aware recall：优先用当前 prompt 做新鲜召回，回退到 before_agent_start 缓存 let rec = recalled.get(sessionId) ?? { nodes: [], edges: [] }; - if (prompt) { + if (prompt && neo4jGate.isAvailable()) { const cleaned = cleanPrompt(prompt); if (cleaned) { try { - const freshRec = await recaller.recall(cleaned); + const freshRec = await withBudget(recaller.recall(cleaned), RECALL_BUDGET_MS, "[graph-memory-pro] assemble recall"); if (freshRec.nodes.length) { rec = freshRec; recalled.set(sessionId, freshRec); @@ -564,45 +722,52 @@ const graphMemoryProPlugin = { } } } - const totalGmNodes = activeNodes.length + rec.nodes.length; const prepared = prepareAssemblyMessages(messages); - if (totalGmNodes === 0) { - if (prepared.dropped > 0) { - api.logger.info( - `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + - `dropped ${prepared.dropped} older msgs, graph ~0 tok`, - ); + // 图谱段：门控 + 降级 —— Neo4j 掉线/超时时只返回裁剪后的转录， + // 不让错误抛回 host（原实现无 catch，getBySession 失败会炸掉 assemble） + let graphTokens = 0; + let systemPromptAddition: string | undefined; + if (neo4jGate.isAvailable()) { + try { + const activeNodes = await getBySession(driver, sessionId); + const activeEdges: any[] = []; + for (const n of activeNodes) { + activeEdges.push(...await edgesFrom(driver, n.id)); + activeEdges.push(...await edgesTo(driver, n.id)); + } + + if (activeNodes.length + rec.nodes.length > 0) { + const { xml, systemPrompt, tokens } = await assembleContext(driver, { + tokenBudget: budget, + activeNodes, + activeEdges, + recalledNodes: rec.nodes, + recalledEdges: rec.edges, + }); + graphTokens = tokens; + if (xml) { + systemPromptAddition = systemPrompt ? `${systemPrompt}\n\n${xml}` : xml; + } + } + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] assemble: graph context unavailable, transcript-only: ${err}`); } - return { - messages: prepared.messages, - estimatedTokens: prepared.tokens, - }; } - const { xml, systemPrompt, tokens: gmTokens } = await assembleContext(driver, { - tokenBudget: budget, - activeNodes, - activeEdges, - recalledNodes: rec.nodes, - recalledEdges: rec.edges, - }); - if (prepared.dropped > 0) { api.logger.info( `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + - `dropped ${prepared.dropped} older msgs, graph ~${gmTokens} tok`, + `dropped ${prepared.dropped} older msgs, graph ~${graphTokens} tok`, ); } - let systemPromptAddition: string | undefined; - if (xml) { - systemPromptAddition = systemPrompt ? `${systemPrompt}\n\n${xml}` : xml; - } - return { messages: prepared.messages, - estimatedTokens: gmTokens + prepared.tokens, + estimatedTokens: graphTokens + prepared.tokens, ...(systemPromptAddition ? { systemPromptAddition } : {}), }; }, @@ -616,6 +781,10 @@ const graphMemoryProPlugin = { reason: cronCfg.enabled ? "cron session extraction disabled" : "cron session graph disabled", }; } + // 熔断开启时跳过提取：未提取消息保留，恢复后下一次 compact / extract 补上 + if (!neo4jGate.isAvailable()) { + return { ok: true, compacted: false, reason: "neo4j unavailable (circuit open)" }; + } return withExtractLock(sessionId, async () => { const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); @@ -692,7 +861,7 @@ const graphMemoryProPlugin = { const ingestedCount = ingestedSinceTurn.get(sessionId) ?? 0; const missingMessages = missingIngestMessages(newMessages, ingestedCount); for (const message of missingMessages) { - await ingestMessage(sessionId, message); + await persistMessage(sessionId, message); } if (missingMessages.length > 0) { api.logger.warn( @@ -771,7 +940,22 @@ const graphMemoryProPlugin = { return; } - const nodes = await getBySession(driver, sid); + // 熔断开启时跳过 finalize（全是 Neo4j 写）—— 消息已缓冲，恢复后补齐 + if (!neo4jGate.isAvailable()) { + api.logger.warn(`[graph-memory-pro] session_end ${sid.slice(0, 12)}…: neo4j unavailable (circuit open), finalize + maintenance skipped`); + return; + } + + let nodes: Awaited>; + try { + nodes = await getBySession(driver, sid); + neo4jGate.recordSuccess(); + void flushMessageBuffer(); + } catch (err) { + neo4jGate.recordFailure(); + api.logger.error(`[graph-memory-pro] session_end error: ${err}`); + return; + } if (nodes.length) { // 获取图谱摘要 const session = getSession(driver); @@ -812,15 +996,9 @@ const graphMemoryProPlugin = { for (const id of fin.invalidations) await deprecate(driver, id); } - // 图维护 - const embedFn = (recaller as any).embed ?? undefined; - const result = await runMaintenance(driver, cfg, llm, embedFn); - api.logger.info( - `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + - `dedup=${result.dedup.merged}, communities=${result.community.count}, ` + - `summaries=${result.communitySummaries}, ` + - `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, - ); + // 图维护：后台单飞（A1）—— 衰减→去重→PR→社区→LLM 摘要可能耗时数分钟， + // 不再阻塞 session_end；结果只进日志，host 对其零依赖 + scheduleMaintenance(); } catch (err) { api.logger.error(`[graph-memory-pro] session_end error: ${err}`); } finally { @@ -1244,7 +1422,7 @@ const graphMemoryProPlugin = { description: "手动触发图维护：衰减评分 + tier 转换、去重、PageRank、社区检测。", parameters: Type.Object({}), async execute() { - const embedFn = (recaller as any).embed ?? undefined; + const embedFn = recaller.embedFn ?? undefined; const result = await runMaintenance(driver, cfg, llm, embedFn); const t = result.decay.tierTransitions; const totalTransitions = t.coreToWorking + t.workingToPeripheral + t.peripheralToWorking + t.workingToCore; diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index 87ec456..e7fa6f3 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -88,6 +88,12 @@ export class Recaller { setEmbedFn(fn: EmbedFn): void { this.embed = fn; } + /** 是否已接入 embedding（启动 probe 成功或会话级 re-probe 成功）。 */ + hasEmbedFn(): boolean { return this.embed !== null; } + + /** 只读暴露 embedFn（maintenance / gm_maintain 需要），替代 (recaller as any).embed。 */ + get embedFn(): EmbedFn | null { return this.embed; } + async recall(query: string, options?: RecallOptions): Promise { const limit = this.cfg.recallMaxNodes; const timeRange = options ? parseTimeRange(options) : null; diff --git a/src/store/db.ts b/src/store/db.ts index d5d7315..01631f8 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -21,8 +21,9 @@ export function getDriver(cfg: Neo4jConfig): Driver { if (_driver) return _driver; _driver = neo4j.driver(cfg.uri, neo4j.auth.basic(cfg.user, cfg.password), { maxConnectionPoolSize: 50, - connectionAcquisitionTimeout: 60000, - maxTransactionRetryTime: 30000, + // 快速失败配合 gate.ts 熔断：掉线时 ~10s 内报错跳闸，而不是每次卡 30-60s + connectionAcquisitionTimeout: 15_000, + maxTransactionRetryTime: 10_000, }); return _driver; } @@ -40,8 +41,9 @@ export function getSession(driver: Driver): Session { console.log("[graph-memory-pro] reconnecting Neo4j driver..."); _driver = neo4j.driver(_cfg.uri, neo4j.auth.basic(_cfg.user, _cfg.password), { maxConnectionPoolSize: 50, - connectionAcquisitionTimeout: 60000, - maxTransactionRetryTime: 30000, + // 与 getDriver 保持一致：快速失败，让熔断门控尽早接手 + connectionAcquisitionTimeout: 15_000, + maxTransactionRetryTime: 10_000, }); return _driver.session({ database: "neo4j" }); } diff --git a/src/store/gate.ts b/src/store/gate.ts new file mode 100644 index 0000000..7a8e9c3 --- /dev/null +++ b/src/store/gate.ts @@ -0,0 +1,60 @@ +/** + * graph-memory-pro — Neo4j 熔断门控（circuit breaker） + * + * 解决的问题：Neo4j 掉线时，每次 ingest / assemble / recall 都要吃满 + * driver 的重试与连接获取超时（最坏数十秒），对话每轮都被拖住， + * 体验上等同于"卡死"。 + * + * 语义： + * - closed（正常）：isAvailable() === true，所有操作放行。 + * - open（跳闸）：连续失败 >= failureThreshold 次后进入；冷却期内 + * isAvailable() === false，调用方应立即降级（跳过图谱、缓冲消息）， + * 而不是等 driver 超时。 + * - half-open（半开探测）：冷却期结束后 isAvailable() 恢复 true， + * 下一个真实操作充当探测 —— 成功则复位 closed，失败则重新计时冷却。 + * + * 注意：失败计数只应从"纯 Neo4j 调用点"记录（saveMessage / getBySession + * 等）。混合了 LLM / embedding 的调用点（recall、compact）不要记录， + * 否则 LLM 超时会误跳闸。 + */ + +export class Neo4jGate { + private consecutiveFailures = 0; + private open = false; + private openedAt = 0; + + constructor( + /** 连续失败多少次后跳闸 */ + private readonly failureThreshold: number = 2, + /** 跳闸后的冷却时长（ms），到期进入半开 */ + private readonly cooldownMs: number = 120_000, + ) {} + + /** 操作成功：复位计数并闭合熔断。 */ + recordSuccess(): void { + this.consecutiveFailures = 0; + this.open = false; + } + + /** + * 操作失败：累计计数；达到阈值跳闸。 + * 已处于 open 时再次失败（半开探测失败 / 在途请求迟到失败）会 + * 重置冷却计时 —— 但被门控的调用方在 open 期间不会发起操作， + * 所以不会出现"永远无法恢复"的抖动。 + */ + recordFailure(): void { + this.consecutiveFailures += 1; + if (!this.open && this.consecutiveFailures >= this.failureThreshold) { + this.open = true; + } + if (this.open) { + this.openedAt = Date.now(); + } + } + + /** 当前是否放行操作（closed 或 冷却到期的 half-open）。 */ + isAvailable(): boolean { + if (!this.open) return true; + return Date.now() - this.openedAt >= this.cooldownMs; + } +} diff --git a/test/neo4j-gate.test.ts b/test/neo4j-gate.test.ts new file mode 100644 index 0000000..4a430d0 --- /dev/null +++ b/test/neo4j-gate.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { Neo4jGate } from "../src/store/gate.ts"; + +describe("Neo4j 熔断门控 (Neo4jGate)", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("closed 状态放行所有操作，成功复位失败计数", () => { + const gate = new Neo4jGate(2, 120_000); + expect(gate.isAvailable()).toBe(true); + + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + + gate.recordSuccess(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + }); + + it("连续失败达到阈值后跳闸，冷却期内不可用", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(119_999); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(1); + expect(gate.isAvailable()).toBe(true); + }); + + it("半开后一次成功即完全恢复（计数归零）", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + vi.advanceTimersByTime(120_000); + expect(gate.isAvailable()).toBe(true); + + gate.recordSuccess(); + gate.recordFailure(); + expect(gate.isAvailable()).toBe(true); + }); + + it("半开探测失败重新进入冷却", () => { + vi.useFakeTimers(); + const gate = new Neo4jGate(2, 120_000); + + gate.recordFailure(); + gate.recordFailure(); + vi.advanceTimersByTime(120_000); + + gate.recordFailure(); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(119_999); + expect(gate.isAvailable()).toBe(false); + + vi.advanceTimersByTime(1); + expect(gate.isAvailable()).toBe(true); + }); +}); diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 8b069bf..c2ba55e 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -58,6 +58,8 @@ vi.mock("../src/engine/embed.ts", () => ({ vi.mock("../src/recaller/recall.ts", () => ({ Recaller: class { setEmbedFn(): void {} + hasEmbedFn(): boolean { return false; } + get embedFn() { return null; } async recall() { return mocks.recall(); } async syncEmbed(): Promise {} }, From 5101f7cddfc8a46c3d68ac0cddfc0bf5c132f546 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:03:11 +0000 Subject: [PATCH 10/11] =?UTF-8?q?Fix:=20=E8=BF=9B=E8=A1=8C=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=AE=A1=E8=AE=A1=E5=90=8E=E6=8D=89=E4=BA=86=E5=87=A0?= =?UTF-8?q?=E4=B8=AAbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.ts | 90 ++++++++++++++++++++++------------- test/session-identity.test.ts | 82 ++++++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 34 deletions(-) diff --git a/index.ts b/index.ts index 6c05142..26f6fd2 100755 --- a/index.ts +++ b/index.ts @@ -345,16 +345,24 @@ const graphMemoryProPlugin = { const extractor = new Extractor(llm); // ── 初始化 embedding ──────────────────────────────────── + // re-probe 状态提前声明：启动 probe 失败时记录时间戳，bootstrap 的 + // 会话级 re-probe 据此退避（端点宕机时不逐会话重试刷日志/打 API） + const embeddingConfigured = !!(cfg.embedding && (cfg.embedding.apiKey || cfg.embedding.baseURL)); + let embedProbeInFlight = false; + let lastEmbedProbeAt = 0; + createEmbedFn(cfg.embedding) .then((fn) => { if (fn) { recaller.setEmbedFn(fn); api.logger.info("[graph-memory-pro] vector search ready"); } else { + lastEmbedProbeAt = Date.now(); api.logger.info("[graph-memory-pro] text search mode (配置 embedding 可启用语义搜索)"); } }) .catch(() => { + lastEmbedProbeAt = Date.now(); api.logger.info("[graph-memory-pro] text search mode"); }); @@ -386,6 +394,9 @@ const graphMemoryProPlugin = { } return withExtractLock(sessionId, async () => { try { + // 先等掉线期间缓冲的消息落库，再判断/标记 extracted——否则行落库晚于 + // markExtracted 时会以 extracted=false 重现，被下一轮 compact 重复提取 + if (messageBuffer.length) await flushMessageBuffer(); if (await isTurnExtracted(driver, sessionId, turnNum)) { api.logger.info(`[graph-memory-pro] turn ${turnNum}: already extracted (compact), skipping`); return; @@ -479,49 +490,61 @@ const graphMemoryProPlugin = { // ── 消息持久化：门控 + 内存缓冲（Neo4j 掉线时兜底） ──── - interface BufferedMessage { sessionId: string; seq: number; role: string; message: any } + interface BufferedMessage { sessionId: string; message: any } const messageBuffer: BufferedMessage[] = []; const MESSAGE_BUFFER_CAP = 2000; - let bufferFlushInFlight = false; + let flushRun: Promise | null = null; /** - * 缓冲一条消息：seq 在缓冲时预分配（不走 getMaxTurnIndex —— 那也是 DB 查询）， - * 保证恢复后落库顺序与到达顺序一致。 + * 缓冲一条消息。不在缓冲时分配 seq：内存 msgSeq 在 session_end 清理 / + * DB 故障时与 DB 脱节，预分配的 seq 会与已有行撞号，saveMessage 的 + * ON MATCH SET 会静默覆盖旧行内容。seq 统一在 flush 时由 ingestMessage + * 分配（那时 DB 可达，getMaxTurnIndex 恢复能正确兜底）。 */ function bufferMessage(sessionId: string, message: any): void { - const seq = (msgSeq.get(sessionId) ?? 0) + 1; - msgSeq.set(sessionId, seq); + // 不可序列化的消息（循环引用 / BigInt）永远写不进 DB——当场丢弃， + // 否则它会永久堵在 flush 队列头并反复重跳熔断 + try { JSON.stringify(message); } catch (err) { + api.logger.warn(`[graph-memory-pro] message not serializable, dropped from outage buffer: ${err}`); + return; + } if (messageBuffer.length >= MESSAGE_BUFFER_CAP) { messageBuffer.shift(); api.logger.warn("[graph-memory-pro] message buffer full, dropping oldest buffered message"); } - messageBuffer.push({ sessionId, seq, role: message?.role ?? "unknown", message }); + messageBuffer.push({ sessionId, message }); } - /** 恢复后把缓冲消息刷回 Neo4j（single-flight；失败保留余量稍后重试）。 */ - async function flushMessageBuffer(): Promise { - if (bufferFlushInFlight || !messageBuffer.length || !neo4jGate.isAvailable()) return; - bufferFlushInFlight = true; - let flushed = 0; - try { - while (messageBuffer.length) { - const next = messageBuffer[0]; - try { - await saveMessage(driver, next.sessionId, next.seq, next.role, next.message); - messageBuffer.shift(); - flushed += 1; - } catch (err) { - neo4jGate.recordFailure(); - api.logger.warn(`[graph-memory-pro] buffered message flush failed, will retry later: ${err}`); - break; + /** + * 恢复后把缓冲消息刷回 Neo4j。single-flight：返回同一个 in-flight + * promise，让 extract / compact 路径能真正等它完成再继续。 + */ + function flushMessageBuffer(): Promise { + if (flushRun) return flushRun; + if (!messageBuffer.length || !neo4jGate.isAvailable()) return Promise.resolve(); + flushRun = (async () => { + let flushed = 0; + try { + while (messageBuffer.length) { + const next = messageBuffer[0]; + try { + await ingestMessage(next.sessionId, next.message); + messageBuffer.shift(); + flushed += 1; + } catch (err) { + neo4jGate.recordFailure(); + api.logger.warn(`[graph-memory-pro] buffered message flush failed, will retry later: ${err}`); + break; + } } + if (flushed > 0) { + api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); + } + } finally { + flushRun = null; } - if (flushed > 0) { - api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); - } - } finally { - bufferFlushInFlight = false; - } + })(); + return flushRun; } /** @@ -607,14 +630,15 @@ const graphMemoryProPlugin = { // ── embedding 会话级 re-probe ────────────────────────── // 启动 probe 失败会让插件停在文本搜索模式直到重启；这里在每个会话开始时 - // 重试一次（single-flight），临时性故障恢复后自动回到向量召回。 + // 重试（single-flight + 5 分钟退避），临时性故障恢复后自动回到向量召回。 - const embeddingConfigured = !!(cfg.embedding && (cfg.embedding.apiKey || cfg.embedding.baseURL)); - let embedProbeInFlight = false; + const EMBED_REPROBE_INTERVAL_MS = 300_000; function ensureEmbeddingReady(): void { if (!embeddingConfigured || recaller.hasEmbedFn() || embedProbeInFlight) return; + if (Date.now() - lastEmbedProbeAt < EMBED_REPROBE_INTERVAL_MS) return; embedProbeInFlight = true; + lastEmbedProbeAt = Date.now(); createEmbedFn(cfg.embedding) .then(fn => { if (fn) { @@ -786,6 +810,8 @@ const graphMemoryProPlugin = { return { ok: true, compacted: false, reason: "neo4j unavailable (circuit open)" }; } return withExtractLock(sessionId, async () => { + // compact 是掉线恢复后的补提取路径：先把缓冲消息刷进 DB 再读未提取集 + if (messageBuffer.length) await flushMessageBuffer(); const msgs = await getUnextracted(driver, sessionId, cfg.compactTurnCount * 3); if (!msgs.length) return { ok: true, compacted: false, reason: "no messages" }; diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index c2ba55e..3253762 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -2,8 +2,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { isCronSessionKey } from "../src/types.ts"; const mocks = vi.hoisted(() => ({ - getBySession: vi.fn(async () => []), - saveMessage: vi.fn(async () => {}), + getBySession: vi.fn(async () => [] as unknown[]), + saveMessage: vi.fn(async ( + _driver: unknown, _sid: string, _turn: number, _role: string, _content: unknown, + ): Promise => {}), getMaxTurnIndex: vi.fn(async () => 0), getUnextracted: vi.fn(async () => []), isTurnExtracted: vi.fn(async () => false), @@ -354,3 +356,79 @@ describe("cron session gating (cron 配置)", () => { expect(mocks.saveMessage).toHaveBeenCalledTimes(1); }); }); + +describe("outage buffering (Neo4j 掉线缓冲)", () => { + beforeEach(() => { + // mockReset：清掉上一个测试可能设置的 mockRejectedValue 等持久实现， + // 否则持续拒绝会泄漏到后续测试，把所有消息都打进缓冲 + mocks.saveMessage.mockReset().mockImplementation(async () => {}); + mocks.saveMessage.mockClear(); + mocks.getUnextracted.mockClear(); + mocks.getMaxTurnIndex.mockClear(); + }); + + it("写失败时消息被缓冲且不向 host 抛错", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValue(new Error("neo4j down")); + + await expect( + engine.ingest({ sessionId: "outage-1", message: { role: "user", content: "hello" } }), + ).resolves.toEqual({ ingested: true }); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + }); + + it("恢复后缓冲消息经 ingestMessage 补写（seq 由 DB 状态分配，不撞号）", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + // 第一条：写失败 → 缓冲（seq 1 被失败尝试消耗） + await engine.ingest({ sessionId: "outage-2", message: { role: "user", content: "first message" } }); + // 第二条：写成功 → 触发 flush → 第一条补写（分配新 seq，绕开撞号） + await engine.ingest({ sessionId: "outage-2", message: { role: "user", content: "second message" } }); + await new Promise(resolve => setTimeout(resolve, 0)); + + expect(mocks.saveMessage).toHaveBeenCalledTimes(3); + const flushed = mocks.saveMessage.mock.calls[2]; + expect(flushed[1]).toBe("outage-2"); + expect(flushed[2]).toBe(3); // seq 3 = max(已写 2) + 1，而非缓冲期的 1 + expect((flushed[4] as { content: string }).content).toContain("first message"); + }); + + it("不可序列化的消息被丢弃，不堵塞后续消息（队头防堵）", async () => { + const { engine } = registerPlugin(); + // 先让写失败一次，把毒消息逼进缓冲路径（真实 saveMessage 会在 stringify 时抛错） + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + const poison: any = { role: "user", content: "ok" }; + poison.self = poison; // 循环引用 → JSON.stringify 抛错 + + await expect( + engine.ingest({ sessionId: "outage-3", message: poison }), + ).resolves.toEqual({ ingested: true }); + await expect( + engine.ingest({ sessionId: "outage-3", message: { role: "user", content: "after poison" } }), + ).resolves.toEqual({ ingested: true }); + await new Promise(resolve => setTimeout(resolve, 0)); + + // 第一次 saveMessage 因序列化失败抛错 → 消息进缓冲即被丢弃； + // 第二条正常直写后 flush 无积压 → 总共 2 次调用，毒消息不重试 + expect(mocks.saveMessage).toHaveBeenCalledTimes(2); + }); + + it("compact 在读取未提取消息前先刷缓冲（恢复补提取顺序）", async () => { + const { engine } = registerPlugin(); + mocks.saveMessage.mockRejectedValueOnce(new Error("neo4j down")); + + await engine.ingest({ sessionId: "outage-4", message: { role: "user", content: "buffered turn" } }); + expect(mocks.saveMessage).toHaveBeenCalledTimes(1); + + const res = await engine.compact({ sessionId: "outage-4" }); + expect(res).toEqual({ ok: true, compacted: false, reason: "no messages" }); + + // flush 先于 getUnextracted：第二条 saveMessage 是缓冲补写，然后才查未提取集 + expect(mocks.saveMessage).toHaveBeenCalledTimes(2); + expect((mocks.saveMessage.mock.calls[1][4] as { content: string }).content).toContain("buffered turn"); + expect(mocks.getUnextracted).toHaveBeenCalledTimes(1); + }); +}); From bb124b76b78b193e1dfec93f21e70546a04f5fe7 Mon Sep 17 00:00:00 2001 From: TriDefender <173548745+TriDefender@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:17:28 +0000 Subject: [PATCH 11/11] =?UTF-8?q?Fix=20=E4=BF=AE=E5=A4=8D=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E5=AE=A1=E8=AE=A1=E5=8F=91=E7=8E=B0=E7=9A=84=E5=B9=B6?= =?UTF-8?q?=E5=8F=91=E3=80=81=E6=95=B0=E6=8D=AE=E6=8D=9F=E5=9D=8F=E4=B8=8E?= =?UTF-8?q?=E6=80=A7=E8=83=BD=E7=BC=BA=E9=99=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 数据损坏/丢失： - gm_maintain 与后台维护链并发（validatedCount 双计、communityId 互踩）： scheduleMaintenance 改为类型化单飞 promise，全部维护入口收敛同一互斥点 - computeGlobalPageRank：GDS write 成功后 drop/topK 失败不再回落 1/(i+1) 覆盖真实 PageRank（drop 失败单独吞，宁泄漏临时投影） - summarizeCommunities：prune 挪到签名复用之后——先 prune 会删掉重编号 场景下的捐赠者社区，导致每轮维护全量重算 LLM 摘要+embedding - graph-memory extract：批边界 await 所有 syncEmbed，closeDriver 不再 丢失最后一批在途向量（markExtracted 后不可自愈） - PUT /nodes：改 type 与非法边清理（EDGE_DIRECTION_RULES 生成谓词）包进 单 executeWrite 事务；name 重名预检 409、空名 400 功能失效： - db.ts：删除不可达的 driver 重连分支（驱动 5.x session() 从不抛错）， getSession 永远优先模块级 _driver 单例 - llm.ts：OAuth 缓存按 oauthPath mtime 失效（CLI 刷新 token 后网关 下一次调用即生效，无需重启）；Anthropic 遍历找 text 块（thinking 块 在前不再误报 empty）；openai/anthropic 路径补 fetchRetry（429/5xx） - recall：双路径 Promise.all 并行 + query 向量算一次共享 + assemble 复用 未变 prompt 的缓存（4 embed/4 投影 → 1 embed/2 并行遍历） - flushMessageBuffer 成功后补排 maintenance（熔断跳过的一轮不再丢失） 竞态/一致性： - upsertNode：撞 *_name 唯一约束退回更新路径（幂等）；records[0] 守卫 - upsertEdge：三段查询合并为单条 apoc.merge.relationship（消除并发重复边） - session_end finalize 包进 withExtractLock；gm_record/gm_link 溯源统一 sessionId；crud 内联 normalizeName 改用 store 导入（受一致性测试保护） 性能： - detectDuplicates：N 次向量查询折叠为单条 UNWIND+CALL - applyDecay：Math.max spread 改 reduce（防超大节点集爆栈） - assemble：2N 次串行边查询改 edgesTouching 单次批量；社区摘要并发预取 - communityRepresentatives 加 totalLimit 封顶；graphWalk clamp maxDepth - recall 超时 30s 退避（withBudget 不取消底层查询，防堆积） 配置/测试： - schema recallMaxNodes 3→6 对齐代码默认；freshTailCount 接线到 sliceLastTurn（默认 5=原硬编码 KEEP_TURNS，行为不变） - integration.recall/routes 支持 NEO4J_TEST_URI（消除 7687 硬编码） - 新增集成用例：TASK→EVENT 改 type 边清理、name 重名 409、社区摘要 --- index.ts | 180 ++++++++++++++++++++------------ openclaw.plugin.json | 4 +- src/cli-extract.ts | 9 +- src/engine/embed.ts | 4 +- src/engine/llm.ts | 27 +++-- src/format/assemble.ts | 14 +-- src/graph/community.ts | 10 +- src/graph/decay.ts | 3 +- src/graph/dedup.ts | 55 +++++----- src/graph/pagerank.ts | 54 ++++++---- src/recaller/recall.ts | 24 +++-- src/routes/crud.ts | 66 +++++++++--- src/store/db.ts | 33 ++---- src/store/store.ts | 175 ++++++++++++++++++++----------- src/types.ts | 5 +- test/integration.graph.test.ts | 14 ++- test/integration.recall.test.ts | 3 +- test/integration.routes.test.ts | 49 ++++++++- test/session-identity.test.ts | 1 + 19 files changed, 480 insertions(+), 250 deletions(-) diff --git a/index.ts b/index.ts index 26f6fd2..4a29eb4 100755 --- a/index.ts +++ b/index.ts @@ -14,7 +14,7 @@ import { markExtracted, isTurnExtracted, upsertNode, upsertEdge, findByName, updateNode, deleteNode, deprecateNodeAndDisconnect, - getBySession, edgesFrom, edgesTo, + getBySession, edgesTouching, deleteEdges, mergeNodes, deprecate, getStats, } from "./src/store/store.ts"; @@ -173,17 +173,18 @@ export function extractUserText(msg: any): string { export function sliceLastTurn( messages: any[], + keepTurns: number = KEEP_TURNS, ): { messages: any[]; tokens: number; dropped: number } { if (!messages.length) { return { messages: [], tokens: 0, dropped: 0 }; } - // 找到最近 N 个 user 消息的位置 + // 找到最近 N 个 user 消息的位置（N = keepTurns，由 cfg.freshTailCount 注入） const userIndices: number[] = []; for (let i = messages.length - 1; i >= 0; i--) { if (messages[i].role === "user") { userIndices.push(i); - if (userIndices.length >= KEEP_TURNS) break; + if (userIndices.length >= keepTurns) break; } } if (!userIndices.length) { @@ -232,8 +233,9 @@ export function sliceLastTurn( /** 图谱为空时也必须执行相同的裁剪、工具配对修复和 content 规范化。 */ export function prepareAssemblyMessages( messages: any[], + keepTurns: number = KEEP_TURNS, ): { messages: any[]; tokens: number; dropped: number } { - const sliced = sliceLastTurn(messages); + const sliced = sliceLastTurn(messages, keepTurns); return { messages: normalizeMessageContent(sanitizeToolUseResultPairing(sliced.messages)), tokens: sliced.tokens, @@ -450,6 +452,8 @@ const graphMemoryProPlugin = { const msgSeq = new Map(); const msgSeqLoaders = new Map>(); const recalled = new Map(); + // recalled 结果对应的 recall prompt（assemble 复用缓存判定用；继承路径不写，查不到则照常新鲜召回） + const recalledPrompt = new Map(); const sessionIdsByKey = new Map(); const pendingSubagentRecall = new Map(); const ingestedSinceTurn = new Map(); @@ -539,6 +543,9 @@ const graphMemoryProPlugin = { } if (flushed > 0) { api.logger.info(`[graph-memory-pro] flushed ${flushed} buffered message(s) to neo4j`); + // 补偿掉线期间被熔断跳过的维护：缓冲消息已补录，趁 gate 可用重排一轮 + // （scheduleMaintenance 自带单飞 + gate 检查，无会话时它是安全的 no-op 调用） + scheduleMaintenance(); } } finally { flushRun = null; @@ -572,6 +579,14 @@ const graphMemoryProPlugin = { const RECALL_BUDGET_MS = 5_000; + // 超时退避：withBudget 只放弃等待、不取消底层查询，反复超时会在后台堆积 + // 占连接的 Neo4j 查询链；冷却窗口内跳过新 recall，直接走缓存/降级。 + const RECALL_BACKOFF_MS = 30_000; + let recallBackoffUntil = 0; + function markRecallBackoffOnTimeout(err: unknown): void { + if (String(err).includes("timed out")) recallBackoffUntil = Date.now() + RECALL_BACKOFF_MS; + } + /** * 给 Promise 加等待上限。不取消底层操作（Neo4j 查询会在后台自然完成、 * 连接归还连接池），只是放弃等待 —— 慢 != 死。 @@ -591,25 +606,34 @@ const graphMemoryProPlugin = { // 全局单飞修掉多会话并发跑维护的竞态；运行期间的再次请求只标记 rerun， // 当前一轮结束后最多补跑一次（覆盖"最后一个结束的会话"）。 - let maintenanceRun: Promise | null = null; + let maintenanceRun: Promise> | { skipped: string } | { failed: string }> | null = null; let maintenanceRerunRequested = false; - function scheduleMaintenance(): void { + /** + * 唯一的维护入口（session_end 与 gm_maintain 共用）： + * - 在跑 → 标记 rerun（保留 session_end 的 trailing 补跑语义）并 join + * 同一个 in-flight promise（gm_maintain 据此拿到结果而非并发裸跑） + * - gate 打开（熔断）→ 返回 skipped 标记，不触碰数据库 + * - 空闲 → 自己成为那一轮 + * 并发跑两条维护链会导致 dedup 双计 validatedCount、communityId 互相覆盖。 + */ + function scheduleMaintenance(): Promise> | { skipped: string } | { failed: string }> { if (!neo4jGate.isAvailable()) { api.logger.info("[graph-memory-pro] maintenance skipped: neo4j unavailable (circuit open)"); - return; + return Promise.resolve({ skipped: "neo4j unavailable (circuit open)" }); } if (maintenanceRun) { maintenanceRerunRequested = true; - api.logger.info("[graph-memory-pro] maintenance already running, rerun queued"); - return; + api.logger.info("[graph-memory-pro] maintenance already running, rerun queued + joining in-flight run"); + return maintenanceRun; } maintenanceRun = (async () => { try { + let result: Awaited>; do { maintenanceRerunRequested = false; const embedFn = recaller.embedFn ?? undefined; - const result = await runMaintenance(driver, cfg, llm, embedFn); + result = await runMaintenance(driver, cfg, llm, embedFn); neo4jGate.recordSuccess(); api.logger.info( `[graph-memory-pro] maintenance: ${result.durationMs}ms, ` + @@ -618,14 +642,17 @@ const graphMemoryProPlugin = { `top_pr=${result.pagerank.topK.slice(0, 3).map(n => `${n.name}(${n.score.toFixed(3)})`).join(",")}`, ); } while (maintenanceRerunRequested && neo4jGate.isAvailable()); + return result; } catch (err) { neo4jGate.recordFailure(); api.logger.error(`[graph-memory-pro] maintenance failed: ${err}`); + return { failed: String(err) }; } finally { maintenanceRun = null; maintenanceRerunRequested = false; } })(); + return maintenanceRun; } // ── embedding 会话级 re-probe ────────────────────────── @@ -663,6 +690,8 @@ const graphMemoryProPlugin = { if (prompt.includes("/new or /reset") || prompt.includes("new session was started")) return; // 熔断开启时跳过召回 —— assemble 也会走降级路径（仅转录文本） if (!neo4jGate.isAvailable()) return; + // 超时冷却窗口内跳过（后台可能仍有在途查询，不再叠加） + if (Date.now() < recallBackoffUntil) return; api.logger.info(`[graph-memory-pro] recall query: "${prompt.slice(0, 80)}"`); @@ -673,10 +702,12 @@ const graphMemoryProPlugin = { if (sessionId) { bindSessionIdentity(sessionId, sessionKey); recalled.set(sessionId, res); + recalledPrompt.set(sessionId, prompt); } api.logger.info(`[graph-memory-pro] recalled ${res.nodes.length} nodes, ${res.edges.length} edges`); } } catch (err) { + markRecallBackoffOnTimeout(err); api.logger.warn(`[graph-memory-pro] recall failed: ${err}`); } }); @@ -717,7 +748,7 @@ const graphMemoryProPlugin = { // cron session 关闭图谱功能：仅做消息裁剪与配对修复，不注入图谱上下文 if (isCronSessionKey(sessionKey) && !cronCfg.enabled) { - const prepared = prepareAssemblyMessages(messages); + const prepared = prepareAssemblyMessages(messages, cfg.freshTailCount); if (prepared.dropped > 0) { api.logger.info( `[graph-memory-pro] assemble: ${prepared.messages.length} msgs (~${prepared.tokens} tok), ` + @@ -730,23 +761,25 @@ const graphMemoryProPlugin = { }; } - // prompt-aware recall：优先用当前 prompt 做新鲜召回，回退到 before_agent_start 缓存 + // prompt-aware recall：clean 后的 prompt 与缓存命中同一查询时直接复用 + // before_agent_start 的结果，只有变化才发起第二次召回 let rec = recalled.get(sessionId) ?? { nodes: [], edges: [] }; - if (prompt && neo4jGate.isAvailable()) { - const cleaned = cleanPrompt(prompt); - if (cleaned) { - try { - const freshRec = await withBudget(recaller.recall(cleaned), RECALL_BUDGET_MS, "[graph-memory-pro] assemble recall"); - if (freshRec.nodes.length) { - rec = freshRec; - recalled.set(sessionId, freshRec); - } - } catch (err) { - api.logger.warn(`[graph-memory-pro] assemble recall failed: ${err}`); + const cachedPrompt = recalledPrompt.get(sessionId); + const cleanedPrompt = prompt ? cleanPrompt(prompt) : ""; + if (cleanedPrompt && neo4jGate.isAvailable() && Date.now() >= recallBackoffUntil && cleanedPrompt !== cachedPrompt) { + try { + const freshRec = await withBudget(recaller.recall(cleanedPrompt), RECALL_BUDGET_MS, "[graph-memory-pro] assemble recall"); + if (freshRec.nodes.length) { + rec = freshRec; + recalled.set(sessionId, freshRec); + recalledPrompt.set(sessionId, cleanedPrompt); } + } catch (err) { + markRecallBackoffOnTimeout(err); + api.logger.warn(`[graph-memory-pro] assemble recall failed: ${err}`); } } - const prepared = prepareAssemblyMessages(messages); + const prepared = prepareAssemblyMessages(messages, cfg.freshTailCount); // 图谱段：门控 + 降级 —— Neo4j 掉线/超时时只返回裁剪后的转录， // 不让错误抛回 host（原实现无 catch，getBySession 失败会炸掉 assemble） @@ -755,11 +788,8 @@ const graphMemoryProPlugin = { if (neo4jGate.isAvailable()) { try { const activeNodes = await getBySession(driver, sessionId); - const activeEdges: any[] = []; - for (const n of activeNodes) { - activeEdges.push(...await edgesFrom(driver, n.id)); - activeEdges.push(...await edgesTo(driver, n.id)); - } + // 单次批量查询替代逐节点 edgesFrom+edgesTo 的 2N 次串行往返 + const activeEdges = await edgesTouching(driver, activeNodes.map(n => n.id)); if (activeNodes.length + rec.nodes.length > 0) { const { xml, systemPrompt, tokens } = await assembleContext(driver, { @@ -925,6 +955,7 @@ const graphMemoryProPlugin = { const childSessionId = sessionIdsByKey.get(childSessionKey); if (childSessionId) { recalled.delete(childSessionId); + recalledPrompt.delete(childSessionId); msgSeq.delete(childSessionId); msgSeqLoaders.delete(childSessionId); extractLocks.delete(childSessionId); @@ -939,6 +970,7 @@ const graphMemoryProPlugin = { msgSeqLoaders.clear(); extractLocks.clear(); recalled.clear(); + recalledPrompt.clear(); sessionIdsByKey.clear(); pendingSubagentRecall.clear(); ingestedSinceTurn.clear(); @@ -983,43 +1015,47 @@ const graphMemoryProPlugin = { return; } if (nodes.length) { - // 获取图谱摘要 - const session = getSession(driver); - let summary = ""; - try { - const summaryResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) - RETURN n.name AS name, n.type AS type, n.validatedCount AS vc, n.pagerank AS pr - ORDER BY n.pagerank DESC LIMIT 20 - `); - summary = summaryResult.records - .map(r => `${r.get("type")}:${r.get("name")}(v${r.get("vc")},pr${(r.get("pr") ?? 0).toFixed?.(3) ?? "0"})`) - .join(", "); - } finally { - await session.close(); - } + // finalize 的 upsert 与 afterTurn/compact 的提取共用 per-session 互斥锁： + // 最后一轮的 afterTurn 提取可能仍在途，不串行化会重复 upsert（validatedCount 双递增） + await withExtractLock(sid, async () => { + // 获取图谱摘要 + const session = getSession(driver); + let summary = ""; + try { + const summaryResult = await session.run(` + MATCH (n:Task|Skill|Event {status: 'active'}) + RETURN n.name AS name, n.type AS type, n.validatedCount AS vc, n.pagerank AS pr + ORDER BY n.pagerank DESC LIMIT 20 + `); + summary = summaryResult.records + .map(r => `${r.get("type")}:${r.get("name")}(v${r.get("vc")},pr${(r.get("pr") ?? 0).toFixed?.(3) ?? "0"})`) + .join(", "); + } finally { + await session.close(); + } - const fin = await extractor.finalize({ sessionNodes: nodes, graphSummary: summary }); + const fin = await extractor.finalize({ sessionNodes: nodes, graphSummary: summary }); - for (const nc of fin.promotedSkills) { - if (nc.name && nc.content) { - await upsertNode(driver, { - type: "SKILL", name: nc.name, - description: nc.description ?? "", content: nc.content, - }, sid); + for (const nc of fin.promotedSkills) { + if (nc.name && nc.content) { + await upsertNode(driver, { + type: "SKILL", name: nc.name, + description: nc.description ?? "", content: nc.content, + }, sid); + } } - } - for (const ec of fin.newEdges) { - const fromNode = await findByName(driver, ec.from); - const toNode = await findByName(driver, ec.to); - if (fromNode && toNode) { - await upsertEdge(driver, { - fromId: fromNode.id, toId: toNode.id, type: ec.type, - instruction: ec.instruction, sessionId: sid, - }); + for (const ec of fin.newEdges) { + const fromNode = await findByName(driver, ec.from); + const toNode = await findByName(driver, ec.to); + if (fromNode && toNode) { + await upsertEdge(driver, { + fromId: fromNode.id, toId: toNode.id, type: ec.type, + instruction: ec.instruction, sessionId: sid, + }); + } } - } - for (const id of fin.invalidations) await deprecate(driver, id); + for (const id of fin.invalidations) await deprecate(driver, id); + }); } // 图维护：后台单飞（A1）—— 衰减→去重→PR→社区→LLM 摘要可能耗时数分钟， @@ -1032,6 +1068,7 @@ const graphMemoryProPlugin = { msgSeqLoaders.delete(sid); extractLocks.delete(sid); recalled.delete(sid); + recalledPrompt.delete(sid); ingestedSinceTurn.delete(sid); if (sessionKey && sessionIdsByKey.get(sessionKey) === sid) { sessionIdsByKey.delete(sessionKey); @@ -1123,7 +1160,8 @@ const graphMemoryProPlugin = { relatedSkill: Type.Optional(Type.String({ description: "关联的已有技能名" })), }), async execute(_toolCallId: string, p: any) { - const sid = ctx?.sessionKey ?? ctx?.sessionId ?? "manual"; + // 溯源统一用 sessionId（与 getBySession 的会话视图对齐）；无会话上下文才落 "manual" + const sid = ctx?.sessionId ?? "manual"; if (!["TASK", "SKILL", "EVENT"].includes(p.type)) { throw new Error(`[graph-memory-pro] 无效节点类型：${String(p.type)}`); } @@ -1257,7 +1295,7 @@ const graphMemoryProPlugin = { ); api.registerTool( - (_ctx: any) => ({ + (ctx: any) => ({ name: "gm_link", label: "Link Graph Memory Nodes", description: @@ -1282,7 +1320,7 @@ const graphMemoryProPlugin = { const stored = await upsertEdge(driver, { fromId: fromNode.id, toId: toNode.id, type: p.type, - instruction: p.instruction, condition: p.condition, sessionId: "manual", + instruction: p.instruction, condition: p.condition, sessionId: ctx?.sessionId ?? "manual", }); if (!stored) { throw new Error( @@ -1448,8 +1486,16 @@ const graphMemoryProPlugin = { description: "手动触发图维护：衰减评分 + tier 转换、去重、PageRank、社区检测。", parameters: Type.Object({}), async execute() { - const embedFn = recaller.embedFn ?? undefined; - const result = await runMaintenance(driver, cfg, llm, embedFn); + // 走 scheduleMaintenance 单飞入口：后台维护在跑时 join 而非并发裸跑 + // （并发会导致 dedup 双计 validatedCount、communityId 互相覆盖） + const result = await scheduleMaintenance(); + if (!("decay" in result)) { + const reason = "skipped" in result ? result.skipped : result.failed; + return { + content: [{ type: "text", text: `⚠️ 图维护未完成：${reason}` }], + details: result, + }; + } const t = result.decay.tierTransitions; const totalTransitions = t.coreToWorking + t.workingToPeripheral + t.peripheralToWorking + t.workingToCore; const text = [ diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 8022351..9a40a38 100755 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -23,9 +23,9 @@ } }, "compactTurnCount": { "type": "number", "default": 6 }, - "recallMaxNodes": { "type": "number", "default": 3 }, + "recallMaxNodes": { "type": "number", "default": 6 }, "recallMaxDepth": { "type": "number", "default": 2 }, - "freshTailCount": { "type": "number", "default": 10 }, + "freshTailCount": { "type": "number", "default": 5, "description": "assemble 转录保留的最近轮数（裁剪窗口）" }, "dedupThreshold": { "type": "number", "default": 0.90 }, "pagerankDamping": { "type": "number", "default": 0.85 }, "pagerankIterations": { "type": "number", "default": 20 }, diff --git a/src/cli-extract.ts b/src/cli-extract.ts index cfe9d18..04a821c 100644 --- a/src/cli-extract.ts +++ b/src/cli-extract.ts @@ -235,6 +235,11 @@ async function extractSessionLoop( const extraction = await extractor.extract({ messages: msgs, existingNames: existing }); const nameToId = new Map(); + // 批内 fire-and-forget 的 syncEmbed 收集到批边界统一 await： + // closeDriver 在 finally 里执行，若不等待，最后一批在途的 embedding + // HTTP 请求会撞上已关闭的 driver 且错误被吞——向量丢失且不可自愈 + // （markExtracted 已执行，重跑 extract 不会补）。 + const pendingEmbeds: Promise[] = []; for (const nc of extraction.nodes) { const { node } = await upsertNode(driver, { type: nc.type, name: nc.name, @@ -242,7 +247,7 @@ async function extractSessionLoop( }, sessionId); nameToId.set(node.name, node.id); stats.nodes += 1; - void recaller.syncEmbed(node).catch(() => {}); + pendingEmbeds.push(recaller.syncEmbed(node).catch(() => {})); } for (const ec of extraction.edges) { @@ -259,6 +264,8 @@ async function extractSessionLoop( } } + await Promise.allSettled(pendingEmbeds); + const maxTurn = msgs.reduce((m, msg) => Math.max(m, msg.turn_index ?? 0), 0); await markExtracted(driver, sessionId, maxTurn); log(` batch ${stats.batches}: ${msgs.length} 消息 -> ${extraction.nodes.length} 节点 / ${extraction.edges.length} 边（累计 ${stats.nodes}/${stats.edges}）`); diff --git a/src/engine/embed.ts b/src/engine/embed.ts index fa98dc7..740a35d 100755 --- a/src/engine/embed.ts +++ b/src/engine/embed.ts @@ -128,8 +128,8 @@ export async function createEmbedFn(cfg: EmbeddingConfig | undefined): Promise => { return callEmbedding(text.slice(0, 8000), mode); }; - } catch (err) { - console.error(`[graph-memory-pro] embedding probe failed:`, err); + } catch { + // probe 失败返回 null（调用方日志已有 "text search mode" 降级提示），不在库代码里写 stdout return null; } } diff --git a/src/engine/llm.ts b/src/engine/llm.ts index c474dc7..2d7cce2 100755 --- a/src/engine/llm.ts +++ b/src/engine/llm.ts @@ -23,6 +23,7 @@ * 超时：AbortController 强制；默认 60s，cfg.llm.timeoutMs 可调（慢速 API 用户可调大）。 */ +import { stat } from "node:fs/promises"; import { loadOAuthSession, needsRefresh, @@ -144,13 +145,19 @@ export function createCompleteFn( // ── OAuth 会话缓存：单飞刷新，避免并发请求同时触发 refresh ── const oauthPath = provider === "oauth" ? llmConfig?.oauthPath : undefined; let cachedSessionPromise: Promise | null = null; + let cachedSessionMtimeMs: number | null = null; let refreshPromise: Promise | null = null; async function getOAuthSession(): Promise { if (!oauthPath) { throw new Error("[graph-memory] provider=oauth 需要 llm.oauthPath"); } - if (!cachedSessionPromise) { + // oauthPath 可能被运行中的其他进程重写（CLI auth login / CLI extract 刷新 token）。 + // 进程内缓存按 mtime 失效：文件更新后下一次调用即重载，无需重启网关。 + let mtimeMs: number | null = null; + try { mtimeMs = (await stat(oauthPath)).mtimeMs; } catch { /* 文件暂不可达：沿用缓存 */ } + if (!cachedSessionPromise || (mtimeMs !== null && mtimeMs !== cachedSessionMtimeMs)) { + cachedSessionMtimeMs = mtimeMs; cachedSessionPromise = loadOAuthSession(oauthPath).catch((error) => { cachedSessionPromise = null; throw error; @@ -163,6 +170,8 @@ export function createCompleteFn( .then(async (s) => { await saveOAuthSession(oauthPath, s); cachedSessionPromise = Promise.resolve(s); + // 同步 mtime 标记，避免下次调用因文件刚写入而多余重载一次 + try { cachedSessionMtimeMs = (await stat(oauthPath)).mtimeMs; } catch {} refreshPromise = null; return s; }) @@ -251,7 +260,7 @@ export function createCompleteFn( ); } const baseURL = (llmConfig?.baseURL ?? ANTHROPIC_DEFAULT_BASE_URL).replace(/\/+$/, ""); - const res = await fetchWithTimeout(`${baseURL}/v1/messages`, { + const res = await fetchRetry(`${baseURL}/v1/messages`, { method: "POST", headers: { "Content-Type": "application/json", @@ -264,13 +273,19 @@ export function createCompleteFn( system, messages: [{ role: "user", content: user }], }), - }, timeoutMs); + }, 3, timeoutMs); if (!res.ok) { const errText = await res.text().catch(() => ""); throw new Error(`[graph-memory] Anthropic API ${res.status}: ${errText.slice(0, 200)}`); } const data = await res.json() as any; - const text = data.content?.[0]?.text; + // 遍历 content 找 text 块：只看 content[0] 时，thinking 块在前会误报 empty content + const text = Array.isArray(data.content) + ? data.content + .filter((b: any) => b?.type === "text" && typeof b.text === "string") + .map((b: any) => b.text) + .join("") + : ""; if (text) return text; const stop = data.choices?.[0]?.finish_reason ?? data.stop_reason; throw new Error( @@ -288,7 +303,7 @@ export function createCompleteFn( ); } const url = `${baseURL.replace(/\/+$/, "")}/chat/completions`; - const res = await fetchWithTimeout(url, { + const res = await fetchRetry(url, { method: "POST", headers: { "Content-Type": "application/json", @@ -303,7 +318,7 @@ export function createCompleteFn( max_tokens: maxTokens, temperature: 0.1, }), - }, timeoutMs); + }, 3, timeoutMs); if (!res.ok) { const errText = await res.text().catch(() => ""); throw new Error(`[graph-memory] LLM API ${res.status}: ${errText.slice(0, 200)}`); diff --git a/src/format/assemble.ts b/src/format/assemble.ts index 00c516d..29838ec 100755 --- a/src/format/assemble.ts +++ b/src/format/assemble.ts @@ -117,13 +117,15 @@ export async function assembleContext( selectedIds.has(e.fromId) && selectedIds.has(e.toId) && !seen.has(e.id) && seen.add(e.id) ); - // 预加载所有需要的社区摘要 - const communityIds = new Set(selected.map(n => n.communityId).filter(Boolean) as string[]); + // 预加载所有需要的社区摘要（并发拉取，避免逐个 await 的串行往返） + const communityIds = Array.from(new Set(selected.map(n => n.communityId).filter(Boolean) as string[])); + const summaries = await Promise.all( + communityIds.map(cid => getCommunitySummary(driver, cid)), + ); const communitySummaries = new Map(); - for (const cid of communityIds) { - const summary = await getCommunitySummary(driver, cid); - if (summary) communitySummaries.set(cid, summary); - } + communityIds.forEach((cid, i) => { + if (summaries[i]) communitySummaries.set(cid, summaries[i]!); + }); // 按社区分组 const byCommunity = new Map(); diff --git a/src/graph/community.ts b/src/graph/community.ts index 71b021a..af76335 100755 --- a/src/graph/community.ts +++ b/src/graph/community.ts @@ -155,7 +155,6 @@ export async function summarizeCommunities( llm: CompleteFn, embedFn?: EmbedFn, ): Promise { - await pruneCommunitySummaries(driver); let generated = 0; for (const [communityId, memberIds] of communities) { @@ -233,10 +232,15 @@ export async function summarizeCommunities( await upsertCommunitySummary(driver, communityId, cleaned, memberIds.length, embedding, memberSignature); generated++; - } catch (err) { - console.log(` [WARN] community summary failed for ${communityId}: ${err}`); + } catch { + // 单社区摘要失败静默跳过（与 syncEmbed 的吞错策略一致）——库代码不直接写 stdout } } + // prune 必须在复用查找之后：detectCommunities 每轮按成员数重编号 c-1..c-N， + // 旧 id 社区（summary/memberSignature/embedding 的持有者）在新编号下"无人引用"， + // 先 prune 会把捐赠者删掉，签名复用永远不生效 → 每轮维护全量重算 LLM 摘要。 + await pruneCommunitySummaries(driver); + return generated; } diff --git a/src/graph/decay.ts b/src/graph/decay.ts index 9859c88..0f3b2dd 100644 --- a/src/graph/decay.ts +++ b/src/graph/decay.ts @@ -219,7 +219,8 @@ export async function applyDecay(driver: Driver, cfg: Pick): return { enabled: true, scanned: 0, tierTransitions: { ...EMPTY_TRANSITIONS }, durationMs: 0 }; } - const maxPagerank = Math.max(...nodes.map(n => n.pagerank), 0.0001); + // reduce 而非 Math.max(...map)：spread 在超大节点集（>10 万）会爆调用栈 + const maxPagerank = nodes.reduce((m, n) => Math.max(m, n.pagerank), 0.0001); const updates: Array<{ id: string; tier: NodeTier; composite: number; tierChanged: boolean }> = []; const transitions: TierTransition = { ...EMPTY_TRANSITIONS }; diff --git a/src/graph/dedup.ts b/src/graph/dedup.ts index 9fac7b5..94e1f4b 100755 --- a/src/graph/dedup.ts +++ b/src/graph/dedup.ts @@ -39,36 +39,37 @@ export async function detectDuplicates(driver: Driver, cfg: GmConfig): Promise ({ + id: r.get("id"), + name: r.get("name"), + embedding: r.get("embedding"), + })); + const searchResult = await session.run(` + UNWIND $nodes AS n + CALL db.index.vector.queryNodes('gm_node_embedding', 5, n.embedding) YIELD node, score + WHERE node.id <> n.id AND node.status = 'active' AND score >= $threshold + RETURN n.id AS nodeA, n.name AS nameA, node.id AS nodeB, node.name AS nameB, score AS similarity + `, { nodes, threshold: cfg.dedupThreshold }); + const pairs: DuplicatePair[] = []; const seenPairs = new Set(); - // 对每个节点做向量搜索 - for (const record of nodesResult.records) { - const nodeId = record.get("id"); - const nodeName = record.get("name"); - const embedding = record.get("embedding"); - - const searchResult = await session.run(` - CALL db.index.vector.queryNodes('gm_node_embedding', 5, $vec) - YIELD node, score - WHERE node.id <> $nodeId AND node.status = 'active' AND score >= $threshold - RETURN node.id AS id, node.name AS name, score - `, { vec: embedding, nodeId, threshold: cfg.dedupThreshold }); - - for (const sr of searchResult.records) { - const otherId = sr.get("id"); - const pairKey = [nodeId, otherId].sort().join("|"); - if (seenPairs.has(pairKey)) continue; - seenPairs.add(pairKey); - - pairs.push({ - nodeA: nodeId, - nodeB: otherId, - nameA: nodeName, - nameB: sr.get("name"), - similarity: sr.get("score"), - }); - } + for (const sr of searchResult.records) { + const nodeId = sr.get("nodeA"); + const otherId = sr.get("nodeB"); + const pairKey = [nodeId, otherId].sort().join("|"); + if (seenPairs.has(pairKey)) continue; + seenPairs.add(pairKey); + + pairs.push({ + nodeA: nodeId, + nodeB: otherId, + nameA: sr.get("nameA"), + nameB: sr.get("nameB"), + similarity: sr.get("similarity"), + }); } return pairs.sort((a, b) => b.similarity - a.similarity); diff --git a/src/graph/pagerank.ts b/src/graph/pagerank.ts index 2cb7030..a39edd6 100755 --- a/src/graph/pagerank.ts +++ b/src/graph/pagerank.ts @@ -122,30 +122,40 @@ export async function computeGlobalPageRank(driver: Driver, cfg: GmConfig): Prom await projectActiveGraph(session, graphName, existingTypes); - await session.run(` - CALL gds.pageRank.write('${graphName}', { - writeProperty: 'pagerank', - dampingFactor: $damping, - maxIterations: toInteger($iterations) - }) - `, { damping: cfg.pagerankDamping, iterations: cfg.pagerankIterations }); - - await session.run(`CALL gds.graph.drop('${graphName}')`); - - const topResult = await session.run(` - MATCH (n:Task|Skill|Event {status: 'active'}) RETURN n.id AS id, n.name AS name, n.pagerank AS score - ORDER BY n.pagerank DESC LIMIT 20 - `); + try { + await session.run(` + CALL gds.pageRank.write('${graphName}', { + writeProperty: 'pagerank', + dampingFactor: $damping, + maxIterations: toInteger($iterations) + }) + `, { damping: cfg.pagerankDamping, iterations: cfg.pagerankIterations }); + } finally { + // drop 失败只能吞掉（宁泄漏一个临时投影）：此分支若抛错落入外层 catch， + // fallback 会用 1/(i+1) 覆盖刚 write 成功的真实 PageRank —— 数据损坏远重于投影泄漏 + try { await session.run(`CALL gds.graph.drop('${graphName}')`); } catch {} + } - const scores = new Map(); - const topK: Array<{ id: string; name: string; score: number }> = []; - for (const r of topResult.records) { - const rawScore = r.get("score"); - const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); - scores.set(r.get("id"), score); - topK.push({ id: r.get("id"), name: r.get("name"), score }); + // write 已成功：pagerank 属性已是真值，后续读取失败只返回空排序， + // 绝不回落外层 catch 的 fallback（那会覆盖全图正确分数） + try { + const topResult = await session.run(` + MATCH (n:Task|Skill|Event {status: 'active'}) RETURN n.id AS id, n.name AS name, n.pagerank AS score + ORDER BY n.pagerank DESC LIMIT 20 + `); + + const scores = new Map(); + const topK: Array<{ id: string; name: string; score: number }> = []; + for (const r of topResult.records) { + const rawScore = r.get("score"); + const score = typeof rawScore === "number" ? rawScore : (rawScore?.toNumber?.() ?? 0); + scores.set(r.get("id"), score); + topK.push({ id: r.get("id"), name: r.get("name"), score }); + } + return { scores, topK }; + } catch { + return { scores: new Map(), topK: [] }; } - return { scores, topK }; } catch { try { await session.run(`CALL gds.graph.drop('${graphName}')`); } catch {} // GDS 不可用时降级为确定性 fallback（与 PPR 一致：按稳定排序赋 1/(i+1)） diff --git a/src/recaller/recall.ts b/src/recaller/recall.ts index e7fa6f3..1c1f515 100755 --- a/src/recaller/recall.ts +++ b/src/recaller/recall.ts @@ -98,8 +98,17 @@ export class Recaller { const limit = this.cfg.recallMaxNodes; const timeRange = options ? parseTimeRange(options) : null; - const precise = await this.recallPrecise(query, limit, timeRange); - const generalized = await this.recallGeneralized(query, limit, timeRange); + // query 向量只算一次，两条路径共享；失败统一落 null（各路径走文本兜底）。 + // 双路径并行执行 —— 原串行 + 各自 embed 会把 2 次调用放大成 4 次 API 调用 + // 与 4 次图遍历，全部压在调用方的预算窗口内。 + const embedPromise: Promise = this.embed + ? this.embed(query, "query").catch(() => null) + : Promise.resolve(null); + + const [precise, generalized] = await Promise.all([ + this.recallPrecise(query, limit, timeRange, embedPromise), + this.recallGeneralized(limit, timeRange, embedPromise), + ]); const merged = this.mergeResults(precise, generalized); return merged; @@ -112,12 +121,13 @@ export class Recaller { query: string, limit: number, timeRange: ParsedTimeRange | null, + embedPromise: Promise, ): Promise { let seeds: GmNode[] = []; - if (this.embed) { + const vec = await embedPromise; + if (vec) { try { - const vec = await this.embed(query, "query"); const scored = await vectorSearchWithScore(this.driver, vec, Math.ceil(limit / 2)); seeds = scored.map(s => s.node); @@ -180,15 +190,15 @@ export class Recaller { * 泛化召回：社区向量搜索 → 图遍历 → PPR 排序 */ private async recallGeneralized( - query: string, limit: number, timeRange: ParsedTimeRange | null, + embedPromise: Promise, ): Promise { let seeds: GmNode[] = []; - if (this.embed) { + const vec = await embedPromise; + if (vec) { try { - const vec = await this.embed(query, "query"); const scoredCommunities = await communityVectorSearch(this.driver, vec); if (scoredCommunities.length > 0) { diff --git a/src/routes/crud.ts b/src/routes/crud.ts index c837ea3..9d4b961 100644 --- a/src/routes/crud.ts +++ b/src/routes/crud.ts @@ -16,11 +16,11 @@ import type { Driver } from "neo4j-driver"; import type { OpenClawPluginApi } from "openclaw/plugin-sdk"; import type { Recaller } from "../recaller/recall.ts"; import type { NodeType, EdgeType } from "../types.ts"; -import { NODE_TYPE_TO_LABEL, isValidEdgeDirection } from "../types.ts"; +import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_DIRECTION_RULES, EDGE_TYPES } from "../types.ts"; import { - upsertNode, findById, allActiveNodes, allEdges, + upsertNode, findById, findByName, allActiveNodes, allEdges, upsertEdge, edgesFrom, edgesTo, deprecate, mergeNodes, - searchNodes, getStats, + searchNodes, getStats, normalizeName, } from "../store/store.ts"; import { getSession } from "../store/db.ts"; @@ -262,12 +262,20 @@ async function handleUpdateNode( params.content = body.content as string; } if (body.name !== undefined) { - // Name change — normalize - const newName = (body.name as string).trim().toLowerCase() - .replace(/[\s_]+/g, "-") - .replace(/[^a-z0-9\u4e00-\u9fff\-]/g, "") - .replace(/-{2,}/g, "-") - .replace(/^-|-$/g, ""); + // 与 store/extractor 的 normalizeName 同源（受 normalize-name.test.ts 跨文件一致性保护） + const newName = normalizeName(body.name as string); + if (!newName) { + json(res, 400, { error: "name normalizes to empty string" }); + return true; + } + if (newName !== existing.name) { + // 重名预检：撞 *_name 唯一约束会让裸 session.run 抛成 500，提前回 409 + const conflict = await findByName(driver, newName); + if (conflict && conflict.id !== existing.id) { + json(res, 409, { error: `Node name already exists: ${newName}` }); + return true; + } + } updates.push("n.name = $newName"); params.newName = newName; } @@ -287,12 +295,40 @@ async function handleUpdateNode( if (updates.length > 1) { // always has updatedAt const session = getSession(driver); try { - await session.run( - `MATCH (n:Task|Skill|Event {id: $id}) - SET ${updates.join(", ")} - ${newType ? `REMOVE n:Task, n:Skill, n:Event SET n:${NODE_TYPE_TO_LABEL[newType]}` : ""}`, - params, - ); + // 单事务：改 type 与非法边清理必须原子 —— 若 DELETE 瞬时失败而 SET 已提交， + // 节点会停留在"新 type + 违反白名单的存量边"状态（正是本端点要修复的不变量） + await session.executeWrite(async tx => { + await tx.run( + `MATCH (n:Task|Skill|Event {id: $id}) + SET ${updates.join(", ")} + ${newType ? `REMOVE n:Task, n:Skill, n:Event SET n:${NODE_TYPE_TO_LABEL[newType]}` : ""}`, + params, + ); + + if (newType) { + // 清理方向白名单外的存量边（gm_* 工具链不允许改 type，仅此端点允许—— + // 必须自己恢复图谱不变量）。合法谓词从 EDGE_DIRECTION_RULES 生成（与 + // isValidEdgeDirection 同一事实源），出/入边各一条定向查询——无向匹配无法区分 + // source/target，from/to 集合不对称时会误删合法边。 + const legal = EDGE_TYPES + .map(t => { + const rule = EDGE_DIRECTION_RULES[t]; + return `(type(r) = '${t}' AND source.type IN ${JSON.stringify(rule.from)} AND target.type IN ${JSON.stringify(rule.to)})`; + }) + .join(" OR "); + const types = JSON.stringify([...EDGE_TYPES]); + await tx.run(` + MATCH (source:Task|Skill|Event {id: $id})-[r]->(target:Task|Skill|Event) + WHERE type(r) IN ${types} AND NOT (${legal}) + DELETE r + `, { id }); + await tx.run(` + MATCH (source:Task|Skill|Event)-[r]->(target:Task|Skill|Event {id: $id}) + WHERE type(r) IN ${types} AND NOT (${legal}) + DELETE r + `, { id }); + } + }); } finally { await session.close(); } diff --git a/src/store/db.ts b/src/store/db.ts index 01631f8..e5a43be 100755 --- a/src/store/db.ts +++ b/src/store/db.ts @@ -1,23 +1,19 @@ /** * graph-memory-pro — Neo4j 连接管理（加固版） * - * 解决 "Pool is closed" 问题： * - driver 是长生命周期单例，不在 dispose 时关闭 - * - getSession 在 driver 被意外关闭时自动重建 + * - getSession 永远优先模块级单例（见函数注释） */ import neo4j, { type Driver, type Session } from "neo4j-driver"; import type { EmbeddingConfig, Neo4jConfig } from "../types.ts"; let _driver: Driver | null = null; -let _cfg: Neo4jConfig | null = null; /** * 获取 Neo4j Driver 单例 - * 保存配置，支持自动重连 */ export function getDriver(cfg: Neo4jConfig): Driver { - _cfg = cfg; if (_driver) return _driver; _driver = neo4j.driver(cfg.uri, neo4j.auth.basic(cfg.user, cfg.password), { maxConnectionPoolSize: 50, @@ -30,25 +26,16 @@ export function getDriver(cfg: Neo4jConfig): Driver { /** * 获取一个 Session（用完必须 close） - * 如果 driver 被关闭了，自动用保存的配置重建 + * + * 永远优先模块级 _driver 单例：调用方（register() 启动时捕获一次并四处传递） + * 持有的旧引用在单例重建后会指向已关闭的池。入参仅作 getDriver 未初始化时 + * 的回退兼容。 + * 注：neo4j-driver 5.x 的 driver.session() 构造阶段不抛错，"Pool is closed" + * 在 session.run() 才报——掉线恢复由 gate 熔断 + 驱动自身连接池重连负责， + * 这里不做（也做不了）session 级重连。 */ -export function getSession(driver: Driver): Session { - try { - return driver.session({ database: "neo4j" }); - } catch (err) { - // Pool is closed — 尝试重建 driver - if (_cfg && String(err).includes("closed")) { - console.log("[graph-memory-pro] reconnecting Neo4j driver..."); - _driver = neo4j.driver(_cfg.uri, neo4j.auth.basic(_cfg.user, _cfg.password), { - maxConnectionPoolSize: 50, - // 与 getDriver 保持一致：快速失败，让熔断门控尽早接手 - connectionAcquisitionTimeout: 15_000, - maxTransactionRetryTime: 10_000, - }); - return _driver.session({ database: "neo4j" }); - } - throw err; - } +export function getSession(passedDriver: Driver): Session { + return (_driver ?? passedDriver).session({ database: "neo4j" }); } /** diff --git a/src/store/store.ts b/src/store/store.ts index b5d6f6a..0538bbc 100755 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -9,7 +9,7 @@ import type { Driver } from "neo4j-driver"; import neo4j from "neo4j-driver"; import { createHash } from "crypto"; import type { GmNode, GmEdge, EdgeType, NodeType, NodeTier } from "../types.ts"; -import { NODE_TYPE_TO_LABEL, isValidEdgeDirection } from "../types.ts"; +import { NODE_TYPE_TO_LABEL, isValidEdgeDirection, EDGE_TYPES } from "../types.ts"; import { getSession } from "./db.ts"; /** Neo4j LIMIT/索引参数必须是 Integer */ @@ -141,6 +141,12 @@ export async function allEdges(driver: Driver): Promise { } } +/** 判断错误是否为 *_name 唯一约束冲突（CREATE 撞上并发创建时用于幂等回退） */ +function isNameConstraintViolation(err: unknown): boolean { + const s = String(err); + return s.includes("ConstraintValidationFailed") || s.includes("already exists with label"); +} + export async function upsertNode( driver: Driver, c: { type: NodeType; name: string; description: string; content: string }, @@ -150,6 +156,35 @@ export async function upsertNode( const label = NODE_TYPE_TO_LABEL[c.type as NodeType]; if (!label) throw new Error(`[graph-memory-pro] Invalid node type: ${String(c.type)}`); const session = getSession(driver); + /** 按 name 更新已存在节点（find 命中与撞约束回退两条路径共用） */ + const updateExisting = async (): Promise<{ node: GmNode; isNew: boolean }> => { + await session.run(` + MATCH (n:Task|Skill|Event {name: $name}) + SET n.content = CASE WHEN size($content) > size(n.content) THEN $content ELSE n.content END, + n.description = CASE WHEN size($description) > size(n.description) THEN $description ELSE n.description END, + n.validatedCount = n.validatedCount + 1, + n.sourceSessions = CASE + WHEN NOT $sessionId IN n.sourceSessions + THEN n.sourceSessions + $sessionId + ELSE n.sourceSessions + END, + n.lastAccessedAt = $now, + n.updatedAt = $now + RETURN n + `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); + + const updated = await session.run( + "MATCH (n:Task|Skill|Event {name: $name}) RETURN n", + { name }, + ); + // 撞约束回退路径存在窄窗口：并发创建的同名节点可能在 MATCH 前被删除 + //（如 maintenance mergeNodes）——守卫让单个节点失败而不是 TypeError 炸整批 + const record = updated.records[0]?.get("n"); + if (!record) { + throw new Error(`[graph-memory-pro] upsertNode: node "${name}" disappeared during update`); + } + return { node: toNode(record), isNew: false }; + }; try { // Try to find existing node with this name across all knowledge labels const existing = await session.run( @@ -158,29 +193,12 @@ export async function upsertNode( ); if (existing.records.length > 0) { - // Update existing node - await session.run(` - MATCH (n:Task|Skill|Event {name: $name}) - SET n.content = CASE WHEN size($content) > size(n.content) THEN $content ELSE n.content END, - n.description = CASE WHEN size($description) > size(n.description) THEN $description ELSE n.description END, - n.validatedCount = n.validatedCount + 1, - n.sourceSessions = CASE - WHEN NOT $sessionId IN n.sourceSessions - THEN n.sourceSessions + $sessionId - ELSE n.sourceSessions - END, - n.lastAccessedAt = $now, - n.updatedAt = $now - RETURN n - `, { name, content: c.content, description: c.description, sessionId, now: Date.now() }); - - const updated = await session.run( - "MATCH (n:Task|Skill|Event {name: $name}) RETURN n", - { name }, - ); - return { node: toNode(updated.records[0].get("n")), isNew: false }; - } else { - // Create new node with specific label + // 必须 return await：否则 finally 的 session.close() 会与闭包内的 + // 第二次 session.run 竞态（closed session 错误） + return await updateExisting(); + } + // Create new node with specific label + try { const now = Date.now(); const result = await session.run(` CREATE (n:MemoryNode:${label} { @@ -198,6 +216,11 @@ export async function upsertNode( sessions: [sessionId], now, }); return { node: toNode(result.records[0].get("n")), isNew: true }; + } catch (err) { + // find-then-create 窗口内并发路径抢先创建了同名节点（撞 *_name 唯一约束） + // → 退回更新路径保持幂等，而不是让整轮提取失败重试 + if (isNameConstraintViolation(err)) return await updateExisting(); + throw err; } } finally { await session.close(); @@ -458,42 +481,25 @@ export async function upsertEdge( const toType = endpoints.records[0].get("toType"); if (!isValidEdgeDirection(e.type, fromType, toType)) return false; - // 检查是否已存在同 from+to+type 的边 - const existing = await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - WHERE type(r) = $type - RETURN r - `, { fromId: e.fromId, toId: e.toId, type: e.type }); - - if (existing.records.length > 0) { - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId})-[r]->(b:Task|Skill|Event {id: $toId}) - WHERE type(r) = $type - SET r.instruction = $instruction - `, { fromId: e.fromId, toId: e.toId, type: e.type, instruction: e.instruction }); - } else { - // 用 APOC 动态创建关系（type 是变量） - await session.run(` - MATCH (a:Task|Skill|Event {id: $fromId}), (b:Task|Skill|Event {id: $toId}) - CALL apoc.create.relationship(a, $type, { - id: $id, - instruction: $instruction, - condition: $condition, - sessionId: $sessionId, - createdAt: $now - }, b) YIELD rel - RETURN rel - `, { - fromId: e.fromId, - toId: e.toId, - type: e.type, - id: uid("e"), - instruction: e.instruction, - condition: e.condition ?? null, - sessionId: e.sessionId, - now: Date.now(), - }); - } + // MERGE 语义：查重 + 创建/更新合并为单条原子语句，消除并发下绕过查重产生重复边的窗口。 + // onCreate 写入全部属性；onMatch 仅刷新 instruction（与原查重-更新分支行为一致）。 + await session.run(` + MATCH (a:Task|Skill|Event {id: $fromId}), (b:Task|Skill|Event {id: $toId}) + CALL apoc.merge.relationship(a, $type, {}, { + id: $id, instruction: $instruction, condition: $condition, + sessionId: $sessionId, createdAt: $now + }, b, { instruction: $instruction }) YIELD rel + RETURN rel + `, { + fromId: e.fromId, + toId: e.toId, + type: e.type, + id: uid("e"), + instruction: e.instruction, + condition: e.condition ?? null, + sessionId: e.sessionId, + now: Date.now(), + }); return true; } finally { await session.close(); @@ -550,6 +556,34 @@ export async function edgesTo(driver: Driver, id: string): Promise { } } +/** 批量查询至少一端在 ids 内的知识边 —— 一次往返替代逐节点 edgesFrom+edgesTo 的 2N 次往返。 */ +export async function edgesTouching(driver: Driver, ids: string[]): Promise { + if (!ids.length) return []; + const session = getSession(driver); + try { + const result = await session.run(` + MATCH (a:Task|Skill|Event)-[r]->(b:Task|Skill|Event) + WHERE (a.id IN $ids OR b.id IN $ids) + AND type(r) IN ${JSON.stringify([...EDGE_TYPES])} + RETURN r.id AS id, a.id AS fromId, b.id AS toId, type(r) AS type, + r.instruction AS instruction, r.condition AS condition, + r.sessionId AS sessionId, r.createdAt AS createdAt + `, { ids }); + return result.records.map(r => ({ + id: r.get("id"), + fromId: r.get("fromId"), + toId: r.get("toId"), + type: r.get("type") as EdgeType, + instruction: r.get("instruction"), + condition: r.get("condition") ?? undefined, + sessionId: r.get("sessionId"), + createdAt: toInt(r.get("createdAt")), + })); + } finally { + await session.close(); + } +} + /** 删除 from→to 之间的边；type 省略时删除所有类型。返回删除条数。 */ export async function deleteEdges( driver: Driver, @@ -743,6 +777,10 @@ export async function graphWalk( ): Promise<{ nodes: GmNode[]; edges: GmEdge[] }> { if (!seedIds.length) return { nodes: [], edges: [] }; + // maxDepth 来自配置且直接内插进 Cypher —— clamp 到 [1,4]，非法值只会得到安全深度而非语法错误 + const parsedDepth = Number(maxDepth); + const depth = Math.max(1, Math.min(4, Number.isFinite(parsedDepth) ? Math.floor(parsedDepth) : 2)); + const session = getSession(driver); try { // 用 Neo4j 的变长路径匹配做图遍历 @@ -751,7 +789,7 @@ export async function graphWalk( WHERE seed.id IN $seedIds AND seed.status = 'active' CALL { WITH seed - MATCH path = (seed)-[*0..${maxDepth}]-(neighbor:Task|Skill|Event {status: 'active'}) + MATCH path = (seed)-[*0..${depth}]-(neighbor:Task|Skill|Event {status: 'active'}) WHERE all(node IN nodes(path) WHERE node.status = 'active') RETURN DISTINCT neighbor } @@ -807,7 +845,16 @@ export async function getBySession(driver: Driver, sessionId: string): Promise { +/** + * 每社区取最近更新的 perCommunity 个代表节点。 + * totalLimit 封顶总返回数（按社区规模降序截断）—— recall 兜底路径用它做 + * graphWalk 种子，社区很多时无上限种子会把遍历放大成全图扫描。 + */ +export async function communityRepresentatives( + driver: Driver, + perCommunity = 2, + totalLimit = 20, +): Promise { const session = getSession(driver); try { const result = await session.run(` @@ -816,9 +863,11 @@ export async function communityRepresentatives(driver: Driver, perCommunity = 2) WITH n.communityId AS cid, n ORDER BY n.updatedAt DESC WITH cid, collect(n) AS members + ORDER BY size(members) DESC UNWIND members[0..toInteger($perCommunity)] AS m RETURN m AS n - `, { perCommunity }); + LIMIT toInteger($totalLimit) + `, { perCommunity, totalLimit }); return result.records.map(r => toNode(r.get("n"))); } finally { await session.close(); diff --git a/src/types.ts b/src/types.ts index ce30fca..a264a1f 100755 --- a/src/types.ts +++ b/src/types.ts @@ -65,7 +65,7 @@ export const EDGE_TYPES = [ export type EdgeType = (typeof EDGE_TYPES)[number]; -const EDGE_DIRECTION_RULES: Record = { @@ -214,6 +214,7 @@ export interface GmConfig { compactTurnCount: number; recallMaxNodes: number; recallMaxDepth: number; + /** assemble 保留的最近轮数（裁剪窗口）；默认 5，与 sliceLastTurn 的回退值一致。 */ freshTailCount: number; embedding?: EmbeddingConfig; llm?: { @@ -247,7 +248,7 @@ export const DEFAULT_CONFIG: GmConfig = { compactTurnCount: 6, recallMaxNodes: 6, recallMaxDepth: 2, - freshTailCount: 10, + freshTailCount: 5, dedupThreshold: 0.90, pagerankDamping: 0.85, pagerankIterations: 20, diff --git a/test/integration.graph.test.ts b/test/integration.graph.test.ts index 8b52d10..61cc0ca 100644 --- a/test/integration.graph.test.ts +++ b/test/integration.graph.test.ts @@ -221,7 +221,19 @@ describe.skipIf(!ENABLED)("graph layer integration (GDS, Docker)", () => { expect(summary?.summary).toBe("容器部署与编排技能"); expect(summary?.memberSignature).toBe(buildCommunityMemberSignature(memberIds)); - // detectCommunities 每轮按成员数重编号（c-1..c-N），ID 变但成员相同 → 按签名跨社区复用 + // detectCommunities 每轮按成员数重编号（c-1..c-N），ID 变但成员相同 → 按签名跨社区复用。 + // 生产链路里 updateCommunities 会先把成员 communityId 改写到新 id 再进 summarize —— + // 这里同样 SET 成员指向新 id（保持生产不变量），旧 id 成为"无人引用"的捐赠者， + // 复用查找发生在 prune 之前，捐赠者复制完摘要后才被 prune 清理。 + const renumber = getSession(driver); + try { + await renumber.run( + "MATCH (n:MemoryNode) WHERE n.id IN $ids SET n.communityId = $cid", + { ids: memberIds, cid: "c-reuse-renumbered" }, + ); + } finally { + await renumber.close(); + } const third = await summarizeCommunities( driver, new Map([["c-reuse-renumbered", memberIds]]), llm, ); diff --git a/test/integration.recall.test.ts b/test/integration.recall.test.ts index 1cf5390..6bb51b5 100644 --- a/test/integration.recall.test.ts +++ b/test/integration.recall.test.ts @@ -17,6 +17,7 @@ import { Recaller, buildNodeEmbeddingText } from "../src/recaller/recall.ts"; import { DEFAULT_CONFIG, type GmConfig } from "../src/types.ts"; const ENABLED = !!process.env.NEO4J_INTEGRATION; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? "bolt://localhost:7687"; let driver: Driver; const TEST_SID = `recall-${Date.now()}`; @@ -24,7 +25,7 @@ const cfg: GmConfig = { ...DEFAULT_CONFIG, recallMaxNodes: 5, recallMaxDepth: 2 describe.skipIf(!ENABLED)("Recaller integration", () => { beforeAll(async () => { - driver = getDriver({ uri: "bolt://localhost:7687", user: "neo4j", password: "graphmemory" }); + driver = getDriver({ uri: NEO4J_URI, user: "neo4j", password: "graphmemory" }); await initSchema(driver); // 构造可被关键词召回的图 diff --git a/test/integration.routes.test.ts b/test/integration.routes.test.ts index 9337726..5077ad1 100644 --- a/test/integration.routes.test.ts +++ b/test/integration.routes.test.ts @@ -6,6 +6,7 @@ import { closeDriver, getDriver, getSession, initSchema } from "../src/store/db. import { findById, upsertNode } from "../src/store/store.ts"; const ENABLED = !!process.env.NEO4J_INTEGRATION; +const NEO4J_URI = process.env.NEO4J_TEST_URI ?? "bolt://localhost:7687"; const TEST_SID = `routes-${Date.now()}`; let driver: Driver; @@ -30,7 +31,7 @@ async function request(method: string, path: string, body?: Record { beforeAll(async () => { - driver = getDriver({ uri: "bolt://localhost:7687", user: "neo4j", password: "graphmemory" }); + driver = getDriver({ uri: NEO4J_URI, user: "neo4j", password: "graphmemory" }); await initSchema(driver); const api = { @@ -109,4 +110,50 @@ describe.skipIf(!ENABLED)("CRUD route integration", () => { }); expect(response.status).toBe(400); }); + + it("drops direction-violating edges when the node type changes (TASK→EVENT)", async () => { + const { node: skill } = await upsertNode(driver, { + type: "SKILL", name: "route-typechange-skill", description: "d", content: "c", + }, TEST_SID); + const { node: task } = await upsertNode(driver, { + type: "TASK", name: "route-typechange-task", description: "d", content: "c", + }, TEST_SID); + + // TASK→SKILL 的 USED_SKILL 合法；节点改成 EVENT 后 USED_SKILL 出边违反白名单 + const created = await request("POST", "edges", { + fromId: task.id, + toId: skill.id, + type: "USED_SKILL", + instruction: "legal before type change", + }); + expect(created.status).toBe(201); + + const changed = await request("PUT", `nodes?id=${task.id}`, { type: "EVENT" }); + expect(changed.status).toBe(200); + + const session = getSession(driver); + try { + const outEdges = await session.run( + "MATCH (n {id: $id})-[r]->() RETURN type(r) AS type", { id: task.id }, + ); + expect(outEdges.records).toHaveLength(0); + } finally { + await session.close(); + } + }); + + it("rejects renaming a node to an existing name with 409", async () => { + const { node: keeper } = await upsertNode(driver, { + type: "SKILL", name: "route-name-keeper", description: "d", content: "c", + }, TEST_SID); + const { node: victim } = await upsertNode(driver, { + type: "TASK", name: "route-name-victim", description: "d", content: "c", + }, TEST_SID); + + const response = await request("PUT", `nodes?id=${victim.id}`, { name: keeper.name }); + expect(response.status).toBe(409); + // 自身同名（标准化后未变）不算冲突 + const selfRename = await request("PUT", `nodes?id=${victim.id}`, { name: "route-name-victim" }); + expect(selfRename.status).toBe(200); + }); }); diff --git a/test/session-identity.test.ts b/test/session-identity.test.ts index 3253762..4068ec3 100644 --- a/test/session-identity.test.ts +++ b/test/session-identity.test.ts @@ -44,6 +44,7 @@ vi.mock("../src/store/store.ts", () => ({ getBySession: mocks.getBySession, edgesFrom: async () => [], edgesTo: async () => [], + edgesTouching: async () => [], deprecate: async () => {}, getStats: async () => ({}), }));