From bcbea147cad8d525a56e53d5a4fc93eeab69f8fb Mon Sep 17 00:00:00 2001 From: rjckkkkk <59609580+rjckkkkk@users.noreply.github.com> Date: Thu, 9 Jul 2026 16:32:41 +0800 Subject: [PATCH] feat: add catalog overlay diagnostics --- AGENTS.md | 4 +- cmd/aima/deploy_failure.go | 129 ++++++++++++++++++++++++++++++ cmd/aima/main.go | 27 ++++--- cmd/aima/main_test.go | 42 ++++++++++ cmd/aima/tooldeps_knowledge.go | 111 +++++++++++++++++++++++++ docs/cli.md | 3 + docs/knowledge.md | 45 ++++++++++- docs/mcp.md | 9 ++- internal/cli/catalog.go | 89 +++++++++++++++++++++ internal/cli/cli_test.go | 50 +++++++++++- internal/knowledge/loader.go | 28 +++++++ internal/knowledge/loader_test.go | 75 +++++++++++++++++ internal/mcp/mcp_test.go | 73 ++++++++++++++++- internal/mcp/tools.go | 2 +- internal/mcp/tools_catalog.go | 80 ++++++++++++++++++ internal/mcp/tools_deps.go | 9 ++- 16 files changed, 753 insertions(+), 23 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7931489c..4516dcd7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,8 +182,8 @@ catalog/ # Knowledge assets (go:embed, compiled in) models/ # Model Asset YAML partitions/ # Partition Strategy YAML stack/ # Stack Component YAML (K3S, HAMi — install config + airgap sources) -# Runtime overlay: ~/.aima/catalog/{hardware,engines,models,partitions,stack}/*.yaml -# Same metadata.name overrides go:embed, new names append. No recompilation needed. +# Runtime overlay: /catalog/{central,user}/{hardware,engines,models,partitions,stack,scenarios}/*.patch.yaml +# Files are *_patch YAML merged as factory -> central -> user. No recompilation needed. ``` ## Key Commands diff --git a/cmd/aima/deploy_failure.go b/cmd/aima/deploy_failure.go index ea22826d..04dc5b41 100644 --- a/cmd/aima/deploy_failure.go +++ b/cmd/aima/deploy_failure.go @@ -3,10 +3,139 @@ package main import ( "context" "encoding/json" + "fmt" "strings" "time" ) +type deploymentErrorCode string + +const ( + deployErrorUnknown deploymentErrorCode = "UNKNOWN_ERROR" + deployErrorOutOfMemory deploymentErrorCode = "OUT_OF_MEMORY" + deployErrorModelNotFound deploymentErrorCode = "MODEL_NOT_FOUND" + deployErrorModelCorrupted deploymentErrorCode = "MODEL_CORRUPTED" + deployErrorModelFormatInvalid deploymentErrorCode = "MODEL_FORMAT_INVALID" + deployErrorPortInUse deploymentErrorCode = "PORT_IN_USE" + deployErrorPermissionDenied deploymentErrorCode = "PERMISSION_DENIED" + deployErrorDownloadFailed deploymentErrorCode = "DOWNLOAD_FAILED" + deployErrorHardwareIncompatible deploymentErrorCode = "HARDWARE_INCOMPATIBLE" + deployErrorTimeout deploymentErrorCode = "TIMEOUT" + deployErrorEngineStartFailed deploymentErrorCode = "ENGINE_START_FAILED" +) + +type deploymentCleanupResult struct { + Attempted bool `json:"attempted"` + Succeeded bool `json:"succeeded"` + Message string `json:"message"` +} + +type deploymentRunError struct { + Code deploymentErrorCode + Message string + Cleanup deploymentCleanupResult +} + +func (e deploymentRunError) Error() string { + msg := fmt.Sprintf("%s: %s", e.Code, e.Message) + if e.Cleanup.Message != "" { + msg += "; cleanup: " + e.Cleanup.Message + } + return msg +} + +func newDeploymentRunError(code deploymentErrorCode, message string, cleanup deploymentCleanupResult) error { + if code == "" { + code = deployErrorUnknown + } + return deploymentRunError{Code: code, Message: strings.TrimSpace(message), Cleanup: cleanup} +} + +func cleanupFailedDeployment(ctx context.Context, deployName string, deleteFn func(context.Context, string) error) deploymentCleanupResult { + deployName = strings.TrimSpace(deployName) + if deployName == "" { + return deploymentCleanupResult{Message: "deployment name unavailable"} + } + if deleteFn == nil { + return deploymentCleanupResult{Message: "deploy.delete unavailable"} + } + if err := deleteFn(ctx, deployName); err != nil { + return deploymentCleanupResult{ + Attempted: true, + Succeeded: false, + Message: "delete failed deployment " + deployName + ": " + err.Error(), + } + } + return deploymentCleanupResult{ + Attempted: true, + Succeeded: true, + Message: "deleted failed deployment " + deployName, + } +} + +func classifyDeploymentFailure(message string) deploymentErrorCode { + lower := strings.ToLower(strings.TrimSpace(message)) + switch { + case lower == "": + return deployErrorUnknown + case strings.Contains(lower, "outofmemoryerror"), + strings.Contains(lower, "out of memory"), + strings.Contains(lower, "oom"), + strings.Contains(lower, "cuda error: out of memory"), + strings.Contains(lower, "hip out of memory"): + return deployErrorOutOfMemory + case strings.Contains(lower, "address already in use"), + strings.Contains(lower, "bind: address"), + strings.Contains(lower, "port is in use"), + strings.Contains(lower, "port already allocated"): + return deployErrorPortInUse + case strings.Contains(lower, "permission denied"), + strings.Contains(lower, "operation not permitted"), + strings.Contains(lower, "access is denied"): + return deployErrorPermissionDenied + case strings.Contains(lower, "not ready within"), + strings.Contains(lower, "timed out"), + strings.Contains(lower, "timeout"): + return deployErrorTimeout + case strings.Contains(lower, "corrupt"), + strings.Contains(lower, "checksum"), + strings.Contains(lower, "incomplete"), + strings.Contains(lower, "safetensorerror"): + return deployErrorModelCorrupted + case strings.Contains(lower, "filenotfounderror"), + strings.Contains(lower, "no such file"), + strings.Contains(lower, "model not found"), + strings.Contains(lower, "cannot find model"), + strings.Contains(lower, "not found"): + return deployErrorModelNotFound + case strings.Contains(lower, "invalid model"), + strings.Contains(lower, "unsupported model"), + strings.Contains(lower, "unknown architecture"), + strings.Contains(lower, "invalid file magic"), + strings.Contains(lower, "invalid gguf"), + strings.Contains(lower, "safetensors header"): + return deployErrorModelFormatInvalid + case strings.Contains(lower, "download"), + strings.Contains(lower, "pull"), + strings.Contains(lower, "connection reset"), + strings.Contains(lower, "tls handshake"), + strings.Contains(lower, "temporary failure in name resolution"): + return deployErrorDownloadFailed + case strings.Contains(lower, "hardware not compatible"), + strings.Contains(lower, "not compatible"), + strings.Contains(lower, "compute capability"), + strings.Contains(lower, "no suitable device"): + return deployErrorHardwareIncompatible + case strings.Contains(lower, "process exited"), + strings.Contains(lower, "startup"), + strings.Contains(lower, "failed core proc"), + strings.Contains(lower, "failed"): + return deployErrorEngineStartFailed + default: + return deployErrorUnknown + } +} + type deploymentFailureDetails struct { Message string StartupMessage string diff --git a/cmd/aima/main.go b/cmd/aima/main.go index bbd1445c..b26dc293 100644 --- a/cmd/aima/main.go +++ b/cmd/aima/main.go @@ -1357,7 +1357,17 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { } } - waitForDeployment := func(deployName, runtimeName, resolvedEngine string, resolvedConfig map[string]any, warmup knowledge.WarmupConfig, deployTimeout time.Duration) (json.RawMessage, error) { + waitForDeployment := func(deployName, runtimeName, resolvedEngine string, resolvedConfig map[string]any, warmup knowledge.WarmupConfig, deployTimeout time.Duration, cleanupOnFailure bool) (json.RawMessage, error) { + cleanup := func() deploymentCleanupResult { + if !cleanupOnFailure { + return deploymentCleanupResult{} + } + result := cleanupFailedDeployment(ctx, deployName, deps.DeployDelete) + if result.Attempted { + notify("cleanup", result.Message) + } + return result + } notify("waiting", deployName) ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() @@ -1376,10 +1386,8 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { case <-ctx.Done(): return nil, ctx.Err() case <-timer.C: - return json.Marshal(map[string]any{ - "name": deployName, "status": "timeout", - "message": fmt.Sprintf("deployment started but not ready within %s", deployTimeout), - }) + msg := fmt.Sprintf("deployment started but not ready within %s", deployTimeout) + return nil, newDeploymentRunError(deployErrorTimeout, msg, cleanup()) case <-ticker.C: statusData, err := deps.DeployStatus(ctx, deployName) if err != nil { @@ -1420,7 +1428,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { StartupMessage: status.StartupMessage, ErrorLines: status.ErrorLines, }, deps.DeployStatus, deps.DeployLogs) - return nil, fmt.Errorf("deployment failed: %s", msg) + return nil, newDeploymentRunError(classifyDeploymentFailure(msg), "deployment failed: "+msg, cleanup()) } phase := status.StartupPhase if phase == "" { @@ -1458,7 +1466,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { return nil, fmt.Errorf("parse resolve result: %w", err) } if !plan.FitReport.Fit { - return nil, fmt.Errorf("hardware not compatible: %s", plan.FitReport.Reason) + return nil, newDeploymentRunError(deployErrorHardwareIncompatible, "hardware not compatible: "+plan.FitReport.Reason, deploymentCleanupResult{}) } notify("resolved", fmt.Sprintf("engine=%s runtime=%s", plan.Engine, plan.Runtime)) for _, warn := range plan.FitReport.Warns { @@ -1501,7 +1509,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { deployTimeout = time.Duration(t) * time.Second } } - return waitForDeployment(deployName, runtimeName, plan.Engine, plan.Config, warmup, deployTimeout) + return waitForDeployment(deployName, runtimeName, plan.Engine, plan.Config, warmup, deployTimeout, false) } } } @@ -1538,6 +1546,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { var deployResult struct { Name string `json:"name"` Runtime string `json:"runtime"` + Reused bool `json:"reused"` } if err := json.Unmarshal(deployData, &deployResult); err != nil || deployResult.Name == "" { return deployData, nil @@ -1551,7 +1560,7 @@ func buildToolDeps(ac *appContext) *mcp.ToolDeps { deployTimeout = time.Duration(t) * time.Second } } - return waitForDeployment(deployResult.Name, deployResult.Runtime, plan.Engine, plan.Config, warmup, deployTimeout) + return waitForDeployment(deployResult.Name, deployResult.Runtime, plan.Engine, plan.Config, warmup, deployTimeout, !deployResult.Reused) } deps = &mcp.ToolDeps{} diff --git a/cmd/aima/main_test.go b/cmd/aima/main_test.go index 89c84a4d..b673563d 100644 --- a/cmd/aima/main_test.go +++ b/cmd/aima/main_test.go @@ -1276,6 +1276,48 @@ func TestApplyScenarioWaitsOnLastStepBeforePostDeploy(t *testing.T) { } } +func TestClassifyDeploymentFailure(t *testing.T) { + tests := []struct { + name string + msg string + want deploymentErrorCode + }{ + {name: "out of memory", msg: "RuntimeError: HIP out of memory while loading weights", want: deployErrorOutOfMemory}, + {name: "model corrupted", msg: "safetensors_rust.SafetensorError: Error while deserializing header: incomplete metadata", want: deployErrorModelCorrupted}, + {name: "model not found", msg: "FileNotFoundError: no such file or directory: /models/demo/config.json", want: deployErrorModelNotFound}, + {name: "port occupied", msg: "bind: address already in use on 0.0.0.0:8000", want: deployErrorPortInUse}, + {name: "permission", msg: "permission denied opening /dev/kfd", want: deployErrorPermissionDenied}, + {name: "timeout", msg: "deployment started but not ready within 30s", want: deployErrorTimeout}, + {name: "generic startup", msg: "process exited before readiness", want: deployErrorEngineStartFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := classifyDeploymentFailure(tt.msg); got != tt.want { + t.Fatalf("classifyDeploymentFailure(%q) = %q, want %q", tt.msg, got, tt.want) + } + }) + } +} + +func TestCleanupFailedDeploymentReportsResult(t *testing.T) { + var deleted string + result := cleanupFailedDeployment(context.Background(), "demo", func(ctx context.Context, name string) error { + deleted = name + return nil + }) + if deleted != "demo" { + t.Fatalf("deleted deployment = %q, want demo", deleted) + } + if !result.Attempted || !result.Succeeded || result.Message != "deleted failed deployment demo" { + t.Fatalf("cleanup result = %#v, want successful deletion", result) + } + + skipped := cleanupFailedDeployment(context.Background(), "demo", nil) + if skipped.Attempted || skipped.Succeeded || !strings.Contains(skipped.Message, "unavailable") { + t.Fatalf("skipped cleanup result = %#v, want unavailable message", skipped) + } +} + func TestVariantQuantizationHint(t *testing.T) { if got := variantQuantizationHint(&knowledge.ModelVariant{ DefaultConfig: map[string]any{"quantization": "gptq"}, diff --git a/cmd/aima/tooldeps_knowledge.go b/cmd/aima/tooldeps_knowledge.go index bbe43d37..9de89ea6 100644 --- a/cmd/aima/tooldeps_knowledge.go +++ b/cmd/aima/tooldeps_knowledge.go @@ -600,6 +600,84 @@ func buildKnowledgeDeps(ac *appContext, deps *mcp.ToolDeps) { return json.Marshal(status) } + deps.CatalogEffective = func(ctx context.Context, kind, name string) (json.RawMessage, error) { + baseKind := strings.TrimSuffix(kind, "_patch") + if knowledge.KindToDir(baseKind) == "" { + return nil, fmt.Errorf("unknown kind %q", kind) + } + if err := validateOverlayAssetName(name); err != nil { + return nil, err + } + assetYAML, found, err := knowledge.CatalogAssetYAML(cat, baseKind, name) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("%s %q not found in effective catalog", baseKind, name) + } + return json.Marshal(map[string]any{ + "kind": baseKind, + "name": name, + "yaml": string(assetYAML), + }) + } + + deps.CatalogDiff = func(ctx context.Context, kind, name string) (json.RawMessage, error) { + baseKind := strings.TrimSuffix(kind, "_patch") + if knowledge.KindToDir(baseKind) == "" { + return nil, fmt.Errorf("unknown kind %q", kind) + } + if err := validateOverlayAssetName(name); err != nil { + return nil, err + } + factoryCat, err := knowledge.LoadCatalog(catalog.FS) + if err != nil { + return nil, fmt.Errorf("load factory catalog: %w", err) + } + factoryYAML, factoryFound, err := knowledge.CatalogAssetYAML(factoryCat, baseKind, name) + if err != nil { + return nil, err + } + effectiveYAML, effectiveFound, err := knowledge.CatalogAssetYAML(cat, baseKind, name) + if err != nil { + return nil, err + } + if !factoryFound && !effectiveFound { + return nil, fmt.Errorf("%s %q not found in factory or effective catalog", baseKind, name) + } + diff := unifiedCatalogYAMLDiff("factory/"+baseKind+"/"+name, "effective/"+baseKind+"/"+name, factoryYAML, effectiveYAML) + return json.Marshal(map[string]any{ + "kind": baseKind, + "name": name, + "changed": string(factoryYAML) != string(effectiveYAML), + "factory_found": factoryFound, + "effective_found": effectiveFound, + "diff": diff, + }) + } + + deps.CatalogValidatePatch = func(ctx context.Context, content string) (json.RawMessage, error) { + effectiveYAML, err := knowledge.ValidateCatalogPatch(cat, []byte(content), "input.patch.yaml") + if err != nil { + return nil, err + } + var probe struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + } `yaml:"metadata"` + } + if err := yaml.Unmarshal(effectiveYAML, &probe); err != nil { + return nil, fmt.Errorf("parse effective patch: %w", err) + } + return json.Marshal(map[string]any{ + "valid": true, + "kind": probe.Kind, + "name": probe.Metadata.Name, + "effective_yaml": string(effectiveYAML), + }) + } + deps.CatalogValidate = func(ctx context.Context) (json.RawMessage, error) { type issue struct { Engine string `json:"engine"` @@ -927,6 +1005,39 @@ func normalizeUserCatalogPatch(kind, name, content string, factoryDigests map[st return out, patchKind, nil } +func unifiedCatalogYAMLDiff(oldLabel, newLabel string, oldData, newData []byte) string { + if string(oldData) == string(newData) { + return "" + } + var b strings.Builder + b.WriteString("--- ") + b.WriteString(oldLabel) + b.WriteByte('\n') + b.WriteString("+++ ") + b.WriteString(newLabel) + b.WriteByte('\n') + b.WriteString("@@\n") + for _, line := range splitCatalogDiffLines(string(oldData)) { + b.WriteByte('-') + b.WriteString(line) + b.WriteByte('\n') + } + for _, line := range splitCatalogDiffLines(string(newData)) { + b.WriteByte('+') + b.WriteString(line) + b.WriteByte('\n') + } + return b.String() +} + +func splitCatalogDiffLines(s string) []string { + s = strings.TrimRight(s, "\n") + if s == "" { + return nil + } + return strings.Split(s, "\n") +} + func parseImportedTimestamp(value string) (time.Time, error) { trimmed := strings.TrimSpace(value) if trimmed == "" { diff --git a/docs/cli.md b/docs/cli.md index 4297f278..4c3c161c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -60,6 +60,9 @@ aima knowledge sync # 与中心服务同步知识 aima knowledge validate # 校验预测与实际性能 aima catalog override # 写入 user-owned catalog patch aima catalog validate # 校验目录资产 +aima catalog validate-patch # 只验证单个 patch，不写盘 +aima catalog effective # 查看 factory/central/user 合并后的有效 YAML +aima catalog diff # 查看 factory 到 effective 的差异 aima catalog status # 查看 factory/overlay 状态 aima benchmark run --model # 在线基准测试（TTFT/TPOT/吞吐量） aima benchmark matrix --model # 组合矩阵测试 diff --git a/docs/knowledge.md b/docs/knowledge.md index 1b207516..af32a954 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -27,7 +27,9 @@ | `knowledge.save` | `knowledge.save` | 保存 Knowledge Note | | `knowledge.evaluate` | `knowledge.evaluate` | 校验知识、引擎切换成本、开放问题 | -静态资产浏览已并入 `catalog.list(kind=profiles|engines|models|scenarios|summary|status|all)`；Pod YAML 生成已并入 `deploy.dry_run(output=pod_yaml)`。 +静态资产浏览已并入 `catalog.list(kind=profiles|engines|models|scenarios|summary|status|all)`； +单个资产诊断使用 `catalog.effective` / `catalog.diff` / `catalog.validate_patch`； +Pod YAML 生成已并入 `deploy.dry_run(output=pod_yaml)`。 --- @@ -40,7 +42,44 @@ | 受众 | 人类、git、go:embed | Agent、MCP 工具 | | 优势 | 可读、可 diff、可版本管理 | 可查询、可 JOIN、可聚合 | | 内容 | 静态知识资产定义 | 静态知识 + 动态实验数据 | -| 变更 | go:embed 需重编译; overlay 目录 (`~/.aima/catalog/`) 免编译热更新 | Agent 探索 → 直接写入 | +| 变更 | factory go:embed 需重编译; `catalog/central` 与 `catalog/user` patch overlay 可随数据目录包下发 | Agent 探索 → 直接写入 | + +### Catalog Overlay 更新能力 + +本次更新补充了面向预装和规模化下发场景的 catalog overlay 基础能力： + +- `catalog/central` 与 `catalog/user` 均使用 `*_patch` YAML，按 `factory -> central -> user` 顺序合并。 +- `aima catalog override ` 继续写入 `catalog/user`，适合单机调试或现场临时覆盖。 +- `aima catalog validate-patch ` 可以在写入前验证单个 patch，并返回合并后的有效 YAML。 +- `aima catalog effective ` 可以查看某个资产在 factory、central、user 合并后的最终 YAML。 +- `aima catalog diff ` 可以查看 factory 到 effective 的差异，便于升级前后审查。 +- `aima catalog status` 可以查看 factory/overlay 状态、覆盖资产和 staleness warning。 + +物理目录由 AIMA 数据目录决定。数据目录解析顺序为： + +1. `AIMA_DATA_DIR` +2. `/etc/aima/data-dir` +3. 默认用户目录 `~/.aima` + +如果需要通过全局默认配置包下发模型参数变更，建议写入 central 层，例如： + +```text +/catalog/central/ + models/.patch.yaml + engines/.patch.yaml + scenarios/.patch.yaml +``` + +上线前建议使用以下命令验证： + +```bash +aima catalog validate-patch ./models/demo.patch.yaml +aima catalog effective model_asset demo-model +aima catalog diff model_asset demo-model +aima catalog status +``` + +`catalog.validate-patch` 只验证输入文件，不写盘；`catalog.effective` 展示当前 factory、central、user 合并后的最终 YAML；`catalog.diff` 用于查看 factory 到 effective 的差异。对于规模化下发，应优先使用 `catalog/central`；`catalog/user` 仍保留给单机覆盖和最终用户本机选择。 ### SQLite 表结构 @@ -307,7 +346,7 @@ verified: ConfigResolver 按优先级合并多层知识: ``` -L0: YAML catalog (go:embed + ~/.aima/catalog/ overlay 合并, staleness digest 检测) +L0: YAML catalog (go:embed factory + catalog/central + catalog/user patch overlay 合并, staleness digest 检测) ↓ merge (高层 override 低层) L1: 用户 CLI --config / --engine / --slot (人类显式指定) ↓ merge diff --git a/docs/mcp.md b/docs/mcp.md index 79eb9ff0..5d3477c1 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -42,7 +42,7 @@ Go Agent (直接调用)，保证行为一致。 --- -## MCP 工具列表 (62 个) +## MCP 工具列表 (65 个) 所有工具统一由 `internal/mcp/tools.go` 的 `RegisterAllTools()` 注册，按领域拆分在 `internal/mcp/tools_*.go` 中实现。下列分组反映当前分支的完整工具前缀集合；具体参数与返回值以各工具的 `inputSchema` 和实现为准。 @@ -65,6 +65,11 @@ Go Agent (直接调用)，保证行为一致。 返回单个部署的完整状态，包含上述 overview 字段，以及 `config`、`labels`、`restarts`、`exit_code`、启动时间戳等 detail 字段。 - 不要依赖 `deploy.list` 提供原始 `config` 或 label map。 如果自动化流程需要精确运行配置或原始 labels，应调用 `deploy.status`。 +- `deploy.run` 在本次新建部署失败或等待超时时会尝试调用 `deploy.delete` 清理残留进程/容器；复用中的既有部署不会被自动删除。 +- `deploy.run` 失败错误会带稳定错误码前缀，便于 UI/日志分类。当前错误码包括： + `OUT_OF_MEMORY`, `MODEL_NOT_FOUND`, `MODEL_CORRUPTED`, `MODEL_FORMAT_INVALID`, + `PORT_IN_USE`, `PERMISSION_DENIED`, `DOWNLOAD_FAILED`, `HARDWARE_INCOMPATIBLE`, + `TIMEOUT`, `ENGINE_START_FAILED`, `UNKNOWN_ERROR`。 ### 知识与调优 @@ -76,7 +81,7 @@ Go Agent (直接调用)，保证行为一致。 ### 协同与集成 -- Catalog (3): `catalog.list`, `catalog.override`, `catalog.validate` +- Catalog (6): `catalog.list`, `catalog.effective`, `catalog.diff`, `catalog.validate_patch`, `catalog.override`, `catalog.validate` - Central (3): `central.sync`, `central.advise`, `central.scenario` - Data (2): `data.export`, `data.import` - Device (4): `device.register`, `device.status`, `device.renew`, `device.reset` diff --git a/internal/cli/catalog.go b/internal/cli/catalog.go index 771a22c9..815d88ac 100644 --- a/internal/cli/catalog.go +++ b/internal/cli/catalog.go @@ -17,6 +17,9 @@ func newCatalogCmd(app *App) *cobra.Command { cmd.AddCommand(newCatalogStatusCmd(app)) cmd.AddCommand(newCatalogOverrideCmd(app)) cmd.AddCommand(newCatalogValidateCmd(app)) + cmd.AddCommand(newCatalogEffectiveCmd(app)) + cmd.AddCommand(newCatalogDiffCmd(app)) + cmd.AddCommand(newCatalogValidatePatchCmd(app)) return cmd } @@ -67,6 +70,92 @@ func newCatalogValidateCmd(app *App) *cobra.Command { } } +func newCatalogEffectiveCmd(app *App) *cobra.Command { + return &cobra.Command{ + Use: "effective ", + Short: "Show the effective YAML for one catalog asset after factory, central, and user overlays", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if app.ToolDeps.CatalogEffective == nil { + return fmt.Errorf("catalog.effective not available") + } + data, err := app.ToolDeps.CatalogEffective(cmd.Context(), args[0], args[1]) + if err != nil { + return err + } + return printCatalogStringField(cmd, data, "yaml", true) + }, + } +} + +func newCatalogDiffCmd(app *App) *cobra.Command { + return &cobra.Command{ + Use: "diff ", + Short: "Show the factory-to-effective diff for one catalog asset", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + if app.ToolDeps.CatalogDiff == nil { + return fmt.Errorf("catalog.diff not available") + } + data, err := app.ToolDeps.CatalogDiff(cmd.Context(), args[0], args[1]) + if err != nil { + return err + } + return printCatalogStringField(cmd, data, "diff", false) + }, + } +} + +func newCatalogValidatePatchCmd(app *App) *cobra.Command { + return &cobra.Command{ + Use: "validate-patch ", + Short: "Validate one catalog patch against the current effective catalog", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if app.ToolDeps.CatalogValidatePatch == nil { + return fmt.Errorf("catalog.validate_patch not available") + } + content, err := os.ReadFile(args[0]) + if err != nil { + return fmt.Errorf("read %s: %w", args[0], err) + } + data, err := app.ToolDeps.CatalogValidatePatch(cmd.Context(), string(content)) + if err != nil { + return err + } + var pretty json.RawMessage = data + out, _ := json.MarshalIndent(pretty, "", " ") + fmt.Fprintln(cmd.OutOrStdout(), string(out)) + return nil + }, + } +} + +func printCatalogStringField(cmd *cobra.Command, data json.RawMessage, field string, fallbackJSON bool) error { + var payload map[string]any + if err := json.Unmarshal(data, &payload); err == nil { + if value, ok := payload[field].(string); ok { + if value == "" && field == "diff" { + fmt.Fprintln(cmd.OutOrStdout(), "no changes") + return nil + } + fmt.Fprint(cmd.OutOrStdout(), value) + if value == "" || value[len(value)-1] != '\n' { + fmt.Fprintln(cmd.OutOrStdout()) + } + return nil + } + } + if fallbackJSON { + var pretty json.RawMessage = data + out, _ := json.MarshalIndent(pretty, "", " ") + fmt.Fprintln(cmd.OutOrStdout(), string(out)) + return nil + } + fmt.Fprintln(cmd.OutOrStdout(), string(data)) + return nil +} + func newCatalogStatusCmd(app *App) *cobra.Command { return &cobra.Command{ Use: "status", diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 6f600679..474595ad 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "testing" @@ -224,6 +225,53 @@ func TestKnowledgeSubcommands(t *testing.T) { } } +func TestCatalogEffectiveCmdPrintsYAML(t *testing.T) { + app := testApp(t) + app.ToolDeps.CatalogEffective = func(ctx context.Context, kind, name string) (json.RawMessage, error) { + return json.RawMessage(`{"kind":"model_asset","name":"demo","yaml":"kind: model_asset\nmetadata:\n name: demo\n"}`), nil + } + root := NewRootCmd(app) + + var buf bytes.Buffer + root.SetOut(&buf) + root.SetArgs([]string{"catalog", "effective", "model_asset", "demo"}) + + if err := root.Execute(); err != nil { + t.Fatalf("catalog effective failed: %v", err) + } + if !strings.Contains(buf.String(), "kind: model_asset") || !strings.Contains(buf.String(), "name: demo") { + t.Fatalf("unexpected catalog effective output: %s", buf.String()) + } +} + +func TestCatalogValidatePatchCmdReadsFile(t *testing.T) { + app := testApp(t) + var gotContent string + app.ToolDeps.CatalogValidatePatch = func(ctx context.Context, content string) (json.RawMessage, error) { + gotContent = content + return json.RawMessage(`{"valid":true,"effective_yaml":"kind: model_asset\nmetadata:\n name: demo\n"}`), nil + } + patchPath := t.TempDir() + "/demo.patch.yaml" + if err := os.WriteFile(patchPath, []byte("kind: model_asset_patch\nmetadata:\n name: demo\n"), 0o644); err != nil { + t.Fatalf("write patch: %v", err) + } + root := NewRootCmd(app) + + var buf bytes.Buffer + root.SetOut(&buf) + root.SetArgs([]string{"catalog", "validate-patch", patchPath}) + + if err := root.Execute(); err != nil { + t.Fatalf("catalog validate-patch failed: %v", err) + } + if !strings.Contains(gotContent, "kind: model_asset_patch") { + t.Fatalf("validate-patch did not read file content: %q", gotContent) + } + if !strings.Contains(buf.String(), `"valid": true`) { + t.Fatalf("unexpected validate-patch output: %s", buf.String()) + } +} + func TestAgentSubcommands(t *testing.T) { app := testApp(t) root := NewRootCmd(app) @@ -505,7 +553,7 @@ func TestCatalogSubcommands(t *testing.T) { t.Fatal("catalog command not found") } - expected := []string{"status", "override"} + expected := []string{"status", "override", "validate", "effective", "diff", "validate-patch"} subs := make(map[string]bool) for _, c := range catalogCmd.Commands() { subs[c.Name()] = true diff --git a/internal/knowledge/loader.go b/internal/knowledge/loader.go index 45256e27..e3fdf36a 100644 --- a/internal/knowledge/loader.go +++ b/internal/knowledge/loader.go @@ -1344,6 +1344,34 @@ func catalogPatchToAssetYAML(base *Catalog, data []byte, path string) ([]byte, e return yaml.Marshal(merged) } +// ValidateCatalogPatch verifies a single catalog patch against the provided +// effective catalog and returns the asset YAML that would be produced. +func ValidateCatalogPatch(base *Catalog, data []byte, path string) ([]byte, error) { + assetData, err := catalogPatchToAssetYAML(base, data, path) + if err != nil { + return nil, err + } + validationCat := &Catalog{EngineProfiles: make(map[string]*EngineProfile)} + if err := validationCat.parseAsset(assetData, path); err != nil { + return nil, err + } + return assetData, nil +} + +// CatalogAssetYAML returns one effective catalog asset as YAML. +func CatalogAssetYAML(cat *Catalog, kind, name string) ([]byte, bool, error) { + baseKind := strings.TrimSuffix(kind, "_patch") + asset, found, err := catalogAssetMap(cat, baseKind, name) + if err != nil || !found { + return nil, found, err + } + data, err := yaml.Marshal(asset) + if err != nil { + return nil, false, err + } + return data, true, nil +} + func patchMetadataName(m map[string]any) string { meta, ok := m["metadata"].(map[string]any) if !ok { diff --git a/internal/knowledge/loader_test.go b/internal/knowledge/loader_test.go index 5b8eaef6..d71de434 100644 --- a/internal/knowledge/loader_test.go +++ b/internal/knowledge/loader_test.go @@ -951,6 +951,81 @@ func TestKindToDir(t *testing.T) { } } +func TestCatalogAssetYAMLAndValidateCatalogPatch(t *testing.T) { + base, err := LoadCatalog(fstest.MapFS{ + "models/demo.yaml": &fstest.MapFile{Data: []byte(`kind: model_asset +metadata: + name: demo-model + type: llm + family: demo + parameter_count: 1b +storage: + formats: [safetensors] + default_path_pattern: models/demo-model + sources: [] +variants: + - name: default + engine: vllm + format: safetensors + hardware: + gpu_arch: Any + vram_min_mib: 1024 + default_config: + max_model_len: 2048 + gpu_memory_utilization: 0.8 + expected_performance: {} +`)}, + }) + if err != nil { + t.Fatalf("LoadCatalog: %v", err) + } + + patch := []byte(`kind: model_asset_patch +metadata: + name: demo-model +variants: + - name: default + default_config: + gpu_memory_utilization: 0.9 +`) + + effective, err := ValidateCatalogPatch(base, patch, "models/demo-model.patch.yaml") + if err != nil { + t.Fatalf("ValidateCatalogPatch: %v", err) + } + if !strings.Contains(string(effective), "gpu_memory_utilization: 0.9") { + t.Fatalf("effective patch YAML missing override:\n%s", string(effective)) + } + if !strings.Contains(string(effective), "max_model_len: 2048") { + t.Fatalf("effective patch YAML lost base config:\n%s", string(effective)) + } + + overlay := &Catalog{ModelAssets: []ModelAsset{}} + if err := overlay.parseAsset(effective, "effective.yaml"); err != nil { + t.Fatalf("parse effective patch: %v", err) + } + merged, _ := MergeCatalog(base, overlay) + yamlData, found, err := CatalogAssetYAML(merged, "model_asset", "demo-model") + if err != nil { + t.Fatalf("CatalogAssetYAML: %v", err) + } + if !found { + t.Fatal("CatalogAssetYAML did not find demo-model") + } + if !strings.Contains(string(yamlData), "gpu_memory_utilization: 0.9") { + t.Fatalf("catalog asset YAML missing effective value:\n%s", string(yamlData)) + } + + badPatch := []byte(`kind: model_asset_patch +metadata: + name: demo-model +variants: wrong-type +`) + if _, err := ValidateCatalogPatch(base, badPatch, "models/demo-model.bad.patch.yaml"); err == nil { + t.Fatal("ValidateCatalogPatch accepted invalid asset schema") + } +} + func TestBenchmarkProfileTiers(t *testing.T) { fs := fstest.MapFS{ "benchmarks/profiles.yaml": &fstest.MapFile{Data: []byte(`kind: benchmark_profiles diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go index 8b18e1e2..cce7f653 100644 --- a/internal/mcp/mcp_test.go +++ b/internal/mcp/mcp_test.go @@ -413,6 +413,9 @@ func TestListToolsForProfile(t *testing.T) { "patrol", "deploy.list", "knowledge.resolve", + "catalog.effective", + "catalog.diff", + "catalog.validate_patch", } for _, name := range toolNames { name := name @@ -469,7 +472,7 @@ func TestListToolsForProfile(t *testing.T) { defs := s.ListToolsForProfile(ProfileOperator) names := namesOf(defs) - included := []string{"hardware.detect", "hardware.metrics", "knowledge.resolve", "deploy.list"} + included := []string{"hardware.detect", "hardware.metrics", "knowledge.resolve", "deploy.list", "catalog.effective", "catalog.diff", "catalog.validate_patch"} for _, name := range included { if !names[name] { t.Errorf("ProfileOperator should include %q", name) @@ -513,7 +516,7 @@ func TestRegisterAllTools(t *testing.T) { "deploy.apply", "deploy.run", "deploy.dry_run", "deploy.delete", "deploy.status", "deploy.list", "knowledge.resolve", "knowledge.search", "knowledge.save", "knowledge.promote", "knowledge.analytics", "knowledge.evaluate", - "catalog.list", "catalog.override", "catalog.validate", + "catalog.list", "catalog.override", "catalog.validate", "catalog.effective", "catalog.diff", "catalog.validate_patch", "central.sync", "central.advise", "central.scenario", "data.export", "data.import", "patrol", "explore", "tuning", "explorer", @@ -612,6 +615,69 @@ func TestCatalogListPartitions(t *testing.T) { } } +func TestCatalogDiagnosticsTools(t *testing.T) { + s := NewServer() + deps := &ToolDeps{ + CatalogEffective: func(ctx context.Context, kind, name string) (json.RawMessage, error) { + if kind != "model_asset" || name != "demo" { + t.Fatalf("CatalogEffective(%q, %q), want model_asset/demo", kind, name) + } + return json.RawMessage(`{"yaml":"kind: model_asset\nmetadata:\n name: demo\n"}`), nil + }, + CatalogDiff: func(ctx context.Context, kind, name string) (json.RawMessage, error) { + if kind != "model_asset" || name != "demo" { + t.Fatalf("CatalogDiff(%q, %q), want model_asset/demo", kind, name) + } + return json.RawMessage(`{"changed":true,"diff":"--- factory\n+++ effective\n"}`), nil + }, + CatalogValidatePatch: func(ctx context.Context, content string) (json.RawMessage, error) { + if !strings.Contains(content, "kind: model_asset_patch") { + t.Fatalf("CatalogValidatePatch content = %q", content) + } + return json.RawMessage(`{"valid":true}`), nil + }, + } + RegisterAllTools(s, deps) + + cases := []struct { + id int + tool string + args string + wantText string + }{ + {id: 1, tool: "catalog.effective", args: `{"kind":"model_asset","name":"demo"}`, wantText: "kind: model_asset"}, + {id: 2, tool: "catalog.diff", args: `{"kind":"model_asset","name":"demo"}`, wantText: "--- factory"}, + {id: 3, tool: "catalog.validate_patch", args: `{"content":"kind: model_asset_patch\nmetadata:\n name: demo\n"}`, wantText: `"valid":true`}, + } + for _, tc := range cases { + t.Run(tc.tool, func(t *testing.T) { + msg := fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":%q,"arguments":%s}}`, tc.id, tc.tool, tc.args) + resp, err := s.HandleMessage(context.Background(), []byte(msg)) + if err != nil { + t.Fatalf("HandleMessage: %v", err) + } + var r jsonrpcResponse + if err := json.Unmarshal(resp, &r); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + if r.Error != nil { + t.Fatalf("unexpected error: %+v", r.Error) + } + raw, _ := json.Marshal(r.Result) + var tr ToolResult + if err := json.Unmarshal(raw, &tr); err != nil { + t.Fatalf("unmarshal tool result: %v", err) + } + if tr.IsError { + t.Fatalf("%s returned error: %+v", tc.tool, tr) + } + if !strings.Contains(tr.Content[0].Text, tc.wantText) { + t.Fatalf("%s response missing %q: %s", tc.tool, tc.wantText, tr.Content[0].Text) + } + }) + } +} + func TestDeployDryRunPodYamlUsesEffectiveOverrides(t *testing.T) { s := NewServer() var gotModel, gotEngine, gotSlot string @@ -864,6 +930,9 @@ func TestProfileMatches(t *testing.T) { // ProfileOperator: exact matches {ProfileOperator, "catalog.list", true}, + {ProfileOperator, "catalog.effective", true}, + {ProfileOperator, "catalog.diff", true}, + {ProfileOperator, "catalog.validate_patch", true}, {ProfileOperator, "openclaw", true}, {ProfileOperator, "support", true}, {ProfileOperator, "knowledge.resolve", true}, diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index 72fd8237..2b630b93 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -27,7 +27,7 @@ var profileIncludes = map[Profile][]string{ ProfileOperator: { "hardware.", "model.", "engine.", "external.", "deploy.", "system.", "fleet.", "scenario.", - "catalog.list", + "catalog.list", "catalog.effective", "catalog.diff", "catalog.validate_patch", "benchmark.run", "benchmark.list", "knowledge.resolve", "knowledge.search", "knowledge.promote", "agent.ask", "agent.status", "agent.rollback", diff --git a/internal/mcp/tools_catalog.go b/internal/mcp/tools_catalog.go index 7fdee85f..4b023659 100644 --- a/internal/mcp/tools_catalog.go +++ b/internal/mcp/tools_catalog.go @@ -142,6 +142,86 @@ func registerCatalogTools(s *Server, deps *ToolDeps) { }, }) + // catalog.effective — inspect one effective asset after all overlay layers + s.RegisterTool(&Tool{ + Name: "catalog.effective", + Description: "Return the effective YAML for one catalog asset after factory, central, and user overlays. Read-only.", + InputSchema: json.RawMessage(`{"type":"object","properties":{"kind":{"type":"string","enum":["engine_profile","engine_asset","model_asset","hardware_profile","partition_strategy","stack_component","deployment_scenario"],"description":"Catalog asset kind"},"name":{"type":"string","description":"metadata.name of the asset"}},"required":["kind","name"]}`), + Handler: func(ctx context.Context, params json.RawMessage) (*ToolResult, error) { + if deps.CatalogEffective == nil { + return ErrorResult("catalog.effective not implemented"), nil + } + var p struct { + Kind string `json:"kind"` + Name string `json:"name"` + } + if err := json.Unmarshal(params, &p); err != nil { + return nil, fmt.Errorf("parse params: %w", err) + } + if p.Kind == "" || p.Name == "" { + return ErrorResult("kind and name are required"), nil + } + data, err := deps.CatalogEffective(ctx, p.Kind, p.Name) + if err != nil { + return nil, fmt.Errorf("catalog effective: %w", err) + } + return TextResult(string(data)), nil + }, + }) + + // catalog.diff — compare embedded factory asset with the current effective asset + s.RegisterTool(&Tool{ + Name: "catalog.diff", + Description: "Return a factory-to-effective diff for one catalog asset. Useful for validating centrally packaged overlays before rollout.", + InputSchema: json.RawMessage(`{"type":"object","properties":{"kind":{"type":"string","enum":["engine_profile","engine_asset","model_asset","hardware_profile","partition_strategy","stack_component","deployment_scenario"],"description":"Catalog asset kind"},"name":{"type":"string","description":"metadata.name of the asset"}},"required":["kind","name"]}`), + Handler: func(ctx context.Context, params json.RawMessage) (*ToolResult, error) { + if deps.CatalogDiff == nil { + return ErrorResult("catalog.diff not implemented"), nil + } + var p struct { + Kind string `json:"kind"` + Name string `json:"name"` + } + if err := json.Unmarshal(params, &p); err != nil { + return nil, fmt.Errorf("parse params: %w", err) + } + if p.Kind == "" || p.Name == "" { + return ErrorResult("kind and name are required"), nil + } + data, err := deps.CatalogDiff(ctx, p.Kind, p.Name) + if err != nil { + return nil, fmt.Errorf("catalog diff: %w", err) + } + return TextResult(string(data)), nil + }, + }) + + // catalog.validate_patch — validate one patch body without writing it + s.RegisterTool(&Tool{ + Name: "catalog.validate_patch", + Description: "Validate a catalog patch body against the current effective catalog and return the merged effective YAML. Read-only; does not write an overlay file.", + InputSchema: json.RawMessage(`{"type":"object","properties":{"content":{"type":"string","description":"YAML content with kind: _patch and metadata.name"}},"required":["content"]}`), + Handler: func(ctx context.Context, params json.RawMessage) (*ToolResult, error) { + if deps.CatalogValidatePatch == nil { + return ErrorResult("catalog.validate_patch not implemented"), nil + } + var p struct { + Content string `json:"content"` + } + if err := json.Unmarshal(params, &p); err != nil { + return nil, fmt.Errorf("parse params: %w", err) + } + if p.Content == "" { + return ErrorResult("content is required"), nil + } + data, err := deps.CatalogValidatePatch(ctx, p.Content) + if err != nil { + return nil, fmt.Errorf("catalog validate_patch: %w", err) + } + return TextResult(string(data)), nil + }, + }) + // catalog.override — write a user-owned YAML patch to the runtime overlay catalog s.RegisterTool(&Tool{ Name: "catalog.override", diff --git a/internal/mcp/tools_deps.go b/internal/mcp/tools_deps.go index 03f0ea6c..97277753 100644 --- a/internal/mcp/tools_deps.go +++ b/internal/mcp/tools_deps.go @@ -75,9 +75,12 @@ type ToolDeps struct { StackStatus func(ctx context.Context) (json.RawMessage, error) // Catalog overlay - CatalogOverride func(ctx context.Context, kind, name, content string) (json.RawMessage, error) - CatalogStatus func(ctx context.Context) (json.RawMessage, error) - CatalogValidate func(ctx context.Context) (json.RawMessage, error) + CatalogOverride func(ctx context.Context, kind, name, content string) (json.RawMessage, error) + CatalogStatus func(ctx context.Context) (json.RawMessage, error) + CatalogValidate func(ctx context.Context) (json.RawMessage, error) + CatalogEffective func(ctx context.Context, kind, name string) (json.RawMessage, error) + CatalogDiff func(ctx context.Context, kind, name string) (json.RawMessage, error) + CatalogValidatePatch func(ctx context.Context, content string) (json.RawMessage, error) // Deploy approval DeployApprove func(ctx context.Context, id int64) (json.RawMessage, error)