From 442ee67c9ebb02f71ef459edc86e16d3dc4183d1 Mon Sep 17 00:00:00 2001 From: Emos21 Date: Tue, 7 Jul 2026 14:48:57 +0300 Subject: [PATCH 1/4] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20You.com=20=E8=81=94?= =?UTF-8?q?=E7=BD=91=E6=A3=80=E7=B4=A2=EF=BC=88endpoint=20+=20MCP=20?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/api/rag/rag.go | 1 + server/api/rag/v1/websearch.go | 17 ++ server/core/websearch/youcom.go | 250 ++++++++++++++++++ server/internal/cmd/cmd.go | 1 + .../controller/rag/rag_v1_websearch.go | 36 +++ server/internal/logic/websearch/websearch.go | 38 +++ server/internal/mcp/websearch.go | 60 +++++ server/manifest/config/config_demo.yaml | 6 + 8 files changed, 409 insertions(+) create mode 100644 server/api/rag/v1/websearch.go create mode 100644 server/core/websearch/youcom.go create mode 100644 server/internal/controller/rag/rag_v1_websearch.go create mode 100644 server/internal/logic/websearch/websearch.go create mode 100644 server/internal/mcp/websearch.go diff --git a/server/api/rag/rag.go b/server/api/rag/rag.go index 779d915..2aed3e4 100644 --- a/server/api/rag/rag.go +++ b/server/api/rag/rag.go @@ -27,4 +27,5 @@ type IRagV1 interface { KBGetList(ctx context.Context, req *v1.KBGetListReq) (res *v1.KBGetListRes, err error) Retriever(ctx context.Context, req *v1.RetrieverReq) (res *v1.RetrieverRes, err error) RetrieverDify(ctx context.Context, req *v1.RetrieverDifyReq) (res *v1.RetrieverDifyRes, err error) + WebSearch(ctx context.Context, req *v1.WebSearchReq) (res *v1.WebSearchRes, err error) } diff --git a/server/api/rag/v1/websearch.go b/server/api/rag/v1/websearch.go new file mode 100644 index 0000000..4dabd15 --- /dev/null +++ b/server/api/rag/v1/websearch.go @@ -0,0 +1,17 @@ +package v1 + +import ( + "github.com/cloudwego/eino/schema" + "github.com/gogf/gf/v2/frame/g" +) + +type WebSearchReq struct { + g.Meta `path:"/v1/websearch" method:"post" tags:"rag"` + Question string `json:"question" v:"required"` + TopK int `json:"top_k"` // 默认为5 +} + +type WebSearchRes struct { + g.Meta `mime:"application/json"` + Document []*schema.Document `json:"document"` +} diff --git a/server/core/websearch/youcom.go b/server/core/websearch/youcom.go new file mode 100644 index 0000000..4e5f641 --- /dev/null +++ b/server/core/websearch/youcom.go @@ -0,0 +1,250 @@ +// Package websearch 基于 You.com Search API 实现联网检索能力， +// 作为 eino retriever.Retriever 的一种实现，独立于现有的向量检索链路。 +package websearch + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" +) + +const ( + // defaultBaseURL You.com Search API 默认地址 + defaultBaseURL = "https://ydc-index.io/v1/search" + // defaultCount 默认返回结果数量 + defaultCount = 5 + // maxCount 单次请求允许的最大返回结果数量 + maxCount = 20 + // defaultTimeout 默认请求超时时间 + defaultTimeout = 10 * time.Second + // sourceName 结果来源标识，写入 MetaData + sourceName = "you.com" + // apiKeyEnv You.com API Key 的环境变量名（团队约定，不可更改） + apiKeyEnv = "YDC_API_KEY" +) + +// Config You.com 联网检索配置 +type Config struct { + APIKey string // You.com Search API Key，留空时读取环境变量 YDC_API_KEY + BaseURL string // 可选，默认 https://ydc-index.io/v1/search + Count int // 默认返回结果数量，默认 5，最大 20 + TimeoutSec int // 请求超时时间（秒），默认 10 +} + +// YouComRetriever 基于 You.com Search API 实现的 eino Retriever +type YouComRetriever struct { + cfg *Config + client *http.Client +} + +// youComResponse You.com Search API 响应结构（仅保留用得到的字段） +type youComResponse struct { + Results struct { + Web []youComResult `json:"web"` + News []youComResult `json:"news"` + } `json:"results"` +} + +// youComResult 单条搜索结果，除 url/title/description/snippets 外的字段均视为可选 +type youComResult struct { + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"description"` + Snippets []string `json:"snippets"` +} + +// NewRetriever 创建 You.com 联网检索 retriever +func NewRetriever(cfg *Config) (*YouComRetriever, error) { + if cfg == nil { + cfg = &Config{} + } + c := *cfg // 拷贝一份，避免修改调用方传入的配置对象 + + if c.BaseURL == "" { + c.BaseURL = defaultBaseURL + } + if c.Count <= 0 { + c.Count = defaultCount + } + if c.Count > maxCount { + c.Count = maxCount + } + + timeout := defaultTimeout + if c.TimeoutSec > 0 { + timeout = time.Duration(c.TimeoutSec) * time.Second + } + + return &YouComRetriever{ + cfg: &c, + client: &http.Client{ + Timeout: timeout, + }, + }, nil +} + +// resolveAPIKey 解析 API Key：优先使用配置文件中的 apiKey，否则读取环境变量 YDC_API_KEY +func (r *YouComRetriever) resolveAPIKey() string { + if r.cfg.APIKey != "" { + return r.cfg.APIKey + } + return os.Getenv(apiKeyEnv) +} + +// Retrieve 调用 You.com Search API 检索网页结果，实现 eino retriever.Retriever 接口 +func (r *YouComRetriever) Retrieve(ctx context.Context, query string, opts ...retriever.Option) ([]*schema.Document, error) { + key := r.resolveAPIKey() + if key == "" { + return nil, fmt.Errorf("未配置 You.com API Key，请设置环境变量 %s 或在配置文件 websearch.youcom.apiKey 中填写", apiKeyEnv) + } + + options := &retriever.Options{} + retriever.GetCommonOptions(options, opts...) + + count := r.cfg.Count + if options.TopK != nil && *options.TopK > 0 { + count = *options.TopK + } + if count > maxCount { + count = maxCount + } + + req, err := r.buildRequest(ctx, key, query, count) + if err != nil { + return nil, err + } + + resp, err := r.client.Do(req) + if err != nil { + return nil, mapRequestError(ctx, err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("读取 you.com 响应失败: %w", err) + } + + if statusErr := mapStatusError(resp.StatusCode); statusErr != nil { + return nil, statusErr + } + + var payload youComResponse + if err = json.Unmarshal(body, &payload); err != nil { + return nil, fmt.Errorf("解析 you.com 响应失败: %w", err) + } + + return toDocuments(&payload), nil +} + +// buildRequest 构建 you.com search 请求：GET + X-API-Key header + query/count 参数 +func (r *YouComRetriever) buildRequest(ctx context.Context, key, query string, count int) (*http.Request, error) { + u, err := url.Parse(r.cfg.BaseURL) + if err != nil { + return nil, fmt.Errorf("you.com baseURL 非法: %w", err) + } + q := u.Query() + q.Set("query", query) + q.Set("count", strconv.Itoa(count)) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("构建 you.com 请求失败: %w", err) + } + req.Header.Set("X-API-Key", key) + return req, nil +} + +// mapRequestError 将底层网络错误转换为更明确的提示，never 回显 API Key +func mapRequestError(ctx context.Context, err error) error { + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("you.com 请求超时: %w", err) + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return fmt.Errorf("you.com 请求超时: %w", err) + } + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("you.com 请求已取消: %w", ctxErr) + } + return fmt.Errorf("you.com 请求失败: %w", err) +} + +// mapStatusError 将 you.com 的错误状态码映射为可读的错误信息，不回显 API Key +func mapStatusError(statusCode int) error { + switch { + case statusCode == http.StatusOK: + return nil + case statusCode == http.StatusUnauthorized: + return fmt.Errorf("you.com 鉴权失败(401)，请检查 API Key 是否正确") + case statusCode == http.StatusForbidden: + return fmt.Errorf("you.com 请求被拒绝(403)，请检查请求地址或账号权限") + case statusCode == http.StatusUnprocessableEntity: + return fmt.Errorf("you.com 请求参数非法(422)，请检查检索参数") + case statusCode == http.StatusTooManyRequests: + return fmt.Errorf("you.com 请求超出速率限制(429)，请稍后重试") + case statusCode >= http.StatusInternalServerError: + return fmt.Errorf("you.com 服务端错误(%d)，可稍后重试", statusCode) + default: + return fmt.Errorf("you.com 返回非预期状态码(%d)", statusCode) + } +} + +// toDocuments 将 you.com 的搜索结果转换为 schema.Document 列表 +// results.web 与 results.news 都会被收录，results.news 缺失时按空处理 +func toDocuments(payload *youComResponse) []*schema.Document { + results := make([]youComResult, 0, len(payload.Results.Web)+len(payload.Results.News)) + results = append(results, payload.Results.Web...) + results = append(results, payload.Results.News...) + + docs := make([]*schema.Document, 0, len(results)) + for i, item := range results { + doc := &schema.Document{ + ID: item.URL, + Content: buildContent(item), + MetaData: map[string]any{ + "url": item.URL, + "title": item.Title, + "source": sourceName, + }, + } + // you.com 按相关性排好序返回，没有分数，这里用排名倒序模拟一个递减分数 + doc.WithScore(rankScore(i)) + docs = append(docs, doc) + } + return docs +} + +// buildContent 拼接 title + description + snippets 作为文档正文 +func buildContent(item youComResult) string { + parts := make([]string, 0, 2+len(item.Snippets)) + if item.Title != "" { + parts = append(parts, item.Title) + } + if item.Description != "" { + parts = append(parts, item.Description) + } + if len(item.Snippets) > 0 { + parts = append(parts, strings.Join(item.Snippets, "\n")) + } + return strings.Join(parts, "\n") +} + +// rankScore 依据排名生成一个递减的分数，排名越靠前分数越高 +func rankScore(rank int) float64 { + const epsilon = 0.001 + return 1.0 - float64(rank)*epsilon +} diff --git a/server/internal/cmd/cmd.go b/server/internal/cmd/cmd.go index 6fa4a38..af80fdb 100644 --- a/server/internal/cmd/cmd.go +++ b/server/internal/cmd/cmd.go @@ -45,6 +45,7 @@ func Mcp(ctx context.Context, s *ghttp.Server) { mcpServer, _ := server.NewServer(trans) mcpServer.RegisterTool(mcp.GetRetrieverTool(), mcp.HandleRetriever) mcpServer.RegisterTool(mcp.GetKnowledgeBaseTool(), mcp.HandleKnowledgeBase) + mcpServer.RegisterTool(mcp.GetWebSearchTool(), mcp.HandleWebSearch) // start mcp Server go func() { mcpServer.Run() diff --git a/server/internal/controller/rag/rag_v1_websearch.go b/server/internal/controller/rag/rag_v1_websearch.go new file mode 100644 index 0000000..841c68f --- /dev/null +++ b/server/internal/controller/rag/rag_v1_websearch.go @@ -0,0 +1,36 @@ +package rag + +import ( + "context" + "sort" + + "github.com/cloudwego/eino/components/retriever" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/wangle201210/go-rag/server/internal/logic/websearch" + + v1 "github.com/wangle201210/go-rag/server/api/rag/v1" +) + +// WebSearch 通过 You.com Search API 实现联网检索，独立于知识库向量检索 +func (c *ControllerV1) WebSearch(ctx context.Context, req *v1.WebSearchReq) (res *v1.WebSearchRes, err error) { + svr := websearch.GetWebSearchSvr() + if svr == nil { + return nil, gerror.New("联网检索未配置，请设置环境变量 YDC_API_KEY 或在配置文件 websearch.youcom.apiKey 中填写 You.com API Key") + } + if req.TopK == 0 { + req.TopK = 5 + } + g.Log().Infof(ctx, "webSearchReq: %v", req) + docs, err := svr.Retrieve(ctx, req.Question, retriever.WithTopK(req.TopK)) + if err != nil { + return nil, err + } + sort.Slice(docs, func(i, j int) bool { + return docs[i].Score() > docs[j].Score() + }) + res = &v1.WebSearchRes{ + Document: docs, + } + return +} diff --git a/server/internal/logic/websearch/websearch.go b/server/internal/logic/websearch/websearch.go new file mode 100644 index 0000000..ddb4eaa --- /dev/null +++ b/server/internal/logic/websearch/websearch.go @@ -0,0 +1,38 @@ +package websearch + +import ( + "sync" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gctx" + "github.com/wangle201210/go-rag/server/core/websearch" +) + +var ( + webSearchSvr *websearch.YouComRetriever + webSearchSvrOnce sync.Once +) + +// GetWebSearchSvr 惰性初始化 You.com 联网检索 retriever +// 与 internal/logic/rag 的 init 模式不同：websearch 是可选功能，未配置时不应导致进程启动失败， +// 因此这里延迟到第一次调用时才读取配置并构建 retriever。 +func GetWebSearchSvr() *websearch.YouComRetriever { + webSearchSvrOnce.Do(func() { + ctx := gctx.New() + + cfg := &websearch.Config{ + APIKey: g.Cfg().MustGet(ctx, "websearch.youcom.apiKey").String(), + Count: g.Cfg().MustGet(ctx, "websearch.youcom.count").Int(), + TimeoutSec: g.Cfg().MustGet(ctx, "websearch.youcom.timeout").Int(), + } + + svr, err := websearch.NewRetriever(cfg) + if err != nil { + // websearch 是可选功能，构建失败只记录日志，不影响其他功能正常使用 + g.Log().Warningf(ctx, "NewRetriever of websearch failed, err=%v", err) + return + } + webSearchSvr = svr + }) + return webSearchSvr +} diff --git a/server/internal/mcp/websearch.go b/server/internal/mcp/websearch.go new file mode 100644 index 0000000..d389f19 --- /dev/null +++ b/server/internal/mcp/websearch.go @@ -0,0 +1,60 @@ +package mcp + +import ( + "context" + "fmt" + + "github.com/ThinkInAIXYZ/go-mcp/protocol" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gctx" + v1 "github.com/wangle201210/go-rag/server/api/rag/v1" +) + +type WebSearchParam struct { + Question string `json:"question" description:"用户提问的问题，将通过 You.com 联网检索" required:"true"` + TopK int `json:"top_k" description:"检索结果的数量，默认为5" required:"false"` // 默认为5 +} + +func GetWebSearchTool() *protocol.Tool { + tool, err := protocol.NewTool("web_search", "You.com 联网检索，返回带来源链接的网页结果", WebSearchParam{}) + if err != nil { + g.Log().Errorf(gctx.New(), "Failed to create tool: %v", err) + return nil + } + return tool +} + +func HandleWebSearch(ctx context.Context, toolReq *protocol.CallToolRequest) (res *protocol.CallToolResult, err error) { + var req WebSearchParam + if err := protocol.VerifyAndUnmarshal(toolReq.RawArguments, &req); err != nil { + return nil, err + } + webSearch, err := c.WebSearch(ctx, &v1.WebSearchReq{ + Question: req.Question, + TopK: req.TopK, + }) + if err != nil { + // 联网检索未配置属于可预期的情况，作为友好文本返回，而不是协议错误 + return &protocol.CallToolResult{ + Content: []protocol.Content{ + &protocol.TextContent{ + Type: "text", + Text: fmt.Sprintf("联网检索暂不可用：%v，请配置 YDC_API_KEY 后重试", err), + }, + }, + }, nil + } + docs := webSearch.Document + msg := fmt.Sprintf("web_search %d documents", len(docs)) + for i, doc := range docs { + msg += fmt.Sprintf("\n%d. score: %.2f, url: %v, content: %s", i+1, doc.Score(), doc.MetaData["url"], doc.Content) + } + return &protocol.CallToolResult{ + Content: []protocol.Content{ + &protocol.TextContent{ + Type: "text", + Text: msg, + }, + }, + }, nil +} diff --git a/server/manifest/config/config_demo.yaml b/server/manifest/config/config_demo.yaml index e3adef9..631dc37 100644 --- a/server/manifest/config/config_demo.yaml +++ b/server/manifest/config/config_demo.yaml @@ -61,3 +61,9 @@ chat: apiKey: "sk-****" baseURL: "https://api.siliconflow.cn/v1" model: "deepseek-ai/DeepSeek-V3" + +websearch: + youcom: + apiKey: "" # You.com Search API Key，留空时读取环境变量 YDC_API_KEY（获取地址 https://you.com/platform/api-keys） + count: 5 # 默认返回结果数量 + timeout: 10 # 请求超时时间（秒） From 19af0e260bd485e2ad49a91dea562e35ad0a8b43 Mon Sep 17 00:00:00 2001 From: Emos21 Date: Tue, 7 Jul 2026 14:49:02 +0300 Subject: [PATCH 2/4] =?UTF-8?q?websearch=20=E5=8D=95=E5=85=83=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/websearch/youcom_integration_test.go | 38 +++ server/core/websearch/youcom_test.go | 304 ++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 server/core/websearch/youcom_integration_test.go create mode 100644 server/core/websearch/youcom_test.go diff --git a/server/core/websearch/youcom_integration_test.go b/server/core/websearch/youcom_integration_test.go new file mode 100644 index 0000000..6fb4c3d --- /dev/null +++ b/server/core/websearch/youcom_integration_test.go @@ -0,0 +1,38 @@ +package websearch + +import ( + "context" + "os" + "testing" + "time" +) + +// TestYouComRetrieverIntegration 是唯一一个会真正访问 You.com 线上接口的测试， +// 未设置 YDC_API_KEY 时自动跳过，避免 CI / 无 key 环境失败。 +func TestYouComRetrieverIntegration(t *testing.T) { + if os.Getenv("YDC_API_KEY") == "" { + t.Skip("YDC_API_KEY 未设置，跳过 you.com 线上集成测试") + } + + r, err := NewRetriever(&Config{Count: 3, TimeoutSec: 10}) + if err != nil { + t.Fatalf("NewRetriever failed: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + docs, err := r.Retrieve(ctx, "You.com API") + if err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if len(docs) == 0 { + t.Fatal("expected at least 1 document from live you.com search") + } + if docs[0].Content == "" { + t.Error("expected non-empty Content in first document") + } + if docs[0].MetaData["url"] == nil || docs[0].MetaData["url"] == "" { + t.Error("expected non-empty MetaData[url] in first document") + } +} diff --git a/server/core/websearch/youcom_test.go b/server/core/websearch/youcom_test.go new file mode 100644 index 0000000..1bd304c --- /dev/null +++ b/server/core/websearch/youcom_test.go @@ -0,0 +1,304 @@ +package websearch + +import ( + "context" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/cloudwego/eino/components/retriever" +) + +// newTestServer 启动一个可控的 you.com 假服务，返回 server 及最近一次收到的请求信息 +func newTestServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return srv +} + +func TestRetrieve_RequestConstruction(t *testing.T) { + var gotPath, gotAPIKey, gotQuery, gotCount string + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAPIKey = r.Header.Get("X-API-Key") + gotQuery = r.URL.Query().Get("query") + gotCount = r.URL.Query().Get("count") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{"web":[]}}`)) + }) + + r, err := NewRetriever(&Config{APIKey: "test-key", BaseURL: srv.URL + "/v1/search", Count: 3}) + if err != nil { + t.Fatalf("NewRetriever failed: %v", err) + } + if _, err = r.Retrieve(context.Background(), "hello world"); err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + + if gotPath != "/v1/search" { + t.Errorf("path = %q, want /v1/search", gotPath) + } + if gotAPIKey != "test-key" { + t.Errorf("X-API-Key header = %q, want test-key", gotAPIKey) + } + if gotQuery != "hello world" { + t.Errorf("query param = %q, want %q", gotQuery, "hello world") + } + if gotCount != "3" { + t.Errorf("count param = %q, want 3", gotCount) + } +} + +func TestRetrieve_MapsWebAndNews(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{ + "web":[{"url":"https://a.com","title":"A","description":"descA","snippets":["s1","s2"]}], + "news":[{"url":"https://b.com","title":"B","description":"descB","snippets":["s3"]}] + }}`)) + }) + + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + docs, err := r.Retrieve(context.Background(), "q") + if err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if len(docs) != 2 { + t.Fatalf("got %d docs, want 2", len(docs)) + } + if docs[0].ID != "https://a.com" || docs[0].MetaData["url"] != "https://a.com" { + t.Errorf("web doc ID/url mismatch: %+v", docs[0]) + } + if docs[1].ID != "https://b.com" { + t.Errorf("news doc ID mismatch: %+v", docs[1]) + } + if docs[1].MetaData["source"] != sourceName { + t.Errorf("source metadata = %v, want %v", docs[1].MetaData["source"], sourceName) + } +} + +func TestRetrieve_NewsAbsent(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{"web":[{"url":"https://a.com","title":"A","description":"d"}]}}`)) + }) + + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + docs, err := r.Retrieve(context.Background(), "q") + if err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if len(docs) != 1 { + t.Fatalf("got %d docs, want 1 (news absent should not break mapping)", len(docs)) + } +} + +func TestRetrieve_OptionalFieldsMissing(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + // 只包含 url，title/description/snippets 均缺失 + w.Write([]byte(`{"results":{"web":[{"url":"https://a.com"}]}}`)) + }) + + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + docs, err := r.Retrieve(context.Background(), "q") + if err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if len(docs) != 1 { + t.Fatalf("got %d docs, want 1", len(docs)) + } + if docs[0].ID != "https://a.com" { + t.Errorf("ID = %q, want https://a.com", docs[0].ID) + } + if docs[0].Content != "" { + t.Errorf("Content = %q, want empty when title/description/snippets absent", docs[0].Content) + } +} + +func TestRetrieve_SnippetsJoined(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{"web":[{"url":"https://a.com","title":"T","description":"D","snippets":["s1","s2","s3"]}]}}`)) + }) + + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + docs, err := r.Retrieve(context.Background(), "q") + if err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + want := "T\nD\ns1\ns2\ns3" + if docs[0].Content != want { + t.Errorf("Content = %q, want %q", docs[0].Content, want) + } +} + +func TestRetrieve_ScoreDescendingOrder(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{"web":[ + {"url":"https://a.com","title":"A"}, + {"url":"https://b.com","title":"B"}, + {"url":"https://c.com","title":"C"} + ]}}`)) + }) + + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + docs, err := r.Retrieve(context.Background(), "q") + if err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if len(docs) != 3 { + t.Fatalf("got %d docs, want 3", len(docs)) + } + for i := 1; i < len(docs); i++ { + if docs[i].Score() >= docs[i-1].Score() { + t.Errorf("scores not strictly descending at %d: %v >= %v", i, docs[i].Score(), docs[i-1].Score()) + } + } +} + +func TestRetrieve_ErrorStatusCodes(t *testing.T) { + cases := []struct { + status int + }{ + {http.StatusUnauthorized}, + {http.StatusForbidden}, + {http.StatusUnprocessableEntity}, + {http.StatusTooManyRequests}, + {http.StatusInternalServerError}, + } + for _, tc := range cases { + t.Run(strconv.Itoa(tc.status), func(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tc.status) + w.Write([]byte(`{"error":"boom"}`)) + }) + r, _ := NewRetriever(&Config{APIKey: "super-secret-key", BaseURL: srv.URL}) + _, err := r.Retrieve(context.Background(), "q") + if err == nil { + t.Fatalf("expected error for status %d, got nil", tc.status) + } + if got := err.Error(); containsKey(got, "super-secret-key") { + t.Errorf("error message leaked API key: %q", got) + } + }) + } +} + +func TestRetrieve_MalformedJSON(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`not json`)) + }) + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + _, err := r.Retrieve(context.Background(), "q") + if err == nil { + t.Fatal("expected error for malformed JSON, got nil") + } +} + +func TestRetrieve_Timeout(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + time.Sleep(200 * time.Millisecond) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{"web":[]}}`)) + }) + r, _ := NewRetriever(&Config{APIKey: "super-secret-key", BaseURL: srv.URL}) + r.client.Timeout = 20 * time.Millisecond // 覆盖为极短超时以加速测试 + + _, err := r.Retrieve(context.Background(), "q") + if err == nil { + t.Fatal("expected timeout error, got nil") + } + if containsKey(err.Error(), "super-secret-key") { + t.Errorf("timeout error leaked API key: %q", err.Error()) + } +} + +func TestRetrieve_NoAPIKeyAnywhere(t *testing.T) { + t.Setenv("YDC_API_KEY", "") + r, _ := NewRetriever(&Config{}) + _, err := r.Retrieve(context.Background(), "q") + if err == nil { + t.Fatal("expected error when no API key is configured anywhere") + } +} + +func TestRetrieve_CountDefaultAndCap(t *testing.T) { + var gotCount string + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + gotCount = r.URL.Query().Get("count") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{"web":[]}}`)) + }) + + t.Run("default", func(t *testing.T) { + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + if _, err := r.Retrieve(context.Background(), "q"); err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if gotCount != strconv.Itoa(defaultCount) { + t.Errorf("count = %q, want default %d", gotCount, defaultCount) + } + }) + + t.Run("cap at construction", func(t *testing.T) { + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL, Count: 999}) + if _, err := r.Retrieve(context.Background(), "q"); err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if gotCount != strconv.Itoa(maxCount) { + t.Errorf("count = %q, want capped %d", gotCount, maxCount) + } + }) + + t.Run("cap via WithTopK option", func(t *testing.T) { + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + if _, err := r.Retrieve(context.Background(), "q", retriever.WithTopK(50)); err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if gotCount != strconv.Itoa(maxCount) { + t.Errorf("count = %q, want capped %d", gotCount, maxCount) + } + }) +} + +func TestRetrieve_APIKeyPrecedence(t *testing.T) { + var gotKey string + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + gotKey = r.Header.Get("X-API-Key") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{"web":[]}}`)) + }) + + t.Run("yaml key wins over env", func(t *testing.T) { + t.Setenv("YDC_API_KEY", "env-key") + r, _ := NewRetriever(&Config{APIKey: "yaml-key", BaseURL: srv.URL}) + if _, err := r.Retrieve(context.Background(), "q"); err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if gotKey != "yaml-key" { + t.Errorf("key used = %q, want yaml-key", gotKey) + } + }) + + t.Run("falls back to env when yaml empty", func(t *testing.T) { + t.Setenv("YDC_API_KEY", "env-key") + r, _ := NewRetriever(&Config{BaseURL: srv.URL}) + if _, err := r.Retrieve(context.Background(), "q"); err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + if gotKey != "env-key" { + t.Errorf("key used = %q, want env-key", gotKey) + } + }) +} + +func containsKey(s, key string) bool { + return len(key) > 0 && strings.Contains(s, key) +} From ce6e72a0287464a090dba0d371d478279bfee86f Mon Sep 17 00:00:00 2001 From: Emos21 Date: Tue, 7 Jul 2026 14:49:06 +0300 Subject: [PATCH 3/4] =?UTF-8?q?README=20=E8=A1=A5=E5=85=85=E8=81=94?= =?UTF-8?q?=E7=BD=91=E6=A3=80=E7=B4=A2=E9=85=8D=E7=BD=AE=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 7197b5a..296f431 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ - [x] chunk 编辑 - [x] 自动生成 QA 对 - [x] 多路召回 +- [x] You.com 联网检索（可选，需 YDC_API_KEY） ## 使用 ### clone项目 @@ -63,6 +64,11 @@ make run # 浏览器打开 http://localhost:8000 ```` +### 联网检索（可选） +如需启用 You.com 联网检索（`POST /v1/websearch` 接口 + MCP `web_search` 工具），设置环境变量 `YDC_API_KEY` +（获取地址 https://you.com/platform/api-keys），或在 `config.yaml` 的 `websearch.youcom.apiKey` 中填写。 +未配置时该功能不影响其他功能正常使用，调用时会返回提示信息。 + ### 安装依赖 *如果有可用的es8和mysql,可以不用安装* 安装es8 From 4d0d81f8919d05df1dc0335e0123414ba665d2ce Mon Sep 17 00:00:00 2001 From: Emos21 Date: Tue, 7 Jul 2026 14:54:13 +0300 Subject: [PATCH 4/4] =?UTF-8?q?websearch=20=E7=BB=93=E6=9E=9C=E6=95=B0?= =?UTF-8?q?=E9=87=8F=E6=8C=89=20TopK=20=E6=88=AA=E6=96=AD=20&=20=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20MCP=20=E9=94=99=E8=AF=AF=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/websearch/youcom.go | 8 +++++++- server/core/websearch/youcom_test.go | 30 ++++++++++++++++++++++++++++ server/internal/mcp/websearch.go | 5 +++-- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/server/core/websearch/youcom.go b/server/core/websearch/youcom.go index 4e5f641..50cf326 100644 --- a/server/core/websearch/youcom.go +++ b/server/core/websearch/youcom.go @@ -146,7 +146,13 @@ func (r *YouComRetriever) Retrieve(ctx context.Context, query string, opts ...re return nil, fmt.Errorf("解析 you.com 响应失败: %w", err) } - return toDocuments(&payload), nil + docs := toDocuments(&payload) + // web 与 news 双通道各返回最多 count 条，这里按总数截断， + // 保证调用方拿到的结果数量与 TopK 语义一致（与 /v1/retriever 行为对齐） + if len(docs) > count { + docs = docs[:count] + } + return docs, nil } // buildRequest 构建 you.com search 请求：GET + X-API-Key header + query/count 参数 diff --git a/server/core/websearch/youcom_test.go b/server/core/websearch/youcom_test.go index 1bd304c..8a0130a 100644 --- a/server/core/websearch/youcom_test.go +++ b/server/core/websearch/youcom_test.go @@ -302,3 +302,33 @@ func TestRetrieve_APIKeyPrecedence(t *testing.T) { func containsKey(s, key string) bool { return len(key) > 0 && strings.Contains(s, key) } + +func TestRetrieve_TopKTruncatesTotalResults(t *testing.T) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results":{ + "web":[ + {"url":"https://a.com","title":"A","description":"d"}, + {"url":"https://b.com","title":"B","description":"d"} + ], + "news":[ + {"url":"https://c.com","title":"C","description":"d"}, + {"url":"https://d.com","title":"D","description":"d"} + ] + }}`)) + }) + + r, _ := NewRetriever(&Config{APIKey: "k", BaseURL: srv.URL}) + docs, err := r.Retrieve(context.Background(), "q", retriever.WithTopK(2)) + if err != nil { + t.Fatalf("Retrieve failed: %v", err) + } + // web 与 news 双通道合计可能超过 TopK，Retrieve 需按总数截断， + // 且 web 结果（分数更高）优先保留 + if len(docs) != 2 { + t.Fatalf("got %d docs, want 2 (TopK must bound total results)", len(docs)) + } + if docs[0].ID != "https://a.com" || docs[1].ID != "https://b.com" { + t.Errorf("expected web docs kept first after truncation, got %v, %v", docs[0].ID, docs[1].ID) + } +} diff --git a/server/internal/mcp/websearch.go b/server/internal/mcp/websearch.go index d389f19..6883d28 100644 --- a/server/internal/mcp/websearch.go +++ b/server/internal/mcp/websearch.go @@ -34,12 +34,13 @@ func HandleWebSearch(ctx context.Context, toolReq *protocol.CallToolRequest) (re TopK: req.TopK, }) if err != nil { - // 联网检索未配置属于可预期的情况，作为友好文本返回，而不是协议错误 + // 联网检索失败属于可预期的情况（未配置、限流等），作为友好文本返回，而不是协议错误 + // 具体原因与处理建议已包含在 err 信息中 return &protocol.CallToolResult{ Content: []protocol.Content{ &protocol.TextContent{ Type: "text", - Text: fmt.Sprintf("联网检索暂不可用：%v，请配置 YDC_API_KEY 后重试", err), + Text: fmt.Sprintf("联网检索暂不可用：%v", err), }, }, }, nil