diff --git a/backend/workflow/opencli_adapter_nodes.py b/backend/workflow/opencli_adapter_nodes.py index 7b468ca..81bf785 100644 --- a/backend/workflow/opencli_adapter_nodes.py +++ b/backend/workflow/opencli_adapter_nodes.py @@ -232,7 +232,6 @@ def _load_opencli_catalog() -> tuple[dict[str, Any], ...]: check=False, text=True, encoding="utf-8", - errors="replace", timeout=_OPENCLI_LIST_TIMEOUT_SECONDS, ) except Exception as exc: diff --git a/frontend/components/flow/command-palette.tsx b/frontend/components/flow/command-palette.tsx index 79fab15..d313b73 100644 --- a/frontend/components/flow/command-palette.tsx +++ b/frontend/components/flow/command-palette.tsx @@ -51,6 +51,11 @@ import { getWorkflowPrimitives, type WorkflowPrimitive, } from "@/lib/workflow/node-primitives" +import { + groupOpenCLIAdapterPlugins, + openCLIKeyboardCandidates, + OPENCLI_SITE_CATEGORIES, +} from "@/lib/plugins/opencli-adapter-catalog" import { openCLIAdapterNodeToCatalogItem } from "@/lib/workflow/opencli-adapter-catalog" import { useWorkflowCapabilities } from "@/lib/workflow/use-workflow-capabilities" import { cn } from "@/lib/utils" @@ -124,7 +129,6 @@ type ToolFilter = "all" | "opencli" | "plugin" type OpenCLIAccessFilter = "all" | "read" | "write" type OpenCLIReadinessFilter = "all" | "runnable" | "blocked" -const OPENCLI_RESULT_LIMIT = 60 const OPENCLI_SEARCH_RESULT_LIMIT = 120 const TAB_META: { id: PickerTab; label: string }[] = [ @@ -157,10 +161,14 @@ const PALETTE_COPY = { noNodes: "没有匹配的节点", opencliCapabilities: "OpenCLI 能力预设", loadingOpencli: "正在读取 OpenCLI 能力目录", - featuredSites: "国内全网 OODA 数据源", + featuredSites: "消息与数据来源", catalogIndex: "能力导航", - featuredDescription: "按 OODA 使用场景浏览常用国内数据源", + featuredDescription: "按行情、官方披露、财经媒体、社交与视频浏览可用来源", moreOpencliPresets: "更多站点能力", + allOpencliSites: "全部 OpenCLI 站点", + siteDirectoryDescription: "按站点浏览完整能力目录，选择后查看该站全部命令", + backToSiteDirectory: "返回站点目录", + siteCommands: "站点命令", pluginTools: "插件与后端工具", noTools: "没有匹配的工具", createImport: "创建与导入", @@ -221,10 +229,14 @@ const PALETTE_COPY = { noNodes: "No matching nodes", opencliCapabilities: "OpenCLI capability presets", loadingOpencli: "Loading the OpenCLI capability catalog", - featuredSites: "China-wide OODA sources", + featuredSites: "Information & data sources", catalogIndex: "Capability navigation", - featuredDescription: "Browse common China data sources by OODA use case", + featuredDescription: "Browse market, official disclosure, media, social, and video sources", moreOpencliPresets: "More site capabilities", + allOpencliSites: "All OpenCLI sites", + siteDirectoryDescription: "Browse the complete catalog by site, then inspect all commands for one site", + backToSiteDirectory: "Back to site directory", + siteCommands: "Site commands", pluginTools: "Plugin & backend tools", noTools: "No matching tools", createImport: "Create & import", @@ -457,6 +469,7 @@ export function CommandPalette({ const [opencliLoading, setOpencliLoading] = useState(false) const [fallbackOpenCLINodes, setFallbackOpenCLINodes] = useState([]) const [fallbackOpenCLIError, setFallbackOpenCLIError] = useState(null) + const [selectedOpenCLISiteId, setSelectedOpenCLISiteId] = useState(null) const [selectedOpenCLI, setSelectedOpenCLI] = useState(null) const [requiredValues, setRequiredValues] = useState>({}) const inputRef = useRef(null) @@ -489,6 +502,7 @@ export function CommandPalette({ setQuery("") setAiMode(false) setAiPrompt("") + setSelectedOpenCLISiteId(null) setSelectedOpenCLI(null) setRequiredValues({}) requestAnimationFrame(() => inputRef.current?.focus()) @@ -715,10 +729,19 @@ export function CommandPalette({ const commonOpenCLIGroups = featuredOpenCLIAdapterGroups(matchingOpenCLINodes, language) const commonOpenCLINodes = commonOpenCLIGroups.flatMap((group) => group.nodes) const commonOpenCLIIds = new Set(commonOpenCLINodes.map((item) => item.id)) - const opencliResultLimit = queryText ? OPENCLI_SEARCH_RESULT_LIMIT : OPENCLI_RESULT_LIMIT - const visibleOpenCLINodes = matchingOpenCLINodes - .filter((item) => !commonOpenCLIIds.has(item.id)) - .slice(0, opencliResultLimit) + const opencliSitePlugins = groupOpenCLIAdapterPlugins(matchingOpenCLINodes) + const opencliSiteCategoryGroups = OPENCLI_SITE_CATEGORIES + .map((category) => ({ + ...category, + sites: opencliSitePlugins.filter((plugin) => plugin.siteCategory === category.key), + })) + .filter((category) => category.sites.length > 0) + const selectedOpenCLISite = opencliSitePlugins.find((plugin) => plugin.id === selectedOpenCLISiteId) ?? null + const visibleOpenCLINodes = queryText + ? matchingOpenCLINodes + .filter((item) => !commonOpenCLIIds.has(item.id)) + .slice(0, OPENCLI_SEARCH_RESULT_LIMIT) + : selectedOpenCLISite?.commands ?? [] const opencliPresetGroups = (() => { const groups = new Map() for (const item of visibleOpenCLINodes) { @@ -736,7 +759,11 @@ export function CommandPalette({ .find((item) => !catalogItemUnavailable(item)) const firstPrimitive = primitiveGroups[0]?.items[0] const firstAuxiliary = auxiliaryOperators[0] - const firstOpenCLI = matchingOpenCLINodes.find( + const firstOpenCLI = openCLIKeyboardCandidates( + queryText, + selectedOpenCLISite, + matchingOpenCLINodes, + ).find( (item) => !openCLIPresetUnavailable(item), ) const firstPluginTool = filteredPluginTools.find((item) => !catalogItemUnavailable(item)) @@ -800,7 +827,7 @@ export function CommandPalette({
{TAB_META.map((tab) => ( - + ))}
@@ -830,7 +857,10 @@ export function CommandPalette({ setQuery(event.target.value)} + onChange={(event) => { + setQuery(event.target.value) + if (event.target.value) setSelectedOpenCLISiteId(null) + }} onKeyDown={(event) => { if (event.key !== "Enter" || event.nativeEvent.isComposing) return if (activeTab === "nodes") { @@ -851,20 +881,20 @@ export function CommandPalette({ {activeTab === "tools" ? (
- {([['all', copy.all], ['opencli', copy.opencliPresets], ['plugin', copy.pluginCapabilities]] as const).map(([id, label]) => )} + {([['all', copy.all], ['opencli', copy.opencliPresets], ['plugin', copy.pluginCapabilities]] as const).map(([id, label]) => )}
{toolFilter !== "plugin" ? (
{copy.accessFilter}
- {([['all', copy.allRoles], ['read', copy.dataRead], ['write', copy.operationTool]] as const).map(([id, label]) => )} + {([['all', copy.allRoles], ['read', copy.dataRead], ['write', copy.operationTool]] as const).map(([id, label]) => )}
{copy.readinessFilter}
- {([['all', copy.allStates], ['runnable', copy.runnable], ['blocked', copy.needsSetup]] as const).map(([id, label]) => )} + {([['all', copy.allStates], ['runnable', copy.runnable], ['blocked', copy.needsSetup]] as const).map(([id, label]) => )}
@@ -927,30 +957,54 @@ export function CommandPalette({
- {commonOpenCLINodes.length > 0 ? ( + {!selectedOpenCLISite && commonOpenCLINodes.length > 0 ? (
@@ -983,7 +1037,101 @@ export function CommandPalette({
) : null} - {opencliPresetGroups.length ? ( + {selectedOpenCLISite ? ( +
+
+ + + + {selectedOpenCLISite.label} + + + {selectedOpenCLISite.site} · {copy.siteCommands} + + + + {selectedOpenCLISite.commands.length} + +
+
+ {opencliPresetGroups.map(([groupKey, items]) => { + const [, presetKind] = groupKey.split(":") + return ( +
+ + {presetKind === "source_slot" ? copy.dataRead : copy.operationTool} + +
+ {items.map((item) => ( + addOpenCLIAdapter(item)} + /> + ))} +
+
+ ) + })} +
+
+ ) : null} + + {!selectedOpenCLISite && !queryText ? ( +
0 && "mt-3", + )} + > +
+ + {copy.allOpencliSites} + {copy.siteDirectoryDescription} + + {opencliSitePlugins.length} +
+
+ {opencliSiteCategoryGroups.map((category) => ( +
+ + {language === "zh-CN" ? category.label : category.labelEn} + +
+ {category.sites.map((site) => ( + + ))} +
+
+ ))} +
+
+ ) : null} + + {!selectedOpenCLISite && queryText && opencliPresetGroups.length ? (
) : null} - {!catalogBusy && matchingOpenCLINodes.length > commonOpenCLINodes.length + visibleOpenCLINodes.length ? ( + {!catalogBusy && queryText && matchingOpenCLINodes.length > commonOpenCLINodes.length + visibleOpenCLINodes.length ? (

{copy.shown} {commonOpenCLINodes.length + visibleOpenCLINodes.length} / {matchingOpenCLINodes.length}. {copy.refineSearch}

diff --git a/frontend/components/flow/inspector.tsx b/frontend/components/flow/inspector.tsx index 5b73202..eecdf97 100644 --- a/frontend/components/flow/inspector.tsx +++ b/frontend/components/flow/inspector.tsx @@ -10,6 +10,7 @@ import { GitBranch, Plus, PlugZap, + RotateCcw, Search, Trash2, Unplug, @@ -68,6 +69,10 @@ import { isOpenCLISourceSlotArray, type OpenCLISourceSlot, } from "@/lib/workflow/node-catalog" +import { + ASHARE_OPENCLI_SOURCES, + OPENCLI_SITUATION_SOURCES, +} from "@/lib/workflow/opencli-business-workflows" import { openCLISlotFromDataSource, SOURCE_ARGUMENT_LABELS, @@ -106,6 +111,28 @@ const edgeTypeHints: Record = { routed: "自动绕开中间节点的正交折线，适合密集流程图。", } +const BUILT_IN_SOURCE_IDS = new Set([ + ...ASHARE_OPENCLI_SOURCES, + ...OPENCLI_SITUATION_SOURCES, +].map((source) => source.id)) + +const SOURCE_ID_ACRONYMS: Record = { + bse: "BSE", + cninfo: "CNInfo", + pdf: "PDF", + sse: "SSE", + szse: "SZSE", + ths: "THS", +} + +function sourceCardLabel(source: OpenCLISourceSlot, language: WorkflowLanguage): string { + if (language === "zh-CN" || !BUILT_IN_SOURCE_IDS.has(source.id)) return source.label + return source.id + .split("-") + .map((part) => SOURCE_ID_ACRONYMS[part] ?? `${part.charAt(0).toUpperCase()}${part.slice(1)}`) + .join(" ") +} + const INSPECTOR_COPY = { "zh-CN": { switchLanguage: "切换节点语言", @@ -173,6 +200,20 @@ const INSPECTOR_COPY = { collectionTopicPlaceholder: "例如：人工智能、贵州茅台", collectionTopicHint: "一次设置会同步到所有搜索型来源。", market: "市场范围", + contentType: "采集内容", + addContent: "添加一类来源", + contentTypes: { + market: "行情", + filings: "公告财报", + macro: "宏观", + news: "新闻", + social: "社区舆情", + video: "视频", + }, + configureSource: "设置", + positionalArgument: "搜索词 / 资源 ID", + removedSource: "已移除", + undoRemove: "撤销", collectionOptions: "采集选项", items: "项", opencliMapping: "OpenCLI 映射", @@ -251,6 +292,20 @@ const INSPECTOR_COPY = { collectionTopicPlaceholder: "Example: artificial intelligence, Apple", collectionTopicHint: "One value is synchronized to every search-based source.", market: "Market scope", + contentType: "Content", + addContent: "Add a source group", + contentTypes: { + market: "Market", + filings: "Filings", + macro: "Macro", + news: "News", + social: "Social", + video: "Video", + }, + configureSource: "Configure", + positionalArgument: "Search term / resource ID", + removedSource: "Removed", + undoRemove: "Undo", collectionOptions: "Collection options", items: "items", opencliMapping: "OpenCLI mapping", @@ -1527,6 +1582,7 @@ export function Inspector({ compact = false, onClose }: { compact?: boolean; onC {openCLISources ? ( updateWorkflowNodeParams(configurationNodeId, { sources })} @@ -1886,6 +1942,7 @@ function OpenCLISourceEditor({ const availableSources = registeredSources.filter((source) => !selectedSourceKeys.has(sourceSlotKey(source))) const businessQuery = sourceBusinessQuery(sources) const market = sourceMarket(sources) + const [removedSource, setRemovedSource] = useState<{ source: OpenCLISourceSlot; index: number } | null>(null) const updateSource = (index: number, patch: Partial) => { onChange(sources.map((source, sourceIndex) => ( @@ -1899,6 +1956,34 @@ function OpenCLISourceEditor({ onChange([...sources, source]) } + const addContentSources = (contentType: string | null) => { + if (!contentType) return + const presets = contentType === "video" + ? OPENCLI_SITUATION_SOURCES.filter((source) => source.sourceGroup?.startsWith("video-")) + : ASHARE_OPENCLI_SOURCES.filter((source) => source.sourceGroup === contentType) + const selectedKeys = new Set(sources.map(sourceSlotKey)) + const additions = presets.filter((source) => !selectedKeys.has(sourceSlotKey(source))) + if (additions.length > 0) onChange([...sources, ...additions]) + } + + const removeSource = (index: number) => { + setRemovedSource({ source: sources[index], index }) + onChange(sources.filter((_, sourceIndex) => sourceIndex !== index)) + } + + const restoreSource = () => { + if (!removedSource) return + const removedKey = sourceSlotKey(removedSource.source) + if (sources.some((source) => sourceSlotKey(source) === removedKey)) { + setRemovedSource(null) + return + } + const restored = [...sources] + restored.splice(Math.min(removedSource.index, restored.length), 0, removedSource.source) + onChange(restored) + setRemovedSource(null) + } + return (
@@ -1944,6 +2029,28 @@ function OpenCLISourceEditor({
+
+ + +

+ {language === "zh-CN" + ? "市场范围只影响行情类卡片；新闻、社区和视频按来源卡片独立配置。" + : "Market scope affects market cards only; news, social, and video are configured per source card."} +

+
{sourceCatalog.isError ? (

{copy.sourceUnavailable} @@ -1998,36 +2105,57 @@ function OpenCLISourceEditor({ ) : null}

+ {removedSource ? ( +
+ {copy.removedSource}: {sourceCardLabel(removedSource.source, language)} + +
+ ) : null} {sources.map((source, index) => { const businessArguments = sourceBusinessArguments(source) + const optionCount = businessArguments.length + (source.positionalArgs?.length ? 1 : 0) + const contentType = source.sourceGroup?.startsWith("video-") ? "video" : source.sourceGroup + const contentLabel = contentType && contentType in copy.contentTypes + ? copy.contentTypes[contentType as keyof typeof copy.contentTypes] + : (language === "zh-CN" ? "数据采集" : "Data collection") return ( -
-
+
+ {source.site.slice(0, 1)}
-

{source.label}

+

{sourceCardLabel(source, language)}

- {source.site} · {source.sourceGroup || (language === "zh-CN" ? "数据采集" : "Data collection")} + {source.site} · {contentLabel}

- -
- {businessArguments.length > 0 ? ( -
- - {copy.collectionOptions} · {businessArguments.length} {copy.items} - -
+ {copy.configureSource} + + +
+ {optionCount > 0 ? ( + <> +

+ {copy.collectionOptions} · {optionCount} {copy.items} +

+ {source.positionalArgs?.length ? ( +
+ + updateSource(index, { positionalArgs: [event.target.value] })} + className={houdiniInputClass} + /> +
+ ) : null} {businessArguments.map(([key, value]) => ( updateSource(index, { args: { ...source.args, [key]: nextValue } })} /> ))} -
-
- ) : null} -
+ + ) : ( +

+ {language === "zh-CN" ? "此来源没有必须填写的业务参数。" : "This source has no required business parameters."} +

+ )} +
+ +
+
+ ) })}
diff --git a/frontend/lib/plugins/opencli-adapter-catalog.ts b/frontend/lib/plugins/opencli-adapter-catalog.ts index c156153..b376c30 100644 --- a/frontend/lib/plugins/opencli-adapter-catalog.ts +++ b/frontend/lib/plugins/opencli-adapter-catalog.ts @@ -24,16 +24,16 @@ export type OpenCLIAdapterPlugin = { } export const OPENCLI_SITE_CATEGORIES = [ - { key: "ai", label: "AI 工具" }, - { key: "social", label: "社交与内容" }, - { key: "news", label: "新闻资讯" }, - { key: "finance", label: "金融数据" }, - { key: "academic", label: "学术研究" }, - { key: "developer", label: "开发者工具" }, - { key: "commerce", label: "电商与生活" }, - { key: "government", label: "政务与行业" }, - { key: "local-app", label: "本地应用" }, - { key: "general", label: "工具与数据" }, + { key: "ai", label: "AI 工具", labelEn: "AI tools" }, + { key: "social", label: "社交与内容", labelEn: "Social & content" }, + { key: "news", label: "新闻资讯", labelEn: "News" }, + { key: "finance", label: "金融数据", labelEn: "Finance" }, + { key: "academic", label: "学术研究", labelEn: "Research" }, + { key: "developer", label: "开发者工具", labelEn: "Developer tools" }, + { key: "commerce", label: "电商与生活", labelEn: "Commerce & life" }, + { key: "government", label: "政务与行业", labelEn: "Government & industry" }, + { key: "local-app", label: "本地应用", labelEn: "Local apps" }, + { key: "general", label: "工具与数据", labelEn: "Tools & data" }, ] as const export type OpenCLISiteCategory = (typeof OPENCLI_SITE_CATEGORIES)[number]["key"] @@ -77,6 +77,10 @@ const SITE_PRESENTATION_OVERRIDES: Record< chatgpt: { label: "ChatGPT 网页版" }, "chatgpt-app": { label: "ChatGPT 桌面应用" }, cnki: { label: "中国知网" }, + "cninfo-reports": { + label: "巨潮财报 CLI", + introduction: "自研 CLI 提供巨潮公告查询、全市场财报分片采集、PDF 下载与完整性审计。", + }, coingecko: { label: "CoinGecko" }, coinglass: { label: "CoinGlass" }, ctrip: { label: "携程" }, @@ -239,6 +243,7 @@ const SITE_CATEGORY_MEMBERS: Record< "chinamoney", "cls", "cninfo", + "cninfo-reports", "cnstock", "coingecko", "coinglass", @@ -467,6 +472,15 @@ export function groupOpenCLIAdapterPlugins( }).sort((left, right) => left.label.localeCompare(right.label)) } +export function openCLIKeyboardCandidates( + queryText: string, + selectedSite: OpenCLIAdapterPlugin | null, + matchingNodes: WorkflowOpenCLIAdapterNode[], +): WorkflowOpenCLIAdapterNode[] { + if (!queryText && !selectedSite) return [] + return selectedSite?.commands ?? matchingNodes +} + export function summarizeOpenCLIAdapterPlugins( plugins: OpenCLIAdapterPlugin[], ): OpenCLIAdapterRegistrySummary { diff --git a/frontend/lib/workflow/backend-opencli-adapter-nodes.ts b/frontend/lib/workflow/backend-opencli-adapter-nodes.ts index b384bda..af20289 100644 --- a/frontend/lib/workflow/backend-opencli-adapter-nodes.ts +++ b/frontend/lib/workflow/backend-opencli-adapter-nodes.ts @@ -96,6 +96,8 @@ const FEATURED_OPENCLI_SOURCES: FeaturedOpenCLISource[] = [ { id: "opencli.adapter.eastmoney.longhu", label: "东方财富 · 龙虎榜", description: "交易所披露的龙虎榜、净流入、成交额与上榜原因", labelEn: "Eastmoney · Dragon-Tiger List", descriptionEn: "Exchange-disclosed active seats, flows, turnover, and reasons" }, { id: "opencli.adapter.eastmoney.rzrq", label: "东方财富 · 融资融券", description: "沪深融资余额、融券余量与历史变化", labelEn: "Eastmoney · Margin Trading", descriptionEn: "Margin financing, securities lending, and historical balances" }, { id: "opencli.adapter.ths.hot", label: "同花顺 · 强势股与题材", description: "当日强势股、题材归因、换手与主力净量", labelEn: "10jqka · Hot Stocks & Themes", descriptionEn: "Intraday hot stocks, theme attribution, turnover, and fund flow" }, + { id: "opencli.adapter.tdx.hot-rank", label: "通达信 · 热度排行", description: "通达信股票热度、涨跌与题材标签排行", labelEn: "Tongdaxin · Hot Ranking", descriptionEn: "Tongdaxin stock heat, price change, and theme rankings" }, + { id: "opencli.adapter.szse.market-summary", label: "深交所 · 市场总览", description: "深市证券数量、成交额与总市值统计", labelEn: "SZSE · Market Summary", descriptionEn: "Shenzhen listings, turnover, and market-cap statistics" }, { id: "opencli.adapter.eastmoney.valuation", label: "东方财富 · A 股估值", description: "A 股 PE、PB、PS 与 PCF 估值横截面", labelEn: "Eastmoney · A-share Valuation", descriptionEn: "A-share PE, PB, PS, and PCF valuation snapshot" }, { id: "opencli.adapter.eastmoney.bbsj-summary", label: "东方财富 · 财务摘要", description: "收入、净利润与 EPS 等上市公司关键财务指标", labelEn: "Eastmoney · Financial Summary", descriptionEn: "Revenue, net profit, EPS, and other company fundamentals" }, { id: "opencli.adapter.eastmoney.research", label: "东方财富 · 券商研报", description: "A 股个股与行业券商研报列表", labelEn: "Eastmoney · Broker Research", descriptionEn: "A-share company and sector broker research" }, @@ -121,6 +123,13 @@ const FEATURED_OPENCLI_SOURCES: FeaturedOpenCLISource[] = [ { id: "opencli.adapter.yicai.kuaixun", label: "第一财经 · 实时快讯", description: "第一财经市场、公司与宏观快讯", labelEn: "Yicai · Live News", descriptionEn: "Yicai market, company, and macro flash news" }, { id: "opencli.adapter.cnstock.kuaixun", label: "上海证券报 · 实时快讯", description: "上证报财经与上市公司快讯", labelEn: "Shanghai Securities News · Live News", descriptionEn: "Shanghai Securities News company and market updates" }, { id: "opencli.adapter.cs.kuaixun", label: "中国证券报 · 实时快讯", description: "中证报财经与资本市场快讯", labelEn: "China Securities Journal · Live News", descriptionEn: "China Securities Journal capital-market updates" }, + { id: "opencli.adapter.douyin.search", label: "抖音 · 视频搜索", description: "按主题搜索抖音视频、作者与互动数据", labelEn: "Douyin · Video Search", descriptionEn: "Search Douyin videos, creators, and engagement by topic" }, + { id: "opencli.adapter.douyin.tophot", label: "抖音 · 实时热点", description: "抖音全站实时热点与传播热度", labelEn: "Douyin · Live Trends", descriptionEn: "Live Douyin trends and propagation signals" }, + { id: "opencli.adapter.bilibili.hot", label: "B 站 · 热门视频", description: "B 站全站热门视频与互动热度", labelEn: "Bilibili · Popular Videos", descriptionEn: "Popular Bilibili videos and engagement signals" }, + { id: "opencli.adapter.bilibili.search", label: "B 站 · 视频搜索", description: "按主题检索 B 站视频和 UP 主", labelEn: "Bilibili · Video Search", descriptionEn: "Search Bilibili videos and creators by topic" }, + { id: "opencli.adapter.bilibili.ranking", label: "B 站 · 视频排行榜", description: "B 站视频分区排行榜与热度指标", labelEn: "Bilibili · Video Rankings", descriptionEn: "Bilibili category rankings and popularity metrics" }, + { id: "opencli.adapter.bilibili.subtitle", label: "B 站 · 字幕提取", description: "按 BV 号提取视频字幕，供内容分析使用", labelEn: "Bilibili · Subtitle Extraction", descriptionEn: "Extract subtitles by BV id for content analysis" }, + { id: "opencli.adapter.bilibili.summary", label: "B 站 · 视频摘要", description: "按 BV 号读取视频摘要与结构化内容", labelEn: "Bilibili · Video Summary", descriptionEn: "Read video summaries and structured content by BV id" }, ] const DOMESTIC_OODA_SOURCE_GROUPS: FeaturedOpenCLISourceGroup[] = [ @@ -141,6 +150,8 @@ const DOMESTIC_OODA_SOURCE_GROUPS: FeaturedOpenCLISourceGroup[] = [ "opencli.adapter.eastmoney.valuation", "opencli.adapter.eastmoney.hot-rank", "opencli.adapter.ths.hot", + "opencli.adapter.tdx.hot-rank", + "opencli.adapter.szse.market-summary", "opencli.adapter.xueqiu.hot-stocks", "opencli.adapter.xueqiu.stock-social", "opencli.adapter.xueqiu.industries", @@ -227,14 +238,23 @@ const DOMESTIC_OODA_SOURCE_GROUPS: FeaturedOpenCLISourceGroup[] = [ "opencli.adapter.eastmoney.guba", "opencli.adapter.weibo.hot", "opencli.adapter.weibo.search", - "opencli.adapter.weixin.search", + "opencli.adapter.weixin.search-articles", "opencli.adapter.xiaohongshu.search", "opencli.adapter.xiaohongshu.feed", + ], + }, + { + id: "video", + label: "视频与多媒体情报", + labelEn: "Video & multimedia intelligence", + sourceIds: [ "opencli.adapter.douyin.search", "opencli.adapter.douyin.tophot", "opencli.adapter.bilibili.hot", "opencli.adapter.bilibili.search", "opencli.adapter.bilibili.ranking", + "opencli.adapter.bilibili.subtitle", + "opencli.adapter.bilibili.summary", ], }, ] @@ -250,10 +270,13 @@ export function featuredOpenCLIAdapterGroups( language: WorkflowLanguage = "zh-CN", ): FeaturedOpenCLIAdapterGroup[] { const byId = new Map(nodes.map((node) => [node.id, node])) + const seen = new Set() return DOMESTIC_OODA_SOURCE_GROUPS.flatMap((group) => { const groupNodes = group.sourceIds.flatMap((id) => { const node = byId.get(id) - return node ? [node] : [] + if (!node || seen.has(node.id)) return [] + seen.add(node.id) + return [node] }) return groupNodes.length ? [{ id: group.id, label: language === "zh-CN" ? group.label : group.labelEn, nodes: groupNodes }] @@ -315,6 +338,8 @@ export function openCLIAdapterNodeMaterialization( export function openCLIAdapterNodeSearchText(node: WorkflowOpenCLIAdapterNode): string { const materialization = openCLIAdapterNodeMaterialization(node) + const chinese = openCLIAdapterNodePresentation(node, "zh-CN") + const english = openCLIAdapterNodePresentation(node, "en-US") const roleAliases = node.access === "read" ? "read source source-slot 数据读取 数据源" : "write tool action 操作工具 写入" @@ -329,6 +354,11 @@ export function openCLIAdapterNodeSearchText(node: WorkflowOpenCLIAdapterNode): node.id, node.label, node.description, + chinese.label, + chinese.description, + chinese.label.replace(/\s+/g, ""), + english.label, + english.description, node.site, node.command, node.capability, diff --git a/frontend/lib/workflow/opencli-business-workflows.ts b/frontend/lib/workflow/opencli-business-workflows.ts index 56142c8..8a17398 100644 --- a/frontend/lib/workflow/opencli-business-workflows.ts +++ b/frontend/lib/workflow/opencli-business-workflows.ts @@ -63,6 +63,14 @@ export const ASHARE_OPENCLI_SOURCES: OpenCLISourceSlot[] = [ command: "hot", args: { limit: 50 }, }, + { + id: "tdx-hot-rank", + label: "通达信股票热度排行", + sourceGroup: "market", + site: "tdx", + command: "hot-rank", + args: { limit: 20 }, + }, { id: "fundamentals", label: "东方财富上市公司财务摘要", @@ -179,6 +187,14 @@ export const OPENCLI_SITUATION_SOURCES: OpenCLISourceSlot[] = [ args: { limit: 20 }, positionalArgs: ["A股 市场"], }, + { + id: "douyin-discovery", + label: "短视频发现 · 抖音", + sourceGroup: "video-discovery", + site: "douyin", + command: "search", + args: { query: "A股 市场", limit: 20 }, + }, { id: "bilibili-transcript", label: "视频字幕证据 · Bilibili", @@ -307,13 +323,14 @@ export function buildAshareMarketWorkflow(name: string) { workflowId: "ashare-market-intelligence", cadence: "5m", sources: ASHARE_OPENCLI_SOURCES, - sourceLabel: "国内全网 OODA 数据源", + sourceLabel: "A 股消息与数据来源", sourceDescription: "行情、公告与 PDF、宏观、新闻、社交五类来源并行采集；逐来源显示完成、空结果或失败", recordsLabel: "A 股金融数据集", maxItemsPerRun: 1_000, allowedDomains: [ "eastmoney.com", "10jqka.com.cn", + "tdx.com.cn", "xueqiu.com", "cninfo.com.cn", "sse.com.cn", @@ -335,6 +352,6 @@ export function buildOpenCLISituationAwarenessWorkflow(name: string) { sourceDescription: "并行采集实时事件、新闻、视频目录与字幕；在运行痕迹中公开来源、时间和采集证据", recordsLabel: "态势证据数据集", maxItemsPerRun: 300, - allowedDomains: ["cls.cn", "sina.com.cn", "bilibili.com", "youtube.com"], + allowedDomains: ["cls.cn", "sina.com.cn", "bilibili.com", "douyin.com", "youtube.com"], }) } diff --git a/frontend/lib/workflow/source-business-config.ts b/frontend/lib/workflow/source-business-config.ts index 0e0a0bc..cef2056 100644 --- a/frontend/lib/workflow/source-business-config.ts +++ b/frontend/lib/workflow/source-business-config.ts @@ -31,6 +31,7 @@ export function openCLISlotFromDataSource(source: DataSource): OpenCLISourceSlot const site = source.channel_config.site const command = source.channel_config.command const args = source.channel_config.args + const positionalArgs = source.channel_config.positional_args ?? source.channel_config.positionalArgs if ( typeof site !== "string" || !site.trim() || @@ -49,12 +50,21 @@ export function openCLISlotFromDataSource(source: DataSource): OpenCLISourceSlot site, command, args: { ...(args as Record) }, + ...(Array.isArray(positionalArgs) && positionalArgs.every((value) => typeof value === "string") + ? { positionalArgs: [...positionalArgs] } + : {}), format: typeof source.channel_config.format === "string" ? source.channel_config.format : undefined, } } -export function sourceSlotKey(source: Pick): string { - return `${source.site}::${source.command}::${stableSerialize(source.args)}` +export function sourceCapabilityKey(source: Pick): string { + return `${source.site.trim().toLowerCase()}::${source.command.trim().toLowerCase()}` +} + +export function sourceSlotKey( + source: Pick, +): string { + return `${sourceCapabilityKey(source)}::${stableSerialize(source.args)}::${stableSerialize(source.positionalArgs ?? [])}` } export function sourceBusinessQuery(sources: OpenCLISourceSlot[]): string | undefined { @@ -77,7 +87,7 @@ export function updateSourceBusinessQuery( export function sourceMarket(sources: OpenCLISourceSlot[]): string | undefined { for (const source of sources) { - if (typeof source.args.market === "string") return source.args.market + if (source.sourceGroup === "market" && typeof source.args.market === "string") return source.args.market } return undefined } @@ -87,7 +97,7 @@ export function updateSourceMarket( value: string, ): OpenCLISourceSlot[] { return sources.map((source) => ( - Object.prototype.hasOwnProperty.call(source.args, "market") + source.sourceGroup === "market" && Object.prototype.hasOwnProperty.call(source.args, "market") ? { ...source, args: { ...source.args, market: value } } : source )) diff --git a/frontend/lib/workflow/studio-templates.ts b/frontend/lib/workflow/studio-templates.ts index 28b47b8..db6a94e 100644 --- a/frontend/lib/workflow/studio-templates.ts +++ b/frontend/lib/workflow/studio-templates.ts @@ -11,7 +11,7 @@ import { import { parseWorkflowProject, workflowNodeSchema, type WorkflowProjectNode } from './schema' export const STUDIO_TEMPLATES = [ - { id: 'ashare-market-intelligence', variant: 'collection-to-consumption', appType: 'workflow', title: 'A 股真实金融数据采集', description: '用 OpenCLI 并行采集国内行情、公告财报、宏观快讯、财经媒体与社区热度，并写入可追溯 Records。', category: '真实业务测试', steps: ['国内全网 OODA 采集', '清洗与准入', '数据工作台'] }, + { id: 'ashare-market-intelligence', variant: 'collection-to-consumption', appType: 'workflow', title: 'A 股真实金融数据采集', description: '用 OpenCLI 并行采集国内行情、公告财报、宏观快讯、财经媒体与社区热度，并写入可追溯 Records。', category: '真实业务测试', steps: ['消息与数据来源', '清洗与准入', '数据工作台'] }, { id: 'opencli-situation-awareness', variant: 'collection-to-consumption', appType: 'workflow', title: 'OpenCLI 态势感知框架', description: '采集实时事件、新闻和视频字幕，保留证据血缘，并投影到数据工作台与逻辑证据页。', category: '真实业务测试', steps: ['多模态证据采集', '证据准入', '数据与证据工作台'] }, { id: 'opencli-live-pipeline', variant: 'collection-to-consumption', appType: 'workflow', title: 'OpenCLI 实时采集清洗发送', description: '从 OpenCLI 动态数据源实时提取，完成标准化、去重、Records 入库并发送结果。', category: '完整链路', steps: ['OpenCLI 实时采集', '清洗与 Records', 'Webhook 发送'] }, { id: 'financial-rss-intelligence', variant: 'collect', appType: 'workflow', title: '财经多源 RSS 情报', description: '并行采集央行政策、监管公告与研究动态，按来源 Group 清洗后写入成果与数据。', category: '采集与监控', steps: ['多源 RSS', 'Group 标准化', 'Records 入库'] }, diff --git a/frontend/scripts/check-node-capability-catalog-regressions.mjs b/frontend/scripts/check-node-capability-catalog-regressions.mjs index 9813427..ecd441f 100644 --- a/frontend/scripts/check-node-capability-catalog-regressions.mjs +++ b/frontend/scripts/check-node-capability-catalog-regressions.mjs @@ -159,6 +159,66 @@ test('core catalog nodes and common parameters expose Chinese and English copy', assert.equal(i18n.shouldPreserveNodeAuthoredText(customNodeData), true) }) +test('market scope changes market cards without rewriting filing source arguments', async () => { + const sourceConfig = await importTypeScript('lib/workflow/source-business-config.ts') + const sources = [ + { id: 'market', label: '行情', sourceGroup: 'market', site: 'eastmoney', command: 'gridlist', args: { market: 'hs-a' } }, + { id: 'filings', label: '公告', sourceGroup: 'filings', site: 'eastmoney', command: 'announcement', args: { market: 'SHA,SZA,BJA' } }, + { id: 'news', label: '新闻', sourceGroup: 'news', site: 'cls', command: 'telegraph', args: { limit: 30 } }, + ] + + assert.equal(sourceConfig.sourceMarket(sources), 'hs-a') + const updated = sourceConfig.updateSourceMarket(sources, 'bj-a') + assert.equal(updated[0].args.market, 'bj-a') + assert.equal(updated[1].args.market, 'SHA,SZA,BJA') + assert.deepStrictEqual(updated[2].args, { limit: 30 }) +}) + +test('source identity separates a capability from its configured query', async () => { + const sourceConfig = await importTypeScript('lib/workflow/source-business-config.ts') + const base = { + site: ' Bilibili ', + command: ' Search ', + args: { limit: 20, type: 'video' }, + positionalArgs: ['A 股'], + } + + assert.equal( + sourceConfig.sourceCapabilityKey(base), + sourceConfig.sourceCapabilityKey({ ...base, site: 'bilibili', command: 'search' }), + ) + assert.equal( + sourceConfig.sourceSlotKey(base), + sourceConfig.sourceSlotKey({ + ...base, + site: 'bilibili', + command: 'search', + args: { type: 'video', limit: 20 }, + }), + ) + assert.notEqual( + sourceConfig.sourceSlotKey(base), + sourceConfig.sourceSlotKey({ ...base, positionalArgs: ['港股'] }), + ) + + const registered = sourceConfig.openCLISlotFromDataSource({ + id: 'source-1', + name: 'B 站视频搜索', + channel_type: 'opencli', + channel_config: { + site: 'bilibili', + command: 'search', + args: { limit: 20 }, + positional_args: ['A 股'], + }, + enabled: true, + tags: ['video'], + created_at: '', + updated_at: '', + }) + assert.deepStrictEqual(registered.positionalArgs, ['A 股']) +}) + test('backend node catalog overlays matching nodes without hiding runnable workflow capabilities', async () => { const [{ mergeBackendNodeCapabilityCatalog }, nodeCatalog] = await Promise.all([ importTypeScript('lib/workflow/backend-node-capability-adapter.ts'), @@ -348,9 +408,10 @@ test('OpenCLI preset results use a navigable two-pane catalog without changing n }) test('Studio materializes every searchable OpenCLI capability preset as a node', async () => { - const [adapterNodes, adapterCatalog, palette, editor] = await Promise.all([ + const [adapterNodes, adapterCatalog, pluginCatalog, palette, editor] = await Promise.all([ importTypeScript('lib/workflow/backend-opencli-adapter-nodes.ts'), importTypeScript('lib/workflow/opencli-adapter-catalog.ts'), + importTypeScript('lib/plugins/opencli-adapter-catalog.ts'), readFrontendSource('components/flow/command-palette.tsx'), readFrontendSource('components/flow/workflow-editor.tsx'), ]) @@ -397,10 +458,25 @@ test('Studio materializes every searchable OpenCLI capability preset as a node', assert.match(adapterNodes.openCLIAdapterNodePresentation(sourcePreset, 'zh-CN').description, /从 example 读取 search 数据/) assert.equal(adapterNodes.openCLIAdapterNodePresentation(sourcePreset, 'en-US').description, 'Search public records') assert.match(adapterNodes.openCLIAdapterNodeSearchText(sourcePreset), /read source source-slot 数据读取/) + assert.match(adapterNodes.openCLIAdapterNodeSearchText({ + ...sourcePreset, + id: 'opencli.adapter.bilibili.subtitle', + site: 'bilibili', + command: 'subtitle', + }), /b 站 · 字幕提取/) + const bilibiliRankingSearchText = adapterNodes.openCLIAdapterNodeSearchText({ + ...sourcePreset, + id: 'opencli.adapter.bilibili.ranking', + site: 'bilibili', + command: 'ranking', + }) + assert.match(bilibiliRankingSearchText, /b 站 · 视频排行榜/) + assert.match(bilibiliRankingSearchText, /b站/) assert.match(adapterNodes.openCLIAdapterNodeSearchText(writePreset), /write tool action 操作工具/) const oodaSourceIds = [ 'opencli.adapter.eastmoney.index-quote', 'opencli.adapter.eastmoney.money-flow', + 'opencli.adapter.tdx.hot-rank', 'opencli.adapter.xueqiu.stock-social', 'opencli.adapter.eastmoney.bbsj-summary', 'opencli.adapter.cninfo.disclosure-pdf', @@ -409,6 +485,8 @@ test('Studio materializes every searchable OpenCLI capability preset as a node', 'opencli.adapter.jin10.kuaixun', 'opencli.adapter.gelonghui.kuaixun', 'opencli.adapter.xueqiu.news', + 'opencli.adapter.douyin.search', + 'opencli.adapter.bilibili.subtitle', ] assert.deepStrictEqual( adapterNodes.featuredOpenCLIAdapterNodes(oodaSourceIds.map((id) => ({ id }))).map((node) => node.id), @@ -417,9 +495,19 @@ test('Studio materializes every searchable OpenCLI capability preset as a node', assert.deepStrictEqual( adapterNodes.featuredOpenCLIAdapterGroups(oodaSourceIds.map((id) => ({ id })), 'zh-CN') .map((group) => group.label), - ['行情、资金与交易结构', '财报、公告、研报与 PDF', '财经媒体与实时快讯', '社交舆情与全网观察'], + ['行情、资金与交易结构', '财报、公告、研报与 PDF', '财经媒体与实时快讯', '社交舆情与全网观察', '视频与多媒体情报'], ) - assert.match(palette, /国内全网 OODA 数据源/) + assert.match(palette, /消息与数据来源/) + assert.doesNotMatch(palette, /国内全网 OODA 数据源/) + assert.match(palette, /行情、官方披露、财经媒体、社交与视频/) + const inspector = await readFrontendSource('components/flow/inspector.tsx') + assert.match(inspector, /addContentSources/) + assert.match(inspector, /OPENCLI_SITUATION_SOURCES/) + assert.match(inspector, /restoreSource/) + assert.match(inspector, /key=\{configurationNodeId\}/) + assert.match(inspector, /presets\.filter\(\(source\) => !selectedKeys\.has\(sourceSlotKey\(source\)\)\)/) + assert.match(inspector, /sourceCardLabel/) + assert.match(inspector, /视频按来源卡片独立配置/) assert.match(palette, /loginRequired:\s*"需登录"/) assert.doesNotMatch(palette, /featuredOpenCLIAdapterNodes\(matchingOpenCLINodes\)\.filter/) assert.equal(adapterCatalog.openCLIAdapterNodeToCatalogItem(sourcePreset).params.opencliAdapterNodeId, sourcePreset.id) @@ -444,11 +532,63 @@ test('Studio materializes every searchable OpenCLI capability preset as a node', }), true, ) + const directoryFixture = [ + ...Array.from({ length: 75 }, (_, index) => ({ + ...sourcePreset, + id: `opencli.adapter.a-site-${String(index).padStart(2, '0')}.list`, + site: `a-site-${String(index).padStart(2, '0')}`, + command: 'list', + adapter: { id: `opencli-a-site-${String(index).padStart(2, '0')}` }, + })), + { + ...sourcePreset, + id: 'opencli.adapter.discord-app.messages', + site: 'discord-app', + command: 'messages', + adapter: { id: 'opencli-discord-app' }, + }, + { + ...sourcePreset, + id: 'opencli.adapter.weixin.search-articles', + site: 'weixin', + command: 'search-articles', + adapter: { id: 'opencli-weixin' }, + }, + { + ...sourcePreset, + id: 'opencli.adapter.cninfo-reports.market-reports', + site: 'cninfo-reports', + command: 'market-reports', + adapter: { id: 'opencli-cninfo-reports' }, + }, + ] + const directory = pluginCatalog.groupOpenCLIAdapterPlugins(directoryFixture) + assert.equal(directory.length, 78) + assert.equal(directory.find((site) => site.site === 'discord-app').siteCategory, 'local-app') + assert.equal(directory.find((site) => site.site === 'weixin').commands[0].id, 'opencli.adapter.weixin.search-articles') + assert.equal(directory.find((site) => site.site === 'cninfo-reports').siteCategory, 'finance') + assert.deepStrictEqual(pluginCatalog.openCLIKeyboardCandidates('', null, directoryFixture), []) + assert.equal( + pluginCatalog.openCLIKeyboardCandidates( + '', + directory.find((site) => site.site === 'weixin'), + directoryFixture, + )[0].id, + 'opencli.adapter.weixin.search-articles', + ) + assert.equal( + adapterNodes.workflowCatalogItemForOpenCLIAdapterNode( + directory.find((site) => site.site === 'cninfo-reports').commands[0], + ).params.opencliAdapterNodeId, + 'opencli.adapter.cninfo-reports.market-reports', + ) assert.match(palette, /adapterCatalogResponse\?\.nodes \?\? fallbackOpenCLINodes/) assert.match(palette, /includeWrite: true, limit: 5000/) assert.match(palette, /!workflowCatalogItemIsOpenCLIAdapterPreset\(item\)/) - assert.match(palette, /OPENCLI_RESULT_LIMIT = 60/) assert.match(palette, /OPENCLI_SEARCH_RESULT_LIMIT = 120/) + assert.match(palette, /groupOpenCLIAdapterPlugins\(matchingOpenCLINodes\)/) + assert.match(palette, /openCLIKeyboardCandidates/) + assert.doesNotMatch(palette, /OPENCLI_RESULT_LIMIT = 60/) assert.match(palette, /opencliPresetGroups/) assert.match(palette, /item\.site.*openCLIPresetKind\(item\)/s) assert.match(palette, /运行前设置/) diff --git a/tests/integration/test_workflow_capabilities_api.py b/tests/integration/test_workflow_capabilities_api.py index 7c708b8..415e978 100644 --- a/tests/integration/test_workflow_capabilities_api.py +++ b/tests/integration/test_workflow_capabilities_api.py @@ -644,6 +644,23 @@ def fake_run(*args, **kwargs): opencli_adapter_nodes.refresh_opencli_adapter_catalog() +def test_opencli_adapter_nodes_reject_invalid_utf8_catalog(monkeypatch): + def fake_run(*args, **kwargs): + assert kwargs["encoding"] == "utf-8" + assert "errors" not in kwargs + raise UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte") + + monkeypatch.setattr(opencli_adapter_nodes, "resolve_opencli_bin", lambda: "opencli-test") + monkeypatch.setattr(opencli_adapter_nodes.subprocess, "run", fake_run) + opencli_adapter_nodes.refresh_opencli_adapter_catalog() + + response = list_opencli_adapter_nodes(refresh=True) + + assert response.total == 0 + assert response.nodes == [] + opencli_adapter_nodes.refresh_opencli_adapter_catalog() + + @pytest.mark.asyncio async def test_opencli_adapter_nodes_endpoint_filters_and_limits(client, monkeypatch): monkeypatch.setattr(