Skip to content

Commit 94ab53d

Browse files
1012839419a-alt1012839419a-alt
authored
feat(doubao): capture conversation_url and classify captcha blocks (#63)
1 parent 7838811 commit 94ab53d

2 files changed

Lines changed: 154 additions & 3 deletions

File tree

‎backend/channels/doubao_research_channel.py‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,18 @@
77
from backend.channels.base import AbstractChannel, Capabilities, ChannelResult
88
from backend.channels.registry import register_channel
99

10-
_URL_RE = re.compile(r"https?://[^\s<>\]\[\](){}\"']+", re.IGNORECASE)
10+
_URL_RE = re.compile(r"https?://[^\s<>\[\](){}'\"]+", re.IGNORECASE)
1111
_TRAILING_URL_PUNCTUATION = ".,;:!?\uff0c\u3002\uff1b\uff1a\uff01\uff1f"
12+
#: OpenCLI adapter reports a captcha wall this way (verified on opencli 1.8.6).
13+
_CAPTCHA_MARKERS = (
14+
"verification challenge",
15+
"captcha",
16+
"blocked the request",
17+
"人机验证",
18+
"验证码",
19+
)
20+
21+
1222
def _citations(text: str) -> list[dict[str, str]]:
1323
"""Extract and de-duplicate URLs while preserving the answer's order."""
1424
seen: set[str] = set()
@@ -37,6 +47,25 @@ def _answer(rows: list[dict[str, Any]]) -> str:
3747
).strip()
3848

3949

50+
def _conversation_url(stdout: str) -> str:
51+
"""Extract https://www.doubao.com/chat/<id> from `doubao status -f json` output."""
52+
try:
53+
rows = _parse_opencli_rows(stdout)
54+
except Exception:
55+
return ""
56+
for row in rows:
57+
url = str(row.get("Url", row.get("url", "")) or "").strip()
58+
if "/chat/" in url:
59+
return url
60+
return ""
61+
62+
63+
def _is_captcha_block(stderr: str, stdout: str) -> bool:
64+
"""True when the adapter reports a captcha/verification wall."""
65+
text = f"{stderr} {stdout}".lower()
66+
return any(marker in text for marker in _CAPTCHA_MARKERS)
67+
68+
4069
async def _run_doubao_command(command: list[str]) -> tuple[int, str, str]:
4170
"""Late import avoids the channel registry's legacy OpenCLI import cycle."""
4271
from backend.channels.opencli_channel import _run_opencli
@@ -102,8 +131,12 @@ async def collect(self, config: dict[str, Any], parameters: dict[str, Any]) -> C
102131
)
103132

104133
if returncode:
134+
# Classify captcha walls so the runner can apply a human-in-the-loop
135+
# or cooldown-retry policy instead of treating it as a permanent failure.
136+
error_type = "captcha_challenge" if _is_captcha_block(stderr, stdout) else None
105137
return ChannelResult.fail(
106-
f"opencli doubao ask exited with code {returncode}: {stderr[:500]}"
138+
f"opencli doubao ask exited with code {returncode}: {stderr[:500]}",
139+
error_type=error_type,
107140
)
108141
try:
109142
answer = _answer(_parse_opencli_rows(stdout))
@@ -114,6 +147,28 @@ async def collect(self, config: dict[str, Any], parameters: dict[str, Any]) -> C
114147
if not answer:
115148
return ChannelResult.fail("Doubao returned no assistant text")
116149

150+
# Best-effort conversation URL: `doubao status -f json` exposes the
151+
# active chat id (https://www.doubao.com/chat/<id>). This is a
152+
# read-only query against the same browser session; a failure here
153+
# must not fail the collect — the answer is already in hand.
154+
conversation_url = ""
155+
if config.get("capture_conversation_url", True):
156+
status_command = [
157+
_opencli_binary(),
158+
"doubao",
159+
"status",
160+
"-f",
161+
"json",
162+
"--site-session",
163+
str(config.get("site_session", "ephemeral")),
164+
]
165+
try:
166+
rc, so, se = await _run_doubao_command(status_command)
167+
if rc == 0:
168+
conversation_url = _conversation_url(so)
169+
except Exception:
170+
conversation_url = ""
171+
117172
citations = _citations(answer) if extract_citations else []
118173
return ChannelResult.ok(
119174
[
@@ -122,6 +177,7 @@ async def collect(self, config: dict[str, Any], parameters: dict[str, Any]) -> C
122177
"content": answer,
123178
"author": "doubao",
124179
"question": question,
180+
"conversation_url": conversation_url,
125181
"citations": citations,
126182
"citation_count": len(citations),
127183
"citation_capture": (

‎tests/unit/channels/test_doubao_research_channel.py‎

Lines changed: 96 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import pytest
22

3-
from backend.channels.doubao_research_channel import DoubaoResearchChannel, _citations
3+
from backend.channels.doubao_research_channel import (
4+
DoubaoResearchChannel,
5+
_citations,
6+
_conversation_url,
7+
)
48
from backend.schemas.source import DataSourceCreate
59

610

@@ -13,6 +17,29 @@ def test_citations_preserve_order_and_strip_punctuation():
1317
]
1418

1519

20+
def test_conversation_url_extracts_chat_id():
21+
status = (
22+
'[{"Status": "Connected", "Url": '
23+
'"https://www.doubao.com/chat/38436240748612354", "Title": "x"}]'
24+
)
25+
assert (
26+
_conversation_url(status)
27+
== "https://www.doubao.com/chat/38436240748612354"
28+
)
29+
30+
31+
def test_conversation_url_ignores_root_chat():
32+
# A freshly opened /chat page has no conversation id yet — must not be picked up.
33+
status = (
34+
'[{"Status": "Connected", "Url": "https://www.doubao.com/chat", "Title": "x"}]'
35+
)
36+
assert _conversation_url(status) == ""
37+
38+
39+
def test_conversation_url_tolerates_garbage():
40+
assert _conversation_url("not json at all") == ""
41+
42+
1643
@pytest.mark.asyncio
1744
async def test_collect_stores_answer_and_citations(monkeypatch):
1845
async def fake_run(command):
@@ -48,8 +75,76 @@ async def fake_run(command):
4875
assert result.items[0]["citations"] == [{"url": "https://example.com/"}]
4976

5077

78+
@pytest.mark.asyncio
79+
async def test_collect_captures_conversation_url(monkeypatch):
80+
calls = []
81+
82+
async def fake_run(command):
83+
calls.append(command)
84+
if command[2] == "ask":
85+
return 0, '[{"Role":"assistant","Text":"回答"}]', ""
86+
if command[2] == "status":
87+
return 0, (
88+
'[{"Status": "Connected", "Url": '
89+
'"https://www.doubao.com/chat/12345", "Title": "t"}]'
90+
), ""
91+
return 0, "", ""
92+
93+
monkeypatch.setattr("backend.channels.doubao_research_channel._run_doubao_command", fake_run)
94+
result = await DoubaoResearchChannel().collect({"question": "测试"}, {})
95+
96+
assert result.success
97+
assert result.items[0]["conversation_url"] == "https://www.doubao.com/chat/12345"
98+
# ask + status both hit the adapter
99+
assert [c[2] for c in calls] == ["ask", "status"]
100+
101+
102+
@pytest.mark.asyncio
103+
async def test_collect_tolerates_status_failure(monkeypatch):
104+
async def fake_run(command):
105+
if command[2] == "ask":
106+
return 0, '[{"Role":"assistant","Text":"回答"}]', ""
107+
return 1, "", "status exploded"
108+
109+
monkeypatch.setattr("backend.channels.doubao_research_channel._run_doubao_command", fake_run)
110+
result = await DoubaoResearchChannel().collect({"question": "测试"}, {})
111+
112+
# A failed status must NOT fail the collect — answer is already in hand.
113+
assert result.success
114+
assert result.items[0]["conversation_url"] == ""
115+
116+
117+
@pytest.mark.asyncio
118+
async def test_collect_classifies_captcha_block(monkeypatch):
119+
async def fake_run(command):
120+
return 1, "", (
121+
"ok: false\nerror:\n code: COMMAND_EXEC\n"
122+
" message: Doubao blocked the request with a verification challenge\n"
123+
" help: 'Detected challenge signal: iframe[src*=\"captcha\"]'"
124+
)
125+
126+
monkeypatch.setattr("backend.channels.doubao_research_channel._run_doubao_command", fake_run)
127+
result = await DoubaoResearchChannel().collect({"question": "测试"}, {})
128+
129+
assert not result.success
130+
assert result.error_type == "captcha_challenge"
131+
132+
133+
@pytest.mark.asyncio
134+
async def test_collect_does_not_classify_generic_error(monkeypatch):
135+
async def fake_run(command):
136+
return 1, "", "some unrelated error"
137+
138+
monkeypatch.setattr("backend.channels.doubao_research_channel._run_doubao_command", fake_run)
139+
result = await DoubaoResearchChannel().collect({"question": "测试"}, {})
140+
141+
assert not result.success
142+
assert result.error_type is None
143+
144+
51145
def test_source_schema_accepts_doubao_research_channel():
52146
source = DataSourceCreate(
53147
name="Doubao research", channel_type="doubao_research", channel_config={"question": "test"}
54148
)
55149
assert source.channel_type == "doubao_research"
150+

0 commit comments

Comments
 (0)