diff --git a/src/refract/ingest.py b/src/refract/ingest.py index b235daa..ccb8b6f 100644 --- a/src/refract/ingest.py +++ b/src/refract/ingest.py @@ -73,6 +73,10 @@ def fetch_url(url: str) -> dict: """ Fetch and extract article text from a URL using trafilatura. Returns a record dict with url, title, text, word_count, article_id. + + No retries: a failed fetch (timeout, 404, dead link) is usually a stale + URL, not a transient blip, so retrying just burns time before the + per-article failure batch_eval.py already handles by moving on. """ headers = { "User-Agent": ( @@ -81,43 +85,35 @@ def fetch_url(url: str) -> dict: ) } - for attempt in range(1, MAX_RETRIES + 1): - try: - downloaded = trafilatura.fetch_url(url) - if not downloaded: - # trafilatura fetch failed — try requests with browser UA - resp = requests.get(url, headers=headers, timeout=20) - if resp.status_code == 404: - raise ValueError(f"404 Not Found: {url}") - resp.raise_for_status() - downloaded = resp.text - if not downloaded: - raise ValueError(f"empty response from {url}") - - text = trafilatura.extract(downloaded, include_comments=False, include_tables=False) - if not text: - raise ValueError(f"trafilatura extracted empty text from {url}") - - meta = trafilatura.extract_metadata(downloaded) - title = meta.title if meta and meta.title else "" - - article_id = _sha256(url, text) - record = { - "article_id": article_id, - "url": url, - "title": title, - "source": _source_from_url(url), - "text": text, - "word_count": len(text.split()), - } - _write_cache(article_id, record) - return record - - except Exception as e: - logger.warning("fetch_url attempt %d failed for %s: %s", attempt, url, e) - if attempt == MAX_RETRIES: - raise - time.sleep(RETRY_BASE_DELAY * (2 ** (attempt - 1))) + downloaded = trafilatura.fetch_url(url) + if not downloaded: + # trafilatura fetch failed — try requests with browser UA + resp = requests.get(url, headers=headers, timeout=20) + if resp.status_code == 404: + raise ValueError(f"404 Not Found: {url}") + resp.raise_for_status() + downloaded = resp.text + if not downloaded: + raise ValueError(f"empty response from {url}") + + text = trafilatura.extract(downloaded, include_comments=False, include_tables=False) + if not text: + raise ValueError(f"trafilatura extracted empty text from {url}") + + meta = trafilatura.extract_metadata(downloaded) + title = meta.title if meta and meta.title else "" + + article_id = _sha256(url, text) + record = { + "article_id": article_id, + "url": url, + "title": title, + "source": _source_from_url(url), + "text": text, + "word_count": len(text.split()), + } + _write_cache(article_id, record) + return record def fetch_guardian(query: str, page_size: int = 10, section: str = "") -> list[dict]: diff --git a/src/refract/llm_client.py b/src/refract/llm_client.py index 383d7ab..0548377 100644 --- a/src/refract/llm_client.py +++ b/src/refract/llm_client.py @@ -234,6 +234,16 @@ def _bare_model(model: str) -> str: return model.split("/", 1)[1] if "/" in model else model +# Groq's catalog namespaces some hosted models under the upstream provider +# (e.g. "openai/gpt-oss-120b"), which collides with this codebase's own +# "groq/" routing prefix used to disambiguate from Cerebras's identically +# named "gpt-oss-120b". Map our routing alias to Groq's real model id before +# calling the API — otherwise the bare-stripped "gpt-oss-120b" 404s. +_GROQ_MODEL_ALIASES = { + "gpt-oss-120b": "openai/gpt-oss-120b", +} + + def _provider(model: str) -> str: """ Infer API provider from model name. @@ -256,7 +266,9 @@ def _provider(model: str) -> str: def _call_groq(prompt: str, model: str, system: str = "", expect_json: bool = True, temperature: float = 0.0) -> tuple[dict | str, int]: - return _call_openai_compat(prompt, _bare_model(model), system, expect_json, temperature, GROQ_BASE, GROQ_API_KEY, "groq") + bare = _bare_model(model) + real_model = _GROQ_MODEL_ALIASES.get(bare, bare) + return _call_openai_compat(prompt, real_model, system, expect_json, temperature, GROQ_BASE, GROQ_API_KEY, "groq") def _parse_json_response(raw: str) -> Any: @@ -331,7 +343,7 @@ def _call_openai_compat( messages.append({"role": "user", "content": prompt}) body: dict[str, Any] = { - "model": _bare_model(model), + "model": model, "messages": messages, "temperature": temperature, }