diff --git a/robosystems_client/clients/document_client.py b/robosystems_client/clients/document_client.py index 40a4cc2..479d642 100644 --- a/robosystems_client/clients/document_client.py +++ b/robosystems_client/clients/document_client.py @@ -20,6 +20,7 @@ from ..api.search.get_document_section import sync_detailed as get_document_section from ..api.search.search_documents import sync_detailed as search_documents from ..client import AuthenticatedClient +from .retry import retrying_authenticated_client from ..models.delete_document_op import DeleteDocumentOp from ..models.document_detail_response import DocumentDetailResponse from ..models.document_list_response import DocumentListResponse @@ -44,12 +45,11 @@ def __init__(self, config: Dict[str, Any]): def _get_client(self) -> AuthenticatedClient: if not self.token: raise Exception("No API key provided. Set X-API-Key in headers.") - return AuthenticatedClient( + return retrying_authenticated_client( base_url=self.base_url, token=self.token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) def upload( diff --git a/robosystems_client/clients/file_client.py b/robosystems_client/clients/file_client.py index 31cd1ff..ecfe427 100644 --- a/robosystems_client/clients/file_client.py +++ b/robosystems_client/clients/file_client.py @@ -137,17 +137,16 @@ def upload( table_name=table_name, ) - from ..client import AuthenticatedClient + from .retry import retrying_authenticated_client if not self.token: raise Exception("No API key provided. Set X-API-Key in headers.") - client = AuthenticatedClient( + client = retrying_authenticated_client( base_url=self.base_url, token=self.token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) kwargs = { @@ -294,17 +293,16 @@ def list( List of FileInfo objects """ try: - from ..client import AuthenticatedClient + from .retry import retrying_authenticated_client if not self.token: raise Exception("No API key provided. Set X-API-Key in headers.") - client = AuthenticatedClient( + client = retrying_authenticated_client( base_url=self.base_url, token=self.token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) kwargs = { @@ -357,17 +355,16 @@ def get(self, graph_id: str, file_id: str) -> Optional[FileInfo]: FileInfo with multi-layer status tracking, or None if not found """ try: - from ..client import AuthenticatedClient + from .retry import retrying_authenticated_client if not self.token: raise Exception("No API key provided. Set X-API-Key in headers.") - client = AuthenticatedClient( + client = retrying_authenticated_client( base_url=self.base_url, token=self.token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) kwargs = { @@ -414,17 +411,16 @@ def delete(self, graph_id: str, file_id: str, cascade: bool = False) -> bool: True if deletion succeeded, False otherwise """ try: - from ..client import AuthenticatedClient + from .retry import retrying_authenticated_client if not self.token: raise Exception("No API key provided. Set X-API-Key in headers.") - client = AuthenticatedClient( + client = retrying_authenticated_client( base_url=self.base_url, token=self.token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) delete_op = DeleteFileOp(file_id=file_id, cascade=cascade) diff --git a/robosystems_client/clients/graph_client.py b/robosystems_client/clients/graph_client.py index 9594cf4..6c2bc64 100644 --- a/robosystems_client/clients/graph_client.py +++ b/robosystems_client/clients/graph_client.py @@ -116,17 +116,16 @@ def operation_client(self) -> OperationClient: def _get_authenticated_client(self): """Build an AuthenticatedClient for API calls.""" - from ..client import AuthenticatedClient + from .retry import retrying_authenticated_client if not self.token: raise ValueError("No API key provided. Set X-API-Key in headers.") - return AuthenticatedClient( + return retrying_authenticated_client( base_url=self.base_url, token=self.token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) # --------------------------------------------------------------------------- diff --git a/robosystems_client/clients/investor_client.py b/robosystems_client/clients/investor_client.py index 4346b80..c571f63 100644 --- a/robosystems_client/clients/investor_client.py +++ b/robosystems_client/clients/investor_client.py @@ -41,6 +41,11 @@ sync_detailed as op_update_security, ) from ..client import AuthenticatedClient +from .retry import ( + DEFAULT_MAX_RETRIES, + DEFAULT_RETRY_DELAY_MS, + retrying_authenticated_client, +) from ..graphql.client import GraphQLClient, strip_none_vars from .token_utils import resolve_config_token from ..graphql.generated.get_investor_holdings import ( @@ -122,12 +127,11 @@ def _get_client(self) -> AuthenticatedClient: token = resolve_config_token(self.config) if not token: raise RuntimeError("No API key provided. Set X-API-Key in headers.") - return AuthenticatedClient( + return retrying_authenticated_client( base_url=self.base_url, token=token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) def _get_graphql_client(self) -> GraphQLClient: @@ -139,6 +143,8 @@ def _get_graphql_client(self) -> GraphQLClient: token=token, headers=self.headers, timeout=self.timeout, + max_retries=self.config.get("max_retries", DEFAULT_MAX_RETRIES), + retry_delay_ms=self.config.get("retry_delay", DEFAULT_RETRY_DELAY_MS), ) def _query( diff --git a/robosystems_client/clients/ledger_client.py b/robosystems_client/clients/ledger_client.py index 66b1b6a..21817fe 100644 --- a/robosystems_client/clients/ledger_client.py +++ b/robosystems_client/clients/ledger_client.py @@ -167,6 +167,11 @@ sync_detailed as op_update_journal_entry, ) from ..client import AuthenticatedClient +from .retry import ( + DEFAULT_MAX_RETRIES, + DEFAULT_RETRY_DELAY_MS, + retrying_authenticated_client, +) from ..graphql.client import GraphQLClient, strip_none_vars from .token_utils import resolve_config_token from ..graphql.generated.get_information_block import ( @@ -585,12 +590,11 @@ def _get_client(self) -> AuthenticatedClient: token = resolve_config_token(self.config) if not token: raise RuntimeError("No API key provided. Set X-API-Key in headers.") - return AuthenticatedClient( + return retrying_authenticated_client( base_url=self.base_url, token=token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) def _get_graphql_client(self) -> GraphQLClient: @@ -609,6 +613,8 @@ def _get_graphql_client(self) -> GraphQLClient: token=token, headers=self.headers, timeout=self.timeout, + max_retries=self.config.get("max_retries", DEFAULT_MAX_RETRIES), + retry_delay_ms=self.config.get("retry_delay", DEFAULT_RETRY_DELAY_MS), ) # ── Helpers ───────────────────────────────────────────────────────── diff --git a/robosystems_client/clients/library_client.py b/robosystems_client/clients/library_client.py index 17a5c00..fd2135c 100644 --- a/robosystems_client/clients/library_client.py +++ b/robosystems_client/clients/library_client.py @@ -22,6 +22,7 @@ from typing import Any from ..graphql.client import GraphQLClient, strip_none_vars +from .retry import DEFAULT_MAX_RETRIES, DEFAULT_RETRY_DELAY_MS from .token_utils import resolve_config_token from ..graphql.generated.get_library_element import ( GetLibraryElement, @@ -110,6 +111,8 @@ def _get_graphql_client(self) -> GraphQLClient: token=token, headers=self.headers, timeout=self.timeout, + max_retries=self.config.get("max_retries", DEFAULT_MAX_RETRIES), + retry_delay_ms=self.config.get("retry_delay", DEFAULT_RETRY_DELAY_MS), ) def _query( diff --git a/robosystems_client/clients/retry.py b/robosystems_client/clients/retry.py new file mode 100644 index 0000000..8ee51ee --- /dev/null +++ b/robosystems_client/clients/retry.py @@ -0,0 +1,176 @@ +"""Rate-limit-aware HTTP client used by the hand-written facades. + +The API rate-limits per user per endpoint category and answers an +exhausted budget with ``429`` plus ``Retry-After`` / ``X-RateLimit-*`` +headers. That rejection is raised by a request dependency *before* the +endpoint handler runs, so the request had no effect and is always safe +to replay — including a ``POST`` carrying no idempotency key. Nothing +other than ``429`` is retried here, precisely because nothing else +carries that guarantee. + +The motivating case is a bulk backfill: an integrator loading a year of +history through per-event write calls runs at the category budget for +minutes at a time, and without this a burst of rejections turns into +silently missing rows. +""" + +from __future__ import annotations + +import random +import time +from typing import Any + +import httpx + +from ..client import AuthenticatedClient + +RETRY_STATUS_CODES = frozenset({429}) + +DEFAULT_MAX_RETRIES = 5 +DEFAULT_RETRY_DELAY_MS = 1000 +MAX_BACKOFF_SECONDS = 30.0 + + +def retry_after_seconds(response: httpx.Response) -> float | None: + """Parse ``Retry-After`` as a delta-seconds value, or ``None``. + + The API always sends the numeric form. The HTTP-date form is ignored + rather than parsed, since treating an unreadable value as "no hint" + degrades to plain backoff instead of to a wrong sleep. + """ + raw = response.headers.get("retry-after") + if not raw: + return None + try: + value = float(raw.strip()) + except ValueError: + return None + return value if value >= 0 else None + + +def backoff_seconds( + attempt: int, retry_delay_ms: int, retry_after: float | None +) -> float: + """Seconds to wait before replaying a rate-limited request. + + Exponential with full jitter, and ``Retry-After`` applied as a + *ceiling* rather than as the sleep itself. The limiter is a sliding + window, so ``Retry-After`` reports the whole window — the worst case + for a client that filled its budget instantaneously. A caller that + merely ran at the sustained rate has slots freeing up within a second + or two, and obeying the header literally would turn a handful of + rejections into minutes of idling. + """ + ceiling = (retry_delay_ms / 1000.0) * (2**attempt) + ceiling = min(ceiling, MAX_BACKOFF_SECONDS) + if retry_after is not None: + ceiling = min(ceiling, retry_after) + return random.uniform(ceiling / 2.0, ceiling) + + +def _is_replayable(request: httpx.Request) -> bool: + """Whether ``request`` can be sent a second time. + + httpx buffers byte and JSON bodies onto the request at construction + and leaves streaming bodies unread; a streamed body is consumed by + the first attempt, so replaying it would send an empty payload. + """ + return hasattr(request, "_content") + + +class RetryingClient(httpx.Client): + """``httpx.Client`` that replays rate-limited requests. + + Subclassed rather than installed as a custom transport on purpose: + httpx disables environment proxy detection whenever ``transport=`` is + supplied (``allow_env_proxies = trust_env and transport is None``), + so a transport-level retry would quietly break proxied callers. + Overriding :meth:`send` leaves proxy, mount, redirect and TLS + handling exactly as httpx configures it. + """ + + def __init__( + self, + *args: Any, + max_retries: int = DEFAULT_MAX_RETRIES, + retry_delay_ms: int = DEFAULT_RETRY_DELAY_MS, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self._max_retries = max(0, int(max_retries)) + self._retry_delay_ms = max(1, int(retry_delay_ms)) + + def send(self, request: httpx.Request, **kwargs: Any) -> httpx.Response: + for attempt in range(self._max_retries): + response = super().send(request, **kwargs) + if response.status_code not in RETRY_STATUS_CODES or not _is_replayable(request): + return response + delay = backoff_seconds( + attempt, self._retry_delay_ms, retry_after_seconds(response) + ) + # The rejection body is small and goes unused, but it holds the + # connection open until it is drained. + response.read() + response.close() + time.sleep(delay) + return super().send(request, **kwargs) + + +def build_httpx_client( + *, + base_url: str, + headers: dict[str, str] | None = None, + timeout: Any = None, + config: dict[str, Any] | None = None, +) -> RetryingClient: + """Build a :class:`RetryingClient` honoring a facade's config dict. + + ``max_retries`` / ``retry_delay`` come from + :class:`~.facade.RoboSystemsClientConfig`; facades built from a bare + dict (the demo scripts, the integration template) fall back to the + defaults. + """ + config = config or {} + return RetryingClient( + base_url=base_url, + headers=headers or {}, + timeout=timeout, + max_retries=config.get("max_retries", DEFAULT_MAX_RETRIES), + retry_delay_ms=config.get("retry_delay", DEFAULT_RETRY_DELAY_MS), + ) + + +def retrying_authenticated_client( + *, + base_url: str, + token: str, + headers: dict[str, str] | None = None, + auth_header_name: str = "X-API-Key", + prefix: str = "", + config: dict[str, Any] | None = None, +) -> AuthenticatedClient: + """An :class:`AuthenticatedClient` whose transport replays 429s. + + Mirrors what ``AuthenticatedClient.get_httpx_client()`` would build — + including stamping the credential onto the header the caller named — + and installs it through the public ``set_httpx_client`` hook, so the + generated ``api/`` layer is untouched and survives ``just + generate-sdk``. + + ``timeout`` is left unset to match the generated client's own + default; callers that need one set it on the facade. + """ + request_headers = dict(headers or {}) + request_headers[auth_header_name] = f"{prefix} {token}" if prefix else token + client = AuthenticatedClient( + base_url=base_url, + token=token, + prefix=prefix, + auth_header_name=auth_header_name, + headers=dict(headers or {}), + ) + return client.set_httpx_client( + build_httpx_client( + base_url=base_url, headers=request_headers, timeout=None, config=config + ) + ) diff --git a/robosystems_client/clients/table_client.py b/robosystems_client/clients/table_client.py index b5ca60a..d41fe06 100644 --- a/robosystems_client/clients/table_client.py +++ b/robosystems_client/clients/table_client.py @@ -62,17 +62,16 @@ def list(self, graph_id: str) -> list[TableInfo]: List of TableInfo objects with metadata """ try: - from ..client import AuthenticatedClient + from .retry import retrying_authenticated_client if not self.token: raise Exception("No API key provided. Set X-API-Key in headers.") - client = AuthenticatedClient( + client = retrying_authenticated_client( base_url=self.base_url, token=self.token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) kwargs = { @@ -133,17 +132,16 @@ def query( request = SqlStatementRequest(sql=final_query) - from ..client import AuthenticatedClient + from .retry import retrying_authenticated_client if not self.token: raise Exception("No API key provided. Set X-API-Key in headers.") - client = AuthenticatedClient( + client = retrying_authenticated_client( base_url=self.base_url, token=self.token, - prefix="", - auth_header_name="X-API-Key", headers=self.headers, + config=self.config, ) kwargs = { diff --git a/robosystems_client/graphql/client.py b/robosystems_client/graphql/client.py index 16a34b4..9f4c274 100644 --- a/robosystems_client/graphql/client.py +++ b/robosystems_client/graphql/client.py @@ -20,7 +20,12 @@ import re from typing import Any -import httpx + +from ..clients.retry import ( + DEFAULT_MAX_RETRIES, + DEFAULT_RETRY_DELAY_MS, + RetryingClient, +) class GraphQLError(Exception): @@ -60,9 +65,13 @@ def __init__( token: str | None = None, headers: dict[str, str] | None = None, timeout: float = 60.0, + max_retries: int = DEFAULT_MAX_RETRIES, + retry_delay_ms: int = DEFAULT_RETRY_DELAY_MS, ): self.base_url = base_url.rstrip("/") self.timeout = timeout + self.max_retries = max_retries + self.retry_delay_ms = retry_delay_ms self._headers: dict[str, str] = {"Content-Type": "application/json"} if headers: self._headers.update(headers) @@ -112,7 +121,11 @@ def execute( payload["operationName"] = operation_name url = self._url_for(graph_id) - with httpx.Client(timeout=self.timeout) as client: + with RetryingClient( + timeout=self.timeout, + max_retries=self.max_retries, + retry_delay_ms=self.retry_delay_ms, + ) as client: response = client.post(url, json=payload, headers=self._headers) if response.status_code >= 400: diff --git a/tests/test_rate_limit_retry.py b/tests/test_rate_limit_retry.py new file mode 100644 index 0000000..75c8786 --- /dev/null +++ b/tests/test_rate_limit_retry.py @@ -0,0 +1,213 @@ +"""Unit tests for the 429 replay behavior shared by every facade. + +The API answers an exhausted category budget with 429 from a request +dependency that runs before the endpoint handler, so the rejected call +had no effect and replaying it is safe. A bulk backfill loop — a year of +history through per-event writes — sits at the category budget for +minutes, and before this the rejections surfaced to callers as ordinary +failures. +""" + +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Any, Iterator + +import httpx +import pytest + +from robosystems_client.clients.ledger_client import LedgerClient +from robosystems_client.clients.retry import ( + RetryingClient, + backoff_seconds, + retry_after_seconds, + retrying_authenticated_client, +) + + +class _Stub: + """A server that rejects the first ``fail_first`` requests with 429.""" + + def __init__(self, fail_first: int, body: bytes = b"{}") -> None: + self.calls = 0 + self.paths: list[str] = [] + self.fail_first = fail_first + self.body = body + stub = self + + class Handler(BaseHTTPRequestHandler): + def _respond(self) -> None: + stub.calls += 1 + stub.paths.append(self.path) + length = int(self.headers.get("content-length", 0)) + if length: + _ = self.rfile.read(length) + if stub.calls <= stub.fail_first: + payload = b'{"detail":"Rate limit exceeded for extensions write operations."}' + self.send_response(429) + # The window, not a usable delay — see backoff_seconds. + self.send_header("Retry-After", "60") + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + _ = self.wfile.write(payload) + return + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(stub.body))) + self.end_headers() + _ = self.wfile.write(stub.body) + + do_GET = _respond + do_POST = _respond + + def log_message(self, *args: Any) -> None: + pass + + self._server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + @property + def base_url(self) -> str: + return f"http://127.0.0.1:{self._server.server_port}" + + def close(self) -> None: + self._server.shutdown() + + +@pytest.fixture +def stub() -> Iterator[_Stub]: + server = _Stub(fail_first=0) + yield server + server.close() + + +@pytest.mark.unit +class TestBackoff: + def test_retry_after_is_a_ceiling_not_the_sleep(self): + # The limiter is a sliding window, so Retry-After reports the whole + # window. Obeying it literally turns a few rejections into minutes. + assert backoff_seconds(0, 1000, 60.0) <= 1.0 + assert backoff_seconds(4, 1000, 2.0) <= 2.0 + + def test_backoff_grows_but_stays_bounded(self): + assert backoff_seconds(0, 1000, None) <= 1.0 + assert backoff_seconds(4, 1000, None) <= 16.0 + assert backoff_seconds(20, 1000, None) <= 30.0 + + def test_retry_after_parsing(self): + def response(value: str | None): + headers = httpx.Headers({"Retry-After": value} if value is not None else {}) + return type("R", (), {"headers": headers})() + + assert retry_after_seconds(response("60")) == 60.0 + assert retry_after_seconds(response(None)) is None + # The HTTP-date form is ignored rather than mis-parsed. + assert retry_after_seconds(response("Wed, 21 Oct 2026 07:28:00 GMT")) is None + + +@pytest.mark.unit +class TestRetryingClient: + def test_replays_until_accepted(self, stub: _Stub): + stub.fail_first = 3 + with RetryingClient(max_retries=5, retry_delay_ms=1) as client: + response = client.post(f"{stub.base_url}/x", json={"event": "invoice_issued"}) + + assert response.status_code == 200 + assert stub.calls == 4 + + def test_surfaces_the_429_once_retries_are_exhausted(self, stub: _Stub): + stub.fail_first = 99 + with RetryingClient(max_retries=2, retry_delay_ms=1) as client: + response = client.post(f"{stub.base_url}/x", json={}) + + assert response.status_code == 429 + assert stub.calls == 3 + assert "Rate limit exceeded" in response.text + + def test_zero_retries_sends_once(self, stub: _Stub): + stub.fail_first = 99 + with RetryingClient(max_retries=0, retry_delay_ms=1) as client: + response = client.post(f"{stub.base_url}/x", json={}) + + assert response.status_code == 429 + assert stub.calls == 1 + + def test_streamed_body_is_not_replayed(self, stub: _Stub): + # httpx leaves a streaming body unread, so the first attempt consumes + # it and a replay would post nothing. + stub.fail_first = 99 + + def body() -> Iterator[bytes]: + yield b'{"a": 1}' + + with RetryingClient(max_retries=3, retry_delay_ms=1) as client: + response = client.post(f"{stub.base_url}/x", content=body()) + + assert response.status_code == 429 + assert stub.calls == 1 + + def test_other_statuses_are_returned_untouched(self, stub: _Stub): + stub.fail_first = 0 + with RetryingClient(max_retries=5, retry_delay_ms=1) as client: + response = client.get(f"{stub.base_url}/x") + + assert response.status_code == 200 + assert stub.calls == 1 + + +@pytest.mark.unit +class TestFacadeWiring: + def test_authenticated_client_carries_the_credential_and_retries(self, stub: _Stub): + stub.fail_first = 2 + client = retrying_authenticated_client( + base_url=stub.base_url, + token="rfs_test", + headers={"X-Trace": "1"}, + config={"max_retries": 5, "retry_delay": 1}, + ) + http = client.get_httpx_client() + response = http.post("/x", json={}) + + assert response.status_code == 200 + assert stub.calls == 3 + assert http.headers["X-API-Key"] == "rfs_test" + assert http.headers["X-Trace"] == "1" + + def test_ledger_write_survives_a_rate_limit_burst(self, stub: _Stub): + # The end-to-end shape the demo backfill hits: create-event-block + # rejected mid-loop, then accepted on replay. + stub.fail_first = 2 + stub.body = ( + b'{"operation": "create-event-block", "operationId": "op_1",' + b' "status": "completed", "at": "2026-02-12T00:00:00+00:00",' + b' "result": {"id": "evt_1", "event_type": "invoice_issued",' + b' "event_category": "sales", "status": "posted",' + b' "occurred_at": "2026-02-12T00:00:00+00:00", "source": "manual",' + b' "currency": "USD", "metadata": {}, "dimension_ids": [],' + b' "event_class": "economic",' + b' "created_at": "2026-02-12T00:00:00+00:00", "created_by": "demo"}}' + ) + ledger = LedgerClient( + { + "base_url": stub.base_url, + "token": "rfs_test", + "headers": {}, + "max_retries": 5, + "retry_delay": 1, + } + ) + + result = ledger.create_event_block( + "kg_test", + { + "event_type": "invoice_issued", + "event_category": "sales", + "source": "manual", + "occurred_at": "2026-02-12T00:00:00+00:00", + "metadata": {"memo": "demo"}, + }, + ) + + assert stub.calls == 3 + assert result.id == "evt_1" + assert all(p.endswith("/operations/create-event-block") for p in stub.paths)