From 0438c700ab46142267b8c36039ab820b96025d98 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 20 Jul 2026 09:49:37 -0400 Subject: [PATCH] Extract PRISM coordinator core owners --- lab/prism/background_services.py | 200 + lab/prism/bounded_executor.py | 171 + lab/prism/bundle_compiler.py | 433 + lab/prism/coordinator_config.py | 1167 ++ lab/prism/coordinator_shutdown.py | 278 + lab/prism/ctv_runtime.py | 397 + lab/prism/job_build_benchmark.py | 20 +- lab/prism/job_bundle.py | 2463 +++ lab/prism/job_delivery.py | 3472 +++ lab/prism/payout_state.py | 1817 ++ lab/prism/prism_coordinator.py | 17399 +++++----------- lab/prism/progress_health.py | 624 + lab/prism/rpc.py | 112 + lab/prism/run_ctv_broadcaster_daemon.py | 3 +- lab/prism/share_ledger.py | 89 +- lab/prism/share_writer.py | 1080 + lab/prism/stratum_session.py | 1166 ++ lab/prism/template_artifacts.py | 455 + lab/prism/tip_refresh.py | 4081 ++++ test/test-prism-postgres-ledger.sh | 12 +- tests/prism_coordinator_test_support.py | 16 +- tests/prism_vardiff_test_support.py | 80 +- tests/test_prism_background_services.py | 527 + tests/test_prism_block_candidates.py | 1055 +- tests/test_prism_bounded_executor.py | 106 + tests/test_prism_coordinator_config.py | 18 +- .../test_prism_coordinator_config_loading.py | 250 + tests/test_prism_coordinator_job_cache.py | 1240 ++ tests/test_prism_coordinator_metrics.py | 70 +- tests/test_prism_coordinator_shutdown.py | 167 +- tests/test_prism_ctv_refresh_priority.py | 4 +- tests/test_prism_ctv_runtime.py | 544 + tests/test_prism_hot_path.py | 33 +- .../test_prism_immutable_refresh_artifacts.py | 39 +- tests/test_prism_initial_job_delivery.py | 79 +- tests/test_prism_job_builder.py | 893 +- tests/test_prism_job_delivery.py | 1663 ++ tests/test_prism_payout_state.py | 127 +- tests/test_prism_progress_health.py | 559 +- tests/test_prism_reconnect_backpressure.py | 40 +- tests/test_prism_refresh_retry_pacing.py | 78 +- .../test_prism_representative_independence.py | 140 +- tests/test_prism_retained_jobs.py | 8 +- tests/test_prism_share_ledger.py | 58 + tests/test_prism_share_writer.py | 28 +- tests/test_prism_share_writer_service.py | 567 + tests/test_prism_stratum_restart_bind.py | 3 + tests/test_prism_stratum_session.py | 522 + tests/test_prism_tip_publication_boundary.py | 119 +- tests/test_prism_tip_refresh.py | 2194 ++ tests/test_prism_tip_refresh_delivery.py | 328 +- tests/test_prism_tip_refresh_validation.py | 840 +- tests/test_prism_vardiff.py | 115 +- 53 files changed, 33316 insertions(+), 14633 deletions(-) create mode 100644 lab/prism/background_services.py create mode 100644 lab/prism/bounded_executor.py create mode 100644 lab/prism/bundle_compiler.py create mode 100644 lab/prism/coordinator_config.py create mode 100644 lab/prism/coordinator_shutdown.py create mode 100644 lab/prism/ctv_runtime.py create mode 100644 lab/prism/job_bundle.py create mode 100644 lab/prism/job_delivery.py create mode 100644 lab/prism/payout_state.py create mode 100644 lab/prism/progress_health.py create mode 100644 lab/prism/rpc.py create mode 100644 lab/prism/share_writer.py create mode 100644 lab/prism/stratum_session.py create mode 100644 lab/prism/template_artifacts.py create mode 100644 lab/prism/tip_refresh.py create mode 100644 tests/test_prism_background_services.py create mode 100644 tests/test_prism_bounded_executor.py create mode 100644 tests/test_prism_coordinator_config_loading.py create mode 100644 tests/test_prism_coordinator_job_cache.py create mode 100644 tests/test_prism_ctv_runtime.py create mode 100644 tests/test_prism_job_delivery.py create mode 100644 tests/test_prism_share_writer_service.py create mode 100644 tests/test_prism_stratum_session.py create mode 100644 tests/test_prism_tip_refresh.py diff --git a/lab/prism/background_services.py b/lab/prism/background_services.py new file mode 100644 index 0000000..b4d4757 --- /dev/null +++ b/lab/prism/background_services.py @@ -0,0 +1,200 @@ +"""Named lifecycle registry for PRISM process-level background loops. + +The registry deliberately has no ``start_all`` operation. The coordinator +starts named services at the existing recovery boundaries, while this module +owns the start-once state and the exact thread handles used during shutdown. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import threading +from typing import Callable, Iterable + + +@dataclass(frozen=True, slots=True) +class BackgroundServiceSpec: + """Immutable construction and shutdown policy for one background loop.""" + + name: str + thread_name: str + target: Callable[[], None] + daemon: bool + join_timeout: float + watchdog_monitored: bool + registration_identity: object | None = None + + def __post_init__(self) -> None: + if not self.name: + raise ValueError("background service name must not be empty") + if not self.thread_name: + raise ValueError("background service thread name must not be empty") + if self.join_timeout < 0: + raise ValueError("background service join timeout must be nonnegative") + + +@dataclass(frozen=True, slots=True) +class BackgroundServiceSnapshot: + """Read-only lifecycle state returned without exposing registry records.""" + + specification: BackgroundServiceSpec + started: bool + thread: threading.Thread | None + + +@dataclass(slots=True) +class _BackgroundServiceRecord: + specification: BackgroundServiceSpec + started: bool = False + thread: threading.Thread | None = None + start_hook_completed: bool = False + + +ThreadFactory = Callable[..., threading.Thread] + + +class BackgroundServiceRegistry: + """Start named process services once and retain their drain handles.""" + + def __init__( + self, + specifications: Iterable[BackgroundServiceSpec] = (), + *, + thread_factory: ThreadFactory = threading.Thread, + ) -> None: + self._lock = threading.Lock() + self._thread_factory = thread_factory + self._records: dict[str, _BackgroundServiceRecord] = {} + self._thread_names: set[str] = set() + for specification in specifications: + self.register(specification) + + def register(self, specification: BackgroundServiceSpec) -> None: + """Register one service without starting it.""" + with self._lock: + self._register_locked(specification) + + def register_if_absent(self, specification: BackgroundServiceSpec) -> bool: + """Atomically install an equivalent dynamic service at most once. + + Returns true when this call registered the service. Concurrent callers + describing the same lifecycle and registration identity receive false; + a conflicting reuse of either name still fails explicitly. + """ + with self._lock: + existing = self._records.get(specification.name) + if existing is not None: + if self._equivalent(existing.specification, specification): + return False + raise ValueError( + "incompatible background service registration for name: " + f"{specification.name}" + ) + self._register_locked(specification) + return True + + def _register_locked(self, specification: BackgroundServiceSpec) -> None: + if specification.name in self._records: + raise ValueError( + f"background service is already registered: {specification.name}" + ) + if specification.thread_name in self._thread_names: + raise ValueError( + "background service thread name is already registered: " + f"{specification.thread_name}" + ) + self._records[specification.name] = _BackgroundServiceRecord( + specification=specification + ) + self._thread_names.add(specification.thread_name) + + @staticmethod + def _equivalent( + existing: BackgroundServiceSpec, + candidate: BackgroundServiceSpec, + ) -> bool: + target_equivalent = ( + existing.target is candidate.target + if existing.registration_identity is None + and candidate.registration_identity is None + else existing.registration_identity == candidate.registration_identity + and existing.registration_identity is not None + and candidate.registration_identity is not None + ) + return bool( + target_equivalent + and existing.name == candidate.name + and existing.thread_name == candidate.thread_name + and existing.daemon == candidate.daemon + and existing.join_timeout == candidate.join_timeout + and existing.watchdog_monitored == candidate.watchdog_monitored + ) + + def contains(self, name: str) -> bool: + with self._lock: + return name in self._records + + def service_names(self) -> tuple[str, ...]: + with self._lock: + return tuple(self._records) + + def snapshot(self, name: str) -> BackgroundServiceSnapshot: + with self._lock: + record = self._records[name] + return BackgroundServiceSnapshot( + specification=record.specification, + started=record.started, + thread=record.thread, + ) + + def start( + self, + name: str, + *, + on_started: Callable[[BackgroundServiceSpec], None] | None = None, + ) -> threading.Thread: + """Start a named service once, returning the same thread thereafter.""" + with self._lock: + record = self._records[name] + if record.thread is not None: + if not record.start_hook_completed and on_started is not None: + on_started(record.specification) + record.start_hook_completed = True + return record.thread + specification = record.specification + thread = self._thread_factory( + target=specification.target, + name=specification.thread_name, + daemon=specification.daemon, + ) + thread.start() + record.thread = thread + record.started = True + if on_started is None: + record.start_hook_completed = True + else: + # The live thread remains registered for shutdown if this + # nonblocking side-effect fails. A later start call retries + # only the hook and never creates a second worker. + on_started(specification) + record.start_hook_completed = True + return thread + + def threads_to_drain(self) -> tuple[tuple[threading.Thread, float], ...]: + """Return started threads in stable registration/shutdown order.""" + with self._lock: + return tuple( + (record.thread, record.specification.join_timeout) + for record in self._records.values() + if record.started and record.thread is not None + ) + + def watchdog_service_names(self, *, started_only: bool = False) -> tuple[str, ...]: + """Derive watchdog keys from the same records used to start loops.""" + with self._lock: + return tuple( + record.specification.name + for record in self._records.values() + if record.specification.watchdog_monitored + and (record.started or not started_only) + ) diff --git a/lab/prism/bounded_executor.py b/lab/prism/bounded_executor.py new file mode 100644 index 0000000..f148989 --- /dev/null +++ b/lab/prism/bounded_executor.py @@ -0,0 +1,171 @@ +"""Bounded priority executor used for PRISM job delivery.""" + +from __future__ import annotations + +from concurrent.futures import Future +import heapq +import queue +import threading +from typing import Any, Callable + + +class _DeliveryQueueFull(RuntimeError): + """The bounded delivery executor cannot admit another task.""" + + +class _BoundedPriorityExecutor: + """Small Future-compatible executor with bounded, priority-ordered work.""" + + # A short poll keeps idle workers promptly joinable without requiring one + # poison-pill queue slot per worker (the queue may be smaller than the + # worker pool). + _WORKER_POLL_SECONDS = 0.01 + + def __init__( + self, + *, + max_workers: int, + max_queue_size: int, + thread_name_prefix: str = "prism-job-delivery", + ) -> None: + self.max_workers = max_workers + self.max_queue_size = max_queue_size + self._queue: queue.PriorityQueue[tuple[object, ...]] = queue.PriorityQueue( + maxsize=max_queue_size + ) + self._lock = threading.Lock() + self._sequence = 0 + self._active_workers = 0 + self._shutdown = False + self._cancel_futures_on_shutdown = False + self._threads = [ + threading.Thread( + target=self._worker, + name=f"{thread_name_prefix}-{index + 1}", + daemon=True, + ) + for index in range(max_workers) + ] + for thread in self._threads: + thread.start() + + def submit( + self, + function: Callable[..., Any], + /, + *args: object, + priority: int = 2, + **kwargs: object, + ) -> Future[Any]: + future: Future[Any] = Future() + with self._lock: + if self._shutdown: + raise RuntimeError("delivery executor is shut down") + self._sequence += 1 + item = ( + int(priority), + self._sequence, + future, + function, + args, + kwargs, + ) + try: + self._queue.put_nowait(item) + except queue.Full as exc: + raise _DeliveryQueueFull("delivery executor queue is full") from exc + return future + + def _worker(self) -> None: + while True: + try: + item = self._queue.get(timeout=self._WORKER_POLL_SECONDS) + except queue.Empty: + with self._lock: + if self._shutdown: + return + continue + _, _, future, function, args, kwargs = item + assert isinstance(future, Future) + with self._lock: + cancel_for_shutdown = ( + self._shutdown and self._cancel_futures_on_shutdown + ) + if cancel_for_shutdown: + future.cancel() + self._queue.task_done() + continue + if not future.set_running_or_notify_cancel(): + self._queue.task_done() + continue + with self._lock: + self._active_workers += 1 + try: + result = function(*args, **kwargs) + except BaseException as exc: + future.set_exception(exc) + else: + future.set_result(result) + finally: + with self._lock: + self._active_workers -= 1 + self._queue.task_done() + + def cancel(self, future: Future[Any]) -> bool: + """Cancel ``future`` and immediately discard it when still queued. + + Cancelling a ``Future`` alone does not release bounded queue capacity + until a worker dequeues it. Removing the exact entry under the queue + mutex makes that capacity available to a replacement immediately. + """ + removed = False + with self._queue.mutex: + queued_items = self._queue.queue + for index, item in enumerate(queued_items): + if item[2] is not future: + continue + queued_items.pop(index) + heapq.heapify(queued_items) + self._queue.unfinished_tasks -= 1 + if self._queue.unfinished_tasks == 0: + self._queue.all_tasks_done.notify_all() + self._queue.not_full.notify() + removed = True + break + # Invoke callbacks only after releasing the queue mutex. A cancellation + # callback may submit the replacement that consumes the reclaimed slot. + future.cancel() + return removed + + def stats(self) -> tuple[int, int]: + with self._lock: + return self._queue.qsize(), self._active_workers + + def shutdown(self, *, wait: bool = True, cancel_futures: bool = False) -> None: + with self._lock: + if self._shutdown: + threads = list(self._threads) + already_shutdown = True + else: + self._shutdown = True + self._cancel_futures_on_shutdown = cancel_futures + threads = list(self._threads) + already_shutdown = False + if already_shutdown: + if wait: + for thread in threads: + thread.join() + return + if cancel_futures: + while True: + try: + item = self._queue.get_nowait() + except queue.Empty: + break + future = item[2] + if isinstance(future, Future): + future.cancel() + self._queue.task_done() + if wait: + for thread in threads: + thread.join() diff --git a/lab/prism/bundle_compiler.py b/lab/prism/bundle_compiler.py new file mode 100644 index 0000000..1012a2a --- /dev/null +++ b/lab/prism/bundle_compiler.py @@ -0,0 +1,433 @@ +"""Cancelable subprocess adapter for PRISM audit-bundle compilation.""" + +from __future__ import annotations + +from contextlib import ExitStack +from dataclasses import dataclass +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +from typing import Any, Callable, Protocol + +from lab.prism.prism_tools import prism_tool_command + + +PRISM_BUILDER_PHASE_METRICS_PREFIX = "qbit-prism-build-phase-metrics " +PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 + + +class CancellationPort(Protocol): + def is_set(self) -> bool: ... + + def raise_if_cancelled(self, phase: str) -> None: ... + + +class BundleBuildControlPort(Protocol): + cancel_event: object + process: subprocess.Popen[str] | None + + +@dataclass(frozen=True) +class BundleCompilerPorts: + payout_policy: Callable[[], dict[str, object]] + ctv_settlement: Callable[[int, str | None], dict[str, object] | None] + signing_seed_hex: Callable[[], str] + ledger_signing_seed_hex: Callable[[], str] + bundle_timeout_seconds: Callable[[], float] + cancel_grace_seconds: Callable[[], float] + phases: Callable[[], dict[str, float]] + record_tip_refresh_phase: Callable[[str, float], None] + record_ipc_bytes: Callable[[str, int], None] + record_worker_failure: Callable[[], None] + record_worker_event: Callable[[str], None] + tip_refresh_metrics_enabled: Callable[[], bool] + active_build_control: Callable[[], BundleBuildControlPort | None] + register_process: Callable[ + [BundleBuildControlPort, subprocess.Popen[str]], None + ] + superseded_error: Callable[[str], BaseException] + + +class BundleCompiler: + """Compile summaries or canonical bundles with exact cancellation rules.""" + + def __init__( + self, + ports: BundleCompilerPorts, + *, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self._ports = ports + self._monotonic = monotonic + + def build_audit_bundle( + self, + *, + shares: list[dict[str, object]], + found_block: dict[str, object], + prior_balances: list[dict[str, object]], + coinbase_script_sig_suffix_hex: str, + witness_merkle_leaves_hex: list[str] | None = None, + ctv_fee_parent_hash: str | None = None, + canonical_output_path: Path | None = None, + summary_only: bool = False, + payout_policy: dict[str, object] | None = None, + ctv_settlement: dict[str, object] | None = None, + cancellation: CancellationPort | None = None, + ) -> dict[str, Any]: + if cancellation is not None: + cancellation.raise_if_cancelled("serialization") + payload: dict[str, object] = { + "found_block": found_block, + "prior_balances": prior_balances, + "payout_policy": ( + self._ports.payout_policy() + if payout_policy is None + else payout_policy + ), + "coinbase_script_sig_suffix_hex": coinbase_script_sig_suffix_hex, + "witness_merkle_leaves_hex": witness_merkle_leaves_hex or [], + } + record_phase_metrics = self._ports.tip_refresh_metrics_enabled() + serialization_copy_seconds = 0.0 + if summary_only: + artifact_started = self._monotonic() + identity_indexes: dict[tuple[str, str, str], int] = {} + identities: list[tuple[str, str, str]] = [] + compact_shares: list[tuple[object, ...]] = [] + for share in shares: + identity = ( + str(share["miner_id"]), + str(share["order_key"]), + str(share["p2mr_program_hex"]), + ) + identity_index = identity_indexes.get(identity) + if identity_index is None: + identity_index = len(identities) + identity_indexes[identity] = identity_index + identities.append(identity) + compact_shares.append( + ( + share["share_seq"], + share["share_id"], + identity_index, + share["share_difficulty"], + share["job_issued_at_ms"], + share["accepted_at_ms"], + share.get("credit_policy"), + ) + ) + payload["compact_share_identities"] = identities + payload["compact_shares"] = compact_shares + if record_phase_metrics: + serialization_copy_seconds += self._monotonic() - artifact_started + else: + payload["shares"] = shares + if ctv_settlement is None and payout_policy is None: + ctv_settlement = self._ports.ctv_settlement( + int(found_block["block_height"]), + ctv_fee_parent_hash, + ) + if ctv_settlement is not None: + payload["ctv_settlement"] = ctv_settlement + if canonical_output_path is not None and summary_only: + raise ValueError( + "canonical output and job summary output are mutually exclusive" + ) + command = prism_tool_command("qbit-prism-build-audit-bundle") + [ + "--input", + "-", + "--signing-key-seed-hex", + self._ports.signing_seed_hex(), + "--ledger-signing-key-seed-hex", + self._ports.ledger_signing_seed_hex(), + ] + command.append("--job-summary-output" if summary_only else "--canonical-output") + if record_phase_metrics: + command.append("--phase-metrics") + if canonical_output_path is not None: + canonical_output_path.parent.mkdir(parents=True, exist_ok=True) + succeeded = False + created_output = False + try: + with ExitStack() as stack: + if canonical_output_path is None: + output = stack.enter_context( + tempfile.TemporaryFile(mode="w+", encoding="utf-8") + ) + else: + output = stack.enter_context( + canonical_output_path.open("x+", encoding="utf-8") + ) + created_output = True + stderr = stack.enter_context( + tempfile.TemporaryFile(mode="w+", encoding="utf-8") + ) + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=output, + stderr=stderr, + text=True, + encoding="utf-8", + close_fds=True, + ) + build_control = self._ports.active_build_control() + if build_control is not None: + self._ports.register_process(build_control, process) + self._ports.record_worker_event("start") + assert process.stdin is not None + input_byte_count = 0 + builder_started = self._monotonic() + worker_deadline = ( + builder_started + self._ports.bundle_timeout_seconds() + ) + compiler = self + + class _CancelableInput: + def __init__(self, stream: Any) -> None: + self.stream = stream + try: + file_descriptor = int(stream.fileno()) + except (AttributeError, OSError, TypeError, ValueError): + self.file_descriptor: int | None = None + else: + os.set_blocking(file_descriptor, False) + self.file_descriptor = file_descriptor + + def check_cancelled(self) -> None: + if cancellation is not None: + cancellation.raise_if_cancelled( + "builder input serialization" + ) + if ( + build_control is not None + and build_control.cancel_event.is_set() # type: ignore[attr-defined] + ): + raise compiler._ports.superseded_error( + "audit-builder input was canceled after supersession" + ) + if compiler._monotonic() >= worker_deadline: + compiler._ports.record_worker_failure() + raise RuntimeError( + "qbit-prism-build-audit-bundle timed out" + ) + + def write(self, value: str) -> int: + nonlocal input_byte_count + self.check_cancelled() + if self.file_descriptor is None: + written = int(self.stream.write(value)) + input_byte_count += len( + value[:written].encode("utf-8") + ) + return written + encoded = value.encode("utf-8") + remaining = memoryview(encoded) + while remaining: + self.check_cancelled() + try: + written = os.write(self.file_descriptor, remaining) + except (BlockingIOError, InterruptedError): + time.sleep( + min( + 0.02, + PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, + ) + ) + continue + if written <= 0: + raise BrokenPipeError( + "audit-builder input pipe closed" + ) + input_byte_count += written + remaining = remaining[written:] + return len(value) + + serialization_started = self._monotonic() + try: + json.dump( + payload, + _CancelableInput(process.stdin), + separators=(",", ":"), + ) + except BrokenPipeError: + pass + except BaseException: + try: + process.kill() + except ProcessLookupError: + pass + process.wait() + if ( + cancellation is not None + and cancellation.is_set() + ) or ( + build_control is not None + and build_control.cancel_event.is_set() # type: ignore[attr-defined] + ): + self._ports.record_worker_event("termination") + raise + finally: + input_serialization_seconds = ( + self._monotonic() - serialization_started + ) + phases = self._ports.phases() + phases["input_serialization"] = phases.get( + "input_serialization", + 0.0, + ) + input_serialization_seconds + if record_phase_metrics: + serialization_copy_seconds += input_serialization_seconds + try: + process.stdin.close() + except (BlockingIOError, BrokenPipeError): + pass + if record_phase_metrics: + self._ports.record_ipc_bytes("input", input_byte_count) + worker_started = self._monotonic() + returncode: int | None = None + if cancellation is None and not hasattr(process, "poll"): + returncode = process.wait() + else: + while returncode is None: + returncode = process.poll() + if returncode is not None: + break + cancelled_by_control = ( + build_control is not None + and build_control.cancel_event.is_set() # type: ignore[attr-defined] + ) + cancelled_by_request = ( + cancellation is not None and cancellation.is_set() + ) + if cancelled_by_control or cancelled_by_request: + process.terminate() + try: + returncode = process.wait( + timeout=max( + 0.0, + self._ports.cancel_grace_seconds(), + ) + ) + except subprocess.TimeoutExpired: + process.kill() + returncode = process.wait() + self._ports.record_worker_event("termination") + break + if self._monotonic() >= worker_deadline: + process.kill() + returncode = process.wait() + self._ports.record_worker_failure() + raise RuntimeError( + "qbit-prism-build-audit-bundle timed out" + ) + time.sleep( + min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS) + ) + phases["worker"] = phases.get("worker", 0.0) + ( + self._monotonic() - worker_started + ) + if cancellation is not None and cancellation.is_set(): + cancellation.raise_if_cancelled("builder worker") + if ( + build_control is not None + and build_control.cancel_event.is_set() # type: ignore[attr-defined] + ): + raise self._ports.superseded_error( + "audit-builder subprocess was canceled after supersession" + ) + stderr.seek(0) + error_text = stderr.read() + if returncode != 0: + self._ports.record_worker_event("crash") + if record_phase_metrics: + self._ports.record_worker_failure() + raise RuntimeError( + f"qbit-prism-build-audit-bundle failed: {error_text}" + ) + if ( + build_control is not None + and build_control.cancel_event.is_set() # type: ignore[attr-defined] + ): + raise self._ports.superseded_error( + "audit-builder result completed after supersession" + ) + output.flush() + output_size = os.fstat(output.fileno()).st_size + if record_phase_metrics: + self._ports.record_ipc_bytes("output", output_size) + if canonical_output_path is not None: + os.fsync(output.fileno()) + output.seek(0) + output_started = self._monotonic() + if cancellation is not None: + cancellation.raise_if_cancelled( + "builder output serialization" + ) + bundle = json.load(output) + output_serialization_seconds = self._monotonic() - output_started + phases["output_serialization"] = phases.get( + "output_serialization", + 0.0, + ) + output_serialization_seconds + if record_phase_metrics: + serialization_copy_seconds += output_serialization_seconds + self._record_phase_metrics( + error_text, + serialization_copy_seconds=serialization_copy_seconds, + ) + if cancellation is not None: + cancellation.raise_if_cancelled("builder verification") + succeeded = True + return bundle + finally: + if canonical_output_path is not None and created_output and not succeeded: + try: + canonical_output_path.unlink() + except FileNotFoundError: + pass + + def _record_phase_metrics( + self, + error_text: str, + *, + serialization_copy_seconds: float, + ) -> None: + rust_serialization = 0.0 + for line in error_text.splitlines(): + if not line.startswith(PRISM_BUILDER_PHASE_METRICS_PREFIX): + continue + raw_metrics = line.removeprefix(PRISM_BUILDER_PHASE_METRICS_PREFIX) + try: + metrics = json.loads(raw_metrics) + phase_seconds = metrics.get("phases_seconds", {}) + if isinstance(phase_seconds, dict): + for phase in ( + "payout_state_derivation", + "ctv_manifest_construction", + "coinbase_bundle_construction", + "signing_verification", + ): + elapsed = phase_seconds.get(phase) + if isinstance(elapsed, (int, float)): + self._ports.record_tip_refresh_phase( + phase, + float(elapsed), + ) + rust_serialization += sum( + float(metrics.get(name, 0.0)) + for name in ( + "input_deserialization_seconds", + "output_serialization_seconds", + ) + ) + except (TypeError, ValueError, json.JSONDecodeError): + pass + self._ports.record_tip_refresh_phase( + "serialization_copy", + serialization_copy_seconds + rust_serialization, + ) diff --git a/lab/prism/coordinator_config.py b/lab/prism/coordinator_config.py new file mode 100644 index 0000000..e415dff --- /dev/null +++ b/lab/prism/coordinator_config.py @@ -0,0 +1,1167 @@ +"""Immutable PRISM coordinator configuration and environment loading.""" + +from __future__ import annotations + +import json +import math +import shlex +from os import environ as _PROCESS_ENVIRON +from dataclasses import dataclass, replace as dataclass_replace +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Mapping + +from lab.auxpow import vardiff +from lab.prism import direct_stratum +from lab.prism.ctv_broadcaster_daemon import MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE +from lab.prism.share_ledger import ( + DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, + DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, + DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, +) + + +DEFAULT_P2MR_SPEND_INPUT_BYTES = 3_680 +DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE = 1 +DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER = 4 +DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS = ( + "tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3" +) +DEFAULT_PRISM_COINBASE_TAG = "/PRISM/" +MAX_PRISM_COINBASE_TAG_BYTES = 40 +DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS = 10_485_760 +DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS = 16 +DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS = 12 +DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION = 1_000 +DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS = 12_000 +TESTNET_QBIT_CHAINS = {"testnet", "testnet3", "testnet4", "signet"} +DEFAULT_PRISM_BLOCKPOLL_SECONDS = 2.0 +DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS = 5.0 +DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS = 1.0 +DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS = 10.0 +DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS = 60.0 +DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE = 5 +DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS = 5.0 +DEFAULT_PRISM_HEALTH_REFRESH_SECONDS = 5.0 +DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS = 15.0 +DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS = 15.0 +DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS = 20.0 +DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS = 384 +DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME = 0 +DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS = 128 +DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS = 30.0 +DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS = 30.0 +DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS = 1.0 +DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG = 1024 +DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS = 10.0 +DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES = 4_096 +DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS = 3_600.0 +DEFAULT_PRISM_STALE_GRACE_SECONDS = 3.0 +DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS = 10.0 +DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS = 30.0 +DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION = 64 +DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS = 16 +DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS = 60.0 +DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS = 0.25 +DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS = 15.0 +DEFAULT_PRISM_WORKER_METRICS_LIMIT = 100 +DEFAULT_SHARE_COMMIT_BATCH_SIZE = 64 +DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS = 5.0 +DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS = 15.0 +DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS = 15.0 +DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS = 120 +DEFAULT_HIGHDIFF_DIFFICULTY = "500000" +DEFAULT_HIGHDIFF_MAX_DIFFICULTY = "4294967296" + + +Env = Mapping[str, str] + + +def _current_environ(environ: Env | None) -> Env: + return _PROCESS_ENVIRON if environ is None else environ + + +def env(name: str, default: str | None = None, *, environ: Env | None = None) -> str: + value = _current_environ(environ).get(name, default) + if value is None or value == "": + raise SystemExit(f"{name} is required") + return value + + +def env_int(name: str, default: int, *, environ: Env | None = None) -> int: + return int(env(name, str(default), environ=environ)) + + +def env_positive_int(name: str, default: int, *, environ: Env | None = None) -> int: + try: + value = env_int(name, default, environ=environ) + except ValueError as exc: + raise SystemExit(f"{name} must be an integer") from exc + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_positive_int_with_legacy( + primary_name: str, + legacy_name: str, + default: int, + *, + environ: Env | None = None, +) -> int: + if env_optional(primary_name, environ=environ) is not None: + return env_positive_int(primary_name, default, environ=environ) + return env_positive_int(legacy_name, default, environ=environ) + + +def env_nonnegative_int(name: str, default: int, *, environ: Env | None = None) -> int: + try: + value = env_int(name, default, environ=environ) + except ValueError as exc: + raise SystemExit(f"{name} must be an integer") from exc + if value < 0: + raise SystemExit(f"{name} must be non-negative") + return value + + +def env_nonnegative_int_with_legacy( + primary_name: str, + legacy_name: str, + default: int, + *, + environ: Env | None = None, +) -> int: + if env_optional(primary_name, environ=environ) is not None: + return env_nonnegative_int(primary_name, default, environ=environ) + return env_nonnegative_int(legacy_name, default, environ=environ) + + +def env_positive_float(name: str, default: float, *, environ: Env | None = None) -> float: + try: + value = float(env(name, str(default), environ=environ)) + except ValueError as exc: + raise SystemExit(f"{name} must be a number") from exc + if not math.isfinite(value): + raise SystemExit(f"{name} must be finite") + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_nonnegative_float(name: str, default: float, *, environ: Env | None = None) -> float: + try: + value = float(env(name, str(default), environ=environ)) + except ValueError as exc: + raise SystemExit(f"{name} must be a number") from exc + if not math.isfinite(value): + raise SystemExit(f"{name} must be finite") + if value < 0: + raise SystemExit(f"{name} must be non-negative") + return value + + +def env_optional_positive_int(name: str, *, environ: Env | None = None) -> int | None: + raw = env_optional(name, environ=environ) + if raw is None: + return None + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"{name} must be an integer") from exc + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_optional_positive_int_with_legacy( + primary_name: str, + legacy_name: str, + *, + environ: Env | None = None, +) -> int | None: + value = env_optional_positive_int(primary_name, environ=environ) + if value is not None: + return value + return env_optional_positive_int(legacy_name, environ=environ) + + +def env_decimal(name: str, default: str, *, environ: Env | None = None) -> Decimal: + try: + value = Decimal(env(name, default, environ=environ)) + except InvalidOperation as exc: + raise SystemExit(f"{name} must be a decimal number") from exc + if not value.is_finite(): + raise SystemExit(f"{name} must be finite") + if value <= 0: + raise SystemExit(f"{name} must be positive") + return value + + +def env_bool(name: str, default: str, *, environ: Env | None = None) -> bool: + return env(name, default, environ=environ).lower() in {"1", "true", "yes", "on"} + + +def env_optional_bool(name: str, *, environ: Env | None = None) -> bool | None: + raw = env_optional(name, environ=environ) + if raw is None: + return None + return raw.lower() in {"1", "true", "yes", "on"} + + +def env_optional(name: str, *, environ: Env | None = None) -> str | None: + value = _current_environ(environ).get(name) + if value is None or value == "": + return None + return value + + +def production_mode(*, environ: Env | None = None) -> bool: + return ( + env_bool("QBIT_PRODUCTION", "0", environ=environ) + or env_bool("QBIT_TOOLS_PRODUCTION", "0", environ=environ) + or env("QBIT_CHAIN", "regtest", environ=environ).lower() in {"main", "mainnet"} + ) + + +def validate_same_tip_job_retention_limits( + *, + retention_seconds: float, + per_connection: int, + max_connections: int, + production: bool, +) -> None: + if retention_seconds <= 0: + return + if per_connection <= 0: + raise SystemExit( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION must be positive " + "when same-tip retention is enabled" + ) + if production and max_connections <= 0: + raise SystemExit( + "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS " + "when same-tip retention is enabled" + ) + + +def require_production_env(name: str, *, environ: Env | None = None) -> str: + value = env_optional(name, environ=environ) + if value is None: + raise SystemExit(f"production mode requires {name}") + return value + + +def validate_prism_production_gate(*, environ: Env | None = None) -> None: + if not production_mode(environ=environ): + return + + for name in ( + "PRISM_ALLOW_MEMORY_LEDGER", + "PRISM_ALLOW_TEST_SIGNING_SEEDS", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", + "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", + ): + if env_bool(name, "0", environ=environ): + raise SystemExit(f"production mode rejects {name}=1") + + if env("QBIT_CHAIN", "regtest", environ=environ).lower() in {"main", "mainnet"} and env_nonnegative_float( + "PRISM_STRATUM_STALE_GRACE_SECONDS", + DEFAULT_PRISM_STALE_GRACE_SECONDS, + environ=environ, + ) != 0: + raise SystemExit("mainnet requires PRISM_STRATUM_STALE_GRACE_SECONDS=0") + + production_difficulties: dict[str, Decimal] = {} + for name in ( + "PRISM_STRATUM_SHARE_DIFF", + "PRISM_STRATUM_VARDIFF_MIN_DIFF", + "PRISM_STRATUM_VARDIFF_START_DIFF", + "PRISM_STRATUM_VARDIFF_MAX_DIFF", + ): + raw_value = require_production_env(name, environ=environ) + if not raw_value: + raise SystemExit(f"production mode requires an explicit {name}") + try: + value = Decimal(raw_value) + except InvalidOperation as exc: + raise SystemExit(f"{name} must be a decimal number") from exc + if not value.is_finite() or value <= 0: + raise SystemExit(f"{name} must be positive") + if value == Decimal("0.000000001"): + raise SystemExit(f"{name} cannot use the lab-only 1e-9 difficulty") + production_difficulties[name] = value + if production_difficulties["PRISM_STRATUM_VARDIFF_MIN_DIFF"] > production_difficulties[ + "PRISM_STRATUM_VARDIFF_START_DIFF" + ]: + raise SystemExit("production vardiff minimum exceeds its start difficulty") + if production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] > production_difficulties[ + "PRISM_STRATUM_VARDIFF_MAX_DIFF" + ]: + raise SystemExit("production vardiff start exceeds its maximum difficulty") + + prism_database_url = env_optional("PRISM_DATABASE_URL", environ=environ) + if prism_database_url is None and env_optional("PRISM_POSTGRES_PSQL_COMMAND", environ=environ) is None: + raise SystemExit("production mode requires PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND") + if env_optional("PRISM_POSTGRES_PASSWORD", environ=environ) == "change-this": + raise SystemExit("production mode requires a non-default PRISM_POSTGRES_PASSWORD") + if prism_database_url is not None and "change-this" in prism_database_url: + raise SystemExit("production mode requires a non-default PRISM_DATABASE_URL") + + require_production_env("PRISM_MANIFEST_SIGNING_SEED_HEX", environ=environ) + require_production_env("PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", environ=environ) + require_production_env("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", environ=environ) + require_production_env("PRISM_LEDGER_WRITER_ID", environ=environ) + require_production_env("PRISM_LEDGER_WRITER_EPOCH", environ=environ) + require_production_env("PRISM_AUDIT_DIR", environ=environ) + require_production_env("PRISM_EVIDENCE_PATH", environ=environ) + + if env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN", environ=environ) is not None: + raise SystemExit( + "production mode requires managed ledger session tokens; unset " + "PRISM_LEDGER_WRITER_SESSION_TOKEN" + ) + if env_nonnegative_int( + "PRISM_STRATUM_MAX_CONNECTIONS", DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, environ=environ + ) <= 0: + raise SystemExit("production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS") + env_positive_int( + "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", + DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + environ=environ, + ) + if env_nonnegative_float( + "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", + DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, + environ=environ, + ) <= 0: + raise SystemExit("production mode requires a positive PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS") + + require_production_env("QBIT_RPC_USER", environ=environ) + qbit_rpc_password = require_production_env("QBIT_RPC_PASSWORD", environ=environ) + if qbit_rpc_password == "change-this": + raise SystemExit("production mode requires a non-default QBIT_RPC_PASSWORD") + + if env("QBIT_CHAIN", "regtest", environ=environ).lower() in {"main", "mainnet"} and env_bool( + "PRISM_CTV_SETTLEMENT_ENABLED", "0", environ=environ + ): + require_production_env( + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", environ=environ + ) + env_positive_int( + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", 0, environ=environ + ) + + validate_same_tip_job_retention_limits( + retention_seconds=env_nonnegative_float( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + environ=environ, + ), + per_connection=env_nonnegative_int( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + environ=environ, + ), + max_connections=env_nonnegative_int( + "PRISM_STRATUM_MAX_CONNECTIONS", DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, environ=environ + ), + production=True, + ) + + +def validate_hex(value: str, *, name: str, expected_bytes: int | None = None) -> str: + try: + bytes.fromhex(value) + except ValueError as exc: + raise SystemExit(f"{name} must be hex") from exc + if expected_bytes is not None and len(value) != expected_bytes * 2: + raise SystemExit(f"{name} must be {expected_bytes * 2} hex chars") + return value.lower() + + +def env_seed_hex(name: str, *, test_default: str, environ: Env | None = None) -> str: + value = env_optional(name, environ=environ) + if value is None: + if env_bool("PRISM_ALLOW_TEST_SIGNING_SEEDS", "0", environ=environ): + value = test_default + else: + raise SystemExit(f"{name} is required") + return validate_hex(value, name=name, expected_bytes=32) + + +def load_prism_vardiff_config( + startup_difficulty: Decimal, *, environ: Env | None = None +) -> vardiff.VardiffConfig: + return vardiff.VardiffConfig( + enabled=env_bool("PRISM_STRATUM_VARDIFF", "1", environ=environ), + target_share_interval_seconds=env_decimal( + "PRISM_STRATUM_VARDIFF_TARGET_SECONDS", "15", environ=environ + ), + min_difficulty=env_decimal( + "PRISM_STRATUM_VARDIFF_MIN_DIFF", str(startup_difficulty), environ=environ + ), + max_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_MAX_DIFF", "1024", environ=environ), + retarget_interval_seconds=env_decimal( + "PRISM_STRATUM_VARDIFF_RETARGET_SECONDS", "90", environ=environ + ), + max_step_factor=env_decimal("PRISM_STRATUM_VARDIFF_MAX_STEP_UP", "4", environ=environ), + startup_difficulty=env_decimal( + "PRISM_STRATUM_VARDIFF_START_DIFF", str(startup_difficulty), environ=environ + ), + max_step_down_factor=env_decimal( + "PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN", "4", environ=environ + ), + ewma_alpha=env_decimal("PRISM_STRATUM_VARDIFF_EWMA_ALPHA", "0.4", environ=environ), + retarget_tolerance=env_decimal( + "PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE", "0.25", environ=environ + ), + ) + + +@dataclass(frozen=True) +class StratumListenerProfile: + name: str + bind: str + port: int + share_difficulty: Decimal + vardiff_config: vardiff.VardiffConfig + heartbeat_name: str + minimum_advertised_difficulty: Decimal = Decimal("0") + + +def load_prism_highdiff_listener( + base_bind: str, + base_vardiff_config: vardiff.VardiffConfig, + *, + environ: Env | None = None, +) -> StratumListenerProfile | None: + port_value = env_optional("PRISM_STRATUM_HIGHDIFF_PORT", environ=environ) + if port_value is None: + return None + try: + port = int(port_value) + except ValueError as exc: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be an integer") from exc + if not 0 < port < 65536: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be a valid TCP port") + min_difficulty = env_decimal( + "PRISM_STRATUM_HIGHDIFF_MIN_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY, environ=environ + ) + start_difficulty = env_decimal( + "PRISM_STRATUM_HIGHDIFF_START_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY, environ=environ + ) + max_difficulty = env_decimal( + "PRISM_STRATUM_HIGHDIFF_MAX_DIFF", DEFAULT_HIGHDIFF_MAX_DIFFICULTY, environ=environ + ) + if min_difficulty > start_difficulty: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_MIN_DIFF exceeds PRISM_STRATUM_HIGHDIFF_START_DIFF") + if start_difficulty > max_difficulty: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_START_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") + share_value = env_optional("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF", environ=environ) + if share_value is None: + share_difficulty = start_difficulty + else: + try: + share_difficulty = Decimal(share_value) + except Exception as exc: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be a decimal") from exc + if not share_difficulty.is_finite() or share_difficulty <= 0: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be positive") + if share_difficulty < min_difficulty: + raise SystemExit( + "PRISM_STRATUM_HIGHDIFF_SHARE_DIFF is below PRISM_STRATUM_HIGHDIFF_MIN_DIFF" + ) + if share_difficulty > max_difficulty: + raise SystemExit( + "PRISM_STRATUM_HIGHDIFF_SHARE_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF" + ) + try: + config = dataclass_replace( + base_vardiff_config, + min_difficulty=min_difficulty, + max_difficulty=max_difficulty, + startup_difficulty=start_difficulty, + ) + except ValueError as exc: + raise SystemExit(f"invalid PRISM_STRATUM_HIGHDIFF_* difficulty bounds: {exc}") from exc + return StratumListenerProfile( + name="highdiff", + bind=env_optional("PRISM_STRATUM_HIGHDIFF_BIND", environ=environ) or base_bind, + port=port, + share_difficulty=share_difficulty, + vardiff_config=config, + heartbeat_name="stratum_accept_highdiff", + minimum_advertised_difficulty=min_difficulty, + ) + + +def default_prism_payout_policy(*, environ: Env | None = None) -> dict[str, object]: + policy: dict[str, object] = { + "p2mr_spend_input_bytes": env_positive_int( + "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES", DEFAULT_P2MR_SPEND_INPUT_BYTES, environ=environ + ), + "target_feerate_sats_per_byte": env_positive_int_with_legacy( + "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE", + "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE", + DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE, + environ=environ, + ), + "safety_multiplier": env_positive_int( + "PRISM_PAYOUT_SAFETY_MULTIPLIER", + DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER, + environ=environ, + ), + } + min_output_sats = env_optional_positive_int_with_legacy( + "PRISM_PAYOUT_MIN_OUTPUT_BITS", "PRISM_PAYOUT_MIN_OUTPUT_SATS", environ=environ + ) + if min_output_sats is not None: + policy["min_output_sats"] = min_output_sats + return policy + + +def default_prism_coinbase_tag_hex(*, environ: Env | None = None) -> str: + tag = _current_environ(environ).get("PRISM_COINBASE_TAG", DEFAULT_PRISM_COINBASE_TAG) + try: + tag_bytes = tag.encode("ascii") + except UnicodeEncodeError as exc: + raise SystemExit("PRISM_COINBASE_TAG must be ASCII") from exc + if len(tag_bytes) > MAX_PRISM_COINBASE_TAG_BYTES: + raise SystemExit(f"PRISM_COINBASE_TAG must be at most {MAX_PRISM_COINBASE_TAG_BYTES} bytes") + if any(byte < 0x20 or byte > 0x7E for byte in tag_bytes): + raise SystemExit("PRISM_COINBASE_TAG must contain printable ASCII only") + return tag_bytes.hex() + + +def default_prism_username_fallback_address(*, environ: Env | None = None) -> str | None: + configured = env_optional("PRISM_USERNAME_FALLBACK_ADDRESS", environ=environ) + if configured is not None: + return configured + if (_current_environ(environ).get("QBIT_CHAIN") or "regtest").lower() in TESTNET_QBIT_CHAINS: + return DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS + return None + + +def _parse_share_weights(raw: str) -> tuple[tuple[str, int], ...]: + if not raw: + return () + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + raise SystemExit(f"PRISM_STRATUM_SHARE_WEIGHTS_JSON is not valid JSON: {exc}") from exc + if not isinstance(parsed, dict): + raise SystemExit("PRISM_STRATUM_SHARE_WEIGHTS_JSON must be an object") + weights: list[tuple[str, int]] = [] + for username, weight in parsed.items(): + parsed_weight = int(weight) + if parsed_weight <= 0: + raise SystemExit(f"share weight for {username} must be positive") + weights.append((str(username), parsed_weight)) + return tuple(weights) + + +def load_share_weights(*, environ: Env | None = None) -> dict[str, int]: + """Load the legacy username-weight mapping for compatibility callers.""" + + source = _current_environ(environ) + return dict(_parse_share_weights(source.get("PRISM_STRATUM_SHARE_WEIGHTS_JSON", ""))) + + +@dataclass(frozen=True) +class RpcConfig: + host: str + port: int + user: str + password: str + chain: str + expected_genesis_hash: str | None + minimum_peers_raw: str | None + + +@dataclass(frozen=True) +class StratumConfig: + bind: str + port: int + extranonce2_size: int + stale_grace_seconds: float + send_timeout_seconds: float + max_connections: int + max_connections_per_username: int + max_pending_initial_jobs: int + initial_job_timeout_seconds: float + accept_resource_exhaustion_backoff_seconds: float + listen_backlog: int + bind_retry_seconds: float + share_difficulty: Decimal + vardiff_config: vardiff.VardiffConfig + vardiff_idle_sweep_seconds: float + listener_profiles: tuple[StratumListenerProfile, ...] + default_share_weight: int + share_weights_by_username: tuple[tuple[str, int], ...] + username_fallback_address: str | None + fallback_version_mask: int + same_tip_job_retention_seconds: float + same_tip_job_retention_per_connection: int + payout_address_cache_max_entries: int + payout_address_cache_ttl_seconds: float + + +@dataclass(frozen=True) +class JobPipelineConfig: + blockpoll_seconds: float + blockwait_enabled: bool + blockwait_timeout_seconds: float + tip_refresh_failure_holdoff_seconds: float + submit_tip_max_age_seconds: float + tip_refresh_max_workers: int + job_build_timeout_seconds: float + job_build_cancel_grace_seconds: float + worker_metrics_limit: int + reorg_reconciler_enabled: bool + job_bundle_cache_seconds: float + bundle_build_timeout_seconds: float + template_cache_seconds: float + template_refresh_failure_exit_seconds: float + reorg_reconcile_cache_seconds: float + min_ready_miners: int + payout_environment: tuple[tuple[str, str], ...] + pool_fee_enabled_raw: str | None + pool_fee_bps_raw: str | None + pool_fee_address: str | None + pool_fee_program_hex: str | None + pool_fee_recipient_id: str | None + pool_fee_order_key: str | None + template_max_age_raw: str | None + + +@dataclass(frozen=True) +class LedgerConfig: + psql_command: str + database_url: str | None + allow_memory_ledger: bool + native_client_mode: str + writer_id: str + writer_epoch: int + writer_session_token: str | None + initialize_schema: bool + lease_ttl_seconds: float + read_concurrency: int + accepted_stats_cache_seconds: float + reward_window_cache_seconds: float + signing_seed_hex: str + attestation_signing_seed_hex: str + writer_public_key_hex: str | None + share_commit_batch_size: int + share_commit_linger_seconds: float + share_commit_timeout_seconds: float + share_recovery_path: Path + + +@dataclass(frozen=True) +class AuditConfig: + evidence_path: Path + directory: Path + share_segment_size: int + live_bundle_retention: int + candidate_retention_seconds: int + bind: str | None + port: int + + +@dataclass(frozen=True) +class CtvConfig: + settlement_enabled_raw: str | None + broadcaster_enabled: bool + broadcaster_wallet: str | None + broadcaster_fee_sats: int + broadcaster_limit: int + broadcaster_chunk_size: int + broadcaster_interval_seconds: float + broadcast_attempt_detail_limit: int + broadcast_retry_backoff_seconds: int + settlement_environment: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class LifecycleConfig: + health_refresh_seconds: float + pending_refresh_health_deadline_seconds: float + coherent_tip_poll_health_deadline_seconds: float + mining_health_startup_grace_seconds: float + writer_quiescence_timeout_seconds: float + watchdog_enabled: bool + watchdog_timeout_seconds: float + watchdog_interval_seconds: float + + +@dataclass(frozen=True) +class CoordinatorConfig: + rpc: RpcConfig + stratum: StratumConfig + jobs: JobPipelineConfig + ledger: LedgerConfig + audit: AuditConfig + ctv: CtvConfig + lifecycle: LifecycleConfig + production: bool + hot_path_log_enabled: bool + coinbase_tag_hex: str + stop_after_block: bool + max_blocks: int + + +def _selected_environment(source: Env, names: tuple[str, ...]) -> tuple[tuple[str, str], ...]: + return tuple((name, source[name]) for name in names if name in source) + + +def load_coordinator_config(environ: Env | None = None) -> CoordinatorConfig: + """Load and validate one immutable configuration snapshot. + + Passing a mapping gives tests and embedding callers a no-global-environment + construction path. The zero-argument production path snapshots ``os.environ``. + """ + + source: Env = dict(_PROCESS_ENVIRON) if environ is None else dict(environ) + validate_prism_production_gate(environ=source) + production = production_mode(environ=source) + + rpc = RpcConfig( + host=env("QBIT_RPC_HOST", environ=source), + port=env_int("QBIT_RPC_PORT", 18452, environ=source), + user=env("QBIT_RPC_USER", environ=source), + password=env("QBIT_RPC_PASSWORD", environ=source), + chain=env("QBIT_CHAIN", "regtest", environ=source), + expected_genesis_hash=env_optional("QBIT_EXPECTED_GENESIS_HASH", environ=source), + minimum_peers_raw=source.get("PRISM_MIN_PEERS"), + ) + + blockpoll_seconds = env_positive_float( + "PRISM_BLOCKPOLL_SECONDS", DEFAULT_PRISM_BLOCKPOLL_SECONDS, environ=source + ) + same_tip_seconds = env_nonnegative_float( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + environ=source, + ) + same_tip_per_connection = env_nonnegative_int( + "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + environ=source, + ) + max_connections = env_nonnegative_int( + "PRISM_STRATUM_MAX_CONNECTIONS", DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, environ=source + ) + max_pending_initial_jobs = env_positive_int( + "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", + DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + environ=source, + ) + if max_connections > 0 and max_pending_initial_jobs > max_connections: + raise SystemExit( + "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS cannot exceed PRISM_STRATUM_MAX_CONNECTIONS" + ) + validate_same_tip_job_retention_limits( + retention_seconds=same_tip_seconds, + per_connection=same_tip_per_connection, + max_connections=max_connections, + production=production, + ) + tip_refresh_max_workers = env_positive_int( + "PRISM_TIP_REFRESH_MAX_WORKERS", DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, environ=source + ) + if tip_refresh_max_workers > DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS: + raise SystemExit( + "PRISM_TIP_REFRESH_MAX_WORKERS cannot exceed " + f"{DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS}" + ) + template_refresh_failure_exit_seconds = env_nonnegative_float( + "PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + environ=source, + ) + if production and template_refresh_failure_exit_seconds <= 0: + raise SystemExit( + "production mode requires a positive PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS" + ) + + bind = env("PRISM_STRATUM_BIND", "127.0.0.1", environ=source) + port = env_int("PRISM_STRATUM_PORT", 3340, environ=source) + share_difficulty = env_decimal("PRISM_STRATUM_SHARE_DIFF", "0.000000001", environ=source) + vardiff_config = load_prism_vardiff_config(share_difficulty, environ=source) + listener_profiles = [ + StratumListenerProfile( + name="default", + bind=bind, + port=port, + share_difficulty=share_difficulty, + vardiff_config=vardiff_config, + heartbeat_name="stratum_accept", + ) + ] + highdiff_profile = load_prism_highdiff_listener(bind, vardiff_config, environ=source) + if highdiff_profile is not None: + if highdiff_profile.port == port and highdiff_profile.bind == bind: + raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must differ from PRISM_STRATUM_PORT") + listener_profiles.append(highdiff_profile) + default_share_weight = env_int("PRISM_STRATUM_SHARE_WEIGHT", 1, environ=source) + if default_share_weight <= 0: + raise SystemExit("PRISM_STRATUM_SHARE_WEIGHT must be positive") + try: + fallback_version_mask = direct_stratum.normalize_version_rolling_mask( + env( + "PRISM_VERSION_ROLLING_MASK", + direct_stratum.QBIT_VERSION_ROLLING_MASK_HEX, + environ=source, + ), + field_name="PRISM_VERSION_ROLLING_MASK", + ) + except ValueError as exc: + raise SystemExit(str(exc)) from exc + + stratum = StratumConfig( + bind=bind, + port=port, + extranonce2_size=env_int("PRISM_STRATUM_EXTRANONCE2_SIZE", 8, environ=source), + stale_grace_seconds=env_nonnegative_float( + "PRISM_STRATUM_STALE_GRACE_SECONDS", DEFAULT_PRISM_STALE_GRACE_SECONDS, environ=source + ), + send_timeout_seconds=env_nonnegative_float( + "PRISM_STRATUM_SEND_TIMEOUT_SECONDS", + DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + environ=source, + ), + max_connections=max_connections, + max_connections_per_username=env_nonnegative_int( + "PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, + environ=source, + ), + max_pending_initial_jobs=max_pending_initial_jobs, + initial_job_timeout_seconds=env_nonnegative_float( + "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", + DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, + environ=source, + ), + accept_resource_exhaustion_backoff_seconds=env_positive_float( + "PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS", + DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, + environ=source, + ), + listen_backlog=env_positive_int( + "PRISM_STRATUM_LISTEN_BACKLOG", DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, environ=source + ), + bind_retry_seconds=env_nonnegative_float( + "PRISM_STRATUM_BIND_RETRY_SECONDS", + DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, + environ=source, + ), + share_difficulty=share_difficulty, + vardiff_config=vardiff_config, + vardiff_idle_sweep_seconds=env_nonnegative_float( + "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS", + DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS, + environ=source, + ), + listener_profiles=tuple(listener_profiles), + default_share_weight=default_share_weight, + share_weights_by_username=_parse_share_weights(source.get("PRISM_STRATUM_SHARE_WEIGHTS_JSON", "")), + username_fallback_address=default_prism_username_fallback_address(environ=source), + fallback_version_mask=fallback_version_mask, + same_tip_job_retention_seconds=same_tip_seconds, + same_tip_job_retention_per_connection=same_tip_per_connection, + payout_address_cache_max_entries=env_nonnegative_int( + "PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES", + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, + environ=source, + ), + payout_address_cache_ttl_seconds=env_nonnegative_float( + "PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS", + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, + environ=source, + ), + ) + + jobs = JobPipelineConfig( + blockpoll_seconds=blockpoll_seconds, + blockwait_enabled=env_bool("PRISM_BLOCKWAIT_ENABLED", "1", environ=source), + blockwait_timeout_seconds=env_positive_float( + "PRISM_BLOCKWAIT_TIMEOUT_SECONDS", + DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, + environ=source, + ), + tip_refresh_failure_holdoff_seconds=env_nonnegative_float( + "PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS", + DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, + environ=source, + ), + submit_tip_max_age_seconds=env_nonnegative_float( + "PRISM_SUBMIT_TIP_MAX_AGE_SECONDS", + DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, + environ=source, + ), + tip_refresh_max_workers=tip_refresh_max_workers, + job_build_timeout_seconds=env_positive_float( + "PRISM_JOB_BUILD_TIMEOUT_SECONDS", DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, environ=source + ), + job_build_cancel_grace_seconds=env_nonnegative_float( + "PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS", + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + environ=source, + ), + worker_metrics_limit=env_nonnegative_int( + "PRISM_WORKER_METRICS_LIMIT", DEFAULT_PRISM_WORKER_METRICS_LIMIT, environ=source + ), + reorg_reconciler_enabled=env_bool("PRISM_REORG_RECONCILER_ENABLED", "1", environ=source), + job_bundle_cache_seconds=env_nonnegative_float( + "PRISM_JOB_BUNDLE_CACHE_SECONDS", + DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, + environ=source, + ), + bundle_build_timeout_seconds=env_positive_float( + "PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS", + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + environ=source, + ), + template_cache_seconds=env_nonnegative_float( + "PRISM_TEMPLATE_CACHE_SECONDS", blockpoll_seconds, environ=source + ), + template_refresh_failure_exit_seconds=template_refresh_failure_exit_seconds, + reorg_reconcile_cache_seconds=env_nonnegative_float( + "PRISM_REORG_RECONCILE_CACHE_SECONDS", + DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + environ=source, + ), + min_ready_miners=env_int("PRISM_MIN_READY_MINERS", 3, environ=source), + payout_environment=_selected_environment( + source, + ( + "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES", + "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE", + "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE", + "PRISM_PAYOUT_SAFETY_MULTIPLIER", + "PRISM_PAYOUT_MIN_OUTPUT_BITS", + "PRISM_PAYOUT_MIN_OUTPUT_SATS", + ), + ), + pool_fee_enabled_raw=source.get("PRISM_POOL_FEE_ENABLED"), + pool_fee_bps_raw=env_optional("PRISM_POOL_FEE_BPS", environ=source), + pool_fee_address=env_optional("PRISM_POOL_FEE_ADDRESS", environ=source), + pool_fee_program_hex=env_optional("PRISM_POOL_FEE_P2MR_PROGRAM_HEX", environ=source), + pool_fee_recipient_id=env_optional("PRISM_POOL_FEE_RECIPIENT_ID", environ=source), + pool_fee_order_key=env_optional("PRISM_POOL_FEE_ORDER_KEY", environ=source), + template_max_age_raw=source.get("PRISM_TEMPLATE_MAX_AGE_SECONDS"), + ) + + evidence_path = Path(env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json", environ=source)) + audit_dir = Path(env("PRISM_AUDIT_DIR", str(evidence_path.parent), environ=source)) + audit = AuditConfig( + evidence_path=evidence_path, + directory=audit_dir, + share_segment_size=env_nonnegative_int( + "PRISM_AUDIT_SHARE_SEGMENT_SIZE", DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, environ=source + ), + live_bundle_retention=env_nonnegative_int( + "PRISM_AUDIT_LIVE_BUNDLE_RETENTION", 5, environ=source + ), + candidate_retention_seconds=env_nonnegative_int( + "PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS", 24 * 60 * 60, environ=source + ), + bind=source.get("PRISM_AUDIT_BIND"), + port=int(source.get("PRISM_AUDIT_PORT", "0") or "0"), + ) + + configured_writer_public_key = env_optional( + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", environ=source + ) + if configured_writer_public_key is not None: + writer_public_key = validate_hex( + configured_writer_public_key, + name="PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", + expected_bytes=32, + ) + elif env_bool("PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", "0", environ=source): + writer_public_key = None + else: + raise SystemExit( + "PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX is required; " + "set PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY=1 only for local tests" + ) + psql_command = source.get("PRISM_POSTGRES_PSQL_COMMAND", "") + database_url = source.get("PRISM_DATABASE_URL", "") + if not psql_command and database_url: + psql_command = f"psql {shlex.quote(database_url)}" + writer_session_token = env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN", environ=source) + if writer_session_token is not None and not env_bool( + "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", "0", environ=source + ): + raise SystemExit( + "PRISM_LEDGER_WRITER_SESSION_TOKEN requires " + "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 for local tests" + ) + ledger = LedgerConfig( + psql_command=psql_command, + database_url=database_url or None, + allow_memory_ledger=env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0", environ=source), + native_client_mode=env("PRISM_POSTGRES_NATIVE_CLIENT", "auto", environ=source), + writer_id=env("PRISM_LEDGER_WRITER_ID", "prism-coordinator", environ=source), + writer_epoch=env_int("PRISM_LEDGER_WRITER_EPOCH", 1, environ=source), + writer_session_token=writer_session_token, + initialize_schema=env("PRISM_POSTGRES_INIT_SCHEMA", "0", environ=source) + in {"1", "true", "yes"}, + lease_ttl_seconds=env_positive_float( + "PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0, environ=source + ), + read_concurrency=env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4, environ=source), + accepted_stats_cache_seconds=env_nonnegative_float( + "PRISM_ACCEPTED_STATS_CACHE_SECONDS", 60.0, environ=source + ), + reward_window_cache_seconds=env_nonnegative_float( + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", 30.0, environ=source + ), + signing_seed_hex=env_seed_hex( + "PRISM_MANIFEST_SIGNING_SEED_HEX", test_default="42" * 32, environ=source + ), + attestation_signing_seed_hex=env_seed_hex( + "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", + test_default="43" * 32, + environ=source, + ), + writer_public_key_hex=writer_public_key, + share_commit_batch_size=env_positive_int( + "PRISM_SHARE_COMMIT_BATCH_SIZE", DEFAULT_SHARE_COMMIT_BATCH_SIZE, environ=source + ), + share_commit_linger_seconds=env_nonnegative_float( + "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS", + DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS, + environ=source, + ) + / 1000.0, + share_commit_timeout_seconds=env_positive_float( + "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS", DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS, environ=source + ), + share_recovery_path=Path( + env( + "PRISM_SHARE_RECOVERY_PATH", + str(audit_dir / "prism-unpersisted-shares.jsonl"), + environ=source, + ) + ), + ) + + configured_broadcaster_enabled = env_optional_bool( + "PRISM_CTV_BROADCASTER_ENABLED", environ=source + ) + broadcaster_enabled = ( + configured_broadcaster_enabled + if configured_broadcaster_enabled is not None + else env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0", environ=source) + ) + broadcaster_wallet = env_optional("PRISM_CTV_BROADCASTER_WALLET", environ=source) + broadcaster_fee_sats = env_nonnegative_int_with_legacy( + "PRISM_CTV_BROADCASTER_FEE_BITS", + "PRISM_CTV_BROADCASTER_FEE_SATS", + 0, + environ=source, + ) + if broadcaster_enabled and broadcaster_fee_sats > 0 and not broadcaster_wallet: + raise SystemExit( + "PRISM_CTV_BROADCASTER_WALLET is required when " + "PRISM_CTV_BROADCASTER_FEE_BITS is positive" + ) + broadcaster_chunk_size = env_positive_int( + "PRISM_CTV_BROADCASTER_CHUNK_SIZE", + DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, + environ=source, + ) + if broadcaster_chunk_size > MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE: + raise SystemExit( + "PRISM_CTV_BROADCASTER_CHUNK_SIZE must be at most " + f"{MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE}" + ) + ctv = CtvConfig( + settlement_enabled_raw=source.get("PRISM_CTV_SETTLEMENT_ENABLED"), + broadcaster_enabled=broadcaster_enabled, + broadcaster_wallet=broadcaster_wallet, + broadcaster_fee_sats=broadcaster_fee_sats, + broadcaster_limit=env_positive_int("PRISM_CTV_BROADCASTER_LIMIT", 100, environ=source), + broadcaster_chunk_size=broadcaster_chunk_size, + broadcaster_interval_seconds=env_positive_float( + "PRISM_CTV_BROADCASTER_INTERVAL_SECONDS", 30.0, environ=source + ), + broadcast_attempt_detail_limit=env_nonnegative_int( + "PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT", + DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, + environ=source, + ), + broadcast_retry_backoff_seconds=env_nonnegative_int( + "PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS", + DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, + environ=source, + ), + settlement_environment=_selected_environment( + source, + ( + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", + "PRISM_RESERVED_COINBASE_OUTPUTS", + "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", + "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", + "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", + "PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", + "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", + ), + ), + ) + + lifecycle = LifecycleConfig( + health_refresh_seconds=env_positive_float( + "PRISM_HEALTH_REFRESH_SECONDS", DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, environ=source + ), + pending_refresh_health_deadline_seconds=env_positive_float( + "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS", + DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, + environ=source, + ), + coherent_tip_poll_health_deadline_seconds=env_positive_float( + "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS", + DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, + environ=source, + ), + mining_health_startup_grace_seconds=env_nonnegative_float( + "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS", + DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, + environ=source, + ), + writer_quiescence_timeout_seconds=env_positive_float( + "PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS", + DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, + environ=source, + ), + watchdog_enabled=env_bool("PRISM_WATCHDOG_ENABLED", "1", environ=source), + watchdog_timeout_seconds=env_positive_float( + "PRISM_WATCHDOG_TIMEOUT_SECONDS", 120.0, environ=source + ), + watchdog_interval_seconds=env_positive_float( + "PRISM_WATCHDOG_INTERVAL_SECONDS", 15.0, environ=source + ), + ) + max_blocks = env_int("PRISM_MAX_BLOCKS", 1, environ=source) + if max_blocks <= 0: + raise SystemExit("PRISM_MAX_BLOCKS must be positive") + return CoordinatorConfig( + rpc=rpc, + stratum=stratum, + jobs=jobs, + ledger=ledger, + audit=audit, + ctv=ctv, + lifecycle=lifecycle, + production=production, + hot_path_log_enabled=env_bool("PRISM_HOT_PATH_LOG", "0", environ=source), + coinbase_tag_hex=default_prism_coinbase_tag_hex(environ=source), + stop_after_block=env("PRISM_STOP_AFTER_BLOCK", "1", environ=source) + in {"1", "true", "yes"}, + max_blocks=max_blocks, + ) diff --git a/lab/prism/coordinator_shutdown.py b/lab/prism/coordinator_shutdown.py new file mode 100644 index 0000000..1c98838 --- /dev/null +++ b/lab/prism/coordinator_shutdown.py @@ -0,0 +1,278 @@ +"""Writer admission and shutdown state for the PRISM coordinator.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager +from functools import wraps +import signal +import threading +import time +from typing import Any, Callable, Protocol + + +class ShutdownInProgress(RuntimeError): + """Raised when work that could mutate the ledger arrives after shutdown.""" + + +class _WriterOperationToken: + """One transferable writer admission held until durable work completes.""" + + def __init__(self, controller: "CoordinatorShutdownController", component: str): + self.controller = controller + self.component = component + self.finished = False + + def finish(self) -> None: + self.controller.finish_token(self) + + +class CoordinatorShutdownController: + """Coordinates the writer barrier, one-shot lease release, and final drain. + + Writer operations enter through :meth:`enter_writer` before shutdown or + inherit an already-admitted operation on the same thread. Queue admissions + use transferable tokens so a share remains visible to the barrier while it + moves from a client thread to the group-commit writer. + """ + + def __init__(self, writer_quiescence_timeout_seconds: float): + self.writer_quiescence_timeout_seconds = writer_quiescence_timeout_seconds + self.condition = threading.Condition(threading.RLock()) + self.local = threading.local() + self.phase = "running" + self.reason: str | None = None + self.signal_number: int | None = None + self.sigterm_monotonic: float | None = None + self.shutdown_started_monotonic: float | None = None + self.active_writers: dict[str, int] = {} + self.shutdowns_total = 0 + self.writer_quiescence_outcomes = {"success": 0, "timeout": 0} + self.writer_quiescence_seconds = 0.0 + self.lease_release_attempts_total = 0 + self.lease_release_outcomes = { + "success": 0, + "not_held": 0, + "unsupported": 0, + "failure": 0, + } + self.lease_release_seconds = 0.0 + self.lease_release_attempted = False + self.lease_release_succeeded = False + self.lease_release_withheld = False + self.sigterm_to_lease_release_seconds = 0.0 + self.sigterm_release_observed = False + self.release_withheld_total = 0 + self.non_writer_drain_seconds = 0.0 + self.non_writer_drains_total = 0 + self._drain_claimed = False + + def request_shutdown(self, signum: int | None) -> None: + """Close admission atomically; the caller only needs to set its event.""" + now = time.monotonic() + with self.condition: + if signum == signal.SIGTERM and self.sigterm_monotonic is None: + self.sigterm_monotonic = now + if self.signal_number is None and signum is not None: + self.signal_number = signum + if self.phase == "running": + self.phase = "requested" + self.condition.notify_all() + + def begin_shutdown(self, reason: str) -> bool: + with self.condition: + if self.phase not in {"running", "requested"}: + return False + self.phase = "quiescing_writers" + self.reason = reason + self.shutdown_started_monotonic = time.monotonic() + self.shutdowns_total += 1 + self.condition.notify_all() + return True + + def wait_for_lease_handling(self) -> bool: + """Wait for the one shutdown owner to release or safely withhold.""" + in_progress = { + "requested", + "quiescing_writers", + "writers_quiesced", + "releasing_lease", + } + with self.condition: + while self.phase in in_progress: + self.condition.wait() + return self.lease_release_succeeded + + def _thread_writer_depth(self) -> int: + return int(getattr(self.local, "writer_depth", 0)) + + def _admit_writer_locked(self, component: str, *, inherited: bool) -> _WriterOperationToken: + if self.lease_release_attempted: + raise ShutdownInProgress("PRISM writer lease release has already started") + if self.phase != "running" and not inherited: + raise ShutdownInProgress("PRISM coordinator is shutting down") + self.active_writers[component] = self.active_writers.get(component, 0) + 1 + return _WriterOperationToken(self, component) + + def enter_writer(self, component: str) -> _WriterOperationToken: + depth = self._thread_writer_depth() + with self.condition: + token = self._admit_writer_locked(component, inherited=depth > 0) + self.local.writer_depth = depth + 1 + return token + + def exit_writer(self, token: _WriterOperationToken) -> None: + depth = self._thread_writer_depth() + self.local.writer_depth = max(0, depth - 1) + token.finish() + + def reserve_writer(self, component: str) -> _WriterOperationToken: + """Reserve work that will finish on another thread.""" + with self.condition: + return self._admit_writer_locked( + component, + inherited=self._thread_writer_depth() > 0, + ) + + def finish_token(self, token: _WriterOperationToken) -> None: + with self.condition: + if token.finished: + return + token.finished = True + remaining = self.active_writers.get(token.component, 0) - 1 + if remaining > 0: + self.active_writers[token.component] = remaining + else: + self.active_writers.pop(token.component, None) + self.condition.notify_all() + + def has_active_writer(self, components: set[str]) -> bool: + with self.condition: + return any(self.active_writers.get(component, 0) for component in components) + + def wait_for_no_active_writer( + self, + components: set[str], + timeout_seconds: float, + ) -> bool: + """Wait once for the selected writer classes to become quiescent.""" + with self.condition: + if not any( + self.active_writers.get(component, 0) for component in components + ): + return True + self.condition.wait(max(0.0, timeout_seconds)) + return not any( + self.active_writers.get(component, 0) for component in components + ) + + def writer_admission_closed(self) -> bool: + with self.condition: + return self.phase != "running" + + def wait_for_writer_quiescence(self) -> tuple[bool, float, dict[str, int]]: + started = time.monotonic() + deadline = started + self.writer_quiescence_timeout_seconds + with self.condition: + while self.active_writers: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + self.condition.wait(remaining) + elapsed = max(0.0, time.monotonic() - started) + quiesced = not self.active_writers + blockers = dict(sorted(self.active_writers.items())) + outcome = "success" if quiesced else "timeout" + self.writer_quiescence_outcomes[outcome] += 1 + self.writer_quiescence_seconds = elapsed + if quiesced: + self.phase = "writers_quiesced" + else: + self.phase = "release_withheld" + self.lease_release_withheld = True + self.release_withheld_total += 1 + self.condition.notify_all() + return quiesced, elapsed, blockers + + def claim_lease_release(self) -> tuple[bool, dict[str, int]]: + with self.condition: + if self.lease_release_attempted or self.lease_release_withheld: + return False, {} + if self.active_writers: + return False, dict(sorted(self.active_writers.items())) + self.lease_release_attempted = True + self.lease_release_attempts_total += 1 + self.phase = "releasing_lease" + self.condition.notify_all() + return True, {} + + def finish_lease_release(self, outcome: str, elapsed: float) -> None: + with self.condition: + self.lease_release_outcomes[outcome] += 1 + self.lease_release_seconds = elapsed + self.lease_release_succeeded = outcome != "failure" + self.phase = "lease_released" if outcome != "failure" else "lease_release_failed" + if outcome != "failure" and self.sigterm_monotonic is not None: + self.sigterm_to_lease_release_seconds = max( + 0.0, + time.monotonic() - self.sigterm_monotonic, + ) + self.sigterm_release_observed = True + self.condition.notify_all() + + def claim_non_writer_drain(self) -> bool: + with self.condition: + if self._drain_claimed: + return False + if self.phase not in { + "lease_released", + "lease_release_failed", + "release_withheld", + }: + return False + self._drain_claimed = True + self.phase = "draining_non_writers" + return True + + def finish_non_writer_drain(self, elapsed: float) -> None: + with self.condition: + self.non_writer_drain_seconds = elapsed + self.non_writer_drains_total += 1 + self.phase = "complete" + self.condition.notify_all() + + def snapshot(self) -> dict[str, Any]: + with self.condition: + return { + "phase": self.phase, + "active_writers": dict(self.active_writers), + "shutdowns_total": self.shutdowns_total, + "writer_quiescence_outcomes": dict(self.writer_quiescence_outcomes), + "writer_quiescence_seconds": self.writer_quiescence_seconds, + "lease_release_attempts_total": self.lease_release_attempts_total, + "lease_release_outcomes": dict(self.lease_release_outcomes), + "lease_release_seconds": self.lease_release_seconds, + "lease_release_withheld": self.lease_release_withheld, + "sigterm_to_lease_release_seconds": self.sigterm_to_lease_release_seconds, + "sigterm_release_observed": self.sigterm_release_observed, + "release_withheld_total": self.release_withheld_total, + "non_writer_drain_seconds": self.non_writer_drain_seconds, + "non_writer_drains_total": self.non_writer_drains_total, + } + + +class _WriterOperationOwner(Protocol): + def _writer_operation(self, component: str) -> AbstractContextManager[object]: ... + + +def ledger_writer_operation(component: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Decorate an entry point that can mutate the PRISM ledger.""" + + def decorate(method: Callable[..., Any]) -> Callable[..., Any]: + @wraps(method) + def guarded(self: _WriterOperationOwner, *args: Any, **kwargs: Any) -> Any: + with self._writer_operation(component): + return method(self, *args, **kwargs) + + return guarded + + return decorate diff --git a/lab/prism/ctv_runtime.py b/lab/prism/ctv_runtime.py new file mode 100644 index 0000000..fe106a4 --- /dev/null +++ b/lab/prism/ctv_runtime.py @@ -0,0 +1,397 @@ +"""Coordinator-facing lifecycle and metrics for the CTV broadcaster daemon. + +The CTV broadcast engine and durable daemon remain in their existing modules. +This service owns only the process runtime seam: daemon construction, writer +admission, the dedicated loop, and its bounded-cardinality metrics. +""" + +from __future__ import annotations + +from contextlib import AbstractContextManager +from dataclasses import dataclass, replace +import threading +import time +import traceback +from typing import Any, Callable, Protocol + +from lab.prism.background_services import BackgroundServiceSpec +from lab.prism.coordinator_config import CtvConfig +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.ctv_broadcaster import CtvFanoutBroadcaster +from lab.prism.ctv_broadcaster_daemon import ( + CtvFanoutBroadcastDaemon, + CtvFanoutChunkResult, + CtvFanoutDaemonResult, +) + + +CTV_FANOUT_BROADCASTER_SERVICE_NAME = "ctv_fanout_broadcaster" +CTV_BROADCAST_STATE_COMPONENT = "ctv_broadcast_state" +PRISM_CTV_BROADCASTER_SECONDS_BUCKETS = ( + 1.0, + 5.0, + 10.0, + 30.0, + 60.0, + 120.0, + 300.0, + 600.0, +) +PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS = (1, 2, 5, 10, 25, 50, 100) + + +class StopSignal(Protocol): + def is_set(self) -> bool: ... + + def wait(self, timeout: float) -> bool: ... + + +@dataclass(frozen=True, slots=True) +class CtvRuntimeConfig: + enabled: bool + wallet: str | None + fee_sats: int + limit: int + chunk_size: int + interval_seconds: float + + @classmethod + def from_coordinator_config(cls, config: CtvConfig) -> "CtvRuntimeConfig": + return cls( + enabled=config.broadcaster_enabled, + wallet=config.broadcaster_wallet, + fee_sats=config.broadcaster_fee_sats, + limit=config.broadcaster_limit, + chunk_size=config.broadcaster_chunk_size, + interval_seconds=config.broadcaster_interval_seconds, + ) + + +@dataclass(frozen=True, slots=True) +class CtvRuntimeMetricsSnapshot: + pass_seconds_bucket_counts: dict[float, int] + pass_seconds_sum: float + pass_count: int + processed_rows_total: int + yielded_total: int + chunk_seconds_bucket_counts: dict[float, int] + chunk_rows_bucket_counts: dict[int, int] + chunk_seconds_sum: float + chunk_rows_sum: int + chunk_count: int + + +class CtvRuntimeService: + """Own the coordinator's CTV daemon instance, loop, and metric state.""" + + def __init__( + self, + *, + rpc_call: Callable[..., Any], + ledger: object, + writer_admission: Callable[[str], AbstractContextManager[object]], + tip_refresh_pending: Callable[[], bool], + heartbeat: Callable[[], None], + stop_event: StopSignal, + config: CtvRuntimeConfig, + daemon_type: Callable[..., CtvFanoutBroadcastDaemon] = CtvFanoutBroadcastDaemon, + broadcaster_type: Callable[..., CtvFanoutBroadcaster] = CtvFanoutBroadcaster, + monotonic: Callable[[], float] = time.monotonic, + print_exception: Callable[[], None] = traceback.print_exc, + ) -> None: + self._rpc_call = rpc_call + self._ledger = ledger + self._writer_admission = writer_admission + self._tip_refresh_pending = tip_refresh_pending + self._heartbeat = heartbeat + self._stop_event = stop_event + self._daemon_type = daemon_type + self._broadcaster_type = broadcaster_type + self._monotonic = monotonic + self._print_exception = print_exception + self._config_lock = threading.Lock() + self._config = config + + self._daemon_lock = threading.Lock() + self._daemon: CtvFanoutBroadcastDaemon | None = None + self._metrics_lock = threading.Lock() + self._pass_seconds_bucket_counts = { + bucket: 0 for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS + } + self._pass_seconds_sum = 0.0 + self._pass_count = 0 + self._processed_rows_total = 0 + self._yielded_total = 0 + self._chunk_seconds_bucket_counts = { + bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS + } + self._chunk_rows_bucket_counts = { + bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS + } + self._chunk_seconds_sum = 0.0 + self._chunk_rows_sum = 0 + self._chunk_count = 0 + + @property + def daemon(self) -> CtvFanoutBroadcastDaemon | None: + with self._daemon_lock: + return self._daemon + + @daemon.setter + def daemon(self, daemon: CtvFanoutBroadcastDaemon | None) -> None: + with self._daemon_lock: + self._daemon = daemon + + @property + def config(self) -> CtvRuntimeConfig: + """Return one immutable configuration snapshot under service ownership.""" + with self._config_lock: + return self._config + + def replace_config(self, **changes: object) -> None: + """Support temporary coordinator compatibility properties.""" + with self._daemon_lock: + with self._config_lock: + previous = self._config + replacement = replace(previous, **changes) + self._config = replacement + if ( + previous.wallet != replacement.wallet + or previous.fee_sats != replacement.fee_sats + ): + self._daemon = None + + def make_daemon( + self, + config: CtvRuntimeConfig | None = None, + ) -> CtvFanoutBroadcastDaemon: + config = self.config if config is None else config + if config.fee_sats > 0 and not config.wallet: + raise ValueError( + "ctv_broadcaster_wallet is required when " + "ctv_broadcaster_fee_sats is positive" + ) + broadcaster = self._broadcaster_type( + self._rpc_call, + funding_wallet=config.wallet, + ) + return self._daemon_type( + self._ledger, + broadcaster, + fee_sats=config.fee_sats, + ) + + def _configured_daemon( + self, + ) -> tuple[CtvRuntimeConfig, CtvFanoutBroadcastDaemon]: + with self._daemon_lock: + with self._config_lock: + config = self._config + if self._daemon is None: + self._daemon = self.make_daemon(config) + return config, self._daemon + + def run_once( + self, + *, + progress_callback: Callable[[], None] | None = None, + chunk_callback: Callable[[CtvFanoutChunkResult], None] | None = None, + ) -> CtvFanoutDaemonResult: + with self._writer_admission(CTV_BROADCAST_STATE_COMPONENT): + config, daemon = self._configured_daemon() + return daemon.run_once( + limit=config.limit, + progress_callback=progress_callback, + chunk_size=config.chunk_size, + tip_refresh_pending=self._tip_refresh_pending, + chunk_callback=( + self.observe_chunk if chunk_callback is None else chunk_callback + ), + ) + + def record_progress(self) -> None: + self._heartbeat() + with self._metrics_lock: + self._processed_rows_total += 1 + + def observe_pass(self, elapsed_seconds: float) -> None: + with self._metrics_lock: + self._pass_count += 1 + self._pass_seconds_sum += elapsed_seconds + for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + self._pass_seconds_bucket_counts[bucket] += 1 + + def observe_chunk(self, result: CtvFanoutChunkResult) -> None: + self._heartbeat() + with self._metrics_lock: + self._chunk_count += 1 + self._chunk_seconds_sum += result.elapsed_seconds + self._chunk_rows_sum += result.processed_count + for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS: + if result.elapsed_seconds <= bucket: + self._chunk_seconds_bucket_counts[bucket] += 1 + for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS: + if result.processed_count <= bucket: + self._chunk_rows_bucket_counts[bucket] += 1 + + def record_yield(self) -> None: + with self._metrics_lock: + self._yielded_total += 1 + + def loop( + self, + *, + run_once: Callable[..., CtvFanoutDaemonResult] | None = None, + progress_callback: Callable[[], None] | None = None, + observe_pass: Callable[[float], None] | None = None, + record_yield: Callable[[], None] | None = None, + ) -> None: + """Run the dedicated broadcaster loop until shutdown. + + The optional call ports retain the coordinator's temporary method-level + compatibility seam; normal registered service threads use the owned + methods directly. + """ + pass_runner = self.run_once if run_once is None else run_once + progress_recorder = ( + self.record_progress if progress_callback is None else progress_callback + ) + pass_observer = self.observe_pass if observe_pass is None else observe_pass + yield_recorder = self.record_yield if record_yield is None else record_yield + while not self._stop_event.is_set(): + self._heartbeat() + started = self._monotonic() + shutdown_admission_closed = False + try: + try: + result = pass_runner(progress_callback=progress_recorder) + except ShutdownInProgress: + shutdown_admission_closed = True + return + finally: + # Stamp completion before logging or entering the interval + # wait. A blocked row never reaches this finally clause, so + # the watchdog remains able to recover a wedged operation. + self._heartbeat() + if not shutdown_admission_closed: + pass_observer(max(0.0, self._monotonic() - started)) + except Exception: + print("prism coordinator: CTV fanout broadcaster pass failed", flush=True) + self._print_exception() + else: + if result.yielded_to_tip_refresh: + yield_recorder() + if result.scanned_count or result.submitted_count or result.failed_count: + print( + "prism coordinator: CTV fanout broadcaster " + f"scanned={result.scanned_count} " + f"submitted={result.submitted_count} " + f"updated={result.updated_count} " + f"failed={result.failed_count}", + flush=True, + ) + if self._stop_event.wait(self.config.interval_seconds): + break + + def background_service_spec(self) -> BackgroundServiceSpec: + return BackgroundServiceSpec( + name=CTV_FANOUT_BROADCASTER_SERVICE_NAME, + thread_name="prism-ctv-fanout-broadcaster", + target=self.loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + ) + + def startup_summary(self) -> str: + config = self.config + return ( + "prism coordinator: CTV fanout broadcaster enabled " + f"mode={'cpfp' if config.fee_sats > 0 else 'direct'} " + f"fee_bits={config.fee_sats} " + f"wallet={'configured' if config.wallet else 'none'} " + f"interval={config.interval_seconds:g}s " + f"limit={config.limit} " + f"chunk_size={config.chunk_size}" + ) + + def metrics_snapshot(self) -> CtvRuntimeMetricsSnapshot: + with self._metrics_lock: + return CtvRuntimeMetricsSnapshot( + pass_seconds_bucket_counts=dict(self._pass_seconds_bucket_counts), + pass_seconds_sum=self._pass_seconds_sum, + pass_count=self._pass_count, + processed_rows_total=self._processed_rows_total, + yielded_total=self._yielded_total, + chunk_seconds_bucket_counts=dict(self._chunk_seconds_bucket_counts), + chunk_rows_bucket_counts=dict(self._chunk_rows_bucket_counts), + chunk_seconds_sum=self._chunk_seconds_sum, + chunk_rows_sum=self._chunk_rows_sum, + chunk_count=self._chunk_count, + ) + + @property + def processed_rows_total(self) -> int: + return self.metrics_snapshot().processed_rows_total + + @property + def pass_count(self) -> int: + return self.metrics_snapshot().pass_count + + def metrics_lines(self) -> list[str]: + snapshot = self.metrics_snapshot() + metric_name = "qbit_prism_ctv_fanout_broadcaster_pass_seconds" + chunk_seconds_name = "qbit_prism_ctv_fanout_broadcaster_chunk_seconds" + chunk_rows_name = "qbit_prism_ctv_fanout_broadcaster_chunk_rows" + return [ + "# HELP qbit_prism_ctv_fanout_broadcaster_processed_rows_total CTV fanout rows completed by the broadcaster loop.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_processed_rows_total counter", + f"qbit_prism_ctv_fanout_broadcaster_processed_rows_total {snapshot.processed_rows_total}", + "# HELP qbit_prism_ctv_fanout_broadcaster_pass_seconds CTV fanout broadcaster pass wall time.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_pass_seconds histogram", + *[ + f'{metric_name}_bucket{{le="{bucket:g}"}} ' + f"{snapshot.pass_seconds_bucket_counts.get(bucket, 0)}" + for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS + ], + f'{metric_name}_bucket{{le="+Inf"}} {snapshot.pass_count}', + f"{metric_name}_sum {snapshot.pass_seconds_sum:.6f}", + f"{metric_name}_count {snapshot.pass_count}", + "# HELP qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total CTV broadcaster passes yielding between committed chunks for a pending tip refresh.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total counter", + f"qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total {snapshot.yielded_total}", + "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_seconds CTV broadcaster committed chunk wall time.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_seconds histogram", + *[ + f'{chunk_seconds_name}_bucket{{le="{bucket:g}"}} ' + f"{snapshot.chunk_seconds_bucket_counts.get(bucket, 0)}" + for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS + ], + f'{chunk_seconds_name}_bucket{{le="+Inf"}} {snapshot.chunk_count}', + f"{chunk_seconds_name}_sum {snapshot.chunk_seconds_sum:.6f}", + f"{chunk_seconds_name}_count {snapshot.chunk_count}", + "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_rows Rows processed per committed CTV broadcaster chunk.", + "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_rows histogram", + *[ + f'{chunk_rows_name}_bucket{{le="{bucket}"}} ' + f"{snapshot.chunk_rows_bucket_counts.get(bucket, 0)}" + for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS + ], + f'{chunk_rows_name}_bucket{{le="+Inf"}} {snapshot.chunk_count}', + f"{chunk_rows_name}_sum {snapshot.chunk_rows_sum}", + f"{chunk_rows_name}_count {snapshot.chunk_count}", + ] diff --git a/lab/prism/job_build_benchmark.py b/lab/prism/job_build_benchmark.py index 0282f91..348cc75 100644 --- a/lab/prism/job_build_benchmark.py +++ b/lab/prism/job_build_benchmark.py @@ -461,7 +461,7 @@ def main() -> int: if not server.ensure_reorg_reconciled_for_tip(rpc.template["previousblockhash"]): raise RuntimeError("benchmark payout reconciliation failed") server._prepare_payout_ledger_artifact( - server._payout_state_generation, + server._ensure_payout_state_service().snapshot().generation, artifacts.network_difficulty, ) # The benchmark's timed path models an already-published artifact, not @@ -500,6 +500,8 @@ def observing(elapsed_seconds: float, phases: dict[str, float]) -> None: finally: pool.close() + tip_refresh_metrics = server._ensure_tip_refresh_service().metrics_snapshot() + job_bundle_metrics = server._ensure_job_bundle_service().metrics_snapshot() report = { "schema": "qbit.prism.job-build-benchmark.v1", "mode": args.mode, @@ -514,23 +516,19 @@ def observing(elapsed_seconds: float, phases: dict[str, float]) -> None: "sent_job_elapsed_seconds": summarize(samples), "full_refresh_wall_seconds": summarize(flip_wall_times), "tip_to_first_delivery_seconds": summarize_histogram( - server.tip_refresh_histograms["first_delivery"] + tip_refresh_metrics["histograms"]["first_delivery"] ), "tip_to_last_delivery_seconds": summarize_histogram( - server.tip_refresh_histograms["last_delivery"] + tip_refresh_metrics["histograms"]["last_delivery"] ), "phase_totals_seconds": {k: round(v, 3) for k, v in sorted(phase_totals.items())}, "bundle_phase_totals_seconds": { phase: round(float(histogram["sum"]), 4) - for phase, histogram in getattr( - server, - "tip_refresh_build_phase_histograms", - {}, - ).items() + for phase, histogram in tip_refresh_metrics["phase_histograms"].items() }, - "builder_ipc_bytes": dict(getattr(server, "tip_refresh_ipc_bytes", {})), - "cache_hits": dict(server.job_cache_hit_counts), - "cache_misses": dict(server.job_cache_miss_counts), + "builder_ipc_bytes": dict(tip_refresh_metrics["ipc_bytes"]), + "cache_hits": dict(job_bundle_metrics["hit_counts"]), + "cache_misses": dict(job_bundle_metrics["miss_counts"]), "getblocktemplate_calls": rpc.calls.get("getblocktemplate", 0), } text = json.dumps(report, indent=2) diff --git a/lab/prism/job_bundle.py b/lab/prism/job_bundle.py new file mode 100644 index 0000000..5cbfbdd --- /dev/null +++ b/lab/prism/job_bundle.py @@ -0,0 +1,2463 @@ +"""Immutable PRISM job bundles, cache, and bounded latest-wins scheduler.""" + +from __future__ import annotations + +from collections import OrderedDict +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, field, replace as dataclass_replace +from decimal import Context, Decimal, getcontext, localcontext +import hashlib +import json +import subprocess +import threading +import time +import weakref +from typing import Any, Callable, ContextManager, Iterator, Protocol, Sequence + +from lab.prism import direct_stratum +from lab.prism.payout_state import ( + PayoutLedgerArtifact, + PayoutStateArtifact, + PayoutStatePublicationBlocked, + PayoutStateSnapshot, + TemplateRefreshBlocked, + TemplateRefreshSuperseded, +) +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + QbitTipTemplateSnapshot, + TemplateArtifactRepository, + freeze_json, +) + + +MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES = 128 +PRISM_JOB_BUILD_EXECUTOR_WORKERS = 2 +PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX = "00000000" +PRISM_REWARD_WINDOW_MULTIPLIER = 8 +PRISM_SNAPSHOT_WINDOW_MARGIN = 2 +PRISM_JOB_BUILD_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +PRISM_JOB_BUILD_PHASES = ( + "reorg", + "template", + "merkle", + "ledger", + "payout_artifact", + "payout", + "ctv", + "input_serialization", + "worker", + "output_serialization", + "assembly", + "bundle", + "preparation_wait", + "executor_queue", + "client_lock", + "payout_gate", + "stamp", + "socket_send", + "send", +) +PRISM_JOB_CACHE_KINDS = ("template", "bundle") + + +def canonical_json_text(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + + +def canonical_json_sha256(value: object) -> str: + return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() + + +class WorkerIdentityPort(Protocol): + payout_address: str + p2mr_program_hex: str + + +class PayoutStatePort(Protocol): + @property + def prepare_lock(self) -> threading.RLock: ... + + def cache_publication_admission(self) -> ContextManager[None]: ... + + def snapshot(self) -> PayoutStateSnapshot: ... + + def current_artifact( + self, + cancellation: object | None = None, + ) -> PayoutStateArtifact: ... + + def usable_ledger_artifact( + self, + payout_state_generation: int, + network_difficulty: int, + ) -> PayoutLedgerArtifact | None: ... + + +@dataclass(frozen=True) +class JobBuildKey: + """Every immutable input capable of changing one constructed job.""" + + best_tip_hash: str + previous_block_hash: str + template_fingerprint: str + template_generation: int + payout_state_generation: int + payout_artifact_sha256: str + mode: str + collection_identity: tuple[str, str] | None + block_height: int + coinbase_value_sats: int + network_difficulty: int + issued_at_ms: int + payout_policy_sha256: str + ctv_settlement_sha256: str | None + witness_merkle_sha256: str + transaction_set_sha256: str + coinbase_suffix_hex: str + signing_key_sha256: str + ledger_signing_key_sha256: str + numeric_context_sha256: str + share_snapshot_sha256: str = "" + + +@dataclass(frozen=True) +class CachedJobBundle: + """One immutable heavy job build reusable across client stamping.""" + + key: tuple[object, ...] + template: dict[str, Any] + template_fingerprint: str + coinbase_manifest: dict[str, Any] + shares_json: list[dict[str, object]] + prior_balances: list[dict[str, object]] + found_block: dict[str, object] + collection_only: bool + issued_at_ms: int + base_job: direct_stratum.DirectQbitStratumJob + built_monotonic: float + template_generation: int = 0 + payout_state_generation: int = 0 + payout_artifact_generation: int = 0 + collection_identity: tuple[str, str] | None = None + prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None + build_key: JobBuildKey | None = None + + def __post_init__(self) -> None: + for name in ( + "template", + "coinbase_manifest", + "shares_json", + "prior_balances", + "found_block", + ): + object.__setattr__(self, name, freeze_json(getattr(self, name))) + + +class JobBuildCancelled(TemplateRefreshBlocked): + """An immutable build was cancelled or timed out.""" + + +class JobBuildSuperseded(JobBuildCancelled, TemplateRefreshSuperseded): + """A coordination race cooperatively cancelled construction.""" + + +class JobBundleBuildSuperseded(JobBuildSuperseded): + """A newer tip or payout generation canceled deterministic construction.""" + + +class CollectionIdentityUnavailable(TemplateRefreshBlocked): + """Collection work is waiting for an authorized worker identity.""" + + +class JobBuildWaiterCancelled(RuntimeError): + """A bundle waiter became obsolete before acquiring preparation.""" + + +class JobBuildCancellation: + def __init__( + self, + *, + timeout_seconds: float, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self._event = threading.Event() + self._lock = threading.Lock() + self._monotonic = monotonic + self.started_monotonic = monotonic() + self.deadline_monotonic = self.started_monotonic + timeout_seconds + self.last_checkpoint_monotonic = self.started_monotonic + self.cancelled_monotonic: float | None = None + self.reason: str | None = None + + def cancel(self, reason: str) -> bool: + with self._lock: + if self._event.is_set(): + return False + self.reason = reason + self.cancelled_monotonic = self._monotonic() + self._event.set() + return True + + def is_set(self) -> bool: + if self._event.is_set(): + return True + if self._monotonic() >= self.deadline_monotonic: + self.cancel("timeout") + return True + return False + + def raise_if_cancelled(self, phase: str) -> None: + if self.is_set(): + reason = self.reason or "cancelled" + if reason == "timeout": + raise JobBuildCancelled( + f"job build timeout at {phase}; immediate retry scheduled" + ) + raise JobBuildSuperseded( + f"job build {reason} at {phase}; immediate retry scheduled" + ) + with self._lock: + self.last_checkpoint_monotonic = self._monotonic() + + +@dataclass +class JobBundleBuildControl: + key: tuple[object, ...] + previousblockhash: str + payout_state_generation: int + payout_artifact_generation: int + cancel_event: threading.Event = field(default_factory=threading.Event) + process: subprocess.Popen[str] | None = None + + +@dataclass +class JobBuildRequest: + key: JobBuildKey + cache_key: tuple[object, ...] + equivalence_key: tuple[object, ...] + artifacts: CachedTemplateArtifacts + template_json: str + transaction_hexes: tuple[str, ...] + witness_merkle_leaves_hex: tuple[str, ...] + worker: WorkerIdentityPort | None + mode: str + payout_artifact: PayoutStateArtifact + payout_ledger_artifact: PayoutLedgerArtifact | None + payout_policy_json: str + ctv_settlement_json: str | None + decimal_context: Context = field(repr=False) + cancellation: JobBuildCancellation + idle_retarget: bool = False + publication_critical: bool = False + request_source: str = "routine" + priority_admission_recorded: bool = False + promise: Future[CachedJobBundle] = field(default_factory=Future) + requested_monotonic: float = field(default_factory=time.monotonic) + superseded_monotonic: float | None = None + + +@dataclass(eq=False) +class JobBuildFlight: + request: JobBuildRequest + future: Future[CachedJobBundle] | None = None + + +@dataclass(frozen=True) +class JobBundleConfig: + cache_seconds: float + build_timeout_seconds: float + cancel_grace_seconds: float + min_ready_miners: int + extranonce2_size: int + share_difficulty: Decimal + + +class BundleCompilerPort(Protocol): + def build_audit_bundle(self, **kwargs: object) -> dict[str, Any]: ... + + +@dataclass(frozen=True) +class JobBundlePorts: + payout_state: Callable[[], PayoutStatePort] + accepted_share_stats: Callable[[], tuple[int, int]] + snapshot_at_job_issue: Callable[[int, int], Sequence[object]] + snapshot_anchor_ms: Callable[[int], int] + payout_policy: Callable[[], dict[str, object]] + ctv_settlement: Callable[[int, str], dict[str, object] | None] + coinbase_suffix: Callable[[str, str], str] + signing_seed_hex: Callable[[], str] + ledger_signing_seed_hex: Callable[[], str] + await_parent_preview: Callable[[str, int], object] + prior_balances_for_parent: Callable[ + [str, int, Sequence[dict[str, object]]], list[dict[str, object]] + ] + serialize_prior_balance_preview: Callable[ + [list[dict[str, object]]], tuple[tuple[str, str, str, int], ...] + ] + accepted_block_preview_from_bundle: Callable[ + [dict[str, Any], list[dict[str, object]]], list[dict[str, object]] + ] + schedule_refresh_retry: Callable[[], None] + idle_tip_diverged: Callable[[], bool] + artifacts_buildable: Callable[[CachedTemplateArtifacts], bool] + published_snapshot_artifacts: Callable[[CachedTemplateArtifacts], bool] + published_artifacts: Callable[[], CachedTemplateArtifacts | None] + note_tip_refresh_superseded: Callable[[], None] + record_tip_refresh_phase: Callable[[str, float], None] + clear_retained_collection_refresh: Callable[[], None] + readiness_promoted: Callable[[], None] + start_bundle_build: Callable[[], ContextManager[object]] + wall_time_ms: Callable[[], int] + + +class JobBundleService: + """Sole owner of shared bundle state, scheduler, cache, and readiness.""" + + def __init__( + self, + config: JobBundleConfig, + ports: JobBundlePorts, + template_repository: TemplateArtifactRepository, + *, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self._config = config + self._ports = ports + self.template_repository = template_repository + self._monotonic = monotonic + self._dependency_lock = threading.Lock() + self._bundle_compiler: BundleCompilerPort | None = None + self._cache_lock = threading.Lock() + self._active_bundle_builds: dict[ + tuple[object, ...], JobBundleBuildControl + ] = {} + self._admission_lock = threading.Lock() + self._scheduler_lock = threading.RLock() + self._priority_preparations: dict[int, float] = {} + self._priority_preparation_sequence = 0 + self._routine_preparations: dict[ + int, + weakref.ReferenceType[JobBuildCancellation], + ] = {} + self._routine_preparation_sequence = 0 + self._priority_changed = threading.Event() + self._executor: ThreadPoolExecutor | None = None + self._executor_shutdown = False + self._active: JobBuildFlight | None = None + self._retiring: JobBuildFlight | None = None + self._pending: JobBuildRequest | None = None + self._issued_at_ms: OrderedDict[int, int] = OrderedDict() + self._bundle_cache: OrderedDict[ + tuple[object, ...], CachedJobBundle + ] = OrderedDict() + self._phase_local = threading.local() + self._readiness_lock = threading.Lock() + self._ready_latched = False + self._prepared_lock = threading.Lock() + self._prepared_ready_bundle: CachedJobBundle | None = None + self._prepared_ready_snapshot: QbitTipTemplateSnapshot | None = None + self._preparation_pending = False + self._failure_count = 0 + self._cache_hits = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} + self._cache_misses = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} + self._build_bucket_counts = { + bucket: 0 for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS + } + self._build_seconds_sum = 0.0 + self._build_count = 0 + self._phase_seconds = {phase: 0.0 for phase in PRISM_JOB_BUILD_PHASES} + self._scheduler_counts = { + "requests": 0, + "starts": 0, + "completions": 0, + "supersessions": 0, + "obsolete_results": 0, + } + self._priority_counts = { + result: 0 + for result in ( + "started", + "coalesced", + "queued", + "routine_deferred", + "routine_preempted", + ) + } + self._priority_admission_seconds = {"sum": 0.0, "count": 0} + self._initial_prepared_work_counts = { + result: 0 for result in ("cache_hit", "singleflight", "deferred") + } + self._cancellation_seconds = {"sum": 0.0, "count": 0} + self._replacement_start_seconds = {"sum": 0.0, "count": 0} + self._worker_counts = { + "starts": 0, + "terminations": 0, + "crashes": 0, + "restarts": 0, + } + self._worker_restart_pending = False + self._shared_build_counts = { + outcome: 0 + for outcome in ("started", "completed", "superseded", "failed") + } + self._preparation_seconds_sum = 0.0 + self._preparation_count = 0 + self._preparation_waiters = 0 + + def bind_bundle_compiler(self, compiler: BundleCompilerPort) -> None: + """Bind the compiler once after constructing both leaf services.""" + with self._dependency_lock: + if self._bundle_compiler is not None: + raise RuntimeError("job bundle compiler is already bound") + self._bundle_compiler = compiler + + def bundle_compiler(self) -> BundleCompilerPort: + with self._dependency_lock: + compiler = self._bundle_compiler + if compiler is None: + raise RuntimeError("job bundle compiler is not bound") + return compiler + + @staticmethod + def collection_identity(worker: WorkerIdentityPort) -> tuple[str, str]: + return worker.payout_address, worker.p2mr_program_hex + + def phases(self) -> dict[str, float]: + phases = getattr(self._phase_local, "phases", None) + if phases is None: + phases = {} + self._phase_local.phases = phases + return phases + + def record_phase(self, phase: str, elapsed: float) -> None: + phases = self.phases() + phases[phase] = phases.get(phase, 0.0) + elapsed + + def observe_elapsed(self, elapsed_seconds: float, phases: dict[str, float]) -> None: + with self._cache_lock: + self._build_count += 1 + self._build_seconds_sum += elapsed_seconds + for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + self._build_bucket_counts[bucket] += 1 + for phase, duration in phases.items(): + if phase in self._phase_seconds: + self._phase_seconds[phase] += duration + + def record_cache_event(self, kind: str, *, hit: bool) -> None: + with self._cache_lock: + counts = self._cache_hits if hit else self._cache_misses + counts[kind] = int(counts.get(kind, 0)) + 1 + + def pool_readiness_latched(self) -> bool: + with self._readiness_lock: + if self._ready_latched: + return True + try: + _, ready_miner_count = self._ports.accepted_share_stats() + except Exception: + return False + if ready_miner_count < self._config.min_ready_miners: + return False + became_ready = False + with self._readiness_lock: + if not self._ready_latched: + self._ready_latched = True + became_ready = True + if became_ready: + self._ports.clear_retained_collection_refresh() + self._ports.readiness_promoted() + return True + + def job_bundle_mode(self, requested_mode: str | None) -> str: + if requested_mode is not None: + if requested_mode not in {"ready", "collection"}: + raise ValueError( + f"unknown PRISM job-bundle mode: {requested_mode}" + ) + return requested_mode + return "ready" if self.pool_readiness_latched() else "collection" + + def job_bundle_key( + self, + artifacts: CachedTemplateArtifacts, + *, + mode: str, + payout_state_generation: int, + payout_artifact_generation: int = 0, + worker: WorkerIdentityPort | None, + ) -> tuple[object, ...]: + if mode == "ready": + return ( + artifacts.fingerprint, + artifacts.previousblockhash, + "ready", + payout_state_generation, + payout_artifact_generation, + ) + if mode != "collection": + raise ValueError(f"unknown PRISM job-bundle mode: {mode}") + if worker is None: + raise CollectionIdentityUnavailable( + "collection-mode worker identity is temporarily unavailable" + ) + return ( + artifacts.fingerprint, + artifacts.previousblockhash, + "collection", + artifacts.generation, + payout_state_generation, + payout_artifact_generation, + *self.collection_identity(worker), + ) + + def lookup_bundle(self, key: tuple[object, ...]) -> CachedJobBundle | None: + now = self._monotonic() + with self._cache_lock: + if self._config.cache_seconds <= 0: + self._bundle_cache.clear() + return None + expired = [ + cache_key + for cache_key, entry in self._bundle_cache.items() + if now - entry.built_monotonic > self._config.cache_seconds + ] + for cache_key in expired: + self._bundle_cache.pop(cache_key, None) + return self._bundle_cache.get(key) + + def bundle_payout_state_current(self, bundle: CachedJobBundle) -> bool: + payout = self._ports.payout_state().snapshot() + artifact = payout.published.artifact + return bool( + bundle.payout_state_generation == payout.generation + and bundle.build_key is not None + and artifact is not None + and bundle.build_key.payout_artifact_sha256 + == artifact.prior_balances_sha256 + ) + + def bundle_entry_usable( + self, + cached: CachedJobBundle | None, + artifacts: CachedTemplateArtifacts, + ) -> bool: + if cached is None or not self._ports.artifacts_buildable(artifacts): + return False + if self._ports.payout_state().snapshot().publication_blocked: + return False + if not self.bundle_payout_state_current(cached): + return False + if not cached.collection_only: + return True + if ( + cached.template is not artifacts.template + or cached.template_generation != artifacts.generation + ): + return False + try: + _, ready_miner_count = self._ports.accepted_share_stats() + except Exception: + return False + return ready_miner_count < self._config.min_ready_miners + + def bind_cached_bundle( + self, + cached: CachedJobBundle, + artifacts: CachedTemplateArtifacts, + ) -> CachedJobBundle: + if ( + cached.template is artifacts.template + and cached.template_generation == artifacts.generation + ): + return cached + manifest = cached.coinbase_manifest + base_job = direct_stratum.make_job_from_builder_manifest( + job_id="prism-template-base", + template=artifacts.template, + manifest=manifest, + extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, + extranonce2_size=self._config.extranonce2_size, + desired_share_difficulty=self._config.share_difficulty, + clean_jobs=True, + transaction_hexes=artifacts.transaction_hexes, + ) + return dataclass_replace( + cached, + template=artifacts.template, + base_job=base_job, + template_generation=artifacts.generation, + build_key=( + dataclass_replace( + cached.build_key, + best_tip_hash=artifacts.previousblockhash, + previous_block_hash=artifacts.previousblockhash, + template_generation=artifacts.generation, + block_height=int(artifacts.template["height"]), + coinbase_value_sats=int(artifacts.template["coinbasevalue"]), + ) + if cached.build_key is not None + else None + ), + ) + + def new_build_request( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None, + *, + mode: str, + payout_state_generation: int, + cache_key: tuple[object, ...], + payout_ledger_artifact: PayoutLedgerArtifact | None = None, + idle_retarget: bool = False, + publication_critical: bool = False, + request_source: str = "routine", + priority_requested_monotonic: float | None = None, + preparation_cancellation: JobBuildCancellation | None = None, + ) -> JobBuildRequest: + cancellation = ( + JobBuildCancellation( + timeout_seconds=max(0.001, self._config.build_timeout_seconds), + monotonic=self._monotonic, + ) + if preparation_cancellation is None + else preparation_cancellation + ) + cancellation.raise_if_cancelled("immutable snapshot") + self._ports.await_parent_preview( + artifacts.previousblockhash, + int(artifacts.template["height"]) - 1, + ) + payout_artifact = self._ports.payout_state().current_artifact(cancellation) + if payout_artifact.generation != payout_state_generation: + raise JobBuildSuperseded( + "payout artifact generation changed before build request" + ) + payout_started = self._monotonic() + payout_policy_json = canonical_json_text(self._ports.payout_policy()) + self.record_phase("payout", self._monotonic() - payout_started) + cancellation.raise_if_cancelled("payout policy") + ctv_started = self._monotonic() + ctv_settlement = self._ports.ctv_settlement( + int(artifacts.template["height"]), + artifacts.previousblockhash, + ) + ctv_settlement_json = ( + canonical_json_text(ctv_settlement) + if ctv_settlement is not None + else None + ) + self.record_phase("ctv", self._monotonic() - ctv_started) + cancellation.raise_if_cancelled("CTV configuration") + suffix_hex = self._ports.coinbase_suffix( + PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, + "00" * self._config.extranonce2_size, + ) + collection_identity = ( + self.collection_identity(worker) + if mode == "collection" and worker is not None + else None + ) + decimal_context = getcontext().copy() + numeric_context_sha256 = canonical_json_sha256( + { + "precision": decimal_context.prec, + "rounding": decimal_context.rounding, + "minimum_exponent": decimal_context.Emin, + "maximum_exponent": decimal_context.Emax, + "capitals": decimal_context.capitals, + "clamp": decimal_context.clamp, + } + ) + with self._cache_lock: + issued_at_ms = self._issued_at_ms.get(artifacts.generation) + if issued_at_ms is None: + issued_at_ms = self._ports.snapshot_anchor_ms( + self._ports.wall_time_ms() + ) + self._issued_at_ms[artifacts.generation] = issued_at_ms + while len(self._issued_at_ms) > 128: + self._issued_at_ms.popitem(last=False) + build_key = JobBuildKey( + best_tip_hash=artifacts.previousblockhash, + previous_block_hash=artifacts.previousblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + payout_state_generation=payout_state_generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + mode=mode, + collection_identity=collection_identity, + block_height=int(artifacts.template["height"]), + coinbase_value_sats=int(artifacts.template["coinbasevalue"]), + network_difficulty=int(artifacts.network_difficulty), + issued_at_ms=issued_at_ms, + payout_policy_sha256=hashlib.sha256( + payout_policy_json.encode() + ).hexdigest(), + ctv_settlement_sha256=( + hashlib.sha256(ctv_settlement_json.encode()).hexdigest() + if ctv_settlement_json is not None + else None + ), + witness_merkle_sha256=canonical_json_sha256( + artifacts.witness_merkle_leaves_hex + ), + transaction_set_sha256=canonical_json_sha256( + artifacts.transaction_hexes + ), + coinbase_suffix_hex=suffix_hex, + signing_key_sha256=hashlib.sha256( + self._ports.signing_seed_hex().encode() + ).hexdigest(), + ledger_signing_key_sha256=hashlib.sha256( + self._ports.ledger_signing_seed_hex().encode() + ).hexdigest(), + numeric_context_sha256=numeric_context_sha256, + ) + immutable_identity: tuple[object, ...] = ( + cache_key, + artifacts.generation, + issued_at_ms, + payout_artifact.prior_balances_sha256, + build_key.payout_policy_sha256, + build_key.ctv_settlement_sha256, + build_key.witness_merkle_sha256, + build_key.transaction_set_sha256, + build_key.coinbase_suffix_hex, + build_key.signing_key_sha256, + build_key.ledger_signing_key_sha256, + build_key.numeric_context_sha256, + ) + return JobBuildRequest( + key=build_key, + cache_key=cache_key, + equivalence_key=immutable_identity, + artifacts=artifacts, + template_json=canonical_json_text(artifacts.template), + transaction_hexes=artifacts.transaction_hexes, + witness_merkle_leaves_hex=artifacts.witness_merkle_leaves_hex, + worker=worker, + mode=mode, + payout_artifact=payout_artifact, + payout_ledger_artifact=payout_ledger_artifact, + payout_policy_json=payout_policy_json, + ctv_settlement_json=ctv_settlement_json, + decimal_context=decimal_context, + cancellation=cancellation, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + requested_monotonic=( + cancellation.started_monotonic + if priority_requested_monotonic is None + else priority_requested_monotonic + ), + ) + + def _executor_locked(self) -> ThreadPoolExecutor: + if self._executor_shutdown: + raise RuntimeError("job build executor is shut down") + executor = self._executor + if executor is None: + executor = ThreadPoolExecutor( + max_workers=PRISM_JOB_BUILD_EXECUTOR_WORKERS, + thread_name_prefix="prism-job-build", + ) + self._executor = executor + return executor + + def _start_locked(self, request: JobBuildRequest) -> JobBuildFlight: + executor = self._executor_locked() + flight = JobBuildFlight(request=request) + self._scheduler_counts["starts"] += 1 + self._shared_build_counts["started"] += 1 + if request.superseded_monotonic is not None: + elapsed = max( + 0.0, + self._monotonic() - request.superseded_monotonic, + ) + self._replacement_start_seconds["sum"] += elapsed + self._replacement_start_seconds["count"] += 1 + flight.future = executor.submit(self._execute_request, request) + self.record_priority_admission_locked(request, "started") + return flight + + def _arm_locked(self, flight: JobBuildFlight) -> None: + future = flight.future + assert future is not None + future.add_done_callback( + lambda completed, build_flight=flight: self._build_done( + build_flight, + completed, + ) + ) + + def _execute_request(self, request: JobBuildRequest) -> CachedJobBundle: + request.cancellation.raise_if_cancelled("start") + control = JobBundleBuildControl( + key=request.equivalence_key, + previousblockhash=request.key.previous_block_hash, + payout_state_generation=request.key.payout_state_generation, + payout_artifact_generation=( + request.payout_ledger_artifact.generation + if request.payout_ledger_artifact is not None + else 0 + ), + ) + with self._cache_lock: + self._active_bundle_builds[control.key] = control + previous_control = getattr( + self._phase_local, + "bundle_build_control", + None, + ) + self._phase_local.bundle_build_control = control + try: + with localcontext(request.decimal_context): + return self.build_shared_job_bundle( + request.artifacts, + request.worker, + mode=request.mode, + payout_state_generation=request.key.payout_state_generation, + payout_artifact=request.payout_ledger_artifact, + key=request.cache_key, + build_request=request, + ) + finally: + self._phase_local.bundle_build_control = previous_control + with self._cache_lock: + if self._active_bundle_builds.get(control.key) is control: + self._active_bundle_builds.pop(control.key, None) + control.process = None + + @staticmethod + def collection_builds_independent( + first: JobBuildRequest, + second: JobBuildRequest, + ) -> bool: + return ( + first.mode == "collection" + and second.mode == "collection" + and first.key.collection_identity != second.key.collection_identity + and dataclass_replace(first.key, collection_identity=None) + == dataclass_replace(second.key, collection_identity=None) + ) + + @staticmethod + def requests_can_share( + first: JobBuildRequest, + second: JobBuildRequest, + ) -> bool: + return first.equivalence_key == second.equivalence_key or ( + first.mode == "ready" + and second.mode == "ready" + and first.cache_key == second.cache_key + ) + + @staticmethod + def ready_precedes_collection( + first: JobBuildRequest, + second: JobBuildRequest, + ) -> bool: + return ( + first.mode == "ready" + and second.mode == "collection" + and not first.cancellation.is_set() + ) + + @staticmethod + def defer_collection( + *blockers: Future[CachedJobBundle], + ) -> Future[CachedJobBundle]: + deferred: Future[CachedJobBundle] = Future() + wake_lock = threading.Lock() + + def wake_for_retry(_completed: Future[CachedJobBundle]) -> None: + with wake_lock: + if not deferred.done(): + deferred.set_exception( + JobBuildSuperseded( + "collection build capacity became available; retrying" + ) + ) + + for blocker in blockers: + blocker.add_done_callback(wake_for_retry) + return deferred + + @staticmethod + def is_publication_critical(request: object) -> bool: + return bool(getattr(request, "publication_critical", False)) + + def record_priority_admission_locked( + self, + request: JobBuildRequest, + result: str, + ) -> None: + if not self.is_publication_critical(request): + return + self._priority_counts[result] += 1 + if result not in {"started", "coalesced"}: + return + if request.priority_admission_recorded: + return + request.priority_admission_recorded = True + elapsed = max(0.0, self._monotonic() - request.requested_monotonic) + self._priority_admission_seconds["sum"] += elapsed + self._priority_admission_seconds["count"] += 1 + + def record_initial_prepared_work_locked(self, result: str) -> None: + self._initial_prepared_work_counts[result] += 1 + + def new_cancellation(self) -> JobBuildCancellation: + return JobBuildCancellation( + timeout_seconds=max(0.001, self._config.build_timeout_seconds), + monotonic=self._monotonic, + ) + + def begin_priority_preparation( + self, + requested_monotonic: float | None = None, + ) -> tuple[int, float]: + """Reserve publication priority before immutable request construction.""" + started = ( + self._monotonic() + if requested_monotonic is None + else requested_monotonic + ) + with self._scheduler_lock: + self._priority_preparation_sequence += 1 + token = self._priority_preparation_sequence + self._priority_preparations[token] = started + for routine_ref in tuple(self._routine_preparations.values()): + routine = routine_ref() + if routine is not None: + routine.cancel("publication priority") + self._routine_preparations.clear() + self._priority_changed.set() + return token, started + + def finish_priority_preparation(self, token: int) -> None: + with self._scheduler_lock: + self._priority_preparations.pop(token, None) + self._priority_changed.set() + + def begin_routine_preparation( + self, + *, + request_source: str, + cancelled: Callable[[], bool] | None, + ) -> tuple[int, JobBuildCancellation]: + """Atomically admit cancellable routine request construction.""" + deferred_recorded = False + while True: + self._priority_changed.clear() + with self._scheduler_lock: + if not self.publication_priority_scheduled_locked(): + self._routine_preparation_sequence += 1 + token = self._routine_preparation_sequence + preparation = self.new_cancellation() + service_ref = weakref.ref(self) + + def remove_dead_preparation( + dead_ref: weakref.ReferenceType[JobBuildCancellation], + *, + preparation_token: int = token, + ) -> None: + service = service_ref() + if service is None: + return + with service._scheduler_lock: + if ( + service._routine_preparations.get( + preparation_token + ) + is dead_ref + ): + service._routine_preparations.pop( + preparation_token, + None, + ) + + self._routine_preparations[token] = weakref.ref( + preparation, + remove_dead_preparation, + ) + return token, preparation + if not deferred_recorded: + self._priority_counts["routine_deferred"] += 1 + if request_source == "initial": + self.record_initial_prepared_work_locked("deferred") + deferred_recorded = True + stopped = self._executor_shutdown + if cancelled is not None and cancelled(): + raise JobBuildWaiterCancelled( + "job bundle request was cancelled behind publication priority" + ) + if stopped: + raise JobBuildWaiterCancelled( + "job builder stopped behind publication priority" + ) + self._priority_changed.wait(0.05) + + def finish_routine_preparation(self, token: int) -> None: + with self._scheduler_lock: + self._routine_preparations.pop(token, None) + + def publication_priority_scheduled_locked(self) -> bool: + if self._priority_preparations: + return True + pending = self._pending + if ( + pending is not None + and not pending.cancellation.is_set() + and self.is_publication_critical(pending) + ): + return True + return any( + flight is not None + and not flight.request.cancellation.is_set() + and self.is_publication_critical(flight.request) + for flight in (self._active, self._retiring) + ) + + def can_inherit_publication_priority( + self, + existing: JobBuildRequest, + incoming: JobBuildRequest, + ) -> bool: + if ( + not self.is_publication_critical(incoming) + or self.is_publication_critical(existing) + ): + return True + cancellation = existing.cancellation + total_budget = max( + 0.001, + cancellation.deadline_monotonic - cancellation.started_monotonic, + ) + remaining_budget = cancellation.deadline_monotonic - self._monotonic() + progress_age = self._monotonic() - cancellation.last_checkpoint_monotonic + return bool( + remaining_budget >= total_budget / 2.0 + and progress_age <= max(0.001, self._config.cancel_grace_seconds) + ) + + def _cancel_flight_locked( + self, + flight: JobBuildFlight, + reason: str, + *, + now: float | None = None, + ) -> bool: + if not flight.request.cancellation.cancel(reason): + return False + flight.request.superseded_monotonic = ( + self._monotonic() if now is None else now + ) + self._scheduler_counts["supersessions"] += 1 + if reason == "publication priority": + self._priority_counts["routine_preempted"] += 1 + self._priority_changed.set() + return True + + def _promote_pending_locked(self) -> None: + pending = self._pending + if pending is None: + return + active = self._active + retiring = self._retiring + if ( + not self.is_publication_critical(pending) + and any( + flight is not None + and not flight.request.cancellation.is_set() + and self.is_publication_critical(flight.request) + for flight in (active, retiring) + ) + ): + return + if active is not None: + if retiring is not None: + return + if self.ready_precedes_collection( + active.request, + pending, + ) and not self.is_publication_critical(pending): + return + if not self.collection_builds_independent(active.request, pending): + reason = ( + "publication priority" + if self.is_publication_critical(pending) + and not self.is_publication_critical(active.request) + else "superseded" + ) + self._cancel_flight_locked(active, reason) + self._retiring = active + self._active = None + elif retiring is not None: + if self.ready_precedes_collection( + retiring.request, + pending, + ) and not self.is_publication_critical(pending): + return + if not self.collection_builds_independent(retiring.request, pending): + reason = ( + "publication priority" + if self.is_publication_critical(pending) + and not self.is_publication_critical(retiring.request) + else "superseded" + ) + self._cancel_flight_locked(retiring, reason) + self._pending = None + flight = self._start_locked(pending) + self._active = flight + self._arm_locked(flight) + + def _build_done( + self, + flight: JobBuildFlight, + future: Future[CachedJobBundle], + ) -> None: + request = flight.request + result: CachedJobBundle | None = None + error: BaseException | None = None + try: + result = future.result() + except BaseException as exc: + error = exc + with self._scheduler_lock: + if error is None and request.cancellation.is_set(): + if request.cancellation.reason == "timeout": + error = JobBuildCancelled( + "job build completed after its timeout" + ) + else: + error = JobBuildSuperseded( + "obsolete job build completed after cancellation" + ) + self._scheduler_counts["completions"] += 1 + self._preparation_count += 1 + self._preparation_seconds_sum += max( + 0.0, + self._monotonic() - request.cancellation.started_monotonic, + ) + if request.cancellation.cancelled_monotonic is not None: + elapsed = max( + 0.0, + self._monotonic() + - request.cancellation.cancelled_monotonic, + ) + self._cancellation_seconds["sum"] += elapsed + self._cancellation_seconds["count"] += 1 + coordination_cancelled = isinstance(error, JobBuildSuperseded) or ( + request.cancellation.is_set() + and request.cancellation.reason != "timeout" + ) + if error is not None and coordination_cancelled: + self._scheduler_counts["obsolete_results"] += 1 + self._shared_build_counts["superseded"] += 1 + self._ports.note_tip_refresh_superseded() + elif error is not None: + self._shared_build_counts["failed"] += 1 + else: + self._shared_build_counts["completed"] += 1 + if self._active is flight: + self._active = None + if self._retiring is flight: + self._retiring = None + self._promote_pending_locked() + if not request.promise.done(): + if error is not None: + request.promise.set_exception(error) + else: + assert result is not None + request.promise.set_result(result) + self._priority_changed.set() + + def request_build(self, request: JobBuildRequest) -> Future[CachedJobBundle]: + with self._scheduler_lock: + if request.idle_retarget and self._ports.idle_tip_diverged(): + request.cancellation.cancel( + "idle retarget deferred during unpublished tip refresh" + ) + if not request.promise.done(): + request.promise.set_exception( + JobBuildSuperseded( + "idle retarget deferred during unpublished tip refresh" + ) + ) + return request.promise + self._scheduler_counts["requests"] += 1 + active = self._active + retiring = self._retiring + pending = self._pending + publication_critical = self.is_publication_critical(request) + if ( + active is not None + and not active.request.cancellation.is_set() + and self.requests_can_share(active.request, request) + and self.can_inherit_publication_priority(active.request, request) + ): + if publication_critical: + active.request.publication_critical = True + active.request.request_source = request.request_source + active.request.requested_monotonic = request.requested_monotonic + active.request.priority_admission_recorded = True + self.record_priority_admission_locked(request, "coalesced") + if request.request_source == "initial": + self.record_initial_prepared_work_locked("singleflight") + return active.request.promise + if ( + retiring is not None + and not retiring.request.cancellation.is_set() + and self.requests_can_share(retiring.request, request) + and self.can_inherit_publication_priority(retiring.request, request) + ): + if publication_critical: + retiring.request.publication_critical = True + retiring.request.request_source = request.request_source + retiring.request.requested_monotonic = request.requested_monotonic + retiring.request.priority_admission_recorded = True + self.record_priority_admission_locked(request, "coalesced") + if request.request_source == "initial": + self.record_initial_prepared_work_locked("singleflight") + return retiring.request.promise + if ( + pending is not None + and not pending.cancellation.is_set() + and self.requests_can_share(pending, request) + and self.can_inherit_publication_priority(pending, request) + ): + if publication_critical: + pending.publication_critical = True + pending.request_source = request.request_source + pending.requested_monotonic = request.requested_monotonic + self.record_priority_admission_locked(request, "queued") + now = self._monotonic() + for occupied in (active, retiring): + if ( + occupied is not None + and not self.requests_can_share( + occupied.request, + pending, + ) + and not self.is_publication_critical(occupied.request) + ): + self._cancel_flight_locked( + occupied, + "publication priority", + now=now, + ) + self._promote_pending_locked() + if request.request_source == "initial": + self.record_initial_prepared_work_locked("singleflight") + return pending.promise + + if not publication_critical: + priority_blockers = tuple( + blocker + for blocker in ( + active.request if active is not None else None, + retiring.request if retiring is not None else None, + pending, + ) + if blocker is not None + and not blocker.cancellation.is_set() + and self.is_publication_critical(blocker) + ) + if priority_blockers: + self._priority_counts["routine_deferred"] += 1 + if request.request_source == "initial": + self.record_initial_prepared_work_locked("deferred") + return self.defer_collection( + *(blocker.promise for blocker in priority_blockers) + ) + + if request.mode == "collection": + possible_blockers = ( + pending, + active.request if active is not None else None, + retiring.request if retiring is not None else None, + ) + for blocker in possible_blockers: + if ( + blocker is not None + and not blocker.cancellation.is_set() + and self.ready_precedes_collection(blocker, request) + and not ( + publication_critical + and not self.is_publication_critical(blocker) + ) + ): + return self.defer_collection(blocker.promise) + if active is None: + if pending is not None: + if self.collection_builds_independent(pending, request): + self._pending = None + flight = self._start_locked(pending) + if retiring is None: + replacement = self._start_locked(request) + self._retiring = flight + self._active = replacement + self._arm_locked(flight) + self._arm_locked(replacement) + return request.promise + self._active = flight + self._arm_locked(flight) + return self.defer_collection( + flight.request.promise, + retiring.request.promise, + ) + pending.cancellation.cancel("superseded while pending") + if not pending.promise.done(): + pending.promise.set_exception( + JobBuildSuperseded( + "pending job build was superseded" + ) + ) + self._pending = None + self._scheduler_counts["supersessions"] += 1 + if ( + retiring is not None + and not self.collection_builds_independent( + retiring.request, + request, + ) + ): + now = self._monotonic() + reason = ( + "publication priority" + if publication_critical + and not self.is_publication_critical(retiring.request) + else "superseded" + ) + if self._cancel_flight_locked( + retiring, + reason, + now=now, + ): + request.superseded_monotonic = now + flight = self._start_locked(request) + self._active = flight + self._arm_locked(flight) + return request.promise + if self.collection_builds_independent(active.request, request): + if self._retiring is None: + self._retiring = active + flight = self._start_locked(request) + self._active = flight + self._arm_locked(flight) + return request.promise + if pending is None: + self._pending = request + if publication_critical: + self.record_priority_admission_locked(request, "queued") + now = self._monotonic() + for occupied in (active, self._retiring): + if ( + occupied is not None + and not self.is_publication_critical( + occupied.request + ) + ): + self._cancel_flight_locked( + occupied, + "publication priority", + now=now, + ) + return request.promise + assert retiring is not None + if publication_critical: + pending.cancellation.cancel("superseded while pending") + if not pending.promise.done(): + pending.promise.set_exception( + JobBuildSuperseded( + "pending job build was superseded by publication priority" + ) + ) + self._scheduler_counts["supersessions"] += 1 + self._pending = request + self.record_priority_admission_locked(request, "queued") + now = self._monotonic() + for occupied in (active, retiring): + if not self.is_publication_critical(occupied.request): + self._cancel_flight_locked( + occupied, + "publication priority", + now=now, + ) + return request.promise + return self.defer_collection( + active.request.promise, + retiring.request.promise, + ) + now = self._monotonic() + for obsolete in (active, retiring): + if obsolete is not None: + reason = ( + "publication priority" + if publication_critical + and not self.is_publication_critical(obsolete.request) + else "superseded" + ) + self._cancel_flight_locked( + obsolete, + reason, + now=now, + ) + request.superseded_monotonic = now + if retiring is None: + self._retiring = active + flight = self._start_locked(request) + self._active = flight + self._arm_locked(flight) + return request.promise + previous_pending = self._pending + if previous_pending is not None: + previous_pending.cancellation.cancel("superseded while pending") + if not previous_pending.promise.done(): + previous_pending.promise.set_exception( + JobBuildSuperseded( + "pending job build was superseded" + ) + ) + self._scheduler_counts["supersessions"] += 1 + self._pending = request + if publication_critical: + self.record_priority_admission_locked(request, "queued") + return request.promise + + def cancel_obsolete_builds( + self, + reason: str, + *, + keep_published_snapshot: bool = False, + ) -> None: + def keep(request: JobBuildRequest) -> bool: + return bool( + keep_published_snapshot + and self._published_snapshot_matches( + request.artifacts, + exact_generation=( + getattr(request, "mode", "ready") == "collection" + ), + ) + ) + + with self._scheduler_lock: + for flight in (self._active, self._retiring): + if ( + flight is not None + and not keep(flight.request) + and flight.request.cancellation.cancel(reason) + ): + flight.request.superseded_monotonic = self._monotonic() + self._scheduler_counts["supersessions"] += 1 + pending = self._pending + if pending is not None and not keep(pending): + pending.cancellation.cancel(reason) + if not pending.promise.done(): + pending.promise.set_exception( + JobBuildSuperseded(f"pending job build {reason}") + ) + self._pending = None + self._scheduler_counts["supersessions"] += 1 + + def cancel_obsolete_bundle_processes( + self, + *, + current_tip: str | None = None, + payout_state_generation: int | None = None, + ) -> None: + processes: list[subprocess.Popen[str]] = [] + with self._cache_lock: + for control in self._active_bundle_builds.values(): + obsolete = ( + current_tip is not None + and control.previousblockhash != current_tip + ) or ( + payout_state_generation is not None + and control.payout_state_generation + != int(payout_state_generation) + ) + if not obsolete or control.cancel_event.is_set(): + continue + control.cancel_event.set() + if control.process is not None: + processes.append(control.process) + for process in processes: + if process.poll() is not None: + continue + try: + process.terminate() + except ProcessLookupError: + pass + + def register_process( + self, + control: JobBundleBuildControl, + process: subprocess.Popen[str], + ) -> None: + terminate = False + with self._cache_lock: + if ( + self._active_bundle_builds.get(control.key) is not control + or control.cancel_event.is_set() + ): + terminate = True + else: + control.process = process + if terminate and process.poll() is None: + try: + process.terminate() + except ProcessLookupError: + pass + + def active_build_control(self) -> JobBundleBuildControl | None: + value = getattr(self._phase_local, "bundle_build_control", None) + return value if isinstance(value, JobBundleBuildControl) else None + + def shutdown(self) -> None: + with self._scheduler_lock: + for flight in (self._active, self._retiring): + if flight is not None: + flight.request.cancellation.cancel("shutdown") + pending = self._pending + if pending is not None: + pending.cancellation.cancel("shutdown") + if not pending.promise.done(): + pending.promise.set_exception( + JobBuildSuperseded( + "pending job build cancelled by shutdown" + ) + ) + self._pending = None + executor = self._executor + self._executor = None + self._executor_shutdown = True + if executor is not None: + executor.shutdown(wait=True, cancel_futures=True) + + def on_template_artifacts_changed( + self, + artifacts: CachedTemplateArtifacts, + fingerprint_changed: bool, + ) -> None: + published_artifacts = self._ports.published_artifacts() + + def keep(entry: CachedJobBundle) -> bool: + current = entry.template_fingerprint == artifacts.fingerprint and ( + not entry.collection_only + or entry.template_generation == artifacts.generation + ) + published = ( + published_artifacts is not None + and entry.template_fingerprint == published_artifacts.fingerprint + and ( + not entry.collection_only + or entry.template_generation == published_artifacts.generation + ) + ) + return current or published + + with self._cache_lock: + self._bundle_cache = OrderedDict( + (key, entry) + for key, entry in self._bundle_cache.items() + if keep(entry) + ) + if fingerprint_changed: + self.cancel_obsolete_builds( + "template fingerprint superseded", + keep_published_snapshot=True, + ) + + def on_template_artifacts_cleared( + self, + _artifacts: CachedTemplateArtifacts, + ) -> None: + published_artifacts = self._ports.published_artifacts() + keep_published = bool( + published_artifacts is not None + and self._ports.published_snapshot_artifacts(published_artifacts) + ) + + def keep(entry: CachedJobBundle) -> bool: + return bool( + keep_published + and published_artifacts is not None + and entry.template_fingerprint == published_artifacts.fingerprint + and ( + not entry.collection_only + or entry.template_generation == published_artifacts.generation + ) + ) + + with self._cache_lock: + self._bundle_cache = OrderedDict( + (key, entry) + for key, entry in self._bundle_cache.items() + if keep(entry) + ) + self.cancel_obsolete_builds( + "template artifacts cleared", + keep_published_snapshot=keep_published, + ) + + def clear_cache(self) -> None: + with self._cache_lock: + self._bundle_cache.clear() + + def cache_bundle_if_current( + self, + built: CachedJobBundle, + artifacts: CachedTemplateArtifacts, + ) -> bool: + payout_state = self._ports.payout_state() + if not self._bundle_cache_admissible(built, artifacts, payout_state): + return False + # P1 publication never takes the template fence. Keep the one-way + # payout -> template -> cache order so neither invalidation path can + # invert admission. + with payout_state.cache_publication_admission(): + with self.template_repository.publication_admission(): + if not self._bundle_cache_admissible( + built, + artifacts, + payout_state, + ): + return False + with self._cache_lock: + self._bundle_cache[built.key] = built + self._bundle_cache.move_to_end(built.key) + while len(self._bundle_cache) > MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES: + self._bundle_cache.popitem(last=False) + return True + + def _published_snapshot_matches( + self, + artifacts: CachedTemplateArtifacts, + *, + exact_generation: bool, + ) -> bool: + if not self._ports.published_snapshot_artifacts(artifacts): + return False + if not exact_generation: + return True + published = self._ports.published_artifacts() + return bool( + published is not None + and published.generation == artifacts.generation + and published.fingerprint == artifacts.fingerprint + and published.previousblockhash == artifacts.previousblockhash + ) + + def _bundle_cache_admissible( + self, + built: CachedJobBundle, + artifacts: CachedTemplateArtifacts, + payout_state: PayoutStatePort, + ) -> bool: + if not self._ports.artifacts_buildable(artifacts): + return False + snapshot_pinned = self._published_snapshot_matches( + artifacts, + exact_generation=built.collection_only, + ) + payout = payout_state.snapshot() + published_artifact = payout.published.artifact + if ( + payout.publication_blocked + or built.payout_state_generation != payout.generation + or built.build_key is None + or published_artifact is None + or built.build_key.payout_artifact_sha256 + != published_artifact.prior_balances_sha256 + ): + return False + current = self.template_repository.current_artifacts() + globally_current = ( + current is not None + and current.fingerprint == artifacts.fingerprint + and current.previousblockhash == artifacts.previousblockhash + and ( + not built.collection_only + or current.generation == artifacts.generation + ) + ) + return globally_current or snapshot_pinned + + def shared_job_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None = None, + *, + mode: str | None = None, + cancelled: Callable[[], bool] | None = None, + retry_superseded: bool = True, + idle_retarget: bool = False, + publication_critical: bool = False, + request_source: str = "routine", + priority_requested_monotonic: float | None = None, + ) -> CachedJobBundle: + with self._ports.start_bundle_build(): + priority_token: int | None = None + if publication_critical: + ( + priority_token, + priority_requested_monotonic, + ) = self.begin_priority_preparation( + priority_requested_monotonic + ) + try: + return self._shared_job_bundle( + artifacts, + worker, + mode=mode, + cancelled=cancelled, + retry_superseded=retry_superseded, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + priority_requested_monotonic=( + priority_requested_monotonic + ), + ) + finally: + if priority_token is not None: + self.finish_priority_preparation(priority_token) + + def _shared_job_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None = None, + *, + mode: str | None = None, + cancelled: Callable[[], bool] | None = None, + retry_superseded: bool = True, + idle_retarget: bool = False, + publication_critical: bool = False, + request_source: str = "routine", + priority_requested_monotonic: float | None = None, + ) -> CachedJobBundle: + while True: + routine_token: int | None = None + preparation_cancellation: JobBuildCancellation | None = None + payout_state_generation: int | None = None + request: JobBuildRequest | None = None + try: + if not publication_critical: + ( + routine_token, + preparation_cancellation, + ) = self.begin_routine_preparation( + request_source=request_source, + cancelled=cancelled, + ) + resolved_mode = self.job_bundle_mode(mode) + if preparation_cancellation is not None: + preparation_cancellation.raise_if_cancelled( + "request preparation admission" + ) + if resolved_mode == "collection" and worker is None: + raise CollectionIdentityUnavailable( + "collection-mode worker identity is temporarily unavailable" + ) + payout = self._ports.payout_state().snapshot() + payout_state_generation = payout.generation + payout_artifact = ( + self._ports.payout_state().usable_ledger_artifact( + payout_state_generation, + artifacts.network_difficulty, + ) + if resolved_mode == "ready" + else None + ) + if preparation_cancellation is not None: + preparation_cancellation.raise_if_cancelled( + "payout artifact lookup" + ) + payout_artifact_generation = ( + payout_artifact.generation + if payout_artifact is not None + else 0 + ) + key = self.job_bundle_key( + artifacts, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + payout_artifact_generation=payout_artifact_generation, + worker=worker, + ) + cached = self.lookup_bundle(key) + if self.bundle_entry_usable(cached, artifacts): + if preparation_cancellation is not None: + preparation_cancellation.raise_if_cancelled( + "bundle cache lookup" + ) + if routine_token is not None: + self.finish_routine_preparation(routine_token) + routine_token = None + self.record_cache_event("bundle", hit=True) + if request_source == "initial": + with self._scheduler_lock: + self._initial_prepared_work_counts["cache_hit"] += 1 + assert cached is not None + return self.bind_cached_bundle(cached, artifacts) + if self.job_bundle_mode(mode) != resolved_mode: + continue + self.record_cache_event("bundle", hit=False) + request = self.new_build_request( + artifacts, + worker, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + cache_key=key, + payout_ledger_artifact=payout_artifact, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + priority_requested_monotonic=( + priority_requested_monotonic + ), + preparation_cancellation=preparation_cancellation, + ) + with self._admission_lock: + with self._scheduler_lock: + if routine_token is not None: + self.finish_routine_preparation(routine_token) + routine_token = None + request.cancellation.raise_if_cancelled( + "scheduler admission" + ) + if self.job_bundle_mode(mode) != resolved_mode: + request.cancellation.cancel("worker mode superseded") + continue + if cancelled is not None and cancelled(): + raise JobBuildWaiterCancelled( + "job bundle request was cancelled before preparation" + ) + promise = self.request_build(request) + wait_deadline = self._monotonic() + max( + 0.001, + self._config.build_timeout_seconds + + self._config.cancel_grace_seconds + + 1.0, + ) + while True: + if cancelled is not None and cancelled(): + raise JobBuildWaiterCancelled( + "job bundle waiter was cancelled during preparation" + ) + try: + built = promise.result( + timeout=min( + 0.1, + max(0.001, wait_deadline - self._monotonic()), + ) + ) + break + except TimeoutError: + if self._monotonic() >= wait_deadline: + raise + except TimeoutError as exc: + assert request is not None + request.cancellation.cancel("timeout") + self._ports.schedule_refresh_retry() + raise JobBuildCancelled( + "job build timed out; immediate retry scheduled" + ) from exc + except JobBuildCancelled: + self._ports.schedule_refresh_retry() + if not retry_superseded: + raise + if payout_state_generation is None: + raise + if not self._ports.artifacts_buildable(artifacts): + raise + current = self.template_repository.current_artifacts() + payout_current = ( + payout_state_generation + == self._ports.payout_state().snapshot().generation + ) + if current is artifacts and payout_current: + continue + if current is artifacts: + continue + raise + finally: + if routine_token is not None: + self.finish_routine_preparation(routine_token) + built = self.bind_cached_bundle(built, artifacts) + if not self.cache_bundle_if_current(built, artifacts): + with self._scheduler_lock: + self._scheduler_counts["obsolete_results"] += 1 + if not self._ports.artifacts_buildable(artifacts): + self._ports.note_tip_refresh_superseded() + raise JobBuildSuperseded( + "observed tip changed before cache publication" + ) + payout_current = ( + built.payout_state_generation + == self._ports.payout_state().snapshot().generation + ) + published_artifacts = self._ports.published_artifacts() + if ( + payout_current + and built.template is artifacts.template + and built.template_generation == artifacts.generation + and ( + not retry_superseded + or published_artifacts is artifacts + ) + ): + return built + if ( + retry_superseded + and self.template_repository.current_artifacts() is artifacts + ): + continue + raise JobBuildSuperseded( + "job build key changed before cache publication" + ) + return built + + def build_shared_job_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentityPort | None = None, + *, + mode: str | None = None, + payout_state_generation: int | None = None, + payout_artifact: PayoutLedgerArtifact | None = None, + key: tuple[object, ...] | None = None, + build_request: JobBuildRequest | None = None, + ) -> CachedJobBundle: + resolved_mode = self.job_bundle_mode(mode) + if resolved_mode == "collection" and worker is None: + raise CollectionIdentityUnavailable( + "collection-mode worker identity is temporarily unavailable" + ) + payout_snapshot = self._ports.payout_state().snapshot() + if payout_state_generation is None: + payout_state_generation = payout_snapshot.generation + if payout_snapshot.publication_blocked: + raise PayoutStatePublicationBlocked( + "payout state invalidation is pending publication" + ) + if key is None: + key = self.job_bundle_key( + artifacts, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + payout_artifact_generation=( + payout_artifact.generation + if payout_artifact is not None + else 0 + ), + worker=worker, + ) + if build_request is None: + build_request = self.new_build_request( + artifacts, + worker, + mode=resolved_mode, + payout_state_generation=payout_state_generation, + cache_key=key, + payout_ledger_artifact=payout_artifact, + ) + else: + payout_artifact = build_request.payout_ledger_artifact + cancellation = build_request.cancellation + cancellation.raise_if_cancelled("ledger_snapshot") + template_value = json.loads(build_request.template_json) + if not isinstance(template_value, dict): + raise RuntimeError("immutable job template is not an object") + template: dict[str, Any] = template_value + issued_at_ms = build_request.key.issued_at_ms + started = self._monotonic() + snapshot_window_weight = ( + PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * int(build_request.key.network_difficulty) + ) + if payout_artifact is not None: + if ( + self._ports.payout_state().usable_ledger_artifact( + payout_state_generation, + build_request.key.network_difficulty, + ) + is not payout_artifact + ): + raise JobBuildSuperseded( + "precomputed payout artifact changed before construction" + ) + prior_balances = list(payout_artifact.prior_balances) + if ( + canonical_json_sha256(prior_balances) + != build_request.key.payout_artifact_sha256 + ): + raise JobBuildSuperseded( + "precomputed payout artifact does not match payout generation" + ) + shares = list(payout_artifact.shares_json) + prior_balances = self._ports.prior_balances_for_parent( + str(template["previousblockhash"]), + int(template["height"]) - 1, + prior_balances, + ) + else: + payout_service = self._ports.payout_state() + with payout_service.prepare_lock: + payout_snapshot = payout_service.snapshot() + published_artifact = payout_snapshot.published.artifact + if payout_snapshot.publication_blocked: + raise PayoutStatePublicationBlocked( + "payout state invalidation is pending publication" + ) + if ( + payout_state_generation != payout_snapshot.generation + or published_artifact is None + or published_artifact.prior_balances_sha256 + != build_request.key.payout_artifact_sha256 + ): + raise JobBuildSuperseded( + "payout generation changed before ledger snapshot" + ) + records = ( + self._ports.snapshot_at_job_issue( + issued_at_ms, + snapshot_window_weight, + ) + if resolved_mode == "ready" + else [] + ) + prior_balances = self._ports.prior_balances_for_parent( + str(template["previousblockhash"]), + int(template["height"]) - 1, + build_request.payout_artifact.prior_balances(), + ) + cancellation.raise_if_cancelled("ledger_snapshot_complete") + shares = [] + for index, record in enumerate(records): + if index % 256 == 0: + cancellation.raise_if_cancelled( + "ledger_snapshot_conversion" + ) + shares.append(record.to_prism_json()) + bundle_anchor_ms = ( + payout_artifact.snapshot_anchor_ms + if payout_artifact is not None + and payout_artifact.snapshot_anchor_ms is not None + else issued_at_ms + ) + ledger_elapsed = self._monotonic() - started + self.record_phase("ledger", ledger_elapsed) + if resolved_mode == "ready": + self._ports.record_tip_refresh_phase( + "ledger_snapshot", + ledger_elapsed, + ) + final_build_key = dataclass_replace( + build_request.key, + share_snapshot_sha256=canonical_json_sha256(shares), + ) + cancellation.raise_if_cancelled("payout_derivation") + started = self._monotonic() + placeholder_suffix_hex = final_build_key.coinbase_suffix_hex + collection_identity: tuple[str, str] | None = None + previous_metrics_scope = bool( + getattr(self._phase_local, "tip_refresh_metrics", False) + ) + self._phase_local.tip_refresh_metrics = resolved_mode == "ready" + try: + if resolved_mode == "ready": + if not shares: + raise RuntimeError( + "ready-pool ledger snapshot contained no payout shares" + ) + cancellation.raise_if_cancelled("ctv_manifest") + cancellation.raise_if_cancelled("signing_verification") + bundle = self.bundle_compiler().build_audit_bundle( + shares=shares, + found_block={ + "block_height": int(template["height"]), + "coinbase_value_sats": int(template["coinbasevalue"]), + "network_difficulty": artifacts.network_difficulty, + "anchor_job_issued_at_ms": bundle_anchor_ms, + }, + prior_balances=prior_balances, + coinbase_script_sig_suffix_hex=placeholder_suffix_hex, + witness_merkle_leaves_hex=list( + build_request.witness_merkle_leaves_hex + ), + ctv_fee_parent_hash=str(template["previousblockhash"]), + summary_only=True, + payout_policy=json.loads(build_request.payout_policy_json), + ctv_settlement=( + json.loads(build_request.ctv_settlement_json) + if build_request.ctv_settlement_json is not None + else None + ), + cancellation=cancellation, + ) + collection_only = False + else: + assert worker is not None + cancellation.raise_if_cancelled("ctv_manifest") + cancellation.raise_if_cancelled("signing_verification") + bundle = self.build_collection_bundle( + template=template, + transaction_hexes=build_request.transaction_hexes, + worker=worker, + network_difficulty=final_build_key.network_difficulty, + issued_at_ms=issued_at_ms, + suffix_hex=placeholder_suffix_hex, + summary_only=True, + payout_policy=json.loads(build_request.payout_policy_json), + ctv_settlement=( + json.loads(build_request.ctv_settlement_json) + if build_request.ctv_settlement_json is not None + else None + ), + cancellation=cancellation, + ) + shares = [] + collection_only = True + collection_identity = self.collection_identity(worker) + finally: + self._phase_local.tip_refresh_metrics = previous_metrics_scope + manifest = bundle["signed_coinbase_manifest"]["manifest"] + prospective_prior_balances: ( + tuple[tuple[str, str, str, int], ...] | None + ) = None + payout_policy_manifest = bundle.get("payout_policy_manifest") + if isinstance(payout_policy_manifest, dict) and isinstance( + payout_policy_manifest.get("accounts"), + list, + ): + prospective_prior_balances = ( + self._ports.serialize_prior_balance_preview( + self._ports.accepted_block_preview_from_bundle( + bundle, + prior_balances, + ) + ) + ) + cancellation.raise_if_cancelled("bundle_assembly") + assembly_started = self._monotonic() + base_job = direct_stratum.make_job_from_builder_manifest( + job_id="prism-template-base", + template=template, + manifest=manifest, + extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, + extranonce2_size=self._config.extranonce2_size, + desired_share_difficulty=self._config.share_difficulty, + clean_jobs=True, + transaction_hexes=build_request.transaction_hexes, + ) + self.record_phase("assembly", self._monotonic() - assembly_started) + cancellation.raise_if_cancelled("serialization") + cancellation.raise_if_cancelled("bundle_publication") + self.record_phase("bundle", self._monotonic() - started) + return CachedJobBundle( + key=key, + template=artifacts.template, + template_fingerprint=artifacts.fingerprint, + coinbase_manifest=manifest, + shares_json=shares, + prior_balances=prior_balances, + found_block=bundle["found_block"], + collection_only=collection_only, + issued_at_ms=issued_at_ms, + base_job=base_job, + built_monotonic=self._monotonic(), + template_generation=artifacts.generation, + payout_state_generation=payout_state_generation, + payout_artifact_generation=( + payout_artifact.generation if payout_artifact is not None else 0 + ), + collection_identity=collection_identity, + prospective_prior_balances=prospective_prior_balances, + build_key=final_build_key, + ) + + def build_collection_bundle( + self, + *, + template: dict[str, Any], + transaction_hexes: tuple[str, ...], + worker: WorkerIdentityPort, + network_difficulty: int, + issued_at_ms: int, + suffix_hex: str, + summary_only: bool = False, + payout_policy: dict[str, object] | None = None, + ctv_settlement: dict[str, object] | None = None, + cancellation: JobBuildCancellation | None = None, + ) -> dict[str, Any]: + if cancellation is not None: + cancellation.raise_if_cancelled("collection payout derivation") + share = { + "share_seq": 1, + "share_id": "bootstrap-share", + "miner_id": worker.payout_address, + "order_key": worker.payout_address, + "p2mr_program_hex": worker.p2mr_program_hex, + "share_difficulty": network_difficulty, + "network_difficulty": network_difficulty, + "template_height": int(template["height"]) - 1, + "job_id": "bootstrap-job", + "job_issued_at_ms": issued_at_ms, + "accepted_at_ms": issued_at_ms, + "ntime": int(template["curtime"]), + } + return self.bundle_compiler().build_audit_bundle( + shares=[share], + found_block={ + "block_height": int(template["height"]), + "coinbase_value_sats": int(template["coinbasevalue"]), + "network_difficulty": network_difficulty, + "anchor_job_issued_at_ms": issued_at_ms, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex=suffix_hex, + witness_merkle_leaves_hex=( + direct_stratum.witness_merkle_leaves_hex(transaction_hexes) + ), + ctv_fee_parent_hash=str(template["previousblockhash"]), + summary_only=summary_only, + payout_policy=payout_policy, + ctv_settlement=ctv_settlement, + cancellation=cancellation, + ) + + def set_preparation_pending(self, pending: bool) -> None: + with self._prepared_lock: + self._preparation_pending = bool(pending) + + def set_prepared_ready( + self, + snapshot: QbitTipTemplateSnapshot | None, + bundle: CachedJobBundle | None, + ) -> None: + with self._prepared_lock: + self._prepared_ready_snapshot = snapshot + self._prepared_ready_bundle = bundle + + def clear_prepared_ready(self) -> None: + self.set_prepared_ready(None, None) + + def prepared_ready_snapshot( + self, + ) -> tuple[ + CachedJobBundle | None, + QbitTipTemplateSnapshot | None, + bool, + ]: + with self._prepared_lock: + return ( + self._prepared_ready_bundle, + self._prepared_ready_snapshot, + self._preparation_pending, + ) + + def record_failure(self) -> None: + with self._cache_lock: + self._failure_count += 1 + + def record_worker_event(self, event: str) -> None: + with self._scheduler_lock: + if event == "start": + if self._worker_restart_pending: + self._worker_counts["restarts"] += 1 + self._worker_restart_pending = False + self._worker_counts["starts"] += 1 + return + if event == "termination": + self._worker_counts["terminations"] += 1 + self._worker_restart_pending = True + return + if event == "crash": + self._worker_counts["crashes"] += 1 + self._worker_restart_pending = True + return + raise ValueError(f"unknown job builder worker event: {event}") + + def tip_refresh_metrics_enabled(self) -> bool: + return bool(getattr(self._phase_local, "tip_refresh_metrics", False)) + + def shared_preparation_metrics(self) -> dict[str, object]: + with self._scheduler_lock: + return { + "build_counts": dict(self._shared_build_counts), + "preparation_sum": self._preparation_seconds_sum, + "preparation_count": self._preparation_count, + "waiters": self._preparation_waiters, + } + + def metrics_snapshot(self) -> dict[str, object]: + with self._cache_lock: + cache = { + "bucket_counts": dict(self._build_bucket_counts), + "build_sum": self._build_seconds_sum, + "build_count": self._build_count, + "phase_seconds": dict(self._phase_seconds), + "hit_counts": dict(self._cache_hits), + "miss_counts": dict(self._cache_misses), + "failure_count": self._failure_count, + } + with self._scheduler_lock: + now = self._monotonic() + priority_requests = tuple( + request + for request in ( + self._active.request if self._active is not None else None, + ( + self._retiring.request + if self._retiring is not None + else None + ), + self._pending, + ) + if request is not None + and not request.cancellation.is_set() + and self.is_publication_critical(request) + ) + priority_preparations = tuple(self._priority_preparations.values()) + scheduler = { + "scheduler_counts": dict(self._scheduler_counts), + "priority_counts": dict(self._priority_counts), + "priority_admission_seconds": dict( + self._priority_admission_seconds + ), + "initial_prepared_counts": dict( + self._initial_prepared_work_counts + ), + "cancellation_seconds": dict(self._cancellation_seconds), + "replacement_seconds": dict(self._replacement_start_seconds), + "worker_counts": dict(self._worker_counts), + "active_builds": int(self._active is not None), + "pending_builds": int(self._pending is not None), + "priority_active": int( + bool(priority_requests or priority_preparations) + ), + "priority_age_seconds": max( + ( + *(now - request.requested_monotonic for request in priority_requests), + *(now - started for started in priority_preparations), + 0.0, + ) + ), + } + return {**cache, **scheduler} + + def bundle_cache_snapshot(self) -> tuple[CachedJobBundle, ...]: + with self._cache_lock: + return tuple(self._bundle_cache.values()) + + def cached_bundle_for_key( + self, + key: tuple[object, ...], + ) -> CachedJobBundle | None: + with self._cache_lock: + return self._bundle_cache.get(key) + + @contextmanager + def cache_admission( + self, + key: tuple[object, ...], + bundle: CachedJobBundle, + *, + allow_uncached: bool, + ) -> Iterator[bool]: + """Hold exact cache identity through an external final commit guard. + + V1/S2 still perform the client/tip guard while this admission is held. + This preserves the existing cache-before-client lock order until those + domains move; no callback is invoked while the service lock is held. + """ + with self._cache_lock: + if self._config.cache_seconds <= 0: + yield bool(allow_uncached and bundle.key == key) + return + cached = self._bundle_cache.get(key) + matches = bool( + cached is not None + and ( + cached is bundle + or ( + bundle.key == cached.key + and bundle.coinbase_manifest is cached.coinbase_manifest + and bundle.shares_json is cached.shares_json + and bundle.prior_balances is cached.prior_balances + and bundle.found_block is cached.found_block + and bundle.collection_only == cached.collection_only + and bundle.issued_at_ms == cached.issued_at_ms + and bundle.built_monotonic == cached.built_monotonic + and bundle.payout_state_generation + == cached.payout_state_generation + and bundle.payout_artifact_generation + == cached.payout_artifact_generation + and bundle.collection_identity == cached.collection_identity + ) + ) + and self._monotonic() - cached.built_monotonic + <= self._config.cache_seconds + ) + yield matches + + def clear_issued_at_for_test(self) -> None: + with self._cache_lock: + self._issued_at_ms.clear() + + def set_ready_for_test(self, ready: bool) -> None: + with self._readiness_lock: + self._ready_latched = bool(ready) + + def ready_latched(self) -> bool: + with self._readiness_lock: + return self._ready_latched + + def active_bundle_builds_for_test( + self, + ) -> dict[tuple[object, ...], JobBundleBuildControl]: + with self._cache_lock: + return dict(self._active_bundle_builds) + + def scheduler_state_for_test( + self, + ) -> tuple[JobBuildFlight | None, JobBuildFlight | None, JobBuildRequest | None]: + with self._scheduler_lock: + return self._active, self._retiring, self._pending + + def replace_config_for_test(self, config: JobBundleConfig) -> None: + self._config = config + + def set_cache_seconds_for_test(self, seconds: float) -> None: + self._config = dataclass_replace( + self._config, + cache_seconds=float(seconds), + ) + + def set_min_ready_miners_for_test(self, count: int) -> None: + self._config = dataclass_replace( + self._config, + min_ready_miners=int(count), + ) diff --git a/lab/prism/job_delivery.py b/lab/prism/job_delivery.py new file mode 100644 index 0000000..ceab7b8 --- /dev/null +++ b/lab/prism/job_delivery.py @@ -0,0 +1,3472 @@ +"""Per-session PRISM job delivery and retained-job ownership. + +This module deliberately has no dependency on :mod:`prism_coordinator`. The +coordinator is the construction root and supplies narrow runtime operations; +session membership remains authoritative in :class:`SessionRegistry` and the +heavy shared bundle build remains in J1's job-bundle service. +""" + +from __future__ import annotations + +from collections import OrderedDict +from concurrent.futures import Future +from dataclasses import dataclass, field, replace as dataclass_replace +from decimal import Decimal +import threading +import time +import traceback +from typing import Any, Callable, Mapping, MutableMapping, Protocol + +from lab.auxpow import vardiff +from lab.prism import direct_stratum +from lab.prism.bounded_executor import _BoundedPriorityExecutor, _DeliveryQueueFull +from lab.prism.coordinator_config import ( + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + DEFAULT_PRISM_STALE_GRACE_SECONDS, +) +from lab.prism.job_bundle import CachedJobBundle, JobBuildWaiterCancelled +from lab.prism.payout_state import TemplateRefreshBlocked, TemplateRefreshSuperseded +from lab.prism.stratum_session import ( + ClientState, + SessionRegistry, + StratumError, + WorkerIdentity, + client_vardiff_lock, + client_can_receive_jobs as session_client_can_receive_jobs, +) +from lab.prism.template_artifacts import CachedTemplateArtifacts, QbitTipTemplateSnapshot +from lab.prism.tip_refresh import ( + FanoutCancellation, + RefreshClientTarget, + RefreshResult, + TipRefreshValidationToken, +) + + +MAX_ACTIVE_PRISM_JOBS_PER_CLIENT = 16 +DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS = 1.0 +PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 +PRISM_DELIVERY_PRIORITY_NEW_TIP = 0 +PRISM_DELIVERY_PRIORITY_INITIAL = 1 +PRISM_DELIVERY_PRIORITY_SAME_TIP = 2 +PRISM_EVICTED_JOB_CLASSES = ("same_tip", "stale_grace") +PRISM_EVICTED_JOB_SUBMIT_OUTCOMES = ( + "accepted_same_tip", + "credited_stale_grace", +) +PRISM_EVICTED_JOB_CAPACITY_SCOPES = ("connection",) +PRISM_CREDIT_POLICY_STALE_GRACE = "stale-grace" +DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS = 4 + + +@dataclass(frozen=True) +class PrismJobContext: + job: direct_stratum.DirectQbitStratumJob + template: dict[str, Any] + shares_json: list[dict[str, object]] + prior_balances: list[dict[str, object]] + found_block: dict[str, object] + share_weight: int + collection_only: bool + worker: WorkerIdentity + issued_at_ms: int + template_fingerprint: str | None = None + template_generation: int = 0 + payout_state_generation: int = 0 + prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None + payout_artifact_generation: int = 0 + connection_id: int = 0 + authorization_generation: int = 0 + difficulty_generation: int = 0 + + +@dataclass(frozen=True) +class EvictedJobEntry: + context: PrismJobContext + connection_id: int + evicted_monotonic: float + previousblockhash: str + client: ClientState | None = None + + +@dataclass(eq=False) +class PendingInitialJob: + client: ClientState + authorization_generation: int + worker: WorkerIdentity + requested_monotonic: float + deadline_monotonic: float | None + connection_id: int | None = None + difficulty_generation: int | None = None + cancelled: threading.Event = field(default_factory=threading.Event) + future: Future[bool] | None = None + predecessor: Future[bool] | None = None + + +@dataclass(frozen=True) +class InitialJobConfig: + max_pending: int + timeout_seconds: float + max_workers: int = DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS + + +@dataclass(frozen=True) +class InitialJobSnapshot: + max_workers: int + pending_count: int + queue_rejection_count: int + timeout_count: int + cancelled_count: int + coalesced_count: int + sent_count: int + failed_count: int + superseded_count: int + queue_capacity_reclaimed_count: int + delivery_latency_seconds_sum: float + delivery_latency_count: int + last_delivery_monotonic: float | None + + +@dataclass(frozen=True) +class CurrentJobSource: + payout_generation: int + current_tip: str | None + published_template: QbitTipTemplateSnapshot | None + + +@dataclass(frozen=True) +class RefreshSource: + ready_latched: bool + payout_generation: int + + +@dataclass(frozen=True) +class DeliverySourceAuthority: + """Immutable source identity revalidated under the shared S1/R1 lock.""" + + kind: str + payout_generation: int + template_generation: int + observation_sequence: int + template_fingerprint: str | None = None + artifacts: CachedTemplateArtifacts | None = None + snapshot: QbitTipTemplateSnapshot | None = None + token: TipRefreshValidationToken | None = None + bundle: CachedJobBundle | None = None + payout_snapshot: Any = None + context_parent: str = "" + lapsed_live_validated: bool = False + + +@dataclass(frozen=True) +class AdmittedIdleBundleSource: + """Owner-issued J1 lease for one exact immutable idle bundle source. + + J1 fixes cache freshness at admission and returns the exact artifact, + bundle, and cache identity. S2 may finish delivery after cache eviction + without reacquiring a J1 lock; R1, P1, and client authority remain final + commit checks. + """ + + artifacts: CachedTemplateArtifacts + bundle: CachedJobBundle + cache_identity: tuple[object, ...] + allow_uncached: bool + + +@dataclass +class IdleDeliveryAuthority: + """Typed V1 state expected at an idle-retarget delivery commit.""" + + connection_id: int + worker: WorkerIdentity + expected_active_job: PrismJobContext | None + expected_window_started: float + pending_difficulty: Decimal + committed_reset_monotonic: float | None = None + + +class JobBuildFailed(RuntimeError): + """A client job could not be built, without implying socket failure.""" + + +# Compatibility name used during the staged extraction. +_JobBuildFailed = JobBuildFailed + + +class InitialJobTracker: + """Own pending first-job identities independently of build/executor policy.""" + + def __init__( + self, + pending: MutableMapping[ClientState, PendingInitialJob] | None = None, + ) -> None: + self.pending = pending if pending is not None else {} + + def adopt( + self, + pending: MutableMapping[ClientState, PendingInitialJob], + ) -> None: + self.pending = pending + + def request_current_locked( + self, + request: PendingInitialJob, + *, + clients: object, + stopping: bool, + ) -> bool: + client = request.client + return bool( + not stopping + and self.pending.get(client) is request + and client in clients # type: ignore[operator] + and ( + request.connection_id is None + or client.connection_id == request.connection_id + ) + and client.authorized + and client.subscribed + and client.worker == request.worker + and int(client.authorization_generation) + == request.authorization_generation + and ( + request.difficulty_generation is None + or int(client.difficulty_generation) + == request.difficulty_generation + ) + and not client.closing + and ( + request.deadline_monotonic is None + or time.monotonic() < request.deadline_monotonic + ) + and not request.cancelled.is_set() + ) + + def cancel_locked( + self, + client: ClientState, + ) -> PendingInitialJob | None: + request = self.pending.pop(client, None) + if request is None: + return None + request.cancelled.set() + return request + + def expire_locked(self, now: float) -> tuple[PendingInitialJob, ...]: + expired: list[PendingInitialJob] = [] + for request in tuple(self.pending.values()): + if ( + request.deadline_monotonic is None + or request.deadline_monotonic > now + or self.pending.get(request.client) is not request + ): + continue + self.pending.pop(request.client, None) + request.cancelled.set() + # This state transition is the admission fence for a concurrent + # reauthorization before the coordinator performs socket close. + request.client.closing = True + expired.append(request) + return tuple(expired) + + def shutdown_locked(self) -> tuple[PendingInitialJob, ...]: + pending = tuple(self.pending.values()) + self.pending.clear() + for request in pending: + request.cancelled.set() + return pending + + +class InitialJobState: + """S2-owned first-job configuration, lifecycle, and metrics state.""" + + def __init__( + self, + config: InitialJobConfig, + pending: MutableMapping[ClientState, PendingInitialJob] | None = None, + ) -> None: + self.config = config + self.tracker = InitialJobTracker(pending) + self.queue_rejection_count = 0 + self.timeout_count = 0 + self.cancelled_count = 0 + self.coalesced_count = 0 + self.sent_count = 0 + self.failed_count = 0 + self.superseded_count = 0 + self.queue_capacity_reclaimed_count = 0 + self.delivery_latency_seconds_sum = 0.0 + self.delivery_latency_count = 0 + self.last_delivery_monotonic: float | None = None + + @property + def pending(self) -> MutableMapping[ClientState, PendingInitialJob]: + return self.tracker.pending + + def adopt_pending( + self, + pending: MutableMapping[ClientState, PendingInitialJob], + ) -> None: + self.tracker.adopt(pending) + + def reconfigure( + self, + *, + max_pending: int | None = None, + timeout_seconds: float | None = None, + max_workers: int | None = None, + ) -> None: + self.config = InitialJobConfig( + max_pending=( + self.config.max_pending if max_pending is None else int(max_pending) + ), + timeout_seconds=( + self.config.timeout_seconds + if timeout_seconds is None + else float(timeout_seconds) + ), + max_workers=( + self.config.max_workers + if max_workers is None + else int(max_workers) + ), + ) + + def snapshot(self) -> InitialJobSnapshot: + return InitialJobSnapshot( + max_workers=self.config.max_workers, + pending_count=len(self.pending), + queue_rejection_count=self.queue_rejection_count, + timeout_count=self.timeout_count, + cancelled_count=self.cancelled_count, + coalesced_count=self.coalesced_count, + sent_count=self.sent_count, + failed_count=self.failed_count, + superseded_count=self.superseded_count, + queue_capacity_reclaimed_count=self.queue_capacity_reclaimed_count, + delivery_latency_seconds_sum=self.delivery_latency_seconds_sum, + delivery_latency_count=self.delivery_latency_count, + last_delivery_monotonic=self.last_delivery_monotonic, + ) + + +@dataclass(frozen=True) +class DeliveryAuthority: + """Immutable identity that every delivery path must revalidate. + + ``expected_active_job`` is the context observed before registration. Once + registration has happened, ``registered_context`` becomes the final send + guard. Authorization and difficulty generations are both intentional: + unlike health snapshots, delivery authority is generation-sensitive. + """ + + connection_id: int + authorization_generation: int + difficulty_generation: int + worker: WorkerIdentity + expected_active_job: object | None + template_fingerprint: str | None + template_generation: int + payout_state_generation: int + + @classmethod + def capture( + cls, + client: ClientState, + *, + context: object, + expected_active_job: object | None, + ) -> DeliveryAuthority: + worker = client.worker + if worker is None: + raise StratumError(20, "client is not authorized") + return cls( + connection_id=int(client.connection_id), + authorization_generation=int(client.authorization_generation), + difficulty_generation=int(client.difficulty_generation), + worker=worker, + expected_active_job=expected_active_job, + template_fingerprint=getattr(context, "template_fingerprint", None), + template_generation=int(getattr(context, "template_generation", 0)), + payout_state_generation=int( + getattr(context, "payout_state_generation", 0) + ), + ) + + def client_matches( + self, + client: ClientState, + *, + active_job: object | None, + ) -> bool: + return bool( + not client.closing + and int(client.connection_id) == self.connection_id + and client.subscribed + and client.authorized + and client.worker == self.worker + and int(client.authorization_generation) + == self.authorization_generation + and int(client.difficulty_generation) == self.difficulty_generation + and active_job is self.expected_active_job + ) + + +class RetainedJobIndex: + """Bounded, indexed ownership for same-tip and stale-grace contexts. + + The caller follows ``job_update_lock -> SessionRegistry.lock -> lock``. + This class never performs RPC, socket I/O, or callbacks while ``lock`` is + held. Tip-parent information is supplied as an immutable cached value. + """ + + def __init__( + self, + *, + lock: threading.RLock | None = None, + graveyard: OrderedDict[str, EvictedJobEntry] | None = None, + by_connection: dict[int, OrderedDict[str, None]] | None = None, + same_tip_by_connection: dict[int, OrderedDict[str, None]] | None = None, + same_tip_job_ids: OrderedDict[str, None] | None = None, + same_tip_ttl_seconds: float = DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + same_tip_per_connection: int = ( + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION + ), + stale_grace_seconds: float = DEFAULT_PRISM_STALE_GRACE_SECONDS, + ) -> None: + self.lock = lock if lock is not None else threading.RLock() + converted_graveyard: OrderedDict[str, EvictedJobEntry] = OrderedDict() + graveyard_requires_conversion = False + for job_id, entry in (graveyard or {}).items(): + if not isinstance(entry, EvictedJobEntry): + graveyard_requires_conversion = True + context, connection_id, evicted_monotonic = entry # type: ignore[misc] + entry = EvictedJobEntry( + context=context, + connection_id=connection_id, + evicted_monotonic=evicted_monotonic, + previousblockhash=str(context.template["previousblockhash"]), + ) + converted_graveyard[job_id] = entry + self.graveyard = ( + graveyard + if isinstance(graveyard, OrderedDict) and not graveyard_requires_conversion + else converted_graveyard + ) + self.by_connection = by_connection if by_connection is not None else {} + self.same_tip_by_connection = ( + same_tip_by_connection if same_tip_by_connection is not None else {} + ) + self.same_tip_job_ids = ( + same_tip_job_ids if same_tip_job_ids is not None else OrderedDict() + ) + self.same_tip_ttl_seconds = float(same_tip_ttl_seconds) + self.same_tip_per_connection = int(same_tip_per_connection) + self.stale_grace_seconds = float(stale_grace_seconds) + self.index_tip_hash: str | None = None + self.next_prune_monotonic = 0.0 + self.expiration_counts = {name: 0 for name in PRISM_EVICTED_JOB_CLASSES} + self.capacity_eviction_counts = { + name: 0 for name in PRISM_EVICTED_JOB_CAPACITY_SCOPES + } + self.submit_counts = { + name: 0 for name in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES + } + + def adopt( + self, + *, + graveyard: MutableMapping[str, EvictedJobEntry] | None, + by_connection: dict[int, OrderedDict[str, None]] | None, + same_tip_by_connection: dict[int, OrderedDict[str, None]] | None, + same_tip_job_ids: OrderedDict[str, None] | None, + current_tip: str | None, + ) -> None: + """Adopt focused-test/embedding replacements without stale aliases.""" + with self.lock: + graveyard_replaced = graveyard is not self.graveyard + if graveyard_replaced: + converted: OrderedDict[str, EvictedJobEntry] = OrderedDict() + requires_conversion = False + for job_id, entry in (graveyard or {}).items(): + if not isinstance(entry, EvictedJobEntry): + requires_conversion = True + context, connection_id, evicted_monotonic = entry # type: ignore[misc] + entry = EvictedJobEntry( + context=context, + connection_id=connection_id, + evicted_monotonic=evicted_monotonic, + previousblockhash=str( + context.template["previousblockhash"] + ), + ) + converted[job_id] = entry + self.graveyard = ( + graveyard + if isinstance(graveyard, OrderedDict) and not requires_conversion + else converted + ) + maps_replaced = False + if by_connection is not None and by_connection is not self.by_connection: + self.by_connection = by_connection + maps_replaced = True + if ( + same_tip_by_connection is not None + and same_tip_by_connection is not self.same_tip_by_connection + ): + self.same_tip_by_connection = same_tip_by_connection + maps_replaced = True + if same_tip_job_ids is not None and same_tip_job_ids is not self.same_tip_job_ids: + self.same_tip_job_ids = same_tip_job_ids + maps_replaced = True + if ( + graveyard_replaced + or maps_replaced + or self.index_tip_hash != current_tip + ): + self._rebuild_locked(current_tip) + + def _job_class_locked( + self, entry: EvictedJobEntry, current_tip: str | None + ) -> str: + if current_tip is None or entry.previousblockhash == current_tip: + return "same_tip" + return "stale_grace" + + def _coerce_entry_locked( + self, + job_id: str, + entry: EvictedJobEntry | object, + ) -> EvictedJobEntry: + if isinstance(entry, EvictedJobEntry): + return entry + context, connection_id, evicted_monotonic = entry # type: ignore[misc] + converted = EvictedJobEntry( + context=context, + connection_id=connection_id, + evicted_monotonic=evicted_monotonic, + previousblockhash=str(context.template["previousblockhash"]), + ) + self.graveyard[job_id] = converted + return converted + + def _remove_locked(self, job_id: str) -> EvictedJobEntry | None: + entry = self.graveyard.pop(job_id, None) + if entry is None: + return None + for mapping in (self.by_connection, self.same_tip_by_connection): + connection_jobs = mapping.get(entry.connection_id) + if connection_jobs is not None: + connection_jobs.pop(job_id, None) + if not connection_jobs: + mapping.pop(entry.connection_id, None) + self.same_tip_job_ids.pop(job_id, None) + return entry + + def _index_locked( + self, + job_id: str, + entry: EvictedJobEntry, + current_tip: str | None, + ) -> None: + self.by_connection.setdefault(entry.connection_id, OrderedDict())[job_id] = None + if self._job_class_locked(entry, current_tip) != "same_tip": + return + self.same_tip_by_connection.setdefault( + entry.connection_id, OrderedDict() + )[job_id] = None + self.same_tip_job_ids[job_id] = None + + def _rebuild_locked(self, current_tip: str | None) -> None: + self.by_connection.clear() + self.same_tip_by_connection.clear() + self.same_tip_job_ids.clear() + for job_id, entry in tuple(self.graveyard.items()): + entry = self._coerce_entry_locked(job_id, entry) + self._index_locked(job_id, entry, current_tip) + self.index_tip_hash = current_tip + self._enforce_capacity_locked() + + def _enforce_capacity_locked(self, connection_id: int | None = None) -> None: + connection_ids = ( + (connection_id,) + if connection_id is not None + else tuple(self.same_tip_by_connection) + ) + for candidate in connection_ids: + job_ids = self.same_tip_by_connection.get(candidate) + while job_ids is not None and len(job_ids) > self.same_tip_per_connection: + self._remove_locked(next(iter(job_ids))) + self.capacity_eviction_counts["connection"] += 1 + job_ids = self.same_tip_by_connection.get(candidate) + + def retain( + self, + client: ClientState, + job_id: str, + context: PrismJobContext, + *, + current_tip: str | None, + now: float | None = None, + ) -> None: + with self.lock: + if self.index_tip_hash != current_tip: + self._rebuild_locked(current_tip) + self._remove_locked(job_id) + entry = EvictedJobEntry( + context=context, + connection_id=int(client.connection_id), + evicted_monotonic=time.monotonic() if now is None else now, + previousblockhash=str(context.template["previousblockhash"]), + client=client, + ) + self.graveyard[job_id] = entry + self._index_locked(job_id, entry, current_tip) + self._enforce_capacity_locked(client.connection_id) + + def _expired_locked( + self, + entry: EvictedJobEntry, + *, + now: float, + current_tip: str | None, + current_tip_first_delivery: float | None, + cached_parent: str | None, + ) -> tuple[str, bool]: + job_class = self._job_class_locked(entry, current_tip) + if job_class == "same_tip": + return ( + job_class, + self.same_tip_ttl_seconds <= 0 + or now - entry.evicted_monotonic > self.same_tip_ttl_seconds, + ) + if self.stale_grace_seconds <= 0 or current_tip is None: + return job_class, True + if cached_parent is not None and entry.previousblockhash != cached_parent: + return job_class, True + client = entry.client + if client is not None: + delivered = client.tip_work_delivered + if delivered is None or delivered[0] != current_tip: + return job_class, False + anchor = float(delivered[1]) + elif current_tip_first_delivery is None: + return job_class, True + else: + anchor = current_tip_first_delivery + return job_class, now - anchor > self.stale_grace_seconds + + def prune( + self, + *, + current_tip: str | None, + current_tip_first_delivery: float | None, + cached_parent: str | None, + now: float | None = None, + force: bool = True, + ) -> None: + with self.lock: + if self.index_tip_hash != current_tip: + self._rebuild_locked(current_tip) + if not self.graveyard: + return + now = time.monotonic() if now is None else now + if not force and now < self.next_prune_monotonic: + return + self.next_prune_monotonic = ( + now + DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS + ) + for job_id, entry in tuple(self.graveyard.items()): + entry = self._coerce_entry_locked(job_id, entry) + job_class, expired = self._expired_locked( + entry, + now=now, + current_tip=current_tip, + current_tip_first_delivery=current_tip_first_delivery, + cached_parent=cached_parent, + ) + if expired: + self._remove_locked(job_id) + self.expiration_counts[job_class] += 1 + + def lookup( + self, + client: ClientState, + job_id: str, + *, + current_tip: str | None, + current_tip_first_delivery: float | None, + cached_parent: str | None, + now: float | None = None, + ) -> EvictedJobEntry | None: + """Constant-time lookup; no scan of the graveyard or client pool.""" + with self.lock: + if self.index_tip_hash != current_tip: + self._rebuild_locked(current_tip) + entry = self.graveyard.get(job_id) + if entry is None: + return None + entry = self._coerce_entry_locked(job_id, entry) + if entry.connection_id != client.connection_id: + return None + job_class, expired = self._expired_locked( + entry, + now=time.monotonic() if now is None else now, + current_tip=current_tip, + current_tip_first_delivery=current_tip_first_delivery, + cached_parent=cached_parent, + ) + if expired: + self._remove_locked(job_id) + self.expiration_counts[job_class] += 1 + return None + return entry + + def peek(self, job_id: str) -> EvictedJobEntry | None: + """Return one retained entry for explicit compatibility observation.""" + with self.lock: + entry = self.graveyard.get(job_id) + if entry is None: + return None + return self._coerce_entry_locked(job_id, entry) + + def retire_connection(self, connection_id: int) -> tuple[str, ...]: + with self.lock: + retired = tuple(self.by_connection.get(connection_id, ())) + for job_id in retired: + self._remove_locked(job_id) + return retired + + def note_submit(self, credit_policy: str | None) -> None: + outcome = ( + "credited_stale_grace" + if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE + else "accepted_same_tip" + ) + with self.lock: + self.submit_counts[outcome] += 1 + + def job_class( + self, + entry: EvictedJobEntry, + *, + current_tip: str | None, + ) -> str: + with self.lock: + return self._job_class_locked(entry, current_tip) + + +class JobDeliveryRuntimePort(Protocol): + def desired_share_difficulty(self, client: ClientState) -> Decimal: ... + def minimum_advertised_difficulty(self, client: ClientState) -> Decimal: ... + def share_weight(self, worker: WorkerIdentity) -> int: ... + def vardiff_config(self, client: ClientState) -> vardiff.VardiffConfig: ... + def send_difficulty( + self, client: ClientState, job: direct_stratum.DirectQbitStratumJob + ) -> None: ... + def send_job( + self, client: ClientState, job: direct_stratum.DirectQbitStratumJob + ) -> None: ... + def send_job_batch( + self, client: ClientState, job: direct_stratum.DirectQbitStratumJob + ) -> None: ... + + +@dataclass(frozen=True) +class JobDeliveryRuntime(JobDeliveryRuntimePort): + """Callable-backed narrow port used by the coordinator construction root.""" + + desired_share_difficulty_fn: Callable[[ClientState], Decimal] + minimum_advertised_difficulty_fn: Callable[[ClientState], Decimal] + share_weight_fn: Callable[[WorkerIdentity], int] + vardiff_config_fn: Callable[[ClientState], vardiff.VardiffConfig] + send_difficulty_fn: Callable[ + [ClientState, direct_stratum.DirectQbitStratumJob], None + ] + send_job_fn: Callable[[ClientState, direct_stratum.DirectQbitStratumJob], None] + send_job_batch_fn: Callable[ + [ClientState, direct_stratum.DirectQbitStratumJob], None + ] + + def desired_share_difficulty(self, client: ClientState) -> Decimal: + return self.desired_share_difficulty_fn(client) + + def minimum_advertised_difficulty(self, client: ClientState) -> Decimal: + return self.minimum_advertised_difficulty_fn(client) + + def share_weight(self, worker: WorkerIdentity) -> int: + return self.share_weight_fn(worker) + + def vardiff_config(self, client: ClientState) -> vardiff.VardiffConfig: + return self.vardiff_config_fn(client) + + def send_difficulty( + self, + client: ClientState, + job: direct_stratum.DirectQbitStratumJob, + ) -> None: + self.send_difficulty_fn(client, job) + + def send_job( + self, + client: ClientState, + job: direct_stratum.DirectQbitStratumJob, + ) -> None: + self.send_job_fn(client, job) + + def send_job_batch( + self, + client: ClientState, + job: direct_stratum.DirectQbitStratumJob, + ) -> None: + self.send_job_batch_fn(client, job) + + +@dataclass(frozen=True) +class RetentionAuthority: + current_tip: str | None + current_tip_first_delivery: float | None + cached_parent: str | None + + +class JobPreparationPort(Protocol): + def ensure_reorg_current(self) -> bool: ... + def issuance_artifacts(self) -> CachedTemplateArtifacts: ... + def shared_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentity, + *, + cancelled: Callable[[], bool] | None = None, + request_source: str = "routine", + ) -> CachedJobBundle: ... + def artifacts_current(self, artifacts: CachedTemplateArtifacts) -> bool: ... + def clear_artifacts(self, artifacts: CachedTemplateArtifacts) -> None: ... + def record_failure(self) -> None: ... + def phases(self) -> dict[str, float]: ... + def retained_artifacts(self) -> CachedTemplateArtifacts | None: ... + def chain_view_untrusted(self) -> bool: ... + def admit_idle_bundle_source( + self, + client: ClientState, + bundle: CachedJobBundle, + *, + allow_uncached: bool, + ) -> AdmittedIdleBundleSource | None: ... + def observe_elapsed( + self, + elapsed_seconds: float, + phases: Mapping[str, float], + ) -> None: ... + def collection_identity(self, worker: WorkerIdentity) -> object: ... + def ready_latched(self) -> bool: ... + def template_fingerprint(self, template: Mapping[str, object]) -> str: ... + + +class TipAuthorityPort(Protocol): + def live_tip(self) -> str: ... + def observe_tip(self, tip_hash: str) -> object: ... + def published_authority(self) -> tuple[str, float | None] | None: ... + def published_authoritative(self, now: float) -> bool: ... + def current_tip_locked(self) -> str | None: ... + def published_template_locked(self) -> QbitTipTemplateSnapshot | None: ... + def snapshot_current_locked( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> bool: ... + def artifacts_parent_current_locked( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: ... + def ensure_artifacts_parent_observed( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: ... + def schedule_retry(self) -> None: ... + def prepared_obsolete( + self, + validation_token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + cancel_event: FanoutCancellation | None, + ) -> bool: ... + def prepared_token_current_locked( + self, + validation_token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + payout_snapshot: Any, + ) -> bool: ... + def record_cancellation(self, stage: str) -> None: ... + def retention_authority_locked(self) -> RetentionAuthority: ... + def consume_retained_refresh(self, context: PrismJobContext) -> None: ... + def published_current_locked( + self, + context_parent: str, + *, + template_fingerprint: str | None, + template_generation: int, + lapsed_live_validated: bool, + payout_generation: int, + ) -> bool: ... + + +class PayoutDeliveryPort(Protocol): + def snapshot(self) -> Any: ... + def generation(self) -> int: ... + def initial_admission( + self, + cancelled: Callable[[], bool], + *, + generation: int, + ) -> Any: ... + def admission( + self, + cancelled: Callable[[], bool], + *, + generation: int, + priority: bool, + ) -> Any: ... + def observe_admission( + self, + admission: object, + *, + generation: int, + fallback_wait_seconds: float, + ) -> None: ... + def record_first_delivery( + self, + generation: int, + delivered_monotonic: float, + ) -> None: ... +class InitialJobRuntimePort(Protocol): + def stopping(self) -> bool: ... + def wait(self, timeout: float) -> bool: ... + def disconnect(self, client: ClientState) -> None: ... + def submit_initial( + self, + function: Callable[[PendingInitialJob], bool], + request: PendingInitialJob, + *, + priority: int, + ) -> Future[Any]: ... + + +@dataclass(frozen=True) +class DeliveryCompatibilityHooks: + """Named legacy monkeypatch resolvers; never a domain dependency port.""" + + run_initial_override: Callable[[], Callable[[PendingInitialJob], bool] | None] + deliver_initial_override: Callable[[], Callable[..., bool | None] | None] + submit_initial_override: Callable[ + [], Callable[[PendingInitialJob], bool] | None + ] + maybe_send_override: Callable[[], Callable[..., bool] | None] + send_prepared_override: Callable[[], Callable[..., RefreshResult] | None] + build_job_override: Callable[[], Callable[..., PrismJobContext] | None] + stamp_job_override: Callable[[], Callable[..., PrismJobContext] | None] + apply_difficulty_override: Callable[[], Callable[..., None] | None] + send_update_override: Callable[[], Callable[..., None] | None] + needs_refresh_override: Callable[[], Callable[..., bool] | None] + retained_classify_override: Callable[[], Callable[..., str] | None] + split_send_enabled: Callable[[], bool] + hot_path_logging_enabled: Callable[[], bool] + reorg_reconciler_enabled: Callable[[], bool] + + +class ProgressDeliveryPort(Protocol): + def record_health_delivery( + self, + client: ClientState, + context: PrismJobContext, + delivered_monotonic: float, + ) -> None: ... + def reconcile_health_eligibility(self) -> None: ... + + +class JobDeliveryService: + """Coordinator-free owner of stamping, mutations, guards, and wire pairs.""" + + def __init__( + self, + *, + registry: SessionRegistry, + runtime: JobDeliveryRuntimePort, + jobs: MutableMapping[str, PrismJobContext], + retained: RetainedJobIndex, + preparation: JobPreparationPort | None = None, + tip_authority: TipAuthorityPort | None = None, + payout: PayoutDeliveryPort | None = None, + initial_runtime: InitialJobRuntimePort | None = None, + hooks: DeliveryCompatibilityHooks | None = None, + progress: ProgressDeliveryPort | None = None, + initial_state: InitialJobState | None = None, + job_counter: int = 0, + delivery_health_updated: Callable[[str], None] | None = None, + ) -> None: + self.registry = registry + self.runtime = runtime + self.jobs = jobs + self.retained = retained + self.preparation = preparation + self.tip_authority = tip_authority + self.payout = payout + self.initial_runtime = initial_runtime + self.hooks = hooks + self.progress = progress + self.delivery_health_updated = delivery_health_updated + self.initial_state = initial_state or InitialJobState( + InitialJobConfig(max_pending=0, timeout_seconds=0.0) + ) + self._job_counter_lock = threading.Lock() + self._job_counter = int(job_counter) + self._send_override_local = threading.local() + self._initial_executor_lock = threading.Lock() + self._initial_executor: _BoundedPriorityExecutor | None = None + self._initial_executor_shutdown = False + + def adopt_ports( + self, + *, + preparation: JobPreparationPort, + tip_authority: TipAuthorityPort, + payout: PayoutDeliveryPort, + initial_runtime: InitialJobRuntimePort, + hooks: DeliveryCompatibilityHooks, + progress: ProgressDeliveryPort, + initial_state: InitialJobState, + delivery_health_updated: Callable[[str], None] | None = None, + ) -> None: + self.preparation = preparation + self.tip_authority = tip_authority + self.payout = payout + self.initial_runtime = initial_runtime + self.hooks = hooks + self.progress = progress + self.initial_state = initial_state + self.delivery_health_updated = delivery_health_updated + + @staticmethod + def _required(port: object | None, name: str) -> Any: + if port is None: + raise RuntimeError(f"job delivery {name} port is not configured") + return port + + def initial_snapshot(self) -> InitialJobSnapshot: + with self.registry.lock: + return self.initial_state.snapshot() + + def initial_executor(self) -> _BoundedPriorityExecutor: + with self._initial_executor_lock: + if self._initial_executor_shutdown: + raise RuntimeError("initial job executor is shut down") + executor = self._initial_executor + if executor is None: + executor = _BoundedPriorityExecutor( + max_workers=self.initial_state.config.max_workers, + max_queue_size=self.initial_state.config.max_pending, + thread_name_prefix="prism-initial-job-delivery", + ) + self._initial_executor = executor + return executor + + def initial_executor_stats(self) -> tuple[int, int]: + with self._initial_executor_lock: + executor = self._initial_executor + return (0, 0) if executor is None else executor.stats() + + def cancel_initial_future(self, future: Future[Any]) -> bool: + with self._initial_executor_lock: + executor = self._initial_executor + reclaimed = bool(executor is not None and executor.cancel(future)) + if executor is None: + future.cancel() + if reclaimed: + with self.registry.lock: + self.initial_state.queue_capacity_reclaimed_count += 1 + return reclaimed + + def shutdown_initial_executor(self) -> None: + self.shutdown_initial_jobs() + with self._initial_executor_lock: + executor = self._initial_executor + self._initial_executor = None + self._initial_executor_shutdown = True + if executor is not None: + executor.shutdown(wait=True, cancel_futures=True) + + @property + def job_counter(self) -> int: + with self._job_counter_lock: + return self._job_counter + + def adopt_job_counter(self, value: int) -> None: + with self._job_counter_lock: + self._job_counter = int(value) + + def next_job_id(self) -> str: + with self._job_counter_lock: + self._job_counter += 1 + return f"prism-{self._job_counter}" + + def initial_request_current_locked(self, request: PendingInitialJob) -> bool: + initial_runtime = self._required(self.initial_runtime, "initial runtime") + return self.initial_state.tracker.request_current_locked( + request, + clients=self.registry.clients, + stopping=initial_runtime.stopping(), + ) + + def initial_request_cancelled(self, request: PendingInitialJob) -> bool: + initial_runtime = self._required(self.initial_runtime, "initial runtime") + if request.cancelled.is_set() or initial_runtime.stopping(): + return True + with self.registry.lock: + return not self.initial_request_current_locked(request) + + def cancel_initial_job_locked( + self, + client: ClientState, + *, + count: bool, + ) -> PendingInitialJob | None: + request = self.initial_state.tracker.cancel_locked(client) + if request is not None and count: + self.initial_state.cancelled_count += 1 + return request + + def cancel_initial_job( + self, + client: ClientState, + *, + count: bool, + ) -> PendingInitialJob | None: + with self.registry.lock: + request = self.cancel_initial_job_locked(client, count=count) + if request is not None and request.future is not None: + self.cancel_initial_future(request.future) + return request + + def shutdown_initial_jobs_locked(self) -> tuple[PendingInitialJob, ...]: + return self.initial_state.tracker.shutdown_locked() + + def shutdown_initial_jobs(self) -> tuple[PendingInitialJob, ...]: + with self.registry.lock: + pending = self.shutdown_initial_jobs_locked() + self.initial_state.cancelled_count += len(pending) + for request in pending: + if request.future is not None: + self.cancel_initial_future(request.future) + return pending + + def current_job_source(self) -> CurrentJobSource: + payout = self._required(self.payout, "payout") + tip = self._required(self.tip_authority, "tip authority") + return CurrentJobSource( + payout_generation=payout.generation(), + current_tip=tip.current_tip_locked(), + published_template=tip.published_template_locked(), + ) + + def reauthorization_has_capacity(self, client: ClientState) -> bool: + """Preserve live work when no superseding initial-job slot exists.""" + source = self.current_job_source() + with self.registry.lock: + return not ( + client not in self.initial_state.pending + and len(self.initial_state.pending) + >= self.initial_state.config.max_pending + and self.client_has_current_tip_job_locked(client, source) + ) + + @staticmethod + def client_has_current_tip_job_locked( + client: ClientState, + source: CurrentJobSource, + ) -> bool: + context = client.active_job + if context is None: + return False + generation = source.payout_generation + if int(getattr(context, "payout_state_generation", generation)) != generation: + return False + current_tip = source.current_tip + if current_tip is None: + return False + snapshot = source.published_template + if snapshot is None: + return str(context.template.get("previousblockhash", "")) == current_tip + if snapshot.bestblockhash != current_tip or snapshot.template_artifacts is None: + return False + return bool( + str(context.template.get("previousblockhash", "")) == current_tip + and getattr(context, "template_fingerprint", None) + == snapshot.template_fingerprint + and int(getattr(context, "template_generation", 0)) + == snapshot.template_generation + and context.template is snapshot.template_artifacts.template + and int(getattr(context, "connection_id", client.connection_id)) + == client.connection_id + and int(getattr(context, "authorization_generation", 0)) + == int(client.authorization_generation) + and int(getattr(context, "difficulty_generation", 0)) + == int(client.difficulty_generation) + ) + + @staticmethod + def client_has_delivered_work_locked(client: ClientState) -> bool: + """Return whether a socket write completed for any usable job.""" + + return bool( + client.tip_work_delivered is not None + or client._progress_delivered_context is not None + ) + + def note_initial_job_delivered( + self, + client: ClientState, + *, + validated_current: bool = False, + ) -> None: + source = None if validated_current else self.current_job_source() + future: Future[bool] | None = None + with self.registry.lock: + if ( + source is not None + and not self.client_has_current_tip_job_locked(client, source) + ): + return + request = self.initial_state.pending.pop(client, None) + if request is None: + return + request.cancelled.set() + future = request.future + delivered = time.monotonic() + self.initial_state.sent_count += 1 + self.initial_state.delivery_latency_seconds_sum += max( + 0.0, + delivered - request.requested_monotonic, + ) + self.initial_state.delivery_latency_count += 1 + self.initial_state.last_delivery_monotonic = delivered + if future is not None: + self.cancel_initial_future(future) + + def schedule_initial_job(self, client: ClientState) -> bool: + initial_runtime = self._required(self.initial_runtime, "initial runtime") + hooks = self._required(self.hooks, "compatibility hooks") + maybe_send_override = hooks.maybe_send_override() + if callable(maybe_send_override): + return bool(maybe_send_override(client, clean_jobs=True)) + + now = time.monotonic() + reject = False + deferred = False + request: PendingInitialJob | None = None + superseded_future: Future[bool] | None = None + already_current = False + source = self.current_job_source() + with self.registry.lock: + if ( + not client.subscribed + or not client.authorized + or client.worker is None + or client.closing + ): + return True + generation = int(client.authorization_generation) + difficulty_generation = int(client.difficulty_generation) + existing = self.initial_state.pending.get(client) + if ( + existing is not None + and existing.connection_id == client.connection_id + and existing.authorization_generation == generation + and existing.difficulty_generation == difficulty_generation + and existing.worker == client.worker + ): + self.initial_state.coalesced_count += 1 + return True + if existing is not None: + existing.cancelled.set() + superseded_future = existing.future + self.initial_state.cancelled_count += 1 + self.initial_state.superseded_count += 1 + if self.client_has_current_tip_job_locked(client, source): + if existing is not None: + self.initial_state.pending.pop(client, None) + already_current = True + if not already_current: + if ( + existing is None + and len(self.initial_state.pending) + >= self.initial_state.config.max_pending + ): + self.initial_state.queue_rejection_count += 1 + reject = True + else: + timeout = self.initial_state.config.timeout_seconds + predecessor = None + if existing is not None: + for candidate in (existing.future, existing.predecessor): + if candidate is not None and not candidate.done(): + predecessor = candidate + break + request = PendingInitialJob( + client=client, + connection_id=client.connection_id, + authorization_generation=generation, + difficulty_generation=difficulty_generation, + worker=client.worker, + requested_monotonic=now, + deadline_monotonic=now + timeout if timeout > 0 else None, + predecessor=predecessor, + ) + self.initial_state.pending[client] = request + deferred = predecessor is not None + if superseded_future is not None: + self.cancel_initial_future(superseded_future) + if already_current: + return True + if reject or request is None: + initial_runtime.disconnect(client) + return False + if deferred: + return True + submit_override = hooks.submit_initial_override() + if callable(submit_override): + return bool(submit_override(request)) + return self.submit_initial_job_request(request) + + def submit_initial_job_request(self, request: PendingInitialJob) -> bool: + initial_runtime = self._required(self.initial_runtime, "initial runtime") + hooks = self._required(self.hooks, "compatibility hooks") + client = request.client + run_override = hooks.run_initial_override() + run = ( + run_override + if callable(run_override) + else lambda pending: self.run_initial_job(pending) + ) + try: + future = self.initial_executor().submit( + run, + request, + priority=PRISM_DELIVERY_PRIORITY_INITIAL, + ) + except (_DeliveryQueueFull, RuntimeError): + disconnect = False + with self.registry.lock: + if self.initial_state.pending.get(client) is request: + self.initial_state.pending.pop(client, None) + request.cancelled.set() + self.initial_state.queue_rejection_count += 1 + disconnect = True + if disconnect: + initial_runtime.disconnect(client) + return not disconnect + cancel_future = False + with self.registry.lock: + if self.initial_state.pending.get(client) is request: + request.future = future + else: + cancel_future = True + future.add_done_callback( + lambda completed: self.initial_job_future_finished(request, completed) + ) + if cancel_future: + self.cancel_initial_future(future) + return True + + def initial_job_future_finished( + self, + request: PendingInitialJob, + future: Future[bool], + ) -> None: + initial_runtime = self._required(self.initial_runtime, "initial runtime") + preparation = self._required(self.preparation, "preparation") + hooks = self._required(self.hooks, "compatibility hooks") + delivered = False + if not future.cancelled(): + try: + delivered = bool(future.result()) + except Exception: + preparation.record_failure() + print( + "prism coordinator: initial job task failed " + f"connection={request.client.connection_id}", + flush=True, + ) + traceback.print_exc() + + disconnect = False + replacement: PendingInitialJob | None = None + source = self.current_job_source() if delivered else None + with self.registry.lock: + current = self.initial_state.pending.get(request.client) + if current is not request: + if ( + current is not None + and current.future is None + and current.predecessor is future + ): + current.predecessor = None + replacement = current + elif ( + source is not None + and self.initial_request_current_locked(request) + and self.client_has_current_tip_job_locked(request.client, source) + ): + self.initial_state.pending.pop(request.client, None) + request.cancelled.set() + self.initial_state.last_delivery_monotonic = time.monotonic() + else: + self.initial_state.pending.pop(request.client, None) + request.cancelled.set() + if ( + request.deadline_monotonic is not None + and request.deadline_monotonic <= time.monotonic() + ): + request.client.closing = True + self.initial_state.timeout_count += 1 + self.initial_state.cancelled_count += 1 + else: + self.initial_state.failed_count += 1 + disconnect = True + if replacement is not None: + submit_override = hooks.submit_initial_override() + if callable(submit_override): + submit_override(replacement) + else: + self.submit_initial_job_request(replacement) + if disconnect: + initial_runtime.disconnect(request.client) + + def run_initial_job(self, request: PendingInitialJob) -> bool: + initial_runtime = self._required(self.initial_runtime, "initial runtime") + preparation = self._required(self.preparation, "preparation") + tip = self._required(self.tip_authority, "tip authority") + hooks = self._required(self.hooks, "compatibility hooks") + retry_delay = 0.05 + last_failure_log_monotonic: float | None = None + + def retry_later() -> bool: + nonlocal retry_delay + request.cancelled.wait(retry_delay) + retry_delay = min(1.0, retry_delay * 2) + return not self.initial_request_cancelled(request) + + try: + while not self.initial_request_cancelled(request): + try: + if not preparation.ensure_reorg_current(): + if not retry_later(): + return False + continue + artifacts = preparation.issuance_artifacts() + if self.initial_request_cancelled(request): + return False + bundle = preparation.shared_bundle( + artifacts, + request.worker, + cancelled=lambda: ( + self.initial_request_cancelled(request) + or not preparation.artifacts_current(artifacts) + ), + request_source="initial", + ) + live_tip = tip.live_tip() + if artifacts.previousblockhash != live_tip: + tip.observe_tip(live_tip) + published = tip.published_authority() + pinned_authoritative = bool( + published is not None + and artifacts.previousblockhash == published[0] + and tip.published_authoritative(time.monotonic()) + ) + if not pinned_authoritative: + preparation.clear_artifacts(artifacts) + if not retry_later(): + return False + continue + except (JobBuildWaiterCancelled, TemplateRefreshBlocked): + if self.initial_request_cancelled(request): + return False + if not retry_later(): + return False + continue + except Exception: + preparation.record_failure() + now = time.monotonic() + if ( + last_failure_log_monotonic is None + or now - last_failure_log_monotonic >= 5.0 + ): + last_failure_log_monotonic = now + print( + "prism coordinator: initial job preparation failed " + f"connection={request.client.connection_id}; retrying", + flush=True, + ) + traceback.print_exc() + if not retry_later(): + return False + continue + deliver_override = hooks.deliver_initial_override() + if callable(deliver_override): + delivered = deliver_override(request, artifacts, bundle) + else: + delivered = self.deliver_initial_bundle(request, artifacts, bundle) + if delivered is None: + if not retry_later(): + return False + continue + return bool(delivered) + return False + except OSError: + initial_runtime.disconnect(request.client) + return False + + @staticmethod + def _acquire_client_job_lock( + client: ClientState, + cancelled: Callable[[], bool], + ) -> bool: + while not cancelled(): + if client.job_update_lock.acquire(timeout=0.1): + return True + return False + + def _prune_retained_locked( + self, + *, + authority: RetentionAuthority, + now: float | None = None, + force: bool, + ) -> None: + self.retained.prune( + current_tip=authority.current_tip, + current_tip_first_delivery=authority.current_tip_first_delivery, + cached_parent=authority.cached_parent, + now=time.monotonic() if now is None else now, + force=force, + ) + + def prune_retained( + self, + *, + now: float | None = None, + force: bool = True, + ) -> None: + tip = self._required(self.tip_authority, "tip authority") + authority = tip.retention_authority_locked() + with self.registry.lock: + self._prune_retained_locked( + authority=authority, + now=now, + force=force, + ) + + def bury_retained( + self, + client: ClientState, + job_id: str, + *, + now: float | None = None, + prune: bool = True, + ) -> None: + tip = self._required(self.tip_authority, "tip authority") + authority = tip.retention_authority_locked() + with self.registry.lock: + context = self.jobs.get(job_id) + if context is None: + return + self.retained.retain( + client, + job_id, + context, + current_tip=authority.current_tip, + now=now, + ) + if prune: + self._prune_retained_locked( + authority=authority, + now=now, + force=False, + ) + + def retained_entry( + self, + client: ClientState, + job_id: str, + *, + now: float | None = None, + ) -> EvictedJobEntry | None: + tip = self._required(self.tip_authority, "tip authority") + hooks = self._required(self.hooks, "compatibility hooks") + existing = self.retained.peek(job_id) + classify_override = hooks.retained_classify_override() + if existing is not None and callable(classify_override): + classify_override(existing) + authority = tip.retention_authority_locked() + with self.registry.lock: + return self.retained.lookup( + client, + job_id, + current_tip=authority.current_tip, + current_tip_first_delivery=authority.current_tip_first_delivery, + cached_parent=authority.cached_parent, + now=time.monotonic() if now is None else now, + ) + + def note_retained_submit(self, credit_policy: str | None) -> None: + self.retained.note_submit(credit_policy) + + def retained_job_class(self, entry: EvictedJobEntry) -> str: + tip = self._required(self.tip_authority, "tip authority") + authority = tip.retention_authority_locked() + return self.retained.job_class(entry, current_tip=authority.current_tip) + + def deliver_initial_bundle( + self, + request: PendingInitialJob, + artifacts: CachedTemplateArtifacts, + bundle: CachedJobBundle, + ) -> bool | None: + preparation = self._required(self.preparation, "preparation") + payout = self._required(self.payout, "payout") + tip = self._required(self.tip_authority, "tip authority") + client = request.client + + def cancelled() -> bool: + return self.initial_request_cancelled(request) + + if not self._acquire_client_job_lock(client, cancelled): + return False + try: + if cancelled(): + return False + if not preparation.artifacts_current(artifacts): + return None + gate_started = time.monotonic() + with payout.initial_admission( + cancelled, + generation=bundle.payout_state_generation, + ) as admitted: + payout.observe_admission( + admitted, + generation=bundle.payout_state_generation, + fallback_wait_seconds=time.monotonic() - gate_started, + ) + if not admitted or cancelled(): + return False + if ( + bundle.payout_state_generation != payout.generation() + or not preparation.artifacts_current(artifacts) + ): + return None + with client_vardiff_lock(client): + context = self._stamp_for_client( + client, + bundle, + clean_jobs=True, + ) + retention_authority = tip.retention_authority_locked() + source_authority = DeliverySourceAuthority( + kind="artifacts", + payout_generation=bundle.payout_state_generation, + template_generation=artifacts.generation, + observation_sequence=0, + template_fingerprint=artifacts.fingerprint, + artifacts=artifacts, + ) + with client_vardiff_lock(client), self.registry.lock: + if not self.source_authority_current_locked( + source_authority, + context, + ): + return None + if not self.initial_request_current_locked(request): + return False + if not self.context_matches_client_locked(client, context): + return False + authority = self.capture_authority( + client, + context, + expected_active_job=client.active_job, + ) + if not self.authority_current_locked( + client, + authority, + expected_active_job=client.active_job, + ): + return False + self.register_locked( + client, + context, + clean_jobs=True, + current_tip=retention_authority.current_tip, + ) + self._prune_retained_locked( + authority=retention_authority, + force=False, + ) + + self.send_update(client, context.job, split_send=self._split_send()) + delivered_monotonic = time.monotonic() + if not self.complete_delivery( + client, + authority, + context, + delivered_monotonic, + initial_request=request, + source_authorities=(source_authority,), + ): + return False + mark_delivered = getattr(admitted, "mark_delivered", None) + if callable(mark_delivered): + mark_delivered() + self._apply_delivered_difficulty(client, context.job) + self.note_tip_work_delivered( + client, + str(context.template["previousblockhash"]), + ) + payout.record_first_delivery( + bundle.payout_state_generation, + delivered_monotonic, + ) + self.note_initial_job_delivered(client, validated_current=True) + return True + finally: + client.job_update_lock.release() + + def sweep_initial_job_timeouts(self, *, now: float | None = None) -> int: + initial_runtime = self._required(self.initial_runtime, "initial runtime") + now = time.monotonic() if now is None else now + with self.registry.lock: + timed_out = self.initial_state.tracker.expire_locked(now) + self.initial_state.timeout_count += len(timed_out) + self.initial_state.cancelled_count += len(timed_out) + for request in timed_out: + if request.future is not None: + self.cancel_initial_future(request.future) + initial_runtime.disconnect(request.client) + return len(timed_out) + + def initial_job_timeout_loop(self) -> None: + initial_runtime = self._required(self.initial_runtime, "initial runtime") + while not initial_runtime.wait(1.0): + self.sweep_initial_job_timeouts() + + def _split_send(self) -> bool: + hooks = self._required(self.hooks, "compatibility hooks") + return hooks.split_send_enabled() + + def _apply_delivered_difficulty( + self, + client: ClientState, + job: direct_stratum.DirectQbitStratumJob, + ) -> None: + hooks = self._required(self.hooks, "compatibility hooks") + override = hooks.apply_difficulty_override() + if callable(override): + override(client, job) + return + self.apply_job_difficulty( + client, + job, + config=self.runtime.vardiff_config(client), + ) + + def _stamp_for_client( + self, + client: ClientState, + bundle: CachedJobBundle, + *, + clean_jobs: bool, + ) -> PrismJobContext: + hooks = self._required(self.hooks, "compatibility hooks") + override = hooks.stamp_job_override() + if callable(override): + return override(client, bundle, clean_jobs=clean_jobs) + return self.stamp(client, bundle, clean_jobs=clean_jobs) + + def build_job_for_client( + self, + client: ClientState, + *, + clean_jobs: bool, + ) -> PrismJobContext: + preparation = self._required(self.preparation, "preparation") + if client.worker is None: + raise StratumError(20, "client is not authorized") + artifacts = ( + preparation.retained_artifacts() + or preparation.issuance_artifacts() + ) + return self.build_job_for_client_from_artifacts( + client, + artifacts, + clean_jobs=clean_jobs, + ) + + def build_job_for_client_from_artifacts( + self, + client: ClientState, + artifacts: CachedTemplateArtifacts, + *, + clean_jobs: bool, + ) -> PrismJobContext: + preparation = self._required(self.preparation, "preparation") + phases = preparation.phases() + while True: + worker = client.worker + if worker is None: + raise StratumError(20, "client is not authorized") + cached_bundle = preparation.shared_bundle(artifacts, worker) + current_worker = client.worker + if not cached_bundle.collection_only or current_worker == worker: + break + stamp_started = time.monotonic() + context = self._stamp_for_client( + client, + cached_bundle, + clean_jobs=clean_jobs, + ) + phases["stamp"] = phases.get("stamp", 0.0) + ( + time.monotonic() - stamp_started + ) + return context + + def maybe_send_job( + self, + client: ClientState, + *, + clean_jobs: bool, + raise_on_reorg_failure: bool = False, + raise_on_build_failure: bool = False, + tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, + tip_refresh_observation_sequence: int | None = None, + ) -> bool: + with client.job_update_lock: + return self.maybe_send_job_locked( + client, + clean_jobs=clean_jobs, + raise_on_reorg_failure=raise_on_reorg_failure, + raise_on_build_failure=raise_on_build_failure, + tip_refresh_snapshot=tip_refresh_snapshot, + tip_refresh_observation_sequence=tip_refresh_observation_sequence, + ) + + def idle_authority_current_locked( + self, + client: ClientState, + authority: IdleDeliveryAuthority, + ) -> bool: + return bool( + client in self.registry.clients + and not client.closing + and self.client_can_receive_jobs(client) + and client.connection_id == authority.connection_id + and client.worker == authority.worker + and client.active_job is authority.expected_active_job + and client.vardiff_window_started_monotonic + == authority.expected_window_started + and client.vardiff_window_accepted == 0 + and client.vardiff_window_submitted == 0 + and client.pending_share_difficulty == authority.pending_difficulty + ) + + @staticmethod + def commit_idle_authority_locked( + client: ClientState, + authority: IdleDeliveryAuthority, + ) -> None: + reset_at = time.monotonic() + client.vardiff_window_started_monotonic = reset_at + client.vardiff_window_accepted = 0 + client.vardiff_window_submitted = 0 + client.vardiff_window_work = Decimal("0") + authority.committed_reset_monotonic = reset_at + + def maybe_send_job_locked( + self, + client: ClientState, + *, + clean_jobs: bool, + raise_on_reorg_failure: bool = False, + raise_on_build_failure: bool = False, + tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, + tip_refresh_observation_sequence: int | None = None, + prepared_bundle: CachedJobBundle | None = None, + idle_authority: IdleDeliveryAuthority | None = None, + prepared_bundle_allow_uncached: bool = False, + ) -> bool: + preparation = self._required(self.preparation, "preparation") + tip = self._required(self.tip_authority, "tip authority") + payout = self._required(self.payout, "payout") + hooks = self._required(self.hooks, "compatibility hooks") + if not client.subscribed or not client.authorized or client.worker is None: + return False + started = time.monotonic() + phases = preparation.phases() + phases.clear() + if hooks.hot_path_logging_enabled(): + print( + "prism coordinator: building job " + f"connection={client.connection_id} username={client.username}", + flush=True, + ) + phase_started = time.monotonic() + guarded_refresh = tip_refresh_snapshot is not None + if guarded_refresh != (tip_refresh_observation_sequence is not None): + raise ValueError("tip refresh snapshot and observation sequence must be paired") + if prepared_bundle is not None and guarded_refresh: + raise ValueError("prepared idle bundles cannot be combined with tip refresh guards") + if prepared_bundle_allow_uncached and prepared_bundle is None: + raise ValueError("uncached prepared delivery requires a prepared bundle") + if prepared_bundle is not None: + pass + elif guarded_refresh: + assert tip_refresh_snapshot is not None + assert tip_refresh_observation_sequence is not None + with self.registry.lock: + refresh_current = tip.snapshot_current_locked( + tip_refresh_snapshot, + tip_refresh_observation_sequence, + ) + if not refresh_current: + tip.schedule_retry() + raise TemplateRefreshSuperseded( + "tip refresh snapshot was superseded before client job build" + ) + try: + chain_view_untrusted = bool( + hooks.reorg_reconciler_enabled() + and preparation.chain_view_untrusted() + ) + except Exception as exc: + tip.schedule_retry() + raise TemplateRefreshBlocked( + "qbit chain trust check failed before sequential client job build" + ) from exc + if chain_view_untrusted: + tip.schedule_retry() + raise TemplateRefreshBlocked( + "qbit chain view became untrusted before sequential client job build" + ) + else: + try: + if not preparation.ensure_reorg_current(): + if raise_on_reorg_failure: + raise TemplateRefreshBlocked( + "qbit chain view became untrusted before client job build" + ) + return False + except TemplateRefreshBlocked: + raise + except Exception as exc: + print( + "prism coordinator: reorg reconciliation failed before job build " + f"connection={client.connection_id} username={client.username}; " + "skipping this job", + flush=True, + ) + traceback.print_exc() + if raise_on_reorg_failure: + raise TemplateRefreshBlocked( + "reorg reconciliation failed before client job build" + ) from exc + return False + phases["reorg"] = time.monotonic() - phase_started + build_override = hooks.build_job_override() + built_from_guarded_artifacts = bool( + guarded_refresh + and tip_refresh_snapshot.template_artifacts is not None + and build_override is None + ) + selected_source_artifacts: CachedTemplateArtifacts | None = None + try: + if prepared_bundle is not None: + context = self._stamp_for_client( + client, + prepared_bundle, + clean_jobs=clean_jobs, + ) + elif built_from_guarded_artifacts: + assert tip_refresh_snapshot is not None + assert tip_refresh_snapshot.template_artifacts is not None + context = self.build_job_for_client_from_artifacts( + client, + tip_refresh_snapshot.template_artifacts, + clean_jobs=clean_jobs, + ) + elif callable(build_override): + context = build_override(client, clean_jobs=clean_jobs) + else: + selected_source_artifacts = ( + preparation.retained_artifacts() + or preparation.issuance_artifacts() + ) + context = self.build_job_for_client_from_artifacts( + client, + selected_source_artifacts, + clean_jobs=clean_jobs, + ) + except TemplateRefreshBlocked: + tip.schedule_retry() + if guarded_refresh or raise_on_reorg_failure or raise_on_build_failure: + raise + return False + except Exception as exc: + preparation.record_failure() + print( + "prism coordinator: job build failed " + f"connection={client.connection_id} username={client.username}; " + "keeping client connected and skipping this template", + flush=True, + ) + traceback.print_exc() + if raise_on_build_failure: + raise JobBuildFailed( + f"job build failed for connection {client.connection_id}" + ) from exc + return False + + payout_snapshot = payout.snapshot() + current_payout_generation = payout_snapshot.generation + published_tip = payout_snapshot.published.source_tip_hash + publication_blocked = payout_snapshot.publication_blocked + context_payout_generation = int( + getattr(context, "payout_state_generation", current_payout_generation) + ) + context_template = getattr(context, "template", None) + context_parent = ( + str(context_template.get("previousblockhash", "")) + if isinstance(context_template, dict) + else "" + ) + published_authority = tip.published_authority() + published_authoritative = tip.published_authoritative(time.monotonic()) + pinned_published_delivery = bool( + context_parent + and published_authority is not None + and context_parent == published_authority[0] + and published_authoritative + ) + lapsed_live_validated = False + if ( + not guarded_refresh + and context_parent + and published_authority is not None + and not published_authoritative + ): + try: + lapsed_live_tip = tip.live_tip() + except Exception: + lapsed_live_tip = None + if lapsed_live_tip is not None: + tip.observe_tip(lapsed_live_tip) + if context_parent != lapsed_live_tip: + tip.schedule_retry() + return False + lapsed_live_validated = True + priority_delivery = ( + not publication_blocked + and context_payout_generation == current_payout_generation + and ( + published_tip is None + or context_parent == published_tip + or pinned_published_delivery + ) + ) + payout_gate_started = time.monotonic() + with payout.admission( + lambda: context_payout_generation + != payout.generation(), + generation=context_payout_generation, + priority=priority_delivery, + ) as payout_admitted: + payout_gate_wait = max(0.0, time.monotonic() - payout_gate_started) + phases["payout_gate"] = phases.get("payout_gate", 0.0) + payout_gate_wait + payout.observe_admission( + payout_admitted, + generation=context_payout_generation, + fallback_wait_seconds=payout_gate_wait, + ) + if not payout_admitted: + tip.schedule_retry() + if guarded_refresh: + raise TemplateRefreshSuperseded( + "payout state changed during client job build" + ) + return False + + authority: DeliveryAuthority | None = None + retention_authority = tip.retention_authority_locked() + published_commit = tip.published_authority() + published_commit_authoritative = ( + published_commit is not None + and tip.published_authoritative(time.monotonic()) + ) + if guarded_refresh: + assert tip_refresh_snapshot is not None + assert tip_refresh_observation_sequence is not None + guarded_commit_current = tip.snapshot_current_locked( + tip_refresh_snapshot, + tip_refresh_observation_sequence, + ) + if not guarded_commit_current: + tip.schedule_retry() + raise TemplateRefreshSuperseded( + "tip refresh snapshot was superseded during client job build" + ) + elif ( + published_commit is not None + and context_parent + and ( + ( + published_commit_authoritative + and context_parent != published_commit[0] + ) + or ( + not published_commit_authoritative + and not lapsed_live_validated + ) + ) + ): + tip.schedule_retry() + return False + + source_authority: DeliverySourceAuthority | None + if prepared_bundle is not None: + source_authority = None + elif guarded_refresh: + assert tip_refresh_snapshot is not None + assert tip_refresh_observation_sequence is not None + source_authority = DeliverySourceAuthority( + kind="tip_snapshot", + payout_generation=context_payout_generation, + template_generation=tip_refresh_snapshot.template_generation, + observation_sequence=tip_refresh_observation_sequence, + template_fingerprint=( + tip_refresh_snapshot.template_fingerprint + ), + snapshot=tip_refresh_snapshot, + ) + elif selected_source_artifacts is not None: + source_authority = DeliverySourceAuthority( + kind="artifacts", + payout_generation=context_payout_generation, + template_generation=selected_source_artifacts.generation, + observation_sequence=0, + template_fingerprint=selected_source_artifacts.fingerprint, + artifacts=selected_source_artifacts, + ) + else: + source_authority = DeliverySourceAuthority( + kind="published_tip", + payout_generation=context_payout_generation, + template_generation=int( + getattr(context, "template_generation", 0) + ), + observation_sequence=0, + template_fingerprint=getattr( + context, + "template_fingerprint", + None, + ), + context_parent=context_parent, + lapsed_live_validated=lapsed_live_validated, + ) + + def commit_context_locked() -> bool: + nonlocal authority + if client not in self.registry.clients or client.closing: + return False + if not self.context_matches_client_locked(client, context): + return False + if isinstance(context, PrismJobContext): + if source_authority is None or not ( + self.source_authority_current_locked( + source_authority, + context, + ) + ): + return False + if ( + idle_authority is not None + and not self.idle_authority_current_locked( + client, + idle_authority, + ) + ): + return False + if guarded_refresh: + assert tip_refresh_snapshot is not None + artifacts = tip_refresh_snapshot.template_artifacts + if built_from_guarded_artifacts and artifacts is not None and ( + context.template is not artifacts.template + or context.template_fingerprint != artifacts.fingerprint + or context.template_generation != artifacts.generation + ): + raise TemplateRefreshBlocked( + "client job build did not use the guarded refresh artifacts" + ) + authority = self.capture_authority( + client, + context, + expected_active_job=client.active_job, + ) + self.register_locked( + client, + context, + clean_jobs=clean_jobs, + current_tip=retention_authority.current_tip, + ) + if clean_jobs: + self._prune_retained_locked( + authority=retention_authority, + force=False, + ) + if idle_authority is not None: + self.commit_idle_authority_locked( + client, + idle_authority, + ) + return True + + if prepared_bundle is not None: + admitted_source = preparation.admit_idle_bundle_source( + client, + prepared_bundle, + allow_uncached=prepared_bundle_allow_uncached, + ) + if ( + admitted_source is None + or admitted_source.bundle is not prepared_bundle + or admitted_source.cache_identity != prepared_bundle.key + or admitted_source.allow_uncached + != prepared_bundle_allow_uncached + ): + return False + selected_source_artifacts = admitted_source.artifacts + if not tip.ensure_artifacts_parent_observed( + selected_source_artifacts + ): + return False + source_authority = DeliverySourceAuthority( + kind="artifacts", + payout_generation=context_payout_generation, + template_generation=selected_source_artifacts.generation, + observation_sequence=0, + template_fingerprint=selected_source_artifacts.fingerprint, + artifacts=selected_source_artifacts, + ) + with client_vardiff_lock(client), self.registry.lock: + if not commit_context_locked(): + return False + else: + with client_vardiff_lock(client), self.registry.lock: + if not commit_context_locked(): + return False + phase_started = time.monotonic() + self.send_update(client, context.job, split_send=self._split_send()) + delivered_monotonic = time.monotonic() + if authority is None or not self.complete_delivery( + client, + authority, + context, + delivered_monotonic, + source_authorities=( + (source_authority,) + if ( + isinstance(context, PrismJobContext) + and source_authority is not None + ) + else () + ), + ): + return False + mark_delivered = getattr(payout_admitted, "mark_delivered", None) + if callable(mark_delivered): + mark_delivered() + self._apply_delivered_difficulty(client, context.job) + self.note_tip_work_delivered( + client, + str(context.template["previousblockhash"]), + ) + payout.record_first_delivery( + context_payout_generation, + delivered_monotonic, + ) + tip.consume_retained_refresh(context) + self.note_initial_job_delivered( + client, + validated_current=guarded_refresh, + ) + phases["send"] = delivered_monotonic - phase_started + elapsed = time.monotonic() - started + preparation.observe_elapsed(elapsed, phases) + if hooks.hot_path_logging_enabled(): + phase_report = ",".join( + f"{phase}:{seconds:.3f}" for phase, seconds in phases.items() + ) + print( + "prism coordinator: sent job " + f"connection={client.connection_id} username={client.username} " + f"job={context.job.job_id} collection={context.collection_only} " + f"elapsed={elapsed:.3f}s phases={phase_report}", + flush=True, + ) + return True + + def send_prepared_job( + self, + client: ClientState, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + validation_token: TipRefreshValidationToken, + expected_connection_id: int, + expected_active_job: PrismJobContext | None, + cancel_event: FanoutCancellation | None = None, + submitted_monotonic: float | None = None, + ) -> RefreshResult: + preparation = self._required(self.preparation, "preparation") + tip = self._required(self.tip_authority, "tip authority") + payout = self._required(self.payout, "payout") + hooks = self._required(self.hooks, "compatibility hooks") + worker_started = time.monotonic() + started = worker_started if submitted_monotonic is None else submitted_monotonic + phases = preparation.phases() + phases.clear() + + def cancelled() -> bool: + return tip.prepared_obsolete( + validation_token, + bundle, + snapshot, + cancel_event, + ) or client.closing + + phases["executor_queue"] = max(0.0, worker_started - started) + client_lock_started = worker_started + client_lock_acquired = False + client_lock_attempted = False + try: + while True: + with self.registry.lock: + if ( + client not in self.registry.clients + or client.connection_id != expected_connection_id + or client.closing + ): + return RefreshResult("disconnected") + if cancelled(): + phases["client_lock"] = max( + 0.0, + time.monotonic() - client_lock_started, + ) + tip.record_cancellation( + "client_lock" if client_lock_attempted else "executor_queue" + ) + return RefreshResult("skipped") + client_lock_attempted = True + client_lock_acquired = client.job_update_lock.acquire( + timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS + ) + if client_lock_acquired: + break + phases["client_lock"] = max( + 0.0, + time.monotonic() - client_lock_started, + ) + if cancelled(): + tip.record_cancellation("client_lock") + return RefreshResult("skipped") + refresh_source = self.refresh_source() + with self.registry.lock: + if ( + client not in self.registry.clients + or client.connection_id != expected_connection_id + ): + return RefreshResult("disconnected") + if ( + not self.client_can_receive_jobs(client) + or self.intervening_supersedes( + client.active_job, + expected_active_job, + snapshot, + refresh_source, + ) + or not self.client_needs_refresh_locked( + client, snapshot, refresh_source + ) + ): + return RefreshResult("skipped") + payout_gate_started = time.monotonic() + with payout.admission( + cancelled, + generation=bundle.payout_state_generation, + priority=True, + ) as payout_admitted: + phases["payout_gate"] = max( + 0.0, + time.monotonic() - payout_gate_started, + ) + payout.observe_admission( + payout_admitted, + generation=bundle.payout_state_generation, + fallback_wait_seconds=phases["payout_gate"], + ) + if not payout_admitted or cancelled(): + tip.record_cancellation("payout_gate") + return RefreshResult("skipped") + fanout_admitted = cancel_event is None or cancel_event.begin_delivery() + if not fanout_admitted: + tip.record_cancellation("payout_gate") + return RefreshResult("skipped") + try: + payout_snapshot = payout.snapshot() + token_current = tip.prepared_token_current_locked( + validation_token, + bundle, + snapshot, + payout_snapshot, + ) + if not token_current: + if cancel_event is not None: + cancel_event.cancel() + return RefreshResult("skipped") + refresh_source = self.refresh_source() + with self.registry.lock: + if ( + client not in self.registry.clients + or client.connection_id != expected_connection_id + ): + return RefreshResult("disconnected") + if ( + not self.client_can_receive_jobs(client) + or self.intervening_supersedes( + client.active_job, + expected_active_job, + snapshot, + refresh_source, + ) + or not self.client_needs_refresh_locked( + client, snapshot, refresh_source + ) + ): + return RefreshResult("skipped") + clean_jobs = self.tip_changed( + client, snapshot, refresh_source + ) + + stamp_started = time.monotonic() + with client_vardiff_lock(client): + context = self._stamp_for_client( + client, + bundle, + clean_jobs=clean_jobs, + ) + phases["stamp"] = time.monotonic() - stamp_started + retention_authority = tip.retention_authority_locked() + refresh_source = self.refresh_source() + source_authority = DeliverySourceAuthority( + kind="tip_token", + payout_generation=validation_token.payout_state_generation, + template_generation=snapshot.template_generation, + observation_sequence=validation_token.observation_sequence, + template_fingerprint=snapshot.template_fingerprint, + snapshot=snapshot, + token=validation_token, + bundle=bundle, + payout_snapshot=payout_snapshot, + ) + authority: DeliveryAuthority | None = None + + def register_current() -> str: + nonlocal authority + with client_vardiff_lock(client), self.registry.lock: + if not self.source_authority_current_locked( + source_authority, + context, + ): + return "source" + if ( + client not in self.registry.clients + or client.connection_id != expected_connection_id + ): + return "disconnected" + if ( + not self.client_can_receive_jobs(client) + or self.intervening_supersedes( + client.active_job, + expected_active_job, + snapshot, + refresh_source, + ) + or not self.client_needs_refresh_locked( + client, snapshot, refresh_source + ) + or not self.context_matches_client_locked( + client, context + ) + ): + return "skipped" + authority = self.capture_authority( + client, + context, + expected_active_job=client.active_job, + ) + self.register_locked( + client, + context, + clean_jobs=clean_jobs, + current_tip=retention_authority.current_tip, + ) + if clean_jobs: + self._prune_retained_locked( + authority=retention_authority, + force=False, + ) + return "registered" + + registration = register_current() + if registration == "source": + if cancel_event is not None: + cancel_event.cancel() + return RefreshResult("skipped") + if registration != "registered": + return RefreshResult(registration) + + socket_send_started = time.monotonic() + try: + self.send_update( + client, + context.job, + split_send=self._split_send(), + ) + finally: + socket_send_finished = time.monotonic() + phases["socket_send"] = max( + 0.0, + socket_send_finished - socket_send_started, + ) + delivered_monotonic = time.monotonic() + if not self.complete_delivery( + client, + authority, + context, + delivered_monotonic, + source_authorities=(source_authority,), + ): + with self.registry.lock: + connected = ( + client in self.registry.clients + and client.connection_id == expected_connection_id + and not client.closing + ) + return RefreshResult( + "skipped" if connected else "disconnected" + ) + mark_delivered = getattr(payout_admitted, "mark_delivered", None) + if callable(mark_delivered): + mark_delivered() + self._apply_delivered_difficulty(client, context.job) + self.note_tip_work_delivered( + client, + str(context.template["previousblockhash"]), + ) + self.note_initial_job_delivered( + client, + validated_current=True, + ) + payout.record_first_delivery( + context.payout_state_generation, + delivered_monotonic, + ) + if hooks.hot_path_logging_enabled(): + print( + "prism coordinator: sent prepared job " + f"connection={client.connection_id} " + f"username={client.username} job={context.job.job_id} " + f"elapsed={delivered_monotonic - started:.3f}s", + flush=True, + ) + return RefreshResult("sent", delivered_monotonic) + finally: + if cancel_event is not None: + cancel_event.end_delivery() + finally: + if client_lock_acquired: + client.job_update_lock.release() + preparation.observe_elapsed( + max(0.0, time.monotonic() - started), + phases, + ) + + def advertise_client_difficulty( + self, + client: ClientState, + target: Decimal, + ) -> bool: + with client.job_update_lock: + return self.advertise_client_difficulty_locked(client, target) + + def advertise_client_difficulty_locked( + self, + client: ClientState, + target: Decimal, + ) -> bool: + hooks = self._required(self.hooks, "compatibility hooks") + initial_runtime = self._required(self.initial_runtime, "initial runtime") + applied_directly = False + schedule_initial = False + with client_vardiff_lock(client): + current = client.pending_share_difficulty or client.share_difficulty + if target == current: + return False + if not (client.subscribed and client.authorized) or ( + client.active_job is None + and hooks.maybe_send_override() is None + ): + client.share_difficulty = target + client.pending_share_difficulty = None + client.difficulty_generation = int(client.difficulty_generation) + 1 + applied_directly = True + schedule_initial = bool( + client.subscribed + and client.authorized + and client.worker is not None + ) + else: + prior_pending = client.pending_share_difficulty + prior_generation = int(client.difficulty_generation) + advertised_generation = prior_generation + 1 + client.pending_share_difficulty = target + client.difficulty_generation = advertised_generation + if applied_directly: + if schedule_initial: + self.schedule_initial_job(client) + return False + with self.registry.lock: + initial_pending = client in self.initial_state.pending + if initial_pending: + self.schedule_initial_job(client) + return False + maybe_send_override = hooks.maybe_send_override() + if callable(maybe_send_override): + sent = bool(maybe_send_override(client, clean_jobs=True)) + else: + sent = bool( + not initial_runtime.stopping() + and self.maybe_send_job(client, clean_jobs=True) + ) + if sent: + return True + with client_vardiff_lock(client): + if ( + client.pending_share_difficulty == target + and int(client.difficulty_generation) == advertised_generation + ): + client.pending_share_difficulty = prior_pending + client.difficulty_generation = prior_generation + return False + + def adopt_jobs(self, jobs: MutableMapping[str, PrismJobContext]) -> None: + self.jobs = jobs + + def active_context_locked( + self, + client: ClientState, + job_id: str, + ) -> PrismJobContext | None: + if job_id not in client.active_job_ids: + return None + return self.jobs.get(job_id) + + @staticmethod + def client_can_receive_jobs(client: ClientState) -> bool: + return session_client_can_receive_jobs(client) + + def refresh_source(self) -> RefreshSource: + preparation = self._required(self.preparation, "preparation") + payout = self._required(self.payout, "payout") + return RefreshSource( + ready_latched=preparation.ready_latched(), + payout_generation=payout.generation(), + ) + + def client_needs_refresh( + self, + client: ClientState, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + return self.client_needs_refresh_locked( + client, + snapshot, + self.refresh_source(), + ) + + def client_needs_refresh_locked( + self, + client: ClientState, + snapshot: QbitTipTemplateSnapshot, + source: RefreshSource, + ) -> bool: + context = client.active_job + if context is None: + return True + if getattr(context, "collection_only", False) and source.ready_latched: + return True + template = context.template + previousblockhash = str(template.get("previousblockhash", "")) + context_fingerprint = getattr(context, "template_fingerprint", None) + if context_fingerprint is None: + preparation = self._required(self.preparation, "preparation") + context_fingerprint = preparation.template_fingerprint(template) + return bool( + previousblockhash != snapshot.bestblockhash + or previousblockhash != snapshot.previousblockhash + or context_fingerprint != snapshot.template_fingerprint + or int(getattr(context, "payout_state_generation", 0)) + != source.payout_generation + ) + + def intervening_supersedes( + self, + active_job: PrismJobContext | None, + expected_active_job: PrismJobContext | None, + snapshot: QbitTipTemplateSnapshot, + source: RefreshSource | None = None, + ) -> bool: + if source is None: + source = self.refresh_source() + if active_job is expected_active_job or active_job is None: + return False + if int(active_job.payout_state_generation) < source.payout_generation: + return False + active_parent = str(active_job.template.get("previousblockhash", "")) + if ( + active_parent != snapshot.bestblockhash + or active_parent != snapshot.previousblockhash + ): + return False + active_generation = int(active_job.template_generation) + snapshot_generation = int(snapshot.template_generation) + if active_generation <= 0 or snapshot_generation <= 0: + return True + return active_generation >= snapshot_generation + + def tip_changed( + self, + client: ClientState, + snapshot: QbitTipTemplateSnapshot, + source: RefreshSource | None = None, + ) -> bool: + if source is None: + source = self.refresh_source() + context = client.active_job + if context is None: + return True + previousblockhash = str(context.template.get("previousblockhash", "")) + return bool( + previousblockhash != snapshot.bestblockhash + or previousblockhash != snapshot.previousblockhash + or int(getattr(context, "payout_state_generation", 0)) + != source.payout_generation + ) + + def stamp( + self, + client: ClientState, + cached: CachedJobBundle, + *, + clean_jobs: bool, + ) -> PrismJobContext: + worker = client.worker + if worker is None: + raise StratumError(20, "client is not authorized") + preparation = self._required(self.preparation, "preparation") + collection_identity = preparation.collection_identity(worker) + if cached.collection_only and cached.collection_identity != collection_identity: + raise StratumError( + 20, + "collection bundle payout identity no longer matches client authorization", + ) + share_target = direct_stratum.effective_share_target( + self.runtime.desired_share_difficulty(client), + cached.base_job.qbit_target, + minimum_advertised_difficulty=( + self.runtime.minimum_advertised_difficulty(client) + ), + ) + job = dataclass_replace( + cached.base_job, + job_id=self.next_job_id(), + extranonce1_hex=client.extranonce1_hex, + share_target=share_target, + share_difficulty=direct_stratum.target_difficulty(share_target), + clean_jobs=clean_jobs, + ) + return PrismJobContext( + job=job, + template=cached.template, + shares_json=cached.shares_json, + prior_balances=cached.prior_balances, + found_block=cached.found_block, + share_weight=self.runtime.share_weight(worker), + collection_only=cached.collection_only, + worker=worker, + issued_at_ms=cached.issued_at_ms, + template_fingerprint=cached.template_fingerprint, + template_generation=cached.template_generation, + payout_state_generation=cached.payout_state_generation, + prospective_prior_balances=cached.prospective_prior_balances, + payout_artifact_generation=cached.payout_artifact_generation, + connection_id=client.connection_id, + authorization_generation=int(client.authorization_generation), + difficulty_generation=int(client.difficulty_generation), + ) + + def send_update( + self, + client: ClientState, + job: direct_stratum.DirectQbitStratumJob, + *, + split_send: bool, + ) -> None: + """Send one adjacent set_difficulty/mining.notify pair.""" + if split_send: + self.runtime.send_difficulty(client, job) + self.runtime.send_job(client, job) + else: + override = ( + None + if self.hooks is None + or bool(getattr(self._send_override_local, "active", False)) + else self.hooks.send_update_override() + ) + if callable(override): + self._send_override_local.active = True + try: + override(client, job) + finally: + self._send_override_local.active = False + else: + self.runtime.send_job_batch(client, job) + + @staticmethod + def apply_job_difficulty( + client: ClientState, + job: direct_stratum.DirectQbitStratumJob, + *, + config: vardiff.VardiffConfig, + ) -> None: + with client_vardiff_lock(client): + if not config.enabled: + client.share_difficulty = job.share_difficulty + client.pending_share_difficulty = None + return + pending = client.pending_share_difficulty + client.share_difficulty = job.share_difficulty + if pending is not None and job.share_difficulty == pending: + client.pending_share_difficulty = None + + @staticmethod + def apply_client_difficulty_requests( + client: ClientState, + *, + base: vardiff.VardiffConfig, + ) -> Decimal | None: + """Resolve d=/md=/suggest inputs without owning vardiff policy.""" + with client_vardiff_lock(client): + requested = ( + client.requested_difficulty + if client.requested_difficulty is not None + else client.suggested_difficulty + ) + if requested is None and client.requested_min_difficulty is None: + client.vardiff_config = None + return None + floor = base.min_difficulty + if client.requested_min_difficulty is not None: + floor = vardiff.clamp( + client.requested_min_difficulty, + base.min_difficulty, + base.max_difficulty, + ) + if requested is None: + requested = client.share_difficulty + target = vardiff.clamp(requested, floor, base.max_difficulty) + client.vardiff_config = dataclass_replace( + base, + min_difficulty=floor, + startup_difficulty=target, + ) + return target + + def capture_authority( + self, + client: ClientState, + context: PrismJobContext, + *, + expected_active_job: object | None, + ) -> DeliveryAuthority: + return DeliveryAuthority.capture( + client, + context=context, + expected_active_job=expected_active_job, + ) + + def authority_current_locked( + self, + client: ClientState, + authority: DeliveryAuthority, + *, + expected_active_job: object | None, + ) -> bool: + return bool( + client in self.registry.clients + and authority.client_matches(client, active_job=expected_active_job) + ) + + @staticmethod + def context_matches_client_locked( + client: ClientState, + context: PrismJobContext, + ) -> bool: + return bool( + getattr(context, "worker", client.worker) == client.worker + and int(getattr(context, "connection_id", client.connection_id)) + == int(client.connection_id) + and int( + getattr( + context, + "authorization_generation", + client.authorization_generation, + ) + ) + == int(client.authorization_generation) + and int( + getattr( + context, + "difficulty_generation", + client.difficulty_generation, + ) + ) + == int(client.difficulty_generation) + ) + + def final_delivery_guard_locked( + self, + client: ClientState, + authority: DeliveryAuthority, + context: PrismJobContext, + ) -> bool: + return bool( + client in self.registry.clients + and not client.closing + and int(client.connection_id) == authority.connection_id + and client.subscribed + and client.authorized + and client.worker == authority.worker + and int(client.authorization_generation) + == authority.authorization_generation + and int(client.difficulty_generation) == authority.difficulty_generation + and client.active_job is context + and getattr(context, "template_fingerprint", None) + == authority.template_fingerprint + and int(getattr(context, "template_generation", 0)) + == authority.template_generation + and int(getattr(context, "payout_state_generation", 0)) + == authority.payout_state_generation + ) + + def register_locked( + self, + client: ClientState, + context: PrismJobContext, + *, + clean_jobs: bool, + current_tip: str | None, + ) -> tuple[str, ...]: + retired = self.registry.register_active_job_locked( + client, + context, + job_id=context.job.job_id, + clean_jobs=clean_jobs, + ) + if clean_jobs: + for job_id in retired: + retired_context = self.jobs.pop(job_id, None) + if retired_context is not None: + self.retained.retain( + client, + job_id, + retired_context, + current_tip=current_tip, + ) + self.jobs[context.job.job_id] = context + self.prune_active_locked(client, current_tip=current_tip) + return retired + + def prune_active_locked( + self, + client: ClientState, + *, + current_tip: str | None, + ) -> None: + for job_id in tuple(client.active_job_ids): + if job_id not in self.jobs: + client.active_job_ids.discard(job_id) + ordered = [job_id for job_id in self.jobs if job_id in client.active_job_ids] + while len(ordered) > MAX_ACTIVE_PRISM_JOBS_PER_CLIENT: + job_id = ordered.pop(0) + client.active_job_ids.remove(job_id) + context = self.jobs.pop(job_id, None) + if context is not None: + self.retained.retain( + client, + job_id, + context, + current_tip=current_tip, + ) + + def prune_active(self, client: ClientState) -> None: + tip = self._required(self.tip_authority, "tip authority") + authority = tip.retention_authority_locked() + with self.registry.lock: + self.prune_active_locked( + client, + current_tip=authority.current_tip, + ) + + def record_successful_delivery( + self, + client: ClientState, + authority: DeliveryAuthority, + context: PrismJobContext, + delivered_monotonic: float, + ) -> bool: + """Final guard and S1 proof commit, after a successful socket send.""" + with self.registry.lock: + return self._record_successful_delivery_locked( + client, + authority, + context, + delivered_monotonic, + ) + + def _record_successful_delivery_locked( + self, + client: ClientState, + authority: DeliveryAuthority, + context: PrismJobContext, + delivered_monotonic: float, + ) -> bool: + if not self.final_delivery_guard_locked(client, authority, context): + return False + if client._progress_delivered_context is context: + return False + return self.registry.record_delivery_locked( + client, + context, + delivered_monotonic, + ) + + def source_authority_current_locked( + self, + source: DeliverySourceAuthority, + context: PrismJobContext, + ) -> bool: + """Validate an immutable source identity under the S1/R1 lock.""" + tip = self._required(self.tip_authority, "tip authority") + context_payout_generation = int( + getattr( + context, + "payout_state_generation", + source.payout_generation, + ) + ) + context_template_generation = int( + getattr(context, "template_generation", source.template_generation) + ) + context_template_fingerprint = getattr( + context, + "template_fingerprint", + source.template_fingerprint, + ) + if ( + context_payout_generation != source.payout_generation + or context_template_generation != source.template_generation + or ( + source.template_fingerprint is not None + and context_template_fingerprint != source.template_fingerprint + ) + ): + return False + if source.kind == "artifacts": + artifacts = source.artifacts + if artifacts is None: + return False + return bool( + context.template is artifacts.template + and context_template_fingerprint == artifacts.fingerprint + and context_template_generation == artifacts.generation + and tip.artifacts_parent_current_locked(artifacts) + ) + if source.kind == "tip_snapshot": + snapshot = source.snapshot + if snapshot is None: + return False + return bool( + context_template_fingerprint == snapshot.template_fingerprint + and context_template_generation == snapshot.template_generation + and str(context.template.get("previousblockhash", "")) + == snapshot.previousblockhash + and tip.snapshot_current_locked( + snapshot, + source.observation_sequence, + ) + ) + if source.kind == "tip_token": + if ( + source.token is None + or source.bundle is None + or source.snapshot is None + or source.payout_snapshot is None + ): + return False + return tip.prepared_token_current_locked( + source.token, + source.bundle, + source.snapshot, + source.payout_snapshot, + ) + if source.kind == "published_tip": + return tip.published_current_locked( + source.context_parent, + template_fingerprint=source.template_fingerprint, + template_generation=source.template_generation, + lapsed_live_validated=source.lapsed_live_validated, + payout_generation=source.payout_generation, + ) + return False + + def complete_delivery( + self, + client: ClientState, + authority: DeliveryAuthority, + context: PrismJobContext, + delivered_monotonic: float, + *, + initial_request: PendingInitialJob | None = None, + source_authorities: tuple[DeliverySourceAuthority, ...] = (), + ) -> bool: + """Commit exact S1 proof, then notify G1 after releasing the lock.""" + progress = self._required(self.progress, "progress") + with self.registry.lock: + if any( + not self.source_authority_current_locked(source, context) + for source in source_authorities + ): + return False + if ( + initial_request is not None + and not self.initial_request_current_locked(initial_request) + ): + return False + if not self._record_successful_delivery_locked( + client, + authority, + context, + delivered_monotonic, + ): + return False + progress.record_health_delivery(client, context, delivered_monotonic) + progress.reconcile_health_eligibility() + return True + + def note_tip_work_delivered( + self, + client: ClientState, + job_parent_hash: str, + ) -> None: + """Anchor stale grace at the first successful delivery for each tip.""" + now = time.monotonic() + with self.registry.lock: + delivered = client.tip_work_delivered + if delivered is None or delivered[0] != job_parent_hash: + client.tip_work_delivered = (job_parent_hash, now) + if self.delivery_health_updated is not None: + self.delivery_health_updated(job_parent_hash) + + def retire_client_locked(self, client: ClientState) -> tuple[str, ...]: + retired_active = self.registry.clear_active_jobs_locked(client) + for job_id in retired_active: + self.jobs.pop(job_id, None) + self.retained.retire_connection(client.connection_id) + return retired_active + + +class JobDeliveryTipRefreshPort: + """R1 delivery interface backed by S2 ownership, without coordinator context.""" + + def __init__( + self, + *, + registry: SessionRegistry | Callable[[], SessionRegistry], + delivery: JobDeliveryService, + submit_task: Callable[..., Future[RefreshResult]], + disconnect: Callable[[ClientState], None], + ) -> None: + self._registry = registry + self.delivery = delivery + self._submit_task = submit_task + self._disconnect = disconnect + + def _refresh_source(self) -> RefreshSource: + return self.delivery.refresh_source() + + def _needs_refresh( + self, + client: ClientState, + snapshot: QbitTipTemplateSnapshot, + source: RefreshSource, + ) -> bool: + hooks = self.delivery.hooks + override = None if hooks is None else hooks.needs_refresh_override() + if callable(override): + return bool(override(client, snapshot)) + return self.delivery.client_needs_refresh_locked(client, snapshot, source) + + def _tip_changed_snapshot( + self, + client: ClientState, + snapshot: QbitTipTemplateSnapshot, + source: RefreshSource, + ) -> bool: + return self.delivery.tip_changed(client, snapshot, source) + + @property + def registry(self) -> SessionRegistry: + if callable(self._registry): + return self._registry() + return self._registry + + def eligible_clients(self) -> tuple[object, ...]: + with self.registry.lock: + return tuple( + client + for client in self.registry.clients + if self.delivery.client_can_receive_jobs(client) + ) + + def client_can_receive_jobs(self, client: object) -> bool: + return self.delivery.client_can_receive_jobs(client) # type: ignore[arg-type] + + def client_needs_refresh( + self, + client: object, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + source = self._refresh_source() + return self._needs_refresh(client, snapshot, source) # type: ignore[arg-type] + + def active_job(self, client: object) -> object | None: + with self.registry.lock: + return client.active_job # type: ignore[attr-defined] + + def connection_id(self, client: object) -> int: + return int(client.connection_id) # type: ignore[attr-defined] + + def delivery_priority( + self, + client: object, + snapshot: QbitTipTemplateSnapshot, + expected_active_job: object | None, + ) -> int: + if expected_active_job is None: + return PRISM_DELIVERY_PRIORITY_INITIAL + source = self._refresh_source() + if self._tip_changed_snapshot(client, snapshot, source): # type: ignore[arg-type] + return PRISM_DELIVERY_PRIORITY_NEW_TIP + return PRISM_DELIVERY_PRIORITY_SAME_TIP + + def submit_task( + self, + executor: object, + fn: Callable[..., RefreshResult], + *args: object, + priority: int, + ) -> Future[RefreshResult]: + return self._submit_task(executor, fn, *args, priority=priority) + + def send_prepared_job(self, *args: object) -> RefreshResult: + hooks = self.delivery.hooks + override = None if hooks is None else hooks.send_prepared_override() + if callable(override): + return override(*args) + return self.delivery.send_prepared_job(*args) # type: ignore[arg-type] + + def disconnect(self, client: object) -> None: + self._disconnect(client) # type: ignore[arg-type] + + @staticmethod + def log_identity(client: object) -> str: + return ( + f"connection={getattr(client, 'connection_id', 'unknown')} " + f"username={getattr(client, 'username', '')}" + ) + + def select_targets( + self, + snapshot: QbitTipTemplateSnapshot, + *, + refresh_all: bool, + ) -> tuple[RefreshClientTarget, ...]: + source = self._refresh_source() + with self.registry.lock: + candidates = tuple( + (client, client.active_job) + for client in self.registry.clients + if self.delivery.client_can_receive_jobs(client) + ) + return tuple( + RefreshClientTarget(client, active_job) + for client, active_job in candidates + if refresh_all or self._needs_refresh(client, snapshot, source) + ) + + def merge_poll_start_targets( + self, + targets: tuple[RefreshClientTarget, ...], + poll_start_clients: tuple[object, ...], + snapshot: QbitTipTemplateSnapshot, + *, + refresh_all: bool, + ) -> tuple[RefreshClientTarget, ...]: + source = self._refresh_source() + with self.registry.lock: + connected = { + client: getattr(client, "active_job", None) + for client in self.registry.clients + } + merged = list(targets) + selected = {target.client for target in targets} + for candidate in poll_start_clients: + client = candidate + if client in selected or client not in connected: + continue + if refresh_all or self._needs_refresh( # type: ignore[arg-type] + client, snapshot, source + ): + merged.append(RefreshClientTarget(client, connected[client])) + selected.add(client) + return tuple(merged) + + def revalidate_targets( + self, + targets: tuple[RefreshClientTarget, ...], + snapshot: QbitTipTemplateSnapshot, + ) -> tuple[tuple[RefreshClientTarget, ...], tuple[str, ...]]: + current: list[RefreshClientTarget] = [] + dropped: list[str] = [] + source = self._refresh_source() + with self.registry.lock: + connected = set(self.registry.clients) + for target in targets: + client = target.client + if client not in connected: + dropped.append("disconnected") + elif not self.delivery.client_can_receive_jobs( # type: ignore[arg-type] + client + ): + dropped.append("skipped") + elif not self._needs_refresh( # type: ignore[arg-type] + client, snapshot, source + ): + dropped.append("skipped") + else: + current.append(target) + return tuple(current), tuple(dropped) + + def deliver_collection( + self, + client: object, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> RefreshResult: + source = self._refresh_source() + with self.registry.lock: + connected = client in self.registry.clients + if not connected: + return RefreshResult("disconnected") + eligible = self.delivery.client_can_receive_jobs( # type: ignore[arg-type] + client + ) + if not eligible: + return RefreshResult("skipped") + try: + hooks = self.delivery.hooks + override = None if hooks is None else hooks.maybe_send_override() + send = override if callable(override) else self.delivery.maybe_send_job + if send( + client, + clean_jobs=self._tip_changed_snapshot( # type: ignore[arg-type] + client, snapshot, source + ), + raise_on_reorg_failure=True, + raise_on_build_failure=True, + tip_refresh_snapshot=snapshot, + tip_refresh_observation_sequence=observation_sequence, + ): + return RefreshResult("sent", time.monotonic()) + return RefreshResult("skipped") + except JobBuildFailed: + return RefreshResult("failed") + except OSError: + self._disconnect(client) # type: ignore[arg-type] + return RefreshResult("disconnected") + + def take_post_accept_refresh( + self, + client: object, + ) -> tuple[int, str] | None: + with self.registry.lock: + block = getattr(client, "post_accept_refresh_block", None) + client.post_accept_refresh_block = None # type: ignore[attr-defined] + return block + + +__all__ = [ + "AdmittedIdleBundleSource", + "DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS", + "DeliveryAuthority", + "DeliverySourceAuthority", + "EvictedJobEntry", + "IdleDeliveryAuthority", + "JobBuildFailed", + "JobDeliveryRuntimePort", + "JobDeliveryRuntime", + "JobDeliveryService", + "JobDeliveryTipRefreshPort", + "InitialJobTracker", + "MAX_ACTIVE_PRISM_JOBS_PER_CLIENT", + "PRISM_CREDIT_POLICY_STALE_GRACE", + "PRISM_DELIVERY_PRIORITY_INITIAL", + "PRISM_DELIVERY_PRIORITY_NEW_TIP", + "PRISM_DELIVERY_PRIORITY_SAME_TIP", + "PRISM_EVICTED_JOB_CAPACITY_SCOPES", + "PRISM_EVICTED_JOB_CLASSES", + "PRISM_EVICTED_JOB_SUBMIT_OUTCOMES", + "PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS", + "PendingInitialJob", + "PrismJobContext", + "RetainedJobIndex", + "_JobBuildFailed", +] diff --git a/lab/prism/payout_state.py b/lab/prism/payout_state.py new file mode 100644 index 0000000..5214bb7 --- /dev/null +++ b/lab/prism/payout_state.py @@ -0,0 +1,1817 @@ +"""Payout generation, artifact, preview, and delivery-gate ownership. + +This module deliberately has no dependency on ``prism_coordinator``. The +coordinator wires the service to ledger, job-build, tip-refresh, and progress +health domains through :class:`PayoutStatePorts`. +""" + +from __future__ import annotations + +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass, field, replace as dataclass_replace +import hashlib +import json +import threading +import time +from typing import Callable, Iterator, Mapping, Protocol, Sequence + + +DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS = 5.0 +DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES = 8 +PRISM_PAYOUT_DELIVERY_GENERATIONS = ("current", "stale", "future") +PRISM_PAYOUT_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +PRISM_REWARD_WINDOW_MULTIPLIER = 8 +PRISM_SNAPSHOT_WINDOW_MARGIN = 2 +PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 + + +def canonical_json_text(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + + +def canonical_json_sha256(value: object) -> str: + return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() + + +class TemplateRefreshBlocked(RuntimeError): + """A live template was fetched, but safe work could not be issued.""" + + +class TemplateRefreshSuperseded(TemplateRefreshBlocked): + """Concurrent tip or payout progress invalidated a refresh attempt.""" + + +class PayoutStatePublicationBlocked(TemplateRefreshBlocked): + """Job construction is waiting for a prepared payout publication.""" + + +class CancellationPort(Protocol): + def raise_if_cancelled(self, phase: str) -> None: ... + + +class _FrozenJsonDict(dict[str, object]): + """A JSON object that retains dict serialization without mutation.""" + + __slots__ = () + + @staticmethod + def _immutable(*_args: object, **_kwargs: object) -> None: + raise TypeError("payout ledger JSON is immutable") + + __setitem__ = _immutable + __delitem__ = _immutable + clear = _immutable + pop = _immutable + popitem = _immutable + setdefault = _immutable + update = _immutable + __ior__ = _immutable + + +class _FrozenJsonList(list[object]): + """A JSON array that retains list equality/serialization semantics.""" + + __slots__ = () + + @staticmethod + def _immutable(*_args: object, **_kwargs: object) -> None: + raise TypeError("payout ledger JSON is immutable") + + __setitem__ = _immutable + __delitem__ = _immutable + __iadd__ = _immutable + __imul__ = _immutable + append = _immutable + clear = _immutable + extend = _immutable + insert = _immutable + pop = _immutable + remove = _immutable + reverse = _immutable + sort = _immutable + + +def _freeze_json_value(value: object) -> object: + if isinstance(value, (_FrozenJsonDict, _FrozenJsonList)): + return value + if isinstance(value, Mapping): + frozen = _FrozenJsonDict() + dict.update( + frozen, + ((str(key), _freeze_json_value(item)) for key, item in value.items()), + ) + return frozen + if isinstance(value, (list, tuple)): + frozen = _FrozenJsonList() + list.extend(frozen, (_freeze_json_value(item) for item in value)) + return frozen + return value + + +def _freeze_json_rows( + rows: Sequence[Mapping[str, object]], +) -> tuple[dict[str, object], ...]: + frozen_rows: list[dict[str, object]] = [] + for row in rows: + frozen = _freeze_json_value(row) + if not isinstance(frozen, dict): + raise TypeError("payout ledger JSON row must be an object") + frozen_rows.append(frozen) + return tuple(frozen_rows) + + +@dataclass(frozen=True) +class PayoutStateArtifact: + """Immutable ledger-backed inputs published with one payout generation.""" + + generation: int + source_generation: int + prior_balances_json: str = field(repr=False) + prior_balances_sha256: str + prepared_monotonic: float + + def prior_balances(self) -> list[dict[str, object]]: + value = json.loads(self.prior_balances_json) + if not isinstance(value, list): + raise RuntimeError("published payout artifact is not a balance list") + return value + + +@dataclass(frozen=True) +class PayoutLedgerArtifact: + """Immutable ledger input prepared independently of a qbit template.""" + + generation: int + payout_state_generation: int + network_difficulty: int + accepted_share_count: int + shares_json: tuple[dict[str, object], ...] = field(repr=False) + prior_balances: tuple[dict[str, object], ...] = field(repr=False) + prepared_monotonic: float + snapshot_anchor_ms: int | None = None + + def __post_init__(self) -> None: + # These artifacts are cached and shared across snapshots/candidates. + # Frozen JSON containers retain equality and encoding semantics while + # protecting the accepted-count/balance fence from caller mutation. + object.__setattr__( + self, + "shares_json", + _freeze_json_rows(self.shares_json), + ) + object.__setattr__( + self, + "prior_balances", + _freeze_json_rows(self.prior_balances), + ) + + +@dataclass(frozen=True) +class AcceptedBlockPayoutTransition: + """Prospective balances for one durable candidate across its landing seam.""" + + block_height: int | None = None + landed: bool = False + preview: tuple[tuple[str, str, str, int], ...] | None = None + published_generation: int | None = None + + +@dataclass(frozen=True) +class PayoutStateCandidate: + """Immutable result of payout work prepared outside delivery admission.""" + + base_generation: int + source_generation: int + source_tip_hash: str | None + cause: str + invalidated_monotonic: float + prepared_monotonic: float + accepted_block_hash: str | None = None + accepted_block_preview: tuple[tuple[str, str, str, int], ...] | None = None + accepted_block_withdrawal: bool = False + accepted_block_height: int | None = None + ledger_artifact: PayoutLedgerArtifact | None = field( + default=None, + compare=False, + repr=False, + ) + + +@dataclass(frozen=True) +class PublishedPayoutState: + """The payout snapshot identity to which cached jobs are stamped.""" + + generation: int + source_generation: int + source_tip_hash: str | None + published_monotonic: float + artifact: PayoutStateArtifact | None = field(default=None, repr=False) + + +@dataclass(frozen=True) +class PayoutStateSnapshot: + generation: int + source: tuple[int, str | None, str, float] + published: PublishedPayoutState + ledger_artifact: PayoutLedgerArtifact | None + publication_blocked: bool + + +@dataclass(frozen=True) +class PayoutStateConfig: + accepted_block_preview_wait_seconds: float = ( + DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS + ) + reconcile_supersession_retries: int = ( + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES + ) + + +@dataclass +class PayoutDeliveryAdmission: + admitted: bool + wait_seconds: float + generation: int + published_generation: int + relation: str + delivered: bool = False + + def __bool__(self) -> bool: + return self.admitted + + def mark_delivered(self) -> None: + if not self.admitted: + raise RuntimeError("payout delivery completed without admission") + self.delivered = True + + +class PayoutStateDeliveryGate: + """Order delivery admission around a short payout publication.""" + + def __init__(self) -> None: + self._condition = threading.Condition() + self._active_deliveries = 0 + self._publisher_waiting = False + self._mutation_owner: int | None = None + self._mutation_depth = 0 + self._published_generation = 0 + self._priority_generation: int | None = None + self._delivery_blocked = False + + @staticmethod + def _generation_relation(generation: int, published_generation: int) -> str: + if generation < published_generation: + return "stale" + if generation > published_generation: + return "future" + return "current" + + @contextmanager + def delivery(self) -> Iterator[None]: + with self.delivery_cancelable(lambda: False, priority=True) as admission: + if not admission: + raise RuntimeError("uncancelled payout delivery was not admitted") + yield + admission.mark_delivered() + + @contextmanager + def delivery_cancelable( + self, + cancelled: Callable[[], bool], + *, + generation: int | None = None, + priority: bool = False, + poll_seconds: float = PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, + ) -> Iterator[PayoutDeliveryAdmission]: + started = time.monotonic() + admitted = False + with self._condition: + if generation is None: + generation = self._published_generation + while True: + if cancelled() or self._delivery_blocked: + break + if generation < self._published_generation: + break + publication_blocked = ( + self._publisher_waiting or self._mutation_owner is not None + ) + priority_blocked = ( + self._priority_generation is not None + and generation == self._priority_generation + and not priority + ) + if priority_blocked: + break + future_blocked = generation > self._published_generation + if not publication_blocked and not future_blocked: + self._active_deliveries += 1 + admitted = True + break + self._condition.wait(timeout=poll_seconds) + published_generation = self._published_generation + relation = self._generation_relation(generation, published_generation) + admission = PayoutDeliveryAdmission( + admitted=admitted, + wait_seconds=max(0.0, time.monotonic() - started), + generation=generation, + published_generation=published_generation, + relation=relation, + ) + try: + yield admission + finally: + if admitted: + with self._condition: + if self._active_deliveries <= 0: + raise RuntimeError( + "payout delivery gate released without admission" + ) + self._active_deliveries -= 1 + if ( + priority + and admission.delivered + and generation == self._priority_generation + ): + self._priority_generation = None + if self._active_deliveries == 0: + self._condition.notify_all() + elif self._priority_generation is None: + self._condition.notify_all() + + @contextmanager + def publication(self) -> Iterator[None]: + owner = threading.get_ident() + with self._condition: + if self._mutation_owner == owner: + self._mutation_depth += 1 + else: + while self._mutation_owner is not None or self._publisher_waiting: + self._condition.wait() + self._publisher_waiting = True + while self._active_deliveries: + self._condition.wait() + self._mutation_owner = owner + self._mutation_depth = 1 + self._publisher_waiting = False + try: + yield + finally: + with self._condition: + if self._mutation_owner != owner or self._mutation_depth <= 0: + raise RuntimeError("payout mutation gate released by non-owner") + self._mutation_depth -= 1 + if self._mutation_depth == 0: + self._mutation_owner = None + self._condition.notify_all() + + def publish_generation(self, generation: int, *, prioritize_delivery: bool) -> None: + owner = threading.get_ident() + with self._condition: + if self._mutation_owner != owner: + raise RuntimeError("payout generation published outside atomic section") + if generation <= self._published_generation: + raise RuntimeError("payout generation did not advance") + self._published_generation = generation + self._priority_generation = generation if prioritize_delivery else None + self._delivery_blocked = False + + def block_delivery( + self, + mark_blocked: Callable[[], bool] | None = None, + ) -> bool: + with self._condition: + while self._mutation_owner is not None: + self._condition.wait() + if mark_blocked is not None and not mark_blocked(): + return False + self._delivery_blocked = True + self._condition.notify_all() + return True + + @contextmanager + def mutation(self) -> Iterator[None]: + with self.publication(): + yield + + +@dataclass(frozen=True) +class PayoutStatePorts: + """Narrow callbacks into the ledger, job, refresh, and health domains.""" + + accepted_share_stats: Callable[[], tuple[int, int]] + snapshot_at_job_issue: Callable[[int, int], Sequence[object]] + current_prior_balances: Callable[[], list[dict[str, object]]] + snapshot_anchor_ms: Callable[[int], int] + current_template_network_difficulty: Callable[[], int | None] + pool_ready: Callable[[], bool] + record_build_phase: Callable[[str, float], None] + invalidate_job_cache: Callable[[], None] + clear_retained_collection_refresh: Callable[[], None] + cancel_obsolete_job_builds: Callable[[str], None] + cancel_obsolete_bundle_builds: Callable[[int], None] + payout_invalidated: Callable[[int, float], None] + payout_published: Callable[[int, float], None] + schedule_refresh_retry: Callable[[], None] + chain_block_hash: Callable[[int], str] + stop_requested: Callable[[], bool] + + +class PayoutStateService: + """Single owner of payout state and its publication boundary.""" + + def __init__( + self, + ports: PayoutStatePorts, + *, + monotonic: Callable[[], float] = time.monotonic, + wall_time_ms: Callable[[], int] | None = None, + histogram_buckets: Sequence[float] = PRISM_PAYOUT_SECONDS_BUCKETS, + config: PayoutStateConfig | None = None, + ) -> None: + self._ports = ports + self._config = config or PayoutStateConfig() + self._monotonic = monotonic + self._wall_time_ms = wall_time_ms or (lambda: int(time.time() * 1000)) + self._histogram_buckets = tuple(float(value) for value in histogram_buckets) + self._lock = threading.RLock() + self._prepare_lock = threading.RLock() + self._cache_publication_lock = threading.RLock() + self._balance_mutation_lock = threading.RLock() + self._preview_condition = threading.Condition() + self._metrics_lock = threading.Lock() + self._executor_lock = threading.Lock() + started = self._monotonic() + self._generation = 0 + self._source: tuple[int, str | None, str, float] = ( + 0, + None, + "startup", + started, + ) + self._published = PublishedPayoutState(0, 0, None, started) + self._ledger_artifact: PayoutLedgerArtifact | None = None + self._ledger_artifact_generation = 0 + self._delivery_gate = PayoutStateDeliveryGate() + self._previews: dict[str, AcceptedBlockPayoutTransition] = {} + self._invalidated_previews: dict[str, int | None] = {} + self._publication_blocked = False + self._artifact_executor: ThreadPoolExecutor | None = None + self._artifact_future: Future[None] | None = None + self._artifact_requested: tuple[int, int] | None = None + self._artifact_executor_shutdown = False + self._first_delivery_pending: tuple[int, float] | None = None + self._state_histograms = { + name: self._new_histogram() + for name in ("preparation", "publish", "first_delivery") + } + self._gate_histograms = { + relation: self._new_histogram() + for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS + } + self._discarded_candidates = 0 + + def _new_histogram(self) -> dict[str, object]: + return { + "buckets": {bucket: 0 for bucket in self._histogram_buckets}, + "sum": 0.0, + "count": 0, + } + + def snapshot(self) -> PayoutStateSnapshot: + with self._lock: + return PayoutStateSnapshot( + generation=self._generation, + source=self._source, + published=self._published, + ledger_artifact=self._ledger_artifact, + publication_blocked=self._publication_blocked, + ) + + def current_artifact( + self, + cancellation: CancellationPort | None = None, + ) -> PayoutStateArtifact: + while True: + with self._lock: + if self._publication_blocked: + raise PayoutStatePublicationBlocked( + "payout state invalidation is pending publication" + ) + published = self._published + if ( + published.generation == self._generation + and published.artifact is not None + and published.artifact.generation == published.generation + ): + return published.artifact + generation = self._generation + source_generation = published.source_generation + artifact = self.prepare_artifact( + generation=generation, + source_generation=source_generation, + cancellation=cancellation, + ) + with self._lock: + published = self._published + if ( + self._publication_blocked + or self._generation != generation + or published.source_generation != source_generation + ): + if cancellation is not None: + cancellation.raise_if_cancelled( + "payout artifact publication race" + ) + continue + self._published = dataclass_replace(published, artifact=artifact) + return artifact + + def prepare_artifact( + self, + *, + generation: int, + source_generation: int, + cancellation: CancellationPort | None = None, + ) -> PayoutStateArtifact: + started = self._monotonic() + if cancellation is not None: + cancellation.raise_if_cancelled("payout artifact read") + with self._prepare_lock: + balances = self._ports.current_prior_balances() + if cancellation is not None: + cancellation.raise_if_cancelled("payout artifact serialization") + artifact = self.artifact_from_balances( + generation=generation, + source_generation=source_generation, + balances=balances, + ) + self._ports.record_build_phase( + "payout_artifact", + self._monotonic() - started, + ) + return artifact + + def artifact_from_balances( + self, + *, + generation: int, + source_generation: int, + balances: list[dict[str, object]], + ) -> PayoutStateArtifact: + balances_json = canonical_json_text(balances) + return PayoutStateArtifact( + generation=generation, + source_generation=source_generation, + prior_balances_json=balances_json, + prior_balances_sha256=hashlib.sha256(balances_json.encode()).hexdigest(), + prepared_monotonic=self._monotonic(), + ) + + def build_ledger_artifact( + self, + expected_payout_state_generation: int, + artifact_payout_state_generation: int, + network_difficulty: int, + ) -> PayoutLedgerArtifact | None: + ledger_started = self._monotonic() + try: + accepted_before, _ = self._ports.accepted_share_stats() + with self._prepare_lock: + with self._lock: + if expected_payout_state_generation != self._generation: + return None + snapshot_window_weight = ( + PRISM_REWARD_WINDOW_MULTIPLIER + * PRISM_SNAPSHOT_WINDOW_MARGIN + * int(network_difficulty) + ) + snapshot_anchor_ms = self._ports.snapshot_anchor_ms( + self._wall_time_ms() + ) + records = list( + self._ports.snapshot_at_job_issue( + snapshot_anchor_ms, + snapshot_window_weight, + ) + ) + prior_balances = self._ports.current_prior_balances() + accepted_after, _ = self._ports.accepted_share_stats() + except Exception: + return None + finally: + self._ports.record_build_phase( + "ledger_snapshot", + self._monotonic() - ledger_started, + ) + if accepted_before != accepted_after or not records: + return None + copy_started = self._monotonic() + shares_json = tuple(record.to_prism_json() for record in records) + frozen_balances = tuple(prior_balances) + self._ports.record_build_phase( + "serialization_copy", + self._monotonic() - copy_started, + ) + return PayoutLedgerArtifact( + generation=0, + payout_state_generation=artifact_payout_state_generation, + network_difficulty=int(network_difficulty), + accepted_share_count=accepted_after, + shares_json=shares_json, + prior_balances=frozen_balances, + prepared_monotonic=self._monotonic(), + snapshot_anchor_ms=snapshot_anchor_ms, + ) + + def prepare_ledger_artifact( + self, + payout_state_generation: int, + network_difficulty: int, + ) -> None: + artifact = self.build_ledger_artifact( + payout_state_generation, + payout_state_generation, + network_difficulty, + ) + if artifact is None: + return + with self._lock: + if payout_state_generation != self._generation: + return + self._ledger_artifact_generation += 1 + self._ledger_artifact = dataclass_replace( + artifact, + generation=self._ledger_artifact_generation, + ) + + def _artifact_preparation_loop(self) -> None: + while True: + with self._executor_lock: + request = self._artifact_requested + self._artifact_requested = None + if request is None: + self._artifact_future = None + return + self.prepare_ledger_artifact(*request) + + def schedule_ledger_artifact_preparation( + self, + payout_state_generation: int, + network_difficulty: int, + ) -> None: + with self._executor_lock: + if self._artifact_executor_shutdown: + return + self._artifact_requested = ( + int(payout_state_generation), + int(network_difficulty), + ) + if self._artifact_future is not None: + return + executor = self._artifact_executor + if executor is None: + executor = ThreadPoolExecutor( + max_workers=1, + thread_name_prefix="prism-payout-artifact", + ) + self._artifact_executor = executor + self._artifact_future = executor.submit(self._artifact_preparation_loop) + + def usable_ledger_artifact( + self, + payout_state_generation: int, + network_difficulty: int, + ) -> PayoutLedgerArtifact | None: + with self._lock: + artifact = self._ledger_artifact + published_artifact = self._published.artifact + if ( + artifact is None + or artifact.payout_state_generation != payout_state_generation + or artifact.network_difficulty != int(network_difficulty) + ): + return None + if published_artifact is None: + try: + published_artifact = self.current_artifact() + except Exception: + return None + try: + accepted_share_count, _ = self._ports.accepted_share_stats() + except Exception: + return None + if accepted_share_count != artifact.accepted_share_count: + return None + balances_sha256 = canonical_json_sha256(artifact.prior_balances) + with self._lock: + if ( + self._ledger_artifact is not artifact + or self._generation != payout_state_generation + or self._published.artifact is not published_artifact + ): + return None + if balances_sha256 != published_artifact.prior_balances_sha256: + self._ledger_artifact = None + return None + return artifact + + def schedule_current_ledger_artifact_if_missing(self) -> None: + snapshot = self.snapshot() + network_difficulty = self._ports.current_template_network_difficulty() + if network_difficulty is None: + return + if ( + self.usable_ledger_artifact(snapshot.generation, network_difficulty) + is not None + ): + return + self.schedule_ledger_artifact_preparation( + snapshot.generation, + network_difficulty, + ) + + def shutdown(self) -> None: + with self._executor_lock: + executor = self._artifact_executor + self._artifact_executor = None + self._artifact_executor_shutdown = True + self._artifact_requested = None + if executor is not None: + executor.shutdown(wait=True, cancel_futures=True) + + @contextmanager + def balance_mutation(self) -> Iterator[None]: + with self._balance_mutation_lock: + with self._preview_condition: + landed_transition = any( + transition.landed for transition in self._previews.values() + ) + if landed_transition: + raise TemplateRefreshBlocked( + "accepted block payout confirmation is still pending" + ) + yield + + def begin_accepted_block_preview( + self, + block_hash: str, + *, + block_height: int | None = None, + ) -> None: + key = block_hash.lower() + with self._preview_condition: + self._invalidated_previews.pop(key, None) + existing = self._previews.get(key) + if existing is None: + self._previews[key] = AcceptedBlockPayoutTransition( + block_height=block_height + ) + elif ( + block_height is not None + and existing.block_height is not None + and existing.block_height != block_height + ): + raise RuntimeError("accepted block payout transition height changed") + elif existing.block_height is None and block_height is not None: + self._previews[key] = dataclass_replace( + existing, + block_height=block_height, + ) + + def mark_accepted_block_landed( + self, + block_hash: str, + *, + block_height: int, + ) -> None: + key = block_hash.lower() + with self._preview_condition: + existing = self._previews.get( + key, + AcceptedBlockPayoutTransition(block_height=block_height), + ) + if existing.block_height not in {None, block_height}: + raise RuntimeError("accepted block payout transition height changed") + self._previews[key] = dataclass_replace( + existing, + block_height=block_height, + landed=True, + ) + self._preview_condition.notify_all() + + def publish_accepted_block_preview( + self, + block_hash: str, + balances: list[dict[str, object]], + ) -> list[dict[str, object]]: + normalized = self.normalized_prior_balances(balances) + serialized = self.serialize_prior_balance_preview(normalized) + key = block_hash.lower() + with self._balance_mutation_lock: + with self._preview_condition: + existing = self._previews.get(key) + existing_preview = existing.preview if existing is not None else None + if existing_preview is not None: + if existing_preview != serialized: + raise RuntimeError( + "accepted block payout preview changed during retry" + ) + if existing.published_generation is not None: + return self.materialize_prior_balance_preview(existing_preview) + captured = self.capture_source() + reserved = self.reserve_source_if_current( + captured[1], + "accepted_block_preview", + tip_hash=key, + invalidated_monotonic=self._monotonic(), + ) + candidate = ( + self.current_candidate() + if reserved is None + else self.prepared_candidate(reserved) + ) + candidate = self.accepted_block_preview_candidate( + candidate, + block_hash=key, + preview=serialized, + ) + self.block_publication(force=True) + published = self.publish_candidate(candidate) + if published is None: + for _attempt in range(self._reconcile_retries()): + candidate = self.accepted_block_preview_candidate( + self.current_candidate(), + block_hash=key, + preview=serialized, + ) + published = self.publish_candidate(candidate) + if published is not None: + break + if published is None: + self.block_publication(force=True) + with self._preview_condition: + transition = self._previews.get( + key, + AcceptedBlockPayoutTransition(landed=True), + ) + self._previews[key] = dataclass_replace( + transition, + landed=True, + preview=serialized, + published_generation=None, + ) + self._preview_condition.notify_all() + return self.materialize_prior_balance_preview(serialized) + + def accepted_block_preview_candidate( + self, + candidate: PayoutStateCandidate, + *, + block_hash: str, + preview: tuple[tuple[str, str, str, int], ...], + ) -> PayoutStateCandidate: + ledger_artifact = candidate.ledger_artifact + if ledger_artifact is not None: + ledger_artifact = dataclass_replace( + ledger_artifact, + prior_balances=tuple(self.materialize_prior_balance_preview(preview)), + ) + return dataclass_replace( + candidate, + accepted_block_hash=block_hash, + accepted_block_preview=preview, + ledger_artifact=ledger_artifact, + ) + + @staticmethod + def serialize_prior_balance_preview( + balances: list[dict[str, object]], + ) -> tuple[tuple[str, str, str, int], ...]: + return tuple( + ( + str(balance["recipient_id"]), + str(balance["order_key"]), + str(balance["p2mr_program_hex"]), + int(balance["balance_sats"]), + ) + for balance in balances + ) + + @staticmethod + def materialize_prior_balance_preview( + preview: tuple[tuple[str, str, str, int], ...], + ) -> list[dict[str, object]]: + return [ + { + "recipient_id": recipient_id, + "order_key": order_key, + "p2mr_program_hex": p2mr_program_hex, + "balance_sats": balance_sats, + } + for recipient_id, order_key, p2mr_program_hex, balance_sats in preview + ] + + def accepted_block_preview_from_bundle( + self, + final_bundle: Mapping[str, object], + *, + prior_balances: list[dict[str, object]] | None = None, + ) -> list[dict[str, object]]: + manifest = final_bundle.get("payout_policy_manifest") + if not isinstance(manifest, dict) or not isinstance( + manifest.get("accounts"), list + ): + raise RuntimeError("accepted block payout manifest is missing accounts") + prior_identities: dict[str, tuple[str, str]] = {} + for balance in prior_balances or []: + program = str(balance.get("p2mr_program_hex", "")).lower() + identity = ( + str(balance.get("order_key", "")), + str(balance.get("recipient_id", "")), + ) + prior_identities[program] = min( + identity, + prior_identities.get(program, identity), + ) + balances: list[dict[str, object]] = [] + for account in manifest["accounts"]: + if not isinstance(account, dict): + continue + if str(account.get("account_type", "miner")) == "pool_fee": + continue + balance_sats = int(account.get("carry_forward_balance_sats", 0)) + if balance_sats == 0: + continue + program = str(account.get("p2mr_program_hex", "")).lower() + account_identity = ( + str(account.get("order_key", "")), + str(account.get("recipient_id", "")), + ) + order_key, recipient_id = min( + account_identity, + prior_identities.get(program, account_identity), + ) + balances.append( + { + "recipient_id": recipient_id, + "order_key": order_key, + "p2mr_program_hex": program, + "balance_sats": balance_sats, + } + ) + return self.normalized_prior_balances(balances) + + def clear_accepted_block_preview( + self, + block_hash: str, + *, + invalidate_published: bool = False, + ) -> None: + key = block_hash.lower() + existing: AcceptedBlockPayoutTransition | None = None + with self._balance_mutation_lock: + with self._preview_condition: + existing = self._previews.get(key) + if existing is None: + if not invalidate_published: + self._invalidated_previews.pop(key, None) + self._preview_condition.notify_all() + return + if not invalidate_published: + self._previews.pop(key, None) + self._invalidated_previews.pop(key, None) + self._preview_condition.notify_all() + return + if existing.preview is None: + self._previews.pop(key, None) + if existing.landed: + self._invalidated_previews[key] = existing.block_height + self._preview_condition.notify_all() + return + captured = self.capture_source() + reserved = self.reserve_source_if_current( + captured[1], + "accepted_block_preview_withdrawn", + tip_hash=captured[2], + invalidated_monotonic=self._monotonic(), + ) + candidate = self.prepared_candidate( + reserved if reserved is not None else self.capture_source() + ) + candidate = dataclass_replace( + candidate, + accepted_block_hash=key, + accepted_block_withdrawal=True, + accepted_block_height=existing.block_height, + ) + self.block_publication(force=True) + published = self.publish_candidate(candidate) + if published is None: + for _attempt in range(self._reconcile_retries()): + candidate = dataclass_replace( + self.current_candidate(), + accepted_block_hash=key, + accepted_block_withdrawal=True, + accepted_block_height=existing.block_height, + ) + published = self.publish_candidate(candidate) + if published is not None: + break + if published is None: + self.block_publication(force=True) + with self._preview_condition: + self._previews.pop(key, None) + self._invalidated_previews[key] = existing.block_height + self._preview_condition.notify_all() + + def accepted_block_transition_landed(self, block_hash: str) -> bool: + with self._preview_condition: + transition = self._previews.get(block_hash.lower()) + return transition is not None and transition.landed + + def accepted_block_transition_for_parent( + self, + parent_hash: str, + *, + parent_height: int | None = None, + ) -> tuple[str, bool] | None: + key = parent_hash.lower() + with self._preview_condition: + exact_transition = self._previews.get(key) + exact_invalidated = key in self._invalidated_previews + fail_closed_candidate_hashes = { + candidate_hash + for candidate_hash, transition in self._previews.items() + if transition.landed + } + fail_closed_candidate_hashes.update(self._invalidated_previews) + ancestor_candidates = [ + (candidate_hash, transition.block_height, False) + for candidate_hash, transition in self._previews.items() + if exact_transition is None + and not exact_invalidated + and transition.block_height is not None + and parent_height is not None + and transition.block_height <= parent_height + ] + ancestor_candidates.extend( + (candidate_hash, candidate_height, True) + for candidate_hash, candidate_height in self._invalidated_previews.items() + if exact_transition is None + and not exact_invalidated + and candidate_height is not None + and parent_height is not None + and candidate_height <= parent_height + ) + if exact_transition is not None or exact_invalidated: + return key, exact_invalidated + if not ancestor_candidates: + return None + active_ancestors: list[tuple[int, str, bool]] = [] + try: + for candidate_hash, candidate_height, candidate_invalidated in ancestor_candidates: + assert candidate_height is not None + active_hash = self._ports.chain_block_hash(candidate_height).lower() + if active_hash == candidate_hash: + active_ancestors.append( + (candidate_height, candidate_hash, candidate_invalidated) + ) + except Exception as exc: + self._ports.schedule_refresh_retry() + raise TemplateRefreshBlocked( + "could not validate an accepted payout preview on the active chain" + ) from exc + if not active_ancestors: + if any( + candidate_hash in fail_closed_candidate_hashes + for candidate_hash, _height, _invalidated in ancestor_candidates + ): + self._ports.schedule_refresh_retry() + raise PayoutStatePublicationBlocked( + "accepted payout transition is no longer active" + ) + return None + _, selected_key, selected_invalidated = max(active_ancestors) + return selected_key, selected_invalidated + + def await_pending_parent_preview( + self, + parent_hash: str, + *, + parent_height: int | None = None, + ) -> list[dict[str, object]] | None: + selected = self.accepted_block_transition_for_parent( + parent_hash, + parent_height=parent_height, + ) + if selected is None: + return None + selected_key, selected_invalidated = selected + if selected_invalidated: + self._ports.schedule_refresh_retry() + raise TemplateRefreshBlocked( + "accepted parent payout preview was withdrawn" + ) + wait_seconds = max( + 0.0, + float(self._config.accepted_block_preview_wait_seconds), + ) + deadline = self._monotonic() + wait_seconds + timed_out = False + invalidated = False + with self._preview_condition: + while selected_key in self._previews: + transition = self._previews[selected_key] + if transition.preview is not None: + return self.materialize_prior_balance_preview(transition.preview) + if self._ports.stop_requested(): + raise RuntimeError( + "coordinator stopped while accepted payout preview was pending" + ) + remaining = deadline - self._monotonic() + if remaining <= 0: + timed_out = True + break + self._preview_condition.wait(timeout=min(0.25, remaining)) + invalidated = selected_key in self._invalidated_previews + if invalidated: + self._ports.schedule_refresh_retry() + raise TemplateRefreshBlocked( + "accepted parent payout preview was withdrawn" + ) + if timed_out: + self._ports.schedule_refresh_retry() + raise TemplateRefreshBlocked( + "accepted parent payout preview is not ready yet" + ) + return None + + def prior_balances_for_parent( + self, + parent_hash: str, + *, + parent_height: int | None = None, + fallback_balances: Sequence[dict[str, object]] | None = None, + ) -> list[dict[str, object]]: + preview = self.await_pending_parent_preview( + parent_hash, + parent_height=parent_height, + ) + if preview is not None: + return preview + return ( + list(fallback_balances) + if fallback_balances is not None + else self._ports.current_prior_balances() + ) + + @staticmethod + def normalized_prior_balances( + balances: list[dict[str, object]], + ) -> list[dict[str, object]]: + rows = [ + { + "recipient_id": str(balance.get("recipient_id", "")), + "order_key": str(balance.get("order_key", "")), + "p2mr_program_hex": str(balance.get("p2mr_program_hex", "")), + "balance_sats": int(balance.get("balance_sats", 0)), + } + for balance in balances + ] + rows.sort( + key=lambda row: ( + row["order_key"], + row["recipient_id"], + row["p2mr_program_hex"], + row["balance_sats"], + ) + ) + return rows + + def prior_balances_match_current( + self, + prior_balances: list[dict[str, object]], + ) -> bool: + return self.normalized_prior_balances( + prior_balances + ) == self.normalized_prior_balances(self._ports.current_prior_balances()) + + def reserve_source( + self, + cause: str, + *, + tip_hash: str | None = None, + invalidated_monotonic: float | None = None, + ) -> int: + invalidated = ( + self._monotonic() + if invalidated_monotonic is None + else invalidated_monotonic + ) + with self._lock: + generation = self._source[0] + 1 + self._source = (generation, tip_hash, cause, invalidated) + return generation + + def reserve_source_for_tip_change( + self, + tip_hash: str, + *, + cause: str, + invalidated_monotonic: float, + ) -> int | None: + """Reserve a source only when the observed tip identity changed.""" + + with self._lock: + if self._source[1] == tip_hash: + return None + generation = self._source[0] + 1 + self._source = ( + generation, + tip_hash, + cause, + invalidated_monotonic, + ) + return generation + + def reserve_source_if_current( + self, + expected_source_generation: int, + cause: str, + *, + tip_hash: str | None = None, + invalidated_monotonic: float | None = None, + ) -> tuple[int, int, str | None, str, float] | None: + invalidated = ( + self._monotonic() + if invalidated_monotonic is None + else invalidated_monotonic + ) + with self._lock: + if self._source[0] != expected_source_generation: + return None + source_generation = expected_source_generation + 1 + self._source = (source_generation, tip_hash, cause, invalidated) + return ( + self._generation, + source_generation, + tip_hash, + cause, + invalidated, + ) + + def capture_source(self) -> tuple[int, int, str | None, str, float]: + with self._lock: + source_generation, source_tip, cause, invalidated = self._source + return ( + self._generation, + source_generation, + source_tip, + cause, + invalidated, + ) + + def prepared_candidate( + self, + captured: tuple[int, int, str | None, str, float], + ) -> PayoutStateCandidate: + base_generation, source_generation, source_tip, cause, invalidated = captured + ledger_artifact: PayoutLedgerArtifact | None = None + network_difficulty = self._ports.current_template_network_difficulty() + if network_difficulty is not None and self._ports.pool_ready(): + ledger_artifact = self.build_ledger_artifact( + base_generation, + base_generation + 1, + network_difficulty, + ) + return PayoutStateCandidate( + base_generation=base_generation, + source_generation=source_generation, + source_tip_hash=source_tip, + cause=cause, + invalidated_monotonic=invalidated, + prepared_monotonic=self._monotonic(), + ledger_artifact=ledger_artifact, + ) + + def current_candidate(self) -> PayoutStateCandidate: + return self.prepared_candidate(self.capture_source()) + + def _record_discarded_candidate(self) -> None: + with self._metrics_lock: + self._discarded_candidates += 1 + + def block_publication( + self, + *, + force: bool = False, + supersede_with: tuple[int, str | None, str, float] | None = None, + ) -> None: + pending_source: int | None = None + + def mark_blocked() -> bool: + nonlocal pending_source + with self._lock: + if supersede_with is not None: + expected_source, fallback_tip, cause, invalidated = supersede_with + current_source, current_tip, _, _ = self._source + source_tip = ( + fallback_tip if current_source == expected_source else current_tip + ) + pending_source = current_source + 1 + self._source = ( + pending_source, + source_tip, + cause, + invalidated, + ) + else: + pending_source = self._source[0] + if ( + not force + and supersede_with is None + and pending_source == self._published.source_generation + ): + return False + self._publication_blocked = True + # Publish the blocked state before entering the narrow cache + # fence. Admission either completes before the following clear or + # observes publication_blocked and fails closed. + with self._cache_publication_lock: + self._ports.invalidate_job_cache() + return True + + if not self._delivery_gate.block_delivery(mark_blocked): + return + self._ports.cancel_obsolete_job_builds("payout generation superseded") + with self._lock: + next_generation = self._generation + 1 + invalidated = self._source[3] + self._ports.payout_invalidated(next_generation, invalidated) + + def publication_fenced(self) -> bool: + with self._lock: + return self._publication_blocked + + def source_requires_publication( + self, + candidate: PayoutStateCandidate | None = None, + ) -> bool: + with self._lock: + if candidate is not None: + return candidate.source_generation != self._published.source_generation + return self._source[0] != self._published.source_generation + + def publish_candidate(self, candidate: PayoutStateCandidate) -> int | None: + with self._lock: + if ( + candidate.source_generation != self._source[0] + or candidate.base_generation != self._generation + ): + self._record_discarded_candidate() + return None + try: + if ( + candidate.accepted_block_preview is not None + and not candidate.accepted_block_withdrawal + ): + artifact = self.artifact_from_balances( + generation=candidate.base_generation + 1, + source_generation=candidate.source_generation, + balances=self.materialize_prior_balance_preview( + candidate.accepted_block_preview + ), + ) + else: + artifact = self.prepare_artifact( + generation=candidate.base_generation + 1, + source_generation=candidate.source_generation, + ) + except Exception: + self._ports.schedule_refresh_retry() + raise + with self._lock: + if ( + candidate.source_generation != self._source[0] + or candidate.base_generation != self._generation + ): + self._record_discarded_candidate() + return None + published_generation: int | None = None + publish_started = 0.0 + invalidate_job_cache = False + with self._delivery_gate.publication(), self._cache_publication_lock: + publish_started = self._monotonic() + with self._lock: + source_generation = self._source[0] + if ( + candidate.source_generation == source_generation + and candidate.base_generation == self._generation + ): + published_generation = self._generation + 1 + if candidate.accepted_block_hash is not None: + key = candidate.accepted_block_hash + with self._preview_condition: + transition = self._previews.get( + key, + AcceptedBlockPayoutTransition( + block_height=candidate.accepted_block_height, + landed=True, + ), + ) + if candidate.accepted_block_withdrawal: + self._previews.pop(key, None) + self._invalidated_previews[key] = ( + transition.block_height + if transition.block_height is not None + else candidate.accepted_block_height + ) + else: + existing_preview = transition.preview + if ( + existing_preview is not None + and existing_preview + != candidate.accepted_block_preview + ): + raise RuntimeError( + "accepted block payout preview changed " + "during atomic publication" + ) + self._invalidated_previews.pop(key, None) + self._previews[key] = dataclass_replace( + transition, + landed=True, + preview=candidate.accepted_block_preview, + published_generation=published_generation, + ) + self._preview_condition.notify_all() + self._generation = published_generation + prepared_artifact = candidate.ledger_artifact + if ( + prepared_artifact is not None + and prepared_artifact.payout_state_generation + == published_generation + ): + self._ledger_artifact_generation += 1 + self._ledger_artifact = dataclass_replace( + prepared_artifact, + generation=self._ledger_artifact_generation, + ) + else: + self._ledger_artifact = None + self._published = PublishedPayoutState( + generation=published_generation, + source_generation=candidate.source_generation, + source_tip_hash=candidate.source_tip_hash, + published_monotonic=publish_started, + artifact=artifact, + ) + self._publication_blocked = False + invalidate_job_cache = True + with self._metrics_lock: + self._first_delivery_pending = ( + published_generation, + candidate.invalidated_monotonic, + ) + if invalidate_job_cache: + # Publication still owns the delivery gate, but the payout + # lock is released before callbacks enter J1-owned locks. + self._ports.invalidate_job_cache() + self._ports.clear_retained_collection_refresh() + if published_generation is not None: + self._delivery_gate.publish_generation( + published_generation, + prioritize_delivery=True, + ) + self.observe_seconds( + "publish", + max(0.0, self._monotonic() - publish_started), + ) + if published_generation is None: + self._record_discarded_candidate() + return None + self._ports.cancel_obsolete_bundle_builds(published_generation) + self._ports.payout_published( + published_generation, + candidate.invalidated_monotonic, + ) + self._ports.cancel_obsolete_job_builds("payout generation published") + network_difficulty = self._ports.current_template_network_difficulty() + accepted_preview_pending_durability = ( + candidate.accepted_block_hash is not None + and not candidate.accepted_block_withdrawal + ) + if ( + network_difficulty is not None + and self.usable_ledger_artifact( + published_generation, + network_difficulty, + ) + is None + and not accepted_preview_pending_durability + ): + self.schedule_ledger_artifact_preparation( + published_generation, + network_difficulty, + ) + return published_generation + + def record_first_delivery( + self, + generation: int, + delivered_monotonic: float, + ) -> None: + elapsed: float | None = None + with self._metrics_lock: + pending = self._first_delivery_pending + if pending is not None and pending[0] == generation: + elapsed = max(0.0, delivered_monotonic - pending[1]) + self._first_delivery_pending = None + if elapsed is not None: + self.observe_seconds("first_delivery", elapsed) + + def advance_generation(self) -> int: + self.reserve_source("payout_only") + prepared_started = self._monotonic() + with self._prepare_lock: + self.block_publication(force=True) + self.observe_seconds( + "preparation", + max(0.0, self._monotonic() - prepared_started), + ) + generation = self.publish_current_with_retry_budget(initial_attempted=False) + if generation is None: + raise TemplateRefreshSuperseded( + "payout-only invalidation was superseded; immediate retry scheduled" + ) + return generation + + def publish_current_with_retry_budget( + self, + *, + initial_attempted: bool = False, + ) -> int | None: + attempts = self._reconcile_retries() + (0 if initial_attempted else 1) + for _attempt in range(attempts): + candidate = self.current_candidate() + published = self.publish_candidate(candidate) + if published is not None: + return published + self.block_publication() + return None + + def _reconcile_retries(self) -> int: + return max(0, int(self._config.reconcile_supersession_retries)) + + @property + def reconcile_supersession_retries(self) -> int: + return self._reconcile_retries() + + def replace_config_for_test(self, config: PayoutStateConfig) -> None: + self._config = config + + def set_preview_wait_seconds_for_test(self, seconds: float) -> None: + self._config = dataclass_replace( + self._config, + accepted_block_preview_wait_seconds=float(seconds), + ) + + def set_reconcile_retries_for_test(self, retries: int) -> None: + self._config = dataclass_replace( + self._config, + reconcile_supersession_retries=int(retries), + ) + + @property + def delivery_gate(self) -> PayoutStateDeliveryGate: + return self._delivery_gate + + @contextmanager + def cache_publication_admission(self) -> Iterator[None]: + """Fence J1 cache insertion against payout generation mutation.""" + with self._cache_publication_lock: + yield + + @contextmanager + def delivery( + self, + generation: int, + *, + cancelled: Callable[[], bool], + priority: bool, + ) -> Iterator[PayoutDeliveryAdmission]: + with self._delivery_gate.delivery_cancelable( + cancelled, + generation=generation, + priority=priority, + ) as admission: + yield admission + + def observe_gate_admission( + self, + admission: object, + *, + generation: int, + fallback_wait_seconds: float, + ) -> None: + published_generation = self.snapshot().generation + relation = getattr(admission, "relation", None) + if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: + relation = PayoutStateDeliveryGate._generation_relation( + generation, + published_generation, + ) + wait_seconds = float( + getattr(admission, "wait_seconds", fallback_wait_seconds) + ) + self.observe_seconds( + "gate_wait", + max(0.0, wait_seconds), + relation=relation, + ) + + def observe_seconds( + self, + name: str, + elapsed_seconds: float, + *, + relation: str | None = None, + ) -> None: + with self._metrics_lock: + if name == "gate_wait": + if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: + raise ValueError( + f"unknown payout delivery generation: {relation}" + ) + histogram = self._gate_histograms[str(relation)] + else: + histogram = self._state_histograms[name] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + elapsed_seconds + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in self._histogram_buckets: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def metrics_snapshot(self) -> dict[str, object]: + with self._metrics_lock: + return { + "state_histograms": { + name: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for name, histogram in self._state_histograms.items() + }, + "gate_histograms": { + relation: { + "buckets": dict(histogram["buckets"]), + "sum": float(histogram["sum"]), + "count": int(histogram["count"]), + } + for relation, histogram in self._gate_histograms.items() + }, + "discarded_candidates": self._discarded_candidates, + } + + def metrics_lines(self) -> list[str]: + metrics = self.metrics_snapshot() + state_histograms = metrics["state_histograms"] + gate_histograms = metrics["gate_histograms"] + assert isinstance(state_histograms, dict) + assert isinstance(gate_histograms, dict) + metric_names = { + "preparation": "qbit_prism_payout_preparation_seconds", + "publish": "qbit_prism_payout_publish_seconds", + "first_delivery": ( + "qbit_prism_payout_invalidation_first_delivery_seconds" + ), + } + descriptions = { + "preparation": ( + "Payout reconciliation and candidate preparation outside " + "delivery publication." + ), + "publish": "Atomic payout generation/cache publication gate-hold time.", + "first_delivery": ( + "Payout invalidation to first delivery of the published generation." + ), + } + lines: list[str] = [] + for name, metric_name in metric_names.items(): + histogram = state_histograms[name] + assert isinstance(histogram, dict) + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + f"# HELP {metric_name} {descriptions[name]}", + f"# TYPE {metric_name} histogram", + *[ + f'{metric_name}_bucket{{le="{bucket:g}"}} ' + f'{int(buckets.get(bucket, 0))}' + for bucket in self._histogram_buckets + ], + f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', + f'{metric_name}_sum {float(histogram["sum"]):.6f}', + f'{metric_name}_count {histogram["count"]}', + ] + ) + gate_name = "qbit_prism_payout_gate_wait_seconds" + lines.extend( + [ + "# HELP qbit_prism_payout_gate_wait_seconds Delivery admission " + "wait by generation relationship to the published payout state.", + "# TYPE qbit_prism_payout_gate_wait_seconds histogram", + ] + ) + for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS: + histogram = gate_histograms[relation] + assert isinstance(histogram, dict) + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + *[ + f'{gate_name}_bucket{{generation="{relation}",' + f'le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in self._histogram_buckets + ], + f'{gate_name}_bucket{{generation="{relation}",le="+Inf"}} ' + f'{histogram["count"]}', + f'{gate_name}_sum{{generation="{relation}"}} ' + f'{float(histogram["sum"]):.6f}', + f'{gate_name}_count{{generation="{relation}"}} ' + f'{histogram["count"]}', + ] + ) + lines.extend( + [ + "# HELP qbit_prism_payout_candidates_discarded_total Prepared " + "payout candidates discarded after source supersession.", + "# TYPE qbit_prism_payout_candidates_discarded_total counter", + "qbit_prism_payout_candidates_discarded_total " + f'{metrics["discarded_candidates"]}', + ] + ) + return lines + + # The following accessors keep temporary facade/test compatibility without + # duplicating mutable state on the coordinator. X1 removes these seams. + @property + def prepare_lock(self) -> threading.RLock: + return self._prepare_lock + + @property + def balance_mutation_lock(self) -> threading.RLock: + return self._balance_mutation_lock + + @property + def preview_condition(self) -> threading.Condition: + return self._preview_condition + + @preview_condition.setter + def preview_condition(self, value: threading.Condition) -> None: + self._preview_condition = value + + @property + def previews(self) -> dict[str, AcceptedBlockPayoutTransition]: + return self._previews + + @property + def invalidated_previews(self) -> dict[str, int | None]: + return self._invalidated_previews + + def replace_generation_for_test(self, generation: int) -> None: + with self._lock: + self._generation = int(generation) + + def replace_published_for_test(self, published: PublishedPayoutState) -> None: + with self._lock: + self._published = published + + def replace_ledger_artifact_for_test( + self, + artifact: PayoutLedgerArtifact | None, + ) -> None: + with self._lock: + self._ledger_artifact = artifact + + def replace_delivery_gate_for_test(self, gate: PayoutStateDeliveryGate) -> None: + self._delivery_gate = gate diff --git a/lab/prism/prism_coordinator.py b/lab/prism/prism_coordinator.py index 8f145a0..b3c0348 100644 --- a/lab/prism/prism_coordinator.py +++ b/lab/prism/prism_coordinator.py @@ -3,41 +3,30 @@ from __future__ import annotations -import base64 -import copy from collections import OrderedDict -from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack, contextmanager import dataclasses -import errno -from functools import wraps import hashlib -import heapq -import http.client import json -import math import os import queue -import random import shlex import signal import socket -import struct import subprocess -import tempfile import threading import time import traceback import urllib.parse import urllib.request import uuid -import weakref from types import SimpleNamespace -from dataclasses import dataclass, field, replace as dataclass_replace -from decimal import Context, Decimal, InvalidOperation, ROUND_CEILING, getcontext, localcontext +from dataclasses import dataclass, replace as dataclass_replace +from decimal import Decimal, ROUND_CEILING from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from typing import Any, Callable, Iterator, Sequence +from typing import Any, Callable, Iterator, Mapping, MutableMapping, Sequence import sys @@ -46,13 +35,258 @@ from lab.auxpow import stratum_codec, vardiff from lab.prism import direct_stratum, public_api +# Compatibility re-exports; new callers should import lab.prism.background_services. +from lab.prism.background_services import ( + BackgroundServiceRegistry, # noqa: F401 - compatibility re-export + BackgroundServiceSpec, # noqa: F401 - compatibility re-export +) +# Compatibility re-exports; new callers should import lab.prism.bounded_executor. +from lab.prism.bounded_executor import ( + _BoundedPriorityExecutor, # noqa: F401 - compatibility re-export + _DeliveryQueueFull, # noqa: F401 - compatibility re-export +) from lab.prism.prism_tools import prism_tool_command from lab.prism.ctv_broadcaster import CtvFanoutBroadcaster +from lab.prism.coordinator_config import ( + CoordinatorConfig, + DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS, # noqa: F401 - compatibility re-export + DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, + DEFAULT_HIGHDIFF_DIFFICULTY, # noqa: F401 - compatibility re-export + DEFAULT_HIGHDIFF_MAX_DIFFICULTY, # noqa: F401 - compatibility re-export + DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, + DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, + DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, + DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE, # noqa: F401 - compatibility re-export + DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER, # noqa: F401 - compatibility re-export + DEFAULT_P2MR_SPEND_INPUT_BYTES, # noqa: F401 - compatibility re-export + DEFAULT_PRISM_BLOCKPOLL_SECONDS, + DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, + DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, + DEFAULT_PRISM_COINBASE_TAG, # noqa: F401 - compatibility re-export + DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, + DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, + DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, + DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, + DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, + DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + DEFAULT_PRISM_STALE_GRACE_SECONDS, + DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, + DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, + DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, + DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, + DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, + DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS, # noqa: F401 - compatibility re-export + DEFAULT_PRISM_WORKER_METRICS_LIMIT, + DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, + DEFAULT_SHARE_COMMIT_BATCH_SIZE, # noqa: F401 - compatibility re-export + DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS, # noqa: F401 - compatibility re-export + DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS, # noqa: F401 - compatibility re-export + DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS, # noqa: F401 - compatibility re-export + MAX_PRISM_COINBASE_TAG_BYTES, # noqa: F401 - compatibility re-export + StratumListenerProfile, + TESTNET_QBIT_CHAINS, + default_prism_coinbase_tag_hex, # noqa: F401 - compatibility re-export + default_prism_payout_policy, + default_prism_username_fallback_address, + env, + env_bool, + env_decimal, # noqa: F401 - compatibility re-export + env_int, + env_nonnegative_float, + env_nonnegative_int, + env_nonnegative_int_with_legacy, # noqa: F401 - compatibility re-export + env_optional, + env_optional_bool, # noqa: F401 - compatibility re-export + env_optional_positive_int, # noqa: F401 - compatibility re-export + env_optional_positive_int_with_legacy, + env_positive_float, + env_positive_int, + env_positive_int_with_legacy, + env_seed_hex, # noqa: F401 - compatibility re-export + load_coordinator_config, + load_share_weights, + load_prism_highdiff_listener, # noqa: F401 - compatibility re-export + load_prism_vardiff_config, # noqa: F401 - compatibility re-export + production_mode, # noqa: F401 - compatibility re-export + require_production_env, # noqa: F401 - compatibility re-export + validate_hex, + validate_prism_production_gate, # noqa: F401 - compatibility re-export + validate_same_tip_job_retention_limits, # noqa: F401 - compatibility re-export +) +# Compatibility re-exports; session callers should import the owning module. +from lab.prism.stratum_session import ( + ClientState, + JobDeliveryPort, + P2mrAddressValidator, + ProgressHealthPort, + SessionRegistry, + SessionRuntimePort, + StratumError, + StratumSessionService, + WorkerIdentity, + apply_stratum_send_timeout as apply_socket_send_timeout, + client_vardiff_lock, + difficulty_payload as stratum_difficulty_payload, + error_payload as stratum_error_payload, + job_payload as stratum_job_payload, + parse_stratum_password_options, # noqa: F401 - compatibility re-export + parse_worker_username, # noqa: F401 - compatibility re-export + result_payload as stratum_result_payload, + split_worker_username, # noqa: F401 - compatibility re-export + stratum_accept_heartbeat_names as configured_accept_heartbeat_names, +) from lab.prism.ctv_broadcaster_daemon import ( CtvFanoutBroadcastDaemon, CtvFanoutChunkResult, CtvFanoutDaemonResult, - MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE, + MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE, # noqa: F401 - compatibility re-export +) +# Compatibility re-exports; new callers should import lab.prism.ctv_runtime. +from lab.prism.ctv_runtime import ( + CtvRuntimeConfig, + CtvRuntimeService, + PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS, # noqa: F401 + PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS, # noqa: F401 + PRISM_CTV_BROADCASTER_SECONDS_BUCKETS, # noqa: F401 +) +from lab.prism.rpc import JsonRpc +# Compatibility re-exports; new callers should import the owning J1 modules. +from lab.prism.bundle_compiler import BundleCompiler, BundleCompilerPorts +from lab.prism.job_bundle import ( + CachedJobBundle, + CollectionIdentityUnavailable, # noqa: F401 - compatibility re-export + JobBuildCancellation as _JobBuildCancellation, + JobBuildCancelled, # noqa: F401 - compatibility re-export + JobBuildFlight as _JobBuildFlight, + JobBuildKey, # noqa: F401 - compatibility re-export + JobBuildRequest as _JobBuildRequest, + JobBuildSuperseded, + JobBuildWaiterCancelled as _JobBuildCancelled, # noqa: F401 + JobBundleBuildControl as _JobBundleBuildControl, + JobBundleBuildSuperseded as _JobBundleBuildSuperseded, + JobBundleConfig, + JobBundlePorts, + JobBundleService, +) +# Compatibility re-exports; new callers should import lab.prism.job_delivery. +from lab.prism.job_delivery import ( + AdmittedIdleBundleSource, + DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS, # noqa: F401 + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, + DeliveryCompatibilityHooks, + EvictedJobEntry, + IdleDeliveryAuthority, + InitialJobConfig, + InitialJobSnapshot, # noqa: F401 - compatibility re-export + InitialJobState, + InitialJobTracker, # noqa: F401 - compatibility re-export + InitialJobRuntimePort, + JobPreparationPort, + JobDeliveryRuntime, + JobDeliveryService, + JobDeliveryTipRefreshPort, + MAX_ACTIVE_PRISM_JOBS_PER_CLIENT, # noqa: F401 + PRISM_CREDIT_POLICY_STALE_GRACE, + PRISM_DELIVERY_PRIORITY_INITIAL, # noqa: F401 + PRISM_DELIVERY_PRIORITY_NEW_TIP, # noqa: F401 + PRISM_DELIVERY_PRIORITY_SAME_TIP, # noqa: F401 + PRISM_EVICTED_JOB_CAPACITY_SCOPES, + PRISM_EVICTED_JOB_CLASSES, + PRISM_EVICTED_JOB_SUBMIT_OUTCOMES, + PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, # noqa: F401 + PendingInitialJob, + PayoutDeliveryPort, + PrismJobContext, + ProgressDeliveryPort, + RetentionAuthority, + RetainedJobIndex, + TipAuthorityPort, + _JobBuildFailed, # noqa: F401 +) +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + QbitTipTemplateSnapshot, + TemplateArtifactEventSink, + TemplateArtifactPorts, + TemplateArtifactRepository, +) +# Compatibility re-exports; new callers should import lab.prism.tip_refresh. +from lab.prism.tip_refresh import ( + FanoutCancellation as _FanoutCancellation, + PRISM_TIP_REFRESH_BUILD_PHASES, + PRISM_TIP_REFRESH_CANCELLATION_STAGES, # noqa: F401 - compatibility re-export + PRISM_TIP_REFRESH_RESULTS, # noqa: F401 - compatibility re-export + PRISM_TIP_REFRESH_SECONDS_BUCKETS, + PublishedTipSnapshot, # noqa: F401 - compatibility re-export + RefreshResult, + RetainedCollectionRefresh, # noqa: F401 - compatibility re-export + TipRefreshConfig, + TipRefreshPorts, + TipRefreshService, + TipRefreshValidationToken, +) +# Compatibility re-exports; new callers should import lab.prism.progress_health. +from lab.prism.progress_health import ( + BundleBuildToken, # noqa: F401 - compatibility re-export + DeliveryProof, + EligibilitySnapshot, + PROGRESS_HEALTH_REASONS, + ProgressHealthConfig, + ProgressHealthService, + ProgressHealthSnapshot, + RefreshActivityToken, # noqa: F401 - compatibility re-export + WorkGeneration, + overlay_progress_health, +) +# Compatibility re-exports; new callers should import lab.prism.payout_state. +from lab.prism.payout_state import ( + AcceptedBlockPayoutTransition as _AcceptedBlockPayoutTransition, # noqa: F401 + PayoutDeliveryAdmission as _PayoutDeliveryAdmission, # noqa: F401 + PayoutLedgerArtifact, + PayoutStateArtifact, + PayoutStateCandidate, + PayoutStateConfig, + PayoutStateDeliveryGate as _PayoutStateDeliveryGate, # noqa: F401 + PayoutStatePorts, + PayoutStatePublicationBlocked as _PayoutStatePublicationBlocked, # noqa: F401 + PayoutStateService, + PublishedPayoutState, # noqa: F401 + TemplateRefreshBlocked, + TemplateRefreshSuperseded, +) +# Compatibility re-exports; new callers should import lab.prism.coordinator_shutdown. +from lab.prism.coordinator_shutdown import ( + CoordinatorShutdownController, # noqa: F401 - compatibility re-export + ShutdownInProgress, # noqa: F401 - compatibility re-export + _WriterOperationToken, # noqa: F401 - compatibility re-export + ledger_writer_operation, # noqa: F401 - compatibility re-export +) +# Compatibility re-exports; new callers should import lab.prism.share_writer. +from lab.prism.share_writer import ( + MAX_PENDING_SHARE_APPENDS, + PENDING_SHARE_COMMIT_WARN_SECONDS as PRISM_PENDING_SHARE_COMMIT_WARN_SECONDS, + PendingShareAppend, + PendingShareInput, + ShareWriter, + ShareWriterCompatibilityField, + ShareWriterConfig, + ShareWriterError, + ShareWriterPorts, + ShareWriterQueueFull, ) from lab.prism.share_ledger import ( DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, @@ -64,89 +298,15 @@ sha256_json_hex, ) -DEFAULT_P2MR_SPEND_INPUT_BYTES = 3_680 -DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE = 1 -DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER = 4 -DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS = ( - "tq1zlsq9dpxz8mennhdpr9nf9s0f2tjtq6gxs9m84k6xglhkfp92q2zszzu4m3" -) -DEFAULT_PRISM_COINBASE_TAG = "/PRISM/" -MAX_PRISM_COINBASE_TAG_BYTES = 40 -DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS = 10_485_760 -DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS = 16 -DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS = 12 -DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION = 1_000 -DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS = 12_000 -TESTNET_QBIT_CHAINS = {"testnet", "testnet3", "testnet4", "signet"} -DEFAULT_PRISM_BLOCKPOLL_SECONDS = 2.0 -DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS = 5.0 -DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS = 1.0 -# Fraction of the holdoff added as random jitter so coordinators sharing one -# qbitd do not phase-lock their blocked-refresh re-attempts. -PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION = 0.25 -DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS = 10.0 -DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS = 60.0 MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES = 128 -DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE = 5 -DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS = 5.0 -DEFAULT_PRISM_HEALTH_REFRESH_SECONDS = 5.0 -DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS = 15.0 -DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS = 15.0 -DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS = 20.0 -DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS = 384 -DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME = 0 -DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS = 128 -DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS = 30.0 -DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS = 30.0 -DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS = 1.0 -# Kernel accept backlog per stratum listener. Sized so a whole fleet -# reconnecting inside a restart window parks in the backlog instead of being -# SYN-dropped; the kernel caps the effective value at net.core.somaxconn. -DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG = 1024 -# How long bind() retries EADDRINUSE at startup, covering overlap with a -# predecessor process that is still draining its shutdown while holding the -# port. Zero fails fast (the historical behavior). -DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS = 10.0 -DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES = 4_096 -DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS = 3_600.0 -DEFAULT_PRISM_STALE_GRACE_SECONDS = 3.0 -# How old the refresh-published tip may be before mining.submit stops trusting -# it and falls back to a live getbestblockhash per share. Healthy coordinators -# republish/reconfirm it every blockpoll interval; a bounded divergence lease -# separately covers replacement construction without failing open forever. -DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS = 10.0 -DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS = 30.0 -DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION = 64 -DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS = 1.0 -DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS = 16 -DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS = 4 -PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 -DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS = 60.0 -DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS = 0.25 PRISM_JOB_BUILD_EXECUTOR_WORKERS = 2 -DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS = 15.0 PRISM_VARDIFF_IDLE_RETARGET_MAX_WORKERS = 2 MAX_PENDING_VARDIFF_IDLE_RETARGETS = 8 -DEFAULT_PRISM_WORKER_METRICS_LIMIT = 100 -MAX_ACTIVE_PRISM_JOBS_PER_CLIENT = 16 # Block candidates queue to a dedicated submitter thread so the miner's share # ack never waits on audit/submitblock after the share and intent commit. The # bound limits RAM; overflow only coalesces a wakeup because Postgres retains # the authoritative pending candidate. MAX_PENDING_BLOCK_CANDIDATES = 32 -# Accepted shares use a small, bounded group-commit queue. Every submitter -# waits for its batch's Postgres commit before receiving Stratum success, so -# this is a latency-smoothing bound rather than a durable backlog. -MAX_PENDING_SHARE_APPENDS = 4_096 -# A pending share commit normally clears the snapshot anchor floor within one -# group-commit linger. Holding it longer than this is a wedged writer or a -# leaked release path and is logged loudly (once per share). -PRISM_PENDING_SHARE_COMMIT_WARN_SECONDS = 30.0 -DEFAULT_SHARE_COMMIT_BATCH_SIZE = 64 -DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS = 5.0 -DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS = 15.0 -DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS = 15.0 -DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS = 120 # The reward window is 8x network difficulty (must match PRISM_WINDOW_MULTIPLIER # in crates/qbit-prism/src/lib.rs and the SQL). The job-build snapshot only needs # the shares that window can cover; requesting a margin above it returns a @@ -176,28 +336,6 @@ 10.0, 30.0, ) -PRISM_CTV_BROADCASTER_SECONDS_BUCKETS = ( - 1.0, - 5.0, - 10.0, - 30.0, - 60.0, - 120.0, - 300.0, - 600.0, -) -PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS = PRISM_JOB_BUILD_SECONDS_BUCKETS -PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS = (1, 2, 5, 10, 25, 50, 100) -PRISM_TIP_REFRESH_SECONDS_BUCKETS = PRISM_JOB_BUILD_SECONDS_BUCKETS -PRISM_TIP_REFRESH_BUILD_PHASES = ( - "ledger_snapshot", - "payout_state_derivation", - "ctv_manifest_construction", - "coinbase_bundle_construction", - "signing_verification", - "serialization_copy", - "singleflight_wait", -) PRISM_BUILDER_PHASE_METRICS_PREFIX = "qbit-prism-build-phase-metrics " PRISM_VARDIFF_IDLE_SECONDS_BUCKETS = PRISM_JOB_BUILD_SECONDS_BUCKETS PRISM_VARDIFF_IDLE_SKIP_REASONS = ( @@ -232,25 +370,7 @@ "send", ) PRISM_JOB_CACHE_KINDS = ("template", "bundle") -PRISM_TIP_REFRESH_RESULTS = ("sent", "skipped", "disconnected", "failed") -PRISM_TIP_REFRESH_CANCELLATION_STAGES = ( - "executor_queue", - "client_lock", - "payout_gate", -) -PRISM_DELIVERY_PRIORITY_NEW_TIP = 0 -PRISM_DELIVERY_PRIORITY_INITIAL = 1 -PRISM_DELIVERY_PRIORITY_SAME_TIP = 2 -PRISM_PROGRESS_HEALTH_REASONS = ( - "tip_poll_stale", - "refresh_pending_too_long", - "current_generation_not_published", - "current_generation_not_delivered", - "bundle_build_stuck", -) -PRISM_EVICTED_JOB_CLASSES = ("same_tip", "stale_grace") -PRISM_EVICTED_JOB_SUBMIT_OUTCOMES = ("accepted_same_tip", "credited_stale_grace") -PRISM_EVICTED_JOB_CAPACITY_SCOPES = ("connection",) +PRISM_PROGRESS_HEALTH_REASONS = PROGRESS_HEALTH_REASONS PRISM_REJECTION_STALE_JOB = "stale-job" PRISM_REJECTION_DUPLICATE_SHARE = "duplicate-share" PRISM_REJECTION_LOW_DIFFICULTY = "low-difficulty" @@ -286,7 +406,6 @@ # Credit policies recorded on accepted ledger rows. Normal shares carry no # policy; a policy marks a share that was credited by an explicit pool rule # (documented in docs/prism-rejections.md) so audits can distinguish them. -PRISM_CREDIT_POLICY_STALE_GRACE = "stale-grace" # Aggregation bucket for per-worker share metrics once the distinct-worker # label budget is exhausted. PRISM_WORKER_METRICS_OVERFLOW_LABEL = "_other" @@ -307,16 +426,6 @@ PRISM_REJECTION_BLOCK_STALE, PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, ) -PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS = frozenset( - { - # qbit can legitimately advance these without making already issued - # jobs stale. Rebuilding every miner job for clock-only changes would - # turn the poller into continuous audit-bundle churn. - "curtime", - "longpollid", - "mintime", - } -) class _ObservedRLock: @@ -369,487 +478,22 @@ def contention_snapshot(self) -> tuple[int, float, float]: self._wait_seconds_sum, self._wait_seconds_max, ) - - -def env(name: str, default: str | None = None) -> str: - value = os.environ.get(name, default) - if value is None or value == "": - raise SystemExit(f"{name} is required") - return value - - -def env_int(name: str, default: int) -> int: - return int(env(name, str(default))) - - -def env_positive_int(name: str, default: int) -> int: - try: - value = env_int(name, default) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_positive_int_with_legacy(primary_name: str, legacy_name: str, default: int) -> int: - if env_optional(primary_name) is not None: - return env_positive_int(primary_name, default) - return env_positive_int(legacy_name, default) - - -def env_nonnegative_int(name: str, default: int) -> int: - try: - value = env_int(name, default) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value < 0: - raise SystemExit(f"{name} must be non-negative") - return value - - -def env_nonnegative_int_with_legacy(primary_name: str, legacy_name: str, default: int) -> int: - if env_optional(primary_name) is not None: - return env_nonnegative_int(primary_name, default) - return env_nonnegative_int(legacy_name, default) - - -def env_positive_float(name: str, default: float) -> float: - try: - value = float(env(name, str(default))) - except ValueError as exc: - raise SystemExit(f"{name} must be a number") from exc - if not math.isfinite(value): - raise SystemExit(f"{name} must be finite") - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_nonnegative_float(name: str, default: float) -> float: - try: - value = float(env(name, str(default))) - except ValueError as exc: - raise SystemExit(f"{name} must be a number") from exc - if not math.isfinite(value): - raise SystemExit(f"{name} must be finite") - if value < 0: - raise SystemExit(f"{name} must be non-negative") - return value - - -def env_optional_positive_int(name: str) -> int | None: - raw = env_optional(name) - if raw is None: - return None - try: - value = int(raw) - except ValueError as exc: - raise SystemExit(f"{name} must be an integer") from exc - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_optional_positive_int_with_legacy(primary_name: str, legacy_name: str) -> int | None: - value = env_optional_positive_int(primary_name) - if value is not None: - return value - return env_optional_positive_int(legacy_name) - - -def env_decimal(name: str, default: str) -> Decimal: - value = Decimal(env(name, default)) - if value <= 0: - raise SystemExit(f"{name} must be positive") - return value - - -def env_bool(name: str, default: str) -> bool: - return env(name, default).lower() in {"1", "true", "yes", "on"} - - -def env_optional_bool(name: str) -> bool | None: - raw = env_optional(name) - if raw is None: - return None - return raw.lower() in {"1", "true", "yes", "on"} - - -def env_optional(name: str) -> str | None: - value = os.environ.get(name) - if value is None or value == "": - return None - return value - - -def production_mode() -> bool: - return ( - env_bool("QBIT_PRODUCTION", "0") - or env_bool("QBIT_TOOLS_PRODUCTION", "0") - or env("QBIT_CHAIN", "regtest").lower() in {"main", "mainnet"} - ) - - -def validate_same_tip_job_retention_limits( - *, - retention_seconds: float, - per_connection: int, - max_connections: int, - production: bool, -) -> None: - if retention_seconds <= 0: - return - if per_connection <= 0: - raise SystemExit( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION must be positive " - "when same-tip retention is enabled" - ) - if production and max_connections <= 0: - raise SystemExit( - "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS " - "when same-tip retention is enabled" - ) - - -def require_production_env(name: str) -> str: - value = env_optional(name) - if value is None: - raise SystemExit(f"production mode requires {name}") - return value - - -def validate_prism_production_gate() -> None: - if not production_mode(): - return - - for name in ( - "PRISM_ALLOW_MEMORY_LEDGER", - "PRISM_ALLOW_TEST_SIGNING_SEEDS", - "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY", - "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", - ): - if env_bool(name, "0"): - raise SystemExit(f"production mode rejects {name}=1") - - if env("QBIT_CHAIN", "regtest").lower() in {"main", "mainnet"} and env_nonnegative_float( - "PRISM_STRATUM_STALE_GRACE_SECONDS", - DEFAULT_PRISM_STALE_GRACE_SECONDS, - ) != 0: - raise SystemExit( - "mainnet requires PRISM_STRATUM_STALE_GRACE_SECONDS=0" - ) - - production_difficulties: dict[str, Decimal] = {} - for name in ( - "PRISM_STRATUM_SHARE_DIFF", - "PRISM_STRATUM_VARDIFF_MIN_DIFF", - "PRISM_STRATUM_VARDIFF_START_DIFF", - "PRISM_STRATUM_VARDIFF_MAX_DIFF", - ): - raw_value = require_production_env(name) - if not raw_value: - raise SystemExit(f"production mode requires an explicit {name}") - try: - value = Decimal(raw_value) - except InvalidOperation as exc: - raise SystemExit(f"{name} must be a decimal number") from exc - if not value.is_finite() or value <= 0: - raise SystemExit(f"{name} must be positive") - if value == Decimal("0.000000001"): - raise SystemExit(f"{name} cannot use the lab-only 1e-9 difficulty") - production_difficulties[name] = value - if ( - production_difficulties["PRISM_STRATUM_VARDIFF_MIN_DIFF"] - > production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] - ): - raise SystemExit("production vardiff minimum exceeds its start difficulty") - if ( - production_difficulties["PRISM_STRATUM_VARDIFF_START_DIFF"] - > production_difficulties["PRISM_STRATUM_VARDIFF_MAX_DIFF"] - ): - raise SystemExit("production vardiff start exceeds its maximum difficulty") - - prism_database_url = env_optional("PRISM_DATABASE_URL") - if prism_database_url is None and env_optional("PRISM_POSTGRES_PSQL_COMMAND") is None: - raise SystemExit("production mode requires PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND") - if env_optional("PRISM_POSTGRES_PASSWORD") == "change-this": - raise SystemExit("production mode requires a non-default PRISM_POSTGRES_PASSWORD") - if prism_database_url is not None and "change-this" in prism_database_url: - raise SystemExit("production mode requires a non-default PRISM_DATABASE_URL") - - require_production_env("PRISM_MANIFEST_SIGNING_SEED_HEX") - require_production_env("PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX") - require_production_env("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX") - require_production_env("PRISM_LEDGER_WRITER_ID") - require_production_env("PRISM_LEDGER_WRITER_EPOCH") - require_production_env("PRISM_AUDIT_DIR") - require_production_env("PRISM_EVIDENCE_PATH") - - if env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN") is not None: - raise SystemExit("production mode requires managed ledger session tokens; unset PRISM_LEDGER_WRITER_SESSION_TOKEN") - - if env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) <= 0: - raise SystemExit( - "production mode requires a positive PRISM_STRATUM_MAX_CONNECTIONS" - ) - env_positive_int( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, - ) - if env_nonnegative_float( - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, - ) <= 0: - raise SystemExit( - "production mode requires a positive " - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS" - ) - - require_production_env("QBIT_RPC_USER") - qbit_rpc_password = require_production_env("QBIT_RPC_PASSWORD") - if qbit_rpc_password == "change-this": - raise SystemExit("production mode requires a non-default QBIT_RPC_PASSWORD") - - if env("QBIT_CHAIN", "regtest").lower() in {"main", "mainnet"} and env_bool( - "PRISM_CTV_SETTLEMENT_ENABLED", "0" - ): - require_production_env("PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT") - env_positive_int("PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", 0) - - validate_same_tip_job_retention_limits( - retention_seconds=env_nonnegative_float( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, - ), - per_connection=env_nonnegative_int( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, - ), - max_connections=env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ), - production=True, - ) - - -def validate_hex(value: str, *, name: str, expected_bytes: int | None = None) -> str: - try: - bytes.fromhex(value) - except ValueError as exc: - raise SystemExit(f"{name} must be hex") from exc - if expected_bytes is not None and len(value) != expected_bytes * 2: - raise SystemExit(f"{name} must be {expected_bytes * 2} hex chars") - return value.lower() - - -def env_seed_hex(name: str, *, test_default: str) -> str: - value = env_optional(name) - if value is None: - if env_bool("PRISM_ALLOW_TEST_SIGNING_SEEDS", "0"): - value = test_default - else: - raise SystemExit(f"{name} is required") - return validate_hex(value, name=name, expected_bytes=32) +PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS = frozenset( + { + # qbit can legitimately advance these without making already issued + # jobs stale. Rebuilding every miner job for clock-only changes would + # turn the poller into continuous audit-bundle churn. + "curtime", + "longpollid", + "mintime", + } +) def now_ms() -> int: return int(time.time() * 1000) -def load_prism_vardiff_config(startup_difficulty: Decimal) -> vardiff.VardiffConfig: - return vardiff.VardiffConfig( - enabled=env_bool("PRISM_STRATUM_VARDIFF", "1"), - target_share_interval_seconds=env_decimal("PRISM_STRATUM_VARDIFF_TARGET_SECONDS", "15"), - min_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_MIN_DIFF", str(startup_difficulty)), - max_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_MAX_DIFF", "1024"), - retarget_interval_seconds=env_decimal("PRISM_STRATUM_VARDIFF_RETARGET_SECONDS", "90"), - max_step_factor=env_decimal("PRISM_STRATUM_VARDIFF_MAX_STEP_UP", "4"), - startup_difficulty=env_decimal("PRISM_STRATUM_VARDIFF_START_DIFF", str(startup_difficulty)), - max_step_down_factor=env_decimal("PRISM_STRATUM_VARDIFF_MAX_STEP_DOWN", "4"), - ewma_alpha=env_decimal("PRISM_STRATUM_VARDIFF_EWMA_ALPHA", "0.4"), - retarget_tolerance=env_decimal("PRISM_STRATUM_VARDIFF_RETARGET_TOLERANCE", "0.25"), - ) - - -@dataclass(frozen=True) -class StratumListenerProfile: - """One stratum listener with its own difficulty policy. - - Every listener feeds the same coordinator, ledger, and settlement path; - the profile only decides where to listen and which difficulty bounds its - clients get. - """ - - name: str - bind: str - port: int - share_difficulty: Decimal - vardiff_config: vardiff.VardiffConfig - heartbeat_name: str - # Difficulty this listener never advertises below, even when the qbit - # network target is easier. Zero means no floor: stamped jobs keep the - # network cap (a share is never required to be harder than a block). The - # high-diff listener sets its configured minimum because marketplace - # verification (NiceHash-style) checks the first advertised difficulty, - # which must hold even while network difficulty sits below the floor. - minimum_advertised_difficulty: Decimal = Decimal("0") - - -DEFAULT_HIGHDIFF_DIFFICULTY = "500000" -DEFAULT_HIGHDIFF_MAX_DIFFICULTY = "4294967296" - - -def load_prism_highdiff_listener( - base_bind: str, - base_vardiff_config: vardiff.VardiffConfig, -) -> StratumListenerProfile | None: - """Optional high-difficulty listener for rental-scale miners. - - Disabled unless PRISM_STRATUM_HIGHDIFF_PORT is set. The 500k default floor - matches the NiceHash SHA-256 pool-verification minimum, which must hold - from the first mining.set_difficulty a client sees. - """ - - port_value = env_optional("PRISM_STRATUM_HIGHDIFF_PORT") - if port_value is None: - return None - try: - port = int(port_value) - except ValueError as exc: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be an integer") from exc - if not 0 < port < 65536: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must be a valid TCP port") - min_difficulty = env_decimal("PRISM_STRATUM_HIGHDIFF_MIN_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY) - start_difficulty = env_decimal("PRISM_STRATUM_HIGHDIFF_START_DIFF", DEFAULT_HIGHDIFF_DIFFICULTY) - max_difficulty = env_decimal("PRISM_STRATUM_HIGHDIFF_MAX_DIFF", DEFAULT_HIGHDIFF_MAX_DIFFICULTY) - if min_difficulty > start_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_MIN_DIFF exceeds PRISM_STRATUM_HIGHDIFF_START_DIFF") - if start_difficulty > max_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_START_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") - # The fixed difficulty (used when vardiff is disabled) tracks the start - # difficulty unless explicitly set, and must respect the listener bounds: - # advertising below the floor would break the marketplace verification - # this listener exists for. - share_value = env_optional("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF") - if share_value is None: - share_difficulty = start_difficulty - else: - try: - share_difficulty = Decimal(share_value) - except Exception as exc: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be a decimal") from exc - if not share_difficulty.is_finite() or share_difficulty <= 0: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF must be positive") - if share_difficulty < min_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF is below PRISM_STRATUM_HIGHDIFF_MIN_DIFF") - if share_difficulty > max_difficulty: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_SHARE_DIFF exceeds PRISM_STRATUM_HIGHDIFF_MAX_DIFF") - try: - config = dataclass_replace( - base_vardiff_config, - min_difficulty=min_difficulty, - max_difficulty=max_difficulty, - startup_difficulty=start_difficulty, - ) - except ValueError as exc: - raise SystemExit(f"invalid PRISM_STRATUM_HIGHDIFF_* difficulty bounds: {exc}") from exc - return StratumListenerProfile( - name="highdiff", - # env_optional so an empty value (compose default passthrough) means - # "inherit the default listener bind" instead of a startup failure. - bind=env_optional("PRISM_STRATUM_HIGHDIFF_BIND") or base_bind, - port=port, - share_difficulty=share_difficulty, - vardiff_config=config, - heartbeat_name="stratum_accept_highdiff", - minimum_advertised_difficulty=min_difficulty, - ) - - -def parse_stratum_password_options(password: str) -> tuple[Decimal | None, Decimal | None]: - """Extract the pool-side d=N / md=N difficulty convention from a password. - - Unknown tokens and malformed values are ignored: miners routinely send - junk passwords ("x") and rejecting them would break every such rig. - Returns (requested_difficulty, requested_min_difficulty). - """ - - requested: Decimal | None = None - requested_min: Decimal | None = None - for token in password.split(","): - key, separator, raw_value = token.strip().partition("=") - if not separator: - continue - key = key.strip().lower() - if key not in {"d", "md"}: - continue - try: - value = Decimal(raw_value.strip()) - except Exception: - continue - if not value.is_finite() or value <= 0: - continue - if key == "d": - requested = value - else: - requested_min = value - return requested, requested_min - - -def default_prism_payout_policy() -> dict[str, object]: - policy: dict[str, object] = { - "p2mr_spend_input_bytes": env_positive_int( - "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES", - DEFAULT_P2MR_SPEND_INPUT_BYTES, - ), - "target_feerate_sats_per_byte": env_positive_int_with_legacy( - "PRISM_PAYOUT_TARGET_FEERATE_BITS_PER_BYTE", - "PRISM_PAYOUT_TARGET_FEERATE_SATS_PER_BYTE", - DEFAULT_MIN_OUTPUT_FEERATE_SATS_PER_BYTE, - ), - "safety_multiplier": env_positive_int( - "PRISM_PAYOUT_SAFETY_MULTIPLIER", - DEFAULT_MIN_OUTPUT_SAFETY_MULTIPLIER, - ), - } - min_output_sats = env_optional_positive_int_with_legacy( - "PRISM_PAYOUT_MIN_OUTPUT_BITS", - "PRISM_PAYOUT_MIN_OUTPUT_SATS", - ) - if min_output_sats is not None: - policy["min_output_sats"] = min_output_sats - return policy - - -def default_prism_coinbase_tag_hex() -> str: - tag = os.environ.get("PRISM_COINBASE_TAG", DEFAULT_PRISM_COINBASE_TAG) - try: - tag_bytes = tag.encode("ascii") - except UnicodeEncodeError as exc: - raise SystemExit("PRISM_COINBASE_TAG must be ASCII") from exc - if len(tag_bytes) > MAX_PRISM_COINBASE_TAG_BYTES: - raise SystemExit( - f"PRISM_COINBASE_TAG must be at most {MAX_PRISM_COINBASE_TAG_BYTES} bytes" - ) - if any(byte < 0x20 or byte > 0x7e for byte in tag_bytes): - raise SystemExit("PRISM_COINBASE_TAG must contain printable ASCII only") - return tag_bytes.hex() - - -def default_prism_username_fallback_address() -> str | None: - configured = env_optional("PRISM_USERNAME_FALLBACK_ADDRESS") - if configured is not None: - return configured - if (os.environ.get("QBIT_CHAIN") or "regtest").lower() in TESTNET_QBIT_CHAINS: - return DEFAULT_TESTNET_USERNAME_FALLBACK_ADDRESS - return None - - def qbit_gbt_rules(chain: str) -> list[str]: rules = ["segwit"] if chain.strip().lower() == "signet": @@ -880,166 +524,6 @@ def canonical_json_sha256(value: object) -> str: return hashlib.sha256(canonical_json_text(value).encode()).hexdigest() -class JsonRpc: - def __init__(self, *, host: str, port: int, user: str, password: str): - self.host = host - self.port = port - self.url = f"http://{host}:{port}" - credentials = f"{user}:{password}".encode() - self.auth = f"Basic {base64.b64encode(credentials).decode()}" - # Keep-alive connections, one per calling thread. qbitd is called on - # the hot share/block paths (a fresh getaddrinfo + TCP connect per call - # was ~seconds of overhead under load); reusing the connection removes - # that. threading.local keeps each thread's HTTPConnection private, so - # concurrent callers never share a non-thread-safe connection. - self._connections = threading.local() - - def _acquire_connection(self, timeout: float) -> http.client.HTTPConnection: - conn = getattr(self._connections, "conn", None) - if conn is None: - conn = http.client.HTTPConnection(self.host, self.port, timeout=timeout) - self._connections.conn = conn - else: - # Reuse: refresh the deadline for this call on the live socket. - conn.timeout = timeout - if conn.sock is not None: - conn.sock.settimeout(timeout) - return conn - - def _drop_connection(self) -> None: - conn = getattr(self._connections, "conn", None) - if conn is not None: - try: - conn.close() - except Exception: - pass - self._connections.conn = None - - def call( - self, - method: str, - params: list[object] | None = None, - *, - wallet: str | None = None, - timeout: float = 10, - ) -> Any: - body = json.dumps( - { - "jsonrpc": "1.0", - "id": method, - "method": method, - "params": params or [], - } - ).encode() - path = "/" - if wallet is not None: - path = f"/wallet/{urllib.parse.quote(wallet, safe='')}" - headers = { - "Authorization": self.auth, - "Content-Type": "application/json", - "User-Agent": "qbit-prism-coordinator/0.1", - } - # One retry with a fresh connection on a transport error. The usual - # cause is the server having closed an idle keep-alive connection, in - # which case the request never reached qbitd, so retrying is safe; the - # only state-changing RPC (submitblock) is idempotent (duplicate -> - # "duplicate") regardless. A second failure raises to the caller, which - # treats it as backend-rpc-unavailable (a rejected share/block, never a - # lost or double-counted block). - last_exc: Exception | None = None - for attempt in range(2): - conn = self._acquire_connection(timeout) - try: - conn.request("POST", path, body=body, headers=headers) - response = conn.getresponse() - data = response.read() # drain so the connection can be reused - except (http.client.HTTPException, OSError) as exc: - last_exc = exc - self._drop_connection() - if attempt == 0: - continue - raise - if response.status != 200: - # Non-200 bodies may hold a JSON-RPC error (qbitd returns the - # error object with a 500 for some methods); surface it as the - # same RuntimeError text callers already match on (e.g. the - # "-32601 / Method not found" blockwait-unsupported probe). - self._drop_connection() - detail = data.decode("utf-8", "replace") - try: - error = json.loads(detail).get("error") - except Exception: - error = None - if error is not None: - raise RuntimeError(f"qbit RPC {method} failed: {error}") - raise RuntimeError(f"qbit RPC {method} HTTP {response.status}: {detail[:200]}") - payload = json.loads(data) - if payload["error"] is not None: - raise RuntimeError(f"qbit RPC {method} failed: {payload['error']}") - return payload["result"] - raise last_exc if last_exc is not None else RuntimeError("qbit RPC call failed") - - -@dataclass(frozen=True) -class WorkerIdentity: - username: str - payout_address: str - worker_name: str | None - script_pubkey_hex: str - p2mr_program_hex: str - - -@dataclass -class _P2mrAddressValidationFlight: - event: threading.Event = field(default_factory=threading.Event) - result: tuple[str, str] | None = None - error: BaseException | None = None - waiters: int = 0 - - -@dataclass(frozen=True) -class PrismJobContext: - job: direct_stratum.DirectQbitStratumJob - template: dict[str, Any] - shares_json: list[dict[str, object]] - prior_balances: list[dict[str, object]] - found_block: dict[str, object] - share_weight: int - collection_only: bool - worker: WorkerIdentity - issued_at_ms: int - template_fingerprint: str | None = None - template_generation: int = 0 - payout_state_generation: int = 0 - prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None - payout_artifact_generation: int = 0 - connection_id: int = 0 - authorization_generation: int = 0 - difficulty_generation: int = 0 - - -@dataclass -class PendingShareAppend: - """A share waiting for the ledger group-commit writer. - - The client thread does not count or acknowledge this share until - ``committed`` is set successfully. A block candidate intent, when - present, is inserted in the same transaction as the share. - """ - - pending_share: PendingShare - username: str - job_id: str - block_hash_hex: str - collection_only: bool - credit_policy: str | None - candidate_intent: dict[str, Any] | None = None - committed: threading.Event = field(default_factory=threading.Event) - record: Any | None = None - error: BaseException | None = None - writer_token: _WriterOperationToken | None = None - - @dataclass(frozen=True) class PrismBlockCandidate: """A block-worthy submission queued for the block-submitter thread. @@ -1062,144 +546,6 @@ class PrismBlockCandidate: credit_share_on_accept: bool = False -@dataclass(frozen=True) -class CachedTemplateArtifacts: - """Template plus everything derivable from it alone, shared by all clients. - - Derived fields are keyed by the template fingerprint: a refetch whose - fingerprint matches (only clock fields moved) reuses the previously - computed transaction hexes and witness merkle leaves instead of re-hashing - the full template. Generation records observation-start order so a slow, - older fetch cannot supersede a newer observation merely by finishing last. - """ - - template: dict[str, Any] - fingerprint: str - previousblockhash: str - transaction_hexes: tuple[str, ...] - witness_merkle_leaves_hex: tuple[str, ...] - network_difficulty: int - fetched_monotonic: float - generation: int = 0 - - -@dataclass(frozen=True) -class QbitTipTemplateSnapshot: - bestblockhash: str - previousblockhash: str - template_fingerprint: str - template_generation: int = 0 - # The observation owns this exact artifact object. It deliberately does - # not participate in snapshot equality: callers compare the stable - # identity fields above, while refresh preparation consumes the exact - # template and derivations that were observed even if the mutable cache's - # current pointer is replaced concurrently. - template_artifacts: CachedTemplateArtifacts | None = field( - default=None, - compare=False, - repr=False, - ) - - -@dataclass(frozen=True) -class RetainedCollectionRefresh: - """Current immutable preparation waiting for a collection identity.""" - - snapshot: QbitTipTemplateSnapshot - observation_sequence: int - payout_state_generation: int - - -@dataclass(frozen=True) -class PayoutStateArtifact: - """Immutable ledger-backed inputs published with one payout generation.""" - - generation: int - source_generation: int - prior_balances_json: str = field(repr=False) - prior_balances_sha256: str - prepared_monotonic: float - - def prior_balances(self) -> list[dict[str, object]]: - value = json.loads(self.prior_balances_json) - if not isinstance(value, list): - raise RuntimeError("published payout artifact is not a balance list") - return value - - -@dataclass(frozen=True) -class JobBuildKey: - """Every immutable input that can affect one constructed mining job.""" - - best_tip_hash: str - previous_block_hash: str - template_fingerprint: str - template_generation: int - payout_state_generation: int - payout_artifact_sha256: str - mode: str - collection_identity: tuple[str, str] | None - block_height: int - coinbase_value_sats: int - network_difficulty: int - issued_at_ms: int - payout_policy_sha256: str - ctv_settlement_sha256: str | None - witness_merkle_sha256: str - transaction_set_sha256: str - coinbase_suffix_hex: str - signing_key_sha256: str - ledger_signing_key_sha256: str - numeric_context_sha256: str - share_snapshot_sha256: str = "" - - -@dataclass(frozen=True) -class CachedJobBundle: - """One heavy job build (ledger snapshot + signed manifest + base job) - shared across every client on the same template. - - The base job is built with the extranonce1 placeholder; per-client jobs - are stamped from it by swapping job_id, extranonce1, difficulty, and the - clean_jobs flag. All other fields are byte-identical across clients - because the stratum coinbase split excludes the extranonce window. - """ - - key: tuple[object, ...] - template: dict[str, Any] - template_fingerprint: str - coinbase_manifest: dict[str, Any] - shares_json: list[dict[str, object]] - prior_balances: list[dict[str, object]] - found_block: dict[str, object] - collection_only: bool - issued_at_ms: int - base_job: direct_stratum.DirectQbitStratumJob - built_monotonic: float - template_generation: int = 0 - payout_state_generation: int = 0 - payout_artifact_generation: int = 0 - # Collection coinbases commit a synthetic share to this exact payout - # identity. Ready bundles have no worker-specific inputs and keep this - # unset, which makes accidental cross-worker stamping fail closed. - collection_identity: tuple[str, str] | None = None - # Compact carry-forward state derived from the immutable job summary. It - # lets an accepted block publish child payout state without retaining the - # full audit bundle (and its duplicate shares tree) in every cached job. - prospective_prior_balances: tuple[tuple[str, str, str, int], ...] | None = None - build_key: JobBuildKey | None = None - - -@dataclass(frozen=True) -class _AcceptedBlockPayoutTransition: - """Prospective balances for one durable candidate across its landing seam.""" - - block_height: int | None = None - landed: bool = False - preview: tuple[tuple[str, str, str, int], ...] | None = None - published_generation: int | None = None - - @dataclass(frozen=True) class _IdleRetargetRequest: """Immutable idle-window identity captured by one bounded sweep.""" @@ -1213,1304 +559,1187 @@ class _IdleRetargetRequest: elapsed_seconds: Decimal -@dataclass(frozen=True) -class EvictedJobEntry: - context: PrismJobContext - connection_id: int - evicted_monotonic: float - previousblockhash: str - client: ClientState | None = None +class _BundlePreparationSuperseded(TemplateRefreshSuperseded): + """The exact work identity lost to a newer tip/template observation. + Subclasses TemplateRefreshSuperseded: losing the shared-bundle build race + to a newer tip/template observation is coordination churn, so it escapes + the poll without arming the template-refresh failure budget. + """ -@dataclass(frozen=True) -class RefreshResult: - result: str - delivered_monotonic: float | None = None +class _CoordinatorSessionRuntime(SessionRuntimePort): + """Dynamic compatibility adapter for the extracted session service.""" -@dataclass(frozen=True, eq=False) -class TipRefreshValidationToken: - """Immutable proof that one prepared refresh passed its expensive guard.""" + def __init__(self, coordinator: PrismCoordinator) -> None: + self.coordinator = coordinator - tip_hash: str - template_fingerprint: str - template_generation: int - payout_state_generation: int - observation_sequence: int - build_key: JobBuildKey - snapshot: QbitTipTemplateSnapshot = field(repr=False) + def running(self) -> bool: + coordinator = self.coordinator + stop_event = getattr(coordinator, "stop_event", None) + if stop_event is not None and stop_event.is_set(): + return False + return coordinator._ensure_shutdown_controller().phase == "running" + def record_heartbeat(self, name: str) -> None: + self.coordinator._record_heartbeat(name) -@dataclass(frozen=True) -class PayoutStateCandidate: - """Immutable result of payout work prepared outside delivery admission.""" - - base_generation: int - source_generation: int - source_tip_hash: str | None - cause: str - invalidated_monotonic: float - prepared_monotonic: float - accepted_block_hash: str | None = None - accepted_block_preview: tuple[tuple[str, str, str, int], ...] | None = None - accepted_block_withdrawal: bool = False - accepted_block_height: int | None = None - ledger_artifact: PayoutLedgerArtifact | None = field( - default=None, - compare=False, - repr=False, - ) + def wait_after_resource_failure(self, heartbeat_name: str) -> None: + coordinator = self.coordinator + remaining_seconds = max( + 0.0, + float( + getattr( + coordinator, + "stratum_accept_resource_exhaustion_backoff_seconds", + DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, + ) + ), + ) + watchdog_timeout_seconds = max( + 0.001, float(getattr(coordinator, "watchdog_timeout_seconds", 120.0)) + ) + heartbeat_interval_seconds = max( + 0.001, min(1.0, watchdog_timeout_seconds / 2.0) + ) + deadline = time.monotonic() + remaining_seconds + while not coordinator.stop_event.is_set(): + coordinator._record_heartbeat(heartbeat_name) + remaining_seconds = deadline - time.monotonic() + if remaining_seconds <= 0: + return + if coordinator.stop_event.wait( + min(remaining_seconds, heartbeat_interval_seconds) + ): + return + def record_resource_exhaustion( + self, + *, + listener_name: str, + location: str, + error_number: int | None, + ) -> None: + coordinator = self.coordinator + with coordinator.lock: + coordinator.accept_resource_exhaustion_count = int( + getattr(coordinator, "accept_resource_exhaustion_count", 0) + ) + 1 + exhaustion_count = coordinator.accept_resource_exhaustion_count + if exhaustion_count == 1 or exhaustion_count % 100 == 0: + print( + "prism coordinator: stratum resource exhaustion " + f"listener={listener_name} location={location} errno={error_number} " + f"count={exhaustion_count}", + flush=True, + ) -@dataclass(frozen=True) -class PublishedPayoutState: - """The payout snapshot identity to which cached jobs are stamped.""" + def record_setup_failure(self) -> int: + coordinator = self.coordinator + with coordinator.lock: + coordinator.connection_setup_failure_count = int( + getattr(coordinator, "connection_setup_failure_count", 0) + ) + 1 + return coordinator.connection_setup_failure_count - generation: int - source_generation: int - source_tip_hash: str | None - published_monotonic: float - artifact: PayoutStateArtifact | None = field(default=None, repr=False) + def sync_registry_metrics(self, registry: SessionRegistry) -> None: + coordinator = self.coordinator + coordinator.clients = registry.clients + coordinator.connection_counter = registry.connection_generation + coordinator.connection_limit_rejection_counts = registry.rejection_counts + coordinator.peak_active_connection_count = registry.peak_active_connections + coordinator.handler_thread_count = registry.handler_thread_count + def max_connections(self) -> int: + return int( + getattr( + self.coordinator, + "stratum_max_connections", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + ) + ) -@dataclass(frozen=True) -class PayoutLedgerArtifact: - """Immutable ledger input prepared independently of a qbit template.""" - - generation: int - payout_state_generation: int - network_difficulty: int - accepted_share_count: int - shares_json: tuple[dict[str, object], ...] = field(repr=False) - prior_balances: tuple[dict[str, object], ...] = field(repr=False) - prepared_monotonic: float - # The anchor the share snapshot was actually taken at. Bundles built from - # this artifact must declare it as anchor_job_issued_at_ms: an auditor - # replaying qbit_audit_share_window at the declared anchor must reproduce - # exactly these shares, which only holds at the snapshot's own anchor. - snapshot_anchor_ms: int | None = None - - -@dataclass -class _PayoutDeliveryAdmission: - admitted: bool - wait_seconds: float - generation: int - published_generation: int - relation: str - delivered: bool = False - - def __bool__(self) -> bool: - return self.admitted - - def mark_delivered(self) -> None: - if not self.admitted: - raise RuntimeError("payout delivery completed without admission") - self.delivered = True - - -@dataclass(eq=False) -class PendingInitialJob: - client: ClientState - authorization_generation: int - worker: WorkerIdentity - requested_monotonic: float - deadline_monotonic: float | None - connection_id: int | None = None - difficulty_generation: int | None = None - cancelled: threading.Event = field(default_factory=threading.Event) - future: Future[bool] | None = None - predecessor: Future[bool] | None = None + def max_connections_per_username(self) -> int: + return int( + getattr( + self.coordinator, + "stratum_max_connections_per_username", + DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, + ) + ) + def client_startup_difficulty(self, profile: StratumListenerProfile) -> Decimal: + return self.coordinator.client_startup_difficulty(profile) -class _DeliveryQueueFull(RuntimeError): - """The bounded delivery executor cannot admit another task.""" + def apply_send_timeout(self, sock: socket.socket) -> None: + self.coordinator.apply_stratum_send_timeout(sock) + def make_client_thread(self, client: ClientState) -> threading.Thread: + return threading.Thread( + target=self.coordinator.handle_client, + args=(client,), + name=f"prism-stratum-client-{client.connection_id}", + daemon=True, + ) -class _JobBuildCancelled(RuntimeError): - """A bundle waiter became obsolete before it acquired preparation.""" + def extranonce2_size(self) -> int: + return int(self.coordinator.extranonce2_size) + def version_mask(self) -> int: + return int(self.coordinator.version_mask) -class _BoundedPriorityExecutor: - """Small Future-compatible executor with bounded, priority-ordered work.""" + def username_fallback_address(self) -> str | None: + return getattr( + self.coordinator, + "username_fallback_address", + default_prism_username_fallback_address(), + ) - def __init__( + def resolve_worker( + self, username: str, fallback: Callable[[], WorkerIdentity] + ) -> WorkerIdentity: + override = self.coordinator.__dict__.get("resolve_worker") + return override(username) if override is not None else fallback() + + def reserve_client_username( self, - *, - max_workers: int, - max_queue_size: int, - thread_name_prefix: str = "prism-job-delivery", + client: ClientState, + worker: WorkerIdentity, + fallback: Callable[[], bool], + ) -> bool: + override = self.coordinator.__dict__.get("reserve_client_username") + return bool(override(client, worker)) if override is not None else fallback() + + def send_result( + self, client: ClientState, request_id: object, result: object ) -> None: - self.max_workers = max_workers - self.max_queue_size = max_queue_size - self._queue: queue.PriorityQueue[tuple[object, ...]] = queue.PriorityQueue( - maxsize=max_queue_size - ) - self._lock = threading.Lock() - self._sequence = 0 - self._active_workers = 0 - self._shutdown = False - self._threads = [ - threading.Thread( - target=self._worker, - name=f"{thread_name_prefix}-{index + 1}", - daemon=True, - ) - for index in range(max_workers) - ] - for thread in self._threads: - thread.start() + override = self.coordinator.__dict__.get("send_result") + if override is not None: + override(client, request_id, result) + else: + client.send(stratum_result_payload(request_id, result)) - def submit( + def send_error( self, - function: Callable[..., Any], - /, - *args: object, - priority: int = PRISM_DELIVERY_PRIORITY_SAME_TIP, - **kwargs: object, - ) -> Future[Any]: - future: Future[Any] = Future() - with self._lock: - if self._shutdown: - raise RuntimeError("delivery executor is shut down") - self._sequence += 1 - item = ( - int(priority), - self._sequence, - future, - function, - args, - kwargs, - ) - try: - self._queue.put_nowait(item) - except queue.Full as exc: - raise _DeliveryQueueFull("delivery executor queue is full") from exc - return future + client: ClientState, + request_id: object, + code: int, + message: str, + *, + reason: str | None, + ) -> None: + override = self.coordinator.__dict__.get("send_error") + if override is not None: + override(client, request_id, code, message, reason=reason) + else: + client.send(stratum_error_payload(request_id, code, message, reason=reason)) - def _worker(self) -> None: - while True: - item = self._queue.get() - _, _, future, function, args, kwargs = item - if function is None: - self._queue.task_done() - return - assert isinstance(future, Future) - if not future.set_running_or_notify_cancel(): - self._queue.task_done() - continue - with self._lock: - self._active_workers += 1 - try: - result = function(*args, **kwargs) - except BaseException as exc: - future.set_exception(exc) - else: - future.set_result(result) - finally: - with self._lock: - self._active_workers -= 1 - self._queue.task_done() - - def cancel(self, future: Future[Any]) -> bool: - """Cancel ``future`` and immediately discard it when still queued. - - ``Future.cancel`` alone leaves the cancelled entry in PriorityQueue - until a worker dequeues it. Removing the exact entry while holding the - queue mutex makes bounded admission available to a replacement at the - cancellation boundary. A worker that already dequeued the entry owns - the normal ``task_done`` path, so the two paths cannot double-release. - """ - removed = False - with self._queue.mutex: - queued_items = self._queue.queue - for index, item in enumerate(queued_items): - if item[2] is not future: - continue - queued_items.pop(index) - heapq.heapify(queued_items) - self._queue.unfinished_tasks -= 1 - if self._queue.unfinished_tasks == 0: - self._queue.all_tasks_done.notify_all() - self._queue.not_full.notify() - removed = True - break - # Invoke callbacks only after releasing the queue mutex. Initial-job - # cancellation callbacks may submit a replacement to this executor. - future.cancel() - return removed - - def stats(self) -> tuple[int, int]: - with self._lock: - return self._queue.qsize(), self._active_workers - - def shutdown(self, *, wait: bool = True, cancel_futures: bool = False) -> None: - with self._lock: - if self._shutdown: - threads = list(self._threads) - already_shutdown = True - else: - self._shutdown = True - threads = list(self._threads) - already_shutdown = False - if already_shutdown: - if wait: - for thread in threads: - thread.join() - return - if cancel_futures: - while True: - try: - item = self._queue.get_nowait() - except queue.Empty: - break - future = item[2] - if isinstance(future, Future): - future.cancel() - self._queue.task_done() - for index in range(len(threads)): - self._queue.put((math.inf, index, None, None, (), {})) - if wait: - for thread in threads: - thread.join() - - -class _FanoutCancellation: - """Cancel a fanout without racing already-admitted deliveries. - - ``cancel`` closes admission without waiting, so workers can call it while - holding a client lock. The fanout coordinator calls ``set`` outside client - locks to wait for deliveries that already passed the final gate. - """ + def disconnect_client( + self, client: ClientState, fallback: Callable[[], None] + ) -> None: + override = self.coordinator.__dict__.get("disconnect_client") + if override is not None: + override(client) + else: + fallback() - def __init__(self) -> None: - self._condition = threading.Condition() - self._cancelling = False - self._active_deliveries = 0 - def is_set(self) -> bool: - with self._condition: - return self._cancelling +class _CoordinatorSessionJobs(JobDeliveryPort): + def __init__(self, coordinator: PrismCoordinator) -> None: + self.coordinator = coordinator - def begin_delivery(self) -> bool: - with self._condition: - if self._cancelling: - return False - self._active_deliveries += 1 - return True + def note_collection_identity_available(self, client: ClientState) -> None: + self.coordinator._note_collection_identity_available(client) - def end_delivery(self) -> None: - with self._condition: - if self._active_deliveries <= 0: - raise RuntimeError("fanout delivery gate released without admission") - self._active_deliveries -= 1 - if self._active_deliveries == 0: - self._condition.notify_all() - - def cancel(self) -> None: - with self._condition: - self._cancelling = True - - def set(self) -> None: - self.cancel() - with self._condition: - while self._active_deliveries: - self._condition.wait() - - -@dataclass -class _JobBundleBuildControl: - key: tuple[object, ...] - previousblockhash: str - payout_state_generation: int - payout_artifact_generation: int - cancel_event: threading.Event = field(default_factory=threading.Event) - process: subprocess.Popen[str] | None = None - - -class _PayoutStateDeliveryGate: - """Order delivery admission around a very short payout publication. - - A publisher first closes admission and drains sends that already crossed - the boundary. It does not own the atomic publication section while that - drain is in progress. Once drained, publication ownership is transferred - to the caller for the generation/cache pointer swap only. - """ + def request_initial_job_delivery(self, client: ClientState) -> None: + self.coordinator.request_initial_job_delivery(client) - def __init__(self) -> None: - self._condition = threading.Condition() - self._active_deliveries = 0 - self._publisher_waiting = False - self._mutation_owner: int | None = None - self._mutation_depth = 0 - self._published_generation = 0 - self._priority_generation: int | None = None - self._delivery_blocked = False + def reauthorization_has_capacity(self, client: ClientState) -> bool: + return self.coordinator._ensure_job_delivery_service().reauthorization_has_capacity( + client + ) - @staticmethod - def _generation_relation(generation: int, published_generation: int) -> str: - if generation < published_generation: - return "stale" - if generation > published_generation: - return "future" - return "current" + def apply_client_difficulty_requests(self, client: ClientState) -> Decimal | None: + return self.coordinator.apply_client_difficulty_requests(client) - @contextmanager - def delivery(self) -> Iterator[None]: - with self.delivery_cancelable(lambda: False, priority=True) as admission: - if not admission: - raise RuntimeError("uncancelled payout delivery was not admitted") - yield - admission.mark_delivered() + def advertise_client_difficulty(self, client: ClientState, target: Decimal) -> bool: + return self.coordinator.advertise_client_difficulty(client, target) - @contextmanager - def delivery_cancelable( - self, - cancelled: Callable[[], bool], - *, - generation: int | None = None, - priority: bool = False, - poll_seconds: float = PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - ) -> Iterator[_PayoutDeliveryAdmission]: - """Admit a delivery unless cancellation wins while mutation owns the gate.""" + def handle_submit(self, client: ClientState, params: list[object]) -> bool: + return self.coordinator.handle_submit(client, params) - started = time.monotonic() - admitted = False - with self._condition: - if generation is None: - generation = self._published_generation - while True: - if cancelled(): - break - if self._delivery_blocked: - break - if generation < self._published_generation: - break - publication_blocked = ( - self._publisher_waiting or self._mutation_owner is not None - ) - priority_blocked = ( - self._priority_generation is not None - and generation == self._priority_generation - and not priority - ) - if priority_blocked: - # A same-generation job that is not for the published tip - # must not occupy a waiter slot indefinitely. Reject it so - # its caller can rebuild current-tip work; the reserved - # first-delivery lane remains available to priority work. - break - future_blocked = generation > self._published_generation - if ( - not publication_blocked - and not future_blocked - ): - self._active_deliveries += 1 - admitted = True - break - self._condition.wait(timeout=poll_seconds) - published_generation = self._published_generation - relation = self._generation_relation(generation, published_generation) - admission = _PayoutDeliveryAdmission( - admitted=admitted, - wait_seconds=max(0.0, time.monotonic() - started), - generation=generation, - published_generation=published_generation, - relation=relation, - ) - try: - yield admission - finally: - if admitted: - with self._condition: - if self._active_deliveries <= 0: - raise RuntimeError("payout delivery gate released without admission") - self._active_deliveries -= 1 - if ( - priority - and admission.delivered - and generation == self._priority_generation - ): - # Keep routine same-generation sends queued until the - # first prioritized current-tip socket delivery exits. - # Privileged synchronization admissions do not consume - # this reservation merely by leaving the gate. - self._priority_generation = None - if self._active_deliveries == 0: - self._condition.notify_all() - elif self._priority_generation is None: - self._condition.notify_all() + def refresh_jobs_after_pending_accepted_block(self, client: ClientState) -> None: + self.coordinator.refresh_jobs_after_pending_accepted_block(client) - @contextmanager - def publication(self) -> Iterator[None]: - owner = threading.get_ident() - with self._condition: - if self._mutation_owner == owner: - self._mutation_depth += 1 - else: - while self._mutation_owner is not None or self._publisher_waiting: - self._condition.wait() - self._publisher_waiting = True - while self._active_deliveries: - self._condition.wait() - self._mutation_owner = owner - self._mutation_depth = 1 - self._publisher_waiting = False - try: - yield - finally: - with self._condition: - if self._mutation_owner != owner or self._mutation_depth <= 0: - raise RuntimeError("payout mutation gate released by non-owner") - self._mutation_depth -= 1 - if self._mutation_depth == 0: - self._mutation_owner = None - self._condition.notify_all() - - def publish_generation(self, generation: int, *, prioritize_delivery: bool) -> None: - owner = threading.get_ident() - with self._condition: - if self._mutation_owner != owner: - raise RuntimeError("payout generation published outside atomic section") - if generation <= self._published_generation: - raise RuntimeError("payout generation did not advance") - self._published_generation = generation - self._priority_generation = generation if prioritize_delivery else None - self._delivery_blocked = False - - def block_delivery( + def cancel_pending_initial_job_locked( self, - mark_blocked: Callable[[], bool] | None = None, - ) -> bool: - """Reject admission until publication, atomically with caller state.""" - - with self._condition: - # A publisher drops the condition while swapping its immutable - # pointer. Wait only for that short section, never for admitted - # socket sends or fanout cancellation. - while self._mutation_owner is not None: - self._condition.wait() - if mark_blocked is not None and not mark_blocked(): - return False - self._delivery_blocked = True - self._condition.notify_all() - return True - - @contextmanager - def mutation(self) -> Iterator[None]: - """Compatibility alias for tests and callers that only need exclusion.""" + client: ClientState, + ) -> Callable[[], object] | None: + request = self.coordinator._cancel_pending_initial_job_locked( + client, + count=True, + ) + if request is None or request.future is None: + return None + return lambda: self.coordinator._cancel_initial_job_future(request.future) - with self.publication(): - yield + def cleanup_disconnected_client(self, client: ClientState) -> None: + coordinator = self.coordinator + with coordinator.lock: + coordinator._ensure_job_delivery_service().retire_client_locked(client) + client.authorized = False + client.worker = None + client.username = "" + def retain_current_collection_refresh_if_unrepresented(self) -> None: + self.coordinator._retain_current_collection_refresh_if_unrepresented() -@dataclass -class _SharedBundlePreparationFlight: - event: threading.Event = field(default_factory=threading.Event) - result: CachedJobBundle | None = None - error: BaseException | None = None - waiters: int = 0 +class _CoordinatorSessionProgress(ProgressHealthPort): + def __init__(self, coordinator: PrismCoordinator) -> None: + self.coordinator = coordinator -@dataclass(eq=False) -class ClientState: - sock: socket.socket - address: tuple[str, int] - connection_id: int - extranonce1_hex: str - subscribed: bool = False - authorized: bool = False - authorization_generation: int = 0 - difficulty_generation: int = 0 - authorized_monotonic: float | None = None - username: str = "" - worker: WorkerIdentity | None = None - version_mask: int = 0 - active_job: PrismJobContext | None = None - # active_job is registered before the potentially blocking socket write. - # These delivery-proof fields advance only after send_job_update succeeds. - _progress_delivered_context: PrismJobContext | None = None - _progress_delivered_template_fingerprint: str | None = None - _progress_delivered_template_generation: int = 0 - _progress_delivered_payout_generation: int = -1 - _progress_delivered_monotonic: float | None = None - listener_name: str = "default" - # Pristine difficulty policy of the accepting listener; never mutated. - listener_vardiff_config: vardiff.VardiffConfig | None = None - # Floor below which stamped jobs never advertise, copied from the - # accepting listener profile. Zero (default listener) keeps the network - # cap authoritative. - minimum_advertised_difficulty: Decimal = Decimal("0") - # Per-client specialization of the listener policy (password d=/md= or - # mining.suggest_difficulty); recomputed from the pristine base on every - # request so repeat applications cannot compound. - vardiff_config: vardiff.VardiffConfig | None = None - requested_difficulty: Decimal | None = None - requested_min_difficulty: Decimal | None = None - suggested_difficulty: Decimal | None = None - share_difficulty: Decimal = Decimal("1") - pending_share_difficulty: Decimal | None = None - vardiff_window_started_monotonic: float = field(default_factory=time.monotonic) - vardiff_window_accepted: int = 0 - vardiff_window_submitted: int = 0 - vardiff_window_work: Decimal = Decimal("0") - vardiff_difficulty_estimate: Decimal | None = None - # Owns all per-client difficulty policy, estimates, and Vardiff windows. - # Share handlers release it before acquiring job_update_lock or the - # coordinator lock; job delivery may acquire it while job_update_lock is - # held. Keeping this lock per connection prevents one miner's accounting - # or slow socket from convoying tip publication for the whole coordinator. - vardiff_lock: threading.RLock = field(default_factory=threading.RLock) - active_job_ids: set[str] = field(default_factory=set) - post_accept_refresh_block: tuple[int, str] | None = None - # (job previousblockhash, monotonic) of the FIRST job this connection was - # sent for that tip. Anchors the per-connection stale-grace window: a - # prior-tip share is in flight until shortly after this connection - # received replacement work, however long the refresh pass took to reach - # it. See stale_grace_deadline_open. - tip_work_delivered: tuple[str, float] | None = None - # Protected by the coordinator lock. Disconnect retirement sets this before - # waiting for any per-client job update so queued work can reject the client. - closing: bool = False - # Serializes every job build/register/send transition for this connection. - # The coordinator lock may be acquired while this lock is held, never in - # the reverse order. RLock permits authorize/retarget helpers to call the - # common maybe_send_job path while retaining the same serialization scope. - job_update_lock: threading.RLock = field(default_factory=threading.RLock) - send_lock: threading.Lock = field(default_factory=threading.Lock) - handler_thread_registered: bool = False - - def send(self, payload: dict[str, object]) -> None: - data = json.dumps(payload).encode() + b"\n" - with self.send_lock: - self.sock.sendall(data) - - def send_batch(self, payloads: list[dict[str, object]]) -> None: - # Tests and embedders may replace ``send`` with an in-memory recorder; - # retain that seam while production sockets write the whole difficulty - # + notify pair under one send lock with no response interleaving. - if "send" in self.__dict__: - for payload in payloads: - self.send(payload) - return - data = b"".join(json.dumps(payload).encode() + b"\n" for payload in payloads) - with self.send_lock: - self.sock.sendall(data) + def record_delivery( + self, + client: ClientState, + context: object, + delivered_monotonic: float, + ) -> None: + self.coordinator._record_progress_delivery_to_health( + client, + context, # type: ignore[arg-type] + delivered_monotonic, + ) - def close(self) -> None: - try: - self.sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - try: - self.sock.close() - except OSError: - pass + def reconcile_eligibility(self) -> None: + coordinator = self.coordinator + service = getattr(coordinator, "progress_health_service", None) + if service is not None: + service.reconcile_pending(coordinator._progress_eligibility_snapshot()) -class StratumError(RuntimeError): +class _CoordinatorJobPreparation(JobPreparationPort): def __init__( self, - code: int, - message: str, *, - reason: str | None = None, - disconnect: bool = False, - ): - super().__init__(message) - self.code = code - self.message = message - self.reason = reason - self.disconnect = disconnect - - -class ShutdownInProgress(RuntimeError): - """Raised when work that could mutate the ledger arrives after shutdown.""" + ensure_reorg_current: Callable[[], bool], + issuance_artifacts: Callable[[], CachedTemplateArtifacts], + shared_bundle: Callable[..., CachedJobBundle], + artifacts_current: Callable[[CachedTemplateArtifacts], bool], + clear_artifacts: Callable[[CachedTemplateArtifacts], None], + record_failure: Callable[[], None], + phases: Callable[[], dict[str, float]], + retained_artifacts: Callable[[], CachedTemplateArtifacts | None], + chain_view_untrusted: Callable[[], bool], + admit_idle_bundle_source: Callable[..., AdmittedIdleBundleSource | None], + observe_elapsed: Callable[[float, Mapping[str, float]], None], + collection_identity: Callable[[WorkerIdentity], object], + ready_latched: Callable[[], bool], + template_fingerprint: Callable[[Mapping[str, object]], str], + ) -> None: + self._ensure_reorg_current = ensure_reorg_current + self._issuance_artifacts = issuance_artifacts + self._shared_bundle = shared_bundle + self._artifacts_current = artifacts_current + self._clear_artifacts = clear_artifacts + self._record_failure = record_failure + self._phases = phases + self._retained_artifacts = retained_artifacts + self._chain_view_untrusted = chain_view_untrusted + self._admit_idle_bundle_source = admit_idle_bundle_source + self._observe_elapsed = observe_elapsed + self._collection_identity = collection_identity + self._ready_latched = ready_latched + self._template_fingerprint = template_fingerprint + + def ensure_reorg_current(self) -> bool: + return self._ensure_reorg_current() + + def issuance_artifacts(self) -> CachedTemplateArtifacts: + return self._issuance_artifacts() + + def shared_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: WorkerIdentity, + *, + cancelled: Callable[[], bool] | None = None, + request_source: str = "routine", + ) -> CachedJobBundle: + return self._shared_bundle( + artifacts, + worker, + cancelled=cancelled, + request_source=request_source, + ) + def artifacts_current(self, artifacts: CachedTemplateArtifacts) -> bool: + return self._artifacts_current(artifacts) -class _WriterOperationToken: - """One transferable writer admission held until durable work completes.""" + def clear_artifacts(self, artifacts: CachedTemplateArtifacts) -> None: + self._clear_artifacts(artifacts) - def __init__(self, controller: "CoordinatorShutdownController", component: str): - self.controller = controller - self.component = component - self.finished = False + def record_failure(self) -> None: + self._record_failure() - def finish(self) -> None: - self.controller.finish_token(self) + def phases(self) -> dict[str, float]: + return self._phases() + def retained_artifacts(self) -> CachedTemplateArtifacts | None: + return self._retained_artifacts() -class CoordinatorShutdownController: - """Coordinates the writer barrier, one-shot lease release, and final drain. + def chain_view_untrusted(self) -> bool: + return self._chain_view_untrusted() - Writer operations enter through :meth:`enter_writer` before shutdown or - inherit an already-admitted operation on the same thread. Queue admissions - use transferable tokens so a share remains visible to the barrier while it - moves from a client thread to the group-commit writer. - """ + def admit_idle_bundle_source( + self, + client: ClientState, + bundle: CachedJobBundle, + *, + allow_uncached: bool, + ) -> AdmittedIdleBundleSource | None: + return self._admit_idle_bundle_source( + client, + bundle, + allow_uncached=allow_uncached, + ) - def __init__(self, writer_quiescence_timeout_seconds: float): - self.writer_quiescence_timeout_seconds = writer_quiescence_timeout_seconds - self.condition = threading.Condition(threading.RLock()) - self.local = threading.local() - self.phase = "running" - self.reason: str | None = None - self.signal_number: int | None = None - self.sigterm_monotonic: float | None = None - self.shutdown_started_monotonic: float | None = None - self.active_writers: dict[str, int] = {} - self.shutdowns_total = 0 - self.writer_quiescence_outcomes = {"success": 0, "timeout": 0} - self.writer_quiescence_seconds = 0.0 - self.lease_release_attempts_total = 0 - self.lease_release_outcomes = { - "success": 0, - "not_held": 0, - "unsupported": 0, - "failure": 0, - } - self.lease_release_seconds = 0.0 - self.lease_release_attempted = False - self.lease_release_succeeded = False - self.lease_release_withheld = False - self.sigterm_to_lease_release_seconds = 0.0 - self.sigterm_release_observed = False - self.release_withheld_total = 0 - self.non_writer_drain_seconds = 0.0 - self.non_writer_drains_total = 0 - self._drain_claimed = False - - def request_shutdown(self, signum: int | None) -> None: - """Close admission atomically; the caller only needs to set its event.""" - now = time.monotonic() - with self.condition: - if signum == signal.SIGTERM and self.sigterm_monotonic is None: - self.sigterm_monotonic = now - if self.signal_number is None and signum is not None: - self.signal_number = signum - if self.phase == "running": - self.phase = "requested" - self.condition.notify_all() - - def begin_shutdown(self, reason: str) -> bool: - with self.condition: - if self.phase not in {"running", "requested"}: - return False - self.phase = "quiescing_writers" - self.reason = reason - self.shutdown_started_monotonic = time.monotonic() - self.shutdowns_total += 1 - self.condition.notify_all() - return True + def observe_elapsed( + self, + elapsed_seconds: float, + phases: Mapping[str, float], + ) -> None: + self._observe_elapsed(elapsed_seconds, phases) - def wait_for_lease_handling(self) -> bool: - """Wait for the one shutdown owner to release or safely withhold.""" - in_progress = { - "requested", - "quiescing_writers", - "writers_quiesced", - "releasing_lease", - } - with self.condition: - while self.phase in in_progress: - self.condition.wait() - return self.lease_release_succeeded - - def _thread_writer_depth(self) -> int: - return int(getattr(self.local, "writer_depth", 0)) - - def _admit_writer_locked(self, component: str, *, inherited: bool) -> _WriterOperationToken: - if self.lease_release_attempted: - raise ShutdownInProgress("PRISM writer lease release has already started") - if self.phase != "running" and not inherited: - raise ShutdownInProgress("PRISM coordinator is shutting down") - self.active_writers[component] = self.active_writers.get(component, 0) + 1 - return _WriterOperationToken(self, component) - - def enter_writer(self, component: str) -> _WriterOperationToken: - depth = self._thread_writer_depth() - with self.condition: - token = self._admit_writer_locked(component, inherited=depth > 0) - self.local.writer_depth = depth + 1 - return token - - def exit_writer(self, token: _WriterOperationToken) -> None: - depth = self._thread_writer_depth() - self.local.writer_depth = max(0, depth - 1) - token.finish() - - def reserve_writer(self, component: str) -> _WriterOperationToken: - """Reserve work that will finish on another thread.""" - with self.condition: - return self._admit_writer_locked( - component, - inherited=self._thread_writer_depth() > 0, - ) + def collection_identity(self, worker: WorkerIdentity) -> object: + return self._collection_identity(worker) - def finish_token(self, token: _WriterOperationToken) -> None: - with self.condition: - if token.finished: - return - token.finished = True - remaining = self.active_writers.get(token.component, 0) - 1 - if remaining > 0: - self.active_writers[token.component] = remaining - else: - self.active_writers.pop(token.component, None) - self.condition.notify_all() + def ready_latched(self) -> bool: + return self._ready_latched() - def has_active_writer(self, components: set[str]) -> bool: - with self.condition: - return any(self.active_writers.get(component, 0) for component in components) + def template_fingerprint(self, template: Mapping[str, object]) -> str: + return self._template_fingerprint(template) - def writer_admission_closed(self) -> bool: - with self.condition: - return self.phase != "running" - def wait_for_writer_quiescence(self) -> tuple[bool, float, dict[str, int]]: - started = time.monotonic() - deadline = started + self.writer_quiescence_timeout_seconds - with self.condition: - while self.active_writers: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - self.condition.wait(remaining) - elapsed = max(0.0, time.monotonic() - started) - quiesced = not self.active_writers - blockers = dict(sorted(self.active_writers.items())) - outcome = "success" if quiesced else "timeout" - self.writer_quiescence_outcomes[outcome] += 1 - self.writer_quiescence_seconds = elapsed - if quiesced: - self.phase = "writers_quiesced" - else: - self.phase = "release_withheld" - self.lease_release_withheld = True - self.release_withheld_total += 1 - self.condition.notify_all() - return quiesced, elapsed, blockers - - def claim_lease_release(self) -> tuple[bool, dict[str, int]]: - with self.condition: - if self.lease_release_attempted or self.lease_release_withheld: - return False, {} - if self.active_writers: - return False, dict(sorted(self.active_writers.items())) - self.lease_release_attempted = True - self.lease_release_attempts_total += 1 - self.phase = "releasing_lease" - self.condition.notify_all() - return True, {} - - def finish_lease_release(self, outcome: str, elapsed: float) -> None: - with self.condition: - self.lease_release_outcomes[outcome] += 1 - self.lease_release_seconds = elapsed - self.lease_release_succeeded = outcome != "failure" - self.phase = "lease_released" if outcome != "failure" else "lease_release_failed" - if outcome != "failure" and self.sigterm_monotonic is not None: - self.sigterm_to_lease_release_seconds = max( - 0.0, - time.monotonic() - self.sigterm_monotonic, - ) - self.sigterm_release_observed = True - self.condition.notify_all() +class _CoordinatorTipAuthority(TipAuthorityPort): + def __init__( + self, + *, + live_tip: Callable[[], str], + observe_tip: Callable[[str], object], + published_authority: Callable[[], tuple[str, float | None] | None], + published_authoritative: Callable[[float], bool], + current_tip_locked: Callable[[], str | None], + published_template_locked: Callable[[], QbitTipTemplateSnapshot | None], + snapshot_current_locked: Callable[[QbitTipTemplateSnapshot, int], bool], + artifacts_parent_current_locked: Callable[..., bool], + ensure_artifacts_parent_observed: Callable[..., bool], + schedule_retry: Callable[[], None], + prepared_obsolete: Callable[..., bool], + prepared_token_current_locked: Callable[..., bool], + record_cancellation: Callable[[str], None], + retention_authority_locked: Callable[[], RetentionAuthority], + consume_retained_refresh: Callable[[PrismJobContext], None], + published_current_locked: Callable[..., bool], + ) -> None: + self._live_tip = live_tip + self._observe_tip = observe_tip + self._published_authority = published_authority + self._published_authoritative = published_authoritative + self._current_tip_locked = current_tip_locked + self._published_template_locked = published_template_locked + self._snapshot_current_locked = snapshot_current_locked + self._artifacts_parent_current_locked = artifacts_parent_current_locked + self._ensure_artifacts_parent_observed = ensure_artifacts_parent_observed + self._schedule_retry = schedule_retry + self._prepared_obsolete = prepared_obsolete + self._prepared_token_current_locked = prepared_token_current_locked + self._record_cancellation = record_cancellation + self._retention_authority_locked = retention_authority_locked + self._consume_retained_refresh = consume_retained_refresh + self._published_current_locked = published_current_locked + + def live_tip(self) -> str: + return self._live_tip() + + def observe_tip(self, tip_hash: str) -> object: + return self._observe_tip(tip_hash) + + def published_authority(self) -> tuple[str, float | None] | None: + return self._published_authority() + + def published_authoritative(self, now: float) -> bool: + return self._published_authoritative(now) + + def current_tip_locked(self) -> str | None: + return self._current_tip_locked() + + def published_template_locked(self) -> QbitTipTemplateSnapshot | None: + return self._published_template_locked() + + def snapshot_current_locked( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> bool: + return self._snapshot_current_locked(snapshot, observation_sequence) - def claim_non_writer_drain(self) -> bool: - with self.condition: - if self._drain_claimed: - return False - if self.phase not in { - "lease_released", - "lease_release_failed", - "release_withheld", - }: - return False - self._drain_claimed = True - self.phase = "draining_non_writers" - return True + def artifacts_parent_current_locked( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: + return self._artifacts_parent_current_locked(artifacts) - def finish_non_writer_drain(self, elapsed: float) -> None: - with self.condition: - self.non_writer_drain_seconds = elapsed - self.non_writer_drains_total += 1 - self.phase = "complete" - self.condition.notify_all() - - def snapshot(self) -> dict[str, Any]: - with self.condition: - return { - "phase": self.phase, - "active_writers": dict(self.active_writers), - "shutdowns_total": self.shutdowns_total, - "writer_quiescence_outcomes": dict(self.writer_quiescence_outcomes), - "writer_quiescence_seconds": self.writer_quiescence_seconds, - "lease_release_attempts_total": self.lease_release_attempts_total, - "lease_release_outcomes": dict(self.lease_release_outcomes), - "lease_release_seconds": self.lease_release_seconds, - "lease_release_withheld": self.lease_release_withheld, - "sigterm_to_lease_release_seconds": self.sigterm_to_lease_release_seconds, - "sigterm_release_observed": self.sigterm_release_observed, - "release_withheld_total": self.release_withheld_total, - "non_writer_drain_seconds": self.non_writer_drain_seconds, - "non_writer_drains_total": self.non_writer_drains_total, - } + def ensure_artifacts_parent_observed( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: + return self._ensure_artifacts_parent_observed(artifacts) + def schedule_retry(self) -> None: + self._schedule_retry() -def ledger_writer_operation(component: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]: - """Decorate an entry point that can mutate the PRISM ledger.""" + def prepared_obsolete( + self, + validation_token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + cancel_event: _FanoutCancellation | None, + ) -> bool: + return self._prepared_obsolete( + validation_token, + bundle, + snapshot, + cancel_event, + ) - def decorate(method: Callable[..., Any]) -> Callable[..., Any]: - @wraps(method) - def guarded(self: "PrismCoordinator", *args: Any, **kwargs: Any) -> Any: - with self._writer_operation(component): - return method(self, *args, **kwargs) + def prepared_token_current_locked( + self, + validation_token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + payout_snapshot: object, + ) -> bool: + return self._prepared_token_current_locked( + validation_token, + bundle, + snapshot, + payout_snapshot, + ) - return guarded + def record_cancellation(self, stage: str) -> None: + self._record_cancellation(stage) - return decorate + def retention_authority_locked(self) -> RetentionAuthority: + return self._retention_authority_locked() + def consume_retained_refresh(self, context: PrismJobContext) -> None: + self._consume_retained_refresh(context) -class TemplateRefreshBlocked(RuntimeError): - """A live template was fetched, but safe work could not be issued.""" + def published_current_locked( + self, + context_parent: str, + *, + template_fingerprint: str | None, + template_generation: int, + lapsed_live_validated: bool, + payout_generation: int, + ) -> bool: + return self._published_current_locked( + context_parent, + template_fingerprint=template_fingerprint, + template_generation=template_generation, + lapsed_live_validated=lapsed_live_validated, + payout_generation=payout_generation, + ) -class TemplateRefreshSuperseded(TemplateRefreshBlocked): - """Concurrent tip/payout progress invalidated this refresh attempt. +class _CoordinatorPayoutDelivery(PayoutDeliveryPort): + def __init__( + self, + *, + snapshot: Callable[[], object], + generation: Callable[[], int], + initial_admission: Callable[..., object], + admission: Callable[..., object], + observe_admission: Callable[..., None], + record_first_delivery: Callable[[int, float], None], + ) -> None: + self._snapshot = snapshot + self._generation = generation + self._initial_admission = initial_admission + self._admission = admission + self._observe_admission = observe_admission + self._record_first_delivery = record_first_delivery - Raised only for coordination races that a scheduled retry resolves on its - own: the tip advanced mid-refresh, the payout-state generation moved, or a - newer observation superseded the prepared work. Unlike its parent, this - subclass never arms the template-refresh failure budget -- a genuine - RPC/build/trust failure must raise plain TemplateRefreshBlocked so - sustained unhealthiness still takes the budgeted restart path. - """ + def snapshot(self) -> object: + return self._snapshot() + def generation(self) -> int: + return self._generation() -class JobBuildCancelled(TemplateRefreshBlocked): - """An immutable build was cancelled or timed out.""" + def initial_admission( + self, + cancelled: Callable[[], bool], + *, + generation: int, + ) -> object: + return self._initial_admission(cancelled, generation=generation) + def admission( + self, + cancelled: Callable[[], bool], + *, + generation: int, + priority: bool, + ) -> object: + return self._admission( + cancelled, generation=generation, priority=priority + ) -class JobBuildSuperseded(JobBuildCancelled, TemplateRefreshSuperseded): - """A coordination race cooperatively cancelled immutable construction.""" + def observe_admission( + self, + admission: object, + *, + generation: int, + fallback_wait_seconds: float, + ) -> None: + self._observe_admission( + admission, + generation=generation, + fallback_wait_seconds=fallback_wait_seconds, + ) + def record_first_delivery( + self, + generation: int, + delivered_monotonic: float, + ) -> None: + self._record_first_delivery(generation, delivered_monotonic) -class _JobBuildCancellation: - def __init__(self, *, timeout_seconds: float) -> None: - self._event = threading.Event() - self._lock = threading.Lock() - self.started_monotonic = time.monotonic() - self.deadline_monotonic = self.started_monotonic + timeout_seconds - self.last_checkpoint_monotonic = self.started_monotonic - self.cancelled_monotonic: float | None = None - self.reason: str | None = None - - def cancel(self, reason: str) -> bool: - with self._lock: - if self._event.is_set(): - return False - self.reason = reason - self.cancelled_monotonic = time.monotonic() - self._event.set() - return True +class _CoordinatorInitialJobRuntime(InitialJobRuntimePort): + def __init__( + self, + *, + stopping: Callable[[], bool], + wait: Callable[[float], bool], + disconnect: Callable[[ClientState], None], + submit_initial: Callable[..., Future[Any]], + ) -> None: + self._stopping = stopping + self._wait = wait + self._disconnect = disconnect + self._submit_initial = submit_initial - def is_set(self) -> bool: - if self._event.is_set(): - return True - if time.monotonic() >= self.deadline_monotonic: - self.cancel("timeout") - return True - return False + def stopping(self) -> bool: + return self._stopping() - def raise_if_cancelled(self, phase: str) -> None: - if self.is_set(): - reason = self.reason or "cancelled" - if reason == "timeout": - raise JobBuildCancelled( - f"job build timeout at {phase}; immediate retry scheduled" - ) - raise JobBuildSuperseded( - f"job build {reason} at {phase}; immediate retry scheduled" - ) - with self._lock: - self.last_checkpoint_monotonic = time.monotonic() + def wait(self, timeout: float) -> bool: + return self._wait(timeout) + def disconnect(self, client: ClientState) -> None: + self._disconnect(client) -@dataclass -class _JobBuildRequest: - key: JobBuildKey - cache_key: tuple[object, ...] - equivalence_key: tuple[object, ...] - artifacts: CachedTemplateArtifacts - template_json: str - transaction_hexes: tuple[str, ...] - witness_merkle_leaves_hex: tuple[str, ...] - worker: WorkerIdentity | None - mode: str - payout_artifact: PayoutStateArtifact - payout_ledger_artifact: PayoutLedgerArtifact | None - payout_policy_json: str - ctv_settlement_json: str | None - decimal_context: Context = field(repr=False) - cancellation: _JobBuildCancellation - idle_retarget: bool = False - publication_critical: bool = False - request_source: str = "routine" - priority_admission_recorded: bool = False - promise: Future[CachedJobBundle] = field(default_factory=Future) - requested_monotonic: float = field(default_factory=time.monotonic) - superseded_monotonic: float | None = None + def submit_initial( + self, + function: Callable[[PendingInitialJob], bool], + request: PendingInitialJob, + *, + priority: int, + ) -> Future[Any]: + return self._submit_initial(function, request, priority=priority) -@dataclass(eq=False) -class _JobBuildFlight: - request: _JobBuildRequest - future: Future[CachedJobBundle] | None = None +class _CoordinatorProgressDelivery(ProgressDeliveryPort): + def __init__( + self, + *, + record_health_delivery: Callable[[ClientState, PrismJobContext, float], None], + reconcile_health_eligibility: Callable[[], None], + ) -> None: + self._record_health_delivery = record_health_delivery + self._reconcile_health_eligibility = reconcile_health_eligibility + def record_health_delivery( + self, + client: ClientState, + context: PrismJobContext, + delivered_monotonic: float, + ) -> None: + self._record_health_delivery(client, context, delivered_monotonic) -class _PayoutStatePublicationBlocked(TemplateRefreshBlocked): - """Job construction is waiting for a prepared payout publication.""" + def reconcile_health_eligibility(self) -> None: + self._reconcile_health_eligibility() -class _JobBundleBuildSuperseded(JobBuildSuperseded): - """A newer tip or payout generation canceled this deterministic build.""" +class _InitialJobPendingCompatibility: + """Compatibility alias that adopts replacements into S2 ownership.""" + _backing = "_pending_initial_jobs_compat" -class CollectionIdentityUnavailable(TemplateRefreshBlocked): - """Current collection work is waiting for an authorized worker identity.""" + def __get__( + self, + instance: PrismCoordinator | None, + owner: type[PrismCoordinator], + ) -> MutableMapping[ClientState, PendingInitialJob] | _InitialJobPendingCompatibility: + if instance is None: + return self + state = instance.__dict__.get("_initial_job_state") + if state is not None: + return state.pending + pending = instance.__dict__.get(self._backing) + if pending is None: + pending = instance.__dict__.setdefault(self._backing, {}) + return pending + def __set__( + self, + instance: PrismCoordinator, + value: MutableMapping[ClientState, PendingInitialJob], + ) -> None: + state = instance.__dict__.get("_initial_job_state") + if state is None: + instance.__dict__[self._backing] = value + return + state.adopt_pending(value) + instance.__dict__["_initial_job_tracker"] = state.tracker + + +class _InitialJobConfigCompatibility: + def __init__(self, field_name: str, default: int | float, cast: type) -> None: + self.field_name = field_name + self.default = default + self.cast = cast + self.backing = f"_{field_name}_compat" + + def __get__(self, instance: PrismCoordinator | None, owner: type[PrismCoordinator]) -> Any: + if instance is None: + return self + state = instance.__dict__.get("_initial_job_state") + if state is not None: + return getattr(state.config, self.field_name) + return instance.__dict__.get(self.backing, self.default) + + def __set__(self, instance: PrismCoordinator, value: object) -> None: + converted = self.cast(value) + state = instance.__dict__.get("_initial_job_state") + if state is None: + instance.__dict__[self.backing] = converted + return + state.reconfigure(**{self.field_name: converted}) + + +class _InitialJobMetricCompatibility: + def __init__(self, field_name: str, default: object, cast: type | None) -> None: + self.field_name = field_name + self.default = default + self.cast = cast + self.backing = f"_{field_name}_compat" + + def __get__(self, instance: PrismCoordinator | None, owner: type[PrismCoordinator]) -> Any: + if instance is None: + return self + state = instance.__dict__.get("_initial_job_state") + if state is not None: + return getattr(state, self.field_name) + return instance.__dict__.get(self.backing, self.default) + + def __set__(self, instance: PrismCoordinator, value: object) -> None: + converted = value if self.cast is None else self.cast(value) + state = instance.__dict__.get("_initial_job_state") + if state is None: + instance.__dict__[self.backing] = converted + return + setattr(state, self.field_name, converted) -class _JobBuildFailed(RuntimeError): - """Internal signal used to distinguish a skipped build from a no-op.""" +class _JobCounterCompatibility: + _backing = "_job_counter_compat" -class _BundlePreparationSuperseded(TemplateRefreshSuperseded): - """The exact work identity lost to a newer tip/template observation. + def __get__(self, instance: PrismCoordinator | None, owner: type[PrismCoordinator]) -> Any: + if instance is None: + return self + service = instance.__dict__.get("_job_delivery_service") + if service is not None: + return service.job_counter + return int(instance.__dict__.get(self._backing, 0)) - Subclasses TemplateRefreshSuperseded: losing the shared-bundle build race - to a newer tip/template observation is coordination churn, so it escapes - the poll without arming the template-refresh failure budget. - """ + def __set__(self, instance: PrismCoordinator, value: object) -> None: + converted = int(value) + service = instance.__dict__.get("_job_delivery_service") + if service is None: + instance.__dict__[self._backing] = converted + return + service.adopt_job_counter(converted) -def parse_worker_username(username: str) -> tuple[str, str | None]: - payout_address, worker_name = split_worker_username(username) - if not payout_address: - raise StratumError(20, "username base is empty") - return payout_address, worker_name +class _JobsCompatibility: + _backing = "_jobs_compat" + def __get__(self, instance: PrismCoordinator | None, owner: type[PrismCoordinator]) -> Any: + if instance is None: + return self + service = instance.__dict__.get("_job_delivery_service") + if service is not None: + return service.jobs + jobs = instance.__dict__.get(self._backing) + if jobs is None: + jobs = instance.__dict__.setdefault(self._backing, {}) + return jobs -def split_worker_username(username: str) -> tuple[str, str | None]: - payout_address, separator, worker_name = username.partition(".") - return payout_address, worker_name if separator else None + def __set__( + self, + instance: PrismCoordinator, + value: MutableMapping[str, PrismJobContext], + ) -> None: + service = instance.__dict__.get("_job_delivery_service") + if service is None: + instance.__dict__[self._backing] = value + return + service.adopt_jobs(value) + + +class _ClientsCompatibility: + _backing = "_clients_compat" + + def __get__( + self, + instance: PrismCoordinator | None, + owner: type[PrismCoordinator], + ) -> Any: + if instance is None: + return self + registry = instance.__dict__.get("_session_registry") + if registry is not None: + return registry.clients + clients = instance.__dict__.get(self._backing) + if clients is None: + clients = instance.__dict__.setdefault(self._backing, set()) + return clients + + def __set__(self, instance: PrismCoordinator, value: object) -> None: + registry = instance.__dict__.get("_session_registry") + if registry is None: + instance.__dict__[self._backing] = value + return + registry.adopt_clients(value) + + +class _RetainedConfigCompatibility: + def __init__(self, field_name: str, default: int | float, cast: type) -> None: + self.field_name = field_name + self.default = default + self.cast = cast + self.backing = f"_{field_name}_compat" + + def __get__(self, instance: PrismCoordinator | None, owner: type[PrismCoordinator]) -> Any: + if instance is None: + return self + index = instance.__dict__.get("_retained_job_index") + if index is not None: + return getattr(index, self.field_name) + return instance.__dict__.get(self.backing, self.default) + + def __set__(self, instance: PrismCoordinator, value: object) -> None: + converted = self.cast(value) + index = instance.__dict__.get("_retained_job_index") + if index is None: + instance.__dict__[self.backing] = converted + return + setattr(index, self.field_name, converted) + + +class _RetainedStateCompatibility: + _adopted_fields = { + "graveyard", + "by_connection", + "same_tip_by_connection", + "same_tip_job_ids", + } + + def __init__( + self, + field_name: str, + default_factory: Callable[[], object], + ) -> None: + self.field_name = field_name + self.default_factory = default_factory + self.backing = f"_evicted_{field_name}_compat" + + def __get__(self, instance: PrismCoordinator | None, owner: type[PrismCoordinator]) -> Any: + if instance is None: + return self + index = instance.__dict__.get("_retained_job_index") + if index is not None: + return getattr(index, self.field_name) + if self.backing not in instance.__dict__: + instance.__dict__[self.backing] = self.default_factory() + return instance.__dict__[self.backing] + + def __set__(self, instance: PrismCoordinator, value: object) -> None: + index = instance.__dict__.get("_retained_job_index") + if index is None: + instance.__dict__[self.backing] = value + return + if self.field_name not in self._adopted_fields: + setattr(index, self.field_name, value) + return + replacements = { + "graveyard": index.graveyard, + "by_connection": index.by_connection, + "same_tip_by_connection": index.same_tip_by_connection, + "same_tip_job_ids": index.same_tip_job_ids, + } + replacements[self.field_name] = value + index.adopt( + graveyard=replacements["graveyard"], + by_connection=replacements["by_connection"], + same_tip_by_connection=replacements["same_tip_by_connection"], + same_tip_job_ids=replacements["same_tip_job_ids"], + current_tip=instance._job_delivery_current_tip_locked(), + ) class PrismCoordinator: - def __init__(self) -> None: - validate_prism_production_gate() - self.rpc = JsonRpc( - host=env("QBIT_RPC_HOST"), - port=env_int("QBIT_RPC_PORT", 18452), - user=env("QBIT_RPC_USER"), - password=env("QBIT_RPC_PASSWORD"), - ) - self.qbit_chain = env("QBIT_CHAIN", "regtest") - self.bind = env("PRISM_STRATUM_BIND", "127.0.0.1") - self.port = env_int("PRISM_STRATUM_PORT", 3340) - self.extranonce2_size = env_int("PRISM_STRATUM_EXTRANONCE2_SIZE", 8) - self.blockpoll_seconds = env_positive_float( - "PRISM_BLOCKPOLL_SECONDS", - DEFAULT_PRISM_BLOCKPOLL_SECONDS, - ) - # Push-style tip detection rides waitfornewblock; the poll loop above - # stays as the fallback and still covers same-tip template refreshes. - self.blockwait_enabled = env_bool("PRISM_BLOCKWAIT_ENABLED", "1") - self.blockwait_timeout_seconds = env_positive_float( - "PRISM_BLOCKWAIT_TIMEOUT_SECONDS", - DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, - ) - # After a FAILED refresh pass, the fallback poller re-attempts no - # sooner than this many seconds (plus jitter) while the tip the failed - # pass worked against is still current, so a persistent blockage or - # sustained payout churn costs ~1 attempt/second instead of one per - # 0.25s trigger slice. A successful pass or a newly observed tip - # re-arms immediately. Zero restores unspaced retries. - self.tip_refresh_failure_holdoff_seconds = env_nonnegative_float( - "PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS", - DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, - ) - # Zero disables stale-grace crediting (every prior-tip share rejects, - # the pre-grace behavior). - self.stale_grace_seconds = env_nonnegative_float( - "PRISM_STRATUM_STALE_GRACE_SECONDS", - DEFAULT_PRISM_STALE_GRACE_SECONDS, - ) - # Per-share/per-job stdout logging is debug-only: at production share - # rates each print is a journald flush on the Stratum hot path. - self.hot_path_log_enabled = env_bool("PRISM_HOT_PATH_LOG", "0") - # Zero disables the observed-tip reuse (every submit re-reads the tip - # over RPC, the legacy behavior). - self.submit_tip_max_age_seconds = env_nonnegative_float( - "PRISM_SUBMIT_TIP_MAX_AGE_SECONDS", - DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, - ) - self.same_tip_job_retention_seconds = env_nonnegative_float( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_SECONDS", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, - ) - self.same_tip_job_retention_per_connection = env_nonnegative_int( - "PRISM_STRATUM_SAME_TIP_JOB_RETENTION_PER_CONNECTION", - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, - ) - self.tip_refresh_max_workers = env_positive_int( - "PRISM_TIP_REFRESH_MAX_WORKERS", - DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, - ) - if self.tip_refresh_max_workers > DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS: - raise SystemExit( - "PRISM_TIP_REFRESH_MAX_WORKERS cannot exceed " - f"{DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS}" - ) - self.job_build_timeout_seconds = env_positive_float( - "PRISM_JOB_BUILD_TIMEOUT_SECONDS", - DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, - ) - self.job_build_cancel_grace_seconds = env_nonnegative_float( - "PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - self.vardiff_idle_sweep_seconds = env_nonnegative_float( - "PRISM_STRATUM_VARDIFF_IDLE_SWEEP_SECONDS", - DEFAULT_PRISM_VARDIFF_IDLE_SWEEP_SECONDS, - ) - # Zero collapses every worker into the overflow label (per-worker - # metrics effectively off) without touching the aggregate counters. - self.worker_metrics_limit = env_nonnegative_int( - "PRISM_WORKER_METRICS_LIMIT", - DEFAULT_PRISM_WORKER_METRICS_LIMIT, - ) - self.reorg_reconciler_enabled = env_bool("PRISM_REORG_RECONCILER_ENABLED", "1") - # Per-template job caching. A zero disables the corresponding cache - # (every build redoes that stage), which is also the legacy behavior. - self.job_bundle_cache_seconds = env_nonnegative_float( - "PRISM_JOB_BUNDLE_CACHE_SECONDS", - DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, - ) - self.bundle_build_timeout_seconds = env_positive_float( - "PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - self.template_cache_seconds = env_nonnegative_float( - "PRISM_TEMPLATE_CACHE_SECONDS", - self.blockpoll_seconds, - ) - self.template_refresh_failure_exit_seconds = env_nonnegative_float( - "PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - if production_mode() and self.template_refresh_failure_exit_seconds <= 0: - raise SystemExit( - "production mode requires a positive " - "PRISM_TEMPLATE_REFRESH_FAILURE_EXIT_SECONDS" - ) - self.last_successful_template_refresh_monotonic: float | None = None - self.template_refresh_failure_started_monotonic: float | None = None - self.reorg_reconcile_cache_seconds = env_nonnegative_float( - "PRISM_REORG_RECONCILE_CACHE_SECONDS", - DEFAULT_PRISM_REORG_RECONCILE_CACHE_SECONDS, + pending_initial_jobs = _InitialJobPendingCompatibility() + stratum_max_pending_initial_jobs = _InitialJobConfigCompatibility( + "max_pending", + DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + int, + ) + stratum_initial_job_timeout_seconds = _InitialJobConfigCompatibility( + "timeout_seconds", + DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, + float, + ) + initial_job_max_workers = _InitialJobConfigCompatibility( + "max_workers", + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, + int, + ) + initial_job_queue_rejection_count = _InitialJobMetricCompatibility( + "queue_rejection_count", 0, int + ) + initial_job_timeout_count = _InitialJobMetricCompatibility("timeout_count", 0, int) + initial_job_cancelled_count = _InitialJobMetricCompatibility( + "cancelled_count", 0, int + ) + initial_job_coalesced_count = _InitialJobMetricCompatibility( + "coalesced_count", 0, int + ) + initial_job_sent_count = _InitialJobMetricCompatibility("sent_count", 0, int) + initial_job_failed_count = _InitialJobMetricCompatibility("failed_count", 0, int) + initial_job_superseded_count = _InitialJobMetricCompatibility( + "superseded_count", 0, int + ) + initial_job_queue_capacity_reclaimed_count = _InitialJobMetricCompatibility( + "queue_capacity_reclaimed_count", 0, int + ) + initial_job_delivery_latency_seconds_sum = _InitialJobMetricCompatibility( + "delivery_latency_seconds_sum", 0.0, float + ) + initial_job_delivery_latency_count = _InitialJobMetricCompatibility( + "delivery_latency_count", 0, int + ) + last_initial_job_delivery_monotonic = _InitialJobMetricCompatibility( + "last_delivery_monotonic", None, None + ) + job_counter = _JobCounterCompatibility() + jobs = _JobsCompatibility() + clients = _ClientsCompatibility() + share_append_queue = ShareWriterCompatibilityField("share_append_queue", None) + share_commit_batch_size = ShareWriterCompatibilityField( + "share_commit_batch_size", DEFAULT_SHARE_COMMIT_BATCH_SIZE + ) + share_commit_linger_seconds = ShareWriterCompatibilityField( + "share_commit_linger_seconds", + DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS / 1000.0, + ) + share_commit_timeout_seconds = ShareWriterCompatibilityField( + "share_commit_timeout_seconds", DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS + ) + share_writer_active = ShareWriterCompatibilityField("share_writer_active", False) + share_append_failure_count = ShareWriterCompatibilityField( + "share_append_failure_count", 0 + ) + share_recovery_path = ShareWriterCompatibilityField("share_recovery_path", None) + share_recovery_lock = ShareWriterCompatibilityField("share_recovery_lock", None) + shares_recovered_to_disk = ShareWriterCompatibilityField( + "shares_recovered_to_disk", 0 + ) + shares_replayed = ShareWriterCompatibilityField("shares_replayed", 0) + _pending_share_commit_lock = ShareWriterCompatibilityField( + "_pending_share_commit_lock", None + ) + _pending_share_commit_floor = ShareWriterCompatibilityField( + "_pending_share_commit_floor", None + ) + + same_tip_job_retention_seconds = _RetainedConfigCompatibility( + "same_tip_ttl_seconds", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + float, + ) + same_tip_job_retention_per_connection = _RetainedConfigCompatibility( + "same_tip_per_connection", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + int, + ) + stale_grace_seconds = _RetainedConfigCompatibility( + "stale_grace_seconds", + DEFAULT_PRISM_STALE_GRACE_SECONDS, + float, + ) + evicted_job_graveyard = _RetainedStateCompatibility("graveyard", OrderedDict) + evicted_jobs_by_connection = _RetainedStateCompatibility("by_connection", dict) + evicted_same_tip_by_connection = _RetainedStateCompatibility( + "same_tip_by_connection", dict + ) + evicted_same_tip_job_ids = _RetainedStateCompatibility( + "same_tip_job_ids", OrderedDict + ) + evicted_job_index_tip_hash = _RetainedStateCompatibility( + "index_tip_hash", lambda: None + ) + evicted_job_next_prune_monotonic = _RetainedStateCompatibility( + "next_prune_monotonic", lambda: 0.0 + ) + evicted_job_expiration_counts = _RetainedStateCompatibility( + "expiration_counts", lambda: {name: 0 for name in PRISM_EVICTED_JOB_CLASSES} + ) + evicted_job_capacity_eviction_counts = _RetainedStateCompatibility( + "capacity_eviction_counts", + lambda: {name: 0 for name in PRISM_EVICTED_JOB_CAPACITY_SCOPES}, + ) + evicted_job_submit_counts = _RetainedStateCompatibility( + "submit_counts", + lambda: {name: 0 for name in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES}, + ) + + @property + def current_tip_first_seen(self) -> tuple[str, float | None] | None: + return self._ensure_tip_refresh_service().published_snapshot().first_seen + + @current_tip_first_seen.setter + def current_tip_first_seen(self, value: tuple[str, float | None] | None) -> None: + self._ensure_tip_refresh_service().seed_published_for_test(first_seen=value) + + @property + def current_tip_parent(self) -> tuple[str, str] | None: + return self._ensure_tip_refresh_service().published_snapshot().parent + + @current_tip_parent.setter + def current_tip_parent(self, value: tuple[str, str] | None) -> None: + self._ensure_tip_refresh_service().seed_published_for_test(parent=value) + + @property + def current_tip_observation_sequence(self) -> int: + return self._ensure_tip_refresh_service().published_snapshot().observation_sequence + + @current_tip_observation_sequence.setter + def current_tip_observation_sequence(self, value: int) -> None: + self._ensure_tip_refresh_service().seed_published_for_test( + observation_sequence=value ) - self.health_refresh_seconds = env_positive_float( - "PRISM_HEALTH_REFRESH_SECONDS", - DEFAULT_PRISM_HEALTH_REFRESH_SECONDS, + + @property + def current_tip_observed_monotonic(self) -> float | None: + return self._ensure_tip_refresh_service().published_snapshot().observed_monotonic + + @current_tip_observed_monotonic.setter + def current_tip_observed_monotonic(self, value: float | None) -> None: + self._ensure_tip_refresh_service().seed_published_for_test( + observed_monotonic=value ) - self.health_pending_refresh_max_age_seconds = env_positive_float( - "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS", - DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, + + @property + def tip_template_snapshot(self) -> QbitTipTemplateSnapshot | None: + return self._ensure_tip_refresh_service().published_snapshot().template + + @tip_template_snapshot.setter + def tip_template_snapshot(self, value: QbitTipTemplateSnapshot | None) -> None: + self._ensure_tip_refresh_service().seed_published_for_test(template=value) + + @property + def latest_detected_tip(self) -> tuple[str, int] | None: + return self._ensure_tip_refresh_service().snapshot().latest_detected_tip + + @latest_detected_tip.setter + def latest_detected_tip(self, value: tuple[str, int] | None) -> None: + self._ensure_tip_refresh_service().seed_state_for_test( + latest_detected_tip=value ) - self.health_tip_poll_max_age_seconds = env_positive_float( - "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS", - DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, + + @property + def tip_refresh_divergence_started_monotonic(self) -> float | None: + return ( + self._ensure_tip_refresh_service() + .snapshot() + .divergence_started_monotonic ) - self.stratum_send_timeout_seconds = env_nonnegative_float( - "PRISM_STRATUM_SEND_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + + @tip_refresh_divergence_started_monotonic.setter + def tip_refresh_divergence_started_monotonic(self, value: float | None) -> None: + self._ensure_tip_refresh_service().seed_state_for_test( + divergence_started_monotonic=value ) - # Zero remains an explicit unlimited option for focused local/regtest - # use. Deployments default to a conservative ceiling above PRISM's - # normal 200-250 connection population. - self.stratum_max_connections = env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, + + @property + def tip_observation_sequence(self) -> int: + return self._ensure_tip_refresh_service().snapshot().observation_sequence + + @tip_observation_sequence.setter + def tip_observation_sequence(self, value: int) -> None: + self._ensure_tip_refresh_service().seed_state_for_test( + observation_sequence=value ) - self.stratum_max_connections_per_username = env_nonnegative_int( - "PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, + + @property + def last_successful_template_refresh_monotonic(self) -> float | None: + return ( + self._ensure_tip_refresh_service() + .snapshot() + .last_successful_refresh_monotonic ) - self.stratum_max_pending_initial_jobs = env_positive_int( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS", - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + + @last_successful_template_refresh_monotonic.setter + def last_successful_template_refresh_monotonic(self, value: float | None) -> None: + self._ensure_tip_refresh_service().seed_state_for_test( + last_successful_refresh_monotonic=value ) - self.initial_job_max_workers = DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS - if ( - self.stratum_max_connections > 0 - and self.stratum_max_pending_initial_jobs > self.stratum_max_connections - ): - raise SystemExit( - "PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS cannot exceed " - "PRISM_STRATUM_MAX_CONNECTIONS" - ) - self.stratum_initial_job_timeout_seconds = env_nonnegative_float( - "PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS", - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, - ) - self.mining_health_startup_grace_seconds = env_nonnegative_float( - "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS", - DEFAULT_PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS, - ) - validate_same_tip_job_retention_limits( - retention_seconds=self.same_tip_job_retention_seconds, - per_connection=self.same_tip_job_retention_per_connection, - max_connections=self.stratum_max_connections, - production=production_mode(), - ) - self.stratum_accept_resource_exhaustion_backoff_seconds = env_positive_float( - "PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS", - DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, - ) - self.stratum_listen_backlog = env_positive_int( - "PRISM_STRATUM_LISTEN_BACKLOG", - DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, - ) - self.stratum_bind_retry_seconds = env_nonnegative_float( - "PRISM_STRATUM_BIND_RETRY_SECONDS", - DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, - ) - self.payout_address_cache_max_entries = env_nonnegative_int( - "PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, - ) - self.payout_address_cache_ttl_seconds = env_nonnegative_float( - "PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, - ) - self.coinbase_tag_hex = default_prism_coinbase_tag_hex() - self.share_difficulty = env_decimal("PRISM_STRATUM_SHARE_DIFF", "0.000000001") - self.vardiff_config = load_prism_vardiff_config(self.share_difficulty) - self.listener_profiles = [ - StratumListenerProfile( - name="default", - bind=self.bind, - port=self.port, - share_difficulty=self.share_difficulty, - vardiff_config=self.vardiff_config, - heartbeat_name="stratum_accept", - ) - ] - highdiff_profile = load_prism_highdiff_listener(self.bind, self.vardiff_config) - if highdiff_profile is not None: - if highdiff_profile.port == self.port and highdiff_profile.bind == self.bind: - raise SystemExit("PRISM_STRATUM_HIGHDIFF_PORT must differ from PRISM_STRATUM_PORT") - self.listener_profiles.append(highdiff_profile) - self.default_share_weight = env_int("PRISM_STRATUM_SHARE_WEIGHT", 1) - if self.default_share_weight <= 0: - raise SystemExit("PRISM_STRATUM_SHARE_WEIGHT must be positive") - self.share_weights_by_username = self.parse_share_weights() - self.username_fallback_address = default_prism_username_fallback_address() - self.min_ready_miners = env_int("PRISM_MIN_READY_MINERS", 3) - self.signing_seed_hex = env_seed_hex( - "PRISM_MANIFEST_SIGNING_SEED_HEX", - test_default="42" * 32, - ) - self.ledger_attestation_signing_seed_hex = env_seed_hex( - "PRISM_LEDGER_ATTESTATION_SIGNING_SEED_HEX", - test_default="43" * 32, - ) - self.ledger_writer_public_key_hex = self.load_trusted_ledger_writer_public_key() - self.evidence_path = Path(env("PRISM_EVIDENCE_PATH", "prism-live-evidence.json")) - self.audit_dir = Path(env("PRISM_AUDIT_DIR", str(self.evidence_path.parent))) - self.audit_dir.mkdir(parents=True, exist_ok=True) - self.audit_share_segment_size = env_nonnegative_int( - "PRISM_AUDIT_SHARE_SEGMENT_SIZE", - DEFAULT_AUDIT_SHARE_SEGMENT_SIZE, - ) - self.audit_live_bundle_retention = env_nonnegative_int("PRISM_AUDIT_LIVE_BUNDLE_RETENTION", 5) - self.audit_candidate_retention_seconds = env_nonnegative_int( - "PRISM_AUDIT_CANDIDATE_RETENTION_SECONDS", - 24 * 60 * 60, - ) - self.ctv_broadcast_attempt_detail_limit = env_nonnegative_int( - "PRISM_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT", - DEFAULT_CTV_BROADCAST_ATTEMPT_DETAIL_LIMIT, - ) - self.ctv_broadcast_retry_backoff_seconds = env_nonnegative_int( - "PRISM_CTV_BROADCAST_RETRY_BACKOFF_SECONDS", - DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, - ) - self.audit_bind = os.environ.get("PRISM_AUDIT_BIND") - self.audit_port = int(os.environ.get("PRISM_AUDIT_PORT", "0") or "0") - self.stop_after_block = env("PRISM_STOP_AFTER_BLOCK", "1") in {"1", "true", "yes"} - self.max_blocks = env_int("PRISM_MAX_BLOCKS", 1) - if self.max_blocks <= 0: - raise SystemExit("PRISM_MAX_BLOCKS must be positive") - try: - fallback_version_mask = direct_stratum.normalize_version_rolling_mask( - env("PRISM_VERSION_ROLLING_MASK", direct_stratum.QBIT_VERSION_ROLLING_MASK_HEX), - field_name="PRISM_VERSION_ROLLING_MASK", - ) - except ValueError as exc: - raise SystemExit(str(exc)) from exc - self.version_mask_selection = self.resolve_version_rolling_mask(fallback_version_mask) - self.version_mask = self.version_mask_selection.selected_mask - self.writer_quiescence_timeout_seconds = env_positive_float( - "PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS", - DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, + + @property + def template_refresh_failure_started_monotonic(self) -> float | None: + return self._ensure_tip_refresh_service().snapshot().failure_started_monotonic + + @template_refresh_failure_started_monotonic.setter + def template_refresh_failure_started_monotonic(self, value: float | None) -> None: + self._ensure_tip_refresh_service().seed_state_for_test( + failure_started_monotonic=value ) - self.ledger = self.make_ledger() - configured_ctv_broadcaster_enabled = env_optional_bool("PRISM_CTV_BROADCASTER_ENABLED") - self.ctv_broadcaster_enabled = ( - configured_ctv_broadcaster_enabled - if configured_ctv_broadcaster_enabled is not None - else env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0") + + @property + def tip_refresh_job_count(self) -> int: + return self._ensure_tip_refresh_service().snapshot().refresh_job_count + + @tip_refresh_job_count.setter + def tip_refresh_job_count(self, value: int) -> None: + self._ensure_tip_refresh_service().seed_state_for_test(refresh_job_count=value) + + @property + def post_accept_refresh_failure_count(self) -> int: + return ( + self._ensure_tip_refresh_service() + .snapshot() + .post_accept_refresh_failure_count ) - self.ctv_broadcaster_wallet = env_optional("PRISM_CTV_BROADCASTER_WALLET") - self.ctv_broadcaster_fee_sats = env_nonnegative_int_with_legacy( - "PRISM_CTV_BROADCASTER_FEE_BITS", - "PRISM_CTV_BROADCASTER_FEE_SATS", - 0, + + @post_accept_refresh_failure_count.setter + def post_accept_refresh_failure_count(self, value: int) -> None: + self._ensure_tip_refresh_service().seed_state_for_test( + post_accept_refresh_failure_count=value ) - if ( - self.ctv_broadcaster_enabled - and self.ctv_broadcaster_fee_sats > 0 - and not self.ctv_broadcaster_wallet - ): - raise SystemExit( - "PRISM_CTV_BROADCASTER_WALLET is required when " - "PRISM_CTV_BROADCASTER_FEE_BITS is positive" - ) - self.ctv_broadcaster_limit = env_positive_int("PRISM_CTV_BROADCASTER_LIMIT", 100) - self.ctv_broadcaster_chunk_size = env_positive_int( - "PRISM_CTV_BROADCASTER_CHUNK_SIZE", - DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, + + def __init__(self, config: CoordinatorConfig | None = None) -> None: + self.config = load_coordinator_config() if config is None else config + rpc_config = self.config.rpc + stratum_config = self.config.stratum + job_config = self.config.jobs + ledger_config = self.config.ledger + audit_config = self.config.audit + ctv_config = self.config.ctv + lifecycle_config = self.config.lifecycle + + self.rpc = JsonRpc( + host=rpc_config.host, + port=rpc_config.port, + user=rpc_config.user, + password=rpc_config.password, + ) + self.qbit_chain = rpc_config.chain + self.bind = stratum_config.bind + self.port = stratum_config.port + self.extranonce2_size = stratum_config.extranonce2_size + self.blockpoll_seconds = job_config.blockpoll_seconds + self.blockwait_enabled = job_config.blockwait_enabled + self.blockwait_timeout_seconds = job_config.blockwait_timeout_seconds + self.tip_refresh_failure_holdoff_seconds = ( + job_config.tip_refresh_failure_holdoff_seconds + ) + self.stale_grace_seconds = stratum_config.stale_grace_seconds + self.hot_path_log_enabled = self.config.hot_path_log_enabled + self.submit_tip_max_age_seconds = job_config.submit_tip_max_age_seconds + self.same_tip_job_retention_seconds = stratum_config.same_tip_job_retention_seconds + self.same_tip_job_retention_per_connection = ( + stratum_config.same_tip_job_retention_per_connection + ) + self.tip_refresh_max_workers = job_config.tip_refresh_max_workers + self.job_build_timeout_seconds = job_config.job_build_timeout_seconds + self.job_build_cancel_grace_seconds = job_config.job_build_cancel_grace_seconds + self.vardiff_idle_sweep_seconds = stratum_config.vardiff_idle_sweep_seconds + self.worker_metrics_limit = job_config.worker_metrics_limit + self.reorg_reconciler_enabled = job_config.reorg_reconciler_enabled + self.job_bundle_cache_seconds = job_config.job_bundle_cache_seconds + self.bundle_build_timeout_seconds = job_config.bundle_build_timeout_seconds + self.template_cache_seconds = job_config.template_cache_seconds + self.template_refresh_failure_exit_seconds = ( + job_config.template_refresh_failure_exit_seconds + ) + self.reorg_reconcile_cache_seconds = job_config.reorg_reconcile_cache_seconds + self.health_refresh_seconds = lifecycle_config.health_refresh_seconds + self.health_pending_refresh_max_age_seconds = ( + lifecycle_config.pending_refresh_health_deadline_seconds + ) + self.health_tip_poll_max_age_seconds = ( + lifecycle_config.coherent_tip_poll_health_deadline_seconds + ) + self.stratum_send_timeout_seconds = stratum_config.send_timeout_seconds + self.stratum_max_connections = stratum_config.max_connections + self.stratum_max_connections_per_username = stratum_config.max_connections_per_username + self.stratum_max_pending_initial_jobs = stratum_config.max_pending_initial_jobs + self.stratum_initial_job_timeout_seconds = stratum_config.initial_job_timeout_seconds + self.mining_health_startup_grace_seconds = ( + lifecycle_config.mining_health_startup_grace_seconds + ) + self.stratum_accept_resource_exhaustion_backoff_seconds = ( + stratum_config.accept_resource_exhaustion_backoff_seconds + ) + self.stratum_listen_backlog = stratum_config.listen_backlog + self.stratum_bind_retry_seconds = stratum_config.bind_retry_seconds + self.payout_address_cache_max_entries = stratum_config.payout_address_cache_max_entries + self.payout_address_cache_ttl_seconds = stratum_config.payout_address_cache_ttl_seconds + self.coinbase_tag_hex = self.config.coinbase_tag_hex + self.share_difficulty = stratum_config.share_difficulty + self.vardiff_config = stratum_config.vardiff_config + self.listener_profiles = list(stratum_config.listener_profiles) + self.default_share_weight = stratum_config.default_share_weight + self.share_weights_by_username = dict(stratum_config.share_weights_by_username) + self.username_fallback_address = stratum_config.username_fallback_address + self.min_ready_miners = job_config.min_ready_miners + self.signing_seed_hex = ledger_config.signing_seed_hex + self.ledger_attestation_signing_seed_hex = ledger_config.attestation_signing_seed_hex + self.ledger_writer_public_key_hex = ledger_config.writer_public_key_hex + self.evidence_path = audit_config.evidence_path + self.audit_dir = audit_config.directory + self.audit_dir.mkdir(parents=True, exist_ok=True) + self.audit_share_segment_size = audit_config.share_segment_size + self.audit_live_bundle_retention = audit_config.live_bundle_retention + self.audit_candidate_retention_seconds = audit_config.candidate_retention_seconds + self.ctv_broadcast_attempt_detail_limit = ctv_config.broadcast_attempt_detail_limit + self.ctv_broadcast_retry_backoff_seconds = ctv_config.broadcast_retry_backoff_seconds + self.audit_bind = audit_config.bind + self.audit_port = audit_config.port + self.stop_after_block = self.config.stop_after_block + self.max_blocks = self.config.max_blocks + self.version_mask_selection = self.resolve_version_rolling_mask( + stratum_config.fallback_version_mask ) - if self.ctv_broadcaster_chunk_size > MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE: - raise SystemExit( - "PRISM_CTV_BROADCASTER_CHUNK_SIZE must be at most " - f"{MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE}" - ) - self.ctv_broadcaster_interval_seconds = env_positive_float( - "PRISM_CTV_BROADCASTER_INTERVAL_SECONDS", - 30.0, + self.version_mask = self.version_mask_selection.selected_mask + self.writer_quiescence_timeout_seconds = ( + lifecycle_config.writer_quiescence_timeout_seconds ) - self._ctv_broadcaster_metrics_lock = threading.Lock() - self.ctv_broadcaster_pass_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS - } - self.ctv_broadcaster_pass_seconds_sum = 0.0 - self.ctv_broadcaster_pass_count = 0 - self.ctv_broadcaster_processed_rows_total = 0 + self.ledger = self.make_ledger() self._ctv_fanout_market_fee_rate_cache: dict[tuple[int | None, str | None], int] = {} - self.ctv_fanout_broadcast_daemon: CtvFanoutBroadcastDaemon | None = None self.lock = _ObservedRLock() self.clients: set[ClientState] = set() self.connection_limit_rejection_counts = {"global": 0, "username": 0} @@ -2523,7 +1752,6 @@ def __init__(self) -> None: self.initial_job_timeout_count = 0 self.initial_job_cancelled_count = 0 self.initial_job_coalesced_count = 0 - self.initial_job_queue_capacity_reclaimed_count = 0 self.last_initial_job_delivery_monotonic: float | None = None self._mining_overload_started_monotonic: float | None = None self._mining_delivery_failure_started_monotonic: float | None = None @@ -2531,9 +1759,7 @@ def __init__(self) -> None: self._p2mr_address_cache: OrderedDict[ str, tuple[float, tuple[str, str]] ] = OrderedDict() - self._p2mr_address_validation_inflight: dict[ - str, _P2mrAddressValidationFlight - ] = {} + self._p2mr_address_validation_inflight: dict[str, object] = {} self.jobs: dict[str, PrismJobContext] = {} # Share-path ownership is deliberately disjoint from the coordinator # control-plane lock. Deduplication remains process-wide so an exact @@ -2555,7 +1781,6 @@ def __init__(self) -> None: self.duplicate_share_count = 0 self.low_difficulty_share_count = 0 self.collection_block_submission_count = 0 - self._pool_ready_latched = False self.grace_credited_share_count = 0 self.idle_retarget_count = 0 self._ensure_vardiff_idle_state() @@ -2583,23 +1808,6 @@ def __init__(self) -> None: self.evicted_job_submit_counts = { outcome: 0 for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES } - # Published share-validation authority. Detection is kept separately - # in latest_detected_tip so a waiter can cancel obsolete refresh work - # without invalidating jobs before their replacement is ready. - # The flip stamp is None for the startup baseline, which never opens - # the stale-grace window. - self.current_tip_first_seen: tuple[str, float | None] | None = None - self.current_tip_parent: tuple[str, str] | None = None - self.latest_detected_tip: tuple[str, int] | None = None - # Start of the current detected-vs-published divergence epoch. Unlike - # latest_detected_tip, this stamp is not renewed by B -> C -> D while - # published work is still on A; submit authority therefore has a - # bounded lease even during repeated refresh failures. - self.tip_refresh_divergence_started_monotonic: float | None = None - # When the refresh path last published or reconfirmed the authoritative - # tip. Bounds normal submit reuse when no replacement is actively - # pending (see submit_stale_check_tip). - self.current_tip_observed_monotonic: float | None = None # Block candidates are landed by a dedicated submitter thread so a # winning share's ack (and every other client's) never waits on # audit/persist/submitblock; see enqueue_block_candidate. @@ -2615,12 +1823,6 @@ def __init__(self) -> None: ) self.block_candidate_retry_max_seconds = DEFAULT_BLOCK_CANDIDATE_RETRY_MAX_SECONDS self.block_candidate_retry_delays: dict[str, float] = {} - self._block_submitter_retry_state_lock = threading.Lock() - self._block_submitter_backoff_started_monotonic: float | None = None - self._block_submitter_backoff_deadline_monotonic: float | None = None - self._block_submitter_backoff_delay_seconds = 0.0 - # Terminal candidates whose durable outbox update failed; replays for - # these run finalize-only (see _finalize_block_candidate). self._block_candidate_finalize_retries: dict[str, tuple[bool, str]] = {} # A block candidate that loses its tip race (or fails to submit) is a # BLOCK-path event, not a share rejection: under the async model the @@ -2635,34 +1837,19 @@ def __init__(self) -> None: self.share_append_queue: queue.Queue[PendingShareAppend] = queue.Queue( maxsize=MAX_PENDING_SHARE_APPENDS ) - self.share_commit_batch_size = env_positive_int( - "PRISM_SHARE_COMMIT_BATCH_SIZE", DEFAULT_SHARE_COMMIT_BATCH_SIZE - ) - self.share_commit_linger_seconds = ( - env_nonnegative_float( - "PRISM_SHARE_COMMIT_LINGER_MILLISECONDS", - DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS, - ) - / 1000.0 - ) - self.share_commit_timeout_seconds = env_positive_float( - "PRISM_SHARE_COMMIT_TIMEOUT_SECONDS", DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS - ) + self.share_commit_batch_size = ledger_config.share_commit_batch_size + self.share_commit_linger_seconds = ledger_config.share_commit_linger_seconds + self.share_commit_timeout_seconds = ledger_config.share_commit_timeout_seconds self.share_writer_active = False self.share_append_failure_count = 0 # Retain the historical recovery-file reader for clean upgrades from a # release that could acknowledge before Postgres commit. New shares # are never written here: an unavailable ledger produces no success # acknowledgement and an exact retry is idempotent. - self.share_recovery_path = Path( - env("PRISM_SHARE_RECOVERY_PATH", str(self.audit_dir / "prism-unpersisted-shares.jsonl")) - ) + self.share_recovery_path = ledger_config.share_recovery_path self.share_recovery_lock = threading.Lock() self.shares_recovered_to_disk = 0 self.shares_replayed = 0 - self.job_build_failure_count = 0 - self.tip_refresh_job_count = 0 - self.post_accept_refresh_failure_count = 0 self.reorg_inactive_block_count = 0 self.reorg_reactivated_block_count = 0 self.reorg_reconcile_skip_count = 0 @@ -2673,28 +1860,6 @@ def __init__(self) -> None: # it here only to derive one metric pinned the complete share window for # the lifetime of the coordinator. self.latest_coinbase_size_bytes: int | None = None - self.tip_template_snapshot: QbitTipTemplateSnapshot | None = None - self._tip_refresh_lock = threading.Lock() - self._tip_refresh_singleflight_lock = threading.Lock() - self._tip_refresh_executor_lock = threading.Lock() - self._tip_refresh_executor: _BoundedPriorityExecutor | None = None - self._tip_refresh_executor_shutdown = False - self._initial_job_executor_lock = threading.Lock() - self._initial_job_executor: _BoundedPriorityExecutor | None = None - self._initial_job_executor_shutdown = False - self._tip_refresh_pending_event = threading.Event() - self._tip_refresh_pending_counter = 0 - self._tip_refresh_pending_token: int | None = None - self._tip_refresh_retry = threading.Event() - self._tip_refresh_retry_counter = 0 - self._tip_refresh_retry_consumed = 0 - self._tip_refresh_failure_holdoff_until: float | None = None - self._tip_refresh_failure_tip: str | None = None - self._active_tip_refresh: tuple[ - TipRefreshValidationToken, - _FanoutCancellation, - ] | None = None - self._retained_collection_refresh: RetainedCollectionRefresh | None = None self.last_reorg_reconciled_tip_hash: str | None = None self.last_reorg_reconciled_trusted = False self.last_reorg_reconciled_monotonic: float | None = None @@ -2711,9 +1876,14 @@ def __init__(self) -> None: self._heartbeats: dict[str, float] = {} self._watchdog_pauses: dict[str, int] = {} self._heartbeats_lock = threading.Lock() - self.watchdog_enabled = env_bool("PRISM_WATCHDOG_ENABLED", "1") - self.watchdog_timeout_seconds = env_positive_float("PRISM_WATCHDOG_TIMEOUT_SECONDS", 120.0) - self.watchdog_interval_seconds = env_positive_float("PRISM_WATCHDOG_INTERVAL_SECONDS", 15.0) + self.watchdog_enabled = lifecycle_config.watchdog_enabled + self.watchdog_timeout_seconds = lifecycle_config.watchdog_timeout_seconds + self.watchdog_interval_seconds = lifecycle_config.watchdog_interval_seconds + self._ctv_runtime_init_lock = threading.Lock() + self._ctv_runtime = self._make_ctv_runtime_service( + CtvRuntimeConfig.from_coordinator_config(ctv_config) + ) + self._background_services = self._make_background_service_registry() def _ensure_share_hot_path_state(self) -> None: """Backfill dedicated hot-path locks for lightweight test embedders.""" @@ -2733,18 +1903,7 @@ def _ensure_share_hot_path_state(self) -> None: @staticmethod def _client_vardiff_lock(client: ClientState) -> threading.RLock: - # Production ClientState instances always have this field. The lazy - # fallback preserves focused tests and embedders that construct a - # ClientState with __new__ and populate only the attributes they use. - lock = getattr(client, "vardiff_lock", None) - if lock is not None: - return lock - with _HOT_PATH_LOCK_INITIALIZATION_LOCK: - lock = getattr(client, "vardiff_lock", None) - if lock is None: - lock = threading.RLock() - client.vardiff_lock = lock - return lock + return client_vardiff_lock(client) def _reserve_recent_share_key(self, share_key: tuple[object, ...]) -> bool: self._ensure_share_hot_path_state() @@ -2862,22 +2021,10 @@ def resolve_version_rolling_mask(self, fallback_mask: int) -> direct_stratum.Ver raise SystemExit(str(exc)) from exc def parse_share_weights(self) -> dict[str, int]: - raw = os.environ.get("PRISM_STRATUM_SHARE_WEIGHTS_JSON", "") - if not raw: - return {} - try: - parsed = json.loads(raw) - except json.JSONDecodeError as exc: - raise SystemExit(f"PRISM_STRATUM_SHARE_WEIGHTS_JSON is not valid JSON: {exc}") from exc - if not isinstance(parsed, dict): - raise SystemExit("PRISM_STRATUM_SHARE_WEIGHTS_JSON must be an object") - weights: dict[str, int] = {} - for username, weight in parsed.items(): - parsed_weight = int(weight) - if parsed_weight <= 0: - raise SystemExit(f"share weight for {username} must be positive") - weights[str(username)] = parsed_weight - return weights + config = getattr(self, "config", None) + if config is not None: + return dict(config.stratum.share_weights_by_username) + return load_share_weights() def share_weight_for_worker(self, worker: WorkerIdentity) -> int: return self.share_weights_by_username.get( @@ -2886,12 +2033,27 @@ def share_weight_for_worker(self, worker: WorkerIdentity) -> int: ) def make_ledger(self) -> SingleWriterShareLedger | PsqlShareLedger: - psql_command = os.environ.get("PRISM_POSTGRES_PSQL_COMMAND", "") - database_url = os.environ.get("PRISM_DATABASE_URL", "") + config = getattr(self, "config", None) + ledger_config = config.ledger if config is not None else None + psql_command = ( + ledger_config.psql_command + if ledger_config is not None + else env_optional("PRISM_POSTGRES_PSQL_COMMAND") or "" + ) + database_url = ( + ledger_config.database_url or "" + if ledger_config is not None + else env_optional("PRISM_DATABASE_URL") or "" + ) if not psql_command and database_url: psql_command = f"psql {shlex.quote(database_url)}" if not psql_command: - if not env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0"): + allow_memory_ledger = ( + ledger_config.allow_memory_ledger + if ledger_config is not None + else env_bool("PRISM_ALLOW_MEMORY_LEDGER", "0") + ) + if not allow_memory_ledger: raise SystemExit( "PRISM_DATABASE_URL or PRISM_POSTGRES_PSQL_COMMAND is required; " "set PRISM_ALLOW_MEMORY_LEDGER=1 only for local tests" @@ -2908,8 +2070,16 @@ def make_ledger(self) -> SingleWriterShareLedger | PsqlShareLedger: DEFAULT_CTV_BROADCAST_RETRY_BACKOFF_SECONDS, ), ) - writer_session_token = env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN") - if writer_session_token is not None and not env_bool("PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", "0"): + writer_session_token = ( + ledger_config.writer_session_token + if ledger_config is not None + else env_optional("PRISM_LEDGER_WRITER_SESSION_TOKEN") + ) + if ( + ledger_config is None + and writer_session_token is not None + and not env_bool("PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN", "0") + ): raise SystemExit( "PRISM_LEDGER_WRITER_SESSION_TOKEN requires " "PRISM_ALLOW_FIXED_LEDGER_SESSION_TOKEN=1 for local tests" @@ -2918,15 +2088,50 @@ def make_ledger(self) -> SingleWriterShareLedger | PsqlShareLedger: return PsqlShareLedger( psql_command=psql_command, database_url=database_url or None, - native_client_mode=env("PRISM_POSTGRES_NATIVE_CLIENT", "auto"), - writer_id=env("PRISM_LEDGER_WRITER_ID", "prism-coordinator"), - writer_epoch=env_int("PRISM_LEDGER_WRITER_EPOCH", 1), + native_client_mode=( + ledger_config.native_client_mode + if ledger_config is not None + else env("PRISM_POSTGRES_NATIVE_CLIENT", "auto") + ), + writer_id=( + ledger_config.writer_id + if ledger_config is not None + else env("PRISM_LEDGER_WRITER_ID", "prism-coordinator") + ), + writer_epoch=( + ledger_config.writer_epoch + if ledger_config is not None + else env_int("PRISM_LEDGER_WRITER_EPOCH", 1) + ), writer_session_token=writer_session_token, - initialize_schema=env("PRISM_POSTGRES_INIT_SCHEMA", "0") in {"1", "true", "yes"}, - lease_ttl_seconds=env_positive_float("PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0), - read_concurrency=env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4), - accepted_stats_cache_seconds=env_nonnegative_float("PRISM_ACCEPTED_STATS_CACHE_SECONDS", 60.0), - reward_window_cache_seconds=env_nonnegative_float("PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", 30.0), + initialize_schema=( + ledger_config.initialize_schema + if ledger_config is not None + else env("PRISM_POSTGRES_INIT_SCHEMA", "0") in {"1", "true", "yes"} + ), + lease_ttl_seconds=( + ledger_config.lease_ttl_seconds + if ledger_config is not None + else env_positive_float("PRISM_LEDGER_LEASE_TTL_SECONDS", 60.0) + ), + read_concurrency=( + ledger_config.read_concurrency + if ledger_config is not None + else env_positive_int("PRISM_POSTGRES_READ_CONCURRENCY", 4) + ), + accepted_stats_cache_seconds=( + ledger_config.accepted_stats_cache_seconds + if ledger_config is not None + else env_nonnegative_float("PRISM_ACCEPTED_STATS_CACHE_SECONDS", 60.0) + ), + reward_window_cache_seconds=( + ledger_config.reward_window_cache_seconds + if ledger_config is not None + else env_nonnegative_float( + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS", + 30.0, + ) + ), audit_body_dir=str(audit_body_dir) if audit_body_dir is not None else None, audit_share_segment_size=getattr(self, "audit_share_segment_size", DEFAULT_AUDIT_SHARE_SEGMENT_SIZE), ctv_broadcast_attempt_detail_limit=getattr( @@ -2942,6 +2147,9 @@ def make_ledger(self) -> SingleWriterShareLedger | PsqlShareLedger: ) def load_trusted_ledger_writer_public_key(self) -> str | None: + config = getattr(self, "config", None) + if config is not None: + return config.ledger.writer_public_key_hex configured = env_optional("PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX") if configured is not None: return validate_hex(configured, name="PRISM_LEDGER_WRITER_PUBLIC_KEY_HEX", expected_bytes=32) @@ -2957,13 +2165,34 @@ def prism_payout_policy(self) -> dict[str, object]: if cached is not None: return cached - policy = default_prism_payout_policy() - fee_bps_raw = env_optional("PRISM_POOL_FEE_BPS") - fee_enabled = env_bool("PRISM_POOL_FEE_ENABLED", "0") - fee_address = env_optional("PRISM_POOL_FEE_ADDRESS") - fee_program_hex = env_optional("PRISM_POOL_FEE_P2MR_PROGRAM_HEX") - fee_recipient_id = env_optional("PRISM_POOL_FEE_RECIPIENT_ID") - fee_order_key = env_optional("PRISM_POOL_FEE_ORDER_KEY") + config = getattr(self, "config", None) + if config is None: + policy = default_prism_payout_policy() + fee_bps_raw = env_optional("PRISM_POOL_FEE_BPS") + fee_enabled = env_bool("PRISM_POOL_FEE_ENABLED", "0") + fee_address = env_optional("PRISM_POOL_FEE_ADDRESS") + fee_program_hex = env_optional("PRISM_POOL_FEE_P2MR_PROGRAM_HEX") + fee_recipient_id = env_optional("PRISM_POOL_FEE_RECIPIENT_ID") + fee_order_key = env_optional("PRISM_POOL_FEE_ORDER_KEY") + else: + job_config = config.jobs + policy = default_prism_payout_policy( + environ=dict(job_config.payout_environment) + ) + fee_bps_raw = job_config.pool_fee_bps_raw + fee_enabled = env_bool( + "PRISM_POOL_FEE_ENABLED", + "0", + environ=( + {} + if job_config.pool_fee_enabled_raw is None + else {"PRISM_POOL_FEE_ENABLED": job_config.pool_fee_enabled_raw} + ), + ) + fee_address = job_config.pool_fee_address + fee_program_hex = job_config.pool_fee_program_hex + fee_recipient_id = job_config.pool_fee_recipient_id + fee_order_key = job_config.pool_fee_order_key has_fee_config = any( value is not None for value in (fee_bps_raw, fee_address, fee_program_hex, fee_recipient_id, fee_order_key) @@ -3023,30 +2252,89 @@ def prism_ctv_settlement_config( block_height: int | None = None, parent_hash: str | None = None, ) -> dict[str, object] | None: - if not env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0"): + coordinator_config = getattr(self, "config", None) + ctv_config = coordinator_config.ctv if coordinator_config is not None else None + settlement_environment = ( + dict(ctv_config.settlement_environment) if ctv_config is not None else None + ) + settlement_enabled = ( + env_bool( + "PRISM_CTV_SETTLEMENT_ENABLED", + "0", + environ=( + {} + if ctv_config.settlement_enabled_raw is None + else { + "PRISM_CTV_SETTLEMENT_ENABLED": ctv_config.settlement_enabled_raw + } + ), + ) + if ctv_config is not None + else env_bool("PRISM_CTV_SETTLEMENT_ENABLED", "0") + ) + if not settlement_enabled: return None - direct_floor_sats = env_positive_int_with_legacy( - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", - "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", - DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, + direct_floor_sats = ( + env_positive_int_with_legacy( + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", + DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, + environ=settlement_environment, + ) + if settlement_environment is not None + else env_positive_int_with_legacy( + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_SATS", + DEFAULT_DIRECT_COINBASE_PAYOUT_FLOOR_SATS, + ) + ) + reserved_coinbase_outputs = ( + env_int( + "PRISM_RESERVED_COINBASE_OUTPUTS", 0, environ=settlement_environment + ) + if settlement_environment is not None + else env_int("PRISM_RESERVED_COINBASE_OUTPUTS", 0) ) - reserved_coinbase_outputs = env_int("PRISM_RESERVED_COINBASE_OUTPUTS", 0) if reserved_coinbase_outputs < 0: raise SystemExit("PRISM_RESERVED_COINBASE_OUTPUTS must be non-negative") config: dict[str, object] = { "direct_floor_sats": direct_floor_sats, "config": { - "max_coinbase_settlement_outputs": env_positive_int( - "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", - DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, + "max_coinbase_settlement_outputs": ( + env_positive_int( + "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", + DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, + environ=settlement_environment, + ) + if settlement_environment is not None + else env_positive_int( + "PRISM_MAX_COINBASE_SETTLEMENT_OUTPUTS", + DEFAULT_MAX_COINBASE_SETTLEMENT_OUTPUTS, + ) ), - "max_direct_coinbase_outputs": env_positive_int( - "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", - DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, + "max_direct_coinbase_outputs": ( + env_positive_int( + "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", + DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, + environ=settlement_environment, + ) + if settlement_environment is not None + else env_positive_int( + "PRISM_MAX_DIRECT_COINBASE_OUTPUTS", + DEFAULT_MAX_DIRECT_COINBASE_OUTPUTS, + ) ), - "max_fanout_recipients_per_transaction": env_positive_int( - "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", - DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, + "max_fanout_recipients_per_transaction": ( + env_positive_int( + "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", + DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, + environ=settlement_environment, + ) + if settlement_environment is not None + else env_positive_int( + "PRISM_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION", + DEFAULT_MAX_CTV_FANOUT_RECIPIENTS_PER_TRANSACTION, + ) ), "reserved_coinbase_outputs": reserved_coinbase_outputs, }, @@ -3056,9 +2344,14 @@ def prism_ctv_settlement_config( block_height=block_height, parent_hash=parent_hash, ), - "premium_bps": env_positive_int( - "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", - DEFAULT_CTV_FANOUT_FEE_PREMIUM_BPS, + "premium_bps": ( + env_positive_int( + "PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", + 12_000, + environ=settlement_environment, + ) + if settlement_environment is not None + else env_positive_int("PRISM_CTV_FANOUT_FEE_PREMIUM_BPS", 12_000) ), } return config @@ -3069,9 +2362,22 @@ def ctv_fanout_market_fee_rate_bits_per_1000_weight( block_height: int | None = None, parent_hash: str | None = None, ) -> int: - configured_rate = env_optional_positive_int_with_legacy( - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", - "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", + coordinator_config = getattr(self, "config", None) + ctv_config = coordinator_config.ctv if coordinator_config is not None else None + settlement_environment = ( + dict(ctv_config.settlement_environment) if ctv_config is not None else None + ) + configured_rate = ( + env_optional_positive_int_with_legacy( + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", + environ=settlement_environment, + ) + if settlement_environment is not None + else env_optional_positive_int_with_legacy( + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_BITS_PER_1000_WEIGHT", + "PRISM_CTV_FANOUT_FEE_MARKET_RATE_SATS_PER_1000_WEIGHT", + ) ) if configured_rate is not None: return configured_rate @@ -3083,10 +2389,16 @@ def ctv_fanout_market_fee_rate_bits_per_1000_weight( if cache_key in fee_rate_cache: return fee_rate_cache[cache_key] try: - estimate = self.rpc.call( - "estimatesmartfee", - [env_positive_int("PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", 2)], + estimate_target_blocks = ( + env_positive_int( + "PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", + 2, + environ=settlement_environment, + ) + if settlement_environment is not None + else env_positive_int("PRISM_CTV_FANOUT_FEE_ESTIMATE_TARGET_BLOCKS", 2) ) + estimate = self.rpc.call("estimatesmartfee", [estimate_target_blocks]) if not isinstance(estimate, dict): raise RuntimeError("estimatesmartfee returned non-object") errors = estimate.get("errors") @@ -3107,366 +2419,611 @@ def ctv_fanout_market_fee_rate_bits_per_1000_weight( fee_rate_cache[cache_key] = rate return rate + def _ensure_share_writer_service(self) -> ShareWriter: + service = self.__dict__.get("_share_writer_service") + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_share_writer_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("_share_writer_service") + if service is not None: + return service + append_queue = self.__dict__.get("share_append_queue") + if append_queue is None: + append_queue = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) + floor_lock = self.__dict__.get("_pending_share_commit_lock") + if floor_lock is None: + floor_lock = threading.Lock() + floor = self.__dict__.get("_pending_share_commit_floor") + if floor is None: + floor = {} + recovery_lock = self.__dict__.get("share_recovery_lock") + if recovery_lock is None: + recovery_lock = threading.Lock() + service = ShareWriter( + ShareWriterConfig( + batch_size=int( + self.__dict__.get( + "share_commit_batch_size", + DEFAULT_SHARE_COMMIT_BATCH_SIZE, + ) + ), + linger_seconds=float( + self.__dict__.get( + "share_commit_linger_seconds", + DEFAULT_SHARE_COMMIT_LINGER_MILLISECONDS / 1000.0, + ) + ), + enqueue_timeout_seconds=float( + self.__dict__.get( + "share_commit_timeout_seconds", + DEFAULT_SHARE_COMMIT_TIMEOUT_SECONDS, + ) + ), + pending_floor_warn_seconds=PRISM_PENDING_SHARE_COMMIT_WARN_SECONDS, + recovery_path=self.__dict__.get("share_recovery_path"), + ), + ShareWriterPorts( + ledger=lambda: self.ledger, + writer_operation=lambda component: self._writer_operation(component), + reserve_writer=lambda component: ( + self._ensure_shutdown_controller().reserve_writer(component) + ), + writer_admission_closed=lambda: ( + self._ensure_shutdown_controller().writer_admission_closed() + ), + has_active_writer=lambda components: ( + self._ensure_shutdown_controller().has_active_writer(components) + ), + heartbeat=lambda name: self._record_heartbeat(name), + monotonic=lambda: time.monotonic(), + wall_time_ms=lambda: now_ms(), + stop_is_set=lambda: bool( + getattr(self, "stop_event", threading.Event()).is_set() + ), + stop_wait=lambda delay: bool( + getattr(self, "stop_event", threading.Event()).wait(delay) + ), + log=lambda message: print(message, flush=True), + log_exception=traceback.print_exc, + hot_path_log_enabled=lambda: bool( + getattr(self, "hot_path_log_enabled", False) + ), + ), + append_queue=append_queue, + floor_lock=floor_lock, + floor=floor, + recovery_lock=recovery_lock, + active=bool(self.__dict__.get("share_writer_active", False)), + append_failures=int( + self.__dict__.get("share_append_failure_count", 0) + ), + recovered_to_disk=int( + self.__dict__.get("shares_recovered_to_disk", 0) + ), + replayed=int(self.__dict__.get("shares_replayed", 0)), + ) + self.__dict__["_share_writer_service"] = service + for name in ( + "share_append_queue", + "share_commit_batch_size", + "share_commit_linger_seconds", + "share_commit_timeout_seconds", + "share_writer_active", + "share_append_failure_count", + "share_recovery_path", + "share_recovery_lock", + "shares_recovered_to_disk", + "shares_replayed", + "_pending_share_commit_lock", + "_pending_share_commit_floor", + ): + self.__dict__.pop(name, None) + return service + def _ensure_job_cache_state(self) -> None: - if not hasattr(self, "_job_cache_lock"): - self._job_cache_lock = threading.Lock() - if not hasattr(self, "_active_job_bundle_builds"): - self._active_job_bundle_builds: dict[ - tuple[object, ...], _JobBundleBuildControl - ] = {} - if not hasattr(self, "_job_build_lock"): - # Compatibility seam for embedders/tests. Expensive construction is - # coordinated by the bounded latest-wins scheduler below, not held - # under this lock. - self._job_build_lock = threading.Lock() - if not hasattr(self, "_job_build_scheduler_lock"): - self._job_build_scheduler_lock = threading.RLock() - if not hasattr(self, "_job_build_priority_preparations"): - self._job_build_priority_preparations: dict[int, float] = {} - if not hasattr(self, "_job_build_priority_preparation_sequence"): - self._job_build_priority_preparation_sequence = 0 - if not hasattr(self, "_job_build_routine_preparations"): - self._job_build_routine_preparations: dict[ - int, - weakref.ReferenceType[_JobBuildCancellation], - ] = {} - if not hasattr(self, "_job_build_routine_preparation_sequence"): - self._job_build_routine_preparation_sequence = 0 - if not hasattr(self, "_job_build_priority_changed"): - self._job_build_priority_changed = threading.Event() - if not hasattr(self, "_job_build_executor"): - self._job_build_executor: ThreadPoolExecutor | None = None - if not hasattr(self, "_job_build_executor_shutdown"): - self._job_build_executor_shutdown = False - if not hasattr(self, "_job_build_active"): - self._job_build_active: _JobBuildFlight | None = None - if not hasattr(self, "_job_build_retiring"): - self._job_build_retiring: _JobBuildFlight | None = None - if not hasattr(self, "_job_build_pending"): - self._job_build_pending: _JobBuildRequest | None = None - if not hasattr(self, "_job_build_issued_at_ms"): - self._job_build_issued_at_ms: OrderedDict[int, int] = OrderedDict() - if not hasattr(self, "job_build_timeout_seconds"): - self.job_build_timeout_seconds = DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS - if not hasattr(self, "job_build_cancel_grace_seconds"): - self.job_build_cancel_grace_seconds = ( - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS - ) - if not hasattr(self, "_template_artifacts"): - self._template_artifacts: CachedTemplateArtifacts | None = None - if not hasattr(self, "_template_artifact_generation"): - self._template_artifact_generation = int( - getattr(self._template_artifacts, "generation", 0) - ) - if not hasattr(self, "_job_bundle_cache"): - self._job_bundle_cache: OrderedDict[ - tuple[object, ...], CachedJobBundle - ] = OrderedDict() - if not hasattr(self, "_payout_state_generation"): - self._payout_state_generation = 0 - if not hasattr(self, "_payout_state_prepare_lock"): - # Ledger mutations and the ledger reads used to build signed jobs - # share this lock. They may be expensive, but they never block - # delivery of an already-published immutable generation. - self._payout_state_prepare_lock = threading.RLock() - if not hasattr(self, "_payout_state_source"): - self._payout_state_source: tuple[int, str | None, str, float] = ( - 0, - None, - "startup", - time.monotonic(), - ) - if not hasattr(self, "_published_payout_state"): - self._published_payout_state = PublishedPayoutState( - generation=self._payout_state_generation, - source_generation=0, - source_tip_hash=None, - published_monotonic=time.monotonic(), - ) - if not hasattr(self, "_payout_ledger_artifact"): - self._payout_ledger_artifact: PayoutLedgerArtifact | None = None - if not hasattr(self, "_payout_ledger_artifact_generation"): - self._payout_ledger_artifact_generation = 0 - if not hasattr(self, "_pending_share_commit_lock"): - self._pending_share_commit_lock = threading.Lock() - if not hasattr(self, "_pending_share_commit_floor"): - # Shares whose accepted_at_ms has been assigned but whose ledger - # row has not reached a terminal outcome in this process. Job and - # payout-artifact snapshot anchors clamp below every entry so a - # bundle's eligible-share set stays exactly reproducible from the - # durable ledger after those rows commit. - self._pending_share_commit_floor: dict[int, list[object]] = {} - if not hasattr(self, "_payout_artifact_executor_lock"): - self._payout_artifact_executor_lock = threading.Lock() - if not hasattr(self, "_payout_artifact_executor"): - self._payout_artifact_executor: ThreadPoolExecutor | None = None - if not hasattr(self, "_payout_artifact_future"): - self._payout_artifact_future: Future[None] | None = None - if not hasattr(self, "_payout_artifact_requested"): - self._payout_artifact_requested: tuple[int, int] | None = None - if not hasattr(self, "_payout_artifact_executor_shutdown"): - self._payout_artifact_executor_shutdown = False - if not hasattr(self, "_payout_state_delivery_gate"): - # Orders reconciliation mutations against final job-delivery - # admission while preserving parallel sends to different miners. - self._payout_state_delivery_gate = _PayoutStateDeliveryGate() - if not hasattr(self, "_payout_balance_mutation_lock"): - # Keep durable payout-balance transitions serialized even when their - # preparation intentionally runs outside the delivery gate. The - # accepted-block path can hold this lock across expensive writes - # without preventing replacement jobs from reaching miners. - self._payout_balance_mutation_lock = threading.RLock() - if not hasattr(self, "_accepted_block_payout_preview_condition"): - self._accepted_block_payout_preview_condition = threading.Condition() - if not hasattr(self, "_accepted_block_payout_previews"): - # Durable replay registers an unlanded transition. Once its block - # is active, reconciliation is barred and child/descendant builders - # wait for or consume its verified prospective balance snapshot. - self._accepted_block_payout_previews: dict[ - str, _AcceptedBlockPayoutTransition - ] = {} - if not hasattr(self, "_invalidated_accepted_block_payout_previews"): - # A landed transition can be withdrawn before durability catches - # up. Keep a height-bearing tombstone so a new exact/descendant - # build cannot fall through to database balances that omit it in - # the gap before durable retry re-registers the candidate. - self._invalidated_accepted_block_payout_previews: dict[ - str, int | None - ] = {} + self._ensure_job_bundle_service() + self._ensure_share_writer_service() if not hasattr(self, "_accounted_accepted_block_hashes"): self._accounted_accepted_block_hashes: set[str] = set() - if not hasattr(self, "_payout_state_metrics_lock"): - self._payout_state_metrics_lock = threading.Lock() - if not hasattr(self, "payout_state_histograms"): - self.payout_state_histograms = { - name: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for name in ("preparation", "publish", "first_delivery") - } - if not hasattr(self, "payout_gate_wait_histograms"): - self.payout_gate_wait_histograms = { - relation: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS - } - if not hasattr(self, "payout_state_candidates_discarded"): - self.payout_state_candidates_discarded = 0 - if not hasattr(self, "_payout_first_delivery_pending"): - self._payout_first_delivery_pending: tuple[int, float] | None = None - if not hasattr(self, "_payout_state_publication_blocked"): - self._payout_state_publication_blocked = False - if not hasattr(self, "_job_build_phase_local"): - self._job_build_phase_local = threading.local() - if not hasattr(self, "job_cache_hit_counts"): - self.job_cache_hit_counts = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} - if not hasattr(self, "job_cache_miss_counts"): - self.job_cache_miss_counts = {kind: 0 for kind in PRISM_JOB_CACHE_KINDS} - if not hasattr(self, "job_build_seconds_bucket_counts"): - self.job_build_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS - } - if not hasattr(self, "job_build_seconds_sum"): - self.job_build_seconds_sum = 0.0 - if not hasattr(self, "job_build_count"): - self.job_build_count = 0 - if not hasattr(self, "job_build_phase_seconds"): - self.job_build_phase_seconds = {phase: 0.0 for phase in PRISM_JOB_BUILD_PHASES} - if not hasattr(self, "job_build_scheduler_counts"): - self.job_build_scheduler_counts = { - "requests": 0, - "starts": 0, - "completions": 0, - "supersessions": 0, - "obsolete_results": 0, - } - if not hasattr(self, "job_build_priority_counts"): - self.job_build_priority_counts = { - result: 0 - for result in ( - "started", - "coalesced", - "queued", - "routine_deferred", - "routine_preempted", - ) - } - if not hasattr(self, "job_build_priority_admission_seconds"): - self.job_build_priority_admission_seconds = { - "sum": 0.0, - "count": 0, - } - if not hasattr(self, "initial_job_prepared_work_counts"): - self.initial_job_prepared_work_counts = { - result: 0 - for result in ("cache_hit", "singleflight", "deferred") - } - if not hasattr(self, "job_build_cancellation_seconds"): - self.job_build_cancellation_seconds = { - "sum": 0.0, - "count": 0, - } - if not hasattr(self, "job_build_replacement_start_seconds"): - self.job_build_replacement_start_seconds = { - "sum": 0.0, - "count": 0, - } - if not hasattr(self, "job_build_worker_counts"): - self.job_build_worker_counts = { - "starts": 0, - "terminations": 0, - "crashes": 0, - "restarts": 0, - } - if not hasattr(self, "_job_build_worker_restart_pending"): - self._job_build_worker_restart_pending = False + self._ensure_payout_state_service() if not hasattr(self, "_health_snapshot"): self._health_snapshot: dict[str, object] | None = None + if not hasattr(self, "_health_snapshot_lock"): + self._health_snapshot_lock = threading.Lock() if not hasattr(self, "_health_snapshot_monotonic"): self._health_snapshot_monotonic: float | None = None if not hasattr(self, "_health_refresh_loop_running"): self._health_refresh_loop_running = False if not hasattr(self, "health_snapshot_refresh_failure_count"): self.health_snapshot_refresh_failure_count = 0 - if not hasattr(self, "_bundle_preparation_lock"): - self._bundle_preparation_lock = threading.Lock() - if not hasattr(self, "_bundle_preparation_flights"): - self._bundle_preparation_flights: dict[ - tuple[object, ...], _SharedBundlePreparationFlight - ] = {} - if not hasattr(self, "shared_bundle_build_counts"): - self.shared_bundle_build_counts = { - outcome: 0 - for outcome in ("started", "completed", "superseded", "failed") - } - if not hasattr(self, "shared_bundle_preparation_seconds_sum"): - self.shared_bundle_preparation_seconds_sum = 0.0 - if not hasattr(self, "shared_bundle_preparation_count"): - self.shared_bundle_preparation_count = 0 - if not hasattr(self, "shared_bundle_preparation_waiters"): - self.shared_bundle_preparation_waiters = 0 - if not hasattr(self, "_prepared_ready_bundle"): - self._prepared_ready_bundle: CachedJobBundle | None = None - if not hasattr(self, "_prepared_ready_snapshot"): - self._prepared_ready_snapshot: QbitTipTemplateSnapshot | None = None - if not hasattr(self, "job_preparation_pending"): - self.job_preparation_pending = False - if not hasattr(self, "_progress_health_lock"): - self._progress_health_lock = threading.Lock() - if not hasattr(self, "_progress_current_template_generation"): - self._progress_current_template_generation = 0 - if not hasattr(self, "_progress_current_template_fingerprint"): - self._progress_current_template_fingerprint: str | None = None - if not hasattr(self, "_progress_current_payout_generation"): - self._progress_current_payout_generation = self._payout_state_generation - if not hasattr(self, "_progress_published_template_generation"): - self._progress_published_template_generation = 0 - if not hasattr(self, "_progress_published_template_fingerprint"): - self._progress_published_template_fingerprint: str | None = None - if not hasattr(self, "_progress_published_payout_generation"): - self._progress_published_payout_generation = 0 - if not hasattr(self, "_progress_has_published_work"): - self._progress_has_published_work = False - if not hasattr(self, "_progress_last_tip_poll_monotonic"): - self._progress_last_tip_poll_monotonic: float | None = None - if not hasattr(self, "_progress_last_delivery_template_generation"): - self._progress_last_delivery_template_generation = 0 - if not hasattr(self, "_progress_last_delivery_template_fingerprint"): - self._progress_last_delivery_template_fingerprint: str | None = None - if not hasattr(self, "_progress_last_delivery_payout_generation"): - self._progress_last_delivery_payout_generation = 0 - if not hasattr(self, "_progress_last_delivery_monotonic"): - self._progress_last_delivery_monotonic: float | None = None - if not hasattr(self, "_progress_pending_since_monotonic"): - self._progress_pending_since_monotonic: float | None = float( - getattr(self, "started_monotonic", time.monotonic()) + self._ensure_progress_health_service() + + def _ensure_job_bundle_service(self) -> JobBundleService: + service = getattr(self, "_job_bundle_service", None) + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_job_bundle_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = getattr(self, "_job_bundle_service", None) + if service is not None: + return service + repository = TemplateArtifactRepository( + TemplateArtifactPorts( + fetch_template=lambda: self.rpc.call( + "getblocktemplate", + [{"rules": qbit_gbt_rules(getattr(self, "qbit_chain", "regtest"))}], + ), + fetch_bestblockhash=lambda: str( + self.rpc.call("getbestblockhash") + ), + newest_observed_tip=self._job_bundle_newest_observed_tip, + observe_tip=self._submit_tip_observation_for_refresh, + schedule_refresh_retry=self._schedule_tip_refresh_retry, + pinned_issuance_artifacts=( + self._job_bundle_pinned_issuance_artifacts + ), + repinned_issuance_artifacts=( + self._job_bundle_repinned_issuance_artifacts + ), + record_tip=lambda tip_hash: ( + self._ensure_tip_refresh_service().observe_tip(tip_hash) + ), + ), + cache_seconds=float( + getattr(self, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) + ), + scale_network_difficulty=scaled_network_difficulty, ) - if not hasattr(self, "_progress_publication_divergence_since_monotonic"): - self._progress_publication_divergence_since_monotonic: float | None = ( - None - if self._progress_has_published_work - else float(getattr(self, "started_monotonic", time.monotonic())) + service = JobBundleService( + JobBundleConfig( + cache_seconds=float( + getattr( + self, + "job_bundle_cache_seconds", + DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, + ) + ), + build_timeout_seconds=float( + getattr( + self, + "job_build_timeout_seconds", + DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, + ) + ), + cancel_grace_seconds=float( + getattr( + self, + "job_build_cancel_grace_seconds", + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + ) + ), + min_ready_miners=int(getattr(self, "min_ready_miners", 3)), + extranonce2_size=int(getattr(self, "extranonce2_size", 8)), + share_difficulty=getattr(self, "share_difficulty", Decimal("1")), + ), + JobBundlePorts( + payout_state=self._ensure_payout_state_service, + accepted_share_stats=lambda: self.accepted_share_stats(), + snapshot_at_job_issue=lambda anchor, window: ( + self.ledger.snapshot_at_job_issue( + anchor, + window_weight=window, + ) + ), + snapshot_anchor_ms=lambda value: self._job_snapshot_anchor_ms( + value + ), + payout_policy=lambda: self.prism_payout_policy(), + ctv_settlement=lambda height, parent: ( + self.prism_ctv_settlement_config( + block_height=height, + parent_hash=parent, + ) + ), + coinbase_suffix=lambda first, second: ( + self.coinbase_script_sig_suffix_hex(first, second) + ), + signing_seed_hex=lambda: str( + getattr(self, "signing_seed_hex", "") + ), + ledger_signing_seed_hex=lambda: str( + getattr(self, "ledger_attestation_signing_seed_hex", "") + ), + await_parent_preview=lambda parent, height: ( + self._await_pending_parent_payout_preview( + parent, + parent_height=height, + ) + ), + prior_balances_for_parent=lambda parent, height, fallback: ( + self._prior_balances_for_job_parent( + parent, + parent_height=height, + fallback_balances=fallback, + ) + ), + serialize_prior_balance_preview=lambda balances: ( + self._serialize_prior_balance_preview(balances) + ), + accepted_block_preview_from_bundle=lambda bundle, balances: ( + self._accepted_block_payout_preview_from_bundle( + bundle, + prior_balances=balances, + ) + ), + schedule_refresh_retry=lambda: self._schedule_tip_refresh_retry(), + idle_tip_diverged=lambda: self._job_bundle_idle_tip_diverged(), + artifacts_buildable=lambda artifacts: ( + self._job_bundle_artifacts_buildable(artifacts) + ), + published_snapshot_artifacts=( + self._job_bundle_published_snapshot_artifacts + ), + published_artifacts=self._job_bundle_published_artifacts, + note_tip_refresh_superseded=lambda: ( + self._record_job_bundle_tip_superseded() + ), + record_tip_refresh_phase=lambda phase, elapsed: ( + self._observe_tip_refresh_build_phase(phase, elapsed) + ), + clear_retained_collection_refresh=lambda: ( + self._clear_retained_collection_refresh() + ), + readiness_promoted=self._on_job_readiness_promoted, + start_bundle_build=lambda: ( + self._ensure_progress_health_service().start_bundle_build() + ), + wall_time_ms=lambda: now_ms(), + ), + repository, + ) + compiler = self._new_bundle_compiler(service) + repository.bind_event_sink( + TemplateArtifactEventSink( + record_cache_event=lambda hit: service.record_cache_event( + "template", + hit=hit, + ), + record_build_phase=service.record_phase, + artifacts_changed=lambda artifacts, fingerprint_changed: ( + self._on_template_artifacts_changed( + service, + artifacts, + fingerprint_changed, + ) + ), + artifacts_cleared=service.on_template_artifacts_cleared, + ) ) - if not hasattr(self, "_progress_refresh_signal_pending"): - self._progress_refresh_signal_pending = False - if not hasattr(self, "_progress_active_refresh_count"): - self._progress_active_refresh_count = 0 - if not hasattr(self, "_progress_last_refresh_activity_monotonic"): - self._progress_last_refresh_activity_monotonic: float | None = None - if not hasattr(self, "_progress_bundle_build_counter"): - self._progress_bundle_build_counter = 0 - if not hasattr(self, "_progress_bundle_builds"): - self._progress_bundle_builds: dict[int, float] = {} + service.bind_bundle_compiler(compiler) + self._bundle_compiler = compiler + self._job_bundle_service = service + return service - def _job_build_phases(self) -> dict[str, float]: - """Per-thread scratch dict of phase timings for the current build.""" - self._ensure_job_cache_state() - phases = getattr(self._job_build_phase_local, "phases", None) - if phases is None: - phases = {} - self._job_build_phase_local.phases = phases - return phases + def _on_job_readiness_promoted(self) -> None: + self._progress_note_refresh_pending() + self._ensure_tip_refresh_service().readiness_promoted() - def _cancel_obsolete_job_bundle_builds( + def _on_template_artifacts_changed( self, - *, - current_tip: str | None = None, - payout_state_generation: int | None = None, + service: JobBundleService, + artifacts: CachedTemplateArtifacts, + fingerprint_changed: bool, ) -> None: - """Cancel only builds proven obsolete by a newer exact generation.""" - self._ensure_job_cache_state() - processes: list[subprocess.Popen[str]] = [] - with self._job_cache_lock: - for control in self._active_job_bundle_builds.values(): - obsolete = ( - current_tip is not None - and control.previousblockhash != current_tip - ) or ( - payout_state_generation is not None - and control.payout_state_generation - != int(payout_state_generation) - ) - if not obsolete or control.cancel_event.is_set(): - continue - control.cancel_event.set() - if control.process is not None: - processes.append(control.process) - for process in processes: - if process.poll() is not None: - continue - try: - process.terminate() - except ProcessLookupError: - pass + service.on_template_artifacts_changed(artifacts, fingerprint_changed) + if fingerprint_changed: + self._ensure_tip_refresh_service().template_artifacts_changed(artifacts) + + def _job_bundle_newest_observed_tip(self) -> str | None: + self._ensure_tip_refresh_state() + with self.lock: + return self._newest_observed_tip_locked() + + def _job_bundle_pinned_issuance_artifacts( + self, + ) -> CachedTemplateArtifacts | None: + self._ensure_tip_refresh_state() + with self.lock: + published = getattr(self, "current_tip_first_seen", None) + latest_detected = getattr(self, "latest_detected_tip", None) + published_snapshot = getattr(self, "tip_template_snapshot", None) + if ( + published is not None + and latest_detected is not None + and latest_detected[0] != published[0] + and published_snapshot is not None + and published_snapshot.bestblockhash == published[0] + and published_snapshot.template_artifacts is not None + and self._published_tip_authoritative_locked(time.monotonic()) + ): + return published_snapshot.template_artifacts + return None + + def _job_bundle_repinned_issuance_artifacts( + self, + artifacts: CachedTemplateArtifacts, + ) -> CachedTemplateArtifacts | None: + with self.lock: + published = getattr(self, "current_tip_first_seen", None) + published_snapshot = getattr(self, "tip_template_snapshot", None) + if ( + published is not None + and artifacts.previousblockhash != published[0] + and published_snapshot is not None + and published_snapshot.bestblockhash == published[0] + and published_snapshot.template_artifacts is not None + and self._published_tip_authoritative_locked(time.monotonic()) + ): + return published_snapshot.template_artifacts + return None + + def _job_bundle_artifacts_buildable( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: + with self.lock: + return self._artifacts_buildable_locked(artifacts) + + def _job_bundle_published_snapshot_artifacts( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: + with self.lock: + return self._published_snapshot_artifacts_locked(artifacts) + + def _job_bundle_published_artifacts( + self, + ) -> CachedTemplateArtifacts | None: + with self.lock: + snapshot = getattr(self, "tip_template_snapshot", None) + return None if snapshot is None else snapshot.template_artifacts + + def _job_bundle_idle_tip_diverged(self) -> bool: + with self.lock: + return self._vardiff_idle_tip_divergence_locked() + + def _record_job_bundle_tip_superseded(self) -> None: + self._ensure_tip_refresh_service().record_superseded_result() + + def _new_bundle_compiler(self, service: JobBundleService) -> BundleCompiler: + return BundleCompiler( + BundleCompilerPorts( + payout_policy=self.prism_payout_policy, + ctv_settlement=lambda height, parent: ( + self.prism_ctv_settlement_config( + block_height=height, + parent_hash=parent, + ) + ), + signing_seed_hex=lambda: str( + getattr(self, "signing_seed_hex", "") + ), + ledger_signing_seed_hex=lambda: str( + getattr(self, "ledger_attestation_signing_seed_hex", "") + ), + bundle_timeout_seconds=lambda: float( + getattr( + self, + "bundle_build_timeout_seconds", + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + ) + ), + cancel_grace_seconds=lambda: float( + getattr( + self, + "job_build_cancel_grace_seconds", + DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, + ) + ), + phases=service.phases, + record_tip_refresh_phase=self._observe_tip_refresh_build_phase, + record_ipc_bytes=self._record_tip_refresh_ipc_bytes, + record_worker_failure=self._record_bundle_compiler_failure, + record_worker_event=service.record_worker_event, + tip_refresh_metrics_enabled=service.tip_refresh_metrics_enabled, + active_build_control=service.active_build_control, + register_process=lambda control, process: service.register_process( + control, # type: ignore[arg-type] + process, + ), + superseded_error=lambda message: _JobBundleBuildSuperseded( + message + ), + ) + ) + + def _ensure_bundle_compiler(self) -> BundleCompiler: + compiler = getattr(self, "_bundle_compiler", None) + if compiler is not None: + return compiler + self._ensure_job_bundle_service() + compiler = getattr(self, "_bundle_compiler", None) + if compiler is None: + raise RuntimeError("job bundle compiler binding was not published") + return compiler + + def _record_bundle_compiler_failure(self) -> None: + self._ensure_tip_refresh_service().record_worker_failure() + + def _ensure_payout_state_service(self) -> PayoutStateService: + service = getattr(self, "_payout_state_service", None) + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_payout_state_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = getattr(self, "_payout_state_service", None) + if service is not None: + return service + service = PayoutStateService( + PayoutStatePorts( + accepted_share_stats=lambda: self.accepted_share_stats(), + snapshot_at_job_issue=lambda anchor, window: ( + self.ledger.snapshot_at_job_issue( + anchor, + window_weight=window, + ) + ), + current_prior_balances=lambda: ( + self.ledger.current_prior_balances() + ), + snapshot_anchor_ms=lambda issued_at_ms: ( + self._job_snapshot_anchor_ms(issued_at_ms) + ), + current_template_network_difficulty=( + self._payout_template_network_difficulty + ), + pool_ready=lambda: self._ensure_job_bundle_service().ready_latched(), + record_build_phase=self._record_payout_build_phase, + invalidate_job_cache=self._invalidate_payout_job_cache, + clear_retained_collection_refresh=( + self._clear_retained_collection_refresh + ), + cancel_obsolete_job_builds=self._cancel_obsolete_job_builds, + cancel_obsolete_bundle_builds=lambda generation: ( + self._cancel_obsolete_job_bundle_builds( + payout_state_generation=generation + ) + ), + payout_invalidated=self._on_payout_state_invalidated, + payout_published=self._on_payout_state_published, + schedule_refresh_retry=self._schedule_tip_refresh_retry, + chain_block_hash=lambda height: str( + self.rpc.call("getblockhash", [height]) + ), + stop_requested=lambda: bool( + getattr(self, "stop_event", threading.Event()).is_set() + ), + ), + wall_time_ms=lambda: now_ms(), + histogram_buckets=PRISM_TIP_REFRESH_SECONDS_BUCKETS, + config=PayoutStateConfig( + accepted_block_preview_wait_seconds=float( + getattr( + self, + "accepted_block_payout_preview_wait_seconds", + DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, + ) + ), + reconcile_supersession_retries=int( + getattr( + self, + "payout_reconcile_supersession_retries", + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + ) + ), + ), + ) + self._payout_state_service = service + return service + + def _payout_template_network_difficulty(self) -> int | None: + service = getattr(self, "_job_bundle_service", None) + if service is None: + return None + artifacts = service.template_repository.current_artifacts() + return None if artifacts is None else artifacts.network_difficulty + + def _record_payout_build_phase(self, phase: str, elapsed: float) -> None: + if phase in PRISM_TIP_REFRESH_BUILD_PHASES: + self._observe_tip_refresh_build_phase(phase, elapsed) + if phase == "payout_artifact": + phases = self._job_build_phases() + phases[phase] = phases.get(phase, 0.0) + elapsed + + def _invalidate_payout_job_cache(self) -> None: + self._ensure_job_bundle_service().clear_cache() + + def _clear_retained_collection_refresh(self) -> None: + self._ensure_tip_refresh_service().clear_retained_collection_refresh() + + def _on_payout_state_invalidated( + self, + generation: int, + invalidated_monotonic: float, + ) -> None: + self._record_progress_payout_generation( + generation, + invalidated_monotonic, + ) + self._ensure_tip_refresh_service().payout_generation_invalidated(generation) + + def _on_payout_state_published( + self, + generation: int, + invalidated_monotonic: float, + ) -> None: + self._record_progress_payout_generation( + generation, + invalidated_monotonic, + ) + self._ensure_tip_refresh_service().payout_generation_changed(generation) + + def _ensure_progress_health_service(self) -> ProgressHealthService: + service = getattr(self, "progress_health_service", None) + if service is None: + started = float(getattr(self, "started_monotonic", time.monotonic())) + service = ProgressHealthService( + ProgressHealthConfig( + pending_refresh_deadline_seconds=float( + getattr( + self, + "health_pending_refresh_max_age_seconds", + DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, + ) + ), + tip_poll_deadline_seconds=float( + getattr( + self, + "health_tip_poll_max_age_seconds", + DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, + ) + ), + bundle_build_deadline_seconds=float( + getattr( + self, + "bundle_build_timeout_seconds", + DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, + ) + ), + ), + started_monotonic=started, + initial_payout_generation=( + self._ensure_payout_state_service().snapshot().generation + ), + ) + self.progress_health_service = service + return service + + def _job_build_phases(self) -> dict[str, float]: + return self._ensure_job_bundle_service().phases() + + def _cancel_obsolete_job_bundle_builds( + self, + *, + current_tip: str | None = None, + payout_state_generation: int | None = None, + ) -> None: + self._ensure_job_bundle_service().cancel_obsolete_bundle_processes( + current_tip=current_tip, + payout_state_generation=payout_state_generation, + ) def _register_job_bundle_process( self, control: _JobBundleBuildControl, process: subprocess.Popen[str], ) -> None: - terminate = False - with self._job_cache_lock: - if ( - self._active_job_bundle_builds.get(control.key) is not control - or control.cancel_event.is_set() - ): - terminate = True - else: - control.process = process - if terminate and process.poll() is None: - try: - process.terminate() - except ProcessLookupError: - pass + self._ensure_job_bundle_service().register_process(control, process) def _build_payout_ledger_artifact( self, @@ -3474,65 +3031,8 @@ def _build_payout_ledger_artifact( artifact_payout_state_generation: int, network_difficulty: int, ) -> PayoutLedgerArtifact | None: - """Build a stable ledger snapshot without publishing it. - - Accepted-share counts fence both sides of the snapshot. If a writer - commits concurrently, this attempt is discarded rather than publishing - an artifact with an ambiguous cutoff; the normal inline path remains - the fail-closed fallback. - """ - self._ensure_job_cache_state() - ledger_started = time.monotonic() - try: - accepted_before, _ = self.accepted_share_stats() - with self._payout_state_prepare_lock: - with self._job_cache_lock: - if ( - expected_payout_state_generation - != self._payout_state_generation - ): - return None - snapshot_window_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * int(network_difficulty) - ) - snapshot_anchor_ms = self._job_snapshot_anchor_ms(now_ms()) - records = list( - self.ledger.snapshot_at_job_issue( - snapshot_anchor_ms, - window_weight=snapshot_window_weight, - ) - ) - prior_balances = self.ledger.current_prior_balances() - accepted_after, _ = self.accepted_share_stats() - except Exception: - # Artifact preparation is speculative. The synchronous bundle path - # still owns errors when current work actually requires a snapshot. - return None - finally: - self._observe_tip_refresh_build_phase( - "ledger_snapshot", - time.monotonic() - ledger_started, - ) - if accepted_before != accepted_after or not records: - return None - copy_started = time.monotonic() - shares_json = tuple(record.to_prism_json() for record in records) - frozen_balances = tuple(prior_balances) - self._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - copy_started, - ) - return PayoutLedgerArtifact( - generation=0, - payout_state_generation=artifact_payout_state_generation, - network_difficulty=int(network_difficulty), - accepted_share_count=accepted_after, - shares_json=shares_json, - prior_balances=frozen_balances, - prepared_monotonic=time.monotonic(), - snapshot_anchor_ms=snapshot_anchor_ms, + return self._ensure_payout_state_service().build_ledger_artifact( + expected_payout_state_generation, artifact_payout_state_generation, network_difficulty ) def _prepare_payout_ledger_artifact( @@ -3540,144 +3040,46 @@ def _prepare_payout_ledger_artifact( payout_state_generation: int, network_difficulty: int, ) -> None: - """Prepare and atomically publish an artifact for a current generation.""" - artifact = self._build_payout_ledger_artifact( - payout_state_generation, - payout_state_generation, - network_difficulty, + self._ensure_payout_state_service().prepare_ledger_artifact( + payout_state_generation, network_difficulty ) - if artifact is None: - return - with self._job_cache_lock: - if payout_state_generation != self._payout_state_generation: - return - self._payout_ledger_artifact_generation += 1 - self._payout_ledger_artifact = dataclass_replace( - artifact, - generation=self._payout_ledger_artifact_generation, - ) def _payout_artifact_preparation_loop(self) -> None: - while True: - with self._payout_artifact_executor_lock: - request = self._payout_artifact_requested - self._payout_artifact_requested = None - if request is None: - self._payout_artifact_future = None - return - self._prepare_payout_ledger_artifact(*request) + self._ensure_payout_state_service()._artifact_preparation_loop() def _schedule_payout_ledger_artifact_preparation( self, payout_state_generation: int, network_difficulty: int, ) -> None: - """Latest-generation-wins scheduling with one worker and one slot.""" - self._ensure_job_cache_state() - with self._payout_artifact_executor_lock: - if self._payout_artifact_executor_shutdown: - return - self._payout_artifact_requested = ( - int(payout_state_generation), - int(network_difficulty), - ) - if self._payout_artifact_future is not None: - return - executor = self._payout_artifact_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=1, - thread_name_prefix="prism-payout-artifact", - ) - self._payout_artifact_executor = executor - self._payout_artifact_future = executor.submit( - self._payout_artifact_preparation_loop - ) + self._ensure_payout_state_service().schedule_ledger_artifact_preparation( + payout_state_generation, network_difficulty + ) def _usable_payout_ledger_artifact( self, payout_state_generation: int, network_difficulty: int, ) -> PayoutLedgerArtifact | None: - self._ensure_job_cache_state() - with self._job_cache_lock: - artifact = self._payout_ledger_artifact - published_artifact = self._published_payout_state.artifact - if ( - artifact is None - or artifact.payout_state_generation != payout_state_generation - or artifact.network_difficulty != int(network_difficulty) - ): - return None - if published_artifact is None: - try: - published_artifact = self._current_payout_state_artifact() - except Exception: - return None - try: - accepted_share_count, _ = self.accepted_share_stats() - except Exception: - return None - if accepted_share_count != artifact.accepted_share_count: - return None - balances_sha256 = canonical_json_sha256(artifact.prior_balances) - with self._job_cache_lock: - if ( - self._payout_ledger_artifact is not artifact - or self._payout_state_generation != payout_state_generation - or self._published_payout_state.artifact is not published_artifact - ): - return None - if balances_sha256 != published_artifact.prior_balances_sha256: - # A candidate can carry a ledger snapshot prepared before its - # payout state is published. Never keep retrying that stale - # shortcut; the synchronous path will take a fresh snapshot. - self._payout_ledger_artifact = None - return None - return artifact + return self._ensure_payout_state_service().usable_ledger_artifact( + payout_state_generation, network_difficulty + ) def _schedule_current_payout_ledger_artifact_if_missing(self) -> None: - """Resume speculative preparation after a durable preview catches up.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - template_artifacts = self._template_artifacts - if template_artifacts is None: - return - if self._usable_payout_ledger_artifact( - payout_state_generation, - template_artifacts.network_difficulty, - ) is not None: - return - self._schedule_payout_ledger_artifact_preparation( - payout_state_generation, - template_artifacts.network_difficulty, - ) + self._ensure_payout_state_service().schedule_current_ledger_artifact_if_missing() def shutdown_payout_artifact_executor(self) -> None: - self._ensure_job_cache_state() - with self._payout_artifact_executor_lock: - executor = self._payout_artifact_executor - self._payout_artifact_executor = None - self._payout_artifact_executor_shutdown = True - self._payout_artifact_requested = None - if executor is not None: - executor.shutdown(wait=True, cancel_futures=True) + self._ensure_payout_state_service().shutdown() def _job_build_checkpoint( self, phase: str, cancellation: _JobBuildCancellation, ) -> None: - """Cooperative boundary around coordinator and isolated-worker phases.""" - cancellation.raise_if_cancelled(phase) def _record_job_cache_event(self, kind: str, *, hit: bool) -> None: - self._ensure_job_cache_state() - with self._job_cache_lock: - counts = self.job_cache_hit_counts if hit else self.job_cache_miss_counts - counts[kind] = int(counts.get(kind, 0)) + 1 + self._ensure_job_bundle_service().record_cache_event(kind, hit=hit) def _prepare_payout_state_artifact( self, @@ -3686,25 +3088,9 @@ def _prepare_payout_state_artifact( source_generation: int, cancellation: _JobBuildCancellation | None = None, ) -> PayoutStateArtifact: - """Snapshot carry-forward balances once for a payout generation.""" - - started = time.monotonic() - if cancellation is not None: - cancellation.raise_if_cancelled("payout artifact read") - with self._payout_state_prepare_lock: - balances = self.ledger.current_prior_balances() - if cancellation is not None: - cancellation.raise_if_cancelled("payout artifact serialization") - artifact = self._payout_state_artifact_from_balances( - generation=generation, - source_generation=source_generation, - balances=balances, - ) - phases = self._job_build_phases() - phases["payout_artifact"] = phases.get("payout_artifact", 0.0) + ( - time.monotonic() - started + return self._ensure_payout_state_service().prepare_artifact( + generation=generation, source_generation=source_generation, cancellation=cancellation ) - return artifact def _payout_state_artifact_from_balances( self, @@ -3713,826 +3099,86 @@ def _payout_state_artifact_from_balances( source_generation: int, balances: list[dict[str, object]], ) -> PayoutStateArtifact: - balances_json = canonical_json_text(balances) - return PayoutStateArtifact( - generation=generation, - source_generation=source_generation, - prior_balances_json=balances_json, - prior_balances_sha256=hashlib.sha256(balances_json.encode()).hexdigest(), - prepared_monotonic=time.monotonic(), + return self._ensure_payout_state_service().artifact_from_balances( + generation=generation, source_generation=source_generation, balances=balances ) def _current_payout_state_artifact( self, cancellation: _JobBuildCancellation | None = None, ) -> PayoutStateArtifact: - """Return the immutable artifact published for the current generation.""" - - self._ensure_job_cache_state() - while True: - with self._job_cache_lock: - if self._payout_state_publication_blocked: - raise _PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - published = self._published_payout_state - if ( - published.generation == self._payout_state_generation - and published.artifact is not None - and published.artifact.generation == published.generation - ): - return published.artifact - generation = self._payout_state_generation - source_generation = published.source_generation - artifact = self._prepare_payout_state_artifact( - generation=generation, - source_generation=source_generation, - cancellation=cancellation, - ) - with self._job_cache_lock: - published = self._published_payout_state - if ( - self._payout_state_publication_blocked - or self._payout_state_generation != generation - or published.source_generation != source_generation - ): - if cancellation is not None: - cancellation.raise_if_cancelled( - "payout artifact publication race" - ) - continue - self._published_payout_state = dataclass_replace( - published, - artifact=artifact, - ) - return artifact + return self._ensure_payout_state_service().current_artifact(cancellation) def _job_build_executor_locked(self) -> ThreadPoolExecutor: - if self._job_build_executor_shutdown: - raise RuntimeError("job build executor is shut down") - executor = self._job_build_executor - if executor is None: - executor = ThreadPoolExecutor( - max_workers=PRISM_JOB_BUILD_EXECUTOR_WORKERS, - thread_name_prefix="prism-job-build", - ) - self._job_build_executor = executor - return executor + return self._ensure_job_bundle_service()._executor_locked() def _start_job_build_locked(self, request: _JobBuildRequest) -> _JobBuildFlight: - executor = self._job_build_executor_locked() - flight = _JobBuildFlight(request=request) - self.job_build_scheduler_counts["starts"] += 1 - self.shared_bundle_build_counts["started"] += 1 - if request.superseded_monotonic is not None: - elapsed = max(0.0, time.monotonic() - request.superseded_monotonic) - self.job_build_replacement_start_seconds["sum"] += elapsed - self.job_build_replacement_start_seconds["count"] += 1 - future = executor.submit(self._execute_job_build_request, request) - flight.future = future - self._record_priority_admission_locked(request, "started") - return flight + return self._ensure_job_bundle_service()._start_locked(request) def _arm_job_build_locked(self, flight: _JobBuildFlight) -> None: - future = flight.future - assert future is not None - future.add_done_callback( - lambda completed, build_flight=flight: self._job_build_done( - build_flight, - completed, - ) - ) + self._ensure_job_bundle_service()._arm_locked(flight) def _execute_job_build_request( self, request: _JobBuildRequest, ) -> CachedJobBundle: - request.cancellation.raise_if_cancelled("start") - control = _JobBundleBuildControl( - key=request.equivalence_key, - previousblockhash=request.key.previous_block_hash, - payout_state_generation=request.key.payout_state_generation, - payout_artifact_generation=( - request.payout_ledger_artifact.generation - if request.payout_ledger_artifact is not None - else 0 - ), - ) - with self._job_cache_lock: - self._active_job_bundle_builds[control.key] = control - previous_control = getattr( - self._job_build_phase_local, - "bundle_build_control", - None, - ) - self._job_build_phase_local.bundle_build_control = control - try: - with localcontext(request.decimal_context): - return self.build_shared_job_bundle( - request.artifacts, - request.worker, - mode=request.mode, - payout_state_generation=request.key.payout_state_generation, - payout_artifact=request.payout_ledger_artifact, - key=request.cache_key, - build_request=request, - ) - finally: - self._job_build_phase_local.bundle_build_control = previous_control - with self._job_cache_lock: - if self._active_job_bundle_builds.get(control.key) is control: - self._active_job_bundle_builds.pop(control.key, None) - control.process = None + return self._ensure_job_bundle_service()._execute_request(request) @staticmethod def _collection_job_builds_are_independent( first: _JobBuildRequest, second: _JobBuildRequest, ) -> bool: - """Distinct workers in one immutable collection cohort are peers.""" - - return ( - first.mode == "collection" - and second.mode == "collection" - and first.key.collection_identity != second.key.collection_identity - and dataclass_replace(first.key, collection_identity=None) - == dataclass_replace(second.key, collection_identity=None) - ) + return JobBundleService.collection_builds_independent(first, second) @staticmethod def _job_build_requests_can_share( first: _JobBuildRequest, second: _JobBuildRequest, ) -> bool: - """Share exact builds plus ready work stable across clock-only refreshes.""" - - return first.equivalence_key == second.equivalence_key or ( - first.mode == "ready" - and second.mode == "ready" - and first.cache_key == second.cache_key - ) + return JobBundleService.requests_can_share(first, second) @staticmethod def _ready_job_build_precedes_collection( first: _JobBuildRequest, second: _JobBuildRequest, ) -> bool: - """Live ready work cannot be displaced by a collection-mode retry.""" - - return ( - first.mode == "ready" - and second.mode == "collection" - and not first.cancellation.is_set() - ) + return JobBundleService.ready_precedes_collection(first, second) @staticmethod - def _defer_job_build_locked( + def _defer_collection_job_build_locked( *blockers: Future[CachedJobBundle], ) -> Future[CachedJobBundle]: - """Wake a bounded lower-priority waiter when occupied capacity exits.""" - - deferred: Future[CachedJobBundle] = Future() - wake_lock = threading.Lock() - - def wake_for_retry(_completed: Future[CachedJobBundle]) -> None: - with wake_lock: - if not deferred.done(): - deferred.set_exception( - JobBuildSuperseded( - "job build capacity became available; retrying" - ) - ) + return JobBundleService.defer_collection(*blockers) - for blocker in blockers: - blocker.add_done_callback(wake_for_retry) - return deferred + def _cancel_job_build_flight_locked( + self, + flight: _JobBuildFlight, + reason: str, + *, + now: float | None = None, + ) -> bool: + return self._ensure_job_bundle_service()._cancel_flight_locked( + flight, + reason, + now=now, + ) - @staticmethod - def _job_build_is_publication_critical(request: object) -> bool: - return bool(getattr(request, "publication_critical", False)) - - def _record_priority_admission_locked( - self, - request: _JobBuildRequest, - result: str, - ) -> None: - """Observe first builder admission from publication-priority reservation.""" - - if not self._job_build_is_publication_critical(request): - return - self.job_build_priority_counts[result] += 1 - if result not in {"started", "coalesced"}: - return - if request.priority_admission_recorded: - return - request.priority_admission_recorded = True - elapsed = max(0.0, time.monotonic() - request.requested_monotonic) - self.job_build_priority_admission_seconds["sum"] += elapsed - self.job_build_priority_admission_seconds["count"] += 1 - - def _record_initial_prepared_work_locked(self, result: str) -> None: - self.initial_job_prepared_work_counts[result] += 1 - - def _new_job_build_cancellation(self) -> _JobBuildCancellation: - return _JobBuildCancellation( - timeout_seconds=max( - 0.001, - float( - getattr( - self, - "job_build_timeout_seconds", - DEFAULT_PRISM_JOB_BUILD_TIMEOUT_SECONDS, - ) - ), - ) - ) - - def _begin_job_build_priority_preparation( - self, - requested_monotonic: float | None = None, - ) -> tuple[int, float]: - """Reserve publication priority before immutable request construction.""" - - started = ( - time.monotonic() - if requested_monotonic is None - else requested_monotonic - ) - with self._job_build_scheduler_lock: - self._job_build_priority_preparation_sequence += 1 - token = self._job_build_priority_preparation_sequence - self._job_build_priority_preparations[token] = started - for routine_cancellation_ref in tuple( - self._job_build_routine_preparations.values() - ): - routine_cancellation = routine_cancellation_ref() - if routine_cancellation is not None: - routine_cancellation.cancel("publication priority") - self._job_build_routine_preparations.clear() - self._job_build_priority_changed.set() - return token, started - - def _finish_job_build_priority_preparation(self, token: int) -> None: - with self._job_build_scheduler_lock: - self._job_build_priority_preparations.pop(token, None) - self._job_build_priority_changed.set() - - def _begin_routine_job_build_preparation( - self, - *, - request_source: str, - cancelled: Callable[[], bool] | None, - ) -> tuple[int, _JobBuildCancellation]: - """Atomically admit cancellable routine request construction.""" - - deferred_recorded = False - while True: - self._job_build_priority_changed.clear() - with self._job_build_scheduler_lock: - if not self._publication_priority_scheduled_locked(): - self._job_build_routine_preparation_sequence += 1 - token = self._job_build_routine_preparation_sequence - preparation_cancellation = ( - self._new_job_build_cancellation() - ) - coordinator_ref = weakref.ref(self) - - def remove_dead_preparation( - dead_ref: weakref.ReferenceType[ - _JobBuildCancellation - ], - *, - preparation_token: int = token, - ) -> None: - coordinator = coordinator_ref() - if coordinator is None: - return - with coordinator._job_build_scheduler_lock: - if ( - coordinator._job_build_routine_preparations.get( - preparation_token - ) - is dead_ref - ): - coordinator._job_build_routine_preparations.pop( - preparation_token, - None, - ) - - self._job_build_routine_preparations[token] = weakref.ref( - preparation_cancellation, - remove_dead_preparation, - ) - return token, preparation_cancellation - if not deferred_recorded: - self.job_build_priority_counts["routine_deferred"] += 1 - if request_source == "initial": - self._record_initial_prepared_work_locked("deferred") - deferred_recorded = True - if cancelled is not None and cancelled(): - raise _JobBuildCancelled( - "job bundle request was cancelled behind publication priority" - ) - stop_event = getattr(self, "stop_event", None) - if stop_event is not None and stop_event.is_set(): - raise _JobBuildCancelled( - "coordinator stopped behind publication priority" - ) - self._job_build_priority_changed.wait(0.05) - - def _finish_routine_job_build_preparation(self, token: int) -> None: - with self._job_build_scheduler_lock: - self._job_build_routine_preparations.pop(token, None) - - def _publication_priority_scheduled_locked(self) -> bool: - if self._job_build_priority_preparations: - return True - pending = self._job_build_pending - if ( - pending is not None - and not pending.cancellation.is_set() - and self._job_build_is_publication_critical(pending) - ): - return True - return any( - flight is not None - and not flight.request.cancellation.is_set() - and self._job_build_is_publication_critical(flight.request) - for flight in ( - self._job_build_active, - self._job_build_retiring, - ) - ) - - def _job_build_can_inherit_publication_priority( - self, - existing: _JobBuildRequest, - incoming: _JobBuildRequest, - ) -> bool: - """Reject an almost-expired or stalled routine flight as a critical owner.""" - - if ( - not self._job_build_is_publication_critical(incoming) - or self._job_build_is_publication_critical(existing) - ): - return True - cancellation = existing.cancellation - total_budget = max( - 0.001, - cancellation.deadline_monotonic - cancellation.started_monotonic, - ) - remaining_budget = cancellation.deadline_monotonic - time.monotonic() - progress_budget = max( - 0.001, - float( - getattr( - self, - "job_build_cancel_grace_seconds", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - ), - ) - progress_age = ( - time.monotonic() - - float( - getattr( - cancellation, - "last_checkpoint_monotonic", - cancellation.started_monotonic, - ) - ) - ) - return ( - remaining_budget >= total_budget / 2.0 - and progress_age <= progress_budget - ) - - def _cancel_job_build_flight_locked( - self, - flight: _JobBuildFlight, - reason: str, - *, - now: float | None = None, - ) -> bool: - if not flight.request.cancellation.cancel(reason): - return False - flight.request.superseded_monotonic = ( - time.monotonic() if now is None else now - ) - self.job_build_scheduler_counts["supersessions"] += 1 - if reason == "publication priority": - self.job_build_priority_counts["routine_preempted"] += 1 - self._job_build_priority_changed.set() - return True - - def _promote_pending_job_build_locked(self) -> None: - pending = self._job_build_pending - if pending is None: - return - active = self._job_build_active - retiring = self._job_build_retiring - if ( - not self._job_build_is_publication_critical(pending) - and any( - flight is not None - and not flight.request.cancellation.is_set() - and self._job_build_is_publication_critical(flight.request) - for flight in (active, retiring) - ) - ): - # The pending request may predate an exact-flight priority upgrade. - # Preserve the same admission invariant as _request_job_build: - # routine work cannot displace scheduled publication work. - return - if active is not None: - if retiring is not None: - return - if self._ready_job_build_precedes_collection( - active.request, - pending, - ) and not self._job_build_is_publication_critical(pending): - return - if not self._collection_job_builds_are_independent( - active.request, - pending, - ): - reason = ( - "publication priority" - if self._job_build_is_publication_critical(pending) - and not self._job_build_is_publication_critical(active.request) - else "superseded" - ) - self._cancel_job_build_flight_locked(active, reason) - self._job_build_retiring = active - self._job_build_active = None - elif retiring is not None: - if self._ready_job_build_precedes_collection( - retiring.request, - pending, - ) and not self._job_build_is_publication_critical(pending): - return - if not self._collection_job_builds_are_independent( - retiring.request, - pending, - ): - reason = ( - "publication priority" - if self._job_build_is_publication_critical(pending) - and not self._job_build_is_publication_critical(retiring.request) - else "superseded" - ) - self._cancel_job_build_flight_locked(retiring, reason) - self._job_build_pending = None - flight = self._start_job_build_locked(pending) - self._job_build_active = flight - self._arm_job_build_locked(flight) + def _promote_pending_job_build_locked(self) -> None: + self._ensure_job_bundle_service()._promote_pending_locked() def _job_build_done( self, flight: _JobBuildFlight, future: Future[CachedJobBundle], ) -> None: - request = flight.request - result: CachedJobBundle | None = None - error: BaseException | None = None - try: - result = future.result() - if request.cancellation.is_set(): - if request.cancellation.reason == "timeout": - error = JobBuildCancelled( - "job build completed after its timeout" - ) - else: - error = JobBuildSuperseded( - "obsolete job build completed after cancellation" - ) - except BaseException as exc: # noqa: BLE001 - delivered to all waiters - error = exc - with self._job_build_scheduler_lock: - self.job_build_scheduler_counts["completions"] += 1 - self.shared_bundle_preparation_count += 1 - self.shared_bundle_preparation_seconds_sum += max( - 0.0, - time.monotonic() - request.cancellation.started_monotonic, - ) - if request.cancellation.cancelled_monotonic is not None: - elapsed = max( - 0.0, - time.monotonic() - request.cancellation.cancelled_monotonic, - ) - self.job_build_cancellation_seconds["sum"] += elapsed - self.job_build_cancellation_seconds["count"] += 1 - coordination_cancelled = isinstance(error, JobBuildSuperseded) or ( - request.cancellation.is_set() - and request.cancellation.reason != "timeout" - ) - if error is not None and coordination_cancelled: - self.job_build_scheduler_counts["obsolete_results"] += 1 - self.shared_bundle_build_counts["superseded"] += 1 - with self._tip_refresh_metrics_lock: - self.tip_refresh_superseded_results += 1 - elif error is not None: - self.shared_bundle_build_counts["failed"] += 1 - else: - self.shared_bundle_build_counts["completed"] += 1 - if self._job_build_active is flight: - self._job_build_active = None - if self._job_build_retiring is flight: - self._job_build_retiring = None - self._promote_pending_job_build_locked() - if not request.promise.done(): - if error is not None: - request.promise.set_exception(error) - else: - assert result is not None - request.promise.set_result(result) - self._job_build_priority_changed.set() - - def _request_job_build(self, request: _JobBuildRequest) -> Future[CachedJobBundle]: - self._ensure_job_cache_state() - with self._job_build_scheduler_lock: - if getattr(request, "idle_retarget", False): - with self.lock: - defer_idle = self._vardiff_idle_tip_divergence_locked() - if defer_idle: - request.cancellation.cancel( - "idle retarget deferred during unpublished tip refresh" - ) - if not request.promise.done(): - request.promise.set_exception( - JobBuildSuperseded( - "idle retarget deferred during unpublished tip refresh" - ) - ) - return request.promise - self.job_build_scheduler_counts["requests"] += 1 - active = self._job_build_active - retiring = self._job_build_retiring - pending = self._job_build_pending - publication_critical = self._job_build_is_publication_critical( - request - ) - if ( - active is not None - and not active.request.cancellation.is_set() - and self._job_build_requests_can_share(active.request, request) - and self._job_build_can_inherit_publication_priority( - active.request, - request, - ) - ): - if publication_critical: - active.request.publication_critical = True - active.request.request_source = request.request_source - active.request.requested_monotonic = request.requested_monotonic - active.request.priority_admission_recorded = True - self._record_priority_admission_locked(request, "coalesced") - if request.request_source == "initial": - self._record_initial_prepared_work_locked("singleflight") - return active.request.promise - if ( - retiring is not None - and not retiring.request.cancellation.is_set() - and self._job_build_requests_can_share(retiring.request, request) - and self._job_build_can_inherit_publication_priority( - retiring.request, - request, - ) - ): - if publication_critical: - retiring.request.publication_critical = True - retiring.request.request_source = request.request_source - retiring.request.requested_monotonic = request.requested_monotonic - retiring.request.priority_admission_recorded = True - self._record_priority_admission_locked(request, "coalesced") - if request.request_source == "initial": - self._record_initial_prepared_work_locked("singleflight") - return retiring.request.promise - if ( - pending is not None - and not pending.cancellation.is_set() - and self._job_build_requests_can_share(pending, request) - and self._job_build_can_inherit_publication_priority( - pending, - request, - ) - ): - if publication_critical: - pending.publication_critical = True - pending.request_source = request.request_source - pending.requested_monotonic = request.requested_monotonic - self._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, retiring): - if ( - occupied is not None - and not self._job_build_requests_can_share( - occupied.request, - pending, - ) - and not self._job_build_is_publication_critical( - occupied.request - ) - ): - self._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - self._promote_pending_job_build_locked() - if request.request_source == "initial": - self._record_initial_prepared_work_locked("singleflight") - return pending.promise - - if not publication_critical: - priority_blockers = tuple( - blocker - for blocker in ( - active.request if active is not None else None, - retiring.request if retiring is not None else None, - pending, - ) - if blocker is not None - and not blocker.cancellation.is_set() - and self._job_build_is_publication_critical(blocker) - ) - if priority_blockers: - self.job_build_priority_counts["routine_deferred"] += 1 - if request.request_source == "initial": - self._record_initial_prepared_work_locked("deferred") - return self._defer_job_build_locked( - *(blocker.promise for blocker in priority_blockers) - ) - - if request.mode == "collection": - possible_blockers = ( - pending, - active.request if active is not None else None, - retiring.request if retiring is not None else None, - ) - for blocker in possible_blockers: - if ( - blocker is not None - and not blocker.cancellation.is_set() - and self._ready_job_build_precedes_collection( - blocker, - request, - ) - and not ( - publication_critical - and not self._job_build_is_publication_critical( - blocker - ) - ) - ): - return self._defer_job_build_locked(blocker.promise) - if active is None: - if pending is not None: - if self._collection_job_builds_are_independent( - pending, - request, - ): - self._job_build_pending = None - flight = self._start_job_build_locked(pending) - if retiring is None: - replacement = self._start_job_build_locked(request) - self._job_build_retiring = flight - self._job_build_active = replacement - self._arm_job_build_locked(flight) - self._arm_job_build_locked(replacement) - return request.promise - self._job_build_active = flight - self._arm_job_build_locked(flight) - return self._defer_job_build_locked( - flight.request.promise, - retiring.request.promise, - ) - pending.cancellation.cancel("superseded while pending") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded("pending job build was superseded") - ) - self._job_build_pending = None - self.job_build_scheduler_counts["supersessions"] += 1 - if ( - retiring is not None - and not self._collection_job_builds_are_independent( - retiring.request, - request, - ) - ): - now = time.monotonic() - reason = ( - "publication priority" - if publication_critical - and not self._job_build_is_publication_critical( - retiring.request - ) - else "superseded" - ) - if self._cancel_job_build_flight_locked( - retiring, - reason, - now=now, - ): - request.superseded_monotonic = now - flight = self._start_job_build_locked(request) - self._job_build_active = flight - self._arm_job_build_locked(flight) - return request.promise - - if self._collection_job_builds_are_independent( - active.request, - request, - ): - if self._job_build_retiring is None: - self._job_build_retiring = active - flight = self._start_job_build_locked(request) - self._job_build_active = flight - self._arm_job_build_locked(flight) - return request.promise - if pending is None: - self._job_build_pending = request - if publication_critical: - self._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, self._job_build_retiring): - if ( - occupied is not None - and not self._job_build_is_publication_critical( - occupied.request - ) - ): - self._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - return request.promise - assert retiring is not None - if publication_critical: - pending.cancellation.cancel("superseded while pending") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded( - "pending job build was superseded by publication priority" - ) - ) - self.job_build_scheduler_counts["supersessions"] += 1 - self._job_build_pending = request - self._record_priority_admission_locked(request, "queued") - now = time.monotonic() - for occupied in (active, retiring): - if not self._job_build_is_publication_critical( - occupied.request - ): - self._cancel_job_build_flight_locked( - occupied, - "publication priority", - now=now, - ) - return request.promise - return self._defer_job_build_locked( - active.request.promise, - retiring.request.promise, - ) + self._ensure_job_bundle_service()._build_done(flight, future) - now = time.monotonic() - for obsolete in (active, retiring): - if obsolete is not None: - reason = ( - "publication priority" - if publication_critical - and not self._job_build_is_publication_critical( - obsolete.request - ) - else "superseded" - ) - self._cancel_job_build_flight_locked( - obsolete, - reason, - now=now, - ) - request.superseded_monotonic = now - if retiring is None: - self._job_build_retiring = active - flight = self._start_job_build_locked(request) - self._job_build_active = flight - self._arm_job_build_locked(flight) - return request.promise - - previous_pending = self._job_build_pending - if previous_pending is not None: - previous_pending.cancellation.cancel("superseded while pending") - if not previous_pending.promise.done(): - previous_pending.promise.set_exception( - JobBuildSuperseded("pending job build was superseded") - ) - self.job_build_scheduler_counts["supersessions"] += 1 - self._job_build_pending = request - if publication_critical: - self._record_priority_admission_locked(request, "queued") - return request.promise + def _request_job_build( + self, + request: _JobBuildRequest, + ) -> Future[CachedJobBundle]: + return self._ensure_job_bundle_service().request_build(request) def _cancel_obsolete_job_builds( self, @@ -4540,105 +3186,20 @@ def _cancel_obsolete_job_builds( *, keep_published_snapshot: bool = False, ) -> None: - self._ensure_job_cache_state() - published_parent: str | None = None - published_fingerprint: str | None = None - if keep_published_snapshot: - # A per-client build for exactly the published snapshot is still - # valid, creditable work when a same-tip template bump sweeps old - # fingerprints. Detection-time sweeps deliberately do not use this: - # once a replacement tip is detected, even published-tip builds - # must vacate the single-flight lane for the replacement's work. - # Snapshot the identity outside the scheduler lock so the keep - # check stays lock-free below. - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - snapshot = getattr(self, "tip_template_snapshot", None) - snapshot_tip = getattr(snapshot, "bestblockhash", None) - snapshot_fingerprint = getattr( - snapshot, "template_fingerprint", None - ) - if ( - published is not None - and snapshot_tip == published[0] - and snapshot_fingerprint is not None - and self._published_tip_authoritative_locked(time.monotonic()) - ): - published_parent = published[0] - published_fingerprint = snapshot_fingerprint - - def keep(request: _JobBuildRequest) -> bool: - return bool( - published_fingerprint is not None - and request.artifacts.previousblockhash == published_parent - and request.artifacts.fingerprint == published_fingerprint - ) - - with self._job_build_scheduler_lock: - for flight in (self._job_build_active, self._job_build_retiring): - if ( - flight is not None - and not keep(flight.request) - and flight.request.cancellation.cancel(reason) - ): - flight.request.superseded_monotonic = time.monotonic() - self.job_build_scheduler_counts["supersessions"] += 1 - pending = self._job_build_pending - if pending is not None and not keep(pending): - pending.cancellation.cancel(reason) - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded(f"pending job build {reason}") - ) - self._job_build_pending = None - self.job_build_scheduler_counts["supersessions"] += 1 + self._ensure_job_bundle_service().cancel_obsolete_builds( + reason, + keep_published_snapshot=keep_published_snapshot, + ) def shutdown_job_build_executor(self) -> None: - self._ensure_job_cache_state() - with self._job_build_scheduler_lock: - for flight in (self._job_build_active, self._job_build_retiring): - if flight is not None: - flight.request.cancellation.cancel("shutdown") - pending = self._job_build_pending - if pending is not None: - pending.cancellation.cancel("shutdown") - if not pending.promise.done(): - pending.promise.set_exception( - JobBuildSuperseded("pending job build cancelled by shutdown") - ) - self._job_build_pending = None - executor = self._job_build_executor - self._job_build_executor = None - self._job_build_executor_shutdown = True - if executor is not None: - executor.shutdown(wait=True, cancel_futures=True) + self._ensure_job_bundle_service().shutdown() def _job_bundle_payout_state_current(self, bundle: CachedJobBundle) -> bool: - self._ensure_job_cache_state() - with self._job_cache_lock: - artifact = self._published_payout_state.artifact - return bool( - bundle.payout_state_generation == self._payout_state_generation - and bundle.build_key is not None - and artifact is not None - and bundle.build_key.payout_artifact_sha256 - == artifact.prior_balances_sha256 - ) + return self._ensure_job_bundle_service().bundle_payout_state_current(bundle) @contextmanager def _payout_balance_mutation(self) -> Iterator[None]: - """Serialize durable balance changes without excluding delivery.""" - self._ensure_job_cache_state() - with self._payout_balance_mutation_lock: - with self._accepted_block_payout_preview_condition: - landed_transition = any( - transition.landed - for transition in self._accepted_block_payout_previews.values() - ) - if landed_transition: - raise TemplateRefreshBlocked( - "accepted block payout confirmation is still pending" - ) + with self._ensure_payout_state_service().balance_mutation(): yield def _begin_accepted_block_payout_preview( @@ -4647,27 +3208,9 @@ def _begin_accepted_block_payout_preview( *, block_height: int | None = None, ) -> None: - """Prevent child work from snapshotting pre-accept balances.""" - self._ensure_job_cache_state() - key = block_hash.lower() - with self._accepted_block_payout_preview_condition: - self._invalidated_accepted_block_payout_previews.pop(key, None) - existing = self._accepted_block_payout_previews.get(key) - if existing is None: - self._accepted_block_payout_previews[key] = ( - _AcceptedBlockPayoutTransition(block_height=block_height) - ) - elif ( - block_height is not None - and existing.block_height is not None - and existing.block_height != block_height - ): - raise RuntimeError("accepted block payout transition height changed") - elif existing.block_height is None and block_height is not None: - self._accepted_block_payout_previews[key] = dataclass_replace( - existing, - block_height=block_height, - ) + self._ensure_payout_state_service().begin_accepted_block_preview( + block_hash, block_height=block_height + ) def _mark_accepted_block_payout_landed( self, @@ -4675,119 +3218,18 @@ def _mark_accepted_block_payout_landed( *, block_height: int, ) -> None: - """Bar reconciliation after submitblock makes a candidate active.""" - self._ensure_job_cache_state() - key = block_hash.lower() - with self._accepted_block_payout_preview_condition: - existing = self._accepted_block_payout_previews.get( - key, - _AcceptedBlockPayoutTransition(block_height=block_height), - ) - if existing.block_height not in {None, block_height}: - raise RuntimeError("accepted block payout transition height changed") - self._accepted_block_payout_previews[key] = dataclass_replace( - existing, - block_height=block_height, - landed=True, - ) - self._accepted_block_payout_preview_condition.notify_all() + self._ensure_payout_state_service().mark_accepted_block_landed( + block_hash, block_height=block_height + ) def _publish_accepted_block_payout_preview( self, block_hash: str, balances: list[dict[str, object]], ) -> list[dict[str, object]]: - """Publish the balances child work must observe after confirmation. - - Canonicalization happens before the delivery boundary. The gate only - installs an immutable pointer and advances the generation, so no job - bound to the pre-accept balances can land after publication. - """ - normalized = self.normalized_prior_balances(balances) - serialized = self._serialize_prior_balance_preview(normalized) - key = block_hash.lower() - with self._payout_balance_mutation_lock: - with self._accepted_block_payout_preview_condition: - existing = self._accepted_block_payout_previews.get(key) - existing_preview = existing.preview if existing is not None else None - if existing_preview is not None: - if existing_preview != serialized: - raise RuntimeError( - "accepted block payout preview changed during retry" - ) - if ( - existing is not None - and existing.published_generation is not None - ): - return self._materialize_prior_balance_preview( - existing_preview - ) - # A bounded publication loss retains the compact preview - # locally while delivery remains fenced. Matching retries - # must still cross the atomic publication boundary so they - # install a generation and reopen admission. - - captured = self._capture_payout_state_source() - reserved = self._reserve_payout_state_source_if_current( - captured[1], - "accepted_block_preview", - tip_hash=key, - invalidated_monotonic=time.monotonic(), - ) - if reserved is None: - candidate = self._current_payout_state_candidate() - else: - candidate = self._prepared_payout_state_candidate(reserved) - candidate = self._accepted_block_preview_candidate( - candidate, - block_hash=key, - preview=serialized, - ) - # Close delivery admission before exposing the preview. The - # generation publication below performs the only atomic pointer - # swap and reopens admission; no preparation lock is involved. - self._block_payout_state_publication(force=True) - published = self._publish_payout_state_candidate(candidate) - if published is None: - max_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for _attempt in range(max_retries): - candidate = self._accepted_block_preview_candidate( - self._current_payout_state_candidate(), - block_hash=key, - preview=serialized, - ) - published = self._publish_payout_state_candidate(candidate) - if published is not None: - break - if published is None: - self._block_payout_state_publication(force=True) - # Admission is now globally fenced, so retaining the compact - # preview locally is safe even though it did not win an atomic - # generation publication. Finalization may finish; the normal - # retry loop will publish the newest source before delivery - # resumes. - with self._accepted_block_payout_preview_condition: - transition = self._accepted_block_payout_previews.get( - key, - _AcceptedBlockPayoutTransition(landed=True), - ) - self._accepted_block_payout_previews[key] = dataclass_replace( - transition, - landed=True, - preview=serialized, - published_generation=None, - ) - self._accepted_block_payout_preview_condition.notify_all() - return self._materialize_prior_balance_preview(serialized) + return self._ensure_payout_state_service().publish_accepted_block_preview( + block_hash, balances + ) def _accepted_block_preview_candidate( self, @@ -4796,38 +3238,15 @@ def _accepted_block_preview_candidate( block_hash: str, preview: tuple[tuple[str, str, str, int], ...], ) -> PayoutStateCandidate: - """Bind a compact preview to its prepared artifact before gating.""" - ledger_artifact = candidate.ledger_artifact - if ledger_artifact is not None: - # The artifact was prepared before accepted-block persistence, so - # replace only its balance view with the verified prospective - # state. This allocation must stay outside delivery publication. - ledger_artifact = dataclass_replace( - ledger_artifact, - prior_balances=tuple( - self._materialize_prior_balance_preview(preview) - ), - ) - return dataclass_replace( - candidate, - accepted_block_hash=block_hash, - accepted_block_preview=preview, - ledger_artifact=ledger_artifact, + return self._ensure_payout_state_service().accepted_block_preview_candidate( + candidate, block_hash=block_hash, preview=preview ) - @staticmethod def _serialize_prior_balance_preview( + self, balances: list[dict[str, object]], ) -> tuple[tuple[str, str, str, int], ...]: - return tuple( - ( - str(balance["recipient_id"]), - str(balance["order_key"]), - str(balance["p2mr_program_hex"]), - int(balance["balance_sats"]), - ) - for balance in balances - ) + return self._ensure_payout_state_service().serialize_prior_balance_preview(balances) def _accepted_block_payout_preview_from_bundle( self, @@ -4835,62 +3254,15 @@ def _accepted_block_payout_preview_from_bundle( *, prior_balances: list[dict[str, object]] | None = None, ) -> list[dict[str, object]]: - """Derive the confirmed carry-forward view from a verified bundle.""" - manifest = final_bundle.get("payout_policy_manifest") - if not isinstance(manifest, dict) or not isinstance(manifest.get("accounts"), list): - raise RuntimeError("accepted block payout manifest is missing accounts") - prior_identities: dict[str, tuple[str, str]] = {} - for balance in prior_balances or []: - program = str(balance.get("p2mr_program_hex", "")).lower() - identity = ( - str(balance.get("order_key", "")), - str(balance.get("recipient_id", "")), - ) - prior_identities[program] = min( - identity, - prior_identities.get(program, identity), - ) - balances: list[dict[str, object]] = [] - for account in manifest["accounts"]: - if not isinstance(account, dict): - continue - if str(account.get("account_type", "miner")) == "pool_fee": - continue - balance_sats = int(account.get("carry_forward_balance_sats", 0)) - if balance_sats == 0: - continue - program = str(account.get("p2mr_program_hex", "")).lower() - account_identity = ( - str(account.get("order_key", "")), - str(account.get("recipient_id", "")), - ) - order_key, recipient_id = min( - account_identity, - prior_identities.get(program, account_identity), - ) - balances.append( - { - "recipient_id": recipient_id, - "order_key": order_key, - "p2mr_program_hex": program, - "balance_sats": balance_sats, - } - ) - return self.normalized_prior_balances(balances) + return self._ensure_payout_state_service().accepted_block_preview_from_bundle( + final_bundle, prior_balances=prior_balances + ) - @staticmethod def _materialize_prior_balance_preview( + self, preview: tuple[tuple[str, str, str, int], ...], ) -> list[dict[str, object]]: - return [ - { - "recipient_id": recipient_id, - "order_key": order_key, - "p2mr_program_hex": p2mr_program_hex, - "balance_sats": balance_sats, - } - for recipient_id, order_key, p2mr_program_hex, balance_sats in preview - ] + return self._ensure_payout_state_service().materialize_prior_balance_preview(preview) def _clear_accepted_block_payout_preview( self, @@ -4898,93 +3270,12 @@ def _clear_accepted_block_payout_preview( *, invalidate_published: bool = False, ) -> None: - self._ensure_job_cache_state() - key = block_hash.lower() - with self._payout_balance_mutation_lock: - with self._accepted_block_payout_preview_condition: - existing = self._accepted_block_payout_previews.get(key) - if existing is None: - if not invalidate_published: - self._invalidated_accepted_block_payout_previews.pop( - key, - None, - ) - self._accepted_block_payout_preview_condition.notify_all() - return - if not invalidate_published: - # Durable state now equals the published prospective view; - # removing the override changes no logical payout state. - self._accepted_block_payout_previews.pop(key, None) - self._invalidated_accepted_block_payout_previews.pop(key, None) - self._accepted_block_payout_preview_condition.notify_all() - return - if existing.preview is None: - # Nothing crossed a generation boundary. A landed - # transition still becomes a tombstone so descendants - # cannot fall through to an uncertain database snapshot. - self._accepted_block_payout_previews.pop(key, None) - if existing.landed: - self._invalidated_accepted_block_payout_previews[key] = ( - existing.block_height - ) - self._accepted_block_payout_preview_condition.notify_all() - return - - captured = self._capture_payout_state_source() - reserved = self._reserve_payout_state_source_if_current( - captured[1], - "accepted_block_preview_withdrawn", - tip_hash=captured[2], - invalidated_monotonic=time.monotonic(), - ) - candidate = self._prepared_payout_state_candidate( - reserved if reserved is not None else self._capture_payout_state_source() - ) - candidate = dataclass_replace( - candidate, - accepted_block_hash=key, - accepted_block_withdrawal=True, - accepted_block_height=existing.block_height, - ) - self._block_payout_state_publication(force=True) - published = self._publish_payout_state_candidate(candidate) - if published is None: - max_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for _attempt in range(max_retries): - candidate = dataclass_replace( - self._current_payout_state_candidate(), - accepted_block_hash=key, - accepted_block_withdrawal=True, - accepted_block_height=existing.block_height, - ) - published = self._publish_payout_state_candidate(candidate) - if published is not None: - break - if published is None: - # Delivery remains fenced. Install the tombstone immediately - # so even local builders cannot fall through while the newest - # source waits for a retry publication. - with self._accepted_block_payout_preview_condition: - self._accepted_block_payout_previews.pop(key, None) - self._invalidated_accepted_block_payout_previews[key] = ( - existing.block_height - ) - self._accepted_block_payout_preview_condition.notify_all() + self._ensure_payout_state_service().clear_accepted_block_preview( + block_hash, invalidate_published=invalidate_published + ) def _accepted_block_payout_transition_landed(self, block_hash: str) -> bool: - self._ensure_job_cache_state() - with self._accepted_block_payout_preview_condition: - transition = self._accepted_block_payout_previews.get(block_hash.lower()) - return transition is not None and transition.landed + return self._ensure_payout_state_service().accepted_block_transition_landed(block_hash) def _accepted_block_payout_transition_for_parent( self, @@ -4992,102 +3283,9 @@ def _accepted_block_payout_transition_for_parent( *, parent_height: int | None = None, ) -> tuple[str, bool] | None: - """Select the highest active exact/ancestor payout transition. - - The boolean result is true for an invalidated landed transition. The - caller decides whether to wait for a live preview or fail closed on its - tombstone. - """ - self._ensure_job_cache_state() - key = parent_hash.lower() - with self._accepted_block_payout_preview_condition: - exact_transition = self._accepted_block_payout_previews.get(key) - exact_invalidated = ( - key in self._invalidated_accepted_block_payout_previews - ) - fail_closed_candidate_hashes = { - candidate_hash - for candidate_hash, transition in ( - self._accepted_block_payout_previews.items() - ) - if transition.landed - } - fail_closed_candidate_hashes.update( - self._invalidated_accepted_block_payout_previews - ) - ancestor_candidates = [ - (candidate_hash, transition.block_height, False) - for candidate_hash, transition in self._accepted_block_payout_previews.items() - if exact_transition is None - and not exact_invalidated - and transition.block_height is not None - and parent_height is not None - and transition.block_height <= parent_height - ] - ancestor_candidates.extend( - ( - candidate_hash, - candidate_height, - True, - ) - for candidate_hash, candidate_height in ( - self._invalidated_accepted_block_payout_previews.items() - ) - if exact_transition is None - and not exact_invalidated - and candidate_height is not None - and parent_height is not None - and candidate_height <= parent_height - ) - if exact_transition is not None or exact_invalidated: - return key, exact_invalidated - if not ancestor_candidates: - return None - - active_ancestors: list[tuple[int, str, bool]] = [] - try: - for ( - candidate_hash, - candidate_height, - candidate_invalidated, - ) in ancestor_candidates: - assert candidate_height is not None - active_hash = str( - self.rpc.call("getblockhash", [candidate_height]) - ).lower() - if active_hash == candidate_hash: - active_ancestors.append( - ( - candidate_height, - candidate_hash, - candidate_invalidated, - ) - ) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "could not validate an accepted payout preview on the active chain" - ) from exc - if not active_ancestors: - if any( - candidate_hash in fail_closed_candidate_hashes - for candidate_hash, _candidate_height, _invalidated in ( - ancestor_candidates - ) - ): - # A prepared artifact published with an accepted preview carries - # that prospective balance view. If its block left the active - # chain between reconciliation and this build, falling through - # to the artifact would stamp unrelated work with those balances. - # Wait for withdrawal/reconciliation to publish a new payout - # generation instead of using either the artifact or live DB. - self._schedule_tip_refresh_retry() - raise _PayoutStatePublicationBlocked( - "accepted payout transition is no longer active" - ) - return None - _, selected_key, selected_invalidated = max(active_ancestors) - return selected_key, selected_invalidated + return self._ensure_payout_state_service().accepted_block_transition_for_parent( + parent_hash, parent_height=parent_height + ) def _await_pending_parent_payout_preview( self, @@ -5095,73 +3293,9 @@ def _await_pending_parent_payout_preview( *, parent_height: int | None = None, ) -> list[dict[str, object]] | None: - """Wait out a pending accepted-parent transition, returning its preview. - - Returns None when no landed transition governs this parent, without - touching any confirmed payout input. Ordering this wait before every - confirmed read lets children of a pending parent block here instead of - taking (or caching) balances that omit their new parent. - """ - selected = self._accepted_block_payout_transition_for_parent( - parent_hash, - parent_height=parent_height, + return self._ensure_payout_state_service().await_pending_parent_preview( + parent_hash, parent_height=parent_height ) - if selected is None: - return None - selected_key, selected_invalidated = selected - if selected_invalidated: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "accepted parent payout preview was withdrawn" - ) - - wait_seconds = max( - 0.0, - float( - getattr( - self, - "accepted_block_payout_preview_wait_seconds", - DEFAULT_ACCEPTED_BLOCK_PAYOUT_PREVIEW_WAIT_SECONDS, - ) - ), - ) - deadline = time.monotonic() + wait_seconds - timed_out = False - invalidated = False - with self._accepted_block_payout_preview_condition: - while selected_key in self._accepted_block_payout_previews: - transition = self._accepted_block_payout_previews[selected_key] - if transition.preview is not None: - return self._materialize_prior_balance_preview( - transition.preview - ) - if self.stop_event.is_set(): - raise RuntimeError( - "coordinator stopped while accepted payout preview was pending" - ) - remaining = deadline - time.monotonic() - if remaining <= 0: - timed_out = True - break - self._accepted_block_payout_preview_condition.wait( - timeout=min(0.25, remaining) - ) - invalidated = ( - selected_key in self._invalidated_accepted_block_payout_previews - ) - if invalidated: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "accepted parent payout preview was withdrawn" - ) - if timed_out: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "accepted parent payout preview is not ready yet" - ) - # The transition reached a terminal durable state while waiting; the - # caller's confirmed fallback now includes the parent. - return None def _prior_balances_for_job_parent( self, @@ -5170,17 +3304,8 @@ def _prior_balances_for_job_parent( parent_height: int | None = None, fallback_balances: Sequence[dict[str, object]] | None = None, ) -> list[dict[str, object]]: - """Return prospective balances, otherwise a prepared/ledger fallback.""" - preview = self._await_pending_parent_payout_preview( - parent_hash, - parent_height=parent_height, - ) - if preview is not None: - return preview - return ( - list(fallback_balances) - if fallback_balances is not None - else self.ledger.current_prior_balances() + return self._ensure_payout_state_service().prior_balances_for_parent( + parent_hash, parent_height=parent_height, fallback_balances=fallback_balances ) def _observe_payout_state_seconds( @@ -5190,21 +3315,9 @@ def _observe_payout_state_seconds( *, relation: str | None = None, ) -> None: - self._ensure_job_cache_state() - with self._payout_state_metrics_lock: - if name == "gate_wait": - if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: - raise ValueError(f"unknown payout delivery generation: {relation}") - histogram = self.payout_gate_wait_histograms[str(relation)] - else: - histogram = self.payout_state_histograms[name] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed_seconds - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + self._ensure_payout_state_service().observe_seconds( + name, elapsed_seconds, relation=relation + ) def _observe_payout_gate_admission( self, @@ -5213,22 +3326,8 @@ def _observe_payout_gate_admission( generation: int, fallback_wait_seconds: float, ) -> None: - self._ensure_job_cache_state() - with self._job_cache_lock: - published_generation = self._payout_state_generation - relation = getattr(admission, "relation", None) - if relation not in PRISM_PAYOUT_DELIVERY_GENERATIONS: - relation = _PayoutStateDeliveryGate._generation_relation( - generation, - published_generation, - ) - wait_seconds = float( - getattr(admission, "wait_seconds", fallback_wait_seconds) - ) - self._observe_payout_state_seconds( - "gate_wait", - max(0.0, wait_seconds), - relation=relation, + self._ensure_payout_state_service().observe_gate_admission( + admission, generation=generation, fallback_wait_seconds=fallback_wait_seconds ) def _reserve_payout_state_source( @@ -5238,21 +3337,9 @@ def _reserve_payout_state_source( tip_hash: str | None = None, invalidated_monotonic: float | None = None, ) -> int: - self._ensure_job_cache_state() - invalidated = ( - time.monotonic() - if invalidated_monotonic is None - else invalidated_monotonic + return self._ensure_payout_state_service().reserve_source( + cause, tip_hash=tip_hash, invalidated_monotonic=invalidated_monotonic ) - with self.lock: - generation = self._payout_state_source[0] + 1 - self._payout_state_source = ( - generation, - tip_hash, - cause, - invalidated, - ) - return generation def _reserve_payout_state_source_if_current( self, @@ -5262,89 +3349,26 @@ def _reserve_payout_state_source_if_current( tip_hash: str | None = None, invalidated_monotonic: float | None = None, ) -> tuple[int, int, str | None, str, float] | None: - """Reserve and capture a source only if preparation was not superseded.""" - - self._ensure_job_cache_state() - invalidated = ( - time.monotonic() - if invalidated_monotonic is None - else invalidated_monotonic - ) - # Match publication's lock order so the returned base generation and - # newly reserved source form one atomic candidate identity. - with self._job_cache_lock: - with self.lock: - if self._payout_state_source[0] != expected_source_generation: - return None - source_generation = expected_source_generation + 1 - self._payout_state_source = ( - source_generation, - tip_hash, - cause, - invalidated, - ) - return ( - self._payout_state_generation, - source_generation, - tip_hash, - cause, - invalidated, - ) + return self._ensure_payout_state_service().reserve_source_if_current( + expected_source_generation, cause, tip_hash=tip_hash, invalidated_monotonic=invalidated_monotonic + ) def _capture_payout_state_source( self, ) -> tuple[int, int, str | None, str, float]: - self._ensure_job_cache_state() - with self.lock: - source_generation, source_tip, cause, invalidated = ( - self._payout_state_source - ) - with self._job_cache_lock: - base_generation = self._payout_state_generation - return ( - base_generation, - source_generation, - source_tip, - cause, - invalidated, - ) + return self._ensure_payout_state_service().capture_source() def _prepared_payout_state_candidate( self, captured: tuple[int, int, str | None, str, float], ) -> PayoutStateCandidate: - base_generation, source_generation, source_tip, cause, invalidated = captured - ledger_artifact: PayoutLedgerArtifact | None = None - with self._job_cache_lock: - template_artifacts = self._template_artifacts - if ( - template_artifacts is not None - and getattr(self, "_pool_ready_latched", False) - ): - ledger_artifact = self._build_payout_ledger_artifact( - base_generation, - base_generation + 1, - template_artifacts.network_difficulty, - ) - return PayoutStateCandidate( - base_generation=base_generation, - source_generation=source_generation, - source_tip_hash=source_tip, - cause=cause, - invalidated_monotonic=invalidated, - prepared_monotonic=time.monotonic(), - ledger_artifact=ledger_artifact, - ) + return self._ensure_payout_state_service().prepared_candidate(captured) def _current_payout_state_candidate(self) -> PayoutStateCandidate: - return self._prepared_payout_state_candidate( - self._capture_payout_state_source() - ) + return self._ensure_payout_state_service().current_candidate() def _record_discarded_payout_candidate(self) -> None: - self._ensure_job_cache_state() - with self._payout_state_metrics_lock: - self.payout_state_candidates_discarded += 1 + self._ensure_payout_state_service()._record_discarded_candidate() def _block_payout_state_publication( self, @@ -5352,396 +3376,55 @@ def _block_payout_state_publication( force: bool = False, supersede_with: tuple[int, str | None, str, float] | None = None, ) -> None: - """Atomically close delivery, optionally reserving a newer source.""" - - self._ensure_job_cache_state() - - pending_source: int | None = None - - def mark_blocked() -> bool: - nonlocal pending_source - with self._job_cache_lock: - with self.lock: - if supersede_with is not None: - ( - expected_source, - fallback_tip, - cause, - invalidated, - ) = supersede_with - current_source, current_tip, _, _ = ( - self._payout_state_source - ) - # A newer tip/source wins its identity, but it must be - # superseded so no candidate prepared before an - # uncertain durable commit can publish afterward. - source_tip = ( - fallback_tip - if current_source == expected_source - else current_tip - ) - pending_source = current_source + 1 - self._payout_state_source = ( - pending_source, - source_tip, - cause, - invalidated, - ) - else: - pending_source = self._payout_state_source[0] - if ( - not force - and supersede_with is None - and pending_source - == self._published_payout_state.source_generation - ): - return False - self._payout_state_publication_blocked = True - self._job_bundle_cache.clear() - return True - - # Close fleet admission atomically with the cache fence. Escaped - # immutable bundles remain stamped with the old generation, but cannot - # cross the boundary while the ledger has newer unpublished state. - if not self._payout_state_delivery_gate.block_delivery(mark_blocked): - return - # Construction cancellation is independent of fanout activation. An - # obsolete builder may still be in its ledger or subprocess phase and - # must be preempted before a replacement request is submitted. - self._cancel_obsolete_job_builds("payout generation superseded") - with self._job_cache_lock: - next_payout_generation = self._payout_state_generation + 1 - with self.lock: - payout_invalidated_monotonic = self._payout_state_source[3] - self._record_progress_payout_generation( - next_payout_generation, - payout_invalidated_monotonic, + self._ensure_payout_state_service().block_publication( + force=force, supersede_with=supersede_with ) - with self.lock: - active = getattr(self, "_active_tip_refresh", None) - if ( - active is not None - and active[0].payout_state_generation < next_payout_generation - ): - active[1].cancel() - elif active is not None: - return - assert pending_source is not None - self._mark_tip_refresh_pending(next_payout_generation) - self._schedule_tip_refresh_retry() def _payout_state_publication_fenced(self) -> bool: - """Report whether delivery is still blocked awaiting a publication.""" - - self._ensure_job_cache_state() - with self._job_cache_lock: - return self._payout_state_publication_blocked + return self._ensure_payout_state_service().publication_fenced() def _payout_source_requires_publication( self, candidate: PayoutStateCandidate | None = None, ) -> bool: - """Report whether an invalidation source still lacks a publication.""" - - self._ensure_job_cache_state() - with self._job_cache_lock: - published_source = self._published_payout_state.source_generation - if candidate is not None: - return candidate.source_generation != published_source - with self.lock: - return self._payout_state_source[0] != published_source + return self._ensure_payout_state_service().source_requires_publication(candidate) def _publish_payout_state_candidate( self, candidate: PayoutStateCandidate, ) -> int | None: - """Publish a prepared candidate, or reject it if its source moved.""" - - self._ensure_job_cache_state() - published_generation: int | None = None - schedule_retry = False - active_to_cancel: _FanoutCancellation | None = None - publish_started = 0.0 - with self._job_cache_lock: - with self.lock: - if ( - candidate.source_generation != self._payout_state_source[0] - or candidate.base_generation != self._payout_state_generation - ): - self._record_discarded_payout_candidate() - return None - try: - if ( - candidate.accepted_block_preview is not None - and not candidate.accepted_block_withdrawal - ): - # The preview IS the balance state this generation publishes. - # The durable ledger only catches up after persistence, so a - # ledger read here would cache pre-parent balances against the - # post-parent generation and outlive the transition override. - artifact = self._payout_state_artifact_from_balances( - generation=candidate.base_generation + 1, - source_generation=candidate.source_generation, - balances=self._materialize_prior_balance_preview( - candidate.accepted_block_preview - ), - ) - else: - artifact = self._prepare_payout_state_artifact( - generation=candidate.base_generation + 1, - source_generation=candidate.source_generation, - ) - except Exception: - # The delivery/cache fence remains closed. A later reconciliation - # retry can prepare and publish the artifact; no generation lacking - # immutable payout inputs is ever exposed to builders. - self._schedule_tip_refresh_retry() - raise - with self._job_cache_lock: - with self.lock: - if ( - candidate.source_generation != self._payout_state_source[0] - or candidate.base_generation != self._payout_state_generation - ): - self._record_discarded_payout_candidate() - return None - with self._payout_state_delivery_gate.publication(): - # publication() has already drained admitted old sends. Start the - # critical-section timer only now; drain latency is delivery wait, - # not time spent holding the atomic payout mutation section. - publish_started = time.monotonic() - with self._job_cache_lock: - with self.lock: - source_generation = self._payout_state_source[0] - if ( - candidate.source_generation == source_generation - and candidate.base_generation - == self._payout_state_generation - ): - published_generation = self._payout_state_generation + 1 - if candidate.accepted_block_hash is not None: - key = candidate.accepted_block_hash - with self._accepted_block_payout_preview_condition: - transition = self._accepted_block_payout_previews.get( - key, - _AcceptedBlockPayoutTransition( - block_height=candidate.accepted_block_height, - landed=True, - ), - ) - if candidate.accepted_block_withdrawal: - self._accepted_block_payout_previews.pop(key, None) - self._invalidated_accepted_block_payout_previews[ - key - ] = ( - transition.block_height - if transition.block_height is not None - else candidate.accepted_block_height - ) - else: - existing_preview = transition.preview - if ( - existing_preview is not None - and existing_preview - != candidate.accepted_block_preview - ): - raise RuntimeError( - "accepted block payout preview changed " - "during atomic publication" - ) - self._invalidated_accepted_block_payout_previews.pop( - key, - None, - ) - self._accepted_block_payout_previews[key] = ( - dataclass_replace( - transition, - landed=True, - preview=candidate.accepted_block_preview, - published_generation=published_generation, - ) - ) - self._accepted_block_payout_preview_condition.notify_all() - self._payout_state_generation = published_generation - prepared_artifact = candidate.ledger_artifact - if ( - prepared_artifact is not None - and prepared_artifact.payout_state_generation - == published_generation - ): - self._payout_ledger_artifact_generation += 1 - self._payout_ledger_artifact = dataclass_replace( - prepared_artifact, - generation=self._payout_ledger_artifact_generation, - ) - else: - self._payout_ledger_artifact = None - self._published_payout_state = PublishedPayoutState( - generation=published_generation, - source_generation=candidate.source_generation, - source_tip_hash=candidate.source_tip_hash, - published_monotonic=publish_started, - artifact=artifact, - ) - self._payout_state_publication_blocked = False - self._job_bundle_cache.clear() - self._retained_collection_refresh = None - active = getattr(self, "_active_tip_refresh", None) - if active is None: - schedule_retry = True - elif active[0].payout_state_generation < published_generation: - # The payout gate itself rejects this old generation. - # Signal its fanout only after atomic publication. - active_to_cancel = active[1] - schedule_retry = True - with self._payout_state_metrics_lock: - self._payout_first_delivery_pending = ( - published_generation, - candidate.invalidated_monotonic, - ) - if published_generation is not None: - # The mutation owner still blocks every delivery admission, - # so the pointer swap and gate generation remain one atomic - # publication boundary. Do not acquire the gate condition - # while holding coordinator locks: cancellation callbacks take - # those locks after entering the gate wait loop. - self._payout_state_delivery_gate.publish_generation( - published_generation, - prioritize_delivery=True, - ) - self._observe_payout_state_seconds( - "publish", - max(0.0, time.monotonic() - publish_started), - ) - if published_generation is None: - self._record_discarded_payout_candidate() - return None - self._cancel_obsolete_job_bundle_builds( - payout_state_generation=published_generation - ) - self._record_progress_payout_generation( - published_generation, - candidate.invalidated_monotonic, - ) - if active_to_cancel is not None: - active_to_cancel.cancel() - self._cancel_obsolete_job_builds("payout generation published") - if schedule_retry: - self._mark_tip_refresh_pending(published_generation) - self._schedule_tip_refresh_retry() - with self._job_cache_lock: - current_artifacts = self._template_artifacts - published_artifact_usable = ( - self._usable_payout_ledger_artifact( - published_generation, - current_artifacts.network_difficulty, - ) - if current_artifacts is not None - else None - ) - accepted_preview_pending_durability = ( - candidate.accepted_block_hash is not None - and not candidate.accepted_block_withdrawal - ) - if ( - current_artifacts is not None - and published_artifact_usable is None - and not accepted_preview_pending_durability - ): - # A background artifact built before accepted-block confirmation - # would carry the old database balances under the new payout - # generation. Child builders use the compact preview until the - # durable state catches up; artifact preparation resumes then. - self._schedule_payout_ledger_artifact_preparation( - published_generation, - current_artifacts.network_difficulty, - ) - return published_generation + return self._ensure_payout_state_service().publish_candidate(candidate) def _record_first_payout_delivery( self, generation: int, delivered_monotonic: float, ) -> None: - self._ensure_job_cache_state() - elapsed: float | None = None - with self._payout_state_metrics_lock: - pending = self._payout_first_delivery_pending - if pending is not None and pending[0] == generation: - elapsed = max(0.0, delivered_monotonic - pending[1]) - self._payout_first_delivery_pending = None - if elapsed is not None: - self._observe_payout_state_seconds("first_delivery", elapsed) + self._ensure_payout_state_service().record_first_delivery( + generation, delivered_monotonic + ) def _advance_payout_state_generation(self) -> int: - """Publish a payout-only invalidation with no expensive gate work.""" - # No production caller remains: direct-block finalization publishes - # through submit_block_candidate's reserve/publish path instead. Tests - # keep using this as the smallest complete reserve -> block -> publish - # invalidation cycle. - self._ensure_job_cache_state() - self._reserve_payout_state_source("payout_only") - prepared_started = time.monotonic() - with self._payout_state_prepare_lock: - # Close build/delivery admission before releasing snapshot readers. - # Publication may then drain already-admitted sends without holding - # the preparation lock needed by later ledger work. - self._block_payout_state_publication(force=True) - self._observe_payout_state_seconds( - "preparation", - max(0.0, time.monotonic() - prepared_started), - ) - generation = self._publish_current_payout_state_with_retry_budget() - if generation is None: - raise TemplateRefreshSuperseded( - "payout-only invalidation was superseded; immediate retry scheduled" - ) - return generation + return self._ensure_payout_state_service().advance_generation() def _publish_current_payout_state_with_retry_budget( self, *, initial_attempted: bool = False, ) -> int | None: - """Publish the current source with a bounded supersession budget.""" - - max_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), + return self._ensure_payout_state_service().publish_current_with_retry_budget( + initial_attempted=initial_attempted ) - attempts = max_retries + (0 if initial_attempted else 1) - for _attempt in range(attempts): - candidate = self._current_payout_state_candidate() - published = self._publish_payout_state_candidate(candidate) - if published is not None: - return published - self._block_payout_state_publication() - return None - def observe_job_build_elapsed(self, elapsed_seconds: float, phases: dict[str, float]) -> None: - self._ensure_job_cache_state() - with self._job_cache_lock: - self.job_build_count += 1 - self.job_build_seconds_sum += elapsed_seconds - for bucket in PRISM_JOB_BUILD_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - self.job_build_seconds_bucket_counts[bucket] += 1 - for phase, duration in phases.items(): - if phase in self.job_build_phase_seconds: - self.job_build_phase_seconds[phase] += duration + def observe_job_build_elapsed( + self, + elapsed_seconds: float, + phases: dict[str, float], + ) -> None: + self._ensure_job_bundle_service().observe_elapsed(elapsed_seconds, phases) def _reserve_template_artifact_generation(self) -> int: - """Reserve template ordering when a fetch starts, not when it finishes.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - self._template_artifact_generation += 1 - return self._template_artifact_generation + return self._ensure_job_bundle_service().template_repository.reserve_generation() def _derive_template_artifacts( self, @@ -5749,37 +3432,8 @@ def _derive_template_artifacts( *, generation: int, ) -> CachedTemplateArtifacts: - # Detach the observation from mutable RPC/test caller state. The - # dataclass then owns this exact tree for the lifetime of its snapshot. - template = copy.deepcopy(template) - fingerprint = qbit_template_fingerprint(template) - with self._job_cache_lock: - previous = self._template_artifacts - if previous is not None and previous.fingerprint == fingerprint: - return CachedTemplateArtifacts( - template=template, - fingerprint=fingerprint, - previousblockhash=str(template.get("previousblockhash", "")), - transaction_hexes=previous.transaction_hexes, - witness_merkle_leaves_hex=previous.witness_merkle_leaves_hex, - network_difficulty=previous.network_difficulty, - fetched_monotonic=time.monotonic(), - generation=generation, - ) - phases = self._job_build_phases() - started = time.monotonic() - transaction_hexes = direct_stratum.transaction_hexes_from_template(template) - witness_leaves = tuple(direct_stratum.witness_merkle_leaves_hex(transaction_hexes)) - network_difficulty = scaled_network_difficulty(str(template["bits"])) - phases["merkle"] = phases.get("merkle", 0.0) + (time.monotonic() - started) - return CachedTemplateArtifacts( - template=template, - fingerprint=fingerprint, - previousblockhash=str(template.get("previousblockhash", "")), - transaction_hexes=transaction_hexes, - witness_merkle_leaves_hex=witness_leaves, - network_difficulty=network_difficulty, - fetched_monotonic=time.monotonic(), + return self._ensure_job_bundle_service().template_repository.derive( + template, generation=generation, ) @@ -5787,37 +3441,9 @@ def _store_template_artifacts( self, artifacts: CachedTemplateArtifacts, ) -> bool: - changed = False - with self._job_cache_lock: - previous = self._template_artifacts - if previous is not None and artifacts.generation < previous.generation: - return False - self._template_artifacts = artifacts - if previous is not None and previous.fingerprint != artifacts.fingerprint: - changed = True - # Retain the published snapshot's bundles alongside the new - # fingerprint: until the replacement tip is published, direct - # issuance still serves published-snapshot work, and pruning - # it here would force those paths to defer for the whole - # detected-but-unpublished window. Publication moves the - # snapshot, so the next fingerprint change drops the old - # entries. - keep_fingerprints = {artifacts.fingerprint} - with self.lock: - published_snapshot = getattr(self, "tip_template_snapshot", None) - if published_snapshot is not None: - keep_fingerprints.add(published_snapshot.template_fingerprint) - self._job_bundle_cache = OrderedDict( - (key, entry) - for key, entry in self._job_bundle_cache.items() - if entry.template_fingerprint in keep_fingerprints - ) - if changed: - self._cancel_obsolete_job_builds( - "template fingerprint superseded", - keep_published_snapshot=True, - ) - return True + return self._ensure_job_bundle_service().template_repository.store_artifacts( + artifacts + ) def store_template_artifacts( self, @@ -5825,136 +3451,20 @@ def store_template_artifacts( *, generation: int | None = None, ) -> CachedTemplateArtifacts | None: - """Best-effort cache fill from an already-fetched template (blockpoll). - - Returns None instead of raising so a template the derivation cannot - digest degrades to the legacy per-build fetch path rather than failing - the poll. The returned artifacts describe this exact observation even - if a newer observation already won the cache-write race; blockpoll then - detects the mismatch before fanout. - """ - self._ensure_job_cache_state() - if generation is None: - generation = self._reserve_template_artifact_generation() - try: - artifacts = self._derive_template_artifacts( - template, - generation=generation, - ) - except Exception: - return None - self._store_template_artifacts(artifacts) - return artifacts + return self._ensure_job_bundle_service().template_repository.store( + template, + generation=generation, + ) def job_issuance_template_artifacts(self) -> CachedTemplateArtifacts: - """Template artifacts for direct (non-refresh) job issuance. - - While a detected replacement tip is still unpublished, share - classification remains anchored to the published tip. Direct issuance - paths (initial delivery retries, Vardiff retargets, reauthorization) - must therefore keep handing out published-snapshot work; issuing - detected-tip work early would have every one of its shares rejected - as stale until the refresh publishes. Once the published authority - lapses (divergence lease expired), issuance falls through to the live - template exactly like submit classification falls back to the live - RPC read. - """ - self._ensure_tip_refresh_state() - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - latest_detected = getattr(self, "latest_detected_tip", None) - published_snapshot = getattr(self, "tip_template_snapshot", None) - pinned = bool( - published is not None - and latest_detected is not None - and latest_detected[0] != published[0] - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and published_snapshot.template_artifacts is not None - and self._published_tip_authoritative_locked(time.monotonic()) - ) - if pinned: - assert published_snapshot is not None - assert published_snapshot.template_artifacts is not None - return published_snapshot.template_artifacts - artifacts = self.current_template_artifacts() - # The fetch above may itself be the first observation of a newer tip - # (a template-cache miss racing ahead of blockpoll/blockwait). The - # published tip still owns share classification, so serve its - # snapshot; the recorded detection has already armed the refresh. - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - published_snapshot = getattr(self, "tip_template_snapshot", None) - repinned = bool( - published is not None - and artifacts.previousblockhash != published[0] - and published_snapshot is not None - and published_snapshot.bestblockhash == published[0] - and published_snapshot.template_artifacts is not None - and self._published_tip_authoritative_locked(time.monotonic()) - ) - if repinned: - assert published_snapshot is not None - assert published_snapshot.template_artifacts is not None - return published_snapshot.template_artifacts - return artifacts + return self._ensure_job_bundle_service().template_repository.issuance() def current_template_artifacts(self) -> CachedTemplateArtifacts: - """Return fresh template artifacts, fetching a template on cache miss.""" - self._ensure_job_cache_state() - ttl = getattr(self, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) - now = time.monotonic() - with self._job_cache_lock: - cached = self._template_artifacts - with self.lock: - observed_tip = self._newest_observed_tip_locked() - cached_tip_current = ( - observed_tip is None - or cached is None - or cached.previousblockhash == observed_tip - ) - if ( - cached is not None - and cached_tip_current - and ttl > 0 - and now - cached.fetched_monotonic <= ttl - ): - self._record_job_cache_event("template", hit=True) - return cached - self._record_job_cache_event("template", hit=False) - generation = self._reserve_template_artifact_generation() - phases = self._job_build_phases() - started = time.monotonic() - template = self.rpc.call( - "getblocktemplate", - [{"rules": qbit_gbt_rules(getattr(self, "qbit_chain", "regtest"))}], - ) - if not isinstance(template, dict): - raise RuntimeError("getblocktemplate returned non-object") - phases["template"] = phases.get("template", 0.0) + (time.monotonic() - started) - artifacts = self._derive_template_artifacts( - template, - generation=generation, - ) - if self._store_template_artifacts(artifacts): - # A direct fetch can be the first reader to see qbit advance. - # Record it as a detection like every other live-tip observation, - # or the buildability gates would treat the newer parent as - # unknown while pinned issuance still waits on blockpoll. - self.observe_tip_for_refresh(artifacts.previousblockhash) - return artifacts - # A later fetch completed first. Build from that current observation, - # never from the stale response that lost the cache-write race. - with self._job_cache_lock: - current = self._template_artifacts - if current is None: - raise RuntimeError("newer template artifacts disappeared after cache race") - self.observe_tip_for_refresh(current.previousblockhash) - return current + return self._ensure_job_bundle_service().template_repository.current() @staticmethod def _collection_bundle_identity(worker: WorkerIdentity) -> tuple[str, str]: - return worker.payout_address, worker.p2mr_program_hex + return JobBundleService.collection_identity(worker) def _job_bundle_key( self, @@ -5965,157 +3475,41 @@ def _job_bundle_key( payout_artifact_generation: int = 0, worker: WorkerIdentity | None, ) -> tuple[object, ...]: - if mode == "ready": - return ( - artifacts.fingerprint, - artifacts.previousblockhash, - "ready", - payout_state_generation, - payout_artifact_generation, - ) - if mode != "collection": - raise ValueError(f"unknown PRISM job-bundle mode: {mode}") - if worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - return ( - artifacts.fingerprint, - artifacts.previousblockhash, - "collection", - artifacts.generation, - payout_state_generation, - payout_artifact_generation, - *self._collection_bundle_identity(worker), + return self._ensure_job_bundle_service().job_bundle_key( + artifacts, + mode=mode, + payout_state_generation=payout_state_generation, + payout_artifact_generation=payout_artifact_generation, + worker=worker, ) def _job_bundle_mode(self, requested_mode: str | None) -> str: - if requested_mode is not None: - if requested_mode not in {"ready", "collection"}: - raise ValueError( - f"unknown PRISM job-bundle mode: {requested_mode}" - ) - return requested_mode - return "ready" if self.pool_readiness_latched() else "collection" + return self._ensure_job_bundle_service().job_bundle_mode(requested_mode) def _lookup_job_bundle( self, key: tuple[object, ...], ) -> CachedJobBundle | None: - ttl = getattr(self, "job_bundle_cache_seconds", DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS) - now = time.monotonic() - with self._job_cache_lock: - if ttl <= 0: - self._job_bundle_cache.clear() - return None - # The entry-count cap is not a memory bound: one production entry - # can reference more than 100k shares. Expired entries must release - # their snapshots instead of remaining resident until count eviction. - expired = [ - cache_key - for cache_key, entry in self._job_bundle_cache.items() - if now - entry.built_monotonic > ttl - ] - for cache_key in expired: - self._job_bundle_cache.pop(cache_key, None) - return self._job_bundle_cache.get(key) - return None + return self._ensure_job_bundle_service().lookup_bundle(key) def _job_bundle_entry_usable( self, cached: CachedJobBundle | None, artifacts: CachedTemplateArtifacts, ) -> bool: - """Re-validate readiness for cached collection bundles. - - Readiness is monotonic in practice (the distinct accepted-miner count - only grows), so submit-capable ready bundles are served as-is. A cached - collection bundle is re-checked against the cheap aggregate stats: - once the pool is ready it must stop being served, or jobs would keep - collecting winning shares without submitting blocks for up to the cache - TTL. - """ - if cached is None: - return False - # Usability follows the same split-authority rule as construction: a - # cached bundle for the newest detected tip stays reusable across - # refresh retries while the previous tip is still published, and the - # published snapshot's own bundle stays servable for pinned issuance. - with self.lock: - parent_usable = self._artifacts_buildable_locked(artifacts) - if not parent_usable: - return False - with self._job_cache_lock: - if self._payout_state_publication_blocked: - return False - if not self._job_bundle_payout_state_current(cached): - return False - if not cached.collection_only: - return True - # Collection bundles sign a synthetic bootstrap share containing the - # exact template ntime. A clock-only observation keeps the stable work - # fingerprint, but it must rebuild this signed bundle instead of - # rebinding the old manifest to a new template generation. - if ( - cached.template is not artifacts.template - or cached.template_generation != artifacts.generation - ): - return False - try: - _, ready_miner_count = self.accepted_share_stats() - except Exception: - # If readiness cannot be proven, force the normal build path. That - # path will either build an up-to-date bundle or surface the ledger - # failure instead of continuing to issue no-submit collection jobs. - return False - return ready_miner_count < self.min_ready_miners + return self._ensure_job_bundle_service().bundle_entry_usable( + cached, + artifacts, + ) def _bind_cached_bundle_to_artifacts( self, cached: CachedJobBundle, artifacts: CachedTemplateArtifacts, ) -> CachedJobBundle: - """Return the cached heavy bundle bound to this exact observation. - - Clock-only template changes intentionally keep the stable fingerprint. - Ready bundles may reuse their ledger snapshot and signed manifest, but - the Stratum base job must still carry the observing template's exact - ntime and generation. Collection bundles are filtered before this point - because their signed synthetic share contains the template ntime. - """ - if ( - cached.template is artifacts.template - and cached.template_generation == artifacts.generation - ): - return cached - manifest = cached.coinbase_manifest - base_job = direct_stratum.make_job_from_builder_manifest( - job_id="prism-template-base", - template=artifacts.template, - manifest=manifest, - extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - extranonce2_size=self.extranonce2_size, - desired_share_difficulty=self.share_difficulty, - clean_jobs=True, - transaction_hexes=artifacts.transaction_hexes, - ) - return dataclass_replace( + return self._ensure_job_bundle_service().bind_cached_bundle( cached, - template=artifacts.template, - base_job=base_job, - template_generation=artifacts.generation, - build_key=( - dataclass_replace( - cached.build_key, - best_tip_hash=artifacts.previousblockhash, - previous_block_hash=artifacts.previousblockhash, - template_generation=artifacts.generation, - block_height=int(artifacts.template["height"]), - coinbase_value_sats=int(artifacts.template["coinbasevalue"]), - ) - if cached.build_key is not None - else None - ), + artifacts, ) def _new_job_build_request( @@ -6128,165 +3522,15 @@ def _new_job_build_request( cache_key: tuple[object, ...], payout_ledger_artifact: PayoutLedgerArtifact | None = None, idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - preparation_cancellation: _JobBuildCancellation | None = None, ) -> _JobBuildRequest: - cancellation = ( - self._new_job_build_cancellation() - if preparation_cancellation is None - else preparation_cancellation - ) - cancellation.raise_if_cancelled("immutable snapshot") - # A pending accepted-parent transition owns the balances children must - # observe. Wait for its preview before acquiring any confirmed payout - # input so a child build cannot take -- or cache into the published - # artifact -- confirmed state that omits its new parent. The preview - # itself is re-resolved consistently during construction. - self._await_pending_parent_payout_preview( - artifacts.previousblockhash, - parent_height=int(artifacts.template["height"]) - 1, - ) - payout_artifact = self._current_payout_state_artifact(cancellation) - if payout_artifact.generation != payout_state_generation: - raise JobBuildSuperseded( - "payout artifact generation changed before build request" - ) - - phases = self._job_build_phases() - payout_started = time.monotonic() - payout_policy_json = canonical_json_text(self.prism_payout_policy()) - phases["payout"] = phases.get("payout", 0.0) + ( - time.monotonic() - payout_started - ) - cancellation.raise_if_cancelled("payout policy") - ctv_started = time.monotonic() - ctv_settlement = self.prism_ctv_settlement_config( - block_height=int(artifacts.template["height"]), - parent_hash=artifacts.previousblockhash, - ) - ctv_settlement_json = ( - canonical_json_text(ctv_settlement) - if ctv_settlement is not None - else None - ) - phases["ctv"] = phases.get("ctv", 0.0) + ( - time.monotonic() - ctv_started - ) - cancellation.raise_if_cancelled("CTV configuration") - - suffix_hex = self.coinbase_script_sig_suffix_hex( - PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - "00" * self.extranonce2_size, - ) - collection_identity = ( - self._collection_bundle_identity(worker) - if mode == "collection" and worker is not None - else None - ) - decimal_context = getcontext().copy() - numeric_context_sha256 = canonical_json_sha256( - { - "precision": decimal_context.prec, - "rounding": decimal_context.rounding, - "minimum_exponent": decimal_context.Emin, - "maximum_exponent": decimal_context.Emax, - "capitals": decimal_context.capitals, - "clamp": decimal_context.clamp, - } - ) - with self._job_cache_lock: - issued_at_ms = self._job_build_issued_at_ms.get(artifacts.generation) - if issued_at_ms is None: - # The issued time doubles as the audit window anchor, so it - # must not cover a stamped share whose commit is still in - # flight: the frozen anchor stays reproducible from the - # durable ledger for every rebuild of this generation. - issued_at_ms = self._job_snapshot_anchor_ms(now_ms()) - self._job_build_issued_at_ms[artifacts.generation] = issued_at_ms - while len(self._job_build_issued_at_ms) > 128: - self._job_build_issued_at_ms.popitem(last=False) - build_key = JobBuildKey( - best_tip_hash=artifacts.previousblockhash, - previous_block_hash=artifacts.previousblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - payout_state_generation=payout_state_generation, - payout_artifact_sha256=payout_artifact.prior_balances_sha256, + return self._ensure_job_bundle_service().new_build_request( + artifacts, + worker, mode=mode, - collection_identity=collection_identity, - block_height=int(artifacts.template["height"]), - coinbase_value_sats=int(artifacts.template["coinbasevalue"]), - network_difficulty=int(artifacts.network_difficulty), - issued_at_ms=issued_at_ms, - payout_policy_sha256=hashlib.sha256( - payout_policy_json.encode() - ).hexdigest(), - ctv_settlement_sha256=( - hashlib.sha256(ctv_settlement_json.encode()).hexdigest() - if ctv_settlement_json is not None - else None - ), - witness_merkle_sha256=canonical_json_sha256( - artifacts.witness_merkle_leaves_hex - ), - transaction_set_sha256=canonical_json_sha256( - artifacts.transaction_hexes - ), - coinbase_suffix_hex=suffix_hex, - signing_key_sha256=hashlib.sha256( - str(getattr(self, "signing_seed_hex", "")).encode() - ).hexdigest(), - ledger_signing_key_sha256=hashlib.sha256( - str( - getattr( - self, - "ledger_attestation_signing_seed_hex", - "", - ) - ).encode() - ).hexdigest(), - numeric_context_sha256=numeric_context_sha256, - ) - immutable_identity: tuple[object, ...] = ( - cache_key, - artifacts.generation, - issued_at_ms, - payout_artifact.prior_balances_sha256, - build_key.payout_policy_sha256, - build_key.ctv_settlement_sha256, - build_key.witness_merkle_sha256, - build_key.transaction_set_sha256, - build_key.coinbase_suffix_hex, - build_key.signing_key_sha256, - build_key.ledger_signing_key_sha256, - build_key.numeric_context_sha256, - ) - return _JobBuildRequest( - key=build_key, + payout_state_generation=payout_state_generation, cache_key=cache_key, - equivalence_key=immutable_identity, - artifacts=artifacts, - template_json=canonical_json_text(artifacts.template), - transaction_hexes=artifacts.transaction_hexes, - witness_merkle_leaves_hex=artifacts.witness_merkle_leaves_hex, - worker=worker, - mode=mode, - payout_artifact=payout_artifact, payout_ledger_artifact=payout_ledger_artifact, - payout_policy_json=payout_policy_json, - ctv_settlement_json=ctv_settlement_json, - decimal_context=decimal_context, - cancellation=cancellation, idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - requested_monotonic=( - cancellation.started_monotonic - if priority_requested_monotonic is None - else priority_requested_monotonic - ), ) def _newest_observed_tip_locked(self) -> str | None: @@ -6344,46 +3588,10 @@ def _cache_job_bundle_if_current( built: CachedJobBundle, artifacts: CachedTemplateArtifacts, ) -> bool: - """Cache only current state; report whether payout state stayed valid.""" - with self._job_cache_lock: - with self.lock: - buildable = self._artifacts_buildable_locked(artifacts) - snapshot_pinned = self._published_snapshot_artifacts_locked( - artifacts - ) - published_artifact = self._published_payout_state.artifact - if not buildable: - return False - if ( - built.payout_state_generation != self._payout_state_generation - or built.build_key is None - or published_artifact is None - or built.build_key.payout_artifact_sha256 - != published_artifact.prior_balances_sha256 - ): - return False - current = self._template_artifacts - globally_current = ( - current is not None - and current.fingerprint == artifacts.fingerprint - and current.previousblockhash == artifacts.previousblockhash - and ( - not built.collection_only - or current.generation == artifacts.generation - ) - ) - if not globally_current and not snapshot_pinned: - # Only the current template observation may win the cache - # race; the sole exception is a pinned rebuild of exactly the - # published snapshot, which repeated direct issuance would - # otherwise rebuild for the rest of the unpublished window. - return False - self._job_bundle_cache[built.key] = built - self._job_bundle_cache.move_to_end(built.key) - while len(self._job_bundle_cache) > MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES: - oldest_key = next(iter(self._job_bundle_cache)) - self._job_bundle_cache.pop(oldest_key, None) - return True + return self._ensure_job_bundle_service().cache_bundle_if_current( + built, + artifacts, + ) def shared_job_bundle( self, @@ -6398,259 +3606,18 @@ def shared_job_bundle( request_source: str = "routine", priority_requested_monotonic: float | None = None, ) -> CachedJobBundle: - """Return one immutable heavy build through a work-identity flight. - - Equivalent callers share one future. Publication-critical refreshes - can cancel routine work and start in the bounded replacement slot; - reconnect, Vardiff, authorization, and same-tip work must defer while - that priority work is active. Only the latest pending request is - retained during repeated supersession. - """ - self._ensure_job_cache_state() self._ensure_tip_refresh_state() - priority_token: int | None = None - if publication_critical: - ( - priority_token, - priority_requested_monotonic, - ) = self._begin_job_build_priority_preparation( - priority_requested_monotonic - ) - try: - return self._shared_job_bundle_after_priority_admission( - artifacts, - worker, - mode=mode, - cancelled=cancelled, - retry_superseded=retry_superseded, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=priority_requested_monotonic, - ) - finally: - if priority_token is not None: - self._finish_job_build_priority_preparation(priority_token) - - def _shared_job_bundle_after_priority_admission( - self, - artifacts: CachedTemplateArtifacts, - worker: WorkerIdentity | None = None, - *, - mode: str | None = None, - cancelled: Callable[[], bool] | None = None, - retry_superseded: bool = True, - idle_retarget: bool = False, - publication_critical: bool = False, - request_source: str = "routine", - priority_requested_monotonic: float | None = None, - ) -> CachedJobBundle: - while True: - routine_preparation_token: int | None = None - preparation_cancellation: _JobBuildCancellation | None = None - if not publication_critical: - ( - routine_preparation_token, - preparation_cancellation, - ) = self._begin_routine_job_build_preparation( - request_source=request_source, - cancelled=cancelled, - ) - resolved_mode = self._job_bundle_mode(mode) - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "request preparation admission" - ) - if resolved_mode == "collection" and worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - payout_artifact = ( - self._usable_payout_ledger_artifact( - payout_state_generation, - artifacts.network_difficulty, - ) - if resolved_mode == "ready" - else None - ) - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "payout artifact lookup" - ) - payout_artifact_generation = ( - payout_artifact.generation if payout_artifact is not None else 0 - ) - key = self._job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=payout_artifact_generation, - worker=worker, - ) - cached = self._lookup_job_bundle(key) - if self._job_bundle_entry_usable(cached, artifacts): - if preparation_cancellation is not None: - preparation_cancellation.raise_if_cancelled( - "bundle cache lookup" - ) - if routine_preparation_token is not None: - self._finish_routine_job_build_preparation( - routine_preparation_token - ) - self._record_job_cache_event("bundle", hit=True) - if request_source == "initial": - with self._job_build_scheduler_lock: - self.initial_job_prepared_work_counts["cache_hit"] += 1 - assert cached is not None - return self._bind_cached_bundle_to_artifacts(cached, artifacts) - if self._job_bundle_mode(mode) != resolved_mode: - if routine_preparation_token is not None: - self._finish_routine_job_build_preparation( - routine_preparation_token - ) - continue - self._record_job_cache_event("bundle", hit=False) - try: - request = self._new_job_build_request( - artifacts, - worker, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - cache_key=key, - payout_ledger_artifact=payout_artifact, - idle_retarget=idle_retarget, - publication_critical=publication_critical, - request_source=request_source, - priority_requested_monotonic=( - priority_requested_monotonic - ), - preparation_cancellation=preparation_cancellation, - ) - # Preserve the historical readiness handoff without holding a - # lock across construction: only admission and the final mode - # re-selection are serialized here. - with self._job_build_lock: - with self._job_build_scheduler_lock: - if routine_preparation_token is not None: - self._finish_routine_job_build_preparation( - routine_preparation_token - ) - routine_preparation_token = None - request.cancellation.raise_if_cancelled( - "scheduler admission" - ) - if self._job_bundle_mode(mode) != resolved_mode: - request.cancellation.cancel( - "worker mode superseded" - ) - continue - if cancelled is not None and cancelled(): - raise _JobBuildCancelled( - "job bundle request was cancelled before preparation" - ) - promise = self._request_job_build(request) - wait_deadline = time.monotonic() + max( - 0.001, - float(self.job_build_timeout_seconds) - + float(self.job_build_cancel_grace_seconds) - + 1.0, - ) - while True: - if cancelled is not None and cancelled(): - raise _JobBuildCancelled( - "job bundle waiter was cancelled during preparation" - ) - try: - built = promise.result( - timeout=min( - 0.1, - max(0.001, wait_deadline - time.monotonic()), - ) - ) - break - except TimeoutError: - if time.monotonic() >= wait_deadline: - raise - except TimeoutError as exc: - request.cancellation.cancel("timeout") - self._schedule_tip_refresh_retry() - raise JobBuildCancelled( - "job build timed out; immediate retry scheduled" - ) from exc - except JobBuildCancelled: - if routine_preparation_token is not None: - self._finish_routine_job_build_preparation( - routine_preparation_token - ) - self._schedule_tip_refresh_retry() - if not retry_superseded: - raise - with self.lock: - buildable = self._artifacts_buildable_locked(artifacts) - if not buildable: - raise - with self._job_cache_lock: - current = self._template_artifacts - payout_current = ( - payout_state_generation == self._payout_state_generation - ) - if current is artifacts and payout_current: - continue - if current is artifacts: - continue - raise - built = self._bind_cached_bundle_to_artifacts(built, artifacts) - if not self._cache_job_bundle_if_current(built, artifacts): - with self._job_build_scheduler_lock: - self.job_build_scheduler_counts["obsolete_results"] += 1 - with self.lock: - buildable = self._artifacts_buildable_locked(artifacts) - if not buildable: - with self._tip_refresh_metrics_lock: - self.tip_refresh_superseded_results += 1 - raise JobBuildSuperseded( - "observed tip changed before cache publication" - ) - with self._job_cache_lock: - payout_current = ( - built.payout_state_generation - == self._payout_state_generation - ) - with self.lock: - published_snapshot = getattr( - self, - "tip_template_snapshot", - None, - ) - published_artifacts = ( - published_snapshot.template_artifacts - if published_snapshot is not None - else None - ) - if ( - payout_current - and built.template is artifacts.template - and built.template_generation == artifacts.generation - and ( - not retry_superseded - or published_artifacts is artifacts - ) - ): - # Snapshot-owned work may outlive an unrelated global - # cache fill. Return it only to its refresh validator or - # retained collection delivery; never retain it globally. - return built - if retry_superseded: - with self._job_cache_lock: - current = self._template_artifacts - if current is artifacts: - continue - raise JobBuildSuperseded( - "job build key changed before cache publication" - ) - return built + return self._ensure_job_bundle_service().shared_job_bundle( + artifacts, + worker, + mode=mode, + cancelled=cancelled, + retry_superseded=retry_superseded, + idle_retarget=idle_retarget, + publication_critical=publication_critical, + request_source=request_source, + priority_requested_monotonic=priority_requested_monotonic, + ) def build_shared_job_bundle( self, @@ -6663,273 +3630,14 @@ def build_shared_job_bundle( key: tuple[object, ...] | None = None, build_request: _JobBuildRequest | None = None, ) -> CachedJobBundle: - phases = self._job_build_phases() - resolved_mode = self._job_bundle_mode(mode) - if resolved_mode == "collection" and worker is None: - raise CollectionIdentityUnavailable( - "collection-mode worker identity is temporarily unavailable" - ) - with self._job_cache_lock: - publication_blocked = self._payout_state_publication_blocked - if payout_state_generation is None: - payout_state_generation = self._payout_state_generation - if publication_blocked: - raise _PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - if key is None: - key = self._job_bundle_key( - artifacts, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - payout_artifact_generation=( - payout_artifact.generation - if payout_artifact is not None - else 0 - ), - worker=worker, - ) - if build_request is None: - build_request = self._new_job_build_request( - artifacts, - worker, - mode=resolved_mode, - payout_state_generation=payout_state_generation, - cache_key=key, - payout_ledger_artifact=payout_artifact, - ) - else: - payout_artifact = build_request.payout_ledger_artifact - cancellation = build_request.cancellation - self._job_build_checkpoint("ledger_snapshot", cancellation) - template_value = json.loads(build_request.template_json) - if not isinstance(template_value, dict): - raise RuntimeError("immutable job template is not an object") - template: dict[str, Any] = template_value - issued_at_ms = build_request.key.issued_at_ms - started = time.monotonic() - snapshot_window_weight = ( - PRISM_REWARD_WINDOW_MULTIPLIER - * PRISM_SNAPSHOT_WINDOW_MARGIN - * int(build_request.key.network_difficulty) - ) - if payout_artifact is not None: - if ( - self._usable_payout_ledger_artifact( - payout_state_generation, - build_request.key.network_difficulty, - ) - is not payout_artifact - ): - raise JobBuildSuperseded( - "precomputed payout artifact changed before construction" - ) - prior_balances = list(payout_artifact.prior_balances) - if ( - canonical_json_sha256(prior_balances) - != build_request.key.payout_artifact_sha256 - ): - raise JobBuildSuperseded( - "precomputed payout artifact does not match payout generation" - ) - shares = list(payout_artifact.shares_json) - # The artifact binds the key above; the balances actually used may - # still be an accepted parent's prospective carry state. - prior_balances = self._prior_balances_for_job_parent( - str(template["previousblockhash"]), - parent_height=int(template["height"]) - 1, - fallback_balances=prior_balances, - ) - else: - with self._payout_state_prepare_lock: - with self._job_cache_lock: - published_artifact = self._published_payout_state.artifact - if self._payout_state_publication_blocked: - raise _PayoutStatePublicationBlocked( - "payout state invalidation is pending publication" - ) - if ( - payout_state_generation != self._payout_state_generation - or published_artifact is None - or published_artifact.prior_balances_sha256 - != build_request.key.payout_artifact_sha256 - ): - raise JobBuildSuperseded( - "payout generation changed before ledger snapshot" - ) - records = ( - self.ledger.snapshot_at_job_issue( - issued_at_ms, - window_weight=snapshot_window_weight, - ) - if resolved_mode == "ready" - else [] - ) - # An accepted parent's prospective carry state supersedes the - # published artifact for children built on that parent; the - # published balances remain the fallback for ordinary tips. - prior_balances = self._prior_balances_for_job_parent( - str(template["previousblockhash"]), - parent_height=int(template["height"]) - 1, - fallback_balances=build_request.payout_artifact.prior_balances(), - ) - self._job_build_checkpoint("ledger_snapshot_complete", cancellation) - shares = [] - for index, record in enumerate(records): - if index % 256 == 0: - self._job_build_checkpoint( - "ledger_snapshot_conversion", - cancellation, - ) - shares.append(record.to_prism_json()) - # A reused artifact carries shares snapshotted at its own earlier - # anchor. The bundle must declare that anchor: replaying the audit - # window at this job's fresher anchor could include a share that was - # already durable at artifact build time but stamped after the - # artifact's clamped anchor, and the artifact's share set excludes it - # by construction. - bundle_anchor_ms = ( - payout_artifact.snapshot_anchor_ms - if payout_artifact is not None - and payout_artifact.snapshot_anchor_ms is not None - else issued_at_ms - ) - ledger_elapsed = time.monotonic() - started - phases["ledger"] = phases.get("ledger", 0.0) + ledger_elapsed - if resolved_mode == "ready": - self._observe_tip_refresh_build_phase( - "ledger_snapshot", - ledger_elapsed, - ) - share_snapshot_sha256 = canonical_json_sha256(shares) - final_build_key = dataclass_replace( - build_request.key, - share_snapshot_sha256=share_snapshot_sha256, - ) - self._job_build_checkpoint("payout_derivation", cancellation) - started = time.monotonic() - placeholder_suffix_hex = final_build_key.coinbase_suffix_hex - collection_identity: tuple[str, str] | None = None - previous_metrics_scope = bool( - getattr(self._job_build_phase_local, "tip_refresh_metrics", False) - ) - self._job_build_phase_local.tip_refresh_metrics = resolved_mode == "ready" - try: - if resolved_mode == "ready": - if not shares: - raise RuntimeError( - "ready-pool ledger snapshot contained no payout shares" - ) - self._job_build_checkpoint("ctv_manifest", cancellation) - self._job_build_checkpoint("signing_verification", cancellation) - bundle = self.build_audit_bundle( - shares=shares, - found_block={ - "block_height": int(template["height"]), - "coinbase_value_sats": int(template["coinbasevalue"]), - "network_difficulty": artifacts.network_difficulty, - "anchor_job_issued_at_ms": bundle_anchor_ms, - }, - prior_balances=prior_balances, - coinbase_script_sig_suffix_hex=placeholder_suffix_hex, - witness_merkle_leaves_hex=list( - build_request.witness_merkle_leaves_hex - ), - ctv_fee_parent_hash=str(template["previousblockhash"]), - summary_only=True, - payout_policy=json.loads(build_request.payout_policy_json), - ctv_settlement=( - json.loads(build_request.ctv_settlement_json) - if build_request.ctv_settlement_json is not None - else None - ), - cancellation=cancellation, - ) - collection_only = False - else: - assert worker is not None - self._job_build_checkpoint("ctv_manifest", cancellation) - self._job_build_checkpoint("signing_verification", cancellation) - bundle = self.build_collection_bundle( - template=template, - transaction_hexes=build_request.transaction_hexes, - worker=worker, - network_difficulty=final_build_key.network_difficulty, - issued_at_ms=issued_at_ms, - suffix_hex=placeholder_suffix_hex, - summary_only=True, - payout_policy=json.loads(build_request.payout_policy_json), - ctv_settlement=( - json.loads(build_request.ctv_settlement_json) - if build_request.ctv_settlement_json is not None - else None - ), - cancellation=cancellation, - ) - shares = [] - collection_only = True - collection_identity = self._collection_bundle_identity(worker) - finally: - self._job_build_phase_local.tip_refresh_metrics = previous_metrics_scope - manifest = bundle["signed_coinbase_manifest"]["manifest"] - prospective_prior_balances: ( - tuple[tuple[str, str, str, int], ...] | None - ) = None - payout_policy_manifest = bundle.get("payout_policy_manifest") - if isinstance(payout_policy_manifest, dict) and isinstance( - payout_policy_manifest.get("accounts"), - list, - ): - prospective_prior_balances = self._serialize_prior_balance_preview( - self._accepted_block_payout_preview_from_bundle( - bundle, - prior_balances=prior_balances, - ) - ) - self._job_build_checkpoint("bundle_assembly", cancellation) - assembly_started = time.monotonic() - base_job = direct_stratum.make_job_from_builder_manifest( - job_id="prism-template-base", - template=template, - manifest=manifest, - extranonce1_hex=PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, - extranonce2_size=self.extranonce2_size, - desired_share_difficulty=self.share_difficulty, - clean_jobs=True, - transaction_hexes=build_request.transaction_hexes, - ) - phases["assembly"] = phases.get("assembly", 0.0) + ( - time.monotonic() - assembly_started - ) - self._job_build_checkpoint("serialization", cancellation) - self._job_build_checkpoint("bundle_publication", cancellation) - phases["bundle"] = phases.get("bundle", 0.0) + (time.monotonic() - started) - return CachedJobBundle( - key=key, - # Construction consumed the detached canonical template above; - # publication retains the exact snapshot-owned object identity used - # by the existing validation token. - template=artifacts.template, - template_fingerprint=artifacts.fingerprint, - # Only this manifest is needed to bind later clock-only template - # observations. Retaining the returned logical bundle duplicated - # the entire shares tree already held in shares_json. - coinbase_manifest=manifest, - shares_json=shares, - prior_balances=prior_balances, - found_block=bundle["found_block"], - collection_only=collection_only, - issued_at_ms=issued_at_ms, - base_job=base_job, - built_monotonic=time.monotonic(), - template_generation=artifacts.generation, + return self._ensure_job_bundle_service().build_shared_job_bundle( + artifacts, + worker, + mode=mode, payout_state_generation=payout_state_generation, - payout_artifact_generation=( - payout_artifact.generation if payout_artifact is not None else 0 - ), - collection_identity=collection_identity, - prospective_prior_balances=prospective_prior_balances, - build_key=final_build_key, + payout_artifact=payout_artifact, + key=key, + build_request=build_request, ) def stamp_job_for_client( @@ -6939,54 +3647,11 @@ def stamp_job_for_client( *, clean_jobs: bool, ) -> PrismJobContext: - if client.worker is None: - raise StratumError(20, "client is not authorized") - if cached.collection_only and cached.collection_identity != ( - self._collection_bundle_identity(client.worker) - ): - raise StratumError( - 20, - "collection bundle payout identity no longer matches client authorization", - ) - with self.lock: - self.job_counter += 1 - job_id = f"prism-{self.job_counter}" - share_target = direct_stratum.effective_share_target( - self.desired_client_share_difficulty(client), - cached.base_job.qbit_target, - minimum_advertised_difficulty=self.client_minimum_advertised_difficulty(client), - ) - job = dataclass_replace( - cached.base_job, - job_id=job_id, - extranonce1_hex=client.extranonce1_hex, - share_target=share_target, - share_difficulty=direct_stratum.target_difficulty(share_target), + return self._ensure_job_delivery_service().stamp( + client, + cached, clean_jobs=clean_jobs, ) - return PrismJobContext( - job=job, - template=cached.template, - shares_json=cached.shares_json, - prior_balances=cached.prior_balances, - found_block=cached.found_block, - share_weight=self.share_weight_for_worker(client.worker), - collection_only=cached.collection_only, - worker=client.worker, - issued_at_ms=cached.issued_at_ms, - template_fingerprint=cached.template_fingerprint, - template_generation=cached.template_generation, - payout_state_generation=cached.payout_state_generation, - prospective_prior_balances=cached.prospective_prior_balances, - payout_artifact_generation=cached.payout_artifact_generation, - connection_id=client.connection_id, - authorization_generation=int( - getattr(client, "authorization_generation", 0) - ), - difficulty_generation=int( - getattr(client, "difficulty_generation", 0) - ), - ) def accepted_share_stats(self) -> tuple[int, int]: """Return (accepted share count, distinct miner count) cheaply. @@ -7014,72 +3679,216 @@ def _ensure_watchdog_state(self) -> None: if not hasattr(self, "_watchdog_pauses"): self._watchdog_pauses = {} + def _legacy_ctv_runtime_config(self) -> CtvRuntimeConfig: + coordinator_config = getattr(self, "config", None) + ctv_config = getattr(coordinator_config, "ctv", None) + if ctv_config is not None: + config = CtvRuntimeConfig.from_coordinator_config(ctv_config) + else: + config = CtvRuntimeConfig( + enabled=False, + wallet=None, + fee_sats=0, + limit=100, + chunk_size=DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, + interval_seconds=30.0, + ) + overrides = self.__dict__.get("_ctv_runtime_compat_config", {}) + if overrides: + config = dataclass_replace(config, **overrides) + return config + + def _make_ctv_runtime_service( + self, + config: CtvRuntimeConfig | None = None, + ) -> CtvRuntimeService: + stop_event = getattr(self, "stop_event", None) + if stop_event is None: + stop_event = threading.Event() + self.stop_event = stop_event + runtime = CtvRuntimeService( + rpc_call=lambda *args, **kwargs: self.rpc.call(*args, **kwargs), + ledger=getattr(self, "ledger", None), + writer_admission=lambda component: self._writer_operation(component), + tip_refresh_pending=lambda: self.tip_refresh_is_pending(), + heartbeat=lambda: self._record_heartbeat("ctv_fanout_broadcaster"), + stop_event=stop_event, + config=self._legacy_ctv_runtime_config() if config is None else config, + daemon_type=CtvFanoutBroadcastDaemon, + broadcaster_type=CtvFanoutBroadcaster, + # Preserve temporary coordinator patch points for focused tests. + monotonic=lambda: time.monotonic(), + print_exception=lambda: traceback.print_exc(), + ) + compat_daemon = self.__dict__.pop("_ctv_runtime_compat_daemon", None) + if compat_daemon is not None: + runtime.daemon = compat_daemon + return runtime + + def _ensure_ctv_runtime(self) -> CtvRuntimeService: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + return runtime + init_lock = self.__dict__.get("_ctv_runtime_init_lock") + if init_lock is None: + # CPython's setdefault is atomic under the GIL. Focused tests may + # construct through __new__, while normal instances install this + # lock in __init__ before any process thread can start. + init_lock = self.__dict__.setdefault( + "_ctv_runtime_init_lock", + threading.Lock(), + ) + with init_lock: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + return runtime + config = self._legacy_ctv_runtime_config() + runtime = self._make_ctv_runtime_service(config) + self.__dict__["_ctv_runtime"] = runtime + # The service is now the sole configuration owner. Removing the + # pre-init store prevents an old override from being replayed if a + # later compatibility property updates the live service. + self.__dict__.pop("_ctv_runtime_compat_config", None) + return runtime + + def _ctv_runtime_config_value(self, name: str) -> object: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + return getattr(runtime.config, name) + return getattr(self._legacy_ctv_runtime_config(), name) + + def _set_ctv_runtime_config_value(self, name: str, value: object) -> None: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + runtime.replace_config(**{name: value}) + return + init_lock = self.__dict__.get("_ctv_runtime_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_ctv_runtime_init_lock", + threading.Lock(), + ) + with init_lock: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + runtime.replace_config(**{name: value}) + return + overrides = self.__dict__.setdefault("_ctv_runtime_compat_config", {}) + overrides[name] = value + + @property + def ctv_broadcaster_enabled(self) -> bool: + return bool(self._ctv_runtime_config_value("enabled")) + + @ctv_broadcaster_enabled.setter + def ctv_broadcaster_enabled(self, value: bool) -> None: + self._set_ctv_runtime_config_value("enabled", bool(value)) + + @property + def ctv_broadcaster_wallet(self) -> str | None: + value = self._ctv_runtime_config_value("wallet") + return None if value is None else str(value) + + @ctv_broadcaster_wallet.setter + def ctv_broadcaster_wallet(self, value: str | None) -> None: + self._set_ctv_runtime_config_value("wallet", value) + + @property + def ctv_broadcaster_fee_sats(self) -> int: + return int(self._ctv_runtime_config_value("fee_sats")) + + @ctv_broadcaster_fee_sats.setter + def ctv_broadcaster_fee_sats(self, value: int) -> None: + self._set_ctv_runtime_config_value("fee_sats", int(value)) + + @property + def ctv_broadcaster_limit(self) -> int: + return int(self._ctv_runtime_config_value("limit")) + + @ctv_broadcaster_limit.setter + def ctv_broadcaster_limit(self, value: int) -> None: + self._set_ctv_runtime_config_value("limit", int(value)) + + @property + def ctv_broadcaster_chunk_size(self) -> int: + return int(self._ctv_runtime_config_value("chunk_size")) + + @ctv_broadcaster_chunk_size.setter + def ctv_broadcaster_chunk_size(self, value: int) -> None: + self._set_ctv_runtime_config_value("chunk_size", int(value)) + + @property + def ctv_broadcaster_interval_seconds(self) -> float: + return float(self._ctv_runtime_config_value("interval_seconds")) + + @ctv_broadcaster_interval_seconds.setter + def ctv_broadcaster_interval_seconds(self, value: float) -> None: + self._set_ctv_runtime_config_value("interval_seconds", float(value)) + + @property + def ctv_fanout_broadcast_daemon(self) -> CtvFanoutBroadcastDaemon | None: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + return runtime.daemon + init_lock = self.__dict__.get("_ctv_runtime_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_ctv_runtime_init_lock", + threading.Lock(), + ) + with init_lock: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is None: + return self.__dict__.get("_ctv_runtime_compat_daemon") + return runtime.daemon + + @ctv_fanout_broadcast_daemon.setter + def ctv_fanout_broadcast_daemon( + self, + daemon: CtvFanoutBroadcastDaemon | None, + ) -> None: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is not None: + runtime.daemon = daemon + return + init_lock = self.__dict__.get("_ctv_runtime_init_lock") + if init_lock is None: + init_lock = self.__dict__.setdefault( + "_ctv_runtime_init_lock", + threading.Lock(), + ) + with init_lock: + runtime = self.__dict__.get("_ctv_runtime") + if runtime is None: + self.__dict__["_ctv_runtime_compat_daemon"] = daemon + else: + runtime.daemon = daemon + + @property + def ctv_broadcaster_processed_rows_total(self) -> int: + return self._ensure_ctv_runtime().processed_rows_total + + @property + def ctv_broadcaster_pass_count(self) -> int: + return self._ensure_ctv_runtime().pass_count + def _ensure_ctv_broadcaster_metrics_state(self) -> None: - if not hasattr(self, "_ctv_broadcaster_metrics_lock"): - self._ctv_broadcaster_metrics_lock = threading.Lock() - if not hasattr(self, "ctv_broadcaster_pass_seconds_bucket_counts"): - self.ctv_broadcaster_pass_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS - } - if not hasattr(self, "ctv_broadcaster_pass_seconds_sum"): - self.ctv_broadcaster_pass_seconds_sum = 0.0 - if not hasattr(self, "ctv_broadcaster_pass_count"): - self.ctv_broadcaster_pass_count = 0 - if not hasattr(self, "ctv_broadcaster_processed_rows_total"): - self.ctv_broadcaster_processed_rows_total = 0 - if not hasattr(self, "ctv_broadcaster_yielded_total"): - self.ctv_broadcaster_yielded_total = 0 - if not hasattr(self, "ctv_broadcaster_chunk_seconds_bucket_counts"): - self.ctv_broadcaster_chunk_seconds_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS - } - if not hasattr(self, "ctv_broadcaster_chunk_rows_bucket_counts"): - self.ctv_broadcaster_chunk_rows_bucket_counts = { - bucket: 0 for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS - } - if not hasattr(self, "ctv_broadcaster_chunk_seconds_sum"): - self.ctv_broadcaster_chunk_seconds_sum = 0.0 - if not hasattr(self, "ctv_broadcaster_chunk_rows_sum"): - self.ctv_broadcaster_chunk_rows_sum = 0 - if not hasattr(self, "ctv_broadcaster_chunk_count"): - self.ctv_broadcaster_chunk_count = 0 + self._ensure_ctv_runtime() def _record_ctv_fanout_broadcaster_progress(self) -> None: - self._record_heartbeat("ctv_fanout_broadcaster") - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - self.ctv_broadcaster_processed_rows_total += 1 + self._ensure_ctv_runtime().record_progress() def observe_ctv_fanout_broadcaster_pass(self, elapsed_seconds: float) -> None: - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - self.ctv_broadcaster_pass_count += 1 - self.ctv_broadcaster_pass_seconds_sum += elapsed_seconds - for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - self.ctv_broadcaster_pass_seconds_bucket_counts[bucket] += 1 + self._ensure_ctv_runtime().observe_pass(elapsed_seconds) def observe_ctv_fanout_broadcaster_chunk( self, result: CtvFanoutChunkResult, ) -> None: - self._record_heartbeat("ctv_fanout_broadcaster") - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - self.ctv_broadcaster_chunk_count += 1 - self.ctv_broadcaster_chunk_seconds_sum += result.elapsed_seconds - self.ctv_broadcaster_chunk_rows_sum += result.processed_count - for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS: - if result.elapsed_seconds <= bucket: - self.ctv_broadcaster_chunk_seconds_bucket_counts[bucket] += 1 - for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS: - if result.processed_count <= bucket: - self.ctv_broadcaster_chunk_rows_bucket_counts[bucket] += 1 + self._ensure_ctv_runtime().observe_chunk(result) def _record_ctv_fanout_broadcaster_yield(self) -> None: - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - self.ctv_broadcaster_yielded_total += 1 + self._ensure_ctv_runtime().record_yield() def _ensure_worker_metrics_state(self) -> None: if not hasattr(self, "worker_metrics_lock"): @@ -7089,55 +3898,77 @@ def _ensure_worker_metrics_state(self) -> None: if not hasattr(self, "worker_rejection_counts"): self.worker_rejection_counts = {} - def _ensure_initial_job_state(self) -> None: - if not hasattr(self, "pending_initial_jobs"): - self.pending_initial_jobs: dict[ClientState, PendingInitialJob] = {} - if not hasattr(self, "stratum_max_pending_initial_jobs"): - self.stratum_max_pending_initial_jobs = ( - DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS - ) - if not hasattr(self, "stratum_initial_job_timeout_seconds"): - self.stratum_initial_job_timeout_seconds = ( - DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS - ) - if not hasattr(self, "initial_job_max_workers"): - self.initial_job_max_workers = DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS - if not hasattr(self, "_initial_job_executor_lock"): - self._initial_job_executor_lock = threading.Lock() - if not hasattr(self, "_initial_job_executor"): - self._initial_job_executor: _BoundedPriorityExecutor | None = None - if not hasattr(self, "_initial_job_executor_shutdown"): - self._initial_job_executor_shutdown = False - if not hasattr(self, "initial_job_queue_rejection_count"): - self.initial_job_queue_rejection_count = 0 - if not hasattr(self, "initial_job_timeout_count"): - self.initial_job_timeout_count = 0 - if not hasattr(self, "initial_job_cancelled_count"): - self.initial_job_cancelled_count = 0 - if not hasattr(self, "initial_job_coalesced_count"): - self.initial_job_coalesced_count = 0 - if not hasattr(self, "initial_job_queue_capacity_reclaimed_count"): - self.initial_job_queue_capacity_reclaimed_count = 0 - if not hasattr(self, "initial_job_sent_count"): - self.initial_job_sent_count = 0 - if not hasattr(self, "initial_job_failed_count"): - self.initial_job_failed_count = 0 - if not hasattr(self, "initial_job_superseded_count"): - self.initial_job_superseded_count = 0 - if not hasattr(self, "initial_job_delivery_latency_seconds_sum"): - self.initial_job_delivery_latency_seconds_sum = 0.0 - if not hasattr(self, "initial_job_delivery_latency_count"): - self.initial_job_delivery_latency_count = 0 - if not hasattr(self, "last_initial_job_delivery_monotonic"): - self.last_initial_job_delivery_monotonic = None - if not hasattr(self, "handler_thread_count"): - self.handler_thread_count = 0 - if not hasattr(self, "peak_active_connection_count"): - self.peak_active_connection_count = len(getattr(self, "clients", ())) - if not hasattr(self, "_mining_overload_started_monotonic"): - self._mining_overload_started_monotonic = None + def _ensure_initial_job_state(self) -> InitialJobState: + state = self.__dict__.get("_initial_job_state") + if state is None: + pending = self.__dict__.get("_pending_initial_jobs_compat") + if pending is None: + pending = {} + self.__dict__["_pending_initial_jobs_compat"] = pending + state = InitialJobState( + InitialJobConfig( + max_pending=int( + self.__dict__.get( + "_max_pending_compat", + DEFAULT_PRISM_STRATUM_MAX_PENDING_INITIAL_JOBS, + ) + ), + timeout_seconds=float( + self.__dict__.get( + "_timeout_seconds_compat", + DEFAULT_PRISM_STRATUM_INITIAL_JOB_TIMEOUT_SECONDS, + ) + ), + max_workers=int( + self.__dict__.get( + "_max_workers_compat", + DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, + ) + ), + ), + pending, + ) + state.queue_rejection_count = int( + self.__dict__.get("_queue_rejection_count_compat", 0) + ) + state.timeout_count = int(self.__dict__.get("_timeout_count_compat", 0)) + state.cancelled_count = int( + self.__dict__.get("_cancelled_count_compat", 0) + ) + state.coalesced_count = int( + self.__dict__.get("_coalesced_count_compat", 0) + ) + state.sent_count = int(self.__dict__.get("_sent_count_compat", 0)) + state.failed_count = int(self.__dict__.get("_failed_count_compat", 0)) + state.superseded_count = int( + self.__dict__.get("_superseded_count_compat", 0) + ) + state.queue_capacity_reclaimed_count = int( + self.__dict__.get( + "_queue_capacity_reclaimed_count_compat", + 0, + ) + ) + state.delivery_latency_seconds_sum = float( + self.__dict__.get("_delivery_latency_seconds_sum_compat", 0.0) + ) + state.delivery_latency_count = int( + self.__dict__.get("_delivery_latency_count_compat", 0) + ) + state.last_delivery_monotonic = self.__dict__.get( + "_last_delivery_monotonic_compat" + ) + state = self.__dict__.setdefault("_initial_job_state", state) + self.__dict__["_initial_job_tracker"] = state.tracker + if not hasattr(self, "handler_thread_count"): + self.handler_thread_count = 0 + if not hasattr(self, "peak_active_connection_count"): + self.peak_active_connection_count = len(getattr(self, "clients", ())) + if not hasattr(self, "_mining_overload_started_monotonic"): + self._mining_overload_started_monotonic = None if not hasattr(self, "_mining_delivery_failure_started_monotonic"): self._mining_delivery_failure_started_monotonic = None + return state def delivery_queue_limit(self) -> int: pending_limit = int( @@ -7160,92 +3991,168 @@ def delivery_queue_limit(self) -> int: connection_limit if connection_limit > 0 else pending_limit, ) + def _tip_refresh_prune_evicted_jobs( + self, + now: float, + force: bool, + ) -> None: + """Resolve the retained-prune compatibility seam at call time.""" + override = self.__dict__.get("prune_evicted_job_graveyard") + if callable(override): + override(now=now, force=force) + return + self._ensure_job_delivery_service().prune_retained( + now=now, + force=force, + ) + + def _ensure_tip_refresh_service(self) -> TipRefreshService: + service = self.__dict__.get("_tip_refresh_service") + if service is not None: + return service + init_lock = self.__dict__.setdefault( + "_tip_refresh_service_init_lock", + threading.Lock(), + ) + with init_lock: + service = self.__dict__.get("_tip_refresh_service") + if service is not None: + return service + service = TipRefreshService( + TipRefreshConfig( + blockpoll_seconds=float( + getattr(self, "blockpoll_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) + ), + blockwait_timeout_seconds=float( + getattr( + self, + "blockwait_timeout_seconds", + DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, + ) + ), + failure_holdoff_seconds=float( + getattr( + self, + "tip_refresh_failure_holdoff_seconds", + DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, + ) + ), + max_workers=int( + getattr( + self, + "tip_refresh_max_workers", + DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS, + ) + ), + submit_tip_max_age_seconds=float( + getattr( + self, + "submit_tip_max_age_seconds", + DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, + ) + ), + failure_exit_seconds=float( + getattr( + self, + "template_refresh_failure_exit_seconds", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) + ), + watchdog_timeout_seconds=float( + getattr(self, "watchdog_timeout_seconds", 120.0) + ), + payout_reconcile_supersession_retries=int( + getattr( + self, + "payout_reconcile_supersession_retries", + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + ) + ), + ), + TipRefreshPorts( + rpc_call=self._tip_refresh_rpc_call, + rpc_call_with_timeout=lambda method, params, timeout: ( + self.rpc.call(method, params, timeout=timeout) + ), + payout_state=self._ensure_payout_state_service, + job_bundles=self._ensure_job_bundle_service, + delivery=JobDeliveryTipRefreshPort( + registry=self._ensure_session_registry, + delivery=self._ensure_job_delivery_service(), + submit_task=self._submit_delivery_task, + disconnect=lambda client: self.disconnect_client(client), + ), + mark_progress_pending=self._progress_note_refresh_pending, + observe_progress_tip_poll=self._record_progress_tip_poll, + publish_progress_work=self._record_progress_publication, + start_progress_refresh=lambda: ( + self._ensure_progress_health_service().start_refresh() + ), + cancel_obsolete_bundle_builds=lambda tip, generation: ( + self._cancel_obsolete_job_bundle_builds( + current_tip=tip, + payout_state_generation=generation, + ) + ), + cancel_obsolete_job_builds=self._cancel_obsolete_job_builds, + prune_evicted_jobs=self._tip_refresh_prune_evicted_jobs, + delivery_queue_limit=self.delivery_queue_limit, + stop_requested=self._tip_refresh_stop_requested, + heartbeat=self._record_heartbeat, + remove_heartbeat=self._remove_watchdog_heartbeat, + chain_view_untrusted=lambda: bool( + getattr(self, "reorg_reconciler_enabled", True) + and self.qbit_chain_view_untrusted() + ), + ensure_reorg_current=lambda tip: ( + self.ensure_reorg_reconciled_for_current_tip( + expected_tip_hash=tip + ) + ), + observe_job_build_elapsed=self.observe_job_build_elapsed, + fetch_snapshot=lambda: ( + self._ensure_job_bundle_service() + .template_repository.fetch_coherent_snapshot() + ), + ensure_reorg_tip=lambda tip: self.ensure_reorg_reconciled_for_tip(tip), + wait_for_execution_permit=lambda timeout: ( + self._ensure_shutdown_controller().wait_for_no_active_writer( + {"accepted_block_handling"}, + timeout, + ) + ), + wait_for_stop=self._tip_refresh_wait_for_stop, + hard_exit=lambda code: os._exit(code), + fetch_snapshot_for_tip=lambda observed_tip: ( + self._ensure_job_bundle_service() + .template_repository.fetch_coherent_snapshot(observed_tip) + ), + ), + monotonic=lambda: time.monotonic(), + state_lock=self._ensure_session_registry().lock, + ) + object.__setattr__(self, "_tip_refresh_service", service) + return service + + def _tip_refresh_rpc_call( + self, + method: str, + params: list[object] | None, + ) -> object: + if params is None: + return self.rpc.call(method) + return self.rpc.call(method, params) + + def _tip_refresh_stop_requested(self) -> bool: + stop_event = getattr(self, "stop_event", None) + return bool(stop_event is not None and stop_event.is_set()) + + def _tip_refresh_wait_for_stop(self, seconds: float) -> bool: + stop_event = getattr(self, "stop_event", None) + return bool(stop_event is not None and stop_event.wait(seconds)) + def _ensure_tip_refresh_state(self) -> None: - if not hasattr(self, "_tip_refresh_lock"): - self._tip_refresh_lock = threading.Lock() - if not hasattr(self, "_tip_refresh_singleflight_lock"): - self._tip_refresh_singleflight_lock = threading.Lock() - if not hasattr(self, "_tip_refresh_executor_lock"): - self._tip_refresh_executor_lock = threading.Lock() - if not hasattr(self, "_tip_refresh_executor"): - self._tip_refresh_executor: _BoundedPriorityExecutor | None = None - if not hasattr(self, "_tip_refresh_executor_shutdown"): - self._tip_refresh_executor_shutdown = False - if not hasattr(self, "tip_refresh_max_workers"): - self.tip_refresh_max_workers = DEFAULT_PRISM_TIP_REFRESH_MAX_WORKERS - if not hasattr(self, "_tip_refresh_metrics_lock"): - self._tip_refresh_metrics_lock = threading.Lock() - if not hasattr(self, "tip_refresh_histograms"): - self.tip_refresh_histograms = { - name: { - "buckets": {bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS}, - "sum": 0.0, - "count": 0, - } - for name in ("refresh", "bundle_build", "first_delivery", "last_delivery") - } - if not hasattr(self, "tip_refresh_build_phase_histograms"): - self.tip_refresh_build_phase_histograms = { - phase: { - "buckets": { - bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - }, - "sum": 0.0, - "count": 0, - } - for phase in PRISM_TIP_REFRESH_BUILD_PHASES - } - if not hasattr(self, "tip_refresh_client_counts"): - self.tip_refresh_client_counts = { - result: 0 for result in PRISM_TIP_REFRESH_RESULTS - } - if not hasattr(self, "tip_refresh_cancellation_counts"): - self.tip_refresh_cancellation_counts = { - stage: 0 for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES - } - if not hasattr(self, "tip_refresh_inflight"): - self.tip_refresh_inflight = 0 - if not hasattr(self, "tip_refresh_build_inflight"): - self.tip_refresh_build_inflight = 0 - if not hasattr(self, "tip_refresh_build_queue_depth"): - self.tip_refresh_build_queue_depth = 0 - if not hasattr(self, "tip_refresh_singleflight_hits"): - self.tip_refresh_singleflight_hits = 0 - if not hasattr(self, "tip_refresh_superseded_results"): - self.tip_refresh_superseded_results = 0 - if not hasattr(self, "tip_refresh_worker_failures"): - self.tip_refresh_worker_failures = 0 - if not hasattr(self, "tip_refresh_worker_restarts"): - self.tip_refresh_worker_restarts = 0 - if not hasattr(self, "tip_refresh_ipc_bytes"): - self.tip_refresh_ipc_bytes = {"input": 0, "output": 0} - if not hasattr(self, "_tip_refresh_pending_event"): - self._tip_refresh_pending_event = threading.Event() - if not hasattr(self, "_tip_refresh_pending_counter"): - self._tip_refresh_pending_counter = 0 - if not hasattr(self, "_tip_refresh_pending_token"): - self._tip_refresh_pending_token: int | None = None - if not hasattr(self, "_tip_refresh_retry"): - self._tip_refresh_retry = threading.Event() - if not hasattr(self, "_tip_refresh_retry_counter"): - self._tip_refresh_retry_counter = 0 - if not hasattr(self, "_tip_refresh_retry_consumed"): - self._tip_refresh_retry_consumed = 0 - if not hasattr(self, "_tip_refresh_failure_holdoff_until"): - self._tip_refresh_failure_holdoff_until: float | None = None - if not hasattr(self, "_tip_refresh_failure_tip"): - self._tip_refresh_failure_tip: str | None = None - if not hasattr(self, "_active_tip_refresh"): - self._active_tip_refresh: tuple[ - TipRefreshValidationToken, - _FanoutCancellation, - ] | None = None - if not hasattr(self, "_retained_collection_refresh"): - self._retained_collection_refresh: RetainedCollectionRefresh | None = None - if not hasattr(self, "latest_detected_tip"): - self.latest_detected_tip: tuple[str, int] | None = None - if not hasattr(self, "tip_refresh_divergence_started_monotonic"): - self.tip_refresh_divergence_started_monotonic: float | None = None + self._ensure_tip_refresh_service() def _retain_collection_refresh( self, @@ -7253,36 +4160,11 @@ def _retain_collection_refresh( observation_sequence: int, payout_state_generation: int, ) -> None: - """Retain reusable current work until an eligible identity appears.""" - retained = RetainedCollectionRefresh( - snapshot=snapshot, - observation_sequence=observation_sequence, - payout_state_generation=payout_state_generation, + self._ensure_tip_refresh_service().retain_collection_refresh( + snapshot, + observation_sequence, + payout_state_generation, ) - should_log = False - with self.lock: - if not self._tip_refresh_snapshot_current_locked( - snapshot, - observation_sequence, - ): - return - if any( - self.client_can_receive_jobs(client) - for client in self.clients - ): - return - previous = self._retained_collection_refresh - self._retained_collection_refresh = retained - should_log = previous is None or ( - previous.snapshot.bestblockhash != snapshot.bestblockhash - or previous.payout_state_generation != payout_state_generation - ) - if should_log: - print( - "prism coordinator: collection refresh retained while no " - "authorized worker identity is available", - flush=True, - ) def _retained_collection_artifacts(self) -> CachedTemplateArtifacts | None: """Return retained artifacts while their published work stays current. @@ -7293,110 +4175,35 @@ def _retained_collection_artifacts(self) -> CachedTemplateArtifacts | None: not yet been updated; a new tip or payout generation still invalidates it immediately. """ - self._ensure_job_cache_state() - self._ensure_tip_refresh_state() - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - with self.lock: - retained = self._retained_collection_refresh - if retained is None: - return None - if retained.payout_state_generation != payout_state_generation: - return None - current_tip = getattr(self, "current_tip_first_seen", None) - published_snapshot = self.tip_template_snapshot - if ( - published_snapshot is None - or current_tip is None - or current_tip[0] != published_snapshot.bestblockhash - ): - return None - return self._tip_refresh_artifacts(published_snapshot) + return self._ensure_tip_refresh_service().retained_collection_artifacts() def _retain_current_collection_refresh_if_unrepresented(self) -> None: """Keep the last published collection work when the fleet empties.""" - self._ensure_tip_refresh_state() - if getattr(self, "_pool_ready_latched", False): - return - with self.lock: - if any( - self.client_can_receive_jobs(client) - for client in self.clients - ): - return - snapshot = self.tip_template_snapshot - observation_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if snapshot is None: - return - self._ensure_job_cache_state() - with self._job_cache_lock: - payout_state_generation = self._payout_state_generation - self._retain_collection_refresh( - snapshot, - observation_sequence, - payout_state_generation, - ) + self._ensure_tip_refresh_service().retain_current_collection_refresh_if_unrepresented() def _note_collection_identity_available(self, client: ClientState) -> None: """Wake a retained collection refresh as soon as a client is eligible.""" - if not self.client_can_receive_jobs(client): - return - if self._retained_collection_artifacts() is None: - return - self._mark_tip_refresh_pending(client.connection_id) - self._schedule_tip_refresh_retry() + self._ensure_tip_refresh_service().note_collection_identity_available(client) def _consume_retained_collection_refresh( self, context: PrismJobContext, ) -> None: """Consume retention only after its collection work was delivered.""" - if not context.collection_only: - return - with self.lock: - retained = self._retained_collection_refresh - published_snapshot = self.tip_template_snapshot - artifacts = ( - published_snapshot.template_artifacts - if published_snapshot is not None - else None - ) - if ( - retained is not None - and retained.payout_state_generation - == context.payout_state_generation - and artifacts is not None - and context.template is artifacts.template - and context.template_fingerprint == artifacts.fingerprint - and context.template_generation == artifacts.generation - ): - self._retained_collection_refresh = None + self._ensure_tip_refresh_service().consume_retained_collection_refresh(context) def tip_refresh_is_pending(self) -> bool: return self._tip_refresh_pending() def _tip_refresh_pending(self) -> bool: - self._ensure_tip_refresh_state() - return self._tip_refresh_pending_event.is_set() + return self._ensure_tip_refresh_service().pending() def _mark_tip_refresh_pending(self, _observation: object) -> int: - self._ensure_tip_refresh_state() - with self.lock: - self._tip_refresh_pending_counter += 1 - token = self._tip_refresh_pending_counter - self._tip_refresh_pending_token = token - self._tip_refresh_pending_event.set() - return token + return self._ensure_tip_refresh_service().mark_pending(_observation) def _claim_tip_refresh_pending(self) -> int | None: """Snapshot pending work without replacing a newer producer's token.""" - self._ensure_tip_refresh_state() - with self.lock: - if not self._tip_refresh_pending_event.is_set(): - return None - return self._tip_refresh_pending_token + return self._ensure_tip_refresh_service().claim_pending() def _mark_tip_refresh_pending_for_poll( self, @@ -7404,25 +4211,13 @@ def _mark_tip_refresh_pending_for_poll( _observation: object, ) -> int | None: """Mark poll-owned work only while no newer producer has superseded it.""" - self._ensure_tip_refresh_state() - with self.lock: - if self._tip_refresh_pending_token != owned_token: - return owned_token - if owned_token is not None: - self._tip_refresh_pending_event.set() - return owned_token - self._tip_refresh_pending_counter += 1 - token = self._tip_refresh_pending_counter - self._tip_refresh_pending_token = token - self._tip_refresh_pending_event.set() - return token + return self._ensure_tip_refresh_service().mark_pending_for_poll( + owned_token, + _observation, + ) def _clear_tip_refresh_pending(self, token: int) -> None: - self._ensure_tip_refresh_state() - with self.lock: - if self._tip_refresh_pending_token == token: - self._tip_refresh_pending_token = None - self._tip_refresh_pending_event.clear() + self._ensure_tip_refresh_service().clear_pending(token) def _clear_tip_refresh_pending_for_completed_refresh( self, @@ -7432,305 +4227,72 @@ def _clear_tip_refresh_pending_for_completed_refresh( pending_signal_token: int | None = None, ) -> bool: """Atomically acknowledge pending work handled by a completed poll.""" - self._ensure_job_cache_state() - with self._payout_state_delivery_gate.delivery_cancelable( - lambda: self._payout_state_generation != payout_state_generation, - generation=payout_state_generation, - priority=True, - ) as admission: - if not admission: - return False - with self._job_cache_lock: - payout_state_current = ( - self._payout_state_generation == payout_state_generation - ) - with self.lock: - refresh_current = self._tip_refresh_snapshot_current_locked( - snapshot, - observation_sequence, - ) - pending_owned = ( - self._tip_refresh_pending_token == pending_signal_token - ) - if ( - not payout_state_current - or not refresh_current - or not pending_owned - ): - return False - self._tip_refresh_pending_token = None - self._tip_refresh_pending_event.clear() - with self._progress_health_lock: - published_current = bool( - self._progress_has_published_work - and self._progress_published_template_fingerprint - == snapshot.template_fingerprint - and self._progress_published_payout_generation - == payout_state_generation - ) - if published_current: - # The completion guard above proves this coherent - # snapshot still represents the latest detected and - # published tip. A transient A -> B -> A observation - # therefore closes its publication-divergence epoch - # even when existing A work needed no new delivery. - self._progress_refresh_signal_pending = False - self._progress_publication_divergence_since_monotonic = None - return True - - def _schedule_tip_refresh_retry(self) -> None: - self._ensure_tip_refresh_state() - # Pair the Event with a monotonic generation so a producer cannot set - # it between a waiter's wake and clear and lose the newest retry. The - # event remains the blocking primitive; the generation is the durable - # coalesced work marker. - with self.lock: - self._tip_refresh_retry_counter += 1 - self._tip_refresh_retry.set() - - def _consume_tip_refresh_retry(self) -> bool: - """Consume all retry signals visible at one atomic wake boundary.""" - self._ensure_tip_refresh_state() - with self.lock: - generation = self._tip_refresh_retry_counter - if generation == self._tip_refresh_retry_consumed: - return False - self._tip_refresh_retry_consumed = generation - self._tip_refresh_retry.clear() - return True - - def _note_tip_refresh_attempt_failed( - self, - observed_tip: str | None = None, - ) -> None: - """Stamp a failed refresh pass so the poller spaces its re-attempt. - - ``observed_tip`` is the tip the failed pass worked against; a pass - that failed before learning one stamps the current observation so an - RPC outage with a static tip still gets spaced. - """ - self._ensure_tip_refresh_state() - holdoff = float( - getattr( - self, - "tip_refresh_failure_holdoff_seconds", - DEFAULT_PRISM_TIP_REFRESH_FAILURE_HOLDOFF_SECONDS, - ) - ) - if holdoff <= 0: - return - holdoff += random.uniform( - 0.0, - holdoff * PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION, + return self._ensure_tip_refresh_service().clear_pending_for_completed_refresh( + snapshot, + observation_sequence, + payout_state_generation, + pending_signal_token, ) - with self.lock: - if observed_tip is None: - observed_tip = self._newest_observed_tip_locked() - self._tip_refresh_failure_tip = observed_tip - self._tip_refresh_failure_holdoff_until = time.monotonic() + holdoff - - def _clear_tip_refresh_failure_holdoff(self) -> None: - self._ensure_tip_refresh_state() - with self.lock: - self._tip_refresh_failure_holdoff_until = None - self._tip_refresh_failure_tip = None - - def _tip_refresh_failure_holdoff_remaining(self) -> float: - """Seconds the poller must still wait before re-running a failed pass. - Zero as soon as the newest known tip differs from the one the failed - pass worked against: spacing throttles re-attempts against an - unchanged, blocked or churning view, never the reaction to a - genuinely new tip (including one detected but not yet published). - """ - self._ensure_tip_refresh_state() - with self.lock: - deadline = self._tip_refresh_failure_holdoff_until - failed_tip = self._tip_refresh_failure_tip - current_hash = self._newest_observed_tip_locked() - if deadline is None: - return 0.0 - if current_hash != failed_tip: - return 0.0 - return max(0.0, deadline - time.monotonic()) + def _schedule_tip_refresh_retry(self) -> None: + self._ensure_tip_refresh_service().schedule_retry() def _observe_tip_refresh_seconds(self, name: str, elapsed_seconds: float) -> None: - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - histogram = self.tip_refresh_histograms[name] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + elapsed_seconds - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + self._ensure_tip_refresh_service().observe_seconds(name, elapsed_seconds) def _observe_tip_refresh_build_phase( self, phase: str, elapsed_seconds: float, ) -> None: - if phase not in PRISM_TIP_REFRESH_BUILD_PHASES: - raise ValueError(f"unknown tip refresh build phase: {phase}") - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - histogram = self.tip_refresh_build_phase_histograms[phase] - histogram["count"] = int(histogram["count"]) + 1 - histogram["sum"] = float(histogram["sum"]) + max( - 0.0, elapsed_seconds - ) - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: - if elapsed_seconds <= bucket: - buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + self._ensure_tip_refresh_service().observe_build_phase( + phase, + elapsed_seconds, + ) def _record_tip_refresh_ipc_bytes(self, direction: str, byte_count: int) -> None: - if direction not in {"input", "output"}: - raise ValueError(f"unknown tip refresh IPC direction: {direction}") - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_ipc_bytes[direction] += max(0, int(byte_count)) + self._ensure_tip_refresh_service().record_ipc_bytes(direction, byte_count) def _record_tip_refresh_client_result(self, result: str) -> None: - if result not in PRISM_TIP_REFRESH_RESULTS: - raise ValueError(f"unknown tip refresh result: {result}") - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_client_counts[result] += 1 + self._ensure_tip_refresh_service().record_client_result(result) def _record_tip_refresh_cancellation(self, stage: str) -> None: - if stage not in PRISM_TIP_REFRESH_CANCELLATION_STAGES: - raise ValueError(f"unknown tip refresh cancellation stage: {stage}") - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_cancellation_counts[stage] += 1 + self._ensure_tip_refresh_service().record_cancellation(stage) def _tip_refresh_future_started(self) -> None: - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_inflight += 1 + self._ensure_tip_refresh_service().future_started() def _tip_refresh_future_finished(self, _future: Future[RefreshResult]) -> None: - self._ensure_tip_refresh_state() - with self._tip_refresh_metrics_lock: - self.tip_refresh_inflight = max(0, self.tip_refresh_inflight - 1) + self._ensure_tip_refresh_service().future_finished(_future) def tip_refresh_executor(self) -> _BoundedPriorityExecutor: - self._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - if self._tip_refresh_executor_shutdown: - raise RuntimeError("tip refresh executor is shut down") - executor = self._tip_refresh_executor - if executor is None: - executor = _BoundedPriorityExecutor( - max_workers=self.tip_refresh_max_workers, - max_queue_size=self.delivery_queue_limit(), - thread_name_prefix="prism-tip-refresh-delivery", - ) - self._tip_refresh_executor = executor - return executor + return self._ensure_tip_refresh_service().executor() def initial_job_executor(self) -> _BoundedPriorityExecutor: - self._ensure_initial_job_state() - with self._initial_job_executor_lock: - if self._initial_job_executor_shutdown: - raise RuntimeError("initial job executor is shut down") - executor = self._initial_job_executor - if executor is None: - executor = _BoundedPriorityExecutor( - max_workers=self.initial_job_max_workers, - max_queue_size=self.stratum_max_pending_initial_jobs, - thread_name_prefix="prism-initial-job-delivery", - ) - self._initial_job_executor = executor - return executor + return self._ensure_job_delivery_service().initial_executor() def shutdown_initial_job_executor(self) -> None: - self._ensure_initial_job_state() - with self.lock: - for request in tuple(self.pending_initial_jobs.values()): - if self.pending_initial_jobs.get(request.client) is not request: - continue - self.pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - if request.future is not None: - self._cancel_initial_job_future(request.future) - self.initial_job_cancelled_count += 1 - with self._initial_job_executor_lock: - executor = self._initial_job_executor - self._initial_job_executor = None - self._initial_job_executor_shutdown = True - if executor is not None: - # Running workers observe request cancellation before shutdown - # returns; queued reconnect work is cancelled without starting. - executor.shutdown(wait=True, cancel_futures=True) + self._ensure_job_delivery_service().shutdown_initial_executor() + + def _cancel_initial_job_future(self, future: Future[bool]) -> bool: + return self._ensure_job_delivery_service().cancel_initial_future(future) def shutdown_tip_refresh_executor(self) -> None: - self._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - executor = self._tip_refresh_executor - self._tip_refresh_executor = None - self._tip_refresh_executor_shutdown = True - if executor is not None: - # Stop queued publication work before waiting on reconnect workers. - executor.shutdown(wait=False, cancel_futures=True) - self.shutdown_initial_job_executor() - if executor is not None: - # Running workers may already hold client/job state or be inside a - # socket send. Drain them before serve returns and the writer lease - # is released; queued workers are cancelled without starting. - executor.shutdown(wait=True) + self._ensure_job_delivery_service().shutdown_initial_executor() + self._ensure_tip_refresh_service().shutdown() + # These owners must close even if the refresh scheduler exceeded its + # bounded join. Closing them cancels work that can otherwise keep the + # scheduler, its non-daemon executor workers, and the process alive. self.shutdown_job_build_executor() self.shutdown_payout_artifact_executor() - def _cancel_initial_job_future(self, future: Future[bool]) -> bool: - """Cancel one initial-job future and account physical queue removal.""" - executor = getattr(self, "_initial_job_executor", None) - reclaimed = bool( - executor is not None - and executor.cancel(future) - ) - if executor is None: - future.cancel() - if reclaimed: - with self.lock: - self._ensure_initial_job_state() - self.initial_job_queue_capacity_reclaimed_count += 1 - return reclaimed - def _initial_request_current_locked(self, request: PendingInitialJob) -> bool: - client = request.client - deadline = request.deadline_monotonic - return ( - self.pending_initial_jobs.get(client) is request - and client in self.clients - and not getattr(client, "closing", False) - and ( - request.connection_id is None - or client.connection_id == request.connection_id - ) - and client.authorized - and client.subscribed - and client.worker == request.worker - and int(getattr(client, "authorization_generation", 0)) - == request.authorization_generation - and ( - request.difficulty_generation is None - or int(getattr(client, "difficulty_generation", 0)) - == request.difficulty_generation - ) - and (deadline is None or time.monotonic() < deadline) - and not request.cancelled.is_set() + return self._ensure_job_delivery_service().initial_request_current_locked( + request ) def _initial_request_cancelled(self, request: PendingInitialJob) -> bool: - if request.cancelled.is_set() or self.stop_event.is_set(): - return True - with self.lock: - self._ensure_initial_job_state() - return not self._initial_request_current_locked(request) + return self._ensure_job_delivery_service().initial_request_cancelled(request) def _cancel_pending_initial_job_locked( self, @@ -7738,60 +4300,15 @@ def _cancel_pending_initial_job_locked( *, count: bool, ) -> PendingInitialJob | None: - self._ensure_initial_job_state() - request = self.pending_initial_jobs.pop(client, None) - if request is None: - return None - request.cancelled.set() - if request.future is not None: - self._cancel_initial_job_future(request.future) - if count: - self.initial_job_cancelled_count += 1 - return request - - def _client_has_current_tip_job_locked(self, client: ClientState) -> bool: - context = client.active_job - if context is None: - return False - payout_generation = int(getattr(self, "_payout_state_generation", 0)) - if int(getattr(context, "payout_state_generation", payout_generation)) != payout_generation: - return False - current_tip = self._current_published_tip_hash_locked() - if current_tip is None: - # An active job is not proof that tip observation is alive. Keep - # coverage fail-closed until blockpoll/blockwait has published the - # tip that makes the job current. - return False - snapshot = getattr(self, "tip_template_snapshot", None) - if snapshot is None: - # Focused embedders may only publish the observed tip. Production - # startup and blockpoll publish a full snapshot, in which case the - # exact template identity checks below are mandatory. - return str(context.template.get("previousblockhash", "")) == current_tip - if snapshot.bestblockhash != current_tip or snapshot.template_artifacts is None: - return False - return bool( - str(context.template.get("previousblockhash", "")) == current_tip - and getattr(context, "template_fingerprint", None) - == snapshot.template_fingerprint - and int(getattr(context, "template_generation", 0)) - == snapshot.template_generation - and context.template is snapshot.template_artifacts.template - and int(getattr(context, "connection_id", client.connection_id)) - == client.connection_id - and int(getattr(context, "authorization_generation", 0)) - == int(getattr(client, "authorization_generation", 0)) - and int(getattr(context, "difficulty_generation", 0)) - == int(getattr(client, "difficulty_generation", 0)) + return self._ensure_job_delivery_service().cancel_initial_job_locked( + client, + count=count, ) - @staticmethod - def _client_has_delivered_work_locked(client: ClientState) -> bool: - """Return whether a socket write completed for any usable job.""" - - return bool( - client.tip_work_delivered is not None - or client._progress_delivered_context is not None + def _client_has_current_tip_job_locked(self, client: ClientState) -> bool: + return self._ensure_job_delivery_service().client_has_current_tip_job_locked( + client, + self._ensure_job_delivery_service().current_job_source(), ) def _reset_delivery_failure_if_coverage_restored_locked(self) -> None: @@ -7803,10 +4320,12 @@ def _reset_delivery_failure_if_coverage_restored_locked(self) -> None: if not authorized_clients: self._mining_delivery_failure_started_monotonic = None return + delivery_service = self._ensure_job_delivery_service() + source = delivery_service.current_job_source() current = sum( 1 for client in authorized_clients - if self._client_has_current_tip_job_locked(client) + if delivery_service.client_has_current_tip_job_locked(client, source) ) if current / len(authorized_clients) >= 0.95: self._mining_delivery_failure_started_monotonic = None @@ -7817,322 +4336,49 @@ def note_initial_job_delivered( *, validated_current: bool = False, ) -> None: - with self.lock: - self._ensure_initial_job_state() - if not validated_current and not self._client_has_current_tip_job_locked(client): - return - request = self.pending_initial_jobs.pop(client, None) - if request is not None: - request.cancelled.set() - if request.future is not None: - self._cancel_initial_job_future(request.future) - delivered = time.monotonic() - self.initial_job_sent_count += 1 - self.initial_job_delivery_latency_seconds_sum += max( - 0.0, delivered - request.requested_monotonic - ) - self.initial_job_delivery_latency_count += 1 - self.last_initial_job_delivery_monotonic = delivered + self._ensure_job_delivery_service().note_initial_job_delivered( + client, + validated_current=validated_current, + ) def schedule_initial_job(self, client: ClientState) -> bool: - """Coalesce and enqueue one first-job request without blocking its handler.""" - # Focused tests and embedders replace maybe_send_job on the instance as - # a synchronous seam. Preserve it without affecting the production - # class path, which always uses the bounded executor below. - if "maybe_send_job" in self.__dict__: - return bool(self.maybe_send_job(client, clean_jobs=True)) - - now = time.monotonic() - reject = False - deferred = False - superseded_future: Future[bool] | None = None - with self.lock: - self._ensure_initial_job_state() - if ( - not client.subscribed - or not client.authorized - or client.worker is None - or getattr(client, "closing", False) - ): - return True - generation = int(getattr(client, "authorization_generation", 0)) - difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) - existing = self.pending_initial_jobs.get(client) - if ( - existing is not None - and existing.connection_id == client.connection_id - and existing.authorization_generation == generation - and existing.difficulty_generation == difficulty_generation - and existing.worker == client.worker - ): - self.initial_job_coalesced_count += 1 - return True - if existing is not None: - existing.cancelled.set() - superseded_future = existing.future - self.initial_job_cancelled_count += 1 - self.initial_job_superseded_count += 1 - if self._client_has_current_tip_job_locked(client): - if existing is not None: - self.pending_initial_jobs.pop(client, None) - if superseded_future is not None: - self._cancel_initial_job_future(superseded_future) - return True - if ( - existing is None - and len(self.pending_initial_jobs) - >= self.stratum_max_pending_initial_jobs - ): - self.initial_job_queue_rejection_count += 1 - reject = True - request = None - else: - timeout = float(self.stratum_initial_job_timeout_seconds) - predecessor = None - if existing is not None: - for candidate in (existing.future, existing.predecessor): - if candidate is not None and not candidate.done(): - predecessor = candidate - break - request = PendingInitialJob( - client=client, - connection_id=client.connection_id, - authorization_generation=generation, - difficulty_generation=difficulty_generation, - worker=client.worker, - requested_monotonic=now, - deadline_monotonic=now + timeout if timeout > 0 else None, - predecessor=predecessor, - ) - self.pending_initial_jobs[client] = request - deferred = predecessor is not None - if superseded_future is not None: - # Install the replacement before cancellation callbacks can run; - # the predecessor callback then hands off exactly one client slot - # instead of mistaking the obsolete request for a terminal failure. - self._cancel_initial_job_future(superseded_future) - if reject or request is None: - self.disconnect_client(client) - return False - if deferred: - return True - - return self._submit_initial_job_request(request) + return self._ensure_job_delivery_service().schedule_initial_job(client) def request_initial_job_delivery(self, client: ClientState) -> bool: - """Compatibility name for the single bounded initial-job pipeline.""" - return self.schedule_initial_job(client) + return self._ensure_job_delivery_service().schedule_initial_job(client) def cancel_initial_job_delivery(self, client: ClientState) -> None: - with self.lock: - self._ensure_initial_job_state() - self._cancel_pending_initial_job_locked(client, count=True) + self._ensure_job_delivery_service().cancel_initial_job(client, count=True) def _submit_initial_job_request(self, request: PendingInitialJob) -> bool: - client = request.client - try: - future = self._submit_delivery_task( - self.initial_job_executor(), - self._run_initial_job, - request, - priority=PRISM_DELIVERY_PRIORITY_INITIAL, - ) - except (_DeliveryQueueFull, RuntimeError): - disconnect = False - with self.lock: - if self.pending_initial_jobs.get(client) is request: - self.pending_initial_jobs.pop(client, None) - request.cancelled.set() - self.initial_job_queue_rejection_count += 1 - disconnect = True - if disconnect: - self.disconnect_client(client) - # An obsolete submit can race its already-installed replacement. - # It owns neither the client slot nor the right to retire the live - # session when admission fails. - return not disconnect - with self.lock: - if self.pending_initial_jobs.get(client) is request: - request.future = future - else: - self._cancel_initial_job_future(future) - future.add_done_callback( - lambda completed: self._initial_job_future_finished(request, completed) - ) - return True + return self._ensure_job_delivery_service().submit_initial_job_request(request) def _initial_job_future_finished( self, request: PendingInitialJob, future: Future[bool], ) -> None: - """Release failed first-job requests instead of stranding capacity.""" - delivered = False - if not future.cancelled(): - try: - delivered = bool(future.result()) - except Exception: - with self.lock: - self.job_build_failure_count = int( - getattr(self, "job_build_failure_count", 0) - ) + 1 - print( - "prism coordinator: initial job task failed " - f"connection={request.client.connection_id}", - flush=True, - ) - traceback.print_exc() - - disconnect = False - replacement: PendingInitialJob | None = None - with self.lock: - self._ensure_initial_job_state() - current = self.pending_initial_jobs.get(request.client) - if current is not request: - if ( - current is not None - and current.future is None - and current.predecessor is future - ): - current.predecessor = None - replacement = current - elif ( - delivered - and self._initial_request_current_locked(request) - and self._client_has_current_tip_job_locked(request.client) - ): - self.pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - self.last_initial_job_delivery_monotonic = time.monotonic() - elif current is request: - self.pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - if ( - request.deadline_monotonic is not None - and request.deadline_monotonic <= time.monotonic() - ): - request.client.closing = True - self.initial_job_timeout_count += 1 - self.initial_job_cancelled_count += 1 - else: - self.initial_job_failed_count += 1 - disconnect = True - if replacement is not None: - self._submit_initial_job_request(replacement) - if disconnect: - self.disconnect_client(request.client) + self._ensure_job_delivery_service().initial_job_future_finished( + request, + future, + ) def _run_initial_job(self, request: PendingInitialJob) -> bool: - """Prepare outside client locks, then atomically stamp and send current work.""" - retry_delay = 0.05 - last_failure_log_monotonic: float | None = None - - def retry_later() -> bool: - nonlocal retry_delay - request.cancelled.wait(retry_delay) - retry_delay = min(1.0, retry_delay * 2) - return not self._initial_request_cancelled(request) - - try: - while not self._initial_request_cancelled(request): - try: - if not self.ensure_reorg_reconciled_for_current_tip(): - if not retry_later(): - return False - continue - artifacts = self.job_issuance_template_artifacts() - if self._initial_request_cancelled(request): - return False - bundle = self.shared_job_bundle( - artifacts, - request.worker, - request_source="initial", - cancelled=lambda: ( - self._initial_request_cancelled(request) - or not self._issuance_artifacts_current(artifacts) - ), - ) - live_tip = str(self.rpc.call("getbestblockhash")) - if artifacts.previousblockhash != live_tip: - # Feed the observation into detection like every other - # live-tip reader; the refresh path owns publication. - self.observe_tip_for_refresh(live_tip) - with self.lock: - published = getattr(self, "current_tip_first_seen", None) - pinned_authoritative = bool( - published is not None - and artifacts.previousblockhash == published[0] - and self._published_tip_authoritative_locked( - time.monotonic() - ) - ) - if not pinned_authoritative: - # Authority lapsed: submit classification is on the - # live RPC read now, so published-tip work would be - # rejected. Drop it and rebuild from live state. - with self._job_cache_lock: - if self._template_artifacts is artifacts: - self._template_artifacts = None - if not retry_later(): - return False - continue - except _JobBuildCancelled: - if self._initial_request_cancelled(request): - return False - if not retry_later(): - return False - continue - except TemplateRefreshBlocked: - if self._initial_request_cancelled(request): - return False - if not retry_later(): - return False - continue - except Exception: - with self.lock: - self.job_build_failure_count = int( - getattr(self, "job_build_failure_count", 0) - ) + 1 - now = time.monotonic() - if ( - last_failure_log_monotonic is None - or now - last_failure_log_monotonic >= 5.0 - ): - last_failure_log_monotonic = now - print( - "prism coordinator: initial job preparation failed " - f"connection={request.client.connection_id}; retrying", - flush=True, - ) - traceback.print_exc() - if not retry_later(): - return False - continue - delivered = self._deliver_initial_bundle(request, artifacts, bundle) - if delivered is None: - if not retry_later(): - return False - continue - return delivered - return False - except OSError: - self.disconnect_client(request.client) - return False + return self._ensure_job_delivery_service().run_initial_job(request) def _template_artifacts_are_current(self, artifacts: CachedTemplateArtifacts) -> bool: - self._ensure_job_cache_state() - with self._job_cache_lock: - current = self._template_artifacts - return ( - current is artifacts - or ( - current is not None - and current.fingerprint == artifacts.fingerprint - and current.generation == artifacts.generation - ) + current = ( + self._ensure_job_bundle_service() + .template_repository.current_artifacts() + ) + return ( + current is artifacts + or ( + current is not None + and current.fingerprint == artifacts.fingerprint + and current.generation == artifacts.generation ) + ) def _issuance_artifacts_current(self, artifacts: CachedTemplateArtifacts) -> bool: """Issuance-side currency for direct job delivery. @@ -8159,29 +4405,6 @@ def _issuance_artifacts_current(self, artifacts: CachedTemplateArtifacts) -> boo and self._published_tip_authoritative_locked(time.monotonic()) ) - def _acquire_client_job_lock( - self, - client: ClientState, - cancelled: Callable[[], bool], - ) -> bool: - while not cancelled(): - if client.job_update_lock.acquire(timeout=0.1): - return True - return False - - @contextmanager - def _cancellable_client_job_lock( - self, - client: ClientState, - cancelled: Callable[[], bool], - ) -> Iterator[bool]: - acquired = self._acquire_client_job_lock(client, cancelled) - try: - yield acquired - finally: - if acquired: - client.job_update_lock.release() - def _payout_delivery( self, cancelled: Callable[[], bool], @@ -8189,7 +4412,7 @@ def _payout_delivery( generation: int, ) -> Any: """Use cancellable admission while retaining focused gate test seams.""" - gate = self._payout_state_delivery_gate + gate = self._ensure_payout_state_service().delivery_gate delivery_cancelable = getattr(gate, "delivery_cancelable", None) if callable(delivery_cancelable): return delivery_cancelable( @@ -8221,116 +4444,151 @@ def _deliver_initial_bundle( artifacts: CachedTemplateArtifacts, bundle: CachedJobBundle, ) -> bool | None: - client = request.client - - def cancelled() -> bool: - return self._initial_request_cancelled(request) - - if not self._acquire_client_job_lock(client, cancelled): - return False - try: - if cancelled(): - return False - if not self._issuance_artifacts_current(artifacts): - return None - self._ensure_job_cache_state() - gate_started = time.monotonic() - with self._payout_delivery( - cancelled, - generation=bundle.payout_state_generation, - ) as admitted: - self._observe_payout_gate_admission( - admitted, - generation=bundle.payout_state_generation, - fallback_wait_seconds=time.monotonic() - gate_started, - ) - if not admitted or cancelled(): - return False - with self._job_cache_lock: - payout_current = ( - bundle.payout_state_generation == self._payout_state_generation - ) - if not payout_current or not self._issuance_artifacts_current(artifacts): - return None - # Difficulty state precedes coordinator admission everywhere. - # A share holding this client's Vardiff lock may delay only - # this delivery; it must never make initial-job delivery hold - # the global control-plane lock while waiting. - with self._client_vardiff_lock(client): - with self.lock: - if not self._initial_request_current_locked(request): - return False - context = self.stamp_job_for_client( - client, - bundle, - clean_jobs=True, - ) - client.active_job = context - for job_id in tuple(client.active_job_ids): - self.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - self.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - self.prune_client_active_jobs(client) - - self.send_job_update(client, context.job) - mark_delivered = getattr(admitted, "mark_delivered", None) - if callable(mark_delivered): - mark_delivered() - self.apply_job_difficulty(client, context.job) - self.note_tip_work_delivered( - client, - str(context.template["previousblockhash"]), - ) - delivered_monotonic = time.monotonic() - self._record_first_payout_delivery( - bundle.payout_state_generation, - delivered_monotonic, - ) - self._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - self.note_initial_job_delivered(client, validated_current=True) - return True - finally: - client.job_update_lock.release() + return self._ensure_job_delivery_service().deliver_initial_bundle( + request, + artifacts, + bundle, + ) def sweep_initial_job_timeouts(self, *, now: float | None = None) -> int: - now = time.monotonic() if now is None else now - timed_out: list[PendingInitialJob] = [] - with self.lock: - self._ensure_initial_job_state() - expired = [ - request - for request in self.pending_initial_jobs.values() - if request.deadline_monotonic is not None - and request.deadline_monotonic <= now - ] - for request in expired: - if self.pending_initial_jobs.get(request.client) is not request: - continue - self.pending_initial_jobs.pop(request.client, None) - request.cancelled.set() - if request.future is not None: - self._cancel_initial_job_future(request.future) - # Commit teardown while this request still owns the pending - # slot. A concurrent reauthorization will observe closing and - # cannot install a replacement between expiry and disconnect. - request.client.closing = True - self.initial_job_timeout_count += 1 - self.initial_job_cancelled_count += 1 - timed_out.append(request) - for request in timed_out: - self.disconnect_client(request.client) - return len(timed_out) + return self._ensure_job_delivery_service().sweep_initial_job_timeouts(now=now) def initial_job_timeout_loop(self) -> None: - while not self.stop_event.wait(1.0): - self.sweep_initial_job_timeouts() + self._ensure_job_delivery_service().initial_job_timeout_loop() + + def _make_background_service_registry(self) -> BackgroundServiceRegistry: + """Describe process loops in their historical shutdown-join order.""" + specifications = [ + BackgroundServiceSpec( + name="qbit_blockpoll", + thread_name="prism-qbit-block-poll", + target=self.blockpoll_loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + ), + BackgroundServiceSpec( + name="block_submitter", + thread_name="prism-block-submitter", + target=self.block_submit_loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + ), + ] + if bool(getattr(self, "blockwait_enabled", False)): + specifications.append( + BackgroundServiceSpec( + name="qbit_blockwait", + thread_name="prism-qbit-block-wait", + target=self.blockwait_loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + ) + ) + if float(getattr(self, "vardiff_idle_sweep_seconds", 0.0)) > 0: + specifications.append( + BackgroundServiceSpec( + name="vardiff_idle_sweep", + thread_name="prism-vardiff-idle-sweep", + target=self.vardiff_idle_sweep_loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + ) + ) + if float(getattr(self, "stratum_initial_job_timeout_seconds", 0.0)) > 0: + specifications.append( + BackgroundServiceSpec( + name="initial_job_timeout_sweep", + thread_name="prism-initial-job-timeouts", + target=self.initial_job_timeout_loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + ) + ) + specifications.append( + BackgroundServiceSpec( + name="share_writer", + thread_name="prism-share-writer", + target=self.share_append_loop, + daemon=True, + join_timeout=5.0, + watchdog_monitored=True, + ) + ) + if bool(getattr(self, "ctv_broadcaster_enabled", False)): + specifications.append(self._ensure_ctv_runtime().background_service_spec()) + if bool(getattr(self, "watchdog_enabled", False)): + specifications.append( + BackgroundServiceSpec( + name="watchdog", + thread_name="prism-watchdog", + target=self.watchdog_loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + ) + ) + if bool(getattr(self, "audit_bind", None)) and bool( + getattr(self, "audit_port", 0) + ): + specifications.append(self._health_snapshot_service_spec()) + return BackgroundServiceRegistry(specifications) + + def _health_snapshot_service_spec(self) -> BackgroundServiceSpec: + return BackgroundServiceSpec( + name="health_snapshot_refresher", + thread_name="prism-health-snapshot-refresher", + target=self.health_snapshot_loop, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + ) + + def _ensure_background_services(self) -> BackgroundServiceRegistry: + registry = getattr(self, "_background_services", None) + if registry is None: + registry = self._make_background_service_registry() + self._background_services = registry + return registry + + def _start_background_service(self, name: str) -> threading.Thread: + registry = self._ensure_background_services() + return registry.start( + name, + on_started=lambda specification: ( + self._record_heartbeat(specification.name) + if specification.watchdog_monitored + else None + ), + ) + + def _start_secondary_accept_service( + self, + server: socket.socket, + profile: StratumListenerProfile, + ) -> threading.Thread: + registry = self._ensure_background_services() + service_name = profile.heartbeat_name + registry.register_if_absent( + BackgroundServiceSpec( + name=service_name, + thread_name=f"prism-stratum-accept-{profile.name}", + target=lambda: self.accept_loop(server, profile), + daemon=True, + join_timeout=1.0, + watchdog_monitored=True, + registration_identity=( + "secondary_stratum_accept", + id(server), + id(profile), + ), + ) + ) + return self._start_background_service(service_name) def _record_heartbeat(self, name: str) -> None: self._ensure_watchdog_state() @@ -8375,104 +4633,723 @@ def _registered_watchdog_heartbeat_names(self, *names: str) -> tuple[str, ...]: return tuple(name for name in names if name in self._heartbeats) def stratum_accept_heartbeat_names(self) -> tuple[str, ...]: - profiles = getattr(self, "listener_profiles", None) - if not profiles: - return ("stratum_accept",) - return tuple(profile.heartbeat_name for profile in profiles) - - @contextmanager - def _watchdog_paused(self, *names: str) -> Iterator[None]: - for name in names: - self._pause_watchdog_heartbeat(name) - try: - yield - finally: - for name in reversed(names): - self._resume_watchdog_heartbeat(name) + return configured_accept_heartbeat_names( + getattr(self, "listener_profiles", None) + ) - def watchdog_loop(self) -> None: - while not self.stop_event.wait(self.watchdog_interval_seconds): - now = time.monotonic() - if self.publication_progress_failure_expired(now): - print( - "prism coordinator: publication-progress watchdog firing; " - "current tip/generation remained unpublished past the " - f"template refresh failure budget=" - f"{self.template_refresh_failure_exit_seconds:g}s. " - "Exiting non-zero so the restart policy recovers the process.", - flush=True, - ) - os._exit(1) - overdue = ( - self._overdue_heartbeats(now) - if getattr(self, "watchdog_enabled", True) - else [] + def _ensure_session_registry(self) -> SessionRegistry: + registry = getattr(self, "_session_registry", None) + if registry is not None: + clients = getattr(self, "clients", registry.clients) + if clients is not registry.clients: + registry.adopt_clients(clients) + rejection_counts = getattr( + self, + "connection_limit_rejection_counts", + registry.rejection_counts, ) - if overdue: - print( - "prism coordinator: liveness watchdog firing; unresponsive " - f"subsystems={overdue} timeout={self.watchdog_timeout_seconds:g}s. " - "Exiting non-zero so the restart policy recovers the process.", - flush=True, - ) - # Queued shares have not been acknowledged. Miners reconnect - # and retry them after restart; exact-payload replay is - # idempotent if Postgres committed just before this exit. - os._exit(1) + if rejection_counts is not registry.rejection_counts: + registry.rejection_counts = rejection_counts + _CoordinatorSessionRuntime(self).sync_registry_metrics(registry) + return registry + lock = getattr(self, "lock", None) + if lock is None: + lock = threading.RLock() + self.lock = lock + clients = getattr(self, "clients", None) + if clients is None: + clients = set() + self.clients = clients + rejection_counts = getattr(self, "connection_limit_rejection_counts", None) + if not isinstance(rejection_counts, dict): + rejection_counts = {"global": 0, "username": 0} + self.connection_limit_rejection_counts = rejection_counts + candidate = SessionRegistry( + lock=lock, + clients=clients, + connection_generation=int(getattr(self, "connection_counter", 0)), + rejection_counts=rejection_counts, + ) + candidate.peak_active_connections = max( + candidate.peak_active_connections, + int(getattr(self, "peak_active_connection_count", 0)), + ) + candidate.handler_thread_count = int( + getattr(self, "handler_thread_count", candidate.handler_thread_count) + ) + registry = self.__dict__.setdefault("_session_registry", candidate) + _CoordinatorSessionRuntime(self).sync_registry_metrics(registry) + return registry + + def _job_delivery_current_tip_locked(self) -> str | None: + tip_service = self.__dict__.get("_tip_refresh_service") + if tip_service is None: + return None + published = tip_service.published_snapshot() + first_seen = published.first_seen + if first_seen is not None: + return str(first_seen[0]) + snapshot = published.template + if snapshot is not None: + return str(snapshot.bestblockhash) + return None - def _ensure_shutdown_controller(self) -> CoordinatorShutdownController: - controller = getattr(self, "_shutdown_controller", None) - if controller is not None: - return controller - candidate = CoordinatorShutdownController( - float( - getattr( - self, - "writer_quiescence_timeout_seconds", - DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, - ) + def _ensure_retained_job_index(self) -> RetainedJobIndex: + index = self.__dict__.get("_retained_job_index") + if index is None: + index = RetainedJobIndex( + graveyard=self.__dict__.get("_evicted_graveyard_compat"), + by_connection=self.__dict__.get("_evicted_by_connection_compat"), + same_tip_by_connection=self.__dict__.get( + "_evicted_same_tip_by_connection_compat" + ), + same_tip_job_ids=self.__dict__.get( + "_evicted_same_tip_job_ids_compat" + ), + same_tip_ttl_seconds=float( + self.__dict__.get( + "_same_tip_ttl_seconds_compat", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS, + ) + ), + same_tip_per_connection=int( + self.__dict__.get( + "_same_tip_per_connection_compat", + DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION, + ) + ), + stale_grace_seconds=float( + self.__dict__.get( + "_stale_grace_seconds_compat", + DEFAULT_PRISM_STALE_GRACE_SECONDS, + ) + ), ) + index.expiration_counts = self.__dict__.get( + "_evicted_expiration_counts_compat", index.expiration_counts + ) + index.capacity_eviction_counts = self.__dict__.get( + "_evicted_capacity_eviction_counts_compat", + index.capacity_eviction_counts, + ) + index.submit_counts = self.__dict__.get( + "_evicted_submit_counts_compat", index.submit_counts + ) + index.next_prune_monotonic = float( + self.__dict__.get("_evicted_next_prune_monotonic_compat", 0.0) + ) + index.index_tip_hash = self.__dict__.get( + "_evicted_index_tip_hash_compat" + ) + index = self.__dict__.setdefault("_retained_job_index", index) + current_tip = self._job_delivery_current_tip_locked() + index.adopt( + graveyard=index.graveyard, + by_connection=index.by_connection, + same_tip_by_connection=index.same_tip_by_connection, + same_tip_job_ids=index.same_tip_job_ids, + current_tip=current_tip, ) - # CPython's setdefault is atomic under the GIL. This lazy path exists - # for focused tests that construct a coordinator with __new__; normal - # instances create the controller in __init__ before threads start. - return self.__dict__.setdefault("_shutdown_controller", candidate) + return index - @contextmanager - def _writer_operation(self, component: str) -> Iterator[None]: - controller = self._ensure_shutdown_controller() - token = controller.enter_writer(component) - try: - yield - finally: - controller.exit_writer(token) + def _sync_retained_job_index_compatibility(self) -> None: + self._ensure_retained_job_index() - def request_shutdown(self, signum: int | None = None) -> None: - """Signal-safe-sized shutdown request; the ordered work runs elsewhere.""" - self._ensure_shutdown_controller().request_shutdown(signum) - self.stop_event.set() + def _next_job_delivery_id(self) -> str: + return self._ensure_job_delivery_service().next_job_id() - @staticmethod - def _shutdown_log(event: str, **fields: object) -> None: - print( - "prism coordinator: " - + json.dumps({"event": event, **fields}, sort_keys=True), - flush=True, + def _job_delivery_send_difficulty( + self, client: ClientState, job: direct_stratum.DirectQbitStratumJob + ) -> None: + override = self.__dict__.get("send_difficulty") + if override is not None: + override(client, job) + else: + client.send(stratum_difficulty_payload(job.share_difficulty)) + + def _job_delivery_send_job( + self, client: ClientState, job: direct_stratum.DirectQbitStratumJob + ) -> None: + override = self.__dict__.get("send_job") + if override is not None: + override(client, job) + else: + client.send(stratum_job_payload(job)) + + def _live_tip_hash(self) -> str: + return str(self.rpc.call("getbestblockhash")) + + def _clear_job_template_if_current( + self, + artifacts: CachedTemplateArtifacts, + ) -> None: + self._ensure_job_bundle_service().template_repository.clear_if_current( + artifacts ) - def _cancel_active_tip_refresh_for_shutdown(self) -> None: - self._ensure_tip_refresh_state() - with self.lock: - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._tip_refresh_retry.clear() + def _record_job_build_failure(self) -> None: + self._ensure_job_bundle_service().record_failure() - def shutdown(self, *, reason: str = "graceful") -> bool: - """Quiesce every ledger writer and release its lease exactly once. + def _current_payout_generation(self) -> int: + return int(self._ensure_payout_state_service().snapshot().generation) - Returns true when release completed safely (including a ledger without - lease support or an already-absent exact session lease). A timeout + def _payout_delivery_snapshot(self) -> object: + return self._ensure_payout_state_service().snapshot() + + def _payout_delivery_cancelable( + self, + cancelled: Callable[[], bool], + *, + generation: int, + priority: bool, + ) -> object: + return self._ensure_payout_state_service().delivery_gate.delivery_cancelable( + cancelled, + generation=generation, + priority=priority, + ) + + def _job_delivery_observe_payout_admission( + self, + admission: object, + *, + generation: int, + fallback_wait_seconds: float, + ) -> None: + override = self.__dict__.get("_observe_payout_gate_admission") + if callable(override): + override( + admission, + generation=generation, + fallback_wait_seconds=fallback_wait_seconds, + ) + return + self._ensure_payout_state_service().observe_gate_admission( + admission, + generation=generation, + fallback_wait_seconds=fallback_wait_seconds, + ) + + def _run_initial_job_override( + self, + ) -> Callable[[PendingInitialJob], bool] | None: + override = self.__dict__.get("_run_initial_job") + return override if callable(override) else None + + def _deliver_initial_bundle_override(self) -> Callable[..., bool | None] | None: + override = self.__dict__.get("_deliver_initial_bundle") + return override if callable(override) else None + + def _submit_initial_job_override( + self, + ) -> Callable[[PendingInitialJob], bool] | None: + override = self.__dict__.get("_submit_initial_job_request") + return override if callable(override) else None + + def _maybe_send_job_override(self) -> Callable[..., bool] | None: + override = self.__dict__.get("maybe_send_job") + return override if callable(override) else None + + def _send_prepared_job_override( + self, + ) -> Callable[..., RefreshResult] | None: + override = self.__dict__.get("send_prepared_job") + return override if callable(override) else None + + def _build_job_override(self) -> Callable[..., PrismJobContext] | None: + override = self.__dict__.get("build_job_for_client") + return override if callable(override) else None + + def _stamp_job_override(self) -> Callable[..., PrismJobContext] | None: + override = self.__dict__.get("stamp_job_for_client") + return override if callable(override) else None + + def _apply_job_difficulty_override(self) -> Callable[..., None] | None: + override = self.__dict__.get("apply_job_difficulty") + return override if callable(override) else None + + def _send_job_update_override(self) -> Callable[..., None] | None: + override = self.__dict__.get("send_job_update") + return override if callable(override) else None + + def _client_needs_refresh_override(self) -> Callable[..., bool] | None: + override = self.__dict__.get("client_needs_tip_template_refresh") + return override if callable(override) else None + + def _retained_classify_override(self) -> Callable[..., str] | None: + override = self.__dict__.get("_evicted_job_class_locked") + return override if callable(override) else None + + def _ensure_job_delivery_hooks(self) -> DeliveryCompatibilityHooks: + hooks = self.__dict__.get("_job_delivery_hooks") + if hooks is None: + hooks = DeliveryCompatibilityHooks( + run_initial_override=self._run_initial_job_override, + deliver_initial_override=self._deliver_initial_bundle_override, + submit_initial_override=self._submit_initial_job_override, + maybe_send_override=self._maybe_send_job_override, + send_prepared_override=self._send_prepared_job_override, + build_job_override=self._build_job_override, + stamp_job_override=self._stamp_job_override, + apply_difficulty_override=self._apply_job_difficulty_override, + send_update_override=self._send_job_update_override, + needs_refresh_override=self._client_needs_refresh_override, + retained_classify_override=self._retained_classify_override, + split_send_enabled=lambda: ( + "send_difficulty" in self.__dict__ + or "send_job" in self.__dict__ + ), + hot_path_logging_enabled=lambda: bool( + getattr(self, "hot_path_log_enabled", False) + ), + reorg_reconciler_enabled=lambda: bool( + getattr(self, "reorg_reconciler_enabled", True) + ), + ) + hooks = self.__dict__.setdefault("_job_delivery_hooks", hooks) + return hooks + + def _job_delivery_retention_authority_locked(self) -> RetentionAuthority: + current_tip = self._job_delivery_current_tip_locked() + first_seen = self.current_tip_first_seen + cached_parent = self.current_tip_parent + return RetentionAuthority( + current_tip=current_tip, + current_tip_first_delivery=( + float(first_seen[1]) + if first_seen is not None and first_seen[1] is not None + else None + ), + cached_parent=( + str(cached_parent[1]) + if cached_parent is not None and cached_parent[0] == current_tip + else None + ), + ) + + def _job_delivery_artifacts_parent_current_locked( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: + return self._ensure_tip_refresh_service().artifacts_parent_current_locked( + artifacts, + now=time.monotonic(), + ) + + def _job_delivery_published_current_locked( + self, + context_parent: str, + *, + template_fingerprint: str | None, + template_generation: int, + lapsed_live_validated: bool, + payout_generation: int, + ) -> bool: + service = self._ensure_tip_refresh_service() + published = service.published_snapshot() + snapshot = published.template + if published.first_seen is None or snapshot is None: + return False + if context_parent != published.first_seen[0]: + return False + if ( + template_fingerprint is not None + and snapshot.template_fingerprint != template_fingerprint + ): + return False + if ( + template_generation > 0 + and snapshot.template_generation != template_generation + ): + return False + return bool( + service.published_tip_authoritative(time.monotonic()) + or lapsed_live_validated + ) + + def _submit_initial_delivery( + self, + function: Callable[[PendingInitialJob], bool], + request: PendingInitialJob, + *, + priority: int, + ) -> Future[Any]: + return self._submit_delivery_task( + self.tip_refresh_executor(), + function, + request, + priority=priority, + ) + + def _reconcile_progress_delivery_health(self) -> None: + service = self.__dict__.get("progress_health_service") + if service is not None: + service.reconcile_pending(self._progress_eligibility_snapshot()) + + def _ensure_job_delivery_service(self) -> JobDeliveryService: + initial_state = self._ensure_initial_job_state() + registry = self._ensure_session_registry() + retained = self._ensure_retained_job_index() + preparation = self.__dict__.setdefault( + "_job_preparation_port", + _CoordinatorJobPreparation( + ensure_reorg_current=lambda: ( + self.ensure_reorg_reconciled_for_current_tip() + ), + issuance_artifacts=lambda: self.job_issuance_template_artifacts(), + shared_bundle=lambda artifacts, worker, cancelled=None, request_source="routine": ( + self.shared_job_bundle( + artifacts, + worker, + cancelled=cancelled, + request_source=request_source, + ) + ), + artifacts_current=lambda artifacts: ( + self._issuance_artifacts_current(artifacts) + ), + clear_artifacts=lambda artifacts: ( + self._clear_job_template_if_current(artifacts) + ), + record_failure=lambda: self._record_job_build_failure(), + phases=lambda: self._job_build_phases(), + retained_artifacts=lambda: self._retained_collection_artifacts(), + chain_view_untrusted=lambda: self.qbit_chain_view_untrusted(), + admit_idle_bundle_source=lambda client, bundle, allow_uncached: ( + self._admit_idle_bundle_source( + client, + bundle, + allow_uncached=allow_uncached, + ) + ), + observe_elapsed=lambda elapsed, phases: ( + self.observe_job_build_elapsed(elapsed, dict(phases)) + ), + collection_identity=lambda worker: ( + self._collection_bundle_identity(worker) + ), + ready_latched=lambda: ( + self._ensure_job_bundle_service().ready_latched() + ), + template_fingerprint=lambda template: ( + qbit_template_fingerprint(dict(template)) + ), + ), + ) + tip_authority = self.__dict__.setdefault( + "_job_tip_authority_port", + _CoordinatorTipAuthority( + live_tip=lambda: self._live_tip_hash(), + observe_tip=lambda tip_hash: ( + self._submit_tip_observation_for_refresh(tip_hash) + ), + published_authority=lambda: self.current_tip_first_seen, + published_authoritative=lambda now: ( + self._published_tip_authoritative_locked(now) + ), + current_tip_locked=lambda: self._job_delivery_current_tip_locked(), + published_template_locked=lambda: self.tip_template_snapshot, + snapshot_current_locked=lambda snapshot, sequence: ( + self._tip_refresh_snapshot_current_locked(snapshot, sequence) + ), + artifacts_parent_current_locked=( + self._job_delivery_artifacts_parent_current_locked + ), + ensure_artifacts_parent_observed=lambda artifacts: ( + self._ensure_tip_refresh_service() + .ensure_artifacts_parent_observed(artifacts) + ), + schedule_retry=lambda: self._schedule_tip_refresh_retry(), + prepared_obsolete=lambda *args: ( + self._prepared_tip_refresh_obsolete(*args) + ), + prepared_token_current_locked=lambda *args: ( + self._ensure_tip_refresh_service() + .token_current_for_payout_snapshot(*args) + ), + record_cancellation=lambda stage: ( + self._record_tip_refresh_cancellation(stage) + ), + retention_authority_locked=( + self._job_delivery_retention_authority_locked + ), + consume_retained_refresh=lambda context: ( + self._consume_retained_collection_refresh(context) + ), + published_current_locked=( + self._job_delivery_published_current_locked + ), + ), + ) + payout = self.__dict__.setdefault( + "_job_payout_delivery_port", + _CoordinatorPayoutDelivery( + snapshot=lambda: self._ensure_payout_state_service().snapshot(), + generation=lambda: int( + self._ensure_payout_state_service().snapshot().generation + ), + initial_admission=lambda cancelled, generation: ( + self._payout_delivery(cancelled, generation=generation) + ), + admission=lambda cancelled, generation, priority: ( + self._ensure_payout_state_service() + .delivery_gate.delivery_cancelable( + cancelled, + generation=generation, + priority=priority, + ) + ), + observe_admission=lambda admission, generation, fallback_wait_seconds: ( + self._job_delivery_observe_payout_admission( + admission, + generation=generation, + fallback_wait_seconds=fallback_wait_seconds, + ) + ), + record_first_delivery=lambda generation, delivered: ( + self._ensure_payout_state_service().record_first_delivery( + generation, delivered + ) + ), + ), + ) + initial_runtime = self.__dict__.setdefault( + "_initial_job_runtime_port", + _CoordinatorInitialJobRuntime( + stopping=lambda: self.stop_event.is_set(), + wait=lambda timeout: self.stop_event.wait(timeout), + disconnect=lambda client: self.disconnect_client(client), + submit_initial=self._submit_initial_delivery, + ), + ) + progress = self.__dict__.setdefault( + "_job_progress_delivery_port", + _CoordinatorProgressDelivery( + record_health_delivery=self._record_progress_delivery_to_health, + reconcile_health_eligibility=( + self._reconcile_progress_delivery_health + ), + ), + ) + hooks = self._ensure_job_delivery_hooks() + service = self.__dict__.get("_job_delivery_service") + jobs = getattr(self, "jobs", None) + if jobs is None: + jobs = {} + self.jobs = jobs + if service is not None: + service.registry = registry + service.retained = retained + service.adopt_ports( + preparation=preparation, + tip_authority=tip_authority, + payout=payout, + initial_runtime=initial_runtime, + hooks=hooks, + progress=progress, + initial_state=initial_state, + delivery_health_updated=( + self._note_delivery_health_updated_locked + ), + ) + if jobs is not service.jobs: + service.adopt_jobs(jobs) + return service + candidate = JobDeliveryService( + registry=registry, + runtime=JobDeliveryRuntime( + desired_share_difficulty_fn=lambda client: ( + self.desired_client_share_difficulty(client) + ), + minimum_advertised_difficulty_fn=lambda client: ( + self.client_minimum_advertised_difficulty(client) + ), + share_weight_fn=lambda worker: self.share_weight_for_worker(worker), + vardiff_config_fn=lambda client: self.client_vardiff_config(client), + send_difficulty_fn=self._job_delivery_send_difficulty, + send_job_fn=self._job_delivery_send_job, + send_job_batch_fn=lambda client, job: client.send_batch( + [ + stratum_difficulty_payload(job.share_difficulty), + stratum_job_payload(job), + ] + ), + ), + jobs=jobs, + retained=retained, + preparation=preparation, + tip_authority=tip_authority, + payout=payout, + initial_runtime=initial_runtime, + hooks=hooks, + progress=progress, + initial_state=initial_state, + job_counter=int(self.__dict__.get("_job_counter_compat", 0)), + delivery_health_updated=self._note_delivery_health_updated_locked, + ) + return self.__dict__.setdefault("_job_delivery_service", candidate) + + def _adopt_legacy_delivery_client(self, client: ClientState) -> None: + """Register explicit ``__new__`` focused-test clients. + + Production coordinators always admit through S1. Legacy focused + coordinators have no loaded config and historically called the direct + delivery facade with an otherwise empty compatibility collection. + Exact S1 membership remains mandatory once a real coordinator exists. + """ + if "config" in self.__dict__: + return + registry = self._ensure_session_registry() + with registry.lock: + if client in registry.clients or registry.clients: + return + registry._add_client_locked(client) + self.clients = registry.clients + + def _ensure_stratum_session_service(self) -> StratumSessionService: + service = getattr(self, "_stratum_session_service", None) + if service is not None: + self._ensure_session_registry() + return service + self._ensure_p2mr_address_cache_state(create_service=False) + validator = P2mrAddressValidator( + rpc_call=lambda method, params: self.rpc.call(method, params), + max_entries=lambda: int( + getattr( + self, + "payout_address_cache_max_entries", + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, + ) + ), + ttl_seconds=lambda: float( + getattr( + self, + "payout_address_cache_ttl_seconds", + DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, + ) + ), + cache_lock=self._p2mr_address_cache_lock, + cache=self._p2mr_address_cache, + inflight=self._p2mr_address_validation_inflight, + ) + candidate = StratumSessionService( + registry=self._ensure_session_registry(), + runtime=_CoordinatorSessionRuntime(self), + jobs=_CoordinatorSessionJobs(self), + progress=_CoordinatorSessionProgress(self), + address_validator=validator, + pool_closed_reason=PRISM_REJECTION_POOL_CLOSED, + ) + return self.__dict__.setdefault("_stratum_session_service", candidate) + + @contextmanager + def _watchdog_paused(self, *names: str) -> Iterator[None]: + for name in names: + self._pause_watchdog_heartbeat(name) + try: + yield + finally: + for name in reversed(names): + self._resume_watchdog_heartbeat(name) + + def watchdog_loop(self) -> None: + while not self.stop_event.wait(self.watchdog_interval_seconds): + now = time.monotonic() + if self.publication_progress_failure_expired(now): + publication_budget = float( + getattr( + self, + "template_refresh_failure_exit_seconds", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) + ) + print( + "prism coordinator: publication-progress watchdog firing; " + "current tip/generation remained unpublished past the " + f"template refresh failure budget=" + f"{publication_budget:g}s. " + "Exiting non-zero so the restart policy recovers the process.", + flush=True, + ) + os._exit(1) + overdue = ( + self._overdue_heartbeats(now) + if getattr(self, "watchdog_enabled", True) + else [] + ) + if overdue: + print( + "prism coordinator: liveness watchdog firing; unresponsive " + f"subsystems={overdue} timeout={self.watchdog_timeout_seconds:g}s. " + "Exiting non-zero so the restart policy recovers the process.", + flush=True, + ) + # Queued shares have not been acknowledged. Miners reconnect + # and retry them after restart; exact-payload replay is + # idempotent if Postgres committed just before this exit. + os._exit(1) + + def publication_progress_failure_expired(self, now: float) -> bool: + """Bound detected-tip divergence independently of delivery health.""" + return self._ensure_tip_refresh_service().publication_failure_expired( + now, + budget_seconds=float( + getattr( + self, + "template_refresh_failure_exit_seconds", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) + ), + ) + + def _ensure_shutdown_controller(self) -> CoordinatorShutdownController: + controller = getattr(self, "_shutdown_controller", None) + if controller is not None: + return controller + candidate = CoordinatorShutdownController( + float( + getattr( + self, + "writer_quiescence_timeout_seconds", + DEFAULT_PRISM_WRITER_QUIESCENCE_TIMEOUT_SECONDS, + ) + ) + ) + # CPython's setdefault is atomic under the GIL. This lazy path exists + # for focused tests that construct a coordinator with __new__; normal + # instances create the controller in __init__ before threads start. + return self.__dict__.setdefault("_shutdown_controller", candidate) + + @contextmanager + def _writer_operation(self, component: str) -> Iterator[None]: + controller = self._ensure_shutdown_controller() + token = controller.enter_writer(component) + try: + yield + finally: + controller.exit_writer(token) + + def request_shutdown(self, signum: int | None = None) -> None: + """Signal-safe-sized shutdown request; the ordered work runs elsewhere.""" + self._ensure_shutdown_controller().request_shutdown(signum) + self.stop_event.set() + + @staticmethod + def _shutdown_log(event: str, **fields: object) -> None: + print( + "prism coordinator: " + + json.dumps({"event": event, **fields}, sort_keys=True), + flush=True, + ) + + def _cancel_active_tip_refresh_for_shutdown(self) -> None: + self._ensure_tip_refresh_service().cancel_active() + + def shutdown(self, *, reason: str = "graceful") -> bool: + """Quiesce every ledger writer and release its lease exactly once. + + Returns true when release completed safely (including a ledger without + lease support or an already-absent exact session lease). A timeout deliberately withholds release while a tracked writer may still run. """ controller = self._ensure_shutdown_controller() @@ -8578,7 +5455,14 @@ def drain_non_writer_components( if not controller.claim_non_writer_drain(): return started = time.monotonic() - for thread, timeout in threads or []: + drain_threads: Sequence[tuple[threading.Thread, float]] + if threads is None: + drain_threads = self._ensure_background_services().threads_to_drain() + else: + # Temporary compatibility for focused shutdown callers. Process + # startup itself is fully registry-owned. + drain_threads = threads + for thread, timeout in drain_threads: thread.join(timeout=timeout) self.shutdown_vardiff_idle_executor() self.shutdown_tip_refresh_executor() @@ -8594,65 +5478,26 @@ def drain_non_writer_components( def open_stratum_listeners( self, listener_stack: ExitStack ) -> list[tuple[socket.socket, StratumListenerProfile]] | None: - """Bind and listen on every stratum listener profile. - - Called before the slow parts of startup (qbit readiness, policy - validation, block-work recovery) so miners reconnecting through a - restart park in the kernel accept backlog instead of getting - connection refused, which sends firmware into reconnect backoff or - failover and costs hashrate. bind() retries EADDRINUSE for a bounded - window because a predecessor process may still hold the port while - draining its shutdown. Returns None when shutdown is requested during - the retry, so startup can abort gracefully. - """ - backlog = int( - getattr(self, "stratum_listen_backlog", DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG) - ) - retry_seconds = float( - getattr( - self, - "stratum_bind_retry_seconds", - DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, - ) + return StratumSessionService.open_stratum_listeners( + listener_stack, + self.listener_profiles, + backlog=int( + getattr( + self, + "stratum_listen_backlog", + DEFAULT_PRISM_STRATUM_LISTEN_BACKLOG, + ) + ), + retry_seconds=float( + getattr( + self, + "stratum_bind_retry_seconds", + DEFAULT_PRISM_STRATUM_BIND_RETRY_SECONDS, + ) + ), + stop_event=getattr(self, "stop_event", None), + socket_factory=socket.socket, ) - listeners: list[tuple[socket.socket, StratumListenerProfile]] = [] - for profile in self.listener_profiles: - server = listener_stack.enter_context( - socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ) - server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - bind_deadline = time.monotonic() + retry_seconds - warned = False - while True: - try: - server.bind((profile.bind, profile.port)) - break - except OSError as exc: - if exc.errno != errno.EADDRINUSE or time.monotonic() >= bind_deadline: - raise - stop_event = getattr(self, "stop_event", None) - if stop_event is not None and stop_event.is_set(): - # Shutting down mid-startup: stop contending for a - # port this process will never serve. - print( - f"prism coordinator: shutdown requested while waiting " - f"to bind {profile.bind}:{profile.port}; aborting startup", - flush=True, - ) - return None - if not warned: - print( - f"prism coordinator: {profile.name} listener port " - f"{profile.bind}:{profile.port} is busy; retrying bind " - f"for up to {retry_seconds:g}s", - flush=True, - ) - warned = True - time.sleep(0.1) - server.listen(backlog) - server.settimeout(1) - listeners.append((server, profile)) - return listeners def serve(self) -> None: with ExitStack() as listener_stack: @@ -8723,109 +5568,48 @@ def _serve_with_listener_stack(self, listener_stack: ExitStack) -> None: f"tip={self.tip_template_snapshot.bestblockhash if self.tip_template_snapshot else 'unknown'}", flush=True, ) - # Seed liveness before starting monitored loops so the watchdog - # never fires during startup. + # Seed listener liveness before accepting so the watchdog never fires + # during startup. Background-loop heartbeats derive from their service + # specifications at each named start below. for _, profile in listeners: self._record_heartbeat(profile.heartbeat_name) - self._record_heartbeat("qbit_blockpoll") - blockpoll_thread = threading.Thread(target=self.blockpoll_loop, daemon=True) - blockpoll_thread.start() - blockwait_thread: threading.Thread | None = None + self._start_background_service("qbit_blockpoll") if self.blockwait_enabled: - self._record_heartbeat("qbit_blockwait") - blockwait_thread = threading.Thread(target=self.blockwait_loop, daemon=True) - blockwait_thread.start() - vardiff_idle_sweep_thread: threading.Thread | None = None + self._start_background_service("qbit_blockwait") if self.vardiff_idle_sweep_seconds > 0: - self._record_heartbeat("vardiff_idle_sweep") - vardiff_idle_sweep_thread = threading.Thread( - target=self.vardiff_idle_sweep_loop, - daemon=True, - ) - vardiff_idle_sweep_thread.start() - initial_job_timeout_thread: threading.Thread | None = None + self._start_background_service("vardiff_idle_sweep") if self.stratum_initial_job_timeout_seconds > 0: - initial_job_timeout_thread = threading.Thread( - target=self.initial_job_timeout_loop, - name="prism-initial-job-timeouts", - daemon=True, - ) - initial_job_timeout_thread.start() - self._record_heartbeat("block_submitter") - block_submitter_thread = threading.Thread( - target=self.block_submit_loop, - daemon=True, - ) - block_submitter_thread.start() - drain_threads: list[tuple[threading.Thread, float]] = [ - (blockpoll_thread, 1.0), - (block_submitter_thread, 1.0), - ] - if blockwait_thread is not None: - drain_threads.append((blockwait_thread, 1.0)) - if vardiff_idle_sweep_thread is not None: - drain_threads.append((vardiff_idle_sweep_thread, 1.0)) - if initial_job_timeout_thread is not None: - drain_threads.append((initial_job_timeout_thread, 1.0)) + self._start_background_service("initial_job_timeout_sweep") + share_writer = self._ensure_share_writer_service() + share_writer.begin_startup_recovery() + self._start_background_service("block_submitter") # Replay any shares stranded on disk by a prior ledger-outage # shutdown before serving, so no acked share is lost across restart. - if not self._run_startup_writer_replay( - self.replay_recovered_shares, - drain_threads=drain_threads, - ): - return - self._record_heartbeat("share_writer") + try: + replay_ready = self._run_startup_writer_replay( + self.replay_recovered_shares, + drain_background_services=True, + before_shutdown=share_writer.cancel_startup_recovery, + ) + finally: + share_writer.finish_startup_recovery() + if not replay_ready: + return self.share_writer_active = True - share_writer_thread = threading.Thread( - target=self.share_append_loop, - daemon=True, - ) - share_writer_thread.start() - drain_threads.append((share_writer_thread, 5.0)) - ctv_broadcaster_thread: threading.Thread | None = None + self._start_background_service("share_writer") if self.ctv_broadcaster_enabled: - self._record_heartbeat("ctv_fanout_broadcaster") - ctv_broadcaster_thread = threading.Thread( - target=self.ctv_fanout_broadcaster_loop, - daemon=True, - ) - ctv_broadcaster_thread.start() - drain_threads.append((ctv_broadcaster_thread, 1.0)) - print( - "prism coordinator: CTV fanout broadcaster enabled " - f"mode={'cpfp' if self.ctv_broadcaster_fee_sats > 0 else 'direct'} " - f"fee_bits={self.ctv_broadcaster_fee_sats} " - f"wallet={'configured' if self.ctv_broadcaster_wallet else 'none'} " - f"interval={self.ctv_broadcaster_interval_seconds:g}s " - f"limit={self.ctv_broadcaster_limit} " - f"chunk_size={self.ctv_broadcaster_chunk_size}", - flush=True, - ) - # Publication progress is mandatory even when the operator disables - # the ordinary heartbeat watchdog: advancing retry heartbeats do not - # prove that current work has ever reached publication. - threading.Thread(target=self.watchdog_loop, daemon=True).start() + self._start_background_service("ctv_fanout_broadcaster") + print(self._ensure_ctv_runtime().startup_summary(), flush=True) if self.watchdog_enabled: + self._start_background_service("watchdog") print( - "prism coordinator: liveness and publication-progress watchdog enabled " + "prism coordinator: liveness watchdog enabled " f"timeout={self.watchdog_timeout_seconds:g}s " - f"publication_budget={self.template_refresh_failure_exit_seconds:g}s " - f"interval={self.watchdog_interval_seconds:g}s", - flush=True, - ) - else: - print( - "prism coordinator: publication-progress watchdog enabled " - f"budget={self.template_refresh_failure_exit_seconds:g}s " f"interval={self.watchdog_interval_seconds:g}s", flush=True, ) for extra_server, extra_profile in listeners[1:]: - threading.Thread( - target=self.accept_loop, - args=(extra_server, extra_profile), - daemon=True, - ).start() + self._start_secondary_accept_service(extra_server, extra_profile) try: self.accept_loop(*listeners[0]) finally: @@ -8840,158 +5624,33 @@ def _serve_with_listener_stack(self, listener_stack: ExitStack) -> None: # joins and the tip-refresh executor drain: those may be stuck # in unrelated client delivery or obsolete fanout work. self.shutdown(reason="serve_exit") - self.drain_non_writer_components(drain_threads) + self.drain_non_writer_components() def _run_startup_writer_replay( self, replay: Callable[[], int], *, drain_threads: list[tuple[threading.Thread, float]] | None = None, + drain_background_services: bool = False, + before_shutdown: Callable[[], None] | None = None, ) -> bool: """Run startup ledger replay, stopping cleanly if shutdown wins.""" try: replay() except ShutdownInProgress: - if drain_threads is not None: + if before_shutdown is not None: + before_shutdown() + if drain_threads is not None or drain_background_services: self.shutdown(reason="serve_startup_exit") - self.drain_non_writer_components(drain_threads) + if drain_background_services: + self.drain_non_writer_components() + else: + self.drain_non_writer_components(drain_threads) return False return True def accept_loop(self, server: socket.socket, profile: StratumListenerProfile) -> None: - while not self.stop_event.is_set(): - self._record_heartbeat(profile.heartbeat_name) - try: - sock, address = server.accept() - except socket.timeout: - continue - except OSError as exc: - # The listener socket is torn down by serve()'s ExitStack on - # shutdown while secondary accept threads may still be blocked - # in accept(). Descriptor exhaustion is recoverable: keep the - # accept loop alive and refresh its watchdog heartbeat while - # waiting for client/RPC descriptors to drain. - if self.stop_event.is_set(): - return - if exc.errno in {errno.EMFILE, errno.ENFILE}: - self._record_stratum_resource_exhaustion( - listener_name=profile.name, - location="accept", - error_number=exc.errno, - ) - self._wait_after_stratum_resource_failure(profile.heartbeat_name) - continue - raise - - if ( - self.stop_event.is_set() - or self._ensure_shutdown_controller().phase != "running" - ): - try: - sock.close() - except OSError: - pass - return - - with self.lock: - if ( - self.stop_event.is_set() - or self._ensure_shutdown_controller().phase != "running" - ): - try: - sock.close() - except OSError: - pass - return - max_connections = int( - getattr( - self, - "stratum_max_connections", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, - ) - ) - if max_connections > 0 and len(self.clients) >= max_connections: - rejection_count = self._note_connection_limit_rejection_locked("global") - client = None - else: - self.connection_counter += 1 - connection_id = self.connection_counter - client = ClientState( - sock=sock, - address=address, - connection_id=connection_id, - extranonce1_hex=f"{connection_id & 0xFFFFFFFF:08x}", - listener_name=profile.name, - listener_vardiff_config=profile.vardiff_config, - minimum_advertised_difficulty=profile.minimum_advertised_difficulty, - share_difficulty=self.client_startup_difficulty(profile), - ) - self.clients.add(client) - self._ensure_initial_job_state() - self.peak_active_connection_count = max( - self.peak_active_connection_count, - len(self.clients), - ) - if client is None: - try: - sock.close() - except OSError: - pass - if rejection_count == 1 or rejection_count % 100 == 0: - print( - "prism coordinator: rejected stratum connection at global limit " - f"limit={max_connections} count={rejection_count}", - flush=True, - ) - continue - try: - sock.settimeout(None) - self.apply_stratum_send_timeout(sock) - thread = threading.Thread(target=self.handle_client, args=(client,), daemon=True) - with self.lock: - client.handler_thread_registered = True - self.handler_thread_count += 1 - thread.start() - except (OSError, RuntimeError) as exc: - # Admission is atomic with the global count. Undo it if socket - # setup or thread creation fails before a handler owns cleanup, - # then keep this listener alive for the next connection. - try: - with self.lock: - if client.handler_thread_registered: - client.handler_thread_registered = False - self.handler_thread_count = max( - 0, - self.handler_thread_count - 1, - ) - self.disconnect_client(client) - except Exception: - print( - "prism coordinator: failed to fully close rejected stratum client " - f"address={address}", - flush=True, - ) - traceback.print_exc() - with self.lock: - self.connection_setup_failure_count = int( - getattr(self, "connection_setup_failure_count", 0) - ) + 1 - setup_failure_count = self.connection_setup_failure_count - if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: - self._record_stratum_resource_exhaustion( - listener_name=profile.name, - location="connection-setup", - error_number=exc.errno, - ) - if setup_failure_count == 1 or setup_failure_count % 100 == 0: - print( - "prism coordinator: stratum connection setup failed; backing off " - f"listener={profile.name} address={address} " - f"error={exc!r} count={setup_failure_count}", - flush=True, - ) - self._wait_after_stratum_resource_failure(profile.heartbeat_name) - continue + self._ensure_stratum_session_service().accept_loop(server, profile) def _record_stratum_resource_exhaustion( self, @@ -9000,92 +5659,36 @@ def _record_stratum_resource_exhaustion( location: str, error_number: int | None, ) -> int: - with self.lock: - self.accept_resource_exhaustion_count = int( - getattr(self, "accept_resource_exhaustion_count", 0) - ) + 1 - exhaustion_count = self.accept_resource_exhaustion_count - if exhaustion_count == 1 or exhaustion_count % 100 == 0: - print( - "prism coordinator: stratum resource exhaustion " - f"listener={listener_name} location={location} errno={error_number} " - f"count={exhaustion_count}", - flush=True, - ) - return exhaustion_count + _CoordinatorSessionRuntime(self).record_resource_exhaustion( + listener_name=listener_name, + location=location, + error_number=error_number, + ) + return self.accept_resource_exhaustion_count def _wait_after_stratum_resource_failure(self, heartbeat_name: str) -> None: - backoff_seconds = getattr( - self, - "stratum_accept_resource_exhaustion_backoff_seconds", - DEFAULT_PRISM_STRATUM_ACCEPT_RESOURCE_EXHAUSTION_BACKOFF_SECONDS, - ) - remaining_seconds = max(0.0, float(backoff_seconds)) - watchdog_timeout_seconds = max( - 0.001, - float(getattr(self, "watchdog_timeout_seconds", 120.0)), - ) - heartbeat_interval_seconds = max( - 0.001, - min(1.0, watchdog_timeout_seconds / 2.0), - ) - deadline = time.monotonic() + remaining_seconds - while not self.stop_event.is_set(): - self._record_heartbeat(heartbeat_name) - remaining_seconds = deadline - time.monotonic() - if remaining_seconds <= 0: - return - if self.stop_event.wait(min(remaining_seconds, heartbeat_interval_seconds)): - return + _CoordinatorSessionRuntime(self).wait_after_resource_failure(heartbeat_name) def _ensure_connection_capacity_state(self) -> None: - if not hasattr(self, "connection_limit_rejection_counts"): - self.connection_limit_rejection_counts = {"global": 0, "username": 0} + self._ensure_session_registry() def _note_connection_limit_rejection_locked(self, scope: str) -> int: - self._ensure_connection_capacity_state() - count = int(self.connection_limit_rejection_counts.get(scope, 0)) + 1 - self.connection_limit_rejection_counts[scope] = count - return count + return self._ensure_session_registry()._note_rejection_locked(scope) def reserve_client_username(self, client: ClientState, worker: WorkerIdentity) -> bool: - """Atomically reserve an exact Stratum username for one connection.""" - with self.lock: - self._ensure_connection_capacity_state() - limit = int( - getattr( - self, - "stratum_max_connections_per_username", - DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS_PER_USERNAME, - ) - ) - active_for_username = sum( - 1 - for other in self.clients - if ( - other is not client - and other.worker is not None - and other.username == worker.username - ) - ) - if limit > 0 and active_for_username >= limit: - rejection_count = self._note_connection_limit_rejection_locked("username") - if rejection_count == 1 or rejection_count % 100 == 0: - print( - "prism coordinator: rejected stratum authorization at username limit " - f"username={worker.username!r} limit={limit} count={rejection_count}", - flush=True, - ) - return False - client.worker = worker - client.username = worker.username - return True + return self._ensure_stratum_session_service().reserve_client_username( + client, worker + ) def start_audit_server(self) -> None: self.start_health_snapshot_refresher() handler_cls = make_audit_handler(self) httpd = ThreadingHTTPServer((self.audit_bind or "127.0.0.1", self.audit_port), handler_cls) - thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread = threading.Thread( + target=httpd.serve_forever, + name="prism-audit-http", + daemon=True, + ) thread.start() print( f"prism coordinator: audit HTTP listening on {self.audit_bind}:{self.audit_port}", @@ -9093,171 +5696,28 @@ def start_audit_server(self) -> None: ) def apply_stratum_send_timeout(self, sock: socket.socket) -> None: - """Bound blocking sends to miners without touching receive semantics. - - Job refreshes use a bounded executor, but an unresponsive peer whose - TCP buffer is full must still release its worker eventually. - SO_SNDTIMEO turns that into an OSError, which the refresh path treats - as a dead client without failing delivery to other miners. - A plain socket timeout is not usable here: it would also apply to - recv, disconnecting idle-but-healthy miners. - """ - timeout_seconds = getattr( - self, - "stratum_send_timeout_seconds", - DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + apply_socket_send_timeout( + sock, + float( + getattr( + self, + "stratum_send_timeout_seconds", + DEFAULT_PRISM_STRATUM_SEND_TIMEOUT_SECONDS, + ) + ), ) - if timeout_seconds <= 0: - return - seconds = int(timeout_seconds) - microseconds = int((timeout_seconds - seconds) * 1_000_000) - try: - sock.setsockopt( - socket.SOL_SOCKET, - socket.SO_SNDTIMEO, - struct.pack("ll", seconds, microseconds), - ) - except (AttributeError, OSError, struct.error): - # Platform without SO_SNDTIMEO support: keep legacy blocking sends. - return def _wait_for_blockpoll_trigger(self) -> bool: - """Wait for the normal interval or an immediate coalesced retry.""" - remaining = float(self.blockpoll_seconds) - while remaining > 0: - if self.stop_event.is_set(): - return False - # After a failed pass, leave retry signals unconsumed until the - # spacing window closes. Re-checked per slice: a newly observed - # tip zeroes the holdoff immediately, and signals arriving during - # the hold stay coalesced for the attempt that eventually runs. - holdoff = self._tip_refresh_failure_holdoff_remaining() - if holdoff <= 0 and self._consume_tip_refresh_retry(): - return not self.stop_event.is_set() - wait_seconds = min(remaining, 0.25) - if holdoff > 0: - # The retry event may already be set; pace on the stop event - # so the spacing window cannot be spun through instantly. - # Short slices keep a newly detected tip's release prompt. - # Beat per slice: a deliberately held poller is idle, not - # hung, and must not trip the watchdog when the configured - # holdoff exceeds its timeout. - self._record_heartbeat("qbit_blockpoll") - wait_seconds = min(wait_seconds, holdoff, 0.05) - self.stop_event.wait(wait_seconds) - else: - self._tip_refresh_retry.wait(wait_seconds) - remaining -= wait_seconds - # Interval-driven attempts respect the same spacing so a sub-holdoff - # blockpoll interval cannot re-run a failed pass early. - while not self.stop_event.is_set(): - holdoff = self._tip_refresh_failure_holdoff_remaining() - if holdoff <= 0: - break - self._record_heartbeat("qbit_blockpoll") - self.stop_event.wait(min(holdoff, 0.05)) - self._consume_tip_refresh_retry() - return not self.stop_event.is_set() + return self._ensure_tip_refresh_service().wait_for_blockpoll_trigger() def blockpoll_loop(self) -> None: - self._ensure_tip_refresh_state() - while self._wait_for_blockpoll_trigger(): - # A superseding observation or post-fanout tip change wakes this - # fallback loop immediately. The event coalesces repeated signals; - # ordinary same-tip polling retains its configured interval. - # Heartbeat at the top of each iteration: reaching here proves the - # loop is alive. A transient qbit RPC error still loops and beats; a - # hung RPC call never returns, so the beat goes stale and the - # watchdog restarts the process. - self._record_heartbeat("qbit_blockpoll") - try: - refreshed = self.poll_qbit_tip_template_once() - if refreshed: - print( - f"prism coordinator: refreshed {refreshed} client job(s) after qbit tip/template change", - flush=True, - ) - except ShutdownInProgress: - # Admission can close after the loop condition but before a - # nested reconciliation enters the writer gate. That is an - # intentional shutdown stop, not a template-health failure. - return - except (TemplateRefreshSuperseded, _PayoutStatePublicationBlocked) as exc: - # Coordination-blocked attempts neither record into nor fire - # the failure budget. A clock armed by an earlier budgeted - # failure must wait for the next budgeted failure (or be - # cleared by the next completed refresh): exiting here would - # let a transient blip plus ordinary payout/tip churn restart - # a process whose qbitd RPC is healthy. The retry is already - # scheduled by the raise site. - print( - f"prism coordinator: tip/template refresh superseded; retrying: {exc}", - flush=True, - ) - except Exception: - print("prism coordinator: qbit tip/template poll failed", flush=True) - traceback.print_exc() - if self.template_refresh_failure_expired(time.monotonic()): - print( - "prism coordinator: template refresh failure budget exhausted; " - "exiting non-zero so the restart policy recovers the process", - flush=True, - ) - os._exit(1) + self._ensure_tip_refresh_service().blockpoll_loop() def template_refresh_failure_expired(self, now: float) -> bool: - budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if budget <= 0: - return False - failure_started = getattr(self, "template_refresh_failure_started_monotonic", None) - return failure_started is not None and now - failure_started >= budget - - def publication_progress_failure_expired(self, now: float) -> bool: - """Bound sustained detected/current publication divergence. - - Retry-loop heartbeats prove only that the driver is scheduled. This - dedicated deadline is independent of client-delivery health and is - cleared only by current work publication/delivery, so old delivery - backlog cannot make a later legitimate divergence expire immediately. - """ - budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if budget <= 0: - return False - self._ensure_job_cache_state() - with self._progress_health_lock: - divergence_since = ( - self._progress_publication_divergence_since_monotonic - ) - return bool( - divergence_since is not None - and now - divergence_since >= budget - ) + return self._ensure_tip_refresh_service().template_refresh_failure_expired(now) def _record_template_refresh_failure(self, now: float) -> None: - budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - if ( - budget > 0 - and getattr(self, "template_refresh_failure_started_monotonic", None) is None - ): - self.template_refresh_failure_started_monotonic = now + self._ensure_tip_refresh_service().record_template_refresh_failure(now) def blockwait_once(self, known_tip: str) -> str: """One waitfornewblock round: returns the tip after the wait. @@ -9267,449 +5727,72 @@ def blockwait_once(self, known_tip: str) -> str: between our last poll and this call is reported immediately rather than being missed for a cycle. """ - timeout_seconds = getattr( - self, - "blockwait_timeout_seconds", - DEFAULT_PRISM_BLOCKWAIT_TIMEOUT_SECONDS, - ) - watchdog_timeout = float(getattr(self, "watchdog_timeout_seconds", 120.0)) - max_rpc_timeout = max(1.0, watchdog_timeout * 0.8) - timeout_seconds = min(float(timeout_seconds), max(1.0, max_rpc_timeout - 1.0)) - result = self.rpc.call( - "waitfornewblock", - [max(1, int(timeout_seconds * 1000)), known_tip], - timeout=timeout_seconds + 10.0, - ) - if isinstance(result, dict): - new_tip = str(result.get("hash", "") or "") - if new_tip: - return new_tip - return known_tip + return self._ensure_tip_refresh_service().blockwait_once(known_tip) def blockwait_loop(self) -> None: - """Push-style tip detection alongside the interval poller. - - Stale rejects are dominated by the window between a block connecting - and miners receiving fresh work; the poller alone leaves up to a full - PRISM_BLOCKPOLL_SECONDS of that window. This loop parks inside - waitfornewblock and triggers the same refresh path within milliseconds - of a new tip. The poller stays on as the fallback and still owns - same-tip template refreshes, which waitfornewblock does not signal. - Disabled cleanly when qbitd does not support the RPC. - """ - known_tip: str | None = None - while not self.stop_event.is_set(): - self._record_heartbeat("qbit_blockwait") - try: - if known_tip is None: - known_tip = str(self.rpc.call("getbestblockhash")) - self.observe_tip_for_refresh(known_tip) - new_tip = self.blockwait_once(known_tip) - if new_tip == known_tip: - if self.stop_event.wait(0.25): - return - continue - # Advance the wait cursor before observation/logging. If either - # operation raises, the next wait must not rediscover the same - # already-seen transition and create a notification storm. - known_tip = new_tip - # Detection can supersede/cancel obsolete heavy work, but only - # the blockpoll single-flight owner may fetch, build, and - # publish replacement work. - try: - self.observe_tip_for_refresh(new_tip) - finally: - # Preserve the wake even if observation-side cancellation - # or bookkeeping raises after known_tip advanced. - self._schedule_tip_refresh_retry() - print( - f"prism coordinator: blockwait saw new tip {new_tip}; " - "single-flight refresh scheduled", - flush=True, - ) - except Exception as exc: - if known_tip is not None and self._blockwait_unsupported(exc): - print( - "prism coordinator: waitfornewblock unavailable on this qbitd; " - "tip detection falls back to blockpoll only", - flush=True, - ) - self._remove_watchdog_heartbeat("qbit_blockwait") - return - print("prism coordinator: blockwait pass failed", flush=True) - traceback.print_exc() - if self.stop_event.wait(min(5.0, self.blockpoll_seconds)): - return + self._ensure_tip_refresh_service().blockwait_loop() @staticmethod def _blockwait_unsupported(exc: Exception) -> bool: - detail = str(exc).lower() - return ( - "-32601" in detail - or "-32602" in detail - or "method not found" in detail - or "unknown method" in detail - or "invalid params" in detail - or "invalid parameter" in detail - or "wrong number of" in detail - or "too many parameters" in detail - or "incorrect number of" in detail - ) + return TipRefreshService.blockwait_unsupported(exc) def make_ctv_fanout_broadcast_daemon(self) -> CtvFanoutBroadcastDaemon: - if self.ctv_broadcaster_fee_sats > 0 and not self.ctv_broadcaster_wallet: - raise ValueError( - "ctv_broadcaster_wallet is required when ctv_broadcaster_fee_sats is positive" - ) - broadcaster = CtvFanoutBroadcaster( - self.rpc.call, - funding_wallet=self.ctv_broadcaster_wallet, - ) - return CtvFanoutBroadcastDaemon( - self.ledger, - broadcaster, - fee_sats=self.ctv_broadcaster_fee_sats, - ) + return self._ensure_ctv_runtime().make_daemon() - @ledger_writer_operation("ctv_broadcast_state") def run_ctv_fanout_broadcaster_once( self, *, progress_callback: Callable[[], None] | None = None, ) -> CtvFanoutDaemonResult: - if self.ctv_fanout_broadcast_daemon is None: - self.ctv_fanout_broadcast_daemon = self.make_ctv_fanout_broadcast_daemon() - return self.ctv_fanout_broadcast_daemon.run_once( - limit=self.ctv_broadcaster_limit, + return self._ensure_ctv_runtime().run_once( progress_callback=progress_callback, - chunk_size=int( - getattr( - self, - "ctv_broadcaster_chunk_size", - DEFAULT_PRISM_CTV_BROADCASTER_CHUNK_SIZE, - ) - ), - tip_refresh_pending=self.tip_refresh_is_pending, chunk_callback=self.observe_ctv_fanout_broadcaster_chunk, ) def ctv_fanout_broadcaster_loop(self) -> None: - while not self.stop_event.is_set(): - self._record_heartbeat("ctv_fanout_broadcaster") - started = time.monotonic() - shutdown_admission_closed = False - try: - try: - result = self.run_ctv_fanout_broadcaster_once( - progress_callback=self._record_ctv_fanout_broadcaster_progress, - ) - except ShutdownInProgress: - shutdown_admission_closed = True - return - finally: - # Stamp completion before logging or entering the interval - # wait. A blocked row never reaches this finally clause, so - # the watchdog remains able to recover a wedged operation. - self._record_heartbeat("ctv_fanout_broadcaster") - if not shutdown_admission_closed: - self.observe_ctv_fanout_broadcaster_pass( - max(0.0, time.monotonic() - started) - ) - except Exception: - print("prism coordinator: CTV fanout broadcaster pass failed", flush=True) - traceback.print_exc() - else: - if result.yielded_to_tip_refresh: - self._record_ctv_fanout_broadcaster_yield() - if result.scanned_count or result.submitted_count or result.failed_count: - print( - "prism coordinator: CTV fanout broadcaster " - f"scanned={result.scanned_count} " - f"submitted={result.submitted_count} " - f"updated={result.updated_count} " - f"failed={result.failed_count}", - flush=True, - ) - if self.stop_event.wait(self.ctv_broadcaster_interval_seconds): - break + self._ensure_ctv_runtime().loop( + run_once=self.run_ctv_fanout_broadcaster_once, + progress_callback=self._record_ctv_fanout_broadcaster_progress, + observe_pass=self.observe_ctv_fanout_broadcaster_pass, + record_yield=self._record_ctv_fanout_broadcaster_yield, + ) def _tip_refresh_artifacts( self, snapshot: QbitTipTemplateSnapshot, ) -> CachedTemplateArtifacts: - artifacts = snapshot.template_artifacts - if ( - artifacts is None - or artifacts.fingerprint != snapshot.template_fingerprint - or artifacts.previousblockhash != snapshot.previousblockhash - or artifacts.generation != snapshot.template_generation - or snapshot.bestblockhash != snapshot.previousblockhash - or qbit_template_fingerprint(artifacts.template) != artifacts.fingerprint - or str(artifacts.template.get("previousblockhash", "")) - != artifacts.previousblockhash - ): - raise TemplateRefreshBlocked( - "tip/template snapshot does not own matching exact artifacts" - ) - return artifacts + return self._ensure_tip_refresh_service().artifacts(snapshot) def prepare_tip_refresh_bundle( self, snapshot: QbitTipTemplateSnapshot, - *, - priority_requested_monotonic: float | None = None, ) -> CachedJobBundle: - """Build ready-pool work from immutable shared inputs only. + return self._ensure_tip_refresh_service().prepare_bundle(snapshot) - Client selection belongs exclusively to fanout. In particular, a - connection disappearing before or during this build cannot affect the - signed payout bundle or its cache lifetime. - """ - self._ensure_job_cache_state() - self._ensure_tip_refresh_state() - artifacts = self._tip_refresh_artifacts(snapshot) - build_started = time.monotonic() - progress_build_token = self._progress_bundle_build_started() - priority_token, priority_requested_monotonic = ( - self._begin_job_build_priority_preparation( - priority_requested_monotonic - ) - ) - try: - max_payout_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), - ) - for attempt in range(max_payout_retries + 1): - with self._job_cache_lock: - payout_generation_before_build = ( - self._payout_state_generation - ) - try: - bundle = self.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - publication_critical=True, - request_source="tip_refresh", - priority_requested_monotonic=( - priority_requested_monotonic - ), - ) - break - except JobBuildSuperseded: - with self._job_cache_lock: - payout_generation_after_build = ( - self._payout_state_generation - ) - payout_publication_blocked = ( - self._payout_state_publication_blocked - ) - with self.lock: - artifacts_buildable = self._artifacts_buildable_locked( - artifacts - ) - if ( - attempt >= max_payout_retries - or payout_publication_blocked - or not artifacts_buildable - or payout_generation_after_build - == payout_generation_before_build - ): - raise - # Coalesce a completed payout generation into this same - # tip owner. Every abandoned attempt remains fenced by its - # immutable generation; only the latest coherent retry can - # reach final publication. - continue - else: # pragma: no cover - range always runs at least once - raise TemplateRefreshBlocked( - "payout generation did not stabilize during preparation" - ) - except TemplateRefreshBlocked: - raise - except Exception as exc: - with self.lock: - self.job_build_failure_count += 1 - raise TemplateRefreshBlocked("prepared refresh bundle build failed") from exc - finally: - self._finish_job_build_priority_preparation(priority_token) - self._progress_bundle_build_finished(progress_build_token) - self._observe_tip_refresh_seconds( - "bundle_build", - time.monotonic() - build_started, - ) - # Revalidate only the snapshot-owned object. A concurrent cache fill is - # unrelated to this refresh and cannot replace its exact artifacts. - artifacts = self._tip_refresh_artifacts(snapshot) - if ( - bundle.template is not artifacts.template - or bundle.template_fingerprint != artifacts.fingerprint - or bundle.template_generation != artifacts.generation - or str(bundle.template.get("previousblockhash", "")) - != artifacts.previousblockhash - ): - raise TemplateRefreshBlocked( - "prepared refresh bundle does not match exact template artifacts" - ) - if bundle.collection_only: - raise TemplateRefreshBlocked( - "ready-pool prepared refresh unexpectedly produced a collection bundle" - ) - return bundle - - def prewarm_current_tip_ready_bundle(self) -> CachedJobBundle | None: - """Publish one exact current-tip ready bundle before Stratum accepts.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - self.job_preparation_pending = True - try: - observation_sequence = self._reserve_tip_observation_sequence() - snapshot = self.fetch_qbit_tip_template_snapshot() - self._record_progress_tip_poll(snapshot) - try: - reconciled = self.ensure_reorg_reconciled_for_tip( - snapshot.bestblockhash - ) - except Exception as exc: - raise TemplateRefreshBlocked( - "startup reorg reconciliation failed before job preparation" - ) from exc - if not reconciled: - raise TemplateRefreshBlocked( - "startup chain view remained untrusted during job preparation" - ) - - ready = self.pool_readiness_latched() - bundle: CachedJobBundle | None = None - if ready: - progress_build_token = self._progress_bundle_build_started() - try: - bundle = self.shared_job_bundle( - self._tip_refresh_artifacts(snapshot), - None, - publication_critical=True, - request_source="tip_refresh", - ) - finally: - self._progress_bundle_build_finished(progress_build_token) - if bundle.collection_only: - raise TemplateRefreshBlocked( - "startup ready preparation produced collection work" - ) - if bundle.payout_state_generation != int( - getattr(self, "_payout_state_generation", 0) - ): - raise TemplateRefreshSuperseded( - "payout state changed during startup job preparation" - ) - - if str(self.rpc.call("getbestblockhash")) != snapshot.bestblockhash: - raise TemplateRefreshSuperseded( - "qbit tip changed during startup job preparation" - ) - if not self.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - publish_refresh_observation=True, - published_snapshot=snapshot, - ): - raise TemplateRefreshSuperseded( - "startup job preparation was superseded before publication" - ) - with self._job_cache_lock: - self._prepared_ready_snapshot = snapshot if bundle is not None else None - self._prepared_ready_bundle = bundle - self._record_progress_publication( - snapshot, - ( - bundle.payout_state_generation - if bundle is not None - else int(getattr(self, "_payout_state_generation", 0)) - ), - ) - self.last_successful_template_refresh_monotonic = time.monotonic() - self._record_progress_tip_poll(snapshot) - return bundle - finally: - with self._job_cache_lock: - self.job_preparation_pending = False - - def prewarm_startup_jobs(self) -> CachedJobBundle | None: - """Best-effort startup prewarm; transient blocking defers to blockpoll.""" - try: - return self.prewarm_current_tip_ready_bundle() - except TemplateRefreshBlocked as exc: - # Startup prewarming is an optimization. A transient reconciliation, - # payout-generation, or tip race must not prevent Stratum listeners - # from opening; blockpoll and the bounded initial-job queue retry it. - self._schedule_tip_refresh_retry() - print( - "prism coordinator: startup job preparation deferred " - f"reason={exc}", - flush=True, - ) - return None - - def _tip_refresh_token_current_locked( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - return bool( - self._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ) - and self._tip_refresh_snapshot_current_locked( - snapshot, - token.observation_sequence, - ) - ) - - def _tip_refresh_token_prepublication_current_locked( - self, - token: TipRefreshValidationToken, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - ) -> bool: - published = getattr(self, "_published_payout_state", None) - return bool( - token.snapshot is snapshot - and token.tip_hash == snapshot.bestblockhash - and token.template_fingerprint == snapshot.template_fingerprint - and token.template_generation == snapshot.template_generation - and bundle.template_fingerprint == token.template_fingerprint - and bundle.template_generation == token.template_generation - and bundle.payout_state_generation == token.payout_state_generation - and bundle.build_key is token.build_key - and token.build_key.best_tip_hash == snapshot.bestblockhash - and token.build_key.previous_block_hash == snapshot.previousblockhash - and token.build_key.template_fingerprint - == snapshot.template_fingerprint - and token.build_key.template_generation == snapshot.template_generation - and token.build_key.payout_state_generation - == token.payout_state_generation - and token.payout_state_generation - == int(getattr(self, "_payout_state_generation", 0)) - and published is not None - and published.artifact is not None - and token.build_key.payout_artifact_sha256 - == published.artifact.prior_balances_sha256 - and snapshot.template_artifacts is not None - and bundle.template is snapshot.template_artifacts.template - and not self._detected_tip_supersedes_locked( - snapshot.bestblockhash, - token.observation_sequence, - ) + def prewarm_current_tip_ready_bundle(self) -> CachedJobBundle | None: + return self._ensure_tip_refresh_service().prewarm_current_tip_ready_bundle() + + def prewarm_startup_jobs(self) -> CachedJobBundle | None: + return self._ensure_tip_refresh_service().prewarm_startup_jobs() + + def _tip_refresh_token_current_locked( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + return self._ensure_tip_refresh_service().token_current(token, bundle, snapshot) + + def _tip_refresh_token_prepublication_current_locked( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + return self._ensure_tip_refresh_service().token_prepublication_current( + token, + bundle, + snapshot, ) def _tip_refresh_snapshot_current_locked( @@ -9717,17 +5800,9 @@ def _tip_refresh_snapshot_current_locked( snapshot: QbitTipTemplateSnapshot, observation_sequence: int, ) -> bool: - current_tip = getattr(self, "current_tip_first_seen", None) - return bool( - self.tip_template_snapshot is snapshot - and current_tip is not None - and current_tip[0] == snapshot.bestblockhash - and int(getattr(self, "current_tip_observation_sequence", 0)) - == observation_sequence - and not self._detected_tip_supersedes_locked( - snapshot.bestblockhash, - observation_sequence, - ) + return self._ensure_tip_refresh_service().snapshot_current( + snapshot, + observation_sequence, ) def _validate_prepared_tip_refresh( @@ -9736,70 +5811,11 @@ def _validate_prepared_tip_refresh( snapshot: QbitTipTemplateSnapshot, observation_sequence: int, ) -> TipRefreshValidationToken: - """Validate prepared work before publishing submit authority.""" - artifacts = self._tip_refresh_artifacts(snapshot) - if ( - bundle.template is not artifacts.template - or bundle.template_fingerprint != artifacts.fingerprint - or bundle.template_generation != artifacts.generation - or bundle.build_key is None - or bundle.build_key.best_tip_hash != snapshot.bestblockhash - or bundle.build_key.previous_block_hash != snapshot.previousblockhash - or bundle.build_key.template_fingerprint != artifacts.fingerprint - or bundle.build_key.template_generation != artifacts.generation - or bundle.build_key.mode != "ready" - ): - raise TemplateRefreshBlocked( - "prepared refresh bundle changed before final validation" - ) - try: - current_tip = str(self.rpc.call("getbestblockhash")) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed before prepared fanout" - ) from exc - if current_tip != snapshot.bestblockhash: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed before prepared fanout " - f"expected={snapshot.bestblockhash} current={current_tip}" - ) - try: - chain_view_untrusted = bool( - getattr(self, "reorg_reconciler_enabled", True) - and self.qbit_chain_view_untrusted() - ) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit chain trust check failed before prepared fanout" - ) from exc - if chain_view_untrusted: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit chain view became untrusted before prepared fanout" - ) - token = TipRefreshValidationToken( - tip_hash=snapshot.bestblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - payout_state_generation=bundle.payout_state_generation, - observation_sequence=observation_sequence, - build_key=bundle.build_key, - snapshot=snapshot, + return self._ensure_tip_refresh_service().validate_prepared( + bundle, + snapshot, + observation_sequence, ) - with self.lock: - if not self._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before tip publication" - ) - return token def _activate_tip_refresh( self, @@ -9808,16 +5824,12 @@ def _activate_tip_refresh( snapshot: QbitTipTemplateSnapshot, cancel_event: _FanoutCancellation, ) -> None: - with self.lock: - if not self._tip_refresh_token_current_locked(token, bundle, snapshot): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before cancellation registration" - ) - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._active_tip_refresh = (token, cancel_event) + self._ensure_tip_refresh_service().activate( + token, + bundle, + snapshot, + cancel_event, + ) def _publish_prepared_tip_refresh( self, @@ -9827,107 +5839,19 @@ def _publish_prepared_tip_refresh( *, parent_hash: str | None, ) -> _FanoutCancellation: - """Atomically publish prepared work and register its cancellation token.""" - now = time.monotonic() - cancel_event = _FanoutCancellation() - # Admit a synchronization-only reader of this payout generation. Do - # not mark it delivered: the gate's priority reservation belongs to - # the first current-tip socket delivery, not this publication fence. - with self._payout_state_delivery_gate.delivery_cancelable( - lambda: False, - generation=token.payout_state_generation, - priority=True, - ) as payout_admitted: - if not payout_admitted: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before atomic publication" - ) - # Match payout publication's cache -> coordinator lock order. A - # payout fence can close admission after this reader entered, so - # recheck its blocked marker inside the same atomic section. - with self._job_cache_lock: - with self.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - self._payout_state_publication_blocked - or current_sequence > token.observation_sequence - or not self._tip_refresh_token_prepublication_current_locked( - token, - bundle, - snapshot, - ) - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded before atomic publication" - ) - - first_seen = getattr(self, "current_tip_first_seen", None) - tip_changed = ( - first_seen is not None and first_seen[0] != token.tip_hash - ) - flip_stamp = ( - now - if tip_changed - else first_seen[1] - if first_seen is not None - else None - ) - self.current_tip_first_seen = ( - token.tip_hash, - flip_stamp, - ) - self.current_tip_observation_sequence = token.observation_sequence - self.current_tip_observed_monotonic = now - if parent_hash is not None: - self.current_tip_parent = (token.tip_hash, parent_hash) - else: - # The parent lookup is best-effort cleanup metadata. A - # transient RPC failure during a same-tip republication - # must not wipe the still-valid cached parent; only a - # parent belonging to a different tip is stale here. - prior_parent = getattr(self, "current_tip_parent", None) - if prior_parent is None or prior_parent[0] != token.tip_hash: - self.current_tip_parent = None - self.tip_template_snapshot = snapshot - self.tip_refresh_divergence_started_monotonic = None - if tip_changed: - # Retained collection work, the retained ready bundle, - # and graveyard classification all belong to the - # previously published tip. - self._retained_collection_refresh = None - self._prepared_ready_bundle = None - self._prepared_ready_snapshot = None - self.prune_evicted_job_graveyard(now=now, force=True) - - if not self._tip_refresh_token_current_locked( - token, - bundle, - snapshot, - ): - # The payout admission and both coordinator locks - # stabilize every field used by this predicate. - raise TemplateRefreshBlocked( - "prepared refresh publication did not produce a current token" - ) - active = self._active_tip_refresh - if active is not None: - active[1].cancel() - self._active_tip_refresh = (token, cancel_event) - return cancel_event + return self._ensure_tip_refresh_service().publish_prepared( + token, + bundle, + snapshot, + parent_hash=parent_hash, + ) def _clear_active_tip_refresh( self, token: TipRefreshValidationToken, cancel_event: _FanoutCancellation, ) -> None: - with self.lock: - active = self._active_tip_refresh - if active is not None and active[0] is token and active[1] is cancel_event: - self._active_tip_refresh = None + self._ensure_tip_refresh_service().clear_active(token, cancel_event) def _prepared_tip_refresh_obsolete( self, @@ -9936,1205 +5860,92 @@ def _prepared_tip_refresh_obsolete( snapshot: QbitTipTemplateSnapshot, cancel_event: _FanoutCancellation | None, ) -> bool: - if self.stop_event.is_set() or ( - cancel_event is not None and cancel_event.is_set() - ): - return True - with self.lock: - current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not current and cancel_event is not None: - cancel_event.cancel() - return not current - - def send_prepared_job( - self, - client: ClientState, - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - validation_token: TipRefreshValidationToken, - expected_connection_id: int, - expected_active_job: PrismJobContext | None, - cancel_event: _FanoutCancellation | None = None, - submitted_monotonic: float | None = None, - ) -> RefreshResult: - worker_started = time.monotonic() - started = worker_started if submitted_monotonic is None else submitted_monotonic - phases = self._job_build_phases() - phases.clear() - - def cancelled() -> bool: - return self._prepared_tip_refresh_obsolete( - validation_token, - bundle, - snapshot, - cancel_event, - ) or getattr(client, "closing", False) - - phases["executor_queue"] = max(0.0, worker_started - started) - client_lock_started = worker_started - client_lock_acquired = False - client_lock_attempted = False - try: - while True: - with self.lock: - if ( - client not in self.clients - or client.connection_id != expected_connection_id - or getattr(client, "closing", False) - ): - return RefreshResult("disconnected") - if cancelled(): - phases["client_lock"] = max( - 0.0, - time.monotonic() - client_lock_started, - ) - self._record_tip_refresh_cancellation( - "client_lock" if client_lock_attempted else "executor_queue" - ) - return RefreshResult("skipped") - client_lock_attempted = True - client_lock_acquired = client.job_update_lock.acquire( - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS - ) - if client_lock_acquired: - break - phases["client_lock"] = max( - 0.0, - time.monotonic() - client_lock_started, - ) - if cancelled(): - self._record_tip_refresh_cancellation("client_lock") - return RefreshResult("skipped") - with self.lock: - if ( - client not in self.clients - or client.connection_id != expected_connection_id - ): - return RefreshResult("disconnected") - if ( - not self.client_can_receive_jobs(client) - or self.intervening_job_supersedes_snapshot( - client.active_job, - expected_active_job, - snapshot, - ) - or not self.client_needs_tip_template_refresh(client, snapshot) - ): - return RefreshResult("skipped") - self._ensure_job_cache_state() - payout_gate_started = time.monotonic() - with self._payout_state_delivery_gate.delivery_cancelable( - cancelled, - generation=bundle.payout_state_generation, - priority=True, - ) as payout_admitted: - phases["payout_gate"] = max( - 0.0, - time.monotonic() - payout_gate_started, - ) - self._observe_payout_gate_admission( - payout_admitted, - generation=bundle.payout_state_generation, - fallback_wait_seconds=phases["payout_gate"], - ) - if not payout_admitted or cancelled(): - self._record_tip_refresh_cancellation("payout_gate") - return RefreshResult("skipped") - fanout_admitted = ( - cancel_event is None or cancel_event.begin_delivery() - ) - if not fanout_admitted: - self._record_tip_refresh_cancellation("payout_gate") - return RefreshResult("skipped") - try: - # The validation token binds this immutable bundle to the - # exact observed artifact object. Fanout tasks consult only - # in-memory publication/cancellation state, never RPC or - # the mutable cache. - # Preserve one coherent difficulty snapshot through stamp - # and commit, but acquire it before global control-plane - # admission so a concurrent share cannot form the inverse - # coordinator -> Vardiff edge. - with self._client_vardiff_lock(client): - with self.lock: - token_current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - if cancel_event is not None: - cancel_event.cancel() - return RefreshResult("skipped") - if ( - client not in self.clients - or client.connection_id != expected_connection_id - ): - return RefreshResult("disconnected") - if ( - not self.client_can_receive_jobs(client) - or self.intervening_job_supersedes_snapshot( - client.active_job, - expected_active_job, - snapshot, - ) - or not self.client_needs_tip_template_refresh( - client, - snapshot, - ) - ): - return RefreshResult("skipped") - clean_jobs = self.client_tip_changed_for_snapshot( - client, - snapshot, - ) - stamp_started = time.monotonic() - context = self.stamp_job_for_client( - client, - bundle, - clean_jobs=clean_jobs, - ) - phases["stamp"] = time.monotonic() - stamp_started - client.active_job = context - if clean_jobs: - for job_id in tuple(client.active_job_ids): - self.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - self.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - self.prune_client_active_jobs(client) - - socket_send_started = time.monotonic() - try: - self.send_job_update(client, context.job) - payout_admitted.mark_delivered() - finally: - socket_send_finished = time.monotonic() - phases["socket_send"] = max( - 0.0, - socket_send_finished - socket_send_started, - ) - self.apply_job_difficulty(client, context.job) - self.note_tip_work_delivered( - client, - str(context.template["previousblockhash"]), - ) - self.note_initial_job_delivered(client, validated_current=True) - delivered_monotonic = time.monotonic() - self._record_first_payout_delivery( - context.payout_state_generation, - delivered_monotonic, - ) - self._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - if getattr(self, "hot_path_log_enabled", False): - print( - "prism coordinator: sent prepared job " - f"connection={client.connection_id} username={client.username} " - f"job={context.job.job_id} " - f"elapsed={delivered_monotonic - started:.3f}s", - flush=True, - ) - return RefreshResult("sent", delivered_monotonic) - finally: - if cancel_event is not None: - cancel_event.end_delivery() - finally: - if client_lock_acquired: - client.job_update_lock.release() - self.observe_job_build_elapsed( - max(0.0, time.monotonic() - started), - phases, - ) - - def _fanout_prepared_tip_refresh( - self, - clients: list[ClientState], - bundle: CachedJobBundle, - snapshot: QbitTipTemplateSnapshot, - *, - observation_sequence: int | None = None, - validation_token: TipRefreshValidationToken | None = None, - preactivated_cancel_event: _FanoutCancellation | None = None, - executor: ThreadPoolExecutor | None = None, - expected_active_jobs: dict[ClientState, PrismJobContext | None] | None = None, - heartbeat_name: str, - ) -> tuple[int, float | None, float | None, int]: - executor = executor or self.tip_refresh_executor() - cancel_event = preactivated_cancel_event or _FanoutCancellation() - if observation_sequence is None: - with self.lock: - observation_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if validation_token is None: - validation_token = self._validate_prepared_tip_refresh( - bundle, - snapshot, - observation_sequence, - ) - if preactivated_cancel_event is None: - self._activate_tip_refresh( - validation_token, - bundle, - snapshot, - cancel_event, - ) - futures: dict[Future[RefreshResult], ClientState] = {} - submitted_at: dict[Future[RefreshResult], float] = {} - queued_cancellations: set[Future[RefreshResult]] = set() - if expected_active_jobs is None: - with self.lock: - expected_active_jobs = { - client: client.active_job - for client in clients - } - clients_iter = iter(clients) - max_inflight = max(1, int(self.tip_refresh_max_workers)) - - def record_queued_cancellation(future: Future[RefreshResult]) -> None: - if future in queued_cancellations: - return - queued_cancellations.add(future) - elapsed = max(0.0, time.monotonic() - submitted_at[future]) - self.observe_job_build_elapsed(elapsed, {"executor_queue": elapsed}) - self._record_tip_refresh_cancellation("executor_queue") - - def cancel_pending_futures(pending: set[Future[RefreshResult]]) -> None: - cancel_event.cancel() - for future in pending: - if future.cancel(): - record_queued_cancellation(future) - - def submit_available(pending: set[Future[RefreshResult]]) -> None: - while ( - len(pending) < max_inflight - and not self.stop_event.is_set() - and not cancel_event.is_set() - ): - with self.lock: - token_current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - cancel_event.cancel() - return - try: - client = next(clients_iter) - except StopIteration: - return - submitted = time.monotonic() - active_job = expected_active_jobs.get(client) - if active_job is None: - priority = PRISM_DELIVERY_PRIORITY_INITIAL - elif self.client_tip_changed_for_snapshot(client, snapshot): - priority = PRISM_DELIVERY_PRIORITY_NEW_TIP - else: - priority = PRISM_DELIVERY_PRIORITY_SAME_TIP - future = self._submit_delivery_task( - executor, - self.send_prepared_job, - client, - bundle, - snapshot, - validation_token, - client.connection_id, - expected_active_jobs.get(client), - cancel_event, - submitted, - priority=priority, - ) - self._tip_refresh_future_started() - future.add_done_callback(self._tip_refresh_future_finished) - futures[future] = client - submitted_at[future] = submitted - pending.add(future) - - pending: set[Future[RefreshResult]] = set() - try: - sent = 0 - failed = 0 - first_delivery: float | None = None - last_delivery: float | None = None - invalidation: TemplateRefreshBlocked | None = None - last_live_trust_check = time.monotonic() - try: - submit_available(pending) - except RuntimeError: - cancel_pending_futures(pending) - cancel_event.set() - if pending: - wait(pending) - if not self.stop_event.is_set(): - self._schedule_tip_refresh_retry() - raise - while pending: - self._record_heartbeat(heartbeat_name) - if self.stop_event.is_set() or cancel_event.is_set(): - cancel_pending_futures(pending) - done, pending = wait( - pending, - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - return_when=FIRST_COMPLETED, - ) - for future in done: - client = futures[future] - if future.cancelled(): - if future not in queued_cancellations: - record_queued_cancellation(future) - self._record_tip_refresh_client_result("skipped") - continue - try: - result = future.result() - except OSError: - self._record_tip_refresh_client_result("disconnected") - self.disconnect_client(client) - continue - except TemplateRefreshBlocked as exc: - self._record_tip_refresh_client_result("skipped") - invalidation = exc - cancel_pending_futures(pending) - continue - except Exception: - failed += 1 - self._record_tip_refresh_client_result("failed") - with self.lock: - self.job_build_failure_count += 1 - print( - "prism coordinator: prepared job fanout failed " - f"connection={client.connection_id} username={client.username}", - flush=True, - ) - traceback.print_exc() - continue - self._record_tip_refresh_client_result(result.result) - if result.result == "sent": - sent += 1 - delivered = result.delivered_monotonic - if delivered is not None: - first_delivery = ( - delivered - if first_delivery is None - else min(first_delivery, delivered) - ) - last_delivery = ( - delivered - if last_delivery is None - else max(last_delivery, delivered) - ) - if ( - pending - and invalidation is None - and not self.stop_event.is_set() - and time.monotonic() - last_live_trust_check >= 1.0 - ): - # Validation tokens keep queued per-client deliveries - # RPC-free, but they cannot observe headers advancing - # ahead of blocks while the best-block hash stays fixed. - # Recheck the live chain view from the fanout driver about - # once per second and cancel every delivery still queued - # if the view becomes untrusted. - try: - trusted = self.ensure_reorg_reconciled_for_current_tip( - expected_tip_hash=snapshot.bestblockhash, - ) - if not trusted: - raise TemplateRefreshBlocked( - "qbit chain view became untrusted during prepared fanout" - ) - last_live_trust_check = time.monotonic() - except ShutdownInProgress: - # Admission can close after the stop-event check above - # but before reconciliation enters its writer scope. - # Preserve the intentional shutdown signal so the - # poller cannot consume its template-failure budget. - cancel_pending_futures(pending) - raise - except TemplateRefreshBlocked as exc: - invalidation = exc - except Exception as exc: - invalidation = TemplateRefreshBlocked( - "qbit chain trust check failed during prepared fanout" - ) - invalidation.__cause__ = exc - if invalidation is not None: - cancel_pending_futures(pending) - if invalidation is None: - try: - submit_available(pending) - except RuntimeError: - cancel_pending_futures(pending) - cancel_event.set() - if pending: - wait(pending) - if not self.stop_event.is_set(): - self._schedule_tip_refresh_retry() - raise - if invalidation is not None: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise invalidation - with self.lock: - token_current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh was superseded during fanout; immediate retry scheduled" - ) - try: - post_fanout_tip = str(self.rpc.call("getbestblockhash")) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed after prepared fanout; " - "immediate retry scheduled" - ) from exc - if post_fanout_tip != snapshot.bestblockhash: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed during prepared fanout; immediate retry scheduled " - f"expected={snapshot.bestblockhash} current={post_fanout_tip}" - ) - try: - post_fanout_untrusted = bool( - getattr(self, "reorg_reconciler_enabled", True) - and self.qbit_chain_view_untrusted() - ) - except Exception as exc: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit chain trust check failed after prepared fanout; " - "immediate retry scheduled" - ) from exc - if post_fanout_untrusted: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit chain view became untrusted during prepared fanout; " - "immediate retry scheduled" - ) - with self.lock: - token_current = self._tip_refresh_token_current_locked( - validation_token, - bundle, - snapshot, - ) - if not token_current: - cancel_event.set() - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "prepared refresh payout state changed during post-fanout " - "validation; immediate retry scheduled" - ) - return sent, first_delivery, last_delivery, failed - finally: - self._clear_active_tip_refresh(validation_token, cancel_event) - - def poll_qbit_tip_template_once(self, *, heartbeat_name: str = "qbit_blockpoll") -> int: - self._ensure_tip_refresh_state() - self._ensure_job_cache_state() - refresh_started = time.monotonic() - singleflight_acquired = False - publication_lock_acquired = False - progress_refresh_active = False - observation_sequence = 0 - pending_signal_token: int | None = None - observed_best_tip: str | None = None - try: - observation_sequence = self._reserve_tip_observation_sequence() - # The interval poller has no push notification to mark priority for - # it. Probe the cheap best-tip RPC before fetching and deriving the - # template so CTV maintenance can yield as soon as a changed tip is - # observed, rather than after reconciliation or bundle preparation. - observed_best_tip = str(self.rpc.call("getbestblockhash")) - if not self.observe_tip_for_refresh( - observed_best_tip, - observation_sequence=observation_sequence, - mark_pending=False, - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded before template fetch" - ) - if not self._tip_refresh_singleflight_lock.acquire(blocking=False): - # Observation above remains deliberately outside the owner - # lane: a newer tip cancels the obsolete owner immediately. - # Mark a replacement only when publication actually differs; - # a same-tip contender still coalesces one follow-up template - # fetch without superseding the current owner. - self.observe_tip_for_refresh( - observed_best_tip, - observation_sequence=observation_sequence, - mark_pending=True, - ) - self._schedule_tip_refresh_retry() - return 0 - singleflight_acquired = True - pending_signal_token = self._claim_tip_refresh_pending() - with self.lock: - poll_start_clients = tuple( - client - for client in self.clients - if self.client_can_receive_jobs(client) - ) - with self.lock: - current_tip = getattr(self, "current_tip_first_seen", None) - if current_tip is not None and current_tip[0] != observed_best_tip: - pending_signal_token = self._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - snapshot = self._reuse_current_tip_template_snapshot(observed_best_tip) - if snapshot is None: - self._record_job_cache_event("template", hit=False) - snapshot = self.fetch_qbit_tip_template_snapshot() - if not self.observe_tip_for_refresh( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - mark_pending=False, - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded during template fetch" - ) - with self.lock: - published_after_fetch = getattr( - self, - "current_tip_first_seen", - None, - ) - if ( - published_after_fetch is not None - and published_after_fetch[0] != snapshot.bestblockhash - ): - pending_signal_token = self._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - self._record_progress_tip_poll(snapshot) - self._progress_refresh_started() - progress_refresh_active = True - self.pool_readiness_latched() - payout_generation_before_reconciliation = int( - getattr(self, "_payout_state_generation", 0) - ) - with self.lock: - previous_snapshot = self.tip_template_snapshot - # Generation orders concurrent observations but is not itself - # a template change. Repeated observations of identical work - # must not trigger a clean fanout on every poll. - snapshot_changed = previous_snapshot is not None and ( - previous_snapshot.bestblockhash != snapshot.bestblockhash - or previous_snapshot.previousblockhash != snapshot.previousblockhash - or previous_snapshot.template_fingerprint - != snapshot.template_fingerprint - ) - if snapshot_changed: - clients = [ - client - for client in self.clients - if self.client_can_receive_jobs(client) - ] - else: - clients = [ - client - for client in self.clients - if self.client_can_receive_jobs(client) - and self.client_needs_tip_template_refresh(client, snapshot) - ] - # Capture the exact job each client had when this refresh pass - # selected it. A Vardiff/authorize path may install intervening - # work while the shared bundle is prepared or while its task - # waits in the executor queue. Artifact generations let the - # task replace stale intervening work while preserving work - # produced from a template stored after this snapshot. - expected_active_jobs = { - client: client.active_job - for client in clients - } - - if clients and snapshot_changed: - pending_signal_token = self._mark_tip_refresh_pending_for_poll( - pending_signal_token, - observation_sequence, - ) - - refreshed = 0 - build_failures = 0 - first_delivery: float | None = None - last_delivery: float | None = None - self._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - try: - reorg_reconciled = self.ensure_reorg_reconciled_for_tip( - snapshot.bestblockhash - ) - except ShutdownInProgress: - # Shutdown may close writer admission after this refresh has - # fetched a snapshot. Leave the refresh incomplete and let - # the controlled shutdown proceed without consuming the - # template failure budget or taking the hard-exit path. - return 0 - except Exception as exc: - raise TemplateRefreshBlocked( - "qbit reorg reconciliation failed before refresh preparation" - ) from exc - if not reorg_reconciled: - raise TemplateRefreshBlocked( - "qbit chain view remained untrusted after reorg reconciliation" - ) - payout_generation_after_reconciliation = int( - getattr(self, "_payout_state_generation", 0) - ) - if ( - payout_generation_after_reconciliation - != payout_generation_before_reconciliation - ): - # A same-tip reconciliation can invalidate signed payout state - # even when no client needed template work at initial - # selection. Reselect after the ledger mutation so every old- - # generation job is replaced from the post-reorg snapshot. - with self.lock: - clients = [ - client - for client in self.clients - if self.client_can_receive_jobs(client) - and self.client_needs_tip_template_refresh(client, snapshot) - ] - expected_active_jobs = { - client: client.active_job - for client in clients - } - # Reconciliation itself minted the payout-pending token and - # this pass has deliberately reselected clients from that new - # generation. Adopt exactly that token; completion still checks - # both token ownership and payout/detected-tip currentness, so - # a later producer cannot be cleared accidentally. - pending_signal_token = self._claim_tip_refresh_pending() - with self.lock: - selected_clients_list = list(clients) - selected_client_set = set(clients) - for client in poll_start_clients: - if client in selected_client_set: - continue - if snapshot_changed or self.client_needs_tip_template_refresh( - client, - snapshot, - ): - selected_clients_list.append(client) - selected_client_set.add(client) - expected_active_jobs[client] = client.active_job - selected_clients = tuple(selected_clients_list) - # Prepared-mode selection must cover every client this pass can - # end up serving. Revalidation only filters selected_clients, so - # deriving from the narrower initial list could publish authority - # through the sequential path with no shared bundle while - # poll-start targets still need prepared ready work. - use_prepared_fanout = bool( - selected_clients - and getattr(self, "_pool_ready_latched", False) - ) - ready_mode = bool(getattr(self, "_pool_ready_latched", False)) - bundle: CachedJobBundle | None = None - validation_token: TipRefreshValidationToken | None = None - preactivated_cancel_event: _FanoutCancellation | None = None - prepared_executor: ThreadPoolExecutor | None = None - if use_prepared_fanout: - self._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - try: - bundle = self.prepare_tip_refresh_bundle( - snapshot, - priority_requested_monotonic=refresh_started, - ) - except _PayoutStatePublicationBlocked: - for _client in selected_clients: - self._record_tip_refresh_client_result("skipped") - self._schedule_tip_refresh_retry() - raise - except TemplateRefreshBlocked: - for _client in selected_clients: - self._record_tip_refresh_client_result("failed") - raise - if ( - bundle.payout_state_generation - != payout_generation_after_reconciliation - ): - # Request construction can observe a payout generation - # published after reconciliation and still return a fully - # coherent newest-generation bundle. Adopt it only while - # it remains the current immutable payout pointer, then - # reselect the complete fleet so no old-generation client - # is omitted merely because selection preceded the build. - with self._job_cache_lock: - current_payout_generation = ( - self._payout_state_generation - ) - current_payout_artifact = ( - self._published_payout_state.artifact - ) - if ( - bundle.payout_state_generation - != current_payout_generation - or bundle.build_key is None - or current_payout_artifact is None - or bundle.build_key.payout_artifact_sha256 - != current_payout_artifact.prior_balances_sha256 - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "payout state changed after refresh preparation; " - "immediate retry scheduled" - ) - payout_generation_after_reconciliation = ( - current_payout_generation - ) - with self.lock: - selected_clients = tuple( - client - for client in self.clients - if self.client_can_receive_jobs(client) - and self.client_needs_tip_template_refresh( - client, - snapshot, - ) - ) - expected_active_jobs = { - client: client.active_job - for client in selected_clients - } - pending_signal_token = self._claim_tip_refresh_pending() - - self._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - # Construction is complete. Acquire the short publication lane - # only for final validation, snapshot publication, and current - # client selection. Other observations and replacement builds can - # progress while an obsolete builder is still unwinding. - while not self._tip_refresh_lock.acquire( - timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS - ): - self._record_heartbeat(heartbeat_name) - if self.stop_event.is_set(): - return 0 - self._probe_tip_while_refresh_waiting() - self._raise_if_tip_refresh_superseded( - snapshot, - observation_sequence, - ) - publication_lock_acquired = True - with self._job_cache_lock: - current_payout_generation = self._payout_state_generation - current_payout_artifact = self._published_payout_state.artifact - if ( - current_payout_generation - != payout_generation_after_reconciliation - or ( - bundle is not None - and ( - current_payout_artifact is None - or bundle.build_key is None - or bundle.build_key.payout_artifact_sha256 - != current_payout_artifact.prior_balances_sha256 - ) - ) - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "complete build key changed before refresh publication" - ) - - # A ready-pool pass must build and validate its immutable shared - # bundle before publishing submit authority. Otherwise a cache, - # derivation, or final chain-validation failure can invalidate - # retained work without a deliverable replacement. Sequential / - # collection work has no shared preparation stage, so it commits - # here immediately before its worker-specific builds. - if use_prepared_fanout: - assert bundle is not None - # Acquire infrastructure before publishing authority. Once the - # active token is installed, every subsequent fanout exit is - # covered by its cancellation/cleanup finally block. - prepared_executor = self.tip_refresh_executor() - # Parent metadata is cleanup-only, but fetch it before the - # final live/trust guard so validation remains the last RPC - # boundary before atomic publication. - try: - prepared_parent_hash = self._fetch_tip_parent_hash( - snapshot.bestblockhash - ) - except Exception: - prepared_parent_hash = None - validation_token = self._validate_prepared_tip_refresh( - bundle, - snapshot, - observation_sequence, - ) - preactivated_cancel_event = self._publish_prepared_tip_refresh( - validation_token, - bundle, - snapshot, - parent_hash=prepared_parent_hash, - ) - else: - if not self.observe_tip_first_seen( - snapshot.bestblockhash, - observation_sequence=observation_sequence, - publish_refresh_observation=True, - published_snapshot=snapshot, - ): - raise TemplateRefreshSuperseded( - "tip/template poll was superseded by a newer tip observation" - ) - self.prune_evicted_job_graveyard(force=False) - with self.lock: - current_tip = getattr(self, "current_tip_first_seen", None) - if ( - current_tip is None - or current_tip[0] != snapshot.bestblockhash - or int(getattr(self, "current_tip_observation_sequence", 0)) - != observation_sequence - ): - raise TemplateRefreshSuperseded( - "tip/template poll was superseded before snapshot publication" - ) - if self.tip_template_snapshot is not snapshot: - raise TemplateRefreshBlocked( - "tip/template snapshot was not atomically published" - ) - - dropped_client_results: list[str] = [] - with self.lock: - # Revalidate the originally selected targets at publication. - # New connections keep their own pending/retained wake; this - # pass must not consume work that appeared after construction. - # Preserve the pre-build expected job pointer so an intervening - # authorize/Vardiff delivery is never overwritten. - current_clients: list[ClientState] = [] - for client in selected_clients: - if client not in self.clients: - dropped_client_results.append("disconnected") - elif not self.client_can_receive_jobs(client): - dropped_client_results.append("skipped") - elif not self.client_needs_tip_template_refresh(client, snapshot): - dropped_client_results.append("skipped") - else: - current_clients.append(client) - clients = current_clients - - for result in dropped_client_results: - self._record_tip_refresh_client_result(result) - - if bundle is not None and not bundle.collection_only: - with self._job_cache_lock: - if ( - bundle.payout_state_generation - == self._payout_state_generation - ): - self._prepared_ready_snapshot = snapshot - self._prepared_ready_bundle = bundle - - if ( - not use_prepared_fanout - and clients - and getattr(self, "_pool_ready_latched", False) - ): - # Publication already committed through the sequential path, - # so a ready-pool target that appeared after build selection - # has no shared bundle to receive. Leave the pending marker - # armed and let the immediate retry build for it. - self._mark_tip_refresh_pending(observation_sequence) - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "current clients appeared after build selection; retry scheduled" - ) - self._tip_refresh_lock.release() - publication_lock_acquired = False - - with self.lock: - progress_eligible_client = any( - self.client_can_receive_jobs(client) - for client in self.clients - ) - if use_prepared_fanout or not progress_eligible_client: - # Ready-mode work was prepared above. With no eligible clients, - # publishing the reconciled snapshot is sufficient; collection - # work with a live identity is not published until its worker- - # specific bundle is actually delivered below. - self._record_progress_publication( - snapshot, - payout_generation_after_reconciliation, - ) - - if not ready_mode: - with self.lock: - eligible_collection_client = any( - self.client_can_receive_jobs(client) - for client in self.clients - ) - if not eligible_collection_client: - self._retain_collection_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - ) - - if use_prepared_fanout: - assert bundle is not None - ( - refreshed, - first_delivery, - last_delivery, - build_failures, - ) = self._fanout_prepared_tip_refresh( - clients, - bundle, - snapshot, - observation_sequence=observation_sequence, - validation_token=validation_token, - preactivated_cancel_event=preactivated_cancel_event, - executor=prepared_executor, - expected_active_jobs=expected_active_jobs, - heartbeat_name=heartbeat_name, - ) - else: - for client in clients: - if self.stop_event.is_set(): - break - # Collection bundles are worker-specific, so build and - # validate each selected target independently. - self._record_heartbeat(heartbeat_name) - with self.lock: - target_connected = client in self.clients - target_eligible = ( - target_connected - and self.client_can_receive_jobs(client) - ) - if not target_connected: - self._record_tip_refresh_client_result("disconnected") - continue - if not target_eligible: - self._record_tip_refresh_client_result("skipped") - continue - try: - if self.maybe_send_job( - client, - clean_jobs=self.client_tip_changed_for_snapshot(client, snapshot), - raise_on_reorg_failure=True, - raise_on_build_failure=True, - tip_refresh_snapshot=snapshot, - tip_refresh_observation_sequence=observation_sequence, - ): - delivered = time.monotonic() - refreshed += 1 - first_delivery = ( - delivered - if first_delivery is None - else min(first_delivery, delivered) - ) - last_delivery = ( - delivered - if last_delivery is None - else max(last_delivery, delivered) - ) - self._record_tip_refresh_client_result("sent") - else: - self._record_tip_refresh_client_result("skipped") - except _JobBuildFailed: - build_failures += 1 - self._record_tip_refresh_client_result("failed") - except OSError: - self._record_tip_refresh_client_result("disconnected") - self.disconnect_client(client) - - if not ready_mode: - with self.lock: - eligible_collection_client = any( - self.client_can_receive_jobs(client) - for client in self.clients - ) - if not eligible_collection_client: - self._retain_collection_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - ) - self._record_progress_publication( - snapshot, - payout_generation_after_reconciliation, - ) - - if clients: - try: - post_fanout_tip = str(self.rpc.call("getbestblockhash")) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit tip validation failed after sequential refresh; " - "immediate retry scheduled" - ) from exc - if post_fanout_tip != snapshot.bestblockhash: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed during sequential refresh; " - "immediate retry scheduled " - f"expected={snapshot.bestblockhash} current={post_fanout_tip}" - ) - if int(getattr(self, "_payout_state_generation", 0)) != ( - payout_generation_after_reconciliation - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "payout state changed during sequential refresh; " - "immediate retry scheduled" - ) - - if refreshed == 0 and build_failures: - raise TemplateRefreshBlocked( - f"job builds failed for {build_failures} client(s); no refreshed work was issued" - ) - if refreshed: - with self.lock: - self.tip_refresh_job_count += refreshed - assert first_delivery is not None and last_delivery is not None - self._observe_tip_refresh_seconds( - "first_delivery", - first_delivery - refresh_started, - ) - self._observe_tip_refresh_seconds( - "last_delivery", - last_delivery - refresh_started, - ) - if not self._clear_tip_refresh_pending_for_completed_refresh( - snapshot, - observation_sequence, - payout_generation_after_reconciliation, - pending_signal_token, - ): - # A newer tip or payout mutation won after the last delivery - # guard. Preserve its pending token and retry immediately. - pending_signal_token = None - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip or payout state changed before refresh completion; " - "immediate retry scheduled" - ) - self.last_successful_template_refresh_monotonic = time.monotonic() - self.template_refresh_failure_started_monotonic = None - self._clear_tip_refresh_failure_holdoff() - # A completed pass reconfirms that the coherent snapshot remained - # current through publication and fanout. Refresh the liveness - # stamp so a legitimately long, progressing pass does not become - # stale the instant its active marker is cleared. - self._record_progress_tip_poll(snapshot) - return refreshed - except (TemplateRefreshSuperseded, _PayoutStatePublicationBlocked): - # Coordination-blocked refreshes -- a superseded tip, a pending - # payout publication fence, a refresh raced by payout mutation -- - # are churn between healthy components, not qbitd unhealthiness. - # They must not arm the restart budget: sustained payout churn - # would otherwise self-terminate a process whose RPC is fine, and - # each restart re-triggers the same churn. Re-raise so callers - # still schedule their immediate retry. Plain TemplateRefreshBlocked - # stays budgeted below: it also wraps genuine failures (job builds - # failing, malformed template artifacts, untrusted chain views) - # whose persistence must still take the budgeted restart path. - # The retry-spacing stamp still applies: churn against an - # unchanged tip re-arms the poller no faster than the holdoff, - # while a genuinely newer tip zeroes it immediately. - self._note_tip_refresh_attempt_failed(observed_best_tip) - raise - except Exception: - self._record_template_refresh_failure(time.monotonic()) - self._note_tip_refresh_attempt_failed(observed_best_tip) - raise - finally: - try: - if publication_lock_acquired: - self._tip_refresh_lock.release() - if progress_refresh_active: - self._progress_refresh_finished() - if singleflight_acquired: - self._observe_tip_refresh_seconds( - "refresh", - time.monotonic() - refresh_started, - ) - finally: - if singleflight_acquired: - self._tip_refresh_singleflight_lock.release() + return self._ensure_tip_refresh_service().prepared_obsolete( + validation_token, + bundle, + snapshot, + cancel_event, + ) - def _probe_tip_while_refresh_waiting(self) -> None: - """Detect a changed live tip without entering the heavy refresh lane.""" - observation_sequence = self._reserve_tip_observation_sequence() - try: - observed_tip = str(self.rpc.call("getbestblockhash")) - except Exception: - # The owning refresh still has to unwind or complete. Preserve its - # pending state and let the next bounded lock wait probe again. - return - self.observe_tip_for_refresh( - observed_tip, + def send_prepared_job( + self, + client: ClientState, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + validation_token: TipRefreshValidationToken, + expected_connection_id: int, + expected_active_job: PrismJobContext | None, + cancel_event: _FanoutCancellation | None = None, + submitted_monotonic: float | None = None, + ) -> RefreshResult: + return self._ensure_job_delivery_service().send_prepared_job( + client, + bundle, + snapshot, + validation_token, + expected_connection_id, + expected_active_job, + cancel_event, + submitted_monotonic, + ) + + def _fanout_prepared_tip_refresh( + self, + clients: list[ClientState], + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + *, + observation_sequence: int | None = None, + validation_token: TipRefreshValidationToken | None = None, + preactivated_cancel_event: _FanoutCancellation | None = None, + executor: ThreadPoolExecutor | None = None, + expected_active_jobs: dict[ClientState, PrismJobContext | None] | None = None, + heartbeat_name: str, + ) -> tuple[int, float | None, float | None, int]: + return self._ensure_tip_refresh_service().fanout_prepared( + list(clients), + bundle, + snapshot, observation_sequence=observation_sequence, + validation_token=validation_token, + preactivated_cancel_event=preactivated_cancel_event, + executor=executor, + expected_active_jobs=expected_active_jobs, + heartbeat_name=heartbeat_name, + ) + + def poll_qbit_tip_template_once( + self, + *, + heartbeat_name: str = "qbit_blockpoll", + ) -> int: + return self._ensure_tip_refresh_service().poll_once( + heartbeat_name=heartbeat_name ) + def _probe_tip_while_refresh_waiting(self) -> None: + self._ensure_tip_refresh_service()._probe_tip_while_waiting() + def _detected_tip_supersedes_locked( self, tip_hash: str, observation_sequence: int, ) -> bool: - latest = getattr(self, "latest_detected_tip", None) - return bool( - latest is not None - and latest[0] != tip_hash - and latest[1] > observation_sequence - ) + latest = self._ensure_tip_refresh_service().snapshot().latest_detected_tip + return bool(latest and latest[0] != tip_hash and latest[1] > observation_sequence) def _raise_if_tip_refresh_superseded( self, snapshot: QbitTipTemplateSnapshot, observation_sequence: int, ) -> None: - """Stop obsolete work before entering another expensive phase.""" - with self.lock: - superseded = self._detected_tip_supersedes_locked( - snapshot.bestblockhash, - observation_sequence, - ) - if superseded: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip/template poll was superseded by a newer tip observation " - "before refresh preparation" - ) + self._ensure_tip_refresh_service()._raise_if_superseded( + snapshot, + observation_sequence, + ) def _reserve_tip_observation_sequence(self) -> int: - with self.lock: - sequence = int(getattr(self, "tip_observation_sequence", 0)) + 1 - self.tip_observation_sequence = sequence - return sequence + return self._ensure_tip_refresh_service().reserve_observation_sequence() def observe_tip_for_refresh( self, @@ -11143,102 +5954,17 @@ def observe_tip_for_refresh( observation_sequence: int | None = None, mark_pending: bool = True, ) -> bool: - """Record tip detection without publishing share-validation authority. - - A waiter outside ``_tip_refresh_lock`` must be able to cancel obsolete - bundle construction and fanout promptly. It must not update - ``current_tip_first_seen``: that value invalidates old jobs, so the - winning refresh publishes it only after replacement work is prepared - and validated. - """ - if observation_sequence is None: - observation_sequence = self._reserve_tip_observation_sequence() - self._ensure_job_cache_state() - now = time.monotonic() - active_to_cancel: _FanoutCancellation | None = None - should_mark_pending = False - with self.lock: - latest = getattr(self, "latest_detected_tip", None) - if latest is not None and observation_sequence < latest[1]: - return latest[0] == tip_hash - published = getattr(self, "current_tip_first_seen", None) - prior_detected_hash = ( - latest[0] - if latest is not None - else published[0] - if published is not None - else None - ) - detection_changed = ( - prior_detected_hash is not None - and prior_detected_hash != tip_hash - ) - if ( - detection_changed - and self._payout_state_source[1] != tip_hash - ): - # Supersede an in-progress immutable payout candidate as soon - # as the newer tip is detected, without publishing that tip as - # submit authority before replacement work is ready. - source_generation = self._payout_state_source[0] + 1 - self._payout_state_source = ( - source_generation, - tip_hash, - "external_tip", - now, - ) - self.latest_detected_tip = (tip_hash, observation_sequence) - replacement_needed = published is None or published[0] != tip_hash - if published is not None and published[0] == tip_hash: - # A live observation has returned to (or reconfirmed) the - # published generation. This closes any unpublished divergence - # epoch without changing the published snapshot itself. - self.current_tip_observed_monotonic = now - self.tip_refresh_divergence_started_monotonic = None - elif ( - published is not None - and getattr( - self, - "tip_refresh_divergence_started_monotonic", - None, - ) - is None - ): - # Anchor the lease to the first departure from the published - # tip. Further B -> C observations deliberately do not renew it. - self.tip_refresh_divergence_started_monotonic = now - pending_already = bool( - getattr(self, "_tip_refresh_pending_event", None) - and self._tip_refresh_pending_event.is_set() - ) - active = getattr(self, "_active_tip_refresh", None) - if ( - active is not None - and active[0].tip_hash != tip_hash - and active[0].observation_sequence < observation_sequence - ): - active_to_cancel = active[1] - should_mark_pending = bool( - mark_pending - and ( - detection_changed - or (replacement_needed and not pending_already) - ) - ) + return self._ensure_tip_refresh_service().observe_tip( + tip_hash, + observation_sequence=observation_sequence, + mark_pending=mark_pending, + ) - if active_to_cancel is not None: - active_to_cancel.cancel() - if detection_changed: - self._progress_note_refresh_pending(now) - # In-flight constructions for older parents can no longer win; - # stop them at detection so replacement preparation starts - # immediately, well before publication moves submit authority. - self._cancel_obsolete_job_bundle_builds(current_tip=tip_hash) - self._cancel_obsolete_job_builds("chain tip superseded") - if should_mark_pending: - self._mark_tip_refresh_pending(observation_sequence) - self._schedule_tip_refresh_retry() - return True + def _submit_tip_observation_for_refresh(self, tip_hash: str) -> bool: + return self._ensure_tip_refresh_service().submit_tip_observation( + tip_hash, + reason="blockpoll", + ) def observe_tip_first_seen( self, @@ -11248,158 +5974,18 @@ def observe_tip_first_seen( publish_refresh_observation: bool = False, published_snapshot: QbitTipTemplateSnapshot | None = None, ) -> bool: - """Publish a tip (and, when supplied, its coherent template snapshot). - - Detection and publication are deliberately separate. Callers doing a - prepared refresh must finish bundle construction and final validation - before invoking this method. - """ - if ( - published_snapshot is not None - and published_snapshot.bestblockhash != tip_hash - ): - raise ValueError("published snapshot does not match tip hash") - if observation_sequence is None: - observation_sequence = self._reserve_tip_observation_sequence() - if not self.observe_tip_for_refresh( + return self._ensure_tip_refresh_service().publish_tip( tip_hash, observation_sequence=observation_sequence, - mark_pending=False, - ): - return False - now = time.monotonic() - with self.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - observation_sequence < current_sequence - or self._detected_tip_supersedes_locked( - tip_hash, - observation_sequence, - ) - ): - return False - first_seen = getattr(self, "current_tip_first_seen", None) - if first_seen is not None and first_seen[0] == tip_hash: - # A same-tip re-observation proves the tip view is live; the - # freshness stamp bounds submit_stale_check_tip reuse. - self.current_tip_observed_monotonic = now - active = getattr(self, "_active_tip_refresh", None) - # A routine blockwait/poll observation of the same hash carries - # no newer template. While that hash is actively fanning out, - # do not invalidate its token merely by advancing the global - # observation sequence. The next real refresh observation can - # advance it after the active fanout clears. - if publish_refresh_observation and ( - active is None or active[0].tip_hash != tip_hash - ): - self.current_tip_observation_sequence = observation_sequence - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - self.tip_refresh_divergence_started_monotonic = None - return True - - # Fetch optional cleanup metadata before the atomic publication. A slow - # parent RPC must not create a window where submits see the new tip but - # the winning refresh has not yet published its coherent snapshot. - try: - parent_hash = self._fetch_tip_parent_hash(tip_hash) - except Exception: - parent_hash = None - - with self.lock: - current_sequence = int( - getattr(self, "current_tip_observation_sequence", 0) - ) - if ( - observation_sequence < current_sequence - or self._detected_tip_supersedes_locked( - tip_hash, - observation_sequence, - ) - ): - return False - first_seen = getattr(self, "current_tip_first_seen", None) - if first_seen is not None and first_seen[0] == tip_hash: - self.current_tip_observed_monotonic = now - active = getattr(self, "_active_tip_refresh", None) - if publish_refresh_observation and ( - active is None or active[0].tip_hash != tip_hash - ): - self.current_tip_observation_sequence = observation_sequence - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - self.tip_refresh_divergence_started_monotonic = None - return True - - tip_changed = first_seen is not None - # The first tip this process publishes is a startup baseline, not - # a tip flip: a None stamp keeps stale grace closed. - self.current_tip_first_seen = ( - tip_hash, - now if tip_changed else None, - ) - self.current_tip_observation_sequence = observation_sequence - self.current_tip_observed_monotonic = now - if published_snapshot is not None: - self.tip_template_snapshot = published_snapshot - self.tip_refresh_divergence_started_monotonic = None - # Retained collection work is reusable throughout detection and - # preparation, but never after authority moves to a different tip. - self._retained_collection_refresh = None - if parent_hash is None: - self.current_tip_parent = None - else: - self.current_tip_parent = (tip_hash, parent_hash) - # Reclassify formerly same-tip entries immediately. On mainnet the - # zero stale-grace TTL removes them in this pass; on other chains - # the actual chain parent removes multi-tip-behind entries while - # the independently configured grace lifetime protects one-back. - self.prune_evicted_job_graveyard(now=now, force=True) - if tip_changed: - # A retained ready bundle belongs to the previously published tip - # and can never satisfy the consumer's published-snapshot identity - # check again; release it as soon as authority moves. - with self._job_cache_lock: - self._prepared_ready_bundle = None - self._prepared_ready_snapshot = None - return True + publish_refresh_observation=publish_refresh_observation, + published_snapshot=published_snapshot, + ) def _fetch_tip_parent_hash(self, tip_hash: str) -> str | None: - block = self.rpc.call("getblock", [tip_hash]) - if not isinstance(block, dict): - return None - parent = str(block.get("previousblockhash", "") or "") - if not parent: - return None - return parent + return self._ensure_tip_refresh_service()._fetch_parent_hash(tip_hash) def current_tip_parent_hash(self, tip_hash: str) -> str | None: - with self.lock: - cached = getattr(self, "current_tip_parent", None) - if cached is not None and cached[0] == tip_hash: - return cached[1] - first_seen = getattr(self, "current_tip_first_seen", None) - observed_sequence = ( - int(getattr(self, "current_tip_observation_sequence", 0)) - if first_seen is not None and first_seen[0] == tip_hash - else None - ) - parent = self._fetch_tip_parent_hash(tip_hash) - if parent is None: - return None - with self.lock: - current = getattr(self, "current_tip_first_seen", None) - if ( - observed_sequence is not None - and current is not None - and current[0] == tip_hash - and int(getattr(self, "current_tip_observation_sequence", 0)) - == observed_sequence - ): - self.current_tip_parent = (tip_hash, parent) - return parent + return self._ensure_tip_refresh_service().current_tip_parent_hash(tip_hash) def submit_stale_check_tip(self) -> str: """Best-known chain tip for per-share submit classification. @@ -11422,11 +6008,7 @@ def submit_stale_check_tip(self) -> str: first unpublished divergence; failed refreshes therefore still fall back to the live RPC instead of accepting frozen work indefinitely. """ - with self.lock: - published_tip = self._submit_stale_check_tip_locked(time.monotonic()) - if published_tip is not None: - return published_tip - return str(self.rpc.call("getbestblockhash")) + return self._ensure_tip_refresh_service().submit_authority() def _submit_stale_check_tip_locked(self, now: float) -> str | None: """Return the authoritative published submit tip while holding self.lock.""" @@ -11466,41 +6048,7 @@ def _published_tip_authoritative_locked(self, now: float) -> bool: predicate so work handed to miners is never classified against a different tip than the one it was issued for. """ - max_age = float( - getattr( - self, - "submit_tip_max_age_seconds", - DEFAULT_PRISM_SUBMIT_TIP_MAX_AGE_SECONDS, - ) - ) - if max_age <= 0: - return False - observed = getattr(self, "current_tip_first_seen", None) - if observed is None: - return False - observed_at = getattr(self, "current_tip_observed_monotonic", None) - if observed_at is not None and now - observed_at <= max_age: - return True - latest_detected = getattr(self, "latest_detected_tip", None) - divergence_started = getattr( - self, - "tip_refresh_divergence_started_monotonic", - None, - ) - divergence_budget = float( - getattr( - self, - "template_refresh_failure_exit_seconds", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, - ) - ) - return bool( - latest_detected is not None - and latest_detected[0] != observed[0] - and divergence_started is not None - and divergence_budget > 0 - and now - divergence_started <= divergence_budget - ) + return self._ensure_tip_refresh_service().published_tip_authoritative(now) def stale_grace_deadline_open( self, @@ -11551,84 +6099,17 @@ def context_eligible_for_stale_grace( return bool(parent_hash) and str(context.template["previousblockhash"]) == parent_hash def note_tip_work_delivered(self, client: ClientState, job_parent_hash: str) -> None: - """Record the first time this connection was sent work for a tip. + self._ensure_job_delivery_service().note_tip_work_delivered( + client, job_parent_hash + ) - First delivery wins per tip: same-tip template refreshes must not slide - the connection's stale-grace anchor forward. - """ - now = time.monotonic() - with self.lock: - delivered = client.tip_work_delivered - if delivered is None or delivered[0] != job_parent_hash: - client.tip_work_delivered = (job_parent_hash, now) - self._ensure_initial_job_state() - if job_parent_hash == self._current_published_tip_hash_locked(): - self._reset_delivery_failure_if_coverage_restored_locked() + def _note_delivery_health_updated_locked(self, job_parent_hash: str) -> None: + self._ensure_initial_job_state() + if job_parent_hash == self._current_published_tip_hash_locked(): + self._reset_delivery_failure_if_coverage_restored_locked() def _ensure_evicted_job_state(self) -> None: - graveyard = getattr(self, "evicted_job_graveyard", None) - rebuild_indexes = False - if not isinstance(graveyard, OrderedDict): - converted: OrderedDict[str, EvictedJobEntry] = OrderedDict() - for job_id, entry in (graveyard or {}).items(): - if isinstance(entry, EvictedJobEntry): - converted[job_id] = entry - continue - context, connection_id, evicted_monotonic = entry - client = next( - ( - candidate - for candidate in getattr(self, "clients", ()) - if candidate.connection_id == connection_id - ), - None, - ) - converted[job_id] = EvictedJobEntry( - context=context, - connection_id=connection_id, - evicted_monotonic=evicted_monotonic, - previousblockhash=str(context.template["previousblockhash"]), - client=client, - ) - self.evicted_job_graveyard = converted - rebuild_indexes = True - if not hasattr(self, "evicted_jobs_by_connection"): - self.evicted_jobs_by_connection = {} - rebuild_indexes = True - if not hasattr(self, "evicted_same_tip_by_connection"): - self.evicted_same_tip_by_connection = {} - rebuild_indexes = True - if not hasattr(self, "evicted_same_tip_job_ids"): - self.evicted_same_tip_job_ids = OrderedDict() - rebuild_indexes = True - if not hasattr(self, "evicted_job_index_tip_hash"): - self.evicted_job_index_tip_hash = None - rebuild_indexes = True - if not hasattr(self, "evicted_job_next_prune_monotonic"): - self.evicted_job_next_prune_monotonic = 0.0 - if not hasattr(self, "evicted_job_expiration_counts"): - self.evicted_job_expiration_counts = { - job_class: 0 for job_class in PRISM_EVICTED_JOB_CLASSES - } - if not hasattr(self, "evicted_job_capacity_eviction_counts"): - self.evicted_job_capacity_eviction_counts = { - scope: 0 for scope in PRISM_EVICTED_JOB_CAPACITY_SCOPES - } - if not hasattr(self, "evicted_job_submit_counts"): - self.evicted_job_submit_counts = { - outcome: 0 for outcome in PRISM_EVICTED_JOB_SUBMIT_OUTCOMES - } - if not hasattr(self, "same_tip_job_retention_seconds"): - self.same_tip_job_retention_seconds = DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_SECONDS - if not hasattr(self, "same_tip_job_retention_per_connection"): - self.same_tip_job_retention_per_connection = ( - DEFAULT_PRISM_SAME_TIP_JOB_RETENTION_PER_CONNECTION - ) - current_tip = self._current_published_tip_hash_locked() - if self.evicted_job_index_tip_hash != current_tip: - rebuild_indexes = True - if rebuild_indexes: - self._rebuild_evicted_job_indexes_locked() + self._ensure_retained_job_index() def _current_published_tip_hash_locked(self) -> str | None: first_seen = getattr(self, "current_tip_first_seen", None) @@ -11640,113 +6121,7 @@ def _current_published_tip_hash_locked(self) -> str | None: return None def _evicted_job_class_locked(self, entry: EvictedJobEntry) -> str: - current_tip = self._current_published_tip_hash_locked() - if current_tip is None or entry.previousblockhash == current_tip: - return "same_tip" - return "stale_grace" - - def _remove_evicted_job_locked(self, job_id: str) -> EvictedJobEntry | None: - entry = self.evicted_job_graveyard.pop(job_id, None) - if entry is None: - return None - connection_jobs = self.evicted_jobs_by_connection.get(entry.connection_id) - if connection_jobs is not None: - connection_jobs.pop(job_id, None) - if not connection_jobs: - self.evicted_jobs_by_connection.pop(entry.connection_id, None) - connection_jobs = self.evicted_same_tip_by_connection.get(entry.connection_id) - if connection_jobs is not None: - connection_jobs.pop(job_id, None) - if not connection_jobs: - self.evicted_same_tip_by_connection.pop(entry.connection_id, None) - self.evicted_same_tip_job_ids.pop(job_id, None) - return entry - - def _index_evicted_job_locked(self, job_id: str, entry: EvictedJobEntry) -> None: - self.evicted_jobs_by_connection.setdefault( - entry.connection_id, - OrderedDict(), - )[job_id] = None - if self._evicted_job_class_locked(entry) != "same_tip": - return - self.evicted_same_tip_by_connection.setdefault( - entry.connection_id, - OrderedDict(), - )[job_id] = None - self.evicted_same_tip_job_ids[job_id] = None - - def _rebuild_evicted_job_indexes_locked(self) -> None: - self.evicted_jobs_by_connection = {} - self.evicted_same_tip_by_connection = {} - self.evicted_same_tip_job_ids = OrderedDict() - for job_id, entry in self.evicted_job_graveyard.items(): - self._index_evicted_job_locked(job_id, entry) - self.evicted_job_index_tip_hash = self._current_published_tip_hash_locked() - self._enforce_evicted_same_tip_capacity_locked() - - def _enforce_evicted_same_tip_capacity_locked( - self, - connection_id: int | None = None, - ) -> None: - connection_ids = ( - (connection_id,) - if connection_id is not None - else tuple(self.evicted_same_tip_by_connection) - ) - per_connection_cap = int(self.same_tip_job_retention_per_connection) - for candidate_connection_id in connection_ids: - job_ids = self.evicted_same_tip_by_connection.get(candidate_connection_id) - while job_ids is not None and len(job_ids) > per_connection_cap: - oldest_job_id = next(iter(job_ids)) - self._remove_evicted_job_locked(oldest_job_id) - self.evicted_job_capacity_eviction_counts["connection"] += 1 - job_ids = self.evicted_same_tip_by_connection.get(candidate_connection_id) - - def _stale_grace_entry_expired_locked( - self, - entry: EvictedJobEntry, - *, - now: float, - ttl: float, - ) -> bool: - current_tip = self._current_published_tip_hash_locked() - first_seen = getattr(self, "current_tip_first_seen", None) - if ( - ttl <= 0 - or current_tip is None - or first_seen is None - or str(first_seen[0]) != current_tip - or first_seen[1] is None - ): - return True - - # Submit eligibility is exactly one chain parent behind, so pruning - # must use that same relationship. The prior poll observation can lag - # (for example when authorize/vardiff issued work on an intermediate - # tip), and using it here would drop work submit would still credit. - # Until the parent RPC has populated the cache, retain conservatively; - # submit classification fetches it before granting stale grace. - cached_parent = getattr(self, "current_tip_parent", None) - if ( - cached_parent is not None - and cached_parent[0] == current_tip - and entry.previousblockhash != cached_parent[1] - ): - return True - - client = entry.client - if client is not None: - delivered = client.tip_work_delivered - if delivered is None or delivered[0] != current_tip: - # Match stale_grace_deadline_open: prior-tip shares stay in - # flight until this connection receives replacement work. - return False - anchor = delivered[1] - else: - # Disconnect normally removes these entries. Keep legacy/test - # orphan state bounded from the refresh path's tip-flip anchor. - anchor = float(first_seen[1]) - return now - anchor > ttl + return self._ensure_job_delivery_service().retained_job_class(entry) def bury_evicted_job( self, @@ -11756,44 +6131,8 @@ def bury_evicted_job( now: float | None = None, prune: bool = True, ) -> None: - with self.lock: - self._ensure_evicted_job_state() - context = self.jobs.get(job_id) - if context is None: - return - self._remove_evicted_job_locked(job_id) - self.evicted_job_graveyard[job_id] = EvictedJobEntry( - context=context, - connection_id=client.connection_id, - evicted_monotonic=time.monotonic() if now is None else now, - previousblockhash=str(context.template["previousblockhash"]), - client=client, - ) - self._index_evicted_job_locked(job_id, self.evicted_job_graveyard[job_id]) - self._enforce_evicted_same_tip_capacity_locked(client.connection_id) - if prune: - self.prune_evicted_job_graveyard(now=now, force=False) - - def _evicted_job_expired_locked( - self, - entry: EvictedJobEntry, - *, - now: float, - ) -> tuple[str, bool]: - job_class = self._evicted_job_class_locked(entry) - if job_class == "same_tip": - ttl = float(self.same_tip_job_retention_seconds) - return job_class, ttl <= 0 or now - entry.evicted_monotonic > ttl - return job_class, self._stale_grace_entry_expired_locked( - entry, - now=now, - ttl=float( - getattr( - self, - "stale_grace_seconds", - DEFAULT_PRISM_STALE_GRACE_SECONDS, - ) - ), + self._ensure_job_delivery_service().bury_retained( + client, job_id, now=now, prune=prune ) def prune_evicted_job_graveyard( @@ -11802,41 +6141,14 @@ def prune_evicted_job_graveyard( now: float | None = None, force: bool = True, ) -> None: - with self.lock: - self._ensure_evicted_job_state() - if not self.evicted_job_graveyard: - return - now = time.monotonic() if now is None else now - if not force and now < self.evicted_job_next_prune_monotonic: - return - self.evicted_job_next_prune_monotonic = ( - now + DEFAULT_PRISM_EVICTED_JOB_PRUNE_INTERVAL_SECONDS - ) - for job_id, entry in tuple(self.evicted_job_graveyard.items()): - job_class, expired = self._evicted_job_expired_locked(entry, now=now) - if expired: - self._remove_evicted_job_locked(job_id) - self.evicted_job_expiration_counts[job_class] += 1 + self._ensure_job_delivery_service().prune_retained(now=now, force=force) def evicted_job_entry( self, client: ClientState, job_id: str, ) -> EvictedJobEntry | None: - with self.lock: - self._ensure_evicted_job_state() - entry = getattr(self, "evicted_job_graveyard", {}).get(job_id) - if entry is None or entry.connection_id != client.connection_id: - return None - job_class, expired = self._evicted_job_expired_locked( - entry, - now=time.monotonic(), - ) - if expired: - self._remove_evicted_job_locked(job_id) - self.evicted_job_expiration_counts[job_class] += 1 - return None - return entry + return self._ensure_job_delivery_service().retained_entry(client, job_id) def evicted_submit_context( self, @@ -11852,14 +6164,7 @@ def evicted_submit_context( return context, PRISM_CREDIT_POLICY_STALE_GRACE def note_evicted_job_submit(self, credit_policy: str | None) -> None: - outcome = ( - "credited_stale_grace" - if credit_policy == PRISM_CREDIT_POLICY_STALE_GRACE - else "accepted_same_tip" - ) - with self.lock: - self._ensure_evicted_job_state() - self.evicted_job_submit_counts[outcome] += 1 + self._ensure_job_delivery_service().note_retained_submit(credit_policy) def refresh_jobs_after_pending_accepted_block( self, @@ -11867,145 +6172,24 @@ def refresh_jobs_after_pending_accepted_block( *, heartbeat_name: str = "qbit_blockpoll", ) -> int: - with self.lock: - block = client.post_accept_refresh_block - client.post_accept_refresh_block = None - if block is None: - return 0 - block_height, block_hash = block - return self.refresh_jobs_after_accepted_block( - block_height=block_height, - block_hash=block_hash, + return self._ensure_tip_refresh_service().refresh_after_pending_accepted_block( + client, heartbeat_name=heartbeat_name, ) def refresh_jobs_after_accepted_block( self, *, block_height: int, block_hash: str, heartbeat_name: str = "qbit_blockpoll" ) -> int: - try: - self._record_heartbeat(heartbeat_name) - observation_sequence = self._reserve_tip_observation_sequence() - observed_tip = str(self.rpc.call("getbestblockhash")) - if not self.observe_tip_for_refresh( - observed_tip, - observation_sequence=observation_sequence, - ): - raise TemplateRefreshSuperseded( - "post-accept tip observation was superseded" - ) - except TemplateRefreshSuperseded: - # A newer observation won before this notification could publish - # its tip. The shared driver is woken below; this is coordination - # churn, not a post-accept refresh failure. - return 0 - except Exception: - with self.lock: - self.post_accept_refresh_failure_count += 1 - print( - "prism coordinator: post-accept clean job refresh failed after direct PRISM block " - f"height={block_height} hash={block_hash}", - flush=True, - ) - traceback.print_exc() - return 0 - finally: - # Accepted-block payout/template changes can require a same-tip - # rebuild. Wake the driver even when the immediate best-tip RPC or - # observation fails; the owner will refetch a coherent snapshot. - self._schedule_tip_refresh_retry() - print( - "prism coordinator: scheduled single-flight job refresh after " - f"direct PRISM block height={block_height} hash={block_hash} " - f"observed_tip={observed_tip}", - flush=True, - ) - return 0 - - def _reuse_current_tip_template_snapshot( - self, - observed_best_tip: str, - ) -> QbitTipTemplateSnapshot | None: - """Rebuild a snapshot from cached artifacts while their tip holds. - - Honors the PRISM_TEMPLATE_CACHE_SECONDS window that per-client job - builds already use: within it, a poll pass whose observed best tip - still equals the cached template's parent issues no getblocktemplate. - Rapid failed-refresh re-attempts then cost one cheap best-hash probe - each, while a changed tip or an expired window falls through to the - full fetch, so same-tip template rotation still lands on the normal - cadence. - """ - self._ensure_job_cache_state() - ttl = float( - getattr(self, "template_cache_seconds", DEFAULT_PRISM_BLOCKPOLL_SECONDS) - ) - if ttl <= 0: - return None - with self._job_cache_lock: - cached = self._template_artifacts - if cached is None: - return None - if time.monotonic() - cached.fetched_monotonic > ttl: - return None - if str(observed_best_tip) != cached.previousblockhash: - return None - self._record_job_cache_event("template", hit=True) - return QbitTipTemplateSnapshot( - bestblockhash=cached.previousblockhash, - previousblockhash=cached.previousblockhash, - template_fingerprint=cached.fingerprint, - template_generation=cached.generation, - template_artifacts=cached, + return self._ensure_tip_refresh_service().refresh_after_accepted_block( + block_height=block_height, + block_hash=block_hash, + heartbeat_name=heartbeat_name, ) def fetch_qbit_tip_template_snapshot(self) -> QbitTipTemplateSnapshot: - # Reserve ordering before either RPC: a fetch that started on an older - # view must not become "newer" merely because its template arrived last. - generation = self._reserve_template_artifact_generation() - template = self.rpc.call( - "getblocktemplate", - [{"rules": qbit_gbt_rules(getattr(self, "qbit_chain", "regtest"))}], - ) - if not isinstance(template, dict): - raise RuntimeError("getblocktemplate returned non-object") - previousblockhash = str(template.get("previousblockhash", "") or "") - if not previousblockhash: - raise RuntimeError("getblocktemplate omitted previousblockhash") - # The template parent is the tip this work actually extends. Validate - # it after fetching the template so a tip transition between these RPCs - # cannot produce an old bestblockhash paired with newer work. Reject a - # template that was superseded before it can enter the shared cache or - # drive tip observation/graveyard pruning. - bestblockhash = str(self.rpc.call("getbestblockhash")) - if bestblockhash != previousblockhash: - # Record the discovery before failing: the retry-spacing gate - # releases on a newer DETECTED tip, and without blockwait this - # mismatch is the only place the new tip becomes known. An - # unrecorded discovery would hold the immediate new-tip retry - # for the full failure holdoff. - self.observe_tip_for_refresh(bestblockhash) - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "qbit tip changed while fetching block template " - f"template_parent={previousblockhash} current={bestblockhash}" - ) - # The poll already paid for this template; seed the job-build cache so - # client job builds triggered by the refresh below reuse it instead of - # refetching one template per client. - artifacts = self.store_template_artifacts( - template, - generation=generation, - ) - if artifacts is not None: - return QbitTipTemplateSnapshot( - bestblockhash=bestblockhash, - previousblockhash=artifacts.previousblockhash, - template_fingerprint=artifacts.fingerprint, - template_generation=artifacts.generation, - template_artifacts=artifacts, - ) - raise TemplateRefreshBlocked( - "unable to derive exact artifacts for observed qbit template" + return ( + self._ensure_job_bundle_service() + .template_repository.fetch_coherent_snapshot() ) def ensure_reorg_reconciled_for_current_tip( @@ -12102,7 +6286,12 @@ def validate_live_chain_identity(self) -> None: f"configured qbit chain {configured!r} does not match RPC chain {reported!r}" ) - expected_genesis = env_optional("QBIT_EXPECTED_GENESIS_HASH") + config = getattr(self, "config", None) + expected_genesis = ( + config.rpc.expected_genesis_hash + if config is not None + else env_optional("QBIT_EXPECTED_GENESIS_HASH") + ) if configured in {"main", "mainnet"} and expected_genesis is None: raise RuntimeError("QBIT_EXPECTED_GENESIS_HASH is required on mainnet") if expected_genesis is not None: @@ -12137,7 +6326,19 @@ def validate_live_chain_identity(self) -> None: connections = int(network_info["connections"]) except (KeyError, TypeError, ValueError) as exc: raise RuntimeError("public-chain qbitd did not report a numeric peer count") from exc - minimum_peers = env_positive_int("PRISM_MIN_PEERS", 1) + minimum_peers = ( + env_positive_int( + "PRISM_MIN_PEERS", + 1, + environ=( + {} + if config.rpc.minimum_peers_raw is None + else {"PRISM_MIN_PEERS": config.rpc.minimum_peers_raw} + ), + ) + if config is not None + else env_positive_int("PRISM_MIN_PEERS", 1) + ) if connections < minimum_peers: raise RuntimeError( f"public-chain qbitd has {connections} peers, requires at least {minimum_peers}" @@ -12164,16 +6365,31 @@ def validate_live_template_and_fee_policy(self) -> None: template_time = int(template["curtime"]) except (KeyError, TypeError, ValueError) as exc: raise RuntimeError("getblocktemplate.curtime was missing or not numeric") from exc - max_age = env_nonnegative_int( - "PRISM_TEMPLATE_MAX_AGE_SECONDS", - DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + config = getattr(self, "config", None) + max_age = ( + env_nonnegative_int( + "PRISM_TEMPLATE_MAX_AGE_SECONDS", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + environ=( + {} + if config.jobs.template_max_age_raw is None + else { + "PRISM_TEMPLATE_MAX_AGE_SECONDS": config.jobs.template_max_age_raw + } + ), + ) + if config is not None + else env_nonnegative_int( + "PRISM_TEMPLATE_MAX_AGE_SECONDS", + DEFAULT_PRISM_TEMPLATE_MAX_AGE_SECONDS, + ) ) template_age = int(time.time()) - template_time if template_age > max_age: raise RuntimeError( f"qbit block template is stale: age={template_age}s exceeds {max_age}s" ) - self.last_successful_template_refresh_monotonic = time.monotonic() + self._ensure_tip_refresh_service().record_successful_refresh(time.monotonic()) settlement = self.prism_ctv_settlement_config( block_height=int(template["height"]) if "height" in template else None, @@ -12210,16 +6426,7 @@ def reconcile_prism_pool_blocks_once( _source_reserved: bool = False, ) -> dict[str, object]: """Serialize reconciliation against accepted-block finalization.""" - self._ensure_job_cache_state() - with self._payout_balance_mutation_lock: - with self._accepted_block_payout_preview_condition: - if any( - transition.landed - for transition in self._accepted_block_payout_previews.values() - ): - raise TemplateRefreshBlocked( - "accepted block payout confirmation is still pending" - ) + with self._payout_balance_mutation(): return self._reconcile_prism_pool_blocks_once( tip_hash=tip_hash, _force_publish=_force_publish, @@ -12252,7 +6459,7 @@ def _reconcile_prism_pool_blocks_once( # are asking about a different tip; repeated reconciliation of the # same tip must not supersede otherwise valid prepared work. with self.lock: - current_source_tip = self._payout_state_source[1] + current_source_tip = self._ensure_payout_state_service().snapshot().source[1] if current_source_tip != tip_hash: self._reserve_payout_state_source( "external_tip", @@ -12264,15 +6471,8 @@ def _reconcile_prism_pool_blocks_once( matured_payouts_total = 0 supersession_retries = 0 skip_recorded = False - max_supersession_retries = max( - 0, - int( - getattr( - self, - "payout_reconcile_supersession_retries", - DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, - ) - ), + max_supersession_retries = ( + self._ensure_payout_state_service().reconcile_supersession_retries ) def finish(*, trusted: bool) -> dict[str, object]: @@ -12296,7 +6496,7 @@ def retry_superseded_candidate() -> bool: self._block_payout_state_publication() return False with self.lock: - latest_tip = self._payout_state_source[1] + latest_tip = self._ensure_payout_state_service().snapshot().source[1] tip_hash = latest_tip or tip_hash return True @@ -12305,7 +6505,7 @@ def retry_superseded_candidate() -> bool: error_candidate: PayoutStateCandidate | None = None attempt_trusted = True try: - with self._payout_state_prepare_lock: + with self._ensure_payout_state_service().prepare_lock: prepared_started = time.monotonic() captured_source = self._capture_payout_state_source() payout_changed = False @@ -12519,70 +6719,19 @@ def retry_superseded_candidate() -> bool: return finish(trusted=attempt_trusted) def client_can_receive_jobs(self, client: ClientState) -> bool: - return ( - not getattr(client, "closing", False) - and client.subscribed - and client.authorized - and client.worker is not None - ) + return self._ensure_job_delivery_service().client_can_receive_jobs(client) def pool_readiness_latched(self) -> bool: - """Latch, once, the transition past min_ready_miners. - - Readiness is monotonic (a lifetime distinct-accepted-miner count), so - a single observation is permanent and later checks stay ledger-free. - The poll loop refreshes the latch outside the coordinator lock. - """ - if getattr(self, "_pool_ready_latched", False): - return True - try: - _, ready_miner_count = self.accepted_share_stats() - except Exception: - return False - if ready_miner_count >= getattr(self, "min_ready_miners", 3): - became_ready = False - with self.lock: - if not getattr(self, "_pool_ready_latched", False): - self._pool_ready_latched = True - self._retained_collection_refresh = None - became_ready = True - if became_ready: - # Collection work becomes obsolete without changing the qbit - # template fingerprint or payout generation. Start the health - # deadline at the readiness transition itself. - self._progress_note_refresh_pending() - return True - return False + return self._ensure_job_bundle_service().pool_readiness_latched() def client_needs_tip_template_refresh( self, client: ClientState, snapshot: QbitTipTemplateSnapshot, ) -> bool: - context = client.active_job - if context is None: - return True - if getattr(context, "collection_only", False) and getattr( - self, "_pool_ready_latched", False - ): - # The pool crossed min_ready_miners after this job was issued. - # A collection job keeps settling solved blocks solver-pays-all, - # so replace it with windowed work on the next poller pass. - return True - template = context.template - previousblockhash = str(template.get("previousblockhash", "")) - context_fingerprint = getattr(context, "template_fingerprint", None) - if context_fingerprint is None: - context_fingerprint = qbit_template_fingerprint(template) - context_payout_generation = int( - getattr(context, "payout_state_generation", 0) - ) - return ( - previousblockhash != snapshot.bestblockhash - or previousblockhash != snapshot.previousblockhash - or context_fingerprint != snapshot.template_fingerprint - or context_payout_generation - != int(getattr(self, "_payout_state_generation", 0)) + return self._ensure_job_delivery_service().client_needs_refresh( + client, + snapshot, ) def intervening_job_supersedes_snapshot( @@ -12591,419 +6740,54 @@ def intervening_job_supersedes_snapshot( expected_active_job: PrismJobContext | None, snapshot: QbitTipTemplateSnapshot, ) -> bool: - if active_job is expected_active_job or active_job is None: - return False - active_payout_generation = int( - getattr(active_job, "payout_state_generation", 0) - ) - if active_payout_generation < int( - getattr(self, "_payout_state_generation", 0) - ): - # Template ordering cannot make a payout-stale intervening job - # authoritative. Let the reconciled refresh replace it. - return False - active_parent_hash = str( - getattr(active_job, "template", {}).get("previousblockhash", "") + return self._ensure_job_delivery_service().intervening_supersedes( + active_job, + expected_active_job, + snapshot, ) - if ( - active_parent_hash != snapshot.bestblockhash - or active_parent_hash != snapshot.previousblockhash - ): - # Artifact generations order fetch starts, not chain tips. A fetch - # for the old tip can start after this exact new-tip observation - # and therefore carry a larger generation; it must not prevent - # the new-tip snapshot from replacing that stale work. - return False - active_generation = int(getattr(active_job, "template_generation", 0)) - snapshot_generation = int(getattr(snapshot, "template_generation", 0)) - if active_generation <= 0 or snapshot_generation <= 0: - # Legacy/test contexts without ordering metadata retain the safe - # behavior: never overwrite an unclassified intervening job. - return True - return active_generation >= snapshot_generation def client_tip_changed_for_snapshot( self, client: ClientState, snapshot: QbitTipTemplateSnapshot, ) -> bool: - context = client.active_job - if context is None: - return True - previousblockhash = str(context.template.get("previousblockhash", "")) - return ( - previousblockhash != snapshot.bestblockhash - or previousblockhash != snapshot.previousblockhash - or int(getattr(context, "payout_state_generation", 0)) - != int(getattr(self, "_payout_state_generation", 0)) - ) + return self._ensure_job_delivery_service().tip_changed(client, snapshot) def handle_client(self, client: ClientState) -> None: - reader = None - try: - reader = client.sock.makefile("r", encoding="utf-8", newline="\n") - for line in reader: - if self.stop_event.is_set(): - break - line = line.strip() - if not line: - continue - request_id: object = None - try: - request = json.loads(line) - if not isinstance(request, dict): - raise StratumError(20, "request must be an object") - request_id = request.get("id") - self.handle_request(client, request) - except json.JSONDecodeError as exc: - self.send_error(client, request_id, 20, f"invalid JSON: {exc.msg}") - except StratumError as exc: - self.send_error(client, request_id, exc.code, exc.message, reason=exc.reason) - if exc.disconnect: - break - except Exception: - print( - f"prism coordinator: client thread failed address={client.address}", - flush=True, - ) - traceback.print_exc() - break - except (OSError, ValueError) as exc: - if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: - self._record_stratum_resource_exhaustion( - listener_name=client.listener_name, - location="client-reader", - error_number=exc.errno, - ) - print( - "prism coordinator: stratum client socket failed " - f"address={client.address} error={exc!r}", - flush=True, - ) - finally: - try: - if reader is not None: - reader.close() - except (OSError, ValueError): - pass - finally: - self.disconnect_client(client) - with self.lock: - self._ensure_initial_job_state() - if getattr(client, "handler_thread_registered", False): - client.handler_thread_registered = False - self.handler_thread_count = max( - 0, - self.handler_thread_count - 1, - ) + self._ensure_stratum_session_service().handle_client(client) def disconnect_client(self, client: ClientState) -> None: - # Retire admission and fanout eligibility without waiting behind job - # delivery. Only the first caller owns socket close and final cleanup. - with self.lock: - # The timeout sweeper marks closing while leaving membership in - # place as its atomic handoff token. Whichever disconnect caller - # first removes that membership owns socket close and cleanup. - if getattr(client, "closing", False) and client not in self.clients: - return - client.closing = True - self.clients.discard(client) - self._cancel_pending_initial_job_locked(client, count=True) - - # Do not take send_lock here: shutdown must interrupt an in-flight - # sendall as well as the handler's blocking reader. - try: - client.close() - finally: - # Every mixed lock path uses job_update_lock -> coordinator lock. - # Retirement above holds neither while this potentially waits. - with client.job_update_lock: - with self.lock: - for job_id in tuple(client.active_job_ids): - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - client.active_job = None - self._ensure_evicted_job_state() - for job_id in tuple( - self.evicted_jobs_by_connection.get(client.connection_id, ()) - ): - self._remove_evicted_job_locked(job_id) - client.authorized = False - client.worker = None - client.username = "" - self._retain_current_collection_refresh_if_unrepresented() + self._ensure_stratum_session_service().disconnect_client(client) def handle_request(self, client: ClientState, request: dict[str, object]) -> None: - """Dispatch one request, translating shutdown races to Stratum errors.""" - try: - self._handle_request(client, request) - except ShutdownInProgress as exc: - # A request can pass the initial shutdown check immediately before - # writer admission closes. Preserve the normal protocol response - # instead of surfacing a generic client-thread failure. - raise StratumError( - 20, - "coordinator is shutting down", - reason=PRISM_REJECTION_POOL_CLOSED, - disconnect=True, - ) from exc + self._ensure_stratum_session_service().handle_request(client, request) def _handle_request(self, client: ClientState, request: dict[str, object]) -> None: - if self.stop_event.is_set() or self._ensure_shutdown_controller().phase != "running": - raise StratumError( - 20, - "coordinator is shutting down", - reason=PRISM_REJECTION_POOL_CLOSED, - disconnect=True, - ) - method = request.get("method") - params = request.get("params", []) - request_id = request.get("id") - if not isinstance(method, str): - raise StratumError(20, "missing method") - if not isinstance(params, list): - raise StratumError(20, "params must be an array") - - if method == "mining.configure": - self.handle_configure(client, request_id, params) - return - if method == "mining.subscribe": - with client.job_update_lock: - client.subscribed = True - self.send_result( - client, - request_id, - [[], client.extranonce1_hex, self.extranonce2_size], - ) - self._note_collection_identity_available(client) - needs_initial_job = client.authorized - if needs_initial_job: - self.request_initial_job_delivery(client) - return - if method == "mining.authorize": - username = str(params[0]) if params else "" - password = str(params[1]) if len(params) > 1 and params[1] is not None else "" - # Address validation may use RPC; it is unrelated to client job - # state and therefore stays outside the job-update lock. - worker = self.resolve_worker(username) - with client.job_update_lock: - was_authorized = client.authorized - if was_authorized: - with self.lock: - self._ensure_initial_job_state() - if ( - client not in self.pending_initial_jobs - and len(self.pending_initial_jobs) - >= self.stratum_max_pending_initial_jobs - and self._client_has_current_tip_job_locked(client) - ): - # Reject before mutating identity or difficulty. A - # working session keeps its current authorization - # when there is no live first-job slot for the - # superseding generation. - raise StratumError( - 20, - "initial job delivery capacity unavailable", - disconnect=False, - ) - if not self.reserve_client_username(client, worker): - raise StratumError( - 20, - "too many connections for username", - # A new connection has no useful session to preserve. A - # live miner re-authorizing to a full username does: keep - # its prior worker/session active after returning the - # capacity error. - disconnect=not client.authorized, - ) - # The password is authoritative for password-derived options: a - # re-authorize without d=/md= clears any prior override (a stored - # suggest_difficulty still applies via the request resolution). - with self._client_vardiff_lock(client): - client.requested_difficulty, client.requested_min_difficulty = ( - parse_stratum_password_options(password) - ) - target = self.apply_client_difficulty_requests(client) - if target is not None: - current = client.pending_share_difficulty or client.share_difficulty - if target != current: - if not was_authorized: - client.share_difficulty = target - client.pending_share_difficulty = None - else: - client.pending_share_difficulty = target - client.difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) + 1 - client.authorization_generation = int( - getattr(client, "authorization_generation", 0) - ) + 1 - client.authorized = True - client.authorized_monotonic = time.monotonic() - self.send_result(client, request_id, True) - self._note_collection_identity_available(client) - # Exactly one coalesced delivery represents this authorization, - # including a password-derived difficulty change. - self.request_initial_job_delivery(client) - return - if method == "mining.extranonce.subscribe": - self.send_result(client, request_id, True) - return - if method == "mining.suggest_difficulty": - self.handle_suggest_difficulty(client, request_id, params) - return - if method == "mining.submit": - accepted_and_closed = self.handle_submit(client, params) - try: - self.send_result(client, request_id, True) - finally: - self.refresh_jobs_after_pending_accepted_block(client) - if accepted_and_closed: - client.close() - return - raise StratumError(20, f"unsupported method {method}") + self._ensure_stratum_session_service()._handle_request(client, request) def handle_suggest_difficulty(self, client: ClientState, request_id: object, params: list[object]) -> None: - with client.job_update_lock: - suggested: Decimal | None = None - if params: - try: - suggested = Decimal(str(params[0])) - except Exception: - suggested = None - if suggested is not None and (not suggested.is_finite() or suggested <= 0): - suggested = None - if suggested is not None: - with self._client_vardiff_lock(client): - client.suggested_difficulty = suggested - target = self.apply_client_difficulty_requests(client) - if target is not None: - self.advertise_client_difficulty(client, target) - self.send_result(client, request_id, True) + self._ensure_stratum_session_service().handle_suggest_difficulty( + client, request_id, params + ) def handle_configure(self, client: ClientState, request_id: object, params: list[object]) -> None: - extensions = params[0] if params else [] - extension_params = params[1] if len(params) > 1 and isinstance(params[1], dict) else {} - result: dict[str, object] = {} - if isinstance(extensions, list): - for extension in extensions: - if extension == "version-rolling": - miner_mask = 0xFFFFFFFF - if "version-rolling.mask" in extension_params: - miner_mask = stratum_codec.parse_mask_hex( - extension_params["version-rolling.mask"], - field_name="version-rolling.mask", - ) - client.version_mask = self.version_mask & miner_mask - result["version-rolling"] = client.version_mask != 0 - result["version-rolling.mask"] = stratum_codec.format_mask_hex(client.version_mask) - else: - result[str(extension)] = False - self.send_result(client, request_id, result) + self._ensure_stratum_session_service().handle_configure( + client, request_id, params + ) def send_result(self, client: ClientState, request_id: object, result: object) -> None: - client.send({"id": request_id, "result": result, "error": None}) + client.send(stratum_result_payload(request_id, result)) def send_error(self, client: ClientState, request_id: object, code: int, message: str, *, reason: str | None = None) -> None: - data = {"reason_id": reason} if reason is not None else None - client.send({"id": request_id, "result": None, "error": [code, message, data]}) + client.send(stratum_error_payload(request_id, code, message, reason=reason)) def resolve_worker(self, username: str) -> WorkerIdentity: - payout_address, worker_name = split_worker_username(username) - try: - if not payout_address: - raise StratumError(20, "username base is empty") - script, p2mr_program_hex = self.validate_p2mr_address(payout_address, label="username base") - except StratumError as username_error: - fallback_address = getattr(self, "username_fallback_address", default_prism_username_fallback_address()) - if fallback_address is None: - raise username_error - print( - f"prism coordinator: username {username!r} cannot be used as a payout " - f"({username_error.message}); using fallback payout {fallback_address}", - flush=True, - ) - payout_address = fallback_address - script, p2mr_program_hex = self.validate_p2mr_address( - fallback_address, - label="PRISM_USERNAME_FALLBACK_ADDRESS", - ) - return WorkerIdentity( - username=username, - payout_address=payout_address, - worker_name=worker_name, - script_pubkey_hex=script, - p2mr_program_hex=p2mr_program_hex, - ) + return self._ensure_stratum_session_service().resolve_worker(username) def validate_p2mr_address(self, address: str, *, label: str) -> tuple[str, str]: - self._ensure_p2mr_address_cache_state() - with self._p2mr_address_cache_lock: - cached = self._p2mr_address_cache.get(address) - if cached is not None: - expires_monotonic, cached_result = cached - if expires_monotonic > time.monotonic(): - self._p2mr_address_cache.move_to_end(address) - return cached_result - self._p2mr_address_cache.pop(address, None) - pending = self._p2mr_address_validation_inflight.get(address) - is_leader = pending is None - if pending is None: - pending = _P2mrAddressValidationFlight() - self._p2mr_address_validation_inflight[address] = pending - else: - pending.waiters += 1 - - if not is_leader: - pending.event.wait() - if pending.result is not None: - return pending.result - if pending.error is not None: - self._raise_shared_p2mr_address_validation_error(pending.error) - raise RuntimeError("payout address validation completed without a result") - - try: - validation = self.rpc.call("validateaddress", [address]) - if not isinstance(validation, dict) or not validation.get("isvalid"): - raise StratumError(20, f"{label} is not a valid qbit address: {address}") - script = str(validation.get("scriptPubKey") or "") - if not script.startswith("5220") or len(script) != 68: - raise StratumError(20, f"{label} does not resolve to a P2MR script: {address}") - result = (script, script[4:]) - with self._p2mr_address_cache_lock: - max_entries = int( - getattr( - self, - "payout_address_cache_max_entries", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_MAX_ENTRIES, - ) - ) - ttl_seconds = float( - getattr( - self, - "payout_address_cache_ttl_seconds", - DEFAULT_PRISM_PAYOUT_ADDRESS_CACHE_TTL_SECONDS, - ) - ) - if max_entries > 0 and ttl_seconds > 0: - self._p2mr_address_cache[address] = ( - time.monotonic() + ttl_seconds, - result, - ) - self._p2mr_address_cache.move_to_end(address) - while len(self._p2mr_address_cache) > max_entries: - self._p2mr_address_cache.popitem(last=False) - pending.result = result - return result - except BaseException as exc: - with self._p2mr_address_cache_lock: - pending.error = exc - raise - finally: - with self._p2mr_address_cache_lock: - if self._p2mr_address_validation_inflight.get(address) is pending: - self._p2mr_address_validation_inflight.pop(address, None) - pending.event.set() + return self._ensure_stratum_session_service().address_validator.validate( + address, label=label + ) @staticmethod def _raise_shared_p2mr_address_validation_error(error: BaseException) -> None: @@ -13016,13 +6800,15 @@ def _raise_shared_p2mr_address_validation_error(error: BaseException) -> None: ) from error raise RuntimeError(str(error)) from error - def _ensure_p2mr_address_cache_state(self) -> None: + def _ensure_p2mr_address_cache_state(self, *, create_service: bool = True) -> None: if not hasattr(self, "_p2mr_address_cache_lock"): self._p2mr_address_cache_lock = threading.Lock() if not hasattr(self, "_p2mr_address_cache"): self._p2mr_address_cache = OrderedDict() if not hasattr(self, "_p2mr_address_validation_inflight"): self._p2mr_address_validation_inflight = {} + if create_service: + self._ensure_stratum_session_service() def maybe_send_job( self, @@ -13034,15 +6820,15 @@ def maybe_send_job( tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, tip_refresh_observation_sequence: int | None = None, ) -> bool: - with client.job_update_lock: - return self._maybe_send_job_locked( - client, - clean_jobs=clean_jobs, - raise_on_reorg_failure=raise_on_reorg_failure, - raise_on_build_failure=raise_on_build_failure, - tip_refresh_snapshot=tip_refresh_snapshot, - tip_refresh_observation_sequence=tip_refresh_observation_sequence, - ) + self._adopt_legacy_delivery_client(client) + return self._ensure_job_delivery_service().maybe_send_job( + client, + clean_jobs=clean_jobs, + raise_on_reorg_failure=raise_on_reorg_failure, + raise_on_build_failure=raise_on_build_failure, + tip_refresh_snapshot=tip_refresh_snapshot, + tip_refresh_observation_sequence=tip_refresh_observation_sequence, + ) def _maybe_send_job_locked( self, @@ -13054,359 +6840,23 @@ def _maybe_send_job_locked( tip_refresh_snapshot: QbitTipTemplateSnapshot | None = None, tip_refresh_observation_sequence: int | None = None, prepared_bundle: CachedJobBundle | None = None, - commit_guard: Callable[[], bool] | None = None, - commit_guard_lock: threading.RLock | None = None, + idle_authority: IdleDeliveryAuthority | None = None, prepared_bundle_allow_uncached: bool = False, ) -> bool: - if not client.subscribed or not client.authorized or client.worker is None: - return False - self._ensure_job_cache_state() - started = time.monotonic() - phases = self._job_build_phases() - phases.clear() - if getattr(self, "hot_path_log_enabled", False): - print( - f"prism coordinator: building job connection={client.connection_id} username={client.username}", - flush=True, - ) - phase_started = time.monotonic() - guarded_refresh = tip_refresh_snapshot is not None - if guarded_refresh != (tip_refresh_observation_sequence is not None): - raise ValueError("tip refresh snapshot and observation sequence must be paired") - if prepared_bundle is not None and guarded_refresh: - raise ValueError("prepared idle bundles cannot be combined with tip refresh guards") - if prepared_bundle_allow_uncached and prepared_bundle is None: - raise ValueError("uncached prepared delivery requires a prepared bundle") - if prepared_bundle is not None: - pass - elif guarded_refresh: - assert tip_refresh_snapshot is not None - assert tip_refresh_observation_sequence is not None - with self.lock: - refresh_current = self._tip_refresh_snapshot_current_locked( - tip_refresh_snapshot, - tip_refresh_observation_sequence, - ) - if not refresh_current: - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip refresh snapshot was superseded before client job build" - ) - try: - chain_view_untrusted = bool( - getattr(self, "reorg_reconciler_enabled", True) - and self.qbit_chain_view_untrusted() - ) - except Exception as exc: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit chain trust check failed before sequential client job build" - ) from exc - if chain_view_untrusted: - self._schedule_tip_refresh_retry() - raise TemplateRefreshBlocked( - "qbit chain view became untrusted before sequential client job build" - ) - else: - try: - if not self.ensure_reorg_reconciled_for_current_tip(): - if raise_on_reorg_failure: - raise TemplateRefreshBlocked( - "qbit chain view became untrusted before client job build" - ) - return False - except TemplateRefreshBlocked: - raise - except Exception as exc: - print( - f"prism coordinator: reorg reconciliation failed before job build " - f"connection={client.connection_id} username={client.username}; skipping this job", - flush=True, - ) - traceback.print_exc() - if raise_on_reorg_failure: - raise TemplateRefreshBlocked( - "reorg reconciliation failed before client job build" - ) from exc - return False - phases["reorg"] = time.monotonic() - phase_started - built_from_guarded_artifacts = bool( - guarded_refresh - and tip_refresh_snapshot.template_artifacts is not None - and "build_job_for_client" not in self.__dict__ - ) - try: - if prepared_bundle is not None: - context = self.stamp_job_for_client( - client, - prepared_bundle, - clean_jobs=clean_jobs, - ) - elif built_from_guarded_artifacts: - assert tip_refresh_snapshot is not None - assert tip_refresh_snapshot.template_artifacts is not None - context = self.build_job_for_client_from_artifacts( - client, - tip_refresh_snapshot.template_artifacts, - clean_jobs=clean_jobs, - publication_critical=True, - request_source="tip_refresh", - ) - else: - context = self.build_job_for_client(client, clean_jobs=clean_jobs) - except TemplateRefreshBlocked: - self._schedule_tip_refresh_retry() - if guarded_refresh or raise_on_reorg_failure or raise_on_build_failure: - raise - return False - except Exception as exc: - # A single bad template (e.g. a coinbase whose bytes collide with the - # extranonce placeholder, or a transient getblocktemplate failure) must - # never tear down the miner's connection. Log it, count it, and skip - # this job; the next share/retarget or block change rebuilds a fresh one. - # Only the build is isolated: nothing has been registered or sent yet, so - # there is no stale job state. Downstream send failures still surface to - # handle_client, which disconnects the (now dead) socket and cleans up. - with self.lock: - self.job_build_failure_count += 1 - print( - f"prism coordinator: job build failed connection={client.connection_id} " - f"username={client.username}; keeping client connected and skipping this template", - flush=True, - ) - traceback.print_exc() - if raise_on_build_failure: - raise _JobBuildFailed( - f"job build failed for connection {client.connection_id}" - ) from exc - return False - # Linearize direct delivery against the immutable publication pointer. - # Expensive build and ledger reads happened under the preparation lock, - # outside this admission boundary. - with self._job_cache_lock: - current_payout_generation = self._payout_state_generation - published_tip = self._published_payout_state.source_tip_hash - publication_blocked = self._payout_state_publication_blocked - context_payout_generation = int( - getattr( - context, - "payout_state_generation", - current_payout_generation, - ) - ) - context_template = getattr(context, "template", None) - context_parent = ( - str(context_template.get("previousblockhash", "")) - if isinstance(context_template, dict) - else "" - ) - with self.lock: - published_authority = getattr(self, "current_tip_first_seen", None) - published_authoritative = self._published_tip_authoritative_locked( - time.monotonic() - ) - pinned_published_delivery = bool( - context_parent - and published_authority is not None - and context_parent == published_authority[0] - and published_authoritative - ) - lapsed_live_validated = False - if ( - not guarded_refresh - and context_parent - and published_authority is not None - and not published_authoritative - ): - # The published authority lapsed, so per-share classification has - # fallen back to the live RPC read. Mirror the initial-job path: - # revalidate against the live tip (outside every lock), record - # the observation for the refresh machinery, and drop work that - # would be stale on arrival. - try: - lapsed_live_tip = str(self.rpc.call("getbestblockhash")) - except Exception: - lapsed_live_tip = None - if lapsed_live_tip is not None: - self.observe_tip_for_refresh(lapsed_live_tip) - if context_parent != lapsed_live_tip: - self._schedule_tip_refresh_retry() - return False - lapsed_live_validated = True - priority_delivery = ( - not publication_blocked - and context_payout_generation == current_payout_generation - and ( - published_tip is None - or context_parent == published_tip - # Reconciliation can source payout state at the detected tip - # before submit authority flips. Pinned published-tip work is - # still the currently creditable work, so it must keep the - # priority lane instead of tripping the gate's non-priority - # same-generation rejection for the whole unpublished window. - or pinned_published_delivery - ) + return self._ensure_job_delivery_service().maybe_send_job_locked( + client, + clean_jobs=clean_jobs, + raise_on_reorg_failure=raise_on_reorg_failure, + raise_on_build_failure=raise_on_build_failure, + tip_refresh_snapshot=tip_refresh_snapshot, + tip_refresh_observation_sequence=tip_refresh_observation_sequence, + prepared_bundle=prepared_bundle, + idle_authority=idle_authority, + prepared_bundle_allow_uncached=prepared_bundle_allow_uncached, ) - payout_gate_started = time.monotonic() - with self._payout_state_delivery_gate.delivery_cancelable( - lambda: context_payout_generation != self._payout_state_generation, - generation=context_payout_generation, - priority=priority_delivery, - ) as payout_admitted: - payout_gate_wait = max(0.0, time.monotonic() - payout_gate_started) - phases["payout_gate"] = phases.get("payout_gate", 0.0) + payout_gate_wait - self._observe_payout_gate_admission( - payout_admitted, - generation=context_payout_generation, - fallback_wait_seconds=payout_gate_wait, - ) - if not payout_admitted: - self._schedule_tip_refresh_retry() - if guarded_refresh: - raise TemplateRefreshSuperseded( - "payout state changed during client job build" - ) - return False - def commit_context_locked() -> bool: - if getattr(client, "closing", False): - return False - if commit_guard is not None and not commit_guard(): - return False - if guarded_refresh: - assert tip_refresh_snapshot is not None - assert tip_refresh_observation_sequence is not None - if not self._tip_refresh_snapshot_current_locked( - tip_refresh_snapshot, - tip_refresh_observation_sequence, - ): - self._schedule_tip_refresh_retry() - raise TemplateRefreshSuperseded( - "tip refresh snapshot was superseded during client job build" - ) - artifacts = tip_refresh_snapshot.template_artifacts - if built_from_guarded_artifacts and artifacts is not None and ( - context.template is not artifacts.template - or context.template_fingerprint != artifacts.fingerprint - or context.template_generation != artifacts.generation - ): - raise TemplateRefreshBlocked( - "client job build did not use the guarded refresh artifacts" - ) - else: - published_now = getattr(self, "current_tip_first_seen", None) - if published_now is None: - # Bootstrap: nothing published yet, so there is no - # authority for this work to contradict. - pass - elif self._published_tip_authoritative_locked( - time.monotonic() - ): - if context_parent and context_parent != published_now[0]: - # Authority moved while this direct build waited - # on the payout gate or client lock. Sending now - # would advertise work that classifies stale-job - # on arrival; skip and let the refresh fanout (or - # the pending retry) deliver current work. - self._schedule_tip_refresh_retry() - return False - elif context_parent and not lapsed_live_validated: - # The lease lapsed during the wait and this context - # never passed a live-tip revalidation: cached - # observations may be blind to the tip mining.submit - # now classifies against. Defer to a fresh pass whose - # pre-wait live check settles it without holding the - # coordinator lock across an RPC. - self._schedule_tip_refresh_retry() - return False - client.active_job = context - if clean_jobs: - for job_id in client.active_job_ids: - self.bury_evicted_job(client, job_id, prune=False) - self.jobs.pop(job_id, None) - client.active_job_ids.clear() - self.prune_evicted_job_graveyard(force=False) - self.jobs[context.job.job_id] = context - client.active_job_ids.add(context.job.job_id) - self.prune_client_active_jobs(client) - return True - - def commit_context() -> bool: - if prepared_bundle is not None: - # Exact cache identity and the client/window guard are one - # commit boundary. A tip or payout publication that wins - # this race makes the idle task a no-op instead of - # delivering stale work. - with self._job_cache_lock: - if not self._idle_bundle_current_locked( - client, - prepared_bundle, - allow_uncached=prepared_bundle_allow_uncached, - ): - return False - with self.lock: - return commit_context_locked() - with self.lock: - return commit_context_locked() - - if commit_guard_lock is None: - committed = commit_context() - else: - # Per-client state is acquired before coordinator admission. - # A busy share can delay only this delivery; it cannot hold - # self.lock while the delivery waits for Vardiff state. - with commit_guard_lock: - committed = commit_context() - if not committed: - return False - phase_started = time.monotonic() - self.send_job_update(client, context.job) - payout_admitted.mark_delivered() - self.apply_job_difficulty(client, context.job) - self.note_tip_work_delivered(client, str(context.template["previousblockhash"])) - delivered_monotonic = time.monotonic() - self._record_first_payout_delivery( - context_payout_generation, - delivered_monotonic, - ) - self._record_progress_delivery( - client, - context, - delivered_monotonic, - ) - self._consume_retained_collection_refresh(context) - self.note_initial_job_delivered( - client, - validated_current=guarded_refresh, - ) - phases["send"] = delivered_monotonic - phase_started - elapsed = time.monotonic() - started - self.observe_job_build_elapsed(elapsed, phases) - if getattr(self, "hot_path_log_enabled", False): - phase_report = ",".join( - f"{phase}:{phases[phase]:.3f}" - for phase in PRISM_JOB_BUILD_PHASES - if phase in phases - ) - print( - f"prism coordinator: sent job connection={client.connection_id} username={client.username} " - f"job={context.job.job_id} collection={context.collection_only} elapsed={elapsed:.3f}s " - f"phases={phase_report}", - flush=True, - ) - return True def prune_client_active_jobs(self, client: ClientState) -> None: - for job_id in tuple(client.active_job_ids): - if job_id not in self.jobs: - client.active_job_ids.discard(job_id) - ordered_active_job_ids = [ - job_id for job_id in self.jobs if job_id in client.active_job_ids - ] - while len(ordered_active_job_ids) > MAX_ACTIVE_PRISM_JOBS_PER_CLIENT: - oldest_job_id = ordered_active_job_ids.pop(0) - client.active_job_ids.remove(oldest_job_id) - self.bury_evicted_job(client, oldest_job_id) - self.jobs.pop(oldest_job_id, None) + self._ensure_job_delivery_service().prune_active(client) def send_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: self.send_difficulty_value(client, job.share_difficulty) @@ -13416,11 +6866,7 @@ def send_difficulty_value(self, client: ClientState, difficulty: Decimal) -> Non @staticmethod def difficulty_payload(difficulty: Decimal) -> dict[str, object]: - return { - "id": None, - "method": "mining.set_difficulty", - "params": [float(difficulty)], - } + return stratum_difficulty_payload(difficulty) def client_vardiff_config(self, client: ClientState) -> vardiff.VardiffConfig: """The difficulty policy for one client: its per-client specialization @@ -13468,202 +6914,78 @@ def client_minimum_advertised_difficulty(self, client: ClientState) -> Decimal: return max(client.minimum_advertised_difficulty, config.min_difficulty) def apply_job_difficulty(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: - with self._client_vardiff_lock(client): - config = ( - client.vardiff_config - or client.listener_vardiff_config - or self.vardiff_config - ) - if not config.enabled: - client.share_difficulty = job.share_difficulty - client.pending_share_difficulty = None - return - pending = client.pending_share_difficulty - client.share_difficulty = job.share_difficulty - if pending is not None and job.share_difficulty == pending: - client.pending_share_difficulty = None + self._ensure_job_delivery_service().apply_job_difficulty( + client, + job, + config=self.client_vardiff_config(client), + ) def apply_client_difficulty_requests(self, client: ClientState) -> Decimal | None: - """Specialize the client's difficulty policy from its recorded requests - (password ``d=``/``md=`` and ``mining.suggest_difficulty``), clamped to - the pristine listener bounds. The listener floor always wins: on a - high-diff listener no request can drop a client below the configured - minimum. Explicit ``d=`` outranks a suggestion. Returns the resolved - target difficulty, or None when the client requested nothing.""" - with self._client_vardiff_lock(client): - base = client.listener_vardiff_config or self.vardiff_config - requested = ( - client.requested_difficulty - if client.requested_difficulty is not None - else client.suggested_difficulty - ) - if requested is None and client.requested_min_difficulty is None: - # No live requests: drop any stale specialization so the client - # falls back to the pristine listener policy. - client.vardiff_config = None - return None - floor = base.min_difficulty - if client.requested_min_difficulty is not None: - floor = vardiff.clamp( - client.requested_min_difficulty, - base.min_difficulty, - base.max_difficulty, - ) - if requested is None: - requested = client.share_difficulty - target = vardiff.clamp(requested, floor, base.max_difficulty) - client.vardiff_config = dataclass_replace( - base, - min_difficulty=floor, - startup_difficulty=target, - ) - return target - - def advertise_client_difficulty(self, client: ClientState, target: Decimal) -> bool: - """Move a client to an explicitly requested difficulty. + """Specialize the client's difficulty policy from its recorded requests + (password ``d=``/``md=`` and ``mining.suggest_difficulty``), clamped to + the pristine listener bounds. The listener floor always wins: on a + high-diff listener no request can drop a client below the configured + minimum. Explicit ``d=`` outranks a suggestion. Returns the resolved + target difficulty, or None when the client requested nothing.""" + return self._ensure_job_delivery_service().apply_client_difficulty_requests( + client, + base=client.listener_vardiff_config or self.vardiff_config, + ) - Before the client can receive jobs the value is applied directly (the - first set_difficulty/notify pair picks it up). Afterwards it uses the - same job-gated pending mechanism as vardiff retargets: the difficulty - is advertised together with the job it applies to, or not at all. - Returns True only when a fresh set_difficulty/notify pair went out, so - callers about to send their own job can skip a duplicate pair.""" - with client.job_update_lock: - return self._advertise_client_difficulty_locked(client, target) + def advertise_client_difficulty( + self, + client: ClientState, + target: Decimal, + ) -> bool: + return self._ensure_job_delivery_service().advertise_client_difficulty( + client, + target, + ) def _advertise_client_difficulty_locked( self, client: ClientState, target: Decimal, ) -> bool: - applied_directly = False - schedule_initial = False - with self._client_vardiff_lock(client): - current = client.pending_share_difficulty or client.share_difficulty - if target == current: - return False - if not (client.subscribed and client.authorized) or ( - client.active_job is None and "maybe_send_job" not in self.__dict__ - ): - client.share_difficulty = target - client.pending_share_difficulty = None - client.difficulty_generation = int( - getattr(client, "difficulty_generation", 0) - ) + 1 - applied_directly = True - schedule_initial = bool( - client.subscribed - and client.authorized - and client.worker is not None - ) - else: - prior_pending = client.pending_share_difficulty - prior_generation = int( - getattr(client, "difficulty_generation", 0) - ) - advertised_generation = prior_generation + 1 - client.pending_share_difficulty = target - client.difficulty_generation = advertised_generation - if applied_directly: - if schedule_initial: - # A pending first-job request captured the previous difficulty - # generation. Replace it atomically so its cancellation callback - # hands the client slot to current work instead of disconnecting. - self.request_initial_job_delivery(client) - return False - with self.lock: - self._ensure_initial_job_state() - initial_pending = client in self.pending_initial_jobs - if initial_pending: - self.request_initial_job_delivery(client) - return False - if not self.stop_event.is_set() and self.maybe_send_job(client, clean_jobs=True): - return True - with self._client_vardiff_lock(client): - if ( - client.pending_share_difficulty == target - and int(getattr(client, "difficulty_generation", 0)) - == advertised_generation - ): - client.pending_share_difficulty = prior_pending - client.difficulty_generation = prior_generation - return False + return self._ensure_job_delivery_service().advertise_client_difficulty_locked( + client, + target, + ) def normalized_prior_balances(self, balances: list[dict[str, object]]) -> list[dict[str, object]]: - rows = [ - { - "recipient_id": str(balance.get("recipient_id", "")), - "order_key": str(balance.get("order_key", "")), - "p2mr_program_hex": str(balance.get("p2mr_program_hex", "")), - "balance_sats": int(balance.get("balance_sats", 0)), - } - for balance in balances - ] - rows.sort( - key=lambda row: ( - row["order_key"], - row["recipient_id"], - row["p2mr_program_hex"], - row["balance_sats"], - ) - ) - return rows + return self._ensure_payout_state_service().normalized_prior_balances(balances) def prior_balances_match_current(self, prior_balances: list[dict[str, object]]) -> bool: - return self.normalized_prior_balances(prior_balances) == self.normalized_prior_balances( - self.ledger.current_prior_balances() - ) + return self._ensure_payout_state_service().prior_balances_match_current(prior_balances) def send_job(self, client: ClientState, job: direct_stratum.DirectQbitStratumJob) -> None: client.send(self.job_payload(job)) @staticmethod def job_payload(job: direct_stratum.DirectQbitStratumJob) -> dict[str, object]: - return { - "id": None, - "method": "mining.notify", - "params": [ - job.job_id, - job.prevhash, - job.coinb1, - job.coinb2, - list(job.merkle_branch), - job.version, - job.nbits, - job.ntime, - job.clean_jobs, - ], - } + return stratum_job_payload(job) def send_job_update( self, client: ClientState, job: direct_stratum.DirectQbitStratumJob, ) -> None: - # Preserve instance-level send method replacements used by focused - # tests; normal coordinators use the atomic socket batch below. - if "send_difficulty" in self.__dict__ or "send_job" in self.__dict__: - self.send_difficulty(client, job) - self.send_job(client, job) - return - client.send_batch( - [ - self.difficulty_payload(job.share_difficulty), - self.job_payload(job), - ] + self._ensure_job_delivery_service().send_update( + client, + job, + split_send=( + "send_difficulty" in self.__dict__ or "send_job" in self.__dict__ + ), ) - def build_job_for_client(self, client: ClientState, *, clean_jobs: bool) -> PrismJobContext: - if client.worker is None: - raise StratumError(20, "client is not authorized") - self._ensure_job_cache_state() - artifacts = ( - self._retained_collection_artifacts() - or self.job_issuance_template_artifacts() - ) - return self.build_job_for_client_from_artifacts( + def build_job_for_client( + self, + client: ClientState, + *, + clean_jobs: bool, + ) -> PrismJobContext: + return self._ensure_job_delivery_service().build_job_for_client( client, - artifacts, clean_jobs=clean_jobs, ) @@ -13673,35 +6995,12 @@ def build_job_for_client_from_artifacts( artifacts: CachedTemplateArtifacts, *, clean_jobs: bool, - publication_critical: bool = False, - request_source: str = "routine", ) -> PrismJobContext: - self._ensure_job_cache_state() - phases = self._job_build_phases() - while True: - worker = client.worker - if worker is None: - raise StratumError(20, "client is not authorized") - progress_build_token = self._progress_bundle_build_started() - try: - cached_bundle = self.shared_job_bundle( - artifacts, - worker, - publication_critical=publication_critical, - request_source=request_source, - ) - finally: - self._progress_bundle_build_finished(progress_build_token) - current_worker = client.worker - if not cached_bundle.collection_only or current_worker == worker: - break - # Reauthorization changed a genuine collection input while the - # worker-specific bundle was being built. Re-select the latest - # identity without refetching or discarding the exact artifacts. - stamp_started = time.monotonic() - context = self.stamp_job_for_client(client, cached_bundle, clean_jobs=clean_jobs) - phases["stamp"] = phases.get("stamp", 0.0) + (time.monotonic() - stamp_started) - return context + return self._ensure_job_delivery_service().build_job_for_client_from_artifacts( + client, + artifacts, + clean_jobs=clean_jobs, + ) def build_collection_bundle( self, @@ -13717,34 +7016,13 @@ def build_collection_bundle( ctv_settlement: dict[str, object] | None = None, cancellation: _JobBuildCancellation | None = None, ) -> dict[str, Any]: - if cancellation is not None: - cancellation.raise_if_cancelled("collection payout derivation") - share = { - "share_seq": 1, - "share_id": "bootstrap-share", - "miner_id": worker.payout_address, - "order_key": worker.payout_address, - "p2mr_program_hex": worker.p2mr_program_hex, - "share_difficulty": network_difficulty, - "network_difficulty": network_difficulty, - "template_height": int(template["height"]) - 1, - "job_id": "bootstrap-job", - "job_issued_at_ms": issued_at_ms, - "accepted_at_ms": issued_at_ms, - "ntime": int(template["curtime"]), - } - return self.build_audit_bundle( - shares=[share], - found_block={ - "block_height": int(template["height"]), - "coinbase_value_sats": int(template["coinbasevalue"]), - "network_difficulty": network_difficulty, - "anchor_job_issued_at_ms": issued_at_ms, - }, - prior_balances=[], - coinbase_script_sig_suffix_hex=suffix_hex, - witness_merkle_leaves_hex=direct_stratum.witness_merkle_leaves_hex(transaction_hexes), - ctv_fee_parent_hash=str(template["previousblockhash"]), + return self._ensure_job_bundle_service().build_collection_bundle( + template=template, + transaction_hexes=transaction_hexes, + worker=worker, + network_difficulty=network_difficulty, + issued_at_ms=issued_at_ms, + suffix_hex=suffix_hex, summary_only=summary_only, payout_policy=payout_policy, ctv_settlement=ctv_settlement, @@ -13766,383 +7044,20 @@ def build_audit_bundle( ctv_settlement: dict[str, object] | None = None, cancellation: _JobBuildCancellation | None = None, ) -> dict[str, Any]: - self._ensure_job_cache_state() self._ensure_tip_refresh_state() - if cancellation is not None: - self._job_build_checkpoint("serialization", cancellation) - payload: dict[str, object] = { - "found_block": found_block, - "prior_balances": prior_balances, - "payout_policy": ( - self.prism_payout_policy() - if payout_policy is None - else payout_policy - ), - "coinbase_script_sig_suffix_hex": coinbase_script_sig_suffix_hex, - "witness_merkle_leaves_hex": witness_merkle_leaves_hex or [], - } - job_build_phase_local = getattr(self, "_job_build_phase_local", None) - record_phase_metrics = bool( - getattr(job_build_phase_local, "tip_refresh_metrics", False) - ) - if summary_only: - artifact_started = time.monotonic() - identity_indexes: dict[tuple[str, str, str], int] = {} - identities: list[tuple[str, str, str]] = [] - compact_shares: list[tuple[object, ...]] = [] - for share in shares: - identity = ( - str(share["miner_id"]), - str(share["order_key"]), - str(share["p2mr_program_hex"]), - ) - identity_index = identity_indexes.get(identity) - if identity_index is None: - identity_index = len(identities) - identity_indexes[identity] = identity_index - identities.append(identity) - compact_shares.append( - ( - share["share_seq"], - share["share_id"], - identity_index, - share["share_difficulty"], - share["job_issued_at_ms"], - share["accepted_at_ms"], - share.get("credit_policy"), - ) - ) - payload["compact_share_identities"] = identities - payload["compact_shares"] = compact_shares - if record_phase_metrics: - self._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - artifact_started, - ) - else: - payload["shares"] = shares - if ctv_settlement is None and payout_policy is None: - ctv_settlement = self.prism_ctv_settlement_config( - block_height=int(found_block["block_height"]), - parent_hash=ctv_fee_parent_hash, - ) - if ctv_settlement is not None: - payload["ctv_settlement"] = ctv_settlement - if canonical_output_path is not None and summary_only: - raise ValueError("canonical output and job summary output are mutually exclusive") - command = prism_tool_command("qbit-prism-build-audit-bundle") + [ - "--input", - "-", - "--signing-key-seed-hex", - self.signing_seed_hex, - "--ledger-signing-key-seed-hex", - self.ledger_attestation_signing_seed_hex, - ] - command.append("--job-summary-output" if summary_only else "--canonical-output") - if record_phase_metrics: - command.append("--phase-metrics") - if canonical_output_path is not None: - canonical_output_path.parent.mkdir(parents=True, exist_ok=True) - succeeded = False - created_output = False - try: - with ExitStack() as stack: - if canonical_output_path is None: - output = stack.enter_context( - tempfile.TemporaryFile(mode="w+", encoding="utf-8") - ) - else: - output = stack.enter_context( - canonical_output_path.open("x+", encoding="utf-8") - ) - created_output = True - stderr = stack.enter_context( - tempfile.TemporaryFile(mode="w+", encoding="utf-8") - ) - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=output, - stderr=stderr, - text=True, - encoding="utf-8", - close_fds=True, - ) - build_control = getattr( - job_build_phase_local, - "bundle_build_control", - None, - ) - if isinstance(build_control, _JobBundleBuildControl): - self._register_job_bundle_process(build_control, process) - with self._job_build_scheduler_lock: - if self._job_build_worker_restart_pending: - self.job_build_worker_counts["restarts"] += 1 - self._job_build_worker_restart_pending = False - self.job_build_worker_counts["starts"] += 1 - assert process.stdin is not None - input_byte_count = 0 - builder_started = time.monotonic() - worker_deadline = builder_started + float( - getattr( - self, - "bundle_build_timeout_seconds", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) - ) - coordinator = self - - class _CancelableInput: - def __init__(self, stream: Any) -> None: - self.stream = stream - try: - file_descriptor = int(stream.fileno()) - except (AttributeError, OSError, TypeError, ValueError): - # Lightweight process fakes used by embedders and - # tests do not necessarily expose an OS pipe. - self.file_descriptor: int | None = None - else: - os.set_blocking(file_descriptor, False) - self.file_descriptor = file_descriptor - - def check_cancelled(self) -> None: - if cancellation is not None: - cancellation.raise_if_cancelled( - "builder input serialization" - ) - if ( - isinstance(build_control, _JobBundleBuildControl) - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder input was canceled after supersession" - ) - if time.monotonic() >= worker_deadline: - with coordinator._tip_refresh_metrics_lock: - coordinator.tip_refresh_worker_failures += 1 - raise RuntimeError( - "qbit-prism-build-audit-bundle timed out" - ) - - def write(self, value: str) -> int: - nonlocal input_byte_count - self.check_cancelled() - if self.file_descriptor is None: - written = int(self.stream.write(value)) - input_byte_count += len(value[:written].encode("utf-8")) - return written - encoded = value.encode("utf-8") - remaining = memoryview(encoded) - while remaining: - self.check_cancelled() - try: - written = os.write( - self.file_descriptor, - remaining, - ) - except (BlockingIOError, InterruptedError): - time.sleep( - min( - 0.02, - PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, - ) - ) - continue - if written <= 0: - raise BrokenPipeError( - "audit-builder input pipe closed" - ) - input_byte_count += written - remaining = remaining[written:] - return len(value) - - serialization_started = time.monotonic() - try: - # iterencode writes bounded fragments to the child instead - # of allocating a second full JSON representation in Python. - json.dump( - payload, - _CancelableInput(process.stdin), - separators=(",", ":"), - ) - except BrokenPipeError: - # Prefer the builder's diagnostic below. - pass - except BaseException as exc: - try: - process.kill() - except ProcessLookupError: - pass - process.wait() - if isinstance( - exc, - (JobBuildCancelled, _JobBundleBuildSuperseded), - ): - with self._job_build_scheduler_lock: - self.job_build_worker_counts["terminations"] += 1 - self._job_build_worker_restart_pending = True - raise - finally: - phases = self._job_build_phases() - phases["input_serialization"] = phases.get( - "input_serialization", - 0.0, - ) + (time.monotonic() - serialization_started) - try: - process.stdin.close() - except (BlockingIOError, BrokenPipeError): - pass - if record_phase_metrics: - self._observe_tip_refresh_build_phase( - "serialization_copy", - time.monotonic() - serialization_started, - ) - self._record_tip_refresh_ipc_bytes("input", input_byte_count) - worker_started = time.monotonic() - terminated = False - returncode: int | None = None - if cancellation is None and not hasattr(process, "poll"): - returncode = process.wait() - else: - while returncode is None: - returncode = process.poll() - if returncode is not None: - break - cancelled_by_control = ( - isinstance(build_control, _JobBundleBuildControl) - and build_control.cancel_event.is_set() - ) - cancelled_by_request = ( - cancellation is not None and cancellation.is_set() - ) - if cancelled_by_control or cancelled_by_request: - terminated = True - process.terminate() - try: - returncode = process.wait( - timeout=max( - 0.0, - float( - getattr( - self, - "job_build_cancel_grace_seconds", - DEFAULT_PRISM_JOB_BUILD_CANCEL_GRACE_SECONDS, - ) - ), - ) - ) - except subprocess.TimeoutExpired: - process.kill() - returncode = process.wait() - with self._job_build_scheduler_lock: - self.job_build_worker_counts["terminations"] += 1 - self._job_build_worker_restart_pending = True - break - if time.monotonic() >= worker_deadline: - process.kill() - returncode = process.wait() - with self._tip_refresh_metrics_lock: - self.tip_refresh_worker_failures += 1 - raise RuntimeError( - "qbit-prism-build-audit-bundle timed out" - ) - time.sleep(min(0.02, PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS)) - phases["worker"] = phases.get("worker", 0.0) + ( - time.monotonic() - worker_started - ) - if terminated and cancellation is not None: - if cancellation.is_set(): - cancellation.raise_if_cancelled("builder worker") - if ( - terminated - and isinstance(build_control, _JobBundleBuildControl) - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder subprocess was canceled after supersession" - ) - stderr.seek(0) - error_text = stderr.read() - if returncode != 0: - with self._job_build_scheduler_lock: - self.job_build_worker_counts["crashes"] += 1 - self._job_build_worker_restart_pending = True - if record_phase_metrics: - with self._tip_refresh_metrics_lock: - self.tip_refresh_worker_failures += 1 - raise RuntimeError( - f"qbit-prism-build-audit-bundle failed: {error_text}" - ) - if ( - isinstance(build_control, _JobBundleBuildControl) - and build_control.cancel_event.is_set() - ): - raise _JobBundleBuildSuperseded( - "audit-builder result completed after supersession" - ) - output.flush() - output_size = os.fstat(output.fileno()).st_size - if record_phase_metrics: - self._record_tip_refresh_ipc_bytes("output", output_size) - for line in error_text.splitlines(): - if not line.startswith(PRISM_BUILDER_PHASE_METRICS_PREFIX): - continue - raw_metrics = line.removeprefix( - PRISM_BUILDER_PHASE_METRICS_PREFIX - ) - try: - metrics = json.loads(raw_metrics) - phase_seconds = metrics.get("phases_seconds", {}) - if isinstance(phase_seconds, dict): - for phase in ( - "payout_state_derivation", - "ctv_manifest_construction", - "coinbase_bundle_construction", - "signing_verification", - ): - elapsed = phase_seconds.get(phase) - if isinstance(elapsed, (int, float)): - self._observe_tip_refresh_build_phase( - phase, - float(elapsed), - ) - rust_serialization = sum( - float(metrics.get(name, 0.0)) - for name in ( - "input_deserialization_seconds", - "output_serialization_seconds", - ) - ) - self._observe_tip_refresh_build_phase( - "serialization_copy", - rust_serialization, - ) - except (TypeError, ValueError, json.JSONDecodeError): - # Metrics are diagnostic only. A malformed timing - # line must never invalidate an otherwise valid - # signed bundle. - pass - if canonical_output_path is not None: - os.fsync(output.fileno()) - output.seek(0) - output_started = time.monotonic() - if cancellation is not None: - cancellation.raise_if_cancelled("builder output serialization") - bundle = json.load(output) - phases["output_serialization"] = phases.get( - "output_serialization", - 0.0, - ) + (time.monotonic() - output_started) - if cancellation is not None: - cancellation.raise_if_cancelled("builder verification") - succeeded = True - return bundle - finally: - if canonical_output_path is not None and created_output and not succeeded: - try: - canonical_output_path.unlink() - except FileNotFoundError: - pass + return self._ensure_bundle_compiler().build_audit_bundle( + shares=shares, + found_block=found_block, + prior_balances=prior_balances, + coinbase_script_sig_suffix_hex=coinbase_script_sig_suffix_hex, + witness_merkle_leaves_hex=witness_merkle_leaves_hex, + ctv_fee_parent_hash=ctv_fee_parent_hash, + canonical_output_path=canonical_output_path, + summary_only=summary_only, + payout_policy=payout_policy, + ctv_settlement=ctv_settlement, + cancellation=cancellation, + ) def coinbase_script_sig_suffix_hex(self, extranonce1_hex: str, extranonce2_hex: str) -> str: extranonce1_hex = validate_hex(extranonce1_hex, name="extranonce1") @@ -14370,7 +7285,7 @@ def handle_submit(self, client: ClientState, params: list[object]) -> bool: # then both reflect the real outcome -- never an "accepted" ack with # no ledger row. This path is rare (an honest miner does not submit # below its assigned target), so it does not affect the async - # common-path latency. On failure the submitter already recorded + # common-path latency. On failure the submitter already recorded # the specific block-failure reason; reject the miner as # low-difficulty (the share was, after all, below its target). The # submitter already recorded the specific block-failure reason in @@ -14378,69 +7293,39 @@ def handle_submit(self, client: ClientState, params: list[object]) -> bool: # the miner-facing rejection (globally and per worker) so this rare # synchronous path is not missing from the rejection metrics. persist_intent = getattr(self.ledger, "persist_block_candidate_intent", None) + candidate_intent_durable = False + share_writer = self._ensure_share_writer_service() try: candidate_intent = self.block_candidate_intent(candidate) if callable(persist_intent): persist_intent(candidate_intent) + candidate_intent_durable = True + # Move this exact stamped attempt into independent active-actor + # ownership and ensure the stable retry/outbox holder before + # submission can fail or another same-hash actor can finish. + share_writer.begin_candidate_actor(candidate.pending_share) except BaseException: # No retry slot is safe until the pre-submit outbox boundary is - # durable. Let the miner retry this submission instead. Without - # a durable intent nothing can commit this stamped share, so - # stop holding the snapshot anchor floor under it. - self._finish_pending_share_commit(pending_share) + # durable. If persistence itself failed, release only this + # attempt. Once persistence returned, however, a later dynamic + # actor-promotion failure must conservatively keep the attempt + # floor: the durable row is replayable even if wiring failed. + if not candidate_intent_durable: + self._finish_pending_share_attempt(pending_share) self._forget_recent_share_key(share_key) raise try: - self._mark_block_candidate_attempted( - str(candidate.submission.block_hash_hex).lower() - ) - block_landed = self.submit_block_candidate(candidate) - except BaseException: - self._retain_block_candidate_for_retry(candidate) - self._forget_recent_share_key(share_key) - raise - if not block_landed: - outcome = getattr(self, "_block_candidate_outcome", None) - reason = getattr(outcome, "reason", None) if outcome is not None else None - retryable_reasons = {None, *PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS} - if reason in retryable_reasons: - # The durable outbox may still land and credit this block. - # Close without a Stratum result instead of issuing a false - # definitive rejection for an uncertain outcome. - self._retain_block_candidate_for_retry(candidate) - self._forget_recent_share_key(share_key) - raise RuntimeError( - "block candidate outcome is pending durable retry" - ) - if reason not in retryable_reasons: - # This process will never credit the candidate share now: - # release its snapshot anchor floor entry before the - # terminal outbox update, whose failure would still leave - # only restart replay (a fresh PendingShare) to credit it. - self._finish_pending_share_commit(candidate.pending_share) - finish = getattr(self.ledger, "mark_block_candidate_abandoned", None) - if callable(finish): - finish(block_hash=submission.block_hash_hex, error=reason) - # Once the durable outbox cannot replay this candidate, - # its landed-transition tombstone no longer protects a - # crash seam and would otherwise accumulate forever. - self._clear_accepted_block_payout_preview( - submission.block_hash_hex - ) - self._forget_recent_share_key(share_key) - self.reject_stratum( - 23, - PRISM_REJECTION_LOW_DIFFICULTY, - "low difficulty share", - worker=worker_name, + return self._submit_synchronous_credit_candidate( + candidate, + share_key=share_key, + worker_name=worker_name, + evicted_entry=evicted_entry, + credit_policy=credit_policy, ) - else: - finish = getattr(self.ledger, "mark_block_candidate_submitted", None) - if callable(finish): - finish(block_hash=submission.block_hash_hex) - if evicted_entry is not None: - self.note_evicted_job_submit(credit_policy) - return False + finally: + # The helper either committed credit, reached durable terminal + # noncredit, or published a stable retry before returning. + share_writer.finish_candidate_actor(candidate.pending_share) # A block-worthy submission that met the share target is a valid share # regardless of the block's fate: credit it now, acknowledge the miner # immediately, and land the block from the dedicated submitter thread @@ -14448,6 +7333,11 @@ def handle_submit(self, client: ClientState, params: list[object]) -> bool: # share credit. try: candidate_intent = self.block_candidate_intent(candidate) + except BaseException: + self._finish_pending_share_attempt(pending_share) + self._forget_recent_share_key(share_key) + raise + try: self.append_accepted_share( client, context, @@ -14459,14 +7349,74 @@ def handle_submit(self, client: ClientState, params: list[object]) -> bool: if evicted_entry is not None: self.note_evicted_job_submit(credit_policy) except BaseException: - # Idempotent with append_accepted_share's own release; also covers - # an intent serialization failure before the append started. - self._finish_pending_share_commit(pending_share) + # Once append begins, S3's pre-visibility rollback, queue-visible + # writer, or synchronous append owns this attempt holder. Do not + # race that owner by releasing the floor from the client thread. self._forget_recent_share_key(share_key) raise self.enqueue_block_candidate(candidate) return False + def _submit_synchronous_credit_candidate( + self, + candidate: PrismBlockCandidate, + *, + share_key: tuple[str, str], + worker_name: str, + evicted_entry: EvictedJobEntry | None, + credit_policy: str | None, + ) -> bool: + """Resolve one below-target block while its active S3 actor is held.""" + submission = candidate.submission + try: + block_landed = self.submit_block_candidate(candidate) + except BaseException: + self._retain_block_candidate_for_retry(candidate) + self._forget_recent_share_key(share_key) + raise + if not block_landed: + outcome = getattr(self, "_block_candidate_outcome", None) + reason = getattr(outcome, "reason", None) if outcome is not None else None + retryable_reasons = {None, *PRISM_RETRYABLE_BLOCK_CANDIDATE_REASONS} + if reason in retryable_reasons: + # The durable outbox may still land and credit this block. + # Close without a Stratum result instead of issuing a false + # definitive rejection for an uncertain outcome. + self._retain_block_candidate_for_retry(candidate) + self._forget_recent_share_key(share_key) + raise RuntimeError("block candidate outcome is pending durable retry") + # This process will never credit the candidate share only after the + # durable outbox update becomes terminal. The actor remains held + # across that update, so a concurrent same-hash actor cannot lose + # its older acceptance floor when this stable holder is removed. + finish = getattr(self.ledger, "mark_block_candidate_abandoned", None) + if callable(finish): + try: + finish(block_hash=submission.block_hash_hex, error=reason) + except BaseException: + self._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) + raise + self._finish_pending_share_candidate(candidate.pending_share) + # Once the durable outbox cannot replay this candidate, its landed + # transition tombstone no longer protects a crash seam. + self._clear_accepted_block_payout_preview(submission.block_hash_hex) + self._forget_recent_share_key(share_key) + self.reject_stratum( + 23, + PRISM_REJECTION_LOW_DIFFICULTY, + "low difficulty share", + worker=worker_name, + ) + finish = getattr(self.ledger, "mark_block_candidate_submitted", None) + if callable(finish): + finish(block_hash=submission.block_hash_hex) + self._finish_pending_share_candidate(candidate.pending_share) + if evicted_entry is not None: + self.note_evicted_job_submit(credit_policy) + return False + @staticmethod def block_candidate_intent(candidate: PrismBlockCandidate) -> dict[str, Any]: """Return the immutable JSON needed to resume a candidate after restart.""" @@ -14514,8 +7464,21 @@ def block_candidate_intent(candidate: PrismBlockCandidate) -> dict[str, Any]: json.dumps(intent, separators=(",", ":"), sort_keys=True) return intent - @staticmethod - def block_candidate_from_intent(intent: dict[str, Any]) -> PrismBlockCandidate: + def block_candidate_from_intent( + self, + intent: dict[str, Any] | None = None, + ) -> PrismBlockCandidate: + # This helper was historically a static method. Preserve class-level + # decode calls while instance calls additionally adopt S3's durable + # credit-candidate holder before the reconstructed value is published. + coordinator: PrismCoordinator | None + if intent is None: + if not isinstance(self, dict): + raise TypeError("block candidate intent must be an object") + intent = self + coordinator = None + else: + coordinator = self if intent.get("schema") != "qbit.prism.block-candidate-intent.v1": raise ValueError("unsupported block candidate intent schema") block_hash = str(intent["block_hash_hex"]).lower() @@ -14565,7 +7528,7 @@ def block_candidate_from_intent(intent: dict[str, Any]) -> PrismBlockCandidate: else None ), ) - return PrismBlockCandidate( + candidate = PrismBlockCandidate( context=context, submission=submission, extranonce1_hex=str(intent["extranonce1_hex"]), @@ -14574,12 +7537,18 @@ def block_candidate_from_intent(intent: dict[str, Any]) -> PrismBlockCandidate: client=SimpleNamespace(username=str(intent["username"])), credit_share_on_accept=bool(intent.get("credit_share_on_accept", False)), ) + if candidate.credit_share_on_accept and coordinator is not None: + # A below-target candidate can credit this older accepted stamp + # after durable replay. Adopt its stable logical floor before + # startup prewarm/job issuance. Ordinary asynchronous candidates + # already committed their share and need no floor. + coordinator._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) + return candidate def _ensure_pending_share_commit_state(self) -> None: - if not hasattr(self, "_pending_share_commit_lock"): - self._pending_share_commit_lock = threading.Lock() - if not hasattr(self, "_pending_share_commit_floor"): - self._pending_share_commit_floor = {} + self._ensure_share_writer_service() def _finish_pending_share_commit(self, pending_share: PendingShare) -> None: """Drop a share from the snapshot anchor floor. @@ -14587,12 +7556,20 @@ def _finish_pending_share_commit(self, pending_share: PendingShare) -> None: Called once the share's ledger row reached a terminal outcome in this process: durably committed, rejected back to the miner, recovered to the on-disk replay file, or its block candidate terminally abandoned. - Idempotent, and a no-op for shares this process never registered - (intent/journal replays re-create PendingShare objects from JSON). + Idempotent. Credit-bearing candidate intents are adopted under their + durable share ID during replay, so a reconstructed PendingShare can + release the same logical lease; ordinary already-credited candidate + replays remain unregistered no-ops. """ - self._ensure_pending_share_commit_state() - with self._pending_share_commit_lock: - self._pending_share_commit_floor.pop(id(pending_share), None) + self._ensure_share_writer_service().finish_pending_share(pending_share) + + def _finish_pending_share_attempt(self, pending_share: PendingShare) -> None: + """Release only one stamped submission's process-local floor holder.""" + self._ensure_share_writer_service().finish_pending_attempt(pending_share) + + def _finish_pending_share_candidate(self, pending_share: PendingShare) -> None: + """Release only a terminal durable credit-candidate floor holder.""" + self._ensure_share_writer_service().finish_pending_candidate(pending_share) def _job_snapshot_anchor_ms(self, issued_at_ms: int) -> int: """Clamp a share-snapshot anchor below every pending share commit. @@ -14608,35 +7585,7 @@ def _job_snapshot_anchor_ms(self, issued_at_ms: int) -> int: issued snapshot reproducible without making job builds wait behind the writer connection. """ - self._ensure_pending_share_commit_state() - stale_share_ids: list[str] = [] - floor_ms: int | None = None - now_monotonic = time.monotonic() - with self._pending_share_commit_lock: - for entry in self._pending_share_commit_floor.values(): - share = entry[0] - accepted_at_ms = int(share.accepted_at_ms) - if floor_ms is None or accepted_at_ms < floor_ms: - floor_ms = accepted_at_ms - if ( - not entry[2] - and now_monotonic - float(entry[1]) - > PRISM_PENDING_SHARE_COMMIT_WARN_SECONDS - ): - entry[2] = True - stale_share_ids.append(str(share.share_id)) - for share_id in stale_share_ids: - # A long-held floor entry is a wedged writer or a leaked release - # path, not normal group-commit latency. The share-commit liveness - # watchdog owns recovery; this log makes the anchor clamp visible. - print( - "prism coordinator: pending share commit is holding the job " - f"snapshot anchor floor share_id={share_id}", - flush=True, - ) - if floor_ms is None: - return issued_at_ms - return min(issued_at_ms, floor_ms - 1) + return self._ensure_share_writer_service().snapshot_anchor_ms(issued_at_ms) def pending_share_from_submission( self, @@ -14647,34 +7596,25 @@ def pending_share_from_submission( credit_policy: str | None = None, ) -> PendingShare: share_difficulty = self.accepted_share_difficulty(context) - self._ensure_pending_share_commit_state() - # Assign accepted_at_ms and register the commit-floor entry under one - # lock hold: a snapshot anchored between assignment and registration - # could otherwise anchor at or above this share and miss its later - # commit. Released via _finish_pending_share_commit. - with self._pending_share_commit_lock: - pending = PendingShare( + return self._ensure_share_writer_service().make_pending_share( + PendingShareInput( share_id=f"{context.worker.username}:{submission.block_hash_hex}", miner_id=context.worker.payout_address, order_key=context.worker.payout_address, p2mr_program_hex=context.worker.p2mr_program_hex, share_difficulty=share_difficulty, - network_difficulty=max(1, int(context.found_block["network_difficulty"])), + network_difficulty=max( + 1, + int(context.found_block["network_difficulty"]), + ), template_height=int(context.template["height"]) - 1, job_id=context.job.job_id, job_issued_at_ms=context.issued_at_ms, - accepted_at_ms=now_ms(), ntime=int(ntime_hex, 16), credit_policy=credit_policy, ) - self._pending_share_commit_floor[id(pending)] = [ - pending, - time.monotonic(), - False, - ] - return pending + ) - @ledger_writer_operation("share_persistence") def append_accepted_share( self, client: ClientState, @@ -14695,331 +7635,53 @@ def append_accepted_share( candidate_intent=candidate_intent, ) try: - if getattr(self, "share_writer_active", False): - self.enqueue_share_append(entry, wait=True) - else: - self._append_share_entry(entry) - finally: - # The append reached a terminal outcome for this process: durably - # committed, or surfaced an error the miner will retry with a fresh - # share. Either way the stamped share no longer holds the snapshot - # anchor floor. Idempotent with the group-commit writer's release. - self._finish_pending_share_commit(pending_share) - # Only committed shares affect public accounting, vardiff, and the - # response that handle_request sends immediately after this returns. - self.note_worker_accepted_share(context.worker.username, credit_policy) - self.note_vardiff_accepted_share(client, context.job) - - def enqueue_share_append(self, entry: PendingShareAppend, *, wait: bool = False) -> None: - queue_obj = getattr(self, "share_append_queue", None) - if queue_obj is None: - queue_obj = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) - self.share_append_queue = queue_obj - if entry.writer_token is None: - entry.writer_token = self._ensure_shutdown_controller().reserve_writer( - "share_persistence" - ) - try: - if wait: - queue_obj.put( - entry, - timeout=getattr(self, "share_commit_timeout_seconds", 15.0), - ) - else: - queue_obj.put_nowait(entry) - except queue.Full: - entry.writer_token.finish() - entry.writer_token = None - raise StratumError( - 20, - "share ledger commit queue is full", - reason=PRISM_REJECTION_INTERNAL_ERROR, - ) - if not wait: - return - # Once admitted, wait for a definite transaction outcome. A local - # timeout is ambiguous because Postgres may commit immediately after - # it; the liveness watchdog owns recovery from a wedged writer. - entry.committed.wait() - if entry.error is not None: + self._ensure_share_writer_service().append_and_wait(entry) + except ShareWriterQueueFull as exc: raise StratumError( 20, - f"share ledger commit failed: {entry.error}", + str(exc), reason=PRISM_REJECTION_INTERNAL_ERROR, - ) - - def share_append_loop(self) -> None: - while True: - self._record_heartbeat("share_writer") - queue_obj = getattr(self, "share_append_queue", None) - if queue_obj is None: - queue_obj = queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) - self.share_append_queue = queue_obj - stopping = self.stop_event.is_set() - try: - entry = queue_obj.get(timeout=0.2 if stopping else 1.0) - except queue.Empty: - controller = self._ensure_shutdown_controller() - if ( - stopping - and controller.writer_admission_closed() - and not controller.has_active_writer( - { - "share_submission", - "share_persistence", - "accepted_block_handling", - } - ) - ): - return - continue - batch = [entry] - batch_size = max(1, int(getattr(self, "share_commit_batch_size", 64))) - deadline = time.monotonic() + max( - 0.0, float(getattr(self, "share_commit_linger_seconds", 0.005)) - ) - if entry.candidate_intent is not None: - deadline = time.monotonic() - while len(batch) < batch_size: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - try: - next_entry = queue_obj.get(timeout=remaining) - batch.append(next_entry) - if next_entry.candidate_intent is not None: - break - except queue.Empty: - break - self._append_share_batch(batch) - - def _append_share_batch(self, batch: list[PendingShareAppend]) -> bool: - """Commit a writer batch, then release every waiting submitter.""" - try: - append_batch = getattr(self.ledger, "append_batch", None) - if callable(append_batch): - records = append_batch( - [(entry.pending_share, entry.candidate_intent) for entry in batch] - ) - else: - # Compatibility for lightweight test/tool ledgers. Production's - # Postgres ledger always supplies the atomic batch method. - records = [self.ledger.append(entry.pending_share) for entry in batch] - if len(records) != len(batch): - raise RuntimeError("share ledger returned an incomplete commit batch") - hot_path_log = getattr(self, "hot_path_log_enabled", False) - for entry, record in zip(batch, records, strict=True): - entry.record = record - if hot_path_log: - print( - "prism coordinator: accepted share " - f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " - f"hash={entry.block_hash_hex} collection={entry.collection_only} " - f"credit_policy={entry.credit_policy or 'normal'}", - flush=True, - ) - return True + ) from exc except Exception as exc: - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.share_append_failure_count = ( - int(getattr(self, "share_append_failure_count", 0)) + len(batch) - ) - for entry in batch: - entry.error = exc - print( - f"prism coordinator: share ledger group commit failed count={len(batch)}", - flush=True, - ) - traceback.print_exc() - return False - finally: - for entry in batch: - self._finish_pending_share_commit(entry.pending_share) - entry.committed.set() - if entry.writer_token is not None: - entry.writer_token.finish() - entry.writer_token = None - - def _recover_share_to_disk(self, entry: PendingShareAppend, reason: str) -> None: - """Durably capture an acked share the writer could not persist. + if isinstance(exc, ShareWriterError): + raise StratumError( + 20, + str(exc), + reason=PRISM_REJECTION_INTERNAL_ERROR, + ) from exc + raise + # Only committed shares affect public accounting, vardiff, and the + # response that handle_request sends immediately after this returns. + self.note_worker_accepted_share(context.worker.username, credit_policy) + self.note_vardiff_accepted_share(client, context.job) - Appends the canonical pending-share JSON to the recovery file (fsynced) - so a ledger outage or shutdown never silently loses a share the miner - was told was accepted; replayed on the next start. Best-effort: if even - the recovery write fails, log loudly rather than raise on the writer. - """ - path = getattr(self, "share_recovery_path", None) - if path is None: - print( - "prism coordinator: WOULD LOSE acked share (no recovery path) " - f"share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - return + def enqueue_share_append(self, entry: PendingShareAppend, *, wait: bool = False) -> None: try: - payload = json.dumps(dataclasses.asdict(entry.pending_share), separators=(",", ":")) - except Exception: - payload = None - with getattr(self, "share_recovery_lock", threading.Lock()): - try: - path.parent.mkdir(parents=True, exist_ok=True) - if payload is None: - raise ValueError("pending share is not serializable") - with open(path, "a", encoding="utf-8") as handle: - handle.write(payload + "\n") - handle.flush() - os.fsync(handle.fileno()) - self.shares_recovered_to_disk = ( - int(getattr(self, "shares_recovered_to_disk", 0)) + 1 - ) - print( - "prism coordinator: recovered unpersisted acked share to disk " - f"share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - except Exception: - print( - "prism coordinator: FAILED to recover acked share to disk; " - f"share may be lost share_id={entry.pending_share.share_id} reason={reason}", - flush=True, - ) - traceback.print_exc() + self._ensure_share_writer_service().enqueue(entry, wait=wait) + except ShareWriterQueueFull as exc: + raise StratumError( + 20, + str(exc), + reason=PRISM_REJECTION_INTERNAL_ERROR, + ) from exc - @ledger_writer_operation("share_recovery_replay") - def replay_recovered_shares(self) -> int: - """Replay any recovery-file shares into the ledger at startup. + def share_append_loop(self) -> None: + self._ensure_share_writer_service().run() - Idempotent: both ledgers raise on a duplicate share_id, so a row already - committed by an earlier partial replay is skipped (not double-counted) - and does not stop the pass. The file is cleared only after a clean pass, - so a transient failure here never drops shares. - """ - path = getattr(self, "share_recovery_path", None) - if path is None or not path.exists(): - return 0 - try: - lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] - except Exception: - print("prism coordinator: could not read share recovery file", flush=True) - traceback.print_exc() - return 0 - # Parse line-by-line and skip any single unparseable line rather than - # aborting the whole replay: a crash mid-append can leave the last line - # torn, and one torn line must not block the intact shares before it. - pendings: list[PendingShare] = [] - parse_failed = False - for line in lines: - try: - pendings.append(PendingShare(**json.loads(line))) - except Exception: - parse_failed = True - print("prism coordinator: skipping an unparseable recovered share line", flush=True) - traceback.print_exc() - # Replay in acceptance order. A share recovered out of FIFO order (a - # ledger flap during the shutdown drain, or an overflow-recovered newest - # share) otherwise sorts by file order; ordering by accepted_at_ms lands - # each share with a share_seq consistent with when it was accepted, so - # the reward window stays correctly ordered. - pendings.sort(key=lambda pending: pending.accepted_at_ms) - replayed = 0 - skipped_duplicates = 0 - for pending in pendings: - try: - self.ledger.append(pending) - replayed += 1 - except Exception as exc: - if "duplicate share_id" in str(exc): - # Already committed by an earlier (partial) replay. Both - # ledgers raise on a duplicate share_id; treat it as done and - # keep going so replay is idempotent -- otherwise a retry - # after a partial pass would stop on the first committed row - # and strand every share after it. - skipped_duplicates += 1 - continue - print("prism coordinator: failed to replay a recovered share; keeping the file", flush=True) - traceback.print_exc() - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - return replayed - if skipped_duplicates: - print( - f"prism coordinator: skipped {skipped_duplicates} already-committed " - "recovered share(s) during replay", - flush=True, - ) - if parse_failed: - # Keep the file (with its intact-but-already-replayed lines, which - # the ledger dedups on a re-run) so the torn line is preserved for - # inspection rather than silently discarded. - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - if replayed: - print(f"prism coordinator: replayed {replayed} recovered share(s) into the ledger", flush=True) - return replayed - try: - path.unlink() - except FileNotFoundError: - pass - self.shares_replayed = int(getattr(self, "shares_replayed", 0)) + replayed - if replayed: - print(f"prism coordinator: replayed {replayed} recovered share(s) into the ledger", flush=True) - return replayed + def _append_share_batch(self, batch: list[PendingShareAppend]) -> bool: + return self._ensure_share_writer_service().append_batch(batch) - def _append_share_entry(self, entry: PendingShareAppend, *, retry_until_stopped: bool = False) -> bool: - """Synchronously append one accepted share. + def _recover_share_to_disk(self, entry: PendingShareAppend, reason: str) -> None: + self._ensure_share_writer_service().recover_to_disk(entry, reason) - On the writer thread a transient ledger failure retries with capped - backoff so ordering is preserved and nothing is silently lost; the - synchronous path (no writer) propagates the exception exactly as the - pre-async code did. + def replay_recovered_shares(self) -> int: + return self._ensure_share_writer_service().replay_recovery_file() - Returns True when the share was persisted to the ledger, or False when - it was recovered to disk instead (ledger still down at shutdown). The - caller uses that to keep the shutdown drain in order. - """ - backoff_seconds = 0.5 - while True: - try: - append_batch = getattr(self.ledger, "append_batch", None) - if callable(append_batch): - record = append_batch( - [(entry.pending_share, entry.candidate_intent)] - )[0] - else: - record = self.ledger.append(entry.pending_share) - entry.record = record - break - except Exception: - if not retry_until_stopped: - raise - self._ensure_share_hot_path_state() - with self._share_accounting_lock: - self.share_append_failure_count = ( - int(getattr(self, "share_append_failure_count", 0)) + 1 - ) - print( - "prism coordinator: ledger share append failed; retrying " - f"share_id={entry.pending_share.share_id}", - flush=True, - ) - traceback.print_exc() - if self.stop_event.wait(backoff_seconds): - # Shutting down mid-outage: do not silently drop this - # already-acked, already-counted share -- recover it to - # disk for replay on the next start. - self._recover_share_to_disk(entry, "ledger unavailable at shutdown") - return False - backoff_seconds = min(backoff_seconds * 2, 5.0) - self._record_heartbeat("share_writer") - if getattr(self, "hot_path_log_enabled", False): - print( - "prism coordinator: accepted share " - f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " - f"hash={entry.block_hash_hex} collection={entry.collection_only} " - f"credit_policy={entry.credit_policy or 'normal'}", - flush=True, - ) - entry.committed.set() - return True + def _append_share_entry(self, entry: PendingShareAppend, *, retry_until_stopped: bool = False) -> bool: + return self._ensure_share_writer_service().append_entry( + entry, + retry_until_stopped=retry_until_stopped, + ) def accepted_share_difficulty(self, context: PrismJobContext) -> int: override = self.share_weights_by_username.get( @@ -15135,112 +7797,153 @@ def _observe_vardiff_idle_seconds(self, name: str, elapsed_seconds: float) -> No if elapsed_seconds <= bucket: buckets[bucket] = int(buckets.get(bucket, 0)) + 1 - def _idle_bundle_current_locked( + def _idle_bundle_cache_key( self, client: ClientState, bundle: CachedJobBundle, *, - allow_uncached: bool = False, - ) -> bool: - """Check one exact issuance observation. Caller holds ``_job_cache_lock``. - - Ready bundles can reuse an older same-tip heavy cache entry, but the - prepared copy must be rebound to the current template artifacts before - delivery. During a detected-but-unpublished refresh, issuance stays - pinned to the published snapshot. Accept that copy only when it still - shares the immutable heavy payload with the cache entry from which it - was derived. - """ - artifacts = self._idle_job_issuance_artifacts_locked() - if artifacts is None or self._payout_state_publication_blocked: - return False - payout_artifact = self._published_payout_state.artifact + artifacts: CachedTemplateArtifacts | None = None, + ) -> tuple[object, ...] | None: + if artifacts is None: + artifacts = self._idle_job_issuance_artifacts_locked() + payout = self._ensure_payout_state_service().snapshot() + if artifacts is None or payout.publication_blocked: + return None + payout_artifact = payout.published.artifact if ( bundle.build_key is None or payout_artifact is None or bundle.build_key.payout_artifact_sha256 != payout_artifact.prior_balances_sha256 ): - return False + return None observed_tip = getattr(self, "current_tip_first_seen", None) if observed_tip is not None and observed_tip[0] != artifacts.previousblockhash: - return False + return None if ( bundle.template_fingerprint != artifacts.fingerprint - or bundle.payout_state_generation != self._payout_state_generation + or bundle.payout_state_generation != payout.generation or str(bundle.template.get("previousblockhash", "")) != artifacts.previousblockhash ): - return False - published_tip = self._published_payout_state.source_tip_hash + return None + published_tip = payout.published.source_tip_hash if published_tip is not None and published_tip != artifacts.previousblockhash: - return False + return None worker = client.worker if worker is None: - return False - mode = "ready" if getattr(self, "_pool_ready_latched", False) else "collection" + return None + mode = ( + "ready" + if self._ensure_job_bundle_service().ready_latched() + else "collection" + ) if bundle.collection_only != (mode == "collection"): - return False + return None if ( bundle.template is not artifacts.template or bundle.template_generation != artifacts.generation ): - # Collection manifests sign the template ntime, while ready - # bundles can cheaply rebind their base job. Either way, the - # object stamped for delivery must represent this observation. - return False - key = self._job_bundle_key( + return None + return self._job_bundle_key( artifacts, mode=mode, - payout_state_generation=self._payout_state_generation, + payout_state_generation=payout.generation, payout_artifact_generation=bundle.payout_artifact_generation, worker=worker, ) - ttl = float( - getattr( - self, - "job_bundle_cache_seconds", - DEFAULT_PRISM_JOB_BUNDLE_CACHE_SECONDS, + + @contextmanager + def _idle_bundle_admission( + self, + client: ClientState, + bundle: CachedJobBundle, + *, + allow_uncached: bool = False, + ) -> Iterator[AdmittedIdleBundleSource | None]: + artifacts = self._idle_job_issuance_artifacts_locked() + if artifacts is None: + yield None + return + key = self._idle_bundle_cache_key( + client, + bundle, + artifacts=artifacts, + ) + if key is None: + yield None + return + with self._ensure_job_bundle_service().cache_admission( + key, + bundle, + allow_uncached=allow_uncached, + ) as admitted: + yield ( + AdmittedIdleBundleSource( + artifacts=artifacts, + bundle=bundle, + cache_identity=key, + allow_uncached=allow_uncached, + ) + if admitted + else None ) + + def _admit_idle_bundle_source( + self, + client: ClientState, + bundle: CachedJobBundle, + *, + allow_uncached: bool = False, + ) -> AdmittedIdleBundleSource | None: + """Consume J1 admission and return an immutable exact-source lease.""" + artifacts = self._idle_job_issuance_artifacts_locked() + if artifacts is None: + return None + key = self._idle_bundle_cache_key( + client, + bundle, + artifacts=artifacts, ) - if ttl <= 0: - # A zero TTL deliberately disables global bundle retention. The - # dedicated worker may still deliver the exact bundle it just - # built after every live template/payout/client guard above has - # passed; the cache-only sweep never opts into this exception. - return allow_uncached and bundle.key == key - cached = self._job_bundle_cache.get(key) - if cached is None: - return False - if cached is not bundle and not ( - bundle.key == cached.key - and bundle.coinbase_manifest is cached.coinbase_manifest - and bundle.shares_json is cached.shares_json - and bundle.prior_balances is cached.prior_balances - and bundle.found_block is cached.found_block - and bundle.collection_only == cached.collection_only - and bundle.issued_at_ms == cached.issued_at_ms - and bundle.built_monotonic == cached.built_monotonic - and bundle.payout_state_generation - == cached.payout_state_generation - and bundle.payout_artifact_generation - == cached.payout_artifact_generation - and bundle.collection_identity == cached.collection_identity - ): - return False - return time.monotonic() - cached.built_monotonic <= ttl + if key is None: + return None + with self._ensure_job_bundle_service().cache_admission( + key, + bundle, + allow_uncached=allow_uncached, + ) as admitted: + if not admitted: + return None + return AdmittedIdleBundleSource( + artifacts=artifacts, + bundle=bundle, + cache_identity=key, + allow_uncached=allow_uncached, + ) + + def _idle_bundle_current_locked( + self, + client: ClientState, + bundle: CachedJobBundle, + *, + allow_uncached: bool = False, + ) -> bool: + """Compatibility predicate; S2 consumes an immutable J1 source lease.""" + with self._idle_bundle_admission( + client, + bundle, + allow_uncached=allow_uncached, + ) as admitted: + return admitted is not None def _idle_job_issuance_artifacts_locked( self, ) -> CachedTemplateArtifacts | None: - """Cache-only counterpart of ``job_issuance_template_artifacts``. - - The idle sweep and its final delivery guard cannot fetch a template. - They must still honor the published-snapshot pin used by every other - direct issuance path while a replacement tip is detected but has not - yet been published. - """ - artifacts = self._template_artifacts + """Cache-only counterpart of ``job_issuance_template_artifacts``.""" + artifacts = ( + self._ensure_job_bundle_service() + .template_repository.current_artifacts() + ) with self.lock: published = getattr(self, "current_tip_first_seen", None) latest_detected = getattr(self, "latest_detected_tip", None) @@ -15270,35 +7973,33 @@ def _idle_job_issuance_artifacts_locked( def _cached_idle_job_bundle(self, client: ClientState) -> CachedJobBundle | None: """Return only an exact issuance bundle; never build or query.""" - self._ensure_job_cache_state() - with self._job_cache_lock: - artifacts = self._idle_job_issuance_artifacts_locked() - worker = client.worker - if artifacts is None or worker is None: - return None - mode = "ready" if getattr(self, "_pool_ready_latched", False) else "collection" - payout_artifact = getattr(self, "_payout_ledger_artifact", None) - payout_artifact_generation = ( - payout_artifact.generation - if mode == "ready" - and payout_artifact is not None - and payout_artifact.payout_state_generation - == self._payout_state_generation - and payout_artifact.network_difficulty - == artifacts.network_difficulty - else 0 - ) - key = self._job_bundle_key( - artifacts, - mode=mode, - payout_state_generation=self._payout_state_generation, - payout_artifact_generation=payout_artifact_generation, - worker=worker, - ) - bundle = self._job_bundle_cache.get(key) - if bundle is None or not self._idle_bundle_current_locked(client, bundle): - return None - return bundle + artifacts = self._idle_job_issuance_artifacts_locked() + worker = client.worker + if artifacts is None or worker is None: + return None + service = self._ensure_job_bundle_service() + mode = "ready" if service.ready_latched() else "collection" + payout = self._ensure_payout_state_service().snapshot() + payout_artifact = payout.ledger_artifact + payout_artifact_generation = ( + payout_artifact.generation + if mode == "ready" + and payout_artifact is not None + and payout_artifact.payout_state_generation == payout.generation + and payout_artifact.network_difficulty == artifacts.network_difficulty + else 0 + ) + key = self._job_bundle_key( + artifacts, + mode=mode, + payout_state_generation=payout.generation, + payout_artifact_generation=payout_artifact_generation, + worker=worker, + ) + bundle = service.cached_bundle_for_key(key) + if bundle is None or not self._idle_bundle_current_locked(client, bundle): + return None + return bundle def _build_idle_job_bundle( self, @@ -15448,12 +8149,11 @@ def _run_idle_retarget_task( if reason is not None: self._record_vardiff_idle_skip(reason) return - with self._job_cache_lock: - bundle_current = self._idle_bundle_current_locked( - client, - bundle, - allow_uncached=True, - ) + bundle_current = self._idle_bundle_current_locked( + client, + bundle, + allow_uncached=True, + ) if not bundle_current: self._record_vardiff_idle_skip("superseded") return @@ -15489,12 +8189,11 @@ def _run_idle_retarget_task( if reason is not None: self._record_vardiff_idle_skip(reason) return - with self._job_cache_lock: - bundle_current = self._idle_bundle_current_locked( - client, - bundle, - allow_uncached=True, - ) + bundle_current = self._idle_bundle_current_locked( + client, + bundle, + allow_uncached=True, + ) if not bundle_current: self._record_vardiff_idle_skip("superseded") except JobBuildSuperseded: @@ -15841,47 +8540,32 @@ def _retarget_client_locked( ) prior_pending = client.pending_share_difficulty client.pending_share_difficulty = next_difficulty + idle_authority = ( + IdleDeliveryAuthority( + connection_id=expected_connection_id, + worker=expected_worker, + expected_active_job=expected_active_job, + expected_window_started=expected_window_started, + pending_difficulty=next_difficulty, + ) + if require_idle + else None + ) # Advertise the new difficulty only with its corresponding job. Idle # retargets stamp an already-cached bundle; normal share-driven # retargets retain the existing build path. Either path sends the pair # together or restores the prior pending difficulty/window state. - def idle_commit_guard() -> bool: - nonlocal idle_window_reset_at - if not require_idle: - return True - # _maybe_send_job_locked holds vardiff_lock before entering the - # coordinator commit section, so this guard never waits under - # self.lock. - if ( - self.stop_event.is_set() - or client not in self.clients - or getattr(client, "closing", False) - or not self.client_can_receive_jobs(client) - or client.connection_id != expected_connection_id - or client.worker != expected_worker - or client.active_job is not expected_active_job - or client.vardiff_window_started_monotonic - != expected_window_started - or client.vardiff_window_accepted != 0 - or client.vardiff_window_submitted != 0 - or client.pending_share_difficulty != next_difficulty - ): - return False - idle_window_reset_at = time.monotonic() - client.vardiff_window_started_monotonic = idle_window_reset_at - client.vardiff_window_accepted = 0 - client.vardiff_window_submitted = 0 - client.vardiff_window_work = Decimal("0") - return True - def restore_speculative_retarget() -> None: - with self._client_vardiff_lock(client), self.lock: + reset_at = idle_window_reset_at + if reset_at is None and idle_authority is not None: + reset_at = idle_authority.committed_reset_monotonic + with self.lock: if client.pending_share_difficulty == next_difficulty: client.pending_share_difficulty = prior_pending self._restore_idle_window_state( client, idle_window_state, - idle_window_reset_at, + reset_at, ) try: @@ -15891,12 +8575,15 @@ def restore_speculative_retarget() -> None: clean_jobs=True, raise_on_build_failure=True, prepared_bundle=prepared_bundle, - commit_guard=idle_commit_guard, - commit_guard_lock=self._client_vardiff_lock(client), + idle_authority=idle_authority, prepared_bundle_allow_uncached=( prepared_bundle_allow_uncached ), ) + if idle_authority is not None: + idle_window_reset_at = ( + idle_authority.committed_reset_monotonic + ) else: sent = bool( client.authorized @@ -15997,6 +8684,7 @@ def replay_pending_block_candidates(self) -> int: queued = 0 for durable_row in durable_rows: durable_block_hash = "" + candidate: PrismBlockCandidate | None = None try: if not isinstance(durable_row, dict): raise ValueError("durable block candidate row is not an object") @@ -16018,6 +8706,7 @@ def replay_pending_block_candidates(self) -> int: if self.enqueue_block_candidate(candidate): queued += 1 except Exception: + terminalized = False if durable_block_hash: self._clear_accepted_block_payout_preview( durable_block_hash, @@ -16032,6 +8721,11 @@ def replay_pending_block_candidates(self) -> int: block_hash=durable_block_hash, error="invalid durable candidate intent", ) + # A normal terminal update return (including already + # terminal/missing) means this process has no pending + # outbox source left that can credit the reconstructed + # share. Release its stable S3 floor lease. + terminalized = True self._clear_accepted_block_payout_preview( durable_block_hash ) @@ -16043,6 +8737,12 @@ def replay_pending_block_candidates(self) -> int: ) + 1 except Exception: traceback.print_exc() + if ( + terminalized + and candidate is not None + and candidate.credit_share_on_accept + ): + self._finish_pending_share_candidate(candidate.pending_share) if queued: print( f"prism coordinator: replayed {queued} pending block candidate(s)", @@ -16161,6 +8861,22 @@ def submit_next_block_candidate(self, timeout: float | None = None) -> bool: def _submit_next_block_candidate_writer( self, candidate: PrismBlockCandidate, + ) -> bool: + """Run one candidate with an independent active credit-floor actor.""" + if not candidate.credit_share_on_accept: + return self._submit_next_block_candidate_writer_actor_owned(candidate) + share_writer = self._ensure_share_writer_service() + share_writer.begin_candidate_actor(candidate.pending_share) + try: + return self._submit_next_block_candidate_writer_actor_owned(candidate) + finally: + # The owned body either committed credit, reached terminal + # noncredit, or re-established the stable retry/outbox holder. + share_writer.finish_candidate_actor(candidate.pending_share) + + def _submit_next_block_candidate_writer_actor_owned( + self, + candidate: PrismBlockCandidate, ) -> bool: """Land one dequeued block candidate inside writer admission.""" outcome = getattr(self, "_block_candidate_outcome", None) @@ -16189,12 +8905,6 @@ def _submit_next_block_candidate_writer( registry.get(block_hash) if registry is not None else None ) if pending_finalize is not None: - # Finalize-only replay: submission, terminal accounting, and - # payout persistence already completed on the pass that armed - # this entry; only the durable outbox update remains. Re-running - # submit_block_candidate here would recount terminal - # abandonments and redo the accepted-path audit/persist work - # once per paced retry. accepted, error = pending_finalize return self._finalize_block_candidate( candidate, @@ -16275,13 +8985,7 @@ def _finalize_block_candidate( error: str, outcome: threading.local, ) -> bool: - """Drive a terminal candidate's durable outbox update, with backoff. - - Failure retains the candidate as a finalize-only replay: the next - paced attempt re-enters here directly, never submit_block_candidate, - so terminal abandonment accounting stays once-per-candidate and an - accepted candidate's audit/persist work is not redone per retry. - """ + """Retry only a terminal candidate's durable outbox transition.""" self._clear_accepted_block_payout_preview( block_hash, invalidate_published=not accepted, @@ -16304,9 +9008,6 @@ def _finalize_block_candidate( # replay source left for this process to guard. self._clear_accepted_block_payout_preview(block_hash) except Exception: - # Keep the coordinator alive. The terminal-state update - # failed, so the durable row stays pending and its replay - # must pace like any other retained retry. print( "prism coordinator: could not finalize durable block candidate " f"hash={block_hash}", @@ -16322,19 +9023,12 @@ def _finalize_block_candidate( self._block_candidate_finalize_retries = registry first_failure = block_hash not in registry registry[block_hash] = (accepted, error) - # The share row already reached its terminal outcome in this - # process; only the outbox mark is pending. Release the - # snapshot anchor floor now (idempotent) -- holding it across - # paced retries would clamp job snapshot anchors and - # under-count already-durable shares in reward windows. - self._finish_pending_share_commit(candidate.pending_share) - self._retain_block_candidate_for_retry(candidate) + self._finish_pending_share_candidate(candidate.pending_share) + self._retain_block_candidate_for_retry( + candidate, + retain_share_floor=False, + ) if accepted and first_failure: - # The block is active regardless of the outbox update; - # post-accept fleet refresh must wait for neither ledger - # recovery nor the first backoff. Return unpaced so the - # caller refreshes immediately; the paced ladder starts - # from the first finalize-only replay. outcome.refresh_client = candidate.client return True self._wait_for_block_candidate_retry( @@ -16350,19 +9044,28 @@ def _finalize_block_candidate( if registry is not None: registry.pop(block_hash, None) self._clear_block_candidate_retry_state(block_hash) - # Terminal for this process either way: an accepted candidate credited - # its share during the success tail (a no-op release here), and an - # abandoned one can only be credited by restart replay, which stamps a - # fresh PendingShare. Stop holding the snapshot anchor floor. - self._finish_pending_share_commit(candidate.pending_share) + self._finish_pending_share_candidate(candidate.pending_share) if accepted: outcome.refresh_client = candidate.client return True - def _retain_block_candidate_for_retry(self, candidate: PrismBlockCandidate) -> None: + def _retain_block_candidate_for_retry( + self, + candidate: PrismBlockCandidate, + *, + retain_share_floor: bool = True, + ) -> None: """Keep the oldest unresolved candidate ahead of queued descendants.""" candidate_height = int(candidate.context.template["height"]) candidate_hash = str(candidate.submission.block_hash_hex).lower() + # Publish no retryable credit-bearing candidate until its stable floor + # lease exists. A submitter may pop and terminally finish immediately + # after the coordinator lock is released; adopting afterward could + # otherwise resurrect an already-finished lease. + if candidate.credit_share_on_accept and retain_share_floor: + self._ensure_share_writer_service().adopt_pending_share( + candidate.pending_share + ) with self.lock: self.block_candidate_retry_count = int( getattr(self, "block_candidate_retry_count", 0) @@ -16370,14 +9073,17 @@ def _retain_block_candidate_for_retry(self, candidate: PrismBlockCandidate) -> N existing = getattr(self, "_retry_block_candidate", None) if existing is None: self._retry_block_candidate = candidate - return - existing_height = int(existing.context.template["height"]) - existing_hash = str(existing.submission.block_hash_hex).lower() - if candidate_hash == existing_hash or candidate_height < existing_height: - # Replacing a descendant is safe because its durable outbox row - # will replay after this lower-height parent reaches a terminal - # state. Equal-height competitors preserve first-in ordering. - self._retry_block_candidate = candidate + else: + existing_height = int(existing.context.template["height"]) + existing_hash = str(existing.submission.block_hash_hex).lower() + if candidate_hash == existing_hash or candidate_height < existing_height: + # Replacing a descendant is safe because its durable outbox + # row remains authoritative and replayable. Its stable S3 + # floor lease also remains terminally reachable by share ID. + self._retry_block_candidate = candidate + # The selected and non-selected candidates were each adopted before + # their own first retry-slot publication. Their stable share-ID leases + # remain reachable without nesting the coordinator and S3 locks. def _reject_terminal_prepared_block_candidate( self, @@ -16578,7 +9284,7 @@ def _land_and_confirm_block_candidate( durable_payout_state = bool( getattr(self.ledger, "durable_payout_state", False) ) - with self._payout_balance_mutation_lock: + with self._ensure_payout_state_service().balance_mutation_lock: if self._defer_for_pending_parent_payout_transition( parent_hash=parent_hash, parent_height=expected_height - 1, @@ -16997,7 +9703,7 @@ def _land_and_confirm_block_candidate( and not payout_publication_required ) with self.lock: - pending_cause = self._payout_state_source[2] + pending_cause = self._ensure_payout_state_service().snapshot().source[2] # A bounded preview-publication loss already left the gate # fenced and its retry scheduled. Do not monopolize the # submitter with a second retry budget. Uncertain commits, @@ -17015,7 +9721,7 @@ def _land_and_confirm_block_candidate( True, ): with self.lock: - latest_tip = self._payout_state_source[1] + latest_tip = self._ensure_payout_state_service().snapshot().source[1] summary = self.reconcile_prism_pool_blocks_once( tip_hash=latest_tip, _force_publish=True, @@ -17418,21 +10124,24 @@ def trusted_ledger_writer_public_key_hex(self, bundle: dict[str, Any]) -> str: expected_bytes=32, ) - def _progress_now(self) -> float: - clock = getattr(self, "_progress_monotonic", None) - return float(clock() if callable(clock) else time.monotonic()) + @staticmethod + def _progress_work_generation( + snapshot: QbitTipTemplateSnapshot, + payout_generation: int, + ) -> WorkGeneration: + return WorkGeneration( + template_generation=int(snapshot.template_generation), + template_fingerprint=snapshot.template_fingerprint, + payout_generation=int(payout_generation), + ) - def _progress_note_refresh_pending(self, started_monotonic: float | None = None) -> None: - self._ensure_job_cache_state() - started = self._progress_now() if started_monotonic is None else started_monotonic - with self._progress_health_lock: - pending_since = self._progress_pending_since_monotonic - if pending_since is None or started < pending_since: - self._progress_pending_since_monotonic = started - divergence_since = self._progress_publication_divergence_since_monotonic - if divergence_since is None or started < divergence_since: - self._progress_publication_divergence_since_monotonic = started - self._progress_refresh_signal_pending = True + def _progress_note_refresh_pending( + self, + started_monotonic: float | None = None, + ) -> None: + self._ensure_progress_health_service().mark_refresh_pending( + started_monotonic + ) def _record_progress_tip_poll( self, @@ -17440,72 +10149,21 @@ def _record_progress_tip_poll( observed_monotonic: float | None = None, ) -> None: """Publish a coherent qbit tip/template observation to health state.""" - self._ensure_job_cache_state() - observed = self._progress_now() if observed_monotonic is None else observed_monotonic - payout_generation = int(getattr(self, "_payout_state_generation", 0)) - with self._progress_health_lock: - if snapshot.template_generation < self._progress_current_template_generation: - # A slower concurrent poll can finish after a newer coherent - # observation. It proves only that the obsolete generation was - # coherent, so it must not renew freshness for current work. - return - self._progress_current_template_generation = snapshot.template_generation - self._progress_current_template_fingerprint = snapshot.template_fingerprint - self._progress_current_payout_generation = max( - self._progress_current_payout_generation, - payout_generation, - ) - self._progress_last_tip_poll_monotonic = observed - same_published_work = bool( - self._progress_has_published_work - and self._progress_published_template_fingerprint - == snapshot.template_fingerprint - and self._progress_published_payout_generation - == self._progress_current_payout_generation - ) - if same_published_work: - # Observation generations order RPC races. When the semantic - # template fingerprint and payout generation are unchanged, - # already-issued work remains current and can be reconciled to - # the latest observation without a needless socket delivery. - self._progress_published_template_generation = ( - snapshot.template_generation - ) - else: - pending_since = self._progress_pending_since_monotonic - if pending_since is None or observed < pending_since: - self._progress_pending_since_monotonic = observed - divergence_since = ( - self._progress_publication_divergence_since_monotonic - ) - if divergence_since is None or observed < divergence_since: - self._progress_publication_divergence_since_monotonic = observed + payout_generation = int(self._ensure_payout_state_service().snapshot().generation) + self._ensure_progress_health_service().observe_tip( + self._progress_work_generation(snapshot, payout_generation), + observed_monotonic, + ) def _record_progress_payout_generation( self, generation: int, invalidated_monotonic: float | None = None, ) -> None: - self._ensure_job_cache_state() - invalidated = ( - self._progress_now() - if invalidated_monotonic is None - else invalidated_monotonic - ) - with self._progress_health_lock: - if generation < self._progress_current_payout_generation: - return - self._progress_current_payout_generation = generation - if generation != self._progress_published_payout_generation: - pending_since = self._progress_pending_since_monotonic - if pending_since is None or invalidated < pending_since: - self._progress_pending_since_monotonic = invalidated - divergence_since = ( - self._progress_publication_divergence_since_monotonic - ) - if divergence_since is None or invalidated < divergence_since: - self._progress_publication_divergence_since_monotonic = invalidated - self._progress_refresh_signal_pending = True + self._ensure_progress_health_service().observe_payout_generation( + generation, + invalidated_monotonic, + ) def _record_progress_publication( self, @@ -17513,423 +10171,170 @@ def _record_progress_publication( payout_generation: int, ) -> None: """Record that current in-memory work is available for delivery.""" - self._ensure_job_cache_state() - with self.lock, self._progress_health_lock: - latest_detected = getattr(self, "latest_detected_tip", None) - publication_matches_latest_tip = bool( - latest_detected is None - or latest_detected[0] == snapshot.bestblockhash - ) - if ( - snapshot.template_generation - < self._progress_published_template_generation - or payout_generation - < self._progress_published_payout_generation - ): - return - self._progress_published_template_generation = max( - self._progress_published_template_generation, - snapshot.template_generation, - ) - self._progress_published_template_fingerprint = snapshot.template_fingerprint - self._progress_published_payout_generation = max( - self._progress_published_payout_generation, - payout_generation, - ) - self._progress_has_published_work = True - if ( - publication_matches_latest_tip - and self._progress_current_template_fingerprint - == snapshot.template_fingerprint - and self._progress_current_payout_generation == payout_generation - ): - self._progress_refresh_signal_pending = False - self._progress_publication_divergence_since_monotonic = None - self._progress_note_refresh_activity() - self._progress_reconcile_pending() + service = self._ensure_progress_health_service() + published = service.publish_work( + self._progress_work_generation(snapshot, payout_generation) + ) + if published: + service.reconcile_pending(self._progress_eligibility_snapshot()) def _record_progress_delivery( self, client: ClientState, context: PrismJobContext, delivered_monotonic: float, + ) -> None: + self._ensure_stratum_session_service().record_successful_delivery( + client, context, delivered_monotonic + ) + + def _complete_job_delivery( + self, + client: ClientState, + authority: object, + context: PrismJobContext, + delivered_monotonic: float, + ) -> bool: + return self._ensure_job_delivery_service().complete_delivery( + client, + authority, # type: ignore[arg-type] + context, + delivered_monotonic, + ) + + def _record_progress_delivery_to_health( + self, + client: ClientState, + context: PrismJobContext, + delivered_monotonic: float, ) -> None: """Record a completed current-generation socket delivery.""" - self._ensure_job_cache_state() fingerprint = getattr(context, "template_fingerprint", None) if fingerprint is None: fingerprint = qbit_template_fingerprint(context.template) - payout_generation = int(getattr(context, "payout_state_generation", 0)) - delivered_tip = str(context.template.get("previousblockhash", "")) - recorded = False - with self.lock, self._progress_health_lock: - client._progress_delivered_context = context + work = WorkGeneration( + template_generation=int(getattr(context, "template_generation", 0)), + template_fingerprint=fingerprint, + payout_generation=int( + getattr(context, "payout_state_generation", 0) + ), + ) + ready_mode_required = self._ensure_job_bundle_service().ready_latched() + with self.lock: client._progress_delivered_template_fingerprint = fingerprint - client._progress_delivered_template_generation = int( - getattr(context, "template_generation", 0) - ) - client._progress_delivered_payout_generation = payout_generation - client._progress_delivered_monotonic = delivered_monotonic - ready_mode_required = bool( - getattr(self, "_pool_ready_latched", False) - ) - latest_detected = getattr(self, "latest_detected_tip", None) - delivery_matches_latest_tip = bool( - latest_detected is None or latest_detected[0] == delivered_tip - ) - if ( - fingerprint == self._progress_current_template_fingerprint - and payout_generation == self._progress_current_payout_generation - and not ( - ready_mode_required - and bool(getattr(context, "collection_only", False)) - ) - ): - self._progress_last_delivery_template_generation = int( - getattr(context, "template_generation", 0) - ) - self._progress_last_delivery_template_fingerprint = fingerprint - self._progress_last_delivery_payout_generation = payout_generation - self._progress_last_delivery_monotonic = delivered_monotonic - # A successful delivery proves publication for its coherent - # generation. Only the latest observed tip can close the - # outstanding divergence below. - self._progress_published_template_generation = max( - self._progress_published_template_generation, - self._progress_current_template_generation, - ) - self._progress_published_template_fingerprint = fingerprint - self._progress_published_payout_generation = max( - self._progress_published_payout_generation, - payout_generation, - ) - self._progress_has_published_work = True - self._progress_refresh_signal_pending = False - if delivery_matches_latest_tip: - self._progress_publication_divergence_since_monotonic = None - recorded = True - if recorded: - self._progress_note_refresh_activity(delivered_monotonic) - self._progress_reconcile_pending(now=delivered_monotonic) - - def _progress_refresh_started(self) -> None: - """Track a coherent poll pass while it continues making progress.""" - self._ensure_job_cache_state() - started = self._progress_now() - with self._progress_health_lock: - self._progress_active_refresh_count += 1 - self._progress_last_refresh_activity_monotonic = started - - def _progress_note_refresh_activity( - self, - observed_monotonic: float | None = None, - ) -> None: - self._ensure_job_cache_state() - observed = ( - self._progress_now() - if observed_monotonic is None - else observed_monotonic - ) - with self._progress_health_lock: - if self._progress_active_refresh_count > 0: - last_activity = self._progress_last_refresh_activity_monotonic - if last_activity is None or observed > last_activity: - self._progress_last_refresh_activity_monotonic = observed - - def _progress_refresh_finished(self) -> None: - self._ensure_job_cache_state() - with self._progress_health_lock: - self._progress_active_refresh_count = max( - 0, - self._progress_active_refresh_count - 1, + client._progress_delivered_template_generation = ( + work.template_generation ) + client._progress_delivered_payout_generation = work.payout_generation + service = self._ensure_progress_health_service() + service.record_delivery( + DeliveryProof( + connection_id=client.connection_id, + delivered_work=work, + collection_only=bool( + getattr(context, "collection_only", False) + ), + delivered_monotonic=delivered_monotonic, + ), + ready_mode_required, + ) - def _progress_bundle_build_started(self) -> int: - self._ensure_job_cache_state() - started = self._progress_now() - with self._progress_health_lock: - self._progress_bundle_build_counter += 1 - token = self._progress_bundle_build_counter - self._progress_bundle_builds[token] = started - return token - - def _progress_bundle_build_finished(self, token: int) -> None: - self._ensure_job_cache_state() - with self._progress_health_lock: - self._progress_bundle_builds.pop(token, None) + def _progress_eligibility_snapshot(self) -> EligibilitySnapshot: + sessions = self._ensure_session_registry().eligible_snapshot() + ready_mode_required = self._ensure_job_bundle_service().ready_latched() - def _progress_eligible_client_counts( - self, - template_fingerprint: str | None, - payout_generation: int, - ) -> tuple[int, int]: - with self.lock: - eligible = [ - client - for client in self.clients - if self.client_can_receive_jobs(client) - ] - delivered_work = [ - client._progress_delivered_context - for client in eligible - ] - ready_mode_required = bool( - getattr(self, "_pool_ready_latched", False) + proofs: list[DeliveryProof] = [] + for connection_id, session in sessions.items(): + if session.delivered is None: + continue + delivered_context = session.delivered.context + delivered_monotonic = session.delivered.delivered_monotonic + fingerprint = getattr( + delivered_context, + "template_fingerprint", + None, ) - requiring_refresh = 0 - for delivered_context in delivered_work: - delivered_fingerprint = None - delivered_payout_generation = -1 - if delivered_context is not None: - delivered_fingerprint = getattr( - delivered_context, - "template_fingerprint", - None, - ) - if delivered_fingerprint is None: - delivered_fingerprint = qbit_template_fingerprint( - delivered_context.template - ) - delivered_payout_generation = int( - getattr(delivered_context, "payout_state_generation", 0) + if fingerprint is None: + fingerprint = qbit_template_fingerprint( + delivered_context.template ) - if ( - delivered_fingerprint != template_fingerprint - or delivered_payout_generation != payout_generation - or ( - ready_mode_required - and bool( + proofs.append( + DeliveryProof( + connection_id=connection_id, + delivered_work=WorkGeneration( + template_generation=int( + getattr( + delivered_context, + "template_generation", + 0, + ) + ), + template_fingerprint=fingerprint, + payout_generation=int( + getattr( + delivered_context, + "payout_state_generation", + 0, + ) + ), + ), + collection_only=bool( getattr(delivered_context, "collection_only", False) - ) + ), + delivered_monotonic=float(delivered_monotonic or 0.0), ) - ): - requiring_refresh += 1 - return len(eligible), requiring_refresh - - def _progress_reconcile_pending(self, *, now: float | None = None) -> None: - """Clear pending state exactly when publication/delivery is sufficient.""" - self._ensure_job_cache_state() - current = self._progress_now() if now is None else now - with self._progress_health_lock: - state_key = ( - self._progress_current_template_fingerprint, - self._progress_current_payout_generation, ) - _, requiring_refresh = self._progress_eligible_client_counts( - *state_key + return EligibilitySnapshot( + eligible_connection_ids=tuple(sessions), + delivery_proofs=tuple(proofs), + ready_mode_required=ready_mode_required, ) - with self._progress_health_lock: - if state_key != ( - self._progress_current_template_fingerprint, - self._progress_current_payout_generation, - ): - return - published_current = bool( - self._progress_has_published_work - and self._progress_published_template_fingerprint == state_key[0] - and self._progress_published_payout_generation == state_key[1] - ) - refresh_required = bool( - self._progress_refresh_signal_pending - or not published_current - or requiring_refresh > 0 - ) - if refresh_required: - if self._progress_pending_since_monotonic is None: - self._progress_pending_since_monotonic = current - else: - self._progress_pending_since_monotonic = None - def progress_health_snapshot(self, *, now: float | None = None) -> dict[str, object]: - """Return a bounded, monotonic-only mining progress health snapshot.""" - self._ensure_job_cache_state() - current = self._progress_now() if now is None else now - payout_generation = int(getattr(self, "_payout_state_generation", 0)) - with self._progress_health_lock: - tracked_payout_generation = self._progress_current_payout_generation - if payout_generation > tracked_payout_generation: - self._record_progress_payout_generation(payout_generation, current) - self._progress_reconcile_pending(now=current) - - with self._progress_health_lock: - template_fingerprint = self._progress_current_template_fingerprint - current_template_generation = self._progress_current_template_generation - current_payout_generation = self._progress_current_payout_generation - published_template_generation = self._progress_published_template_generation - published_template_fingerprint = ( - self._progress_published_template_fingerprint - ) - published_payout_generation = self._progress_published_payout_generation - has_published_work = self._progress_has_published_work - last_tip_poll = self._progress_last_tip_poll_monotonic - last_delivery_fingerprint = ( - self._progress_last_delivery_template_fingerprint - ) - last_delivery_payout_generation = ( - self._progress_last_delivery_payout_generation - ) - last_delivery_monotonic = self._progress_last_delivery_monotonic - pending_since = self._progress_pending_since_monotonic - refresh_signal_pending = self._progress_refresh_signal_pending - active_refresh_count = self._progress_active_refresh_count - last_refresh_activity = ( - self._progress_last_refresh_activity_monotonic - ) - bundle_build_starts = tuple(self._progress_bundle_builds.values()) - - eligible_count, requiring_refresh = self._progress_eligible_client_counts( - template_fingerprint, - current_payout_generation, - ) - started = float(getattr(self, "started_monotonic", current)) - tip_poll_reference = started if last_tip_poll is None else last_tip_poll - tip_poll_age = max(0.0, current - tip_poll_reference) - refresh_activity_age = ( - None - if active_refresh_count <= 0 or last_refresh_activity is None - else max(0.0, current - last_refresh_activity) - ) - pending_age = ( - None if pending_since is None else max(0.0, current - pending_since) - ) - oldest_bundle_age = ( - 0.0 - if not bundle_build_starts - else max(0.0, current - min(bundle_build_starts)) - ) - published_current = bool( - has_published_work - and published_template_fingerprint == template_fingerprint - and published_payout_generation == current_payout_generation - ) - delivered_current = bool( - template_fingerprint is not None - and last_delivery_fingerprint == template_fingerprint - and last_delivery_payout_generation == current_payout_generation - ) - delivery_age = ( - max(0.0, current - last_delivery_monotonic) - if delivered_current and last_delivery_monotonic is not None - else None - ) - pending_deadline = float( - getattr( - self, - "health_pending_refresh_max_age_seconds", - DEFAULT_PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS, - ) - ) - tip_poll_deadline = float( - getattr( - self, - "health_tip_poll_max_age_seconds", - DEFAULT_PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS, - ) - ) - bundle_build_deadline = float( - getattr( - self, - "bundle_build_timeout_seconds", - DEFAULT_PRISM_BUNDLE_BUILD_TIMEOUT_SECONDS, - ) + def _progress_health_value( + self, + *, + now: float | None = None, + ) -> ProgressHealthSnapshot: + eligibility = self._progress_eligibility_snapshot() + payout_generation = int(self._ensure_payout_state_service().snapshot().generation) + return self._ensure_progress_health_service().snapshot( + eligibility, + payout_generation, + now=now, ) - reasons: list[str] = [] - refresh_is_progressing = bool( - active_refresh_count > 0 - and refresh_activity_age is not None - and refresh_activity_age <= tip_poll_deadline - ) - if tip_poll_age > tip_poll_deadline and not refresh_is_progressing: - reasons.append("tip_poll_stale") - if oldest_bundle_age > bundle_build_deadline: - reasons.append("bundle_build_stuck") - if not has_published_work: - reasons.append("current_generation_not_published") - elif pending_age is not None and pending_age > pending_deadline: - reasons.append("refresh_pending_too_long") - if refresh_signal_pending or not published_current: - reasons.append("current_generation_not_published") - elif requiring_refresh > 0: - reasons.append("current_generation_not_delivered") - reasons = list(dict.fromkeys(reasons)) - - def rounded(value: float | None) -> float | None: - return None if value is None else round(value, 3) - return { - "ok": not reasons, - "reason": reasons[0] if reasons else None, - "reasons": reasons, - "pending_refresh": pending_since is not None, - "pending_refresh_age_seconds": rounded(pending_age), - "tip_poll_age_seconds": rounded(tip_poll_age), - "tip_refresh_in_progress": active_refresh_count > 0, - "tip_refresh_progress_age_seconds": rounded(refresh_activity_age), - "current_template_generation": current_template_generation, - "published_template_generation": published_template_generation, - "current_payout_generation": current_payout_generation, - "published_payout_generation": published_payout_generation, - "last_valid_delivery_age_seconds": rounded(delivery_age), - "eligible_client_count": eligible_count, - "eligible_clients_requiring_refresh": requiring_refresh, - "bundle_build_oldest_age_seconds": rounded(oldest_bundle_age), - } + def progress_health_snapshot( + self, + *, + now: float | None = None, + ) -> dict[str, object]: + """Return the existing bounded progress-health payload.""" + return self._progress_health_value(now=now).as_mapping() @staticmethod def _apply_progress_health( payload: dict[str, object], progress: dict[str, object], ) -> dict[str, object]: - # A cached snapshot's top-level ``ok`` already includes the progress - # state from snapshot time. Recompute that half from the stable mining - # readiness field so a fresh in-memory progress overlay can both fail - # and recover immediately without hiding an independent mining fault. - base_ok = bool(payload.get("mining_ready", payload.get("ok"))) - result = dict(payload) - result.update(progress) - result["ok"] = base_ok and bool(progress["ok"]) - if progress["ok"]: - result.pop("reason", None) - result.pop("reasons", None) - return result + return dict(overlay_progress_health(payload, progress)) def progress_health_metrics_lines(self) -> list[str]: - progress = self.progress_health_snapshot() - pending_age = progress["pending_refresh_age_seconds"] - delivery_age = progress["last_valid_delivery_age_seconds"] - active_reasons = set(progress["reasons"]) - return [ - "# HELP qbit_prism_refresh_pending Whether current template or payout work still requires publication or delivery.", - "# TYPE qbit_prism_refresh_pending gauge", - f"qbit_prism_refresh_pending {1 if progress['pending_refresh'] else 0}", - "# HELP qbit_prism_refresh_pending_age_seconds Monotonic age of the oldest unresolved current-work refresh.", - "# TYPE qbit_prism_refresh_pending_age_seconds gauge", - f"qbit_prism_refresh_pending_age_seconds {float(pending_age or 0.0):.6f}", - "# HELP qbit_prism_tip_poll_age_seconds Monotonic age of the last coherent qbit tip/template poll.", - "# TYPE qbit_prism_tip_poll_age_seconds gauge", - f"qbit_prism_tip_poll_age_seconds {float(progress['tip_poll_age_seconds']):.6f}", - "# HELP qbit_prism_current_generation_delivery_age_seconds Monotonic age of the last valid current-generation delivery, or -1 when none exists.", - "# TYPE qbit_prism_current_generation_delivery_age_seconds gauge", - f"qbit_prism_current_generation_delivery_age_seconds {float(delivery_age) if delivery_age is not None else -1.0:.6f}", - "# HELP qbit_prism_bundle_build_oldest_age_seconds Monotonic age of the oldest active bundle build.", - "# TYPE qbit_prism_bundle_build_oldest_age_seconds gauge", - f"qbit_prism_bundle_build_oldest_age_seconds {float(progress['bundle_build_oldest_age_seconds']):.6f}", - "# HELP qbit_prism_health_state Current progress-health state by bounded reason.", - "# TYPE qbit_prism_health_state gauge", - f'qbit_prism_health_state{{reason="healthy"}} {1 if progress["ok"] else 0}', - *[ - f'qbit_prism_health_state{{reason="{reason}"}} {1 if reason in active_reasons else 0}' - for reason in PRISM_PROGRESS_HEALTH_REASONS - ], - ] + return list( + self._ensure_progress_health_service().metrics_lines( + self._progress_health_value() + ) + ) def ready_miner_count(self) -> int: return self.accepted_share_stats()[1] def mining_delivery_snapshot(self, *, now: float | None = None) -> dict[str, object]: now = time.monotonic() if now is None else now + delivery_service = self._ensure_job_delivery_service() + current_job_source = delivery_service.current_job_source() with self.lock: - self._ensure_initial_job_state() + initial_state = self._ensure_initial_job_state() + initial_snapshot = initial_state.snapshot() active = len(self.clients) current_tip = self._current_published_tip_hash_locked() published_snapshot = getattr(self, "tip_template_snapshot", None) @@ -17943,11 +10348,13 @@ def mining_delivery_snapshot(self, *, now: float | None = None) -> dict[str, obj clients_with_current_work = [ client for client in authorized_clients - if self._client_has_current_tip_job_locked(client) + if delivery_service.client_has_current_tip_job_locked( + client, current_job_source + ) ] current = len(clients_with_current_work) - pending_requests = list(self.pending_initial_jobs.values()) - pending = len(pending_requests) + pending_requests = list(initial_state.pending.values()) + pending = initial_snapshot.pending_count oldest_age = max( ( max(0.0, now - request.requested_monotonic) @@ -17958,7 +10365,7 @@ def mining_delivery_snapshot(self, *, now: float | None = None) -> dict[str, obj genuinely_pending_initial_clients = [ client for client in authorized_clients - if not self._client_has_delivered_work_locked(client) + if not delivery_service.client_has_delivered_work_locked(client) ] genuine_initial_started = [ started @@ -17966,8 +10373,8 @@ def mining_delivery_snapshot(self, *, now: float | None = None) -> dict[str, obj for started in ( client.authorized_monotonic, ( - self.pending_initial_jobs[client].requested_monotonic - if client in self.pending_initial_jobs + initial_state.pending[client].requested_monotonic + if client in initial_state.pending else None ), ) @@ -17984,7 +10391,7 @@ def mining_delivery_snapshot(self, *, now: float | None = None) -> dict[str, obj DEFAULT_PRISM_STRATUM_MAX_CONNECTIONS, ) ) - pending_limit = int(self.stratum_max_pending_initial_jobs) + pending_limit = int(initial_state.config.max_pending) coverage = current / authorized if authorized else 1.0 cap_saturated = connection_limit > 0 and active >= connection_limit pending_saturated = pending >= pending_limit @@ -18013,23 +10420,25 @@ def mining_delivery_snapshot(self, *, now: float | None = None) -> dict[str, obj if self._mining_overload_started_monotonic is not None else 0.0 ) - timeout = float(self.stratum_initial_job_timeout_seconds) - timeout_disconnects = self.initial_job_timeout_count - queue_rejections = self.initial_job_queue_rejection_count - cancelled = self.initial_job_cancelled_count - coalesced = self.initial_job_coalesced_count + timeout = float(initial_state.config.timeout_seconds) + timeout_disconnects = initial_snapshot.timeout_count + queue_rejections = initial_snapshot.queue_rejection_count + cancelled = initial_snapshot.cancelled_count + coalesced = initial_snapshot.coalesced_count queue_capacity_reclaimed = ( - self.initial_job_queue_capacity_reclaimed_count + initial_snapshot.queue_capacity_reclaimed_count ) peak = self.peak_active_connection_count handlers = self.handler_thread_count - self._ensure_job_cache_state() - with self._job_cache_lock: - prepared_bundle = self._prepared_ready_bundle - prepared_snapshot = self._prepared_ready_snapshot - preparation_pending = bool(self.job_preparation_pending) - payout_generation = int(self._payout_state_generation) + ( + prepared_bundle, + prepared_snapshot, + preparation_pending, + ) = self._ensure_job_bundle_service().prepared_ready_snapshot() + payout_generation = int( + self._ensure_payout_state_service().snapshot().generation + ) prepared_current = bool( prepared_bundle is not None and prepared_snapshot is published_snapshot @@ -18099,8 +10508,9 @@ def mining_delivery_snapshot(self, *, now: float | None = None) -> dict[str, obj if reject_storm: unhealthy_reasons.append("stale-unknown-rejection-storm") mining_ready = not unhealthy_reasons - executor = getattr(self, "_tip_refresh_executor", None) - queue_depth, active_workers = executor.stats() if executor is not None else (0, 0) + queue_depth, active_workers = ( + self._ensure_tip_refresh_service().executor_stats() + ) return { "mining_ready": mining_ready, "mining_delivery_healthy": mining_ready, @@ -18194,7 +10604,7 @@ def health_payload(self) -> dict[str, object]: def refresh_health_snapshot(self) -> dict[str, object]: payload = self.health_payload() self._ensure_job_cache_state() - with self._job_cache_lock: + with self._health_snapshot_lock: self._health_snapshot = payload self._health_snapshot_monotonic = time.monotonic() return payload @@ -18212,7 +10622,7 @@ def cached_health_payload(self) -> tuple[int, dict[str, object]]: refresh_seconds = getattr( self, "health_refresh_seconds", DEFAULT_PRISM_HEALTH_REFRESH_SECONDS ) - with self._job_cache_lock: + with self._health_snapshot_lock: snapshot = self._health_snapshot snapshot_monotonic = self._health_snapshot_monotonic loop_running = self._health_refresh_loop_running @@ -18258,22 +10668,27 @@ def cached_health_payload(self) -> tuple[int, dict[str, object]]: return (200 if payload.get("ok") else 503), payload def health_snapshot_loop(self) -> None: - while not self.stop_event.is_set(): - try: - self.refresh_health_snapshot() - except Exception: - with self._job_cache_lock: - self.health_snapshot_refresh_failure_count += 1 - print("prism coordinator: health snapshot refresh failed", flush=True) - traceback.print_exc() - if self.stop_event.wait( - getattr(self, "health_refresh_seconds", DEFAULT_PRISM_HEALTH_REFRESH_SECONDS) - ): - break + try: + while not self.stop_event.is_set(): + try: + self.refresh_health_snapshot() + except Exception: + with self._health_snapshot_lock: + self.health_snapshot_refresh_failure_count += 1 + print("prism coordinator: health snapshot refresh failed", flush=True) + traceback.print_exc() + if self.stop_event.wait( + getattr(self, "health_refresh_seconds", DEFAULT_PRISM_HEALTH_REFRESH_SECONDS) + ): + break + finally: + self._ensure_job_cache_state() + with self._health_snapshot_lock: + self._health_refresh_loop_running = False def start_health_snapshot_refresher(self) -> None: self._ensure_job_cache_state() - with self._job_cache_lock: + with self._health_snapshot_lock: if self._health_refresh_loop_running: return self._health_refresh_loop_running = True @@ -18282,7 +10697,10 @@ def start_health_snapshot_refresher(self) -> None: except Exception: print("prism coordinator: initial health snapshot refresh failed", flush=True) traceback.print_exc() - threading.Thread(target=self.health_snapshot_loop, daemon=True).start() + registry = self._ensure_background_services() + if not registry.contains("health_snapshot_refresher"): + registry.register(self._health_snapshot_service_spec()) + self._start_background_service("health_snapshot_refresher") def latest_evidence_payload(self) -> dict[str, object] | None: with self.lock: @@ -18404,6 +10822,8 @@ def block_submitter_metrics_lines(self) -> list[str]: def metrics_payload(self) -> str: ledger_metrics = self.ledger.metrics() + job_metrics = self._ensure_job_bundle_service().metrics_snapshot() + share_writer_metrics = self._ensure_share_writer_service().metrics_snapshot() audit_metrics = self.audit_artifact_metrics() mining_metrics = self.mining_delivery_snapshot() process_rss_bytes, process_open_fds = self.process_resource_metrics() @@ -18629,7 +11049,7 @@ def metrics_payload(self) -> str: ], "# HELP qbit_prism_job_build_failures_total Job builds skipped after a template/coinbase error without dropping the client.", "# TYPE qbit_prism_job_build_failures_total counter", - f"qbit_prism_job_build_failures_total {self.job_build_failure_count}", + f"qbit_prism_job_build_failures_total {int(job_metrics['failure_count'])}", "# HELP qbit_prism_block_candidates_dropped_total Legacy counter; durable candidate outbox rows are never dropped on queue overflow.", "# TYPE qbit_prism_block_candidates_dropped_total counter", f"qbit_prism_block_candidates_dropped_total {int(getattr(self, 'block_candidates_dropped', 0))}", @@ -18650,16 +11070,16 @@ def metrics_payload(self) -> str: ], "# HELP qbit_prism_share_append_queue_depth Accepted shares waiting on the ledger writer thread.", "# TYPE qbit_prism_share_append_queue_depth gauge", - f"qbit_prism_share_append_queue_depth {self.share_append_queue.qsize() if getattr(self, 'share_append_queue', None) is not None else 0}", + f"qbit_prism_share_append_queue_depth {share_writer_metrics.queue_depth}", "# HELP qbit_prism_share_append_failures_total Shares in group commits that failed before acknowledgement.", "# TYPE qbit_prism_share_append_failures_total counter", - f"qbit_prism_share_append_failures_total {int(getattr(self, 'share_append_failure_count', 0))}", + f"qbit_prism_share_append_failures_total {share_writer_metrics.append_failures}", "# HELP qbit_prism_shares_recovered_to_disk_total Legacy pre-commit-ACK shares written to the upgrade recovery file.", "# TYPE qbit_prism_shares_recovered_to_disk_total counter", - f"qbit_prism_shares_recovered_to_disk_total {int(getattr(self, 'shares_recovered_to_disk', 0))}", + f"qbit_prism_shares_recovered_to_disk_total {share_writer_metrics.recovered_to_disk}", "# HELP qbit_prism_shares_replayed_total Recovery-file shares replayed into the ledger at startup.", "# TYPE qbit_prism_shares_replayed_total counter", - f"qbit_prism_shares_replayed_total {int(getattr(self, 'shares_replayed', 0))}", + f"qbit_prism_shares_replayed_total {share_writer_metrics.replayed}", "# HELP qbit_prism_tip_refresh_jobs_total Client jobs refreshed after qbit tip/template changes.", "# TYPE qbit_prism_tip_refresh_jobs_total counter", f"qbit_prism_tip_refresh_jobs_total {self.tip_refresh_job_count}", @@ -18884,89 +11304,30 @@ def audit_artifact_kind(name: str) -> str: return "other" def ctv_fanout_broadcaster_metrics_lines(self) -> list[str]: - self._ensure_ctv_broadcaster_metrics_state() - with self._ctv_broadcaster_metrics_lock: - bucket_counts = dict(self.ctv_broadcaster_pass_seconds_bucket_counts) - pass_sum = self.ctv_broadcaster_pass_seconds_sum - pass_count = self.ctv_broadcaster_pass_count - processed_rows_total = self.ctv_broadcaster_processed_rows_total - yielded_total = self.ctv_broadcaster_yielded_total - chunk_seconds_buckets = dict( - self.ctv_broadcaster_chunk_seconds_bucket_counts - ) - chunk_rows_buckets = dict(self.ctv_broadcaster_chunk_rows_bucket_counts) - chunk_seconds_sum = self.ctv_broadcaster_chunk_seconds_sum - chunk_rows_sum = self.ctv_broadcaster_chunk_rows_sum - chunk_count = self.ctv_broadcaster_chunk_count - metric_name = "qbit_prism_ctv_fanout_broadcaster_pass_seconds" - chunk_seconds_name = "qbit_prism_ctv_fanout_broadcaster_chunk_seconds" - chunk_rows_name = "qbit_prism_ctv_fanout_broadcaster_chunk_rows" - return [ - "# HELP qbit_prism_ctv_fanout_broadcaster_processed_rows_total CTV fanout rows completed by the broadcaster loop.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_processed_rows_total counter", - f"qbit_prism_ctv_fanout_broadcaster_processed_rows_total {processed_rows_total}", - "# HELP qbit_prism_ctv_fanout_broadcaster_pass_seconds CTV fanout broadcaster pass wall time.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_pass_seconds histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {bucket_counts.get(bucket, 0)}' - for bucket in PRISM_CTV_BROADCASTER_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {pass_count}', - f"{metric_name}_sum {pass_sum:.6f}", - f"{metric_name}_count {pass_count}", - "# HELP qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total CTV broadcaster passes yielding between committed chunks for a pending tip refresh.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total counter", - f"qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total {yielded_total}", - "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_seconds CTV broadcaster committed chunk wall time.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_seconds histogram", - *[ - f'{chunk_seconds_name}_bucket{{le="{bucket:g}"}} {chunk_seconds_buckets.get(bucket, 0)}' - for bucket in PRISM_CTV_BROADCASTER_CHUNK_SECONDS_BUCKETS - ], - f'{chunk_seconds_name}_bucket{{le="+Inf"}} {chunk_count}', - f"{chunk_seconds_name}_sum {chunk_seconds_sum:.6f}", - f"{chunk_seconds_name}_count {chunk_count}", - "# HELP qbit_prism_ctv_fanout_broadcaster_chunk_rows Rows processed per committed CTV broadcaster chunk.", - "# TYPE qbit_prism_ctv_fanout_broadcaster_chunk_rows histogram", - *[ - f'{chunk_rows_name}_bucket{{le="{bucket}"}} {chunk_rows_buckets.get(bucket, 0)}' - for bucket in PRISM_CTV_BROADCASTER_CHUNK_ROWS_BUCKETS - ], - f'{chunk_rows_name}_bucket{{le="+Inf"}} {chunk_count}', - f"{chunk_rows_name}_sum {chunk_rows_sum}", - f"{chunk_rows_name}_count {chunk_count}", - ] + return self._ensure_ctv_runtime().metrics_lines() def initial_delivery_metrics_lines(self) -> list[str]: - self._ensure_initial_job_state() mining = self.mining_delivery_snapshot() - with self.lock: - counts = { - "sent": self.initial_job_sent_count, - "cancelled": self.initial_job_cancelled_count, - "coalesced": self.initial_job_coalesced_count, - "failed": self.initial_job_failed_count, - "superseded": self.initial_job_superseded_count, - } - latency_sum = self.initial_job_delivery_latency_seconds_sum - latency_count = self.initial_job_delivery_latency_count - queue_capacity_reclaimed = ( - self.initial_job_queue_capacity_reclaimed_count - ) - executor = getattr(self, "_initial_job_executor", None) - queued, slots = executor.stats() if executor is not None else (0, 0) - configured_workers = int( - getattr( - self, - "initial_job_max_workers", - DEFAULT_PRISM_INITIAL_JOB_MAX_WORKERS, - ) - ) - with self._bundle_preparation_lock: - build_counts = dict(self.shared_bundle_build_counts) - preparation_sum = self.shared_bundle_preparation_seconds_sum - preparation_count = self.shared_bundle_preparation_count - waiters = self.shared_bundle_preparation_waiters + initial_snapshot = self._ensure_job_delivery_service().initial_snapshot() + counts = { + "sent": initial_snapshot.sent_count, + "cancelled": initial_snapshot.cancelled_count, + "coalesced": initial_snapshot.coalesced_count, + "failed": initial_snapshot.failed_count, + "superseded": initial_snapshot.superseded_count, + } + latency_sum = initial_snapshot.delivery_latency_seconds_sum + latency_count = initial_snapshot.delivery_latency_count + queued, slots = self._ensure_job_delivery_service().initial_executor_stats() + configured_workers = initial_snapshot.max_workers + preparation = ( + self._ensure_job_bundle_service().shared_preparation_metrics() + ) + build_counts = preparation["build_counts"] + assert isinstance(build_counts, dict) + preparation_sum = float(preparation["preparation_sum"]) + preparation_count = int(preparation["preparation_count"]) + waiters = int(preparation["waiters"]) return [ "# HELP qbit_prism_stratum_subscribed_clients Subscribed Stratum clients.", "# TYPE qbit_prism_stratum_subscribed_clients gauge", @@ -19013,7 +11374,7 @@ def initial_delivery_metrics_lines(self) -> list[str]: ], "# HELP qbit_prism_initial_job_queue_capacity_reclaimed_total Queued initial-job slots reclaimed immediately by cancellation.", "# TYPE qbit_prism_initial_job_queue_capacity_reclaimed_total counter", - f"qbit_prism_initial_job_queue_capacity_reclaimed_total {queue_capacity_reclaimed}", + f"qbit_prism_initial_job_queue_capacity_reclaimed_total {initial_snapshot.queue_capacity_reclaimed_count}", "# HELP qbit_prism_shared_bundle_preparation_seconds Heavy shared bundle preparation wall time.", "# TYPE qbit_prism_shared_bundle_preparation_seconds summary", f"qbit_prism_shared_bundle_preparation_seconds_sum {preparation_sum:.6f}", @@ -19095,207 +11456,41 @@ def vardiff_idle_metrics_lines(self) -> list[str]: ) return lines def tip_refresh_metrics_lines(self) -> list[str]: - self._ensure_tip_refresh_state() - with self._tip_refresh_executor_lock: - executor_workers = ( - self.tip_refresh_max_workers - if self._tip_refresh_executor is not None - else 0 - ) - with self._tip_refresh_metrics_lock: - histograms = { - name: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for name, histogram in self.tip_refresh_histograms.items() - } - phase_histograms = { - phase: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for phase, histogram in self.tip_refresh_build_phase_histograms.items() - } - client_counts = dict(self.tip_refresh_client_counts) - cancellation_counts = dict(self.tip_refresh_cancellation_counts) - inflight = self.tip_refresh_inflight - build_inflight = self.tip_refresh_build_inflight - build_queue_depth = self.tip_refresh_build_queue_depth - singleflight_hits = self.tip_refresh_singleflight_hits - superseded_results = self.tip_refresh_superseded_results - worker_failures = self.tip_refresh_worker_failures - worker_restarts = self.tip_refresh_worker_restarts - ipc_bytes = dict(self.tip_refresh_ipc_bytes) - - metric_names = { - "refresh": "qbit_prism_tip_refresh_seconds", - "bundle_build": "qbit_prism_tip_refresh_bundle_build_seconds", - "first_delivery": "qbit_prism_tip_refresh_first_delivery_seconds", - "last_delivery": "qbit_prism_tip_refresh_last_delivery_seconds", - } - descriptions = { - "refresh": "Full qbit tip/template refresh pass wall time.", - "bundle_build": "Shared ready-pool refresh bundle preparation wall time.", - "first_delivery": "Tip observation to first successful client delivery.", - "last_delivery": "Tip observation to last successful client delivery.", - } - lines: list[str] = [] - for name, metric_name in metric_names.items(): - histogram = histograms[name] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - f"# HELP {metric_name} {descriptions[name]}", - f"# TYPE {metric_name} histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', - f'{metric_name}_sum {float(histogram["sum"]):.6f}', - f'{metric_name}_count {histogram["count"]}', - ] - ) - phase_metric_name = "qbit_prism_tip_refresh_bundle_phase_seconds" - lines.extend( - [ - "# HELP qbit_prism_tip_refresh_bundle_phase_seconds Shared bundle-build phase wall time.", - "# TYPE qbit_prism_tip_refresh_bundle_phase_seconds histogram", - ] - ) - for phase in PRISM_TIP_REFRESH_BUILD_PHASES: - histogram = phase_histograms[phase] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - *[ - f'{phase_metric_name}_bucket{{phase="{phase}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{phase_metric_name}_bucket{{phase="{phase}",le="+Inf"}} {histogram["count"]}', - f'{phase_metric_name}_sum{{phase="{phase}"}} {float(histogram["sum"]):.6f}', - f'{phase_metric_name}_count{{phase="{phase}"}} {histogram["count"]}', - ] - ) - lines.extend( - [ - "# HELP qbit_prism_tip_refresh_clients_total Client outcomes from tip/template refresh passes.", - "# TYPE qbit_prism_tip_refresh_clients_total counter", - *[ - f'qbit_prism_tip_refresh_clients_total{{result="{result}"}} {int(client_counts.get(result, 0))}' - for result in PRISM_TIP_REFRESH_RESULTS - ], - "# HELP qbit_prism_tip_refresh_cancellations_total Obsolete prepared refresh tasks canceled before delivery admission.", - "# TYPE qbit_prism_tip_refresh_cancellations_total counter", - *[ - f'qbit_prism_tip_refresh_cancellations_total{{stage="{stage}"}} {int(cancellation_counts.get(stage, 0))}' - for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES - ], - "# HELP qbit_prism_tip_refresh_inflight Prepared refresh client tasks currently queued or running.", - "# TYPE qbit_prism_tip_refresh_inflight gauge", - f"qbit_prism_tip_refresh_inflight {inflight}", - "# HELP qbit_prism_tip_refresh_executor_workers Configured persistent refresh executor workers, or zero before creation.", - "# TYPE qbit_prism_tip_refresh_executor_workers gauge", - f"qbit_prism_tip_refresh_executor_workers {executor_workers}", - "# HELP qbit_prism_tip_refresh_bundle_inflight Shared bundle builds currently running.", - "# TYPE qbit_prism_tip_refresh_bundle_inflight gauge", - f"qbit_prism_tip_refresh_bundle_inflight {build_inflight}", - "# HELP qbit_prism_tip_refresh_bundle_queue_depth Shared bundle callers waiting on bounded build admission or an identical single-flight.", - "# TYPE qbit_prism_tip_refresh_bundle_queue_depth gauge", - f"qbit_prism_tip_refresh_bundle_queue_depth {build_queue_depth}", - "# HELP qbit_prism_tip_refresh_bundle_singleflight_hits_total Shared bundle callers coalesced behind an identical build.", - "# TYPE qbit_prism_tip_refresh_bundle_singleflight_hits_total counter", - f"qbit_prism_tip_refresh_bundle_singleflight_hits_total {singleflight_hits}", - "# HELP qbit_prism_tip_refresh_bundle_superseded_results_total Completed or canceled shared bundles discarded after supersession.", - "# TYPE qbit_prism_tip_refresh_bundle_superseded_results_total counter", - f"qbit_prism_tip_refresh_bundle_superseded_results_total {superseded_results}", - "# HELP qbit_prism_tip_refresh_builder_worker_failures_total Audit-builder subprocess failures.", - "# TYPE qbit_prism_tip_refresh_builder_worker_failures_total counter", - f"qbit_prism_tip_refresh_builder_worker_failures_total {worker_failures}", - "# HELP qbit_prism_tip_refresh_builder_worker_restarts_total Long-lived builder worker restarts; zero for the inline subprocess design.", - "# TYPE qbit_prism_tip_refresh_builder_worker_restarts_total counter", - f"qbit_prism_tip_refresh_builder_worker_restarts_total {worker_restarts}", - "# HELP qbit_prism_tip_refresh_builder_ipc_bytes_total Bytes copied across audit-builder subprocess IPC.", - "# TYPE qbit_prism_tip_refresh_builder_ipc_bytes_total counter", - *[ - f'qbit_prism_tip_refresh_builder_ipc_bytes_total{{direction="{direction}"}} {int(ipc_bytes.get(direction, 0))}' - for direction in ("input", "output") - ], - ] - ) - return lines + return self._ensure_tip_refresh_service().metrics_lines() def job_build_metrics_lines(self) -> list[str]: self._ensure_job_cache_state() - with self._job_cache_lock: - bucket_counts = dict(self.job_build_seconds_bucket_counts) - build_sum = self.job_build_seconds_sum - build_count = self.job_build_count - phase_seconds = dict(self.job_build_phase_seconds) - hit_counts = dict(self.job_cache_hit_counts) - miss_counts = dict(self.job_cache_miss_counts) + snapshot = self._ensure_job_bundle_service().metrics_snapshot() + bucket_counts = snapshot["bucket_counts"] + build_sum = float(snapshot["build_sum"]) + build_count = int(snapshot["build_count"]) + phase_seconds = snapshot["phase_seconds"] + hit_counts = snapshot["hit_counts"] + miss_counts = snapshot["miss_counts"] + scheduler_counts = snapshot["scheduler_counts"] + priority_counts = snapshot["priority_counts"] + priority_admission_seconds = snapshot["priority_admission_seconds"] + initial_prepared_counts = snapshot["initial_prepared_counts"] + cancellation_seconds = snapshot["cancellation_seconds"] + replacement_seconds = snapshot["replacement_seconds"] + worker_counts = snapshot["worker_counts"] + active_builds = int(snapshot["active_builds"]) + pending_builds = int(snapshot["pending_builds"]) + priority_active = int(snapshot["priority_active"]) + priority_age_seconds = float(snapshot["priority_age_seconds"]) + assert isinstance(bucket_counts, dict) + assert isinstance(phase_seconds, dict) + assert isinstance(hit_counts, dict) + assert isinstance(miss_counts, dict) + assert isinstance(scheduler_counts, dict) + assert isinstance(priority_counts, dict) + assert isinstance(priority_admission_seconds, dict) + assert isinstance(initial_prepared_counts, dict) + assert isinstance(cancellation_seconds, dict) + assert isinstance(replacement_seconds, dict) + assert isinstance(worker_counts, dict) + with self._health_snapshot_lock: health_refresh_failures = self.health_snapshot_refresh_failure_count - with self._job_build_scheduler_lock: - scheduler_counts = dict(self.job_build_scheduler_counts) - priority_counts = dict(self.job_build_priority_counts) - priority_admission_seconds = dict( - self.job_build_priority_admission_seconds - ) - initial_prepared_counts = dict( - self.initial_job_prepared_work_counts - ) - cancellation_seconds = dict(self.job_build_cancellation_seconds) - replacement_seconds = dict(self.job_build_replacement_start_seconds) - worker_counts = dict(self.job_build_worker_counts) - active_builds = int(self._job_build_active is not None) - pending_builds = int(self._job_build_pending is not None) - priority_requests = tuple( - request - for request in ( - ( - self._job_build_active.request - if self._job_build_active is not None - else None - ), - ( - self._job_build_retiring.request - if self._job_build_retiring is not None - else None - ), - self._job_build_pending, - ) - if request is not None - and not request.cancellation.is_set() - and self._job_build_is_publication_critical(request) - ) - priority_preparations = tuple( - self._job_build_priority_preparations.values() - ) - priority_active = int( - bool(priority_requests or priority_preparations) - ) - priority_age_seconds = max( - ( - time.monotonic() - request.requested_monotonic - for request in priority_requests - ), - default=0.0, - ) - priority_age_seconds = max( - priority_age_seconds, - max( - ( - time.monotonic() - started - for started in priority_preparations - ), - default=0.0, - ), - ) lock = getattr(self, "lock", None) if lock is not None: with lock: @@ -19407,85 +11602,7 @@ def job_build_metrics_lines(self) -> list[str]: return lines def payout_state_metrics_lines(self) -> list[str]: - self._ensure_job_cache_state() - with self._payout_state_metrics_lock: - state_histograms = { - name: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for name, histogram in self.payout_state_histograms.items() - } - gate_histograms = { - relation: { - "buckets": dict(histogram["buckets"]), - "sum": float(histogram["sum"]), - "count": int(histogram["count"]), - } - for relation, histogram in self.payout_gate_wait_histograms.items() - } - discarded = self.payout_state_candidates_discarded - - metric_names = { - "preparation": "qbit_prism_payout_preparation_seconds", - "publish": "qbit_prism_payout_publish_seconds", - "first_delivery": "qbit_prism_payout_invalidation_first_delivery_seconds", - } - descriptions = { - "preparation": "Payout reconciliation and candidate preparation outside delivery publication.", - "publish": "Atomic payout generation/cache publication gate-hold time.", - "first_delivery": "Payout invalidation to first delivery of the published generation.", - } - lines: list[str] = [] - for name, metric_name in metric_names.items(): - histogram = state_histograms[name] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - f"# HELP {metric_name} {descriptions[name]}", - f"# TYPE {metric_name} histogram", - *[ - f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', - f'{metric_name}_sum {float(histogram["sum"]):.6f}', - f'{metric_name}_count {histogram["count"]}', - ] - ) - - gate_name = "qbit_prism_payout_gate_wait_seconds" - lines.extend( - [ - "# HELP qbit_prism_payout_gate_wait_seconds Delivery admission wait by generation relationship to the published payout state.", - "# TYPE qbit_prism_payout_gate_wait_seconds histogram", - ] - ) - for relation in PRISM_PAYOUT_DELIVERY_GENERATIONS: - histogram = gate_histograms[relation] - buckets = histogram["buckets"] - assert isinstance(buckets, dict) - lines.extend( - [ - *[ - f'{gate_name}_bucket{{generation="{relation}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' - for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS - ], - f'{gate_name}_bucket{{generation="{relation}",le="+Inf"}} {histogram["count"]}', - f'{gate_name}_sum{{generation="{relation}"}} {float(histogram["sum"]):.6f}', - f'{gate_name}_count{{generation="{relation}"}} {histogram["count"]}', - ] - ) - lines.extend( - [ - "# HELP qbit_prism_payout_candidates_discarded_total Prepared payout candidates discarded after source supersession.", - "# TYPE qbit_prism_payout_candidates_discarded_total counter", - f"qbit_prism_payout_candidates_discarded_total {discarded}", - ] - ) - return lines + return self._ensure_payout_state_service().metrics_lines() def make_audit_handler(coordinator: PrismCoordinator) -> type[BaseHTTPRequestHandler]: diff --git a/lab/prism/progress_health.py b/lab/prism/progress_health.py new file mode 100644 index 0000000..847feb4 --- /dev/null +++ b/lab/prism/progress_health.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""Monotonic mining-progress readiness state for the PRISM coordinator.""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import MappingProxyType +import threading +import time +from typing import Callable, Mapping + + +PROGRESS_HEALTH_REASONS = ( + "tip_poll_stale", + "refresh_pending_too_long", + "current_generation_not_published", + "current_generation_not_delivered", + "bundle_build_stuck", +) + + +@dataclass(frozen=True) +class WorkGeneration: + template_generation: int + template_fingerprint: str | None + payout_generation: int + + +@dataclass(frozen=True) +class DeliveryProof: + connection_id: int + delivered_work: WorkGeneration + collection_only: bool + delivered_monotonic: float + + +@dataclass(frozen=True) +class EligibilitySnapshot: + eligible_connection_ids: tuple[int, ...] + delivery_proofs: tuple[DeliveryProof, ...] + ready_mode_required: bool + + +@dataclass(frozen=True) +class ProgressHealthConfig: + pending_refresh_deadline_seconds: float + tip_poll_deadline_seconds: float + bundle_build_deadline_seconds: float + + +@dataclass(frozen=True) +class ProgressHealthSnapshot: + ok: bool + reason: str | None + reasons: tuple[str, ...] + pending_refresh: bool + pending_refresh_age_seconds: float | None + tip_poll_age_seconds: float + tip_refresh_in_progress: bool + tip_refresh_progress_age_seconds: float | None + current_template_generation: int + published_template_generation: int + current_payout_generation: int + published_payout_generation: int + last_valid_delivery_age_seconds: float | None + eligible_client_count: int + eligible_clients_requiring_refresh: int + bundle_build_oldest_age_seconds: float + + def as_mapping(self) -> dict[str, object]: + """Return the existing mutable HTTP/test compatibility representation.""" + + return { + "ok": self.ok, + "reason": self.reason, + "reasons": list(self.reasons), + "pending_refresh": self.pending_refresh, + "pending_refresh_age_seconds": self.pending_refresh_age_seconds, + "tip_poll_age_seconds": self.tip_poll_age_seconds, + "tip_refresh_in_progress": self.tip_refresh_in_progress, + "tip_refresh_progress_age_seconds": self.tip_refresh_progress_age_seconds, + "current_template_generation": self.current_template_generation, + "published_template_generation": self.published_template_generation, + "current_payout_generation": self.current_payout_generation, + "published_payout_generation": self.published_payout_generation, + "last_valid_delivery_age_seconds": self.last_valid_delivery_age_seconds, + "eligible_client_count": self.eligible_client_count, + "eligible_clients_requiring_refresh": ( + self.eligible_clients_requiring_refresh + ), + "bundle_build_oldest_age_seconds": self.bundle_build_oldest_age_seconds, + } + + +@dataclass(frozen=True) +class _ProgressStateCopy: + current_work: WorkGeneration + published_work: WorkGeneration + has_published_work: bool + last_tip_poll_monotonic: float | None + last_delivery: DeliveryProof | None + pending_since_monotonic: float | None + refresh_signal_pending: bool + active_refresh_count: int + last_refresh_activity_monotonic: float | None + bundle_build_starts: tuple[float, ...] + + +class RefreshActivityToken: + """One idempotent, context-managed refresh activity lifetime.""" + + def __init__(self, service: ProgressHealthService, token_id: int) -> None: + self._service = service + self._token_id = token_id + self._state_lock = threading.Lock() + self._finished = False + + def note_activity(self, observed_monotonic: float | None = None) -> None: + with self._state_lock: + if self._finished: + return + self._service._note_refresh_activity( + self._token_id, + observed_monotonic, + ) + + def finish(self) -> None: + with self._state_lock: + if self._finished: + return + self._finished = True + self._service._finish_refresh(self._token_id) + + cancel = finish + + def __enter__(self) -> RefreshActivityToken: + return self + + def __exit__(self, *_args: object) -> None: + self.finish() + + +class BundleBuildToken: + """One idempotent, context-managed bundle construction lifetime.""" + + def __init__(self, service: ProgressHealthService, token_id: int) -> None: + self._service = service + self._token_id = token_id + self._state_lock = threading.Lock() + self._finished = False + + def finish(self) -> None: + with self._state_lock: + if self._finished: + return + self._finished = True + self._service._finish_bundle_build(self._token_id) + + cancel = finish + + def __enter__(self) -> BundleBuildToken: + return self + + def __exit__(self, *_args: object) -> None: + self.finish() + + +class ProgressHealthService: + """Own aggregate mining-progress state and evaluate bounded readiness.""" + + def __init__( + self, + config: ProgressHealthConfig, + *, + started_monotonic: float, + initial_payout_generation: int = 0, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.config = config + self._monotonic = monotonic + self._started_monotonic = float(started_monotonic) + self._lock = threading.Lock() + self._current_work = WorkGeneration(0, None, initial_payout_generation) + self._published_work = WorkGeneration(0, None, 0) + self._has_published_work = False + self._last_tip_poll_monotonic: float | None = None + self._last_delivery: DeliveryProof | None = None + self._pending_since_monotonic: float | None = self._started_monotonic + self._refresh_signal_pending = False + self._refresh_token_counter = 0 + self._active_refresh_tokens: set[int] = set() + self._last_refresh_activity_monotonic: float | None = None + self._bundle_build_counter = 0 + self._bundle_builds: dict[int, float] = {} + + def now(self) -> float: + return float(self._monotonic()) + + def mark_refresh_pending(self, started_monotonic: float | None = None) -> None: + started = self.now() if started_monotonic is None else started_monotonic + with self._lock: + pending_since = self._pending_since_monotonic + if pending_since is None or started < pending_since: + self._pending_since_monotonic = started + self._refresh_signal_pending = True + + def observe_tip( + self, + work: WorkGeneration, + observed_monotonic: float | None = None, + ) -> None: + observed = self.now() if observed_monotonic is None else observed_monotonic + with self._lock: + if work.template_generation < self._current_work.template_generation: + return + current_work = WorkGeneration( + work.template_generation, + work.template_fingerprint, + max(self._current_work.payout_generation, work.payout_generation), + ) + self._current_work = current_work + self._last_tip_poll_monotonic = observed + same_published_work = bool( + self._has_published_work + and self._published_work.template_fingerprint + == current_work.template_fingerprint + and self._published_work.payout_generation + == current_work.payout_generation + ) + if same_published_work: + self._published_work = WorkGeneration( + current_work.template_generation, + self._published_work.template_fingerprint, + self._published_work.payout_generation, + ) + else: + pending_since = self._pending_since_monotonic + if pending_since is None or observed < pending_since: + self._pending_since_monotonic = observed + + def observe_payout_generation( + self, + generation: int, + invalidated_monotonic: float | None = None, + ) -> None: + invalidated = ( + self.now() + if invalidated_monotonic is None + else invalidated_monotonic + ) + with self._lock: + if generation < self._current_work.payout_generation: + return + self._current_work = WorkGeneration( + self._current_work.template_generation, + self._current_work.template_fingerprint, + generation, + ) + if generation != self._published_work.payout_generation: + pending_since = self._pending_since_monotonic + if pending_since is None or invalidated < pending_since: + self._pending_since_monotonic = invalidated + self._refresh_signal_pending = True + + def publish_work(self, work: WorkGeneration) -> bool: + with self._lock: + if ( + work.template_generation < self._published_work.template_generation + or work.payout_generation < self._published_work.payout_generation + ): + return False + self._published_work = WorkGeneration( + max( + self._published_work.template_generation, + work.template_generation, + ), + work.template_fingerprint, + max( + self._published_work.payout_generation, + work.payout_generation, + ), + ) + self._has_published_work = True + if ( + self._current_work.template_fingerprint + == work.template_fingerprint + and self._current_work.payout_generation == work.payout_generation + ): + self._refresh_signal_pending = False + self._note_any_refresh_activity() + return True + + def record_delivery( + self, + proof: DeliveryProof, + ready_mode_required: bool, + ) -> None: + with self._lock: + work = proof.delivered_work + if ( + work.template_fingerprint + == self._current_work.template_fingerprint + and work.payout_generation == self._current_work.payout_generation + and not (ready_mode_required and proof.collection_only) + ): + self._last_delivery = proof + self._published_work = WorkGeneration( + max( + self._published_work.template_generation, + self._current_work.template_generation, + ), + work.template_fingerprint, + max( + self._published_work.payout_generation, + work.payout_generation, + ), + ) + self._has_published_work = True + self._refresh_signal_pending = False + self._note_refresh_activity_locked(proof.delivered_monotonic) + + def start_refresh(self) -> RefreshActivityToken: + started = self.now() + with self._lock: + self._refresh_token_counter += 1 + token_id = self._refresh_token_counter + self._active_refresh_tokens.add(token_id) + self._last_refresh_activity_monotonic = started + return RefreshActivityToken(self, token_id) + + def _note_refresh_activity( + self, + token_id: int, + observed_monotonic: float | None, + ) -> None: + observed = self.now() if observed_monotonic is None else observed_monotonic + with self._lock: + if token_id in self._active_refresh_tokens: + self._note_refresh_activity_locked(observed) + + def _note_any_refresh_activity( + self, + observed_monotonic: float | None = None, + ) -> None: + observed = self.now() if observed_monotonic is None else observed_monotonic + with self._lock: + self._note_refresh_activity_locked(observed) + + def _note_refresh_activity_locked(self, observed_monotonic: float) -> None: + if self._active_refresh_tokens and ( + self._last_refresh_activity_monotonic is None + or observed_monotonic > self._last_refresh_activity_monotonic + ): + self._last_refresh_activity_monotonic = observed_monotonic + + def _finish_refresh(self, token_id: int) -> None: + with self._lock: + self._active_refresh_tokens.discard(token_id) + + def start_bundle_build(self) -> BundleBuildToken: + started = self.now() + with self._lock: + self._bundle_build_counter += 1 + token_id = self._bundle_build_counter + self._bundle_builds[token_id] = started + return BundleBuildToken(self, token_id) + + def _finish_bundle_build(self, token_id: int) -> None: + with self._lock: + self._bundle_builds.pop(token_id, None) + + @staticmethod + def _requiring_refresh( + eligibility: EligibilitySnapshot, + work: WorkGeneration, + ) -> int: + proofs = { + proof.connection_id: proof + for proof in eligibility.delivery_proofs + } + requiring_refresh = 0 + for connection_id in eligibility.eligible_connection_ids: + proof = proofs.get(connection_id) + if ( + proof is None + or proof.delivered_work.template_fingerprint + != work.template_fingerprint + or proof.delivered_work.payout_generation + != work.payout_generation + or ( + eligibility.ready_mode_required + and proof.collection_only + ) + ): + requiring_refresh += 1 + return requiring_refresh + + def _copy_state_locked(self) -> _ProgressStateCopy: + return _ProgressStateCopy( + current_work=self._current_work, + published_work=self._published_work, + has_published_work=self._has_published_work, + last_tip_poll_monotonic=self._last_tip_poll_monotonic, + last_delivery=self._last_delivery, + pending_since_monotonic=self._pending_since_monotonic, + refresh_signal_pending=self._refresh_signal_pending, + active_refresh_count=len(self._active_refresh_tokens), + last_refresh_activity_monotonic=( + self._last_refresh_activity_monotonic + ), + bundle_build_starts=tuple(self._bundle_builds.values()), + ) + + def reconcile_pending( + self, + eligibility: EligibilitySnapshot, + *, + now: float | None = None, + ) -> None: + """Reconcile pending state from an already-captured client snapshot.""" + + current = self.now() if now is None else now + with self._lock: + reconcile_work = self._current_work + requiring_refresh = self._requiring_refresh(eligibility, reconcile_work) + with self._lock: + if reconcile_work != self._current_work: + return + published_current = bool( + self._has_published_work + and self._published_work.template_fingerprint + == reconcile_work.template_fingerprint + and self._published_work.payout_generation + == reconcile_work.payout_generation + ) + refresh_required = bool( + self._refresh_signal_pending + or not published_current + or requiring_refresh > 0 + ) + if refresh_required: + if self._pending_since_monotonic is None: + self._pending_since_monotonic = current + else: + self._pending_since_monotonic = None + + def snapshot( + self, + eligibility: EligibilitySnapshot, + current_payout_generation: int, + *, + now: float | None = None, + ) -> ProgressHealthSnapshot: + current = self.now() if now is None else now + with self._lock: + tracked_payout_generation = self._current_work.payout_generation + if current_payout_generation > tracked_payout_generation: + self.observe_payout_generation(current_payout_generation, current) + self.reconcile_pending(eligibility, now=current) + with self._lock: + state = self._copy_state_locked() + + requiring_refresh = self._requiring_refresh( + eligibility, + state.current_work, + ) + tip_poll_reference = ( + self._started_monotonic + if state.last_tip_poll_monotonic is None + else state.last_tip_poll_monotonic + ) + tip_poll_age = max(0.0, current - tip_poll_reference) + refresh_activity_age = ( + None + if ( + state.active_refresh_count <= 0 + or state.last_refresh_activity_monotonic is None + ) + else max(0.0, current - state.last_refresh_activity_monotonic) + ) + pending_age = ( + None + if state.pending_since_monotonic is None + else max(0.0, current - state.pending_since_monotonic) + ) + oldest_bundle_age = ( + 0.0 + if not state.bundle_build_starts + else max(0.0, current - min(state.bundle_build_starts)) + ) + published_current = bool( + state.has_published_work + and state.published_work.template_fingerprint + == state.current_work.template_fingerprint + and state.published_work.payout_generation + == state.current_work.payout_generation + ) + delivered_current = bool( + state.current_work.template_fingerprint is not None + and state.last_delivery is not None + and state.last_delivery.delivered_work.template_fingerprint + == state.current_work.template_fingerprint + and state.last_delivery.delivered_work.payout_generation + == state.current_work.payout_generation + ) + delivery_age = ( + max(0.0, current - state.last_delivery.delivered_monotonic) + if delivered_current and state.last_delivery is not None + else None + ) + refresh_is_progressing = bool( + state.active_refresh_count > 0 + and refresh_activity_age is not None + and refresh_activity_age <= self.config.tip_poll_deadline_seconds + ) + reasons: list[str] = [] + if ( + tip_poll_age > self.config.tip_poll_deadline_seconds + and not refresh_is_progressing + ): + reasons.append("tip_poll_stale") + if oldest_bundle_age > self.config.bundle_build_deadline_seconds: + reasons.append("bundle_build_stuck") + if not state.has_published_work: + reasons.append("current_generation_not_published") + elif ( + pending_age is not None + and pending_age > self.config.pending_refresh_deadline_seconds + ): + reasons.append("refresh_pending_too_long") + if state.refresh_signal_pending or not published_current: + reasons.append("current_generation_not_published") + elif requiring_refresh > 0: + reasons.append("current_generation_not_delivered") + reasons = list(dict.fromkeys(reasons)) + + def rounded(value: float | None) -> float | None: + return None if value is None else round(value, 3) + + return ProgressHealthSnapshot( + ok=not reasons, + reason=reasons[0] if reasons else None, + reasons=tuple(reasons), + pending_refresh=state.pending_since_monotonic is not None, + pending_refresh_age_seconds=rounded(pending_age), + tip_poll_age_seconds=round(tip_poll_age, 3), + tip_refresh_in_progress=state.active_refresh_count > 0, + tip_refresh_progress_age_seconds=rounded(refresh_activity_age), + current_template_generation=state.current_work.template_generation, + published_template_generation=state.published_work.template_generation, + current_payout_generation=state.current_work.payout_generation, + published_payout_generation=state.published_work.payout_generation, + last_valid_delivery_age_seconds=rounded(delivery_age), + eligible_client_count=len(eligibility.eligible_connection_ids), + eligible_clients_requiring_refresh=requiring_refresh, + bundle_build_oldest_age_seconds=round(oldest_bundle_age, 3), + ) + + @staticmethod + def overlay( + base_health: Mapping[str, object], + snapshot: ProgressHealthSnapshot, + ) -> Mapping[str, object]: + return overlay_progress_health(base_health, snapshot.as_mapping()) + + @staticmethod + def metrics_lines(snapshot: ProgressHealthSnapshot) -> tuple[str, ...]: + pending_age = snapshot.pending_refresh_age_seconds + delivery_age = snapshot.last_valid_delivery_age_seconds + active_reasons = set(snapshot.reasons) + return ( + "# HELP qbit_prism_refresh_pending Whether current template or payout work still requires publication or delivery.", + "# TYPE qbit_prism_refresh_pending gauge", + f"qbit_prism_refresh_pending {1 if snapshot.pending_refresh else 0}", + "# HELP qbit_prism_refresh_pending_age_seconds Monotonic age of the oldest unresolved current-work refresh.", + "# TYPE qbit_prism_refresh_pending_age_seconds gauge", + f"qbit_prism_refresh_pending_age_seconds {float(pending_age or 0.0):.6f}", + "# HELP qbit_prism_tip_poll_age_seconds Monotonic age of the last coherent qbit tip/template poll.", + "# TYPE qbit_prism_tip_poll_age_seconds gauge", + f"qbit_prism_tip_poll_age_seconds {snapshot.tip_poll_age_seconds:.6f}", + "# HELP qbit_prism_current_generation_delivery_age_seconds Monotonic age of the last valid current-generation delivery, or -1 when none exists.", + "# TYPE qbit_prism_current_generation_delivery_age_seconds gauge", + f"qbit_prism_current_generation_delivery_age_seconds {float(delivery_age) if delivery_age is not None else -1.0:.6f}", + "# HELP qbit_prism_bundle_build_oldest_age_seconds Monotonic age of the oldest active bundle build.", + "# TYPE qbit_prism_bundle_build_oldest_age_seconds gauge", + f"qbit_prism_bundle_build_oldest_age_seconds {snapshot.bundle_build_oldest_age_seconds:.6f}", + "# HELP qbit_prism_health_state Current progress-health state by bounded reason.", + "# TYPE qbit_prism_health_state gauge", + f'qbit_prism_health_state{{reason="healthy"}} {1 if snapshot.ok else 0}', + *( + f'qbit_prism_health_state{{reason="{reason}"}} {1 if reason in active_reasons else 0}' + for reason in PROGRESS_HEALTH_REASONS + ), + ) + + +def overlay_progress_health( + base_health: Mapping[str, object], + progress: Mapping[str, object], +) -> Mapping[str, object]: + """Overlay current progress without masking an independent base failure.""" + + base_ok = bool(base_health.get("mining_ready", base_health.get("ok"))) + result = dict(base_health) + result.update(progress) + result["ok"] = base_ok and bool(progress["ok"]) + if progress["ok"]: + result.pop("reason", None) + result.pop("reasons", None) + return MappingProxyType(result) + + +__all__ = [ + "BundleBuildToken", + "DeliveryProof", + "EligibilitySnapshot", + "PROGRESS_HEALTH_REASONS", + "ProgressHealthConfig", + "ProgressHealthService", + "ProgressHealthSnapshot", + "RefreshActivityToken", + "WorkGeneration", + "overlay_progress_health", +] diff --git a/lab/prism/rpc.py b/lab/prism/rpc.py new file mode 100644 index 0000000..388aada --- /dev/null +++ b/lab/prism/rpc.py @@ -0,0 +1,112 @@ +"""Thread-local JSON-RPC client used by PRISM processes.""" + +from __future__ import annotations + +import base64 +import http.client +import json +import threading +import urllib.parse +from typing import Any + + +class JsonRpc: + """Minimal qbit JSON-RPC client with one keep-alive connection per thread.""" + + def __init__(self, *, host: str, port: int, user: str, password: str): + self.host = host + self.port = port + self.url = f"http://{host}:{port}" + credentials = f"{user}:{password}".encode() + self.auth = f"Basic {base64.b64encode(credentials).decode()}" + # Keep-alive connections, one per calling thread. qbitd is called on + # the hot share/block paths (a fresh getaddrinfo + TCP connect per call + # was ~seconds of overhead under load); reusing the connection removes + # that. threading.local keeps each thread's HTTPConnection private, so + # concurrent callers never share a non-thread-safe connection. + self._connections = threading.local() + + def _acquire_connection(self, timeout: float) -> http.client.HTTPConnection: + conn = getattr(self._connections, "conn", None) + if conn is None: + conn = http.client.HTTPConnection(self.host, self.port, timeout=timeout) + self._connections.conn = conn + else: + # Reuse: refresh the deadline for this call on the live socket. + conn.timeout = timeout + if conn.sock is not None: + conn.sock.settimeout(timeout) + return conn + + def _drop_connection(self) -> None: + conn = getattr(self._connections, "conn", None) + if conn is not None: + try: + conn.close() + except Exception: + pass + self._connections.conn = None + + def call( + self, + method: str, + params: list[object] | None = None, + *, + wallet: str | None = None, + timeout: float = 10, + ) -> Any: + body = json.dumps( + { + "jsonrpc": "1.0", + "id": method, + "method": method, + "params": params or [], + } + ).encode() + path = "/" + if wallet is not None: + path = f"/wallet/{urllib.parse.quote(wallet, safe='')}" + headers = { + "Authorization": self.auth, + "Content-Type": "application/json", + "User-Agent": "qbit-prism-coordinator/0.1", + } + # One retry with a fresh connection on a transport error. The usual + # cause is the server having closed an idle keep-alive connection, in + # which case the request never reached qbitd, so retrying is safe; the + # only state-changing RPC (submitblock) is idempotent (duplicate -> + # "duplicate") regardless. A second failure raises to the caller, which + # treats it as backend-rpc-unavailable (a rejected share/block, never a + # lost or double-counted block). + last_exc: Exception | None = None + for attempt in range(2): + conn = self._acquire_connection(timeout) + try: + conn.request("POST", path, body=body, headers=headers) + response = conn.getresponse() + data = response.read() # drain so the connection can be reused + except (http.client.HTTPException, OSError) as exc: + last_exc = exc + self._drop_connection() + if attempt == 0: + continue + raise + if response.status != 200: + # Non-200 bodies may hold a JSON-RPC error (qbitd returns the + # error object with a 500 for some methods); surface it as the + # same RuntimeError text callers already match on (e.g. the + # "-32601 / Method not found" blockwait-unsupported probe). + self._drop_connection() + detail = data.decode("utf-8", "replace") + try: + error = json.loads(detail).get("error") + except Exception: + error = None + if error is not None: + raise RuntimeError(f"qbit RPC {method} failed: {error}") + raise RuntimeError(f"qbit RPC {method} HTTP {response.status}: {detail[:200]}") + payload = json.loads(data) + if payload["error"] is not None: + raise RuntimeError(f"qbit RPC {method} failed: {payload['error']}") + return payload["result"] + raise last_exc if last_exc is not None else RuntimeError("qbit RPC call failed") diff --git a/lab/prism/run_ctv_broadcaster_daemon.py b/lab/prism/run_ctv_broadcaster_daemon.py index c8cd263..5abe4c3 100644 --- a/lab/prism/run_ctv_broadcaster_daemon.py +++ b/lab/prism/run_ctv_broadcaster_daemon.py @@ -19,8 +19,9 @@ CtvFanoutDaemonResult, MAX_CTV_FANOUT_BROADCASTER_CHUNK_SIZE, ) +from lab.prism.coordinator_config import env, env_bool, env_int, env_positive_float +from lab.prism.rpc import JsonRpc from lab.prism.share_ledger import PsqlShareLedger, SingleWriterShareLedger -from lab.prism.prism_coordinator import JsonRpc, env, env_bool, env_int, env_positive_float def env_positive_int(name: str, default: int | None = None) -> int: diff --git a/lab/prism/share_ledger.py b/lab/prism/share_ledger.py index 2404e12..e793956 100644 --- a/lab/prism/share_ledger.py +++ b/lab/prism/share_ledger.py @@ -99,6 +99,18 @@ class PendingShare: credit_policy: str | None = None +class ShareReplayConflict(RuntimeError): + """A recovery row reused a share ID with a different durable payload.""" + + +@dataclass(frozen=True) +class ShareReplayResult: + """Typed result for one legacy recovery-journal append.""" + + disposition: str + record: AcceptedShareRecord + + class SingleWriterShareLedger: """Assigns canonical share_seq values and returns immutable snapshots. @@ -167,6 +179,46 @@ def append(self, pending: PendingShare) -> AcceptedShareRecord: self._next_share_seq += 1 return record + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + """Append one recovery row with an explicit exact/conflict outcome.""" + if pending.share_difficulty <= 0: + raise ValueError("share_difficulty must be positive") + if pending.network_difficulty <= 0: + raise ValueError("network_difficulty must be positive") + credit_policy = validate_credit_policy(pending.credit_policy) + with self._lock: + existing = self._shares_by_id.get(pending.share_id) + if existing is not None: + if not self._pending_matches_record( + pending, + existing, + credit_policy=credit_policy, + ): + raise ShareReplayConflict( + f"recovered share payload conflicts with {pending.share_id}" + ) + return ShareReplayResult("exact_existing", replace(existing)) + record = AcceptedShareRecord( + share_seq=self._next_share_seq, + share_id=pending.share_id, + miner_id=pending.miner_id, + order_key=pending.order_key, + p2mr_program_hex=pending.p2mr_program_hex, + share_difficulty=pending.share_difficulty, + network_difficulty=pending.network_difficulty, + template_height=pending.template_height, + job_id=pending.job_id, + job_issued_at_ms=pending.job_issued_at_ms, + accepted_at_ms=pending.accepted_at_ms, + ntime=pending.ntime, + credit_policy=credit_policy, + ) + self._shares.append(record) + self._share_ids.add(pending.share_id) + self._shares_by_id[pending.share_id] = record + self._next_share_seq += 1 + return ShareReplayResult("inserted", replace(record)) + @staticmethod def _pending_matches_record( pending: PendingShare, @@ -1556,10 +1608,10 @@ def append(self, pending: PendingShare) -> AcceptedShareRecord: ) return record - def append_batch( + def _append_batch_with_replay_outcomes( self, entries: list[tuple[PendingShare, dict[str, Any] | None]], - ) -> list[AcceptedShareRecord]: + ) -> list[ShareReplayResult]: """Commit accepted shares and optional block intents in one transaction. Replaying the exact same payload is idempotent. Reusing a share ID or @@ -1745,6 +1797,7 @@ def append_batch( WHEN EXISTS (SELECT 1 FROM share_mismatch) THEN json_build_object( 'error', 'duplicate share_id payload mismatch', + 'error_kind', 'share_replay_conflict', 'share_ids', (SELECT json_agg(share_id ORDER BY share_id) FROM share_mismatch) ) WHEN EXISTS (SELECT 1 FROM candidate_mismatch) THEN @@ -1779,6 +1832,8 @@ def append_batch( with self._lock: result = self._run_json(sql) if "error" in result: + if result.get("error_kind") == "share_replay_conflict": + raise ShareReplayConflict(str(result["error"])) raise RuntimeError(str(result["error"])) records = result.get("records") if not isinstance(records, list) or len(records) != len(entries): @@ -1794,7 +1849,33 @@ def append_batch( record, new_miner=bool(payload.get("new_miner", False)), ) - return parsed + return [ + ShareReplayResult( + ( + "inserted" + if bool(payload.get("newly_inserted", True)) + else "exact_existing" + ), + record, + ) + for payload, record in zip(records, parsed, strict=True) + ] + + def append_batch( + self, + entries: list[tuple[PendingShare, dict[str, Any] | None]], + ) -> list[AcceptedShareRecord]: + return [ + outcome.record + for outcome in self._append_batch_with_replay_outcomes(entries) + ] + + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + """Use the exact batch comparator for one typed recovery outcome.""" + outcomes = self._append_batch_with_replay_outcomes([(pending, None)]) + if len(outcomes) != 1: + raise RuntimeError("Postgres recovery append returned an incomplete result") + return outcomes[0] def persist_block_candidate_intent(self, candidate: dict[str, Any]) -> bool: """Persist candidate work that is not yet eligible for share credit.""" @@ -5980,7 +6061,7 @@ def pool_block_state(self, *, block_hash: str) -> dict[str, object] | None: return state def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object]]: - sql = f""" + sql = """ SELECT COALESCE(json_agg(json_build_object( 'block_hash', block_hash, 'block_height', block_height, diff --git a/lab/prism/share_writer.py b/lab/prism/share_writer.py new file mode 100644 index 0000000..ee7714e --- /dev/null +++ b/lab/prism/share_writer.py @@ -0,0 +1,1080 @@ +"""Durable accepted-share writer and legacy recovery journal. + +This module owns the mutable share-persistence boundary. It deliberately has +no dependency on :mod:`lab.prism.prism_coordinator`; the coordinator remains a +construction root and temporary compatibility facade. +""" + +from __future__ import annotations + +from contextlib import AbstractContextManager +import dataclasses +from dataclasses import dataclass, field +import json +import os +from pathlib import Path +import queue +import threading +from typing import Any, Callable, Protocol + +from lab.prism.coordinator_shutdown import ShutdownInProgress, _WriterOperationToken +from lab.prism.share_ledger import ( + PendingShare, + ShareReplayConflict, + ShareReplayResult, +) + + +MAX_PENDING_SHARE_APPENDS = 4_096 +PENDING_SHARE_COMMIT_WARN_SECONDS = 30.0 +_STARTUP_RECOVERY_WAIT_POLL_SECONDS = 0.01 +_WRITER_EXIT_COMPONENTS = frozenset( + {"share_submission", "share_persistence", "accepted_block_handling"} +) + + +class ShareWriterError(RuntimeError): + """A persistence operation failed before it could be acknowledged.""" + + +class ShareWriterQueueFull(ShareWriterError): + """The bounded share group-commit queue rejected an entry.""" + + +class ShareLedgerPort(Protocol): + def append(self, pending: PendingShare) -> Any: ... + + def append_batch( + self, + entries: list[tuple[PendingShare, dict[str, Any] | None]], + ) -> list[Any]: ... + + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: ... + + +@dataclass(frozen=True) +class ShareWriterPorts: + """Narrow, call-time capabilities used by :class:`ShareWriter`.""" + + ledger: Callable[[], ShareLedgerPort] + writer_operation: Callable[[str], AbstractContextManager[object]] + reserve_writer: Callable[[str], _WriterOperationToken] + writer_admission_closed: Callable[[], bool] + has_active_writer: Callable[[set[str]], bool] + heartbeat: Callable[[str], None] + monotonic: Callable[[], float] + wall_time_ms: Callable[[], int] + stop_is_set: Callable[[], bool] + stop_wait: Callable[[float], bool] + log: Callable[[str], None] + log_exception: Callable[[], None] + hot_path_log_enabled: Callable[[], bool] + + +@dataclass +class ShareWriterConfig: + batch_size: int = 64 + linger_seconds: float = 0.005 + enqueue_timeout_seconds: float = 15.0 + pending_floor_warn_seconds: float = PENDING_SHARE_COMMIT_WARN_SECONDS + recovery_path: Path | None = None + + +@dataclass(frozen=True) +class PendingShareInput: + share_id: str + miner_id: str + order_key: str + p2mr_program_hex: str + share_difficulty: int + network_difficulty: int + template_height: int + job_id: str + job_issued_at_ms: int + ntime: int + credit_policy: str | None = None + + +@dataclass +class PendingShareAppend: + """A share waiting for the ledger group-commit writer. + + The client thread does not count or acknowledge this share until + ``committed`` is set successfully. A block candidate intent, when + present, is inserted in the same transaction as the share. + """ + + pending_share: PendingShare + username: str + job_id: str + block_hash_hex: str + collection_only: bool + credit_policy: str | None + candidate_intent: dict[str, Any] | None = None + committed: threading.Event = field(default_factory=threading.Event) + record: Any | None = None + error: BaseException | None = None + writer_token: _WriterOperationToken | None = None + + +@dataclass(frozen=True) +class ShareWriterMetricsSnapshot: + queue_depth: int + active: bool + append_failures: int + recovered_to_disk: int + replayed: int + replay_exact_existing: int + replay_conflicts: int + + +@dataclass +class _PendingFloorHolder: + pending: PendingShare + registered_monotonic: float + anchor_ms: int + warned: bool = False + + +class ShareWriter: + """Own group commit, the pending snapshot floor, and legacy recovery.""" + + def __init__( + self, + config: ShareWriterConfig, + ports: ShareWriterPorts, + *, + append_queue: queue.Queue[PendingShareAppend] | None = None, + floor_lock: threading.Lock | threading.RLock | None = None, + floor: dict[object, list[object]] | None = None, + recovery_lock: threading.Lock | threading.RLock | None = None, + active: bool = False, + append_failures: int = 0, + recovered_to_disk: int = 0, + replayed: int = 0, + ): + self.config = config + self.ports = ports + self._queue = append_queue or queue.Queue(maxsize=MAX_PENDING_SHARE_APPENDS) + self._floor_lock = floor_lock or threading.Lock() + self._floor = floor if floor is not None else {} + self._floor_anchors: dict[str, int] = { + str(entry[0].share_id): int(entry[0].accepted_at_ms) + for entry in self._floor.values() + if entry and hasattr(entry[0], "share_id") + } + # A stamped submission, durable credit-bearing outbox, and active + # candidate actor are independent reasons to hold the same logical + # snapshot floor. Attempts and actors use object identity; durable + # candidates use share_id so restart reconstruction reaches the holder. + self._attempt_holders: dict[int, _PendingFloorHolder] = {} + self._candidate_holders: dict[str, _PendingFloorHolder] = {} + self._candidate_actor_holders: dict[int, _PendingFloorHolder] = {} + self._recovery_lock = recovery_lock or threading.Lock() + self._state_lock = threading.Lock() + self._active = bool(active) + self._running = False + self._recovery_started = False + self._startup_recovery_complete = threading.Event() + self._startup_recovery_complete.set() + self._startup_recovery_cancelled = threading.Event() + self._append_failures = int(append_failures) + self._recovered_to_disk = int(recovered_to_disk) + self._replayed = int(replayed) + self._replay_exact_existing = 0 + self._replay_conflicts = 0 + + # Compatibility state is adopted by identity. Replacement while the loop + # is live would split one logical queue/floor across two owners, so reject + # it rather than silently stranding work. + @property + def append_queue(self) -> queue.Queue[PendingShareAppend]: + return self._queue + + def adopt_queue(self, value: queue.Queue[PendingShareAppend]) -> None: + with self._state_lock: + if self._running and value is not self._queue: + raise RuntimeError("cannot replace the share queue while the writer is running") + self._queue = value + + @property + def floor_lock(self) -> threading.Lock | threading.RLock: + return self._floor_lock + + @property + def floor(self) -> dict[object, list[object]]: + return self._floor + + def adopt_floor_lock(self, value: threading.Lock | threading.RLock) -> None: + with self._state_lock: + if self._running and value is not self._floor_lock: + raise RuntimeError("cannot replace the pending-share floor lock while running") + self._floor_lock = value + + def adopt_floor(self, value: dict[object, list[object]]) -> None: + with self._state_lock: + if self._running and value is not self._floor: + raise RuntimeError("cannot replace the pending-share floor while running") + self._floor = value + self._floor_anchors = { + str(entry[0].share_id): int(entry[0].accepted_at_ms) + for entry in value.values() + if entry and hasattr(entry[0], "share_id") + } + self._attempt_holders = {} + self._candidate_holders = {} + self._candidate_actor_holders = {} + + @property + def recovery_lock(self) -> threading.Lock | threading.RLock: + with self._state_lock: + return self._recovery_lock + + @property + def recovery_path(self) -> Path | None: + with self._state_lock: + return self.config.recovery_path + + def adopt_recovery_lock(self, value: threading.Lock | threading.RLock) -> None: + with self._state_lock: + if self._recovery_started and value is not self._recovery_lock: + raise RuntimeError("cannot replace the share recovery lock after replay starts") + self._recovery_lock = value + + def set_recovery_path(self, value: Path | None) -> None: + with self._state_lock: + if self._recovery_started and value != self.config.recovery_path: + raise RuntimeError("cannot replace the share recovery path after replay starts") + self.config.recovery_path = value + + @property + def active(self) -> bool: + with self._state_lock: + return self._active + + @active.setter + def active(self, value: bool) -> None: + with self._state_lock: + self._active = bool(value) + + @property + def append_failures(self) -> int: + with self._state_lock: + return self._append_failures + + @append_failures.setter + def append_failures(self, value: int) -> None: + with self._state_lock: + self._append_failures = int(value) + + @property + def recovered_to_disk(self) -> int: + with self._state_lock: + return self._recovered_to_disk + + @recovered_to_disk.setter + def recovered_to_disk(self, value: int) -> None: + with self._state_lock: + self._recovered_to_disk = int(value) + + @property + def replayed(self) -> int: + with self._state_lock: + return self._replayed + + @replayed.setter + def replayed(self, value: int) -> None: + with self._state_lock: + self._replayed = int(value) + + def metrics_snapshot(self) -> ShareWriterMetricsSnapshot: + with self._state_lock: + return ShareWriterMetricsSnapshot( + queue_depth=self._queue.qsize(), + active=self._active, + append_failures=self._append_failures, + recovered_to_disk=self._recovered_to_disk, + replayed=self._replayed, + replay_exact_existing=self._replay_exact_existing, + replay_conflicts=self._replay_conflicts, + ) + + def begin_startup_recovery(self) -> None: + """Fence contingent candidate credit behind legacy ACK recovery.""" + # A new epoch must clear the prior outcome before it exposes a closed + # gate. Startup calls this before the candidate submitter is started. + self._startup_recovery_cancelled.clear() + self._startup_recovery_complete.clear() + + def finish_startup_recovery(self) -> None: + """Open a successfully completed recovery epoch.""" + self._startup_recovery_complete.set() + + def cancel_startup_recovery(self) -> None: + """Abort gated candidate credit before shutdown waits for writers.""" + # Publish cancellation first: every waiter woken by the open gate, and + # every later caller that observes it open, must abort without append. + self._startup_recovery_cancelled.set() + self._startup_recovery_complete.set() + + def _wait_for_startup_recovery(self) -> None: + """Wait for normal recovery completion or abort promptly on closure.""" + if self._startup_recovery_complete.is_set(): + # This cancellation check is the fast-path linearization point. + # cancel_startup_recovery publishes cancellation before opening + # the gate, so a cancellation interleaved with is_set() cannot be + # mistaken for an already-normally-open recovery epoch. + if self._startup_recovery_cancelled.is_set(): + raise ShutdownInProgress("PRISM startup share recovery was cancelled") + return + while True: + if ( + self._startup_recovery_cancelled.is_set() + or self.ports.stop_is_set() + or self.ports.writer_admission_closed() + ): + raise ShutdownInProgress( + "PRISM startup share recovery was interrupted by shutdown" + ) + if self._startup_recovery_complete.wait( + _STARTUP_RECOVERY_WAIT_POLL_SECONDS + ): + if ( + self._startup_recovery_cancelled.is_set() + or self.ports.stop_is_set() + or self.ports.writer_admission_closed() + ): + raise ShutdownInProgress( + "PRISM startup share recovery was interrupted by shutdown" + ) + return + + def make_pending_share(self, value: PendingShareInput) -> PendingShare: + """Stamp and register a pending share under one floor-lock hold.""" + with self._floor_lock: + pending = PendingShare( + share_id=value.share_id, + miner_id=value.miner_id, + order_key=value.order_key, + p2mr_program_hex=value.p2mr_program_hex, + share_difficulty=value.share_difficulty, + network_difficulty=value.network_difficulty, + template_height=value.template_height, + job_id=value.job_id, + job_issued_at_ms=value.job_issued_at_ms, + accepted_at_ms=self.ports.wall_time_ms(), + ntime=value.ntime, + credit_policy=value.credit_policy, + ) + self._migrate_legacy_holders_locked(str(pending.share_id)) + self._attempt_holders[id(pending)] = _PendingFloorHolder( + pending=pending, + registered_monotonic=self.ports.monotonic(), + anchor_ms=int(pending.accepted_at_ms), + ) + self._rebuild_floor_locked(str(pending.share_id), preferred=pending) + return pending + + def adopt_pending_share(self, pending: PendingShare) -> None: + """Atomically promote/register a durable candidate by stable share ID. + + If ``pending`` is a live stamped attempt, promotion removes that exact + attempt and acquires the durable holder in the same floor-lock hold. + Startup replay has no attempt holder and simply adds/rebinds the durable + holder. Same-ID retries retain the minimum stamp without letting a + failed newer attempt release the older durable source. + """ + with self._floor_lock: + self._adopt_pending_share_locked(pending) + + def _adopt_pending_share_locked(self, pending: PendingShare) -> None: + logical_key = str(pending.share_id) + self._migrate_legacy_holders_locked(logical_key) + promoted = self._attempt_holders.pop(id(pending), None) + self._ensure_candidate_holder_locked(pending, source=promoted) + self._rebuild_floor_locked(logical_key, preferred=pending) + + def _ensure_candidate_holder_locked( + self, + pending: PendingShare, + *, + source: _PendingFloorHolder | None = None, + ) -> None: + """Create/rebind the stable durable-outbox holder under the floor lock.""" + logical_key = str(pending.share_id) + existing = self._candidate_holders.get(logical_key) + if existing is None: + registered_monotonic = ( + source.registered_monotonic + if source is not None + else self.ports.monotonic() + ) + anchor_ms = min( + int(pending.accepted_at_ms), + source.anchor_ms if source is not None else int(pending.accepted_at_ms), + ) + self._candidate_holders[logical_key] = _PendingFloorHolder( + pending=pending, + registered_monotonic=registered_monotonic, + anchor_ms=anchor_ms, + warned=source.warned if source is not None else False, + ) + else: + existing.pending = pending + existing.anchor_ms = min( + existing.anchor_ms, + int(pending.accepted_at_ms), + source.anchor_ms if source is not None else int(pending.accepted_at_ms), + ) + if source is not None: + existing.registered_monotonic = min( + existing.registered_monotonic, + source.registered_monotonic, + ) + existing.warned = existing.warned or source.warned + + def begin_candidate_actor(self, pending: PendingShare) -> None: + """Acquire one active credit-candidate actor and stable retry holder. + + The exact stamped attempt moves to actor ownership atomically, so share + append cleanup cannot erase it before the actor decides whether credit + committed or a durable retry/terminal transition now owns the floor. + """ + with self._floor_lock: + logical_key = str(pending.share_id) + self._migrate_legacy_holders_locked(logical_key) + promoted = self._attempt_holders.pop(id(pending), None) + actor = self._candidate_actor_holders.get(id(pending)) + if actor is None: + actor = promoted or _PendingFloorHolder( + pending=pending, + registered_monotonic=self.ports.monotonic(), + anchor_ms=int(pending.accepted_at_ms), + ) + self._candidate_actor_holders[id(pending)] = actor + else: + actor.pending = pending + actor.anchor_ms = min(actor.anchor_ms, int(pending.accepted_at_ms)) + if promoted is not None: + actor.anchor_ms = min(actor.anchor_ms, promoted.anchor_ms) + actor.registered_monotonic = min( + actor.registered_monotonic, + promoted.registered_monotonic, + ) + actor.warned = actor.warned or promoted.warned + self._ensure_candidate_holder_locked(pending, source=actor) + self._rebuild_floor_locked(logical_key, preferred=pending) + + def finish_candidate_actor(self, pending: PendingShare) -> None: + """Release only this active candidate object's floor authority.""" + with self._floor_lock: + logical_key = str(getattr(pending, "share_id", "")) + self._migrate_legacy_holders_locked(logical_key) + self._candidate_actor_holders.pop(id(pending), None) + self._rebuild_floor_locked(logical_key) + + def _migrate_legacy_holders_locked(self, logical_key: str) -> None: + """Adopt a directly inserted compatibility floor entry as an attempt.""" + represented_ids = set(self._attempt_holders) + candidate_holder = self._candidate_holders.get(logical_key) + if candidate_holder is not None: + represented_ids.add(id(candidate_holder.pending)) + represented_ids.update( + holder_id + for holder_id, holder in self._candidate_actor_holders.items() + if str(holder.pending.share_id) == logical_key + ) + for key, entry in list(self._floor.items()): + pending = entry[0] if entry else None + if ( + str(getattr(pending, "share_id", "")) != logical_key + or id(pending) in represented_ids + ): + continue + self._attempt_holders[id(pending)] = _PendingFloorHolder( + pending=pending, + registered_monotonic=float(entry[1]), + anchor_ms=int( + self._floor_anchors.get( + logical_key, + getattr(pending, "accepted_at_ms", 0), + ) + ), + warned=bool(entry[2]), + ) + represented_ids.add(id(pending)) + if key != logical_key: + migrated_entry = self._floor.pop(key) + self._floor.setdefault(logical_key, migrated_entry) + + def _holders_for_locked(self, logical_key: str) -> list[_PendingFloorHolder]: + holders = [ + holder + for holder in self._attempt_holders.values() + if str(holder.pending.share_id) == logical_key + ] + candidate_holder = self._candidate_holders.get(logical_key) + if candidate_holder is not None: + holders.append(candidate_holder) + holders.extend( + holder + for holder in self._candidate_actor_holders.values() + if str(holder.pending.share_id) == logical_key + ) + return holders + + def _rebuild_floor_locked( + self, + logical_key: str, + *, + preferred: PendingShare | None = None, + ) -> None: + holders = self._holders_for_locked(logical_key) + if not holders: + self._floor.pop(logical_key, None) + self._floor_anchors.pop(logical_key, None) + return + representative = None + if preferred is not None: + representative = next( + (holder for holder in holders if holder.pending is preferred), + None, + ) + if representative is None: + representative = self._candidate_holders.get(logical_key) or holders[-1] + values = [ + representative.pending, + min(holder.registered_monotonic for holder in holders), + any(holder.warned for holder in holders), + ] + existing = self._floor.get(logical_key) + if isinstance(existing, list) and len(existing) == 3: + existing[:] = values + else: + self._floor[logical_key] = values + self._floor_anchors[logical_key] = min(holder.anchor_ms for holder in holders) + + def finish_pending_attempt(self, pending: PendingShare) -> None: + """Release only one stamped submission attempt holder.""" + with self._floor_lock: + logical_key = str(getattr(pending, "share_id", "")) + self._migrate_legacy_holders_locked(logical_key) + self._attempt_holders.pop(id(pending), None) + self._floor.pop(id(pending), None) + self._rebuild_floor_locked(logical_key) + + def finish_pending_candidate(self, pending: PendingShare) -> None: + """Release only the durable credit-candidate holder for ``share_id``.""" + with self._floor_lock: + logical_key = str(getattr(pending, "share_id", "")) + self._migrate_legacy_holders_locked(logical_key) + self._candidate_holders.pop(logical_key, None) + self._rebuild_floor_locked(logical_key) + + def finish_pending_share(self, pending: PendingShare) -> None: + """Compatibility terminal: remove all holders for one durable identity. + + Product paths use the attempt, durable-candidate, or actor-specific + terminal so one owner cannot release another. + """ + with self._floor_lock: + share_id = getattr(pending, "share_id", None) + if share_id is None: + self._floor.pop(id(pending), None) + self._attempt_holders.pop(id(pending), None) + return + logical_key = str(share_id) + self._attempt_holders = { + holder_id: holder + for holder_id, holder in self._attempt_holders.items() + if str(holder.pending.share_id) != logical_key + } + self._candidate_holders.pop(logical_key, None) + self._candidate_actor_holders = { + holder_id: holder + for holder_id, holder in self._candidate_actor_holders.items() + if str(holder.pending.share_id) != logical_key + } + self._floor.pop(logical_key, None) + self._floor_anchors.pop(logical_key, None) + self._floor.pop(id(pending), None) + # Compatibility tests and old embeddings may have inserted an + # id-keyed entry for an earlier object with the same durable ID. + for key, entry in list(self._floor.items()): + candidate = entry[0] if entry else None + if str(getattr(candidate, "share_id", "")) == logical_key: + self._floor.pop(key, None) + + def transfer_pending_floor(self, old: PendingShare, new: PendingShare) -> None: + """Merge retry attempts without orphaning durable candidate leases. + + Attempts with one durable share ID become one logical lease and retain + the minimum acceptance stamp. Distinct parent/descendant IDs remain + independent because either outbox row may later credit its own share; + both can be released by a reconstructed object carrying that ID. + """ + if old is new: + return + with self._floor_lock: + old_key = str(old.share_id) + new_key = str(new.share_id) + if old_key != new_key: + # Parent/descendant candidates are separate durable outbox + # identities and may each still credit a share. Keep both + # stable leases; later reconstructed candidates release them + # by share ID even when neither Python object remains queued. + self._adopt_pending_share_locked(old) + self._adopt_pending_share_locked(new) + return + self._adopt_pending_share_locked(old) + self._adopt_pending_share_locked(new) + + def snapshot_anchor_ms(self, issued_at_ms: int) -> int: + stale_share_ids: list[str] = [] + floor_ms: int | None = None + current = self.ports.monotonic() + with self._floor_lock: + for entry in self._floor.values(): + pending = entry[0] + accepted_at_ms = self._floor_anchors.get( + str(pending.share_id), + int(pending.accepted_at_ms), + ) + floor_ms = ( + accepted_at_ms + if floor_ms is None + else min(floor_ms, accepted_at_ms) + ) + if ( + not bool(entry[2]) + and current - float(entry[1]) > self.config.pending_floor_warn_seconds + ): + entry[2] = True + logical_key = str(pending.share_id) + for holder in self._holders_for_locked(logical_key): + holder.warned = True + stale_share_ids.append(str(pending.share_id)) + for share_id in stale_share_ids: + self.ports.log( + "prism coordinator: pending share commit is holding the job " + f"snapshot anchor floor share_id={share_id}" + ) + return issued_at_ms if floor_ms is None else min(issued_at_ms, floor_ms - 1) + + def append_and_wait(self, entry: PendingShareAppend) -> Any: + """Persist an entry under admission with one lower terminal owner. + + In normal submit flow this operation inherits the already-admitted + ``share_submission`` token. A direct service caller instead owns a + fresh ``share_persistence`` admission. Once admission enters, enqueue + rollback, the queue-visible writer, or synchronous append owns attempt + cleanup. This wrapper cleans only refusal before that ownership handoff, + so an interrupted waiter cannot drop a still-queued floor. + """ + admission_entered = False + try: + # O1 starts the durable candidate submitter before legacy share + # replay. During that narrow startup window, candidate credit is + # contingent on landing while the journal contains shares already + # acknowledged by an older process. Serialize the two sources so + # scheduling cannot assign nondeterministic cross-source sequence. + self._wait_for_startup_recovery() + with self.ports.writer_operation("share_persistence"): + admission_entered = True + if self.active: + self.enqueue(entry, wait=True) + else: + self.append_entry(entry) + return entry.record + except BaseException: + if not admission_entered: + self.finish_pending_attempt(entry.pending_share) + raise + + def enqueue(self, entry: PendingShareAppend, *, wait: bool = False) -> None: + try: + if entry.writer_token is None: + entry.writer_token = self.ports.reserve_writer("share_persistence") + if wait: + self._queue.put(entry, timeout=self.config.enqueue_timeout_seconds) + else: + self._queue.put_nowait(entry) + except queue.Full as exc: + self._rollback_invisible_enqueue(entry) + raise ShareWriterQueueFull("share ledger commit queue is full") from exc + except BaseException: + self._rollback_invisible_enqueue(entry) + raise + if not wait: + return + entry.committed.wait() + if entry.error is not None: + raise ShareWriterError(f"share ledger commit failed: {entry.error}") + + def _rollback_invisible_enqueue(self, entry: PendingShareAppend) -> None: + """Release all ownership when an entry never became queue-visible.""" + if entry.writer_token is not None: + entry.writer_token.finish() + entry.writer_token = None + self.finish_pending_attempt(entry.pending_share) + + def run(self) -> None: + with self._state_lock: + if self._running: + raise RuntimeError("share writer loop is already running") + self._running = True + try: + while True: + self.ports.heartbeat("share_writer") + stopping = self.ports.stop_is_set() + try: + entry = self._queue.get(timeout=0.2 if stopping else 1.0) + except queue.Empty: + if ( + stopping + and self.ports.writer_admission_closed() + and not self.ports.has_active_writer(set(_WRITER_EXIT_COMPONENTS)) + ): + return + continue + batch = [entry] + batch_size = max(1, int(self.config.batch_size)) + deadline = self.ports.monotonic() + max( + 0.0, float(self.config.linger_seconds) + ) + if entry.candidate_intent is not None: + deadline = self.ports.monotonic() + while len(batch) < batch_size: + remaining = deadline - self.ports.monotonic() + if remaining <= 0: + break + try: + next_entry = self._queue.get(timeout=remaining) + except queue.Empty: + break + batch.append(next_entry) + if next_entry.candidate_intent is not None: + break + self.append_batch(batch) + finally: + with self._state_lock: + self._running = False + + def append_batch(self, batch: list[PendingShareAppend]) -> bool: + """Commit one writer batch and release every waiting submitter.""" + try: + ledger = self.ports.ledger() + append_batch = getattr(ledger, "append_batch", None) + if callable(append_batch): + records = append_batch( + [(entry.pending_share, entry.candidate_intent) for entry in batch] + ) + else: + records = [ledger.append(entry.pending_share) for entry in batch] + if len(records) != len(batch): + raise RuntimeError("share ledger returned an incomplete commit batch") + hot_path_log = self.ports.hot_path_log_enabled() + for entry, record in zip(batch, records, strict=True): + entry.record = record + if hot_path_log: + self._log_committed(entry, record) + return True + except Exception as exc: + with self._state_lock: + self._append_failures += len(batch) + for entry in batch: + entry.error = exc + self.ports.log( + f"prism coordinator: share ledger group commit failed count={len(batch)}" + ) + self.ports.log_exception() + return False + finally: + for entry in batch: + self.finish_pending_attempt(entry.pending_share) + entry.committed.set() + if entry.writer_token is not None: + entry.writer_token.finish() + entry.writer_token = None + + def append_entry( + self, + entry: PendingShareAppend, + *, + retry_until_stopped: bool = False, + ) -> bool: + """Synchronously append one accepted share, preserving retry order.""" + backoff_seconds = 0.5 + try: + while True: + try: + ledger = self.ports.ledger() + append_batch = getattr(ledger, "append_batch", None) + if callable(append_batch): + record = append_batch( + [(entry.pending_share, entry.candidate_intent)] + )[0] + else: + record = ledger.append(entry.pending_share) + entry.record = record + break + except Exception: + if not retry_until_stopped: + raise + with self._state_lock: + self._append_failures += 1 + self.ports.log( + "prism coordinator: ledger share append failed; retrying " + f"share_id={entry.pending_share.share_id}" + ) + self.ports.log_exception() + if self.ports.stop_wait(backoff_seconds): + self.recover_to_disk( + entry, + "ledger unavailable at shutdown", + ) + return False + backoff_seconds = min(backoff_seconds * 2, 5.0) + self.ports.heartbeat("share_writer") + if self.ports.hot_path_log_enabled(): + self._log_committed(entry, record) + entry.committed.set() + return True + finally: + self.finish_pending_attempt(entry.pending_share) + + def _log_committed(self, entry: PendingShareAppend, record: Any) -> None: + self.ports.log( + "prism coordinator: accepted share " + f"seq={record.share_seq} miner={entry.username} job={entry.job_id} " + f"hash={entry.block_hash_hex} collection={entry.collection_only} " + f"credit_policy={entry.credit_policy or 'normal'}" + ) + + def recover_to_disk(self, entry: PendingShareAppend, reason: str) -> None: + with self._state_lock: + self._recovery_started = True + path = self.config.recovery_path + recovery_lock = self._recovery_lock + if path is None: + self.ports.log( + "prism coordinator: WOULD LOSE acked share (no recovery path) " + f"share_id={entry.pending_share.share_id} reason={reason}" + ) + return + try: + payload = json.dumps( + dataclasses.asdict(entry.pending_share), + separators=(",", ":"), + ) + except Exception: + payload = None + with recovery_lock: + try: + path.parent.mkdir(parents=True, exist_ok=True) + if payload is None: + raise ValueError("pending share is not serializable") + with open(path, "a", encoding="utf-8") as handle: + handle.write(payload + "\n") + handle.flush() + os.fsync(handle.fileno()) + with self._state_lock: + self._recovered_to_disk += 1 + self.ports.log( + "prism coordinator: recovered unpersisted acked share to disk " + f"share_id={entry.pending_share.share_id} reason={reason}" + ) + except Exception: + self.ports.log( + "prism coordinator: FAILED to recover acked share to disk; " + f"share may be lost share_id={entry.pending_share.share_id} " + f"reason={reason}" + ) + self.ports.log_exception() + + def replay_recovery_file(self) -> int: + """Replay the recovery journal with typed exact/conflict outcomes.""" + with self.ports.writer_operation("share_recovery_replay"): + return self._replay_recovery_file_admitted() + + def _replay_recovery_file_admitted(self) -> int: + with self._state_lock: + self._recovery_started = True + path = self.config.recovery_path + if path is None: + return 0 + with self._recovery_lock: + if not path.exists(): + return 0 + try: + lines = [ + line + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + except Exception: + self.ports.log("prism coordinator: could not read share recovery file") + self.ports.log_exception() + return 0 + pendings: list[PendingShare] = [] + parse_failed = False + for line in lines: + try: + pendings.append(PendingShare(**json.loads(line))) + except Exception: + parse_failed = True + self.ports.log( + "prism coordinator: skipping an unparseable recovered share line" + ) + self.ports.log_exception() + pendings.sort(key=lambda pending: pending.accepted_at_ms) + replayed = 0 + exact_existing = 0 + conflict = False + for pending in pendings: + ledger = self.ports.ledger() + append_recovered = getattr(ledger, "append_recovered_share", None) + if not callable(append_recovered): + self.ports.log( + "prism coordinator: recovery ledger lacks typed replay support; " + "keeping the file" + ) + break + try: + result = append_recovered(pending) + except ShareReplayConflict: + conflict = True + with self._state_lock: + self._replay_conflicts += 1 + self.ports.log( + "prism coordinator: recovered share conflicts with durable " + f"payload; keeping the file share_id={pending.share_id}" + ) + break + except Exception: + self.ports.log( + "prism coordinator: failed to replay a recovered share; " + "keeping the file" + ) + self.ports.log_exception() + break + disposition = getattr(result, "disposition", None) + if disposition == "inserted": + replayed += 1 + elif disposition == "exact_existing": + exact_existing += 1 + else: + self.ports.log( + "prism coordinator: recovery ledger returned an unsupported " + f"replay disposition {disposition!r}; keeping the file" + ) + break + completed = replayed + exact_existing == len(pendings) + with self._state_lock: + self._replayed += replayed + self._replay_exact_existing += exact_existing + if exact_existing: + self.ports.log( + f"prism coordinator: skipped {exact_existing} already-committed " + "recovered share(s) during replay" + ) + if replayed: + self.ports.log( + f"prism coordinator: replayed {replayed} recovered share(s) " + "into the ledger" + ) + if not parse_failed and not conflict and completed: + try: + path.unlink() + except FileNotFoundError: + pass + return replayed + + +class ShareWriterCompatibilityField: + """Descriptor routing a temporary coordinator attribute to S3 ownership.""" + + def __init__(self, name: str, default: Any): + self.name = name + self.default = default + + def __get__(self, instance: Any, owner: type[Any]) -> Any: + if instance is None: + return self + service = instance.__dict__.get("_share_writer_service") + if service is None: + value = instance.__dict__.get(self.name, self.default) + if callable(value) and getattr(value, "__share_writer_default_factory__", False): + value = value() + instance.__dict__[self.name] = value + return value + return _compat_get(service, self.name) + + def __set__(self, instance: Any, value: Any) -> None: + service = instance.__dict__.get("_share_writer_service") + if service is None: + instance.__dict__[self.name] = value + return + _compat_set(service, self.name, value) + + +def compatibility_default(factory: Callable[[], Any]) -> Callable[[], Any]: + setattr(factory, "__share_writer_default_factory__", True) + return factory + + +def _compat_get(service: ShareWriter, name: str) -> Any: + if name == "share_append_queue": + return service.append_queue + if name == "share_commit_batch_size": + return service.config.batch_size + if name == "share_commit_linger_seconds": + return service.config.linger_seconds + if name == "share_commit_timeout_seconds": + return service.config.enqueue_timeout_seconds + if name == "share_writer_active": + return service.active + if name == "share_append_failure_count": + return service.append_failures + if name == "share_recovery_path": + return service.recovery_path + if name == "share_recovery_lock": + return service.recovery_lock + if name == "shares_recovered_to_disk": + return service.recovered_to_disk + if name == "shares_replayed": + return service.replayed + if name == "_pending_share_commit_lock": + return service.floor_lock + if name == "_pending_share_commit_floor": + return service.floor + raise AttributeError(name) + + +def _compat_set(service: ShareWriter, name: str, value: Any) -> None: + if name == "share_append_queue": + service.adopt_queue(value) + elif name == "share_commit_batch_size": + service.config.batch_size = int(value) + elif name == "share_commit_linger_seconds": + service.config.linger_seconds = float(value) + elif name == "share_commit_timeout_seconds": + service.config.enqueue_timeout_seconds = float(value) + elif name == "share_writer_active": + service.active = bool(value) + elif name == "share_append_failure_count": + service.append_failures = int(value) + elif name == "share_recovery_path": + service.set_recovery_path(Path(value) if value is not None else None) + elif name == "share_recovery_lock": + service.adopt_recovery_lock(value) + elif name == "shares_recovered_to_disk": + service.recovered_to_disk = int(value) + elif name == "shares_replayed": + service.replayed = int(value) + elif name == "_pending_share_commit_lock": + service.adopt_floor_lock(value) + elif name == "_pending_share_commit_floor": + service.adopt_floor(value) + else: + raise AttributeError(name) diff --git a/lab/prism/stratum_session.py b/lab/prism/stratum_session.py new file mode 100644 index 0000000..0accf3b --- /dev/null +++ b/lab/prism/stratum_session.py @@ -0,0 +1,1166 @@ +"""PRISM Stratum listener and session lifecycle. + +This module owns miner-facing connection admission, protocol dispatch, worker +resolution, and the live-session registry. It deliberately knows nothing +about :class:`PrismCoordinator`: construction-root adapters provide the few +job-delivery, progress-health, and runtime operations required by a session. +""" + +from __future__ import annotations + +from collections import OrderedDict +from contextlib import ExitStack +from dataclasses import dataclass, field +from decimal import Decimal +import errno +import json +import socket +import struct +import threading +import time +import traceback +from types import MappingProxyType +from typing import Callable, Mapping, Protocol + +from lab.auxpow import stratum_codec, vardiff +from lab.prism import direct_stratum +from lab.prism.coordinator_config import ( + StratumListenerProfile, + load_prism_highdiff_listener, # noqa: F401 - compatibility re-export +) + + +_VARDIFF_LOCK_INITIALIZATION_LOCK = threading.Lock() +from lab.prism.coordinator_shutdown import ShutdownInProgress + + +@dataclass(frozen=True) +class WorkerIdentity: + username: str + payout_address: str + worker_name: str | None + script_pubkey_hex: str + p2mr_program_hex: str + + +@dataclass(eq=False) +class ClientState: + sock: socket.socket + address: tuple[str, int] + connection_id: int + extranonce1_hex: str + subscribed: bool = False + authorized: bool = False + authorization_generation: int = 0 + difficulty_generation: int = 0 + authorized_monotonic: float | None = None + username: str = "" + worker: WorkerIdentity | None = None + version_mask: int = 0 + active_job: object | None = None + # Compatibility mirrors. SessionRegistry is authoritative for delivery + # proof, but these fields remain available throughout the staged split. + _progress_delivered_context: object | None = None + _progress_delivered_template_fingerprint: str | None = None + _progress_delivered_template_generation: int = 0 + _progress_delivered_payout_generation: int = -1 + _progress_delivered_monotonic: float | None = None + listener_name: str = "default" + listener_vardiff_config: vardiff.VardiffConfig | None = None + minimum_advertised_difficulty: Decimal = Decimal("0") + vardiff_config: vardiff.VardiffConfig | None = None + requested_difficulty: Decimal | None = None + requested_min_difficulty: Decimal | None = None + suggested_difficulty: Decimal | None = None + share_difficulty: Decimal = Decimal("1") + pending_share_difficulty: Decimal | None = None + vardiff_window_started_monotonic: float = field(default_factory=time.monotonic) + vardiff_window_accepted: int = 0 + vardiff_window_submitted: int = 0 + vardiff_window_work: Decimal = Decimal("0") + vardiff_difficulty_estimate: Decimal | None = None + # Serializes vardiff/request state without involving the coordinator's + # control-plane lock. Delivery paths take this before publication locks. + vardiff_lock: threading.RLock = field(default_factory=threading.RLock) + active_job_ids: set[str] = field(default_factory=set) + post_accept_refresh_block: tuple[int, str] | None = None + tip_work_delivered: tuple[str, float] | None = None + closing: bool = False + job_update_lock: threading.RLock = field(default_factory=threading.RLock) + send_lock: threading.Lock = field(default_factory=threading.Lock) + handler_thread_registered: bool = False + + def send(self, payload: dict[str, object]) -> None: + data = json.dumps(payload).encode() + b"\n" + with self.send_lock: + self.sock.sendall(data) + + def send_batch(self, payloads: list[dict[str, object]]) -> None: + # Focused tests and embedders replace send with a recorder. Preserve + # that seam while normal sockets write a paired update atomically. + if "send" in self.__dict__: + for payload in payloads: + self.send(payload) + return + data = b"".join(json.dumps(payload).encode() + b"\n" for payload in payloads) + with self.send_lock: + self.sock.sendall(data) + + def close(self) -> None: + try: + self.sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + self.sock.close() + except OSError: + pass + + +def client_vardiff_lock(client: ClientState) -> threading.RLock: + """Return the per-client vardiff lock, including lightweight embedders.""" + lock = getattr(client, "vardiff_lock", None) + if lock is not None: + return lock + with _VARDIFF_LOCK_INITIALIZATION_LOCK: + lock = getattr(client, "vardiff_lock", None) + if lock is None: + lock = threading.RLock() + client.vardiff_lock = lock + return lock + + +class StratumError(RuntimeError): + def __init__( + self, + code: int, + message: str, + *, + reason: str | None = None, + disconnect: bool = False, + ): + super().__init__(message) + self.code = code + self.message = message + self.reason = reason + self.disconnect = disconnect + + +def parse_stratum_password_options(password: str) -> tuple[Decimal | None, Decimal | None]: + """Extract pool-side d=N / md=N options, ignoring unknown miner tokens.""" + requested: Decimal | None = None + requested_min: Decimal | None = None + for token in password.split(","): + key, separator, raw_value = token.strip().partition("=") + if not separator: + continue + key = key.strip().lower() + if key not in {"d", "md"}: + continue + try: + value = Decimal(raw_value.strip()) + except Exception: + continue + if not value.is_finite() or value <= 0: + continue + if key == "d": + requested = value + else: + requested_min = value + return requested, requested_min + + +def parse_worker_username(username: str) -> tuple[str, str | None]: + payout_address, worker_name = split_worker_username(username) + if not payout_address: + raise StratumError(20, "username base is empty") + return payout_address, worker_name + + +def split_worker_username(username: str) -> tuple[str, str | None]: + payout_address, separator, worker_name = username.partition(".") + return payout_address, worker_name if separator else None + + +def result_payload(request_id: object, result: object) -> dict[str, object]: + return {"id": request_id, "result": result, "error": None} + + +def error_payload( + request_id: object, + code: int, + message: str, + *, + reason: str | None = None, +) -> dict[str, object]: + data = {"reason_id": reason} if reason is not None else None + return {"id": request_id, "result": None, "error": [code, message, data]} + + +def difficulty_payload(difficulty: Decimal) -> dict[str, object]: + return { + "id": None, + "method": "mining.set_difficulty", + "params": [float(difficulty)], + } + + +def job_payload(job: direct_stratum.DirectQbitStratumJob) -> dict[str, object]: + return { + "id": None, + "method": "mining.notify", + "params": [ + job.job_id, + job.prevhash, + job.coinb1, + job.coinb2, + list(job.merkle_branch), + job.version, + job.nbits, + job.ntime, + job.clean_jobs, + ], + } + + +def client_can_receive_jobs(client: ClientState) -> bool: + return ( + not getattr(client, "closing", False) + and client.subscribed + and client.authorized + and client.worker is not None + ) + + +def stratum_accept_heartbeat_names( + profiles: list[StratumListenerProfile] | tuple[StratumListenerProfile, ...] | None, +) -> tuple[str, ...]: + if not profiles: + return ("stratum_accept",) + return tuple(profile.heartbeat_name for profile in profiles) + + +@dataclass(frozen=True) +class DeliveredSessionContext: + context: object + delivered_monotonic: float + + +@dataclass(frozen=True) +class EligibleSession: + connection_id: int + delivered: DeliveredSessionContext | None + + +class SessionRegistry: + """Atomic owner of live membership and connection-scoped session facts.""" + + def __init__( + self, + *, + lock: threading.RLock, + clients: object | None = None, + connection_generation: int = 0, + rejection_counts: dict[str, int] | None = None, + ) -> None: + self.lock = lock + self.clients = clients if clients is not None else set() + self.connection_generation = max( + int(connection_generation), + max( + (int(client.connection_id) for client in self.clients), # type: ignore[union-attr] + default=0, + ), + ) + self.rejection_counts = ( + rejection_counts + if rejection_counts is not None + else {"global": 0, "username": 0} + ) + self.peak_active_connections = len(self.clients) + self.handler_thread_count = sum( + int(getattr(client, "handler_thread_registered", False)) + for client in self.clients + ) + self._delivered_by_connection: dict[int, DeliveredSessionContext] = {} + + def adopt_clients(self, clients: object) -> None: + """Adopt a compatibility replacement without changing its order/type.""" + with self.lock: + self.clients = clients + live_ids = {client.connection_id for client in clients} # type: ignore[union-attr] + self._delivered_by_connection = { + connection_id: delivered + for connection_id, delivered in self._delivered_by_connection.items() + if connection_id in live_ids + } + self.peak_active_connections = max( + self.peak_active_connections, + len(clients), # type: ignore[arg-type] + ) + self.connection_generation = max( + self.connection_generation, + max( + (int(client.connection_id) for client in clients), # type: ignore[union-attr] + default=0, + ), + ) + self.handler_thread_count = sum( + int(getattr(client, "handler_thread_registered", False)) + for client in clients # type: ignore[union-attr] + ) + + def _add_client_locked(self, client: ClientState) -> None: + add = getattr(self.clients, "add", None) + if callable(add): + add(client) + return + append = getattr(self.clients, "append", None) + if callable(append): + append(client) + return + raise TypeError("session membership must support add() or append()") + + def _discard_client_locked(self, client: ClientState) -> None: + discard = getattr(self.clients, "discard", None) + if callable(discard): + discard(client) + return + remove = getattr(self.clients, "remove", None) + if callable(remove): + try: + remove(client) + except ValueError: + pass + return + raise TypeError("session membership must support discard() or remove()") + + def _note_rejection_locked(self, scope: str) -> int: + count = int(self.rejection_counts.get(scope, 0)) + 1 + self.rejection_counts[scope] = count + return count + + def admit( + self, + *, + sock: socket.socket, + address: tuple[str, int], + profile: StratumListenerProfile, + share_difficulty: Decimal, + max_connections: int, + ) -> tuple[ClientState | None, int]: + with self.lock: + if max_connections > 0 and len(self.clients) >= max_connections: + return None, self._note_rejection_locked("global") + self.connection_generation += 1 + connection_id = self.connection_generation + client = ClientState( + sock=sock, + address=address, + connection_id=connection_id, + extranonce1_hex=f"{connection_id & 0xFFFFFFFF:08x}", + listener_name=profile.name, + listener_vardiff_config=profile.vardiff_config, + minimum_advertised_difficulty=profile.minimum_advertised_difficulty, + share_difficulty=share_difficulty, + ) + self._add_client_locked(client) + self.peak_active_connections = max( + self.peak_active_connections, len(self.clients) + ) + return client, 0 + + def reserve_username( + self, + client: ClientState, + worker: WorkerIdentity, + *, + max_connections_per_username: int, + ) -> tuple[bool, int]: + with self.lock: + active_for_username = sum( + 1 + for other in self.clients + if ( + other is not client + and other.worker is not None + and other.username == worker.username + ) + ) + if ( + max_connections_per_username > 0 + and active_for_username >= max_connections_per_username + ): + return False, self._note_rejection_locked("username") + # Commit the replacement only after capacity validation. A failed + # reauthorization therefore leaves the prior live identity intact. + client.worker = worker + client.username = worker.username + return True, 0 + + def register_handler(self, client: ClientState) -> None: + with self.lock: + if client.handler_thread_registered: + return + client.handler_thread_registered = True + self.handler_thread_count += 1 + + def unregister_handler(self, client: ClientState) -> None: + with self.lock: + if not client.handler_thread_registered: + return + client.handler_thread_registered = False + self.handler_thread_count = max(0, self.handler_thread_count - 1) + + def begin_retirement_locked(self, client: ClientState) -> bool: + if getattr(client, "closing", False) and client not in self.clients: + return False + client.closing = True + self._discard_client_locked(client) + self._delivered_by_connection.pop(client.connection_id, None) + return True + + def register_active_job_locked( + self, + client: ClientState, + context: object, + *, + job_id: str, + clean_jobs: bool, + ) -> tuple[str, ...]: + retired = tuple(client.active_job_ids) if clean_jobs else () + if clean_jobs: + client.active_job_ids.clear() + client.active_job = context + client.active_job_ids.add(job_id) + return retired + + def clear_active_jobs_locked(self, client: ClientState) -> tuple[str, ...]: + retired = tuple(client.active_job_ids) + client.active_job_ids.clear() + client.active_job = None + return retired + + def record_delivery( + self, + client: ClientState, + context: object, + delivered_monotonic: float, + ) -> bool: + with self.lock: + return self.record_delivery_locked( + client, + context, + delivered_monotonic, + ) + + def record_delivery_locked( + self, + client: ClientState, + context: object, + delivered_monotonic: float, + ) -> bool: + """Commit proof while the shared registry lock is already held.""" + if client not in self.clients or client.closing: + return False + delivered = DeliveredSessionContext(context, delivered_monotonic) + self._delivered_by_connection[client.connection_id] = delivered + # Compatibility mirrors for staged callers/tests. + client._progress_delivered_context = context + client._progress_delivered_monotonic = delivered_monotonic + return True + + def eligible_snapshot(self) -> Mapping[int, EligibleSession]: + """Return an immutable exact client_can_receive_jobs population.""" + with self.lock: + captured: dict[int, EligibleSession] = {} + for client in self.clients: + if not client_can_receive_jobs(client): + continue + delivered = self._delivered_by_connection.get(client.connection_id) + if delivered is None and client._progress_delivered_context is not None: + delivered = DeliveredSessionContext( + client._progress_delivered_context, + float(client._progress_delivered_monotonic or 0.0), + ) + captured[client.connection_id] = EligibleSession( + connection_id=client.connection_id, + delivered=delivered, + ) + return MappingProxyType(captured) + + +@dataclass +class _P2mrAddressValidationFlight: + event: threading.Event = field(default_factory=threading.Event) + result: tuple[str, str] | None = None + error: BaseException | None = None + waiters: int = 0 + + +class P2mrAddressValidator: + """Bounded LRU and singleflight wrapper around validateaddress RPC.""" + + def __init__( + self, + *, + rpc_call: Callable[[str, list[object]], object], + max_entries: Callable[[], int], + ttl_seconds: Callable[[], float], + cache_lock: threading.Lock | None = None, + cache: OrderedDict[str, tuple[float, tuple[str, str]]] | None = None, + inflight: dict[str, _P2mrAddressValidationFlight] | None = None, + ) -> None: + self.rpc_call = rpc_call + self.max_entries = max_entries + self.ttl_seconds = ttl_seconds + self.cache_lock = cache_lock if cache_lock is not None else threading.Lock() + self.cache = cache if cache is not None else OrderedDict() + self.inflight = inflight if inflight is not None else {} + + def validate(self, address: str, *, label: str) -> tuple[str, str]: + with self.cache_lock: + cached = self.cache.get(address) + if cached is not None: + expires_monotonic, cached_result = cached + if expires_monotonic > time.monotonic(): + self.cache.move_to_end(address) + return cached_result + self.cache.pop(address, None) + pending = self.inflight.get(address) + is_leader = pending is None + if pending is None: + pending = _P2mrAddressValidationFlight() + self.inflight[address] = pending + else: + pending.waiters += 1 + + if not is_leader: + pending.event.wait() + if pending.result is not None: + return pending.result + if pending.error is not None: + self._raise_shared_error(pending.error) + raise RuntimeError("payout address validation completed without a result") + + try: + validation = self.rpc_call("validateaddress", [address]) + if not isinstance(validation, dict) or not validation.get("isvalid"): + raise StratumError(20, f"{label} is not a valid qbit address: {address}") + script = str(validation.get("scriptPubKey") or "") + if not script.startswith("5220") or len(script) != 68: + raise StratumError(20, f"{label} does not resolve to a P2MR script: {address}") + result = (script, script[4:]) + with self.cache_lock: + max_entries = int(self.max_entries()) + ttl_seconds = float(self.ttl_seconds()) + if max_entries > 0 and ttl_seconds > 0: + self.cache[address] = (time.monotonic() + ttl_seconds, result) + self.cache.move_to_end(address) + while len(self.cache) > max_entries: + self.cache.popitem(last=False) + pending.result = result + return result + except BaseException as exc: + with self.cache_lock: + pending.error = exc + raise + finally: + with self.cache_lock: + if self.inflight.get(address) is pending: + self.inflight.pop(address, None) + pending.event.set() + + @staticmethod + def _raise_shared_error(error: BaseException) -> None: + if isinstance(error, StratumError): + raise StratumError( + error.code, + error.message, + reason=error.reason, + disconnect=error.disconnect, + ) from error + raise RuntimeError(str(error)) from error + + +class JobDeliveryPort(Protocol): + """Session-facing seam to job construction/delivery and submit handling.""" + + def note_collection_identity_available(self, client: ClientState) -> None: ... + def request_initial_job_delivery(self, client: ClientState) -> None: ... + def reauthorization_has_capacity(self, client: ClientState) -> bool: ... + def apply_client_difficulty_requests(self, client: ClientState) -> Decimal | None: ... + def advertise_client_difficulty(self, client: ClientState, target: Decimal) -> bool: ... + def handle_submit(self, client: ClientState, params: list[object]) -> bool: ... + def refresh_jobs_after_pending_accepted_block(self, client: ClientState) -> None: ... + def cancel_pending_initial_job_locked( + self, + client: ClientState, + ) -> Callable[[], object] | None: ... + def cleanup_disconnected_client(self, client: ClientState) -> None: ... + def retain_current_collection_refresh_if_unrepresented(self) -> None: ... + + +class ProgressHealthPort(Protocol): + """G1 integration; session code stores proof but owns no health policy.""" + + def record_delivery( + self, + client: ClientState, + context: object, + delivered_monotonic: float, + ) -> None: ... + + def reconcile_eligibility(self) -> None: ... + + +class SessionRuntimePort(Protocol): + def running(self) -> bool: ... + def record_heartbeat(self, name: str) -> None: ... + def wait_after_resource_failure(self, heartbeat_name: str) -> None: ... + def record_resource_exhaustion( + self, *, listener_name: str, location: str, error_number: int | None + ) -> None: ... + def record_setup_failure(self) -> int: ... + def sync_registry_metrics(self, registry: SessionRegistry) -> None: ... + def max_connections(self) -> int: ... + def max_connections_per_username(self) -> int: ... + def client_startup_difficulty(self, profile: StratumListenerProfile) -> Decimal: ... + def apply_send_timeout(self, sock: socket.socket) -> None: ... + def make_client_thread(self, client: ClientState) -> threading.Thread: ... + def extranonce2_size(self) -> int: ... + def version_mask(self) -> int: ... + def username_fallback_address(self) -> str | None: ... + def resolve_worker( + self, username: str, fallback: Callable[[], WorkerIdentity] + ) -> WorkerIdentity: ... + def reserve_client_username( + self, client: ClientState, worker: WorkerIdentity, fallback: Callable[[], bool] + ) -> bool: ... + def send_result( + self, client: ClientState, request_id: object, result: object + ) -> None: ... + def send_error( + self, + client: ClientState, + request_id: object, + code: int, + message: str, + *, + reason: str | None, + ) -> None: ... + def disconnect_client( + self, client: ClientState, fallback: Callable[[], None] + ) -> None: ... + + +class StratumSessionService: + def __init__( + self, + *, + registry: SessionRegistry, + runtime: SessionRuntimePort, + jobs: JobDeliveryPort, + progress: ProgressHealthPort, + address_validator: P2mrAddressValidator, + pool_closed_reason: str, + ) -> None: + self.registry = registry + self.runtime = runtime + self.jobs = jobs + self.progress = progress + self.address_validator = address_validator + self.pool_closed_reason = pool_closed_reason + + @staticmethod + def open_stratum_listeners( + listener_stack: ExitStack, + profiles: list[StratumListenerProfile], + *, + backlog: int, + retry_seconds: float, + stop_event: threading.Event | None, + socket_factory: Callable[[int, int], socket.socket] = socket.socket, + ) -> list[tuple[socket.socket, StratumListenerProfile]] | None: + listeners: list[tuple[socket.socket, StratumListenerProfile]] = [] + for profile in profiles: + server = listener_stack.enter_context( + socket_factory(socket.AF_INET, socket.SOCK_STREAM) + ) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + bind_deadline = time.monotonic() + retry_seconds + warned = False + while True: + try: + server.bind((profile.bind, profile.port)) + break + except OSError as exc: + if exc.errno != errno.EADDRINUSE or time.monotonic() >= bind_deadline: + raise + if stop_event is not None and stop_event.is_set(): + print( + "prism coordinator: shutdown requested while waiting " + f"to bind {profile.bind}:{profile.port}; aborting startup", + flush=True, + ) + return None + if not warned: + print( + f"prism coordinator: {profile.name} listener port " + f"{profile.bind}:{profile.port} is busy; retrying bind " + f"for up to {retry_seconds:g}s", + flush=True, + ) + warned = True + time.sleep(0.1) + server.listen(backlog) + server.settimeout(1) + listeners.append((server, profile)) + return listeners + + def accept_loop(self, server: socket.socket, profile: StratumListenerProfile) -> None: + while self.runtime.running(): + self.runtime.record_heartbeat(profile.heartbeat_name) + try: + sock, address = server.accept() + except socket.timeout: + continue + except OSError as exc: + if not self.runtime.running(): + return + if exc.errno in {errno.EMFILE, errno.ENFILE}: + self.runtime.record_resource_exhaustion( + listener_name=profile.name, + location="accept", + error_number=exc.errno, + ) + self.runtime.wait_after_resource_failure(profile.heartbeat_name) + continue + raise + + if not self.runtime.running(): + try: + sock.close() + except OSError: + pass + return + + # Recheck shutdown and membership admission in one registry + # critical section. Socket close remains outside the lock. + with self.registry.lock: + admission_open = self.runtime.running() + if admission_open: + client, rejection_count = self.registry.admit( + sock=sock, + address=address, + profile=profile, + share_difficulty=self.runtime.client_startup_difficulty( + profile + ), + max_connections=self.runtime.max_connections(), + ) + else: + client, rejection_count = None, 0 + if not admission_open: + try: + sock.close() + except OSError: + pass + return + self.runtime.sync_registry_metrics(self.registry) + if client is None: + try: + sock.close() + except OSError: + pass + if rejection_count == 1 or rejection_count % 100 == 0: + print( + "prism coordinator: rejected stratum connection at global limit " + f"limit={self.runtime.max_connections()} count={rejection_count}", + flush=True, + ) + continue + try: + sock.settimeout(None) + self.runtime.apply_send_timeout(sock) + thread = self.runtime.make_client_thread(client) + self.registry.register_handler(client) + self.runtime.sync_registry_metrics(self.registry) + thread.start() + except (OSError, RuntimeError) as exc: + self.registry.unregister_handler(client) + self.runtime.sync_registry_metrics(self.registry) + try: + self.disconnect_client(client) + except Exception: + print( + "prism coordinator: failed to fully close rejected stratum client " + f"address={address}", + flush=True, + ) + traceback.print_exc() + setup_failure_count = self.runtime.record_setup_failure() + if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: + self.runtime.record_resource_exhaustion( + listener_name=profile.name, + location="connection-setup", + error_number=exc.errno, + ) + if setup_failure_count == 1 or setup_failure_count % 100 == 0: + print( + "prism coordinator: stratum connection setup failed; backing off " + f"listener={profile.name} address={address} " + f"error={exc!r} count={setup_failure_count}", + flush=True, + ) + self.runtime.wait_after_resource_failure(profile.heartbeat_name) + + def reserve_client_username( + self, client: ClientState, worker: WorkerIdentity + ) -> bool: + accepted, rejection_count = self.registry.reserve_username( + client, + worker, + max_connections_per_username=self.runtime.max_connections_per_username(), + ) + self.runtime.sync_registry_metrics(self.registry) + if not accepted and (rejection_count == 1 or rejection_count % 100 == 0): + print( + "prism coordinator: rejected stratum authorization at username limit " + f"username={worker.username!r} " + f"limit={self.runtime.max_connections_per_username()} " + f"count={rejection_count}", + flush=True, + ) + return accepted + + def handle_client(self, client: ClientState) -> None: + reader = None + try: + reader = client.sock.makefile("r", encoding="utf-8", newline="\n") + for line in reader: + if not self.runtime.running(): + break + line = line.strip() + if not line: + continue + request_id: object = None + try: + request = json.loads(line) + if not isinstance(request, dict): + raise StratumError(20, "request must be an object") + request_id = request.get("id") + self.handle_request(client, request) + except json.JSONDecodeError as exc: + self._send_error(client, request_id, 20, f"invalid JSON: {exc.msg}") + except StratumError as exc: + self._send_error( + client, + request_id, + exc.code, + exc.message, + reason=exc.reason, + ) + if exc.disconnect: + break + except Exception: + print( + f"prism coordinator: client thread failed address={client.address}", + flush=True, + ) + traceback.print_exc() + break + except (OSError, ValueError) as exc: + if isinstance(exc, OSError) and exc.errno in {errno.EMFILE, errno.ENFILE}: + self.runtime.record_resource_exhaustion( + listener_name=client.listener_name, + location="client-reader", + error_number=exc.errno, + ) + print( + "prism coordinator: stratum client socket failed " + f"address={client.address} error={exc!r}", + flush=True, + ) + finally: + try: + if reader is not None: + reader.close() + except (OSError, ValueError): + pass + finally: + self.runtime.disconnect_client( + client, lambda: self.disconnect_client(client) + ) + self.registry.unregister_handler(client) + self.runtime.sync_registry_metrics(self.registry) + + def disconnect_client(self, client: ClientState) -> None: + cancel_pending: Callable[[], object] | None = None + with self.registry.lock: + if not self.registry.begin_retirement_locked(client): + return + cancel_pending = self.jobs.cancel_pending_initial_job_locked(client) + + if cancel_pending is not None: + cancel_pending() + + # Socket closure interrupts recv/send before waiting for job delivery. + try: + client.close() + finally: + with client.job_update_lock: + self.jobs.cleanup_disconnected_client(client) + self.jobs.retain_current_collection_refresh_if_unrepresented() + self.progress.reconcile_eligibility() + + def handle_request(self, client: ClientState, request: dict[str, object]) -> None: + try: + self._handle_request(client, request) + except ShutdownInProgress as exc: + raise StratumError( + 20, + "coordinator is shutting down", + reason=self.pool_closed_reason, + disconnect=True, + ) from exc + + def _handle_request(self, client: ClientState, request: dict[str, object]) -> None: + if not self.runtime.running(): + raise StratumError( + 20, + "coordinator is shutting down", + reason=self.pool_closed_reason, + disconnect=True, + ) + method = request.get("method") + params = request.get("params", []) + request_id = request.get("id") + if not isinstance(method, str): + raise StratumError(20, "missing method") + if not isinstance(params, list): + raise StratumError(20, "params must be an array") + + if method == "mining.configure": + self.handle_configure(client, request_id, params) + return + if method == "mining.subscribe": + with client.job_update_lock: + client.subscribed = True + self._send_result( + client, + request_id, + [[], client.extranonce1_hex, self.runtime.extranonce2_size()], + ) + self.jobs.note_collection_identity_available(client) + needs_initial_job = client.authorized + if needs_initial_job: + self.jobs.request_initial_job_delivery(client) + return + if method == "mining.authorize": + username = str(params[0]) if params else "" + password = str(params[1]) if len(params) > 1 and params[1] is not None else "" + # validateaddress RPC stays outside registry and job-update locks. + worker = self.runtime.resolve_worker( + username, lambda: self.resolve_worker(username) + ) + with client.job_update_lock: + was_authorized = client.authorized + if ( + was_authorized + and not self.jobs.reauthorization_has_capacity(client) + ): + raise StratumError( + 20, + "initial job delivery capacity unavailable", + disconnect=False, + ) + if not self.runtime.reserve_client_username( + client, + worker, + lambda: self.reserve_client_username(client, worker), + ): + raise StratumError( + 20, + "too many connections for username", + disconnect=not client.authorized, + ) + client.requested_difficulty, client.requested_min_difficulty = ( + parse_stratum_password_options(password) + ) + target = self.jobs.apply_client_difficulty_requests(client) + if target is not None: + current = client.pending_share_difficulty or client.share_difficulty + if target != current: + if not was_authorized: + client.share_difficulty = target + client.pending_share_difficulty = None + else: + client.pending_share_difficulty = target + client.difficulty_generation = int(client.difficulty_generation) + 1 + client.authorization_generation = int(client.authorization_generation) + 1 + client.authorized = True + client.authorized_monotonic = time.monotonic() + self._send_result(client, request_id, True) + self.jobs.note_collection_identity_available(client) + self.jobs.request_initial_job_delivery(client) + return + if method == "mining.extranonce.subscribe": + self._send_result(client, request_id, True) + return + if method == "mining.suggest_difficulty": + self.handle_suggest_difficulty(client, request_id, params) + return + if method == "mining.submit": + accepted_and_closed = self.jobs.handle_submit(client, params) + try: + self._send_result(client, request_id, True) + finally: + self.jobs.refresh_jobs_after_pending_accepted_block(client) + if accepted_and_closed: + client.close() + return + raise StratumError(20, f"unsupported method {method}") + + def handle_suggest_difficulty( + self, client: ClientState, request_id: object, params: list[object] + ) -> None: + with client.job_update_lock: + suggested: Decimal | None = None + if params: + try: + suggested = Decimal(str(params[0])) + except Exception: + suggested = None + if suggested is not None and ( + not suggested.is_finite() or suggested <= 0 + ): + suggested = None + if suggested is not None: + client.suggested_difficulty = suggested + target = self.jobs.apply_client_difficulty_requests(client) + if target is not None: + self.jobs.advertise_client_difficulty(client, target) + self._send_result(client, request_id, True) + + def handle_configure( + self, client: ClientState, request_id: object, params: list[object] + ) -> None: + extensions = params[0] if params else [] + extension_params = ( + params[1] if len(params) > 1 and isinstance(params[1], dict) else {} + ) + result: dict[str, object] = {} + if isinstance(extensions, list): + for extension in extensions: + if extension == "version-rolling": + miner_mask = 0xFFFFFFFF + if "version-rolling.mask" in extension_params: + miner_mask = stratum_codec.parse_mask_hex( + extension_params["version-rolling.mask"], + field_name="version-rolling.mask", + ) + client.version_mask = self.runtime.version_mask() & miner_mask + result["version-rolling"] = client.version_mask != 0 + result["version-rolling.mask"] = stratum_codec.format_mask_hex( + client.version_mask + ) + else: + result[str(extension)] = False + self._send_result(client, request_id, result) + + @staticmethod + def send_result(client: ClientState, request_id: object, result: object) -> None: + client.send(result_payload(request_id, result)) + + def _send_result( + self, client: ClientState, request_id: object, result: object + ) -> None: + self.runtime.send_result(client, request_id, result) + + @staticmethod + def send_error( + client: ClientState, + request_id: object, + code: int, + message: str, + *, + reason: str | None = None, + ) -> None: + client.send(error_payload(request_id, code, message, reason=reason)) + + def _send_error( + self, + client: ClientState, + request_id: object, + code: int, + message: str, + *, + reason: str | None = None, + ) -> None: + self.runtime.send_error( + client, + request_id, + code, + message, + reason=reason, + ) + + def resolve_worker(self, username: str) -> WorkerIdentity: + payout_address, worker_name = split_worker_username(username) + try: + if not payout_address: + raise StratumError(20, "username base is empty") + script, p2mr_program_hex = self.address_validator.validate( + payout_address, label="username base" + ) + except StratumError as username_error: + fallback_address = self.runtime.username_fallback_address() + if fallback_address is None: + raise username_error + print( + f"prism coordinator: username {username!r} cannot be used as a payout " + f"({username_error.message}); using fallback payout {fallback_address}", + flush=True, + ) + payout_address = fallback_address + script, p2mr_program_hex = self.address_validator.validate( + fallback_address, + label="PRISM_USERNAME_FALLBACK_ADDRESS", + ) + return WorkerIdentity( + username=username, + payout_address=payout_address, + worker_name=worker_name, + script_pubkey_hex=script, + p2mr_program_hex=p2mr_program_hex, + ) + + def record_successful_delivery( + self, + client: ClientState, + context: object, + delivered_monotonic: float, + ) -> None: + if not self.registry.record_delivery( + client, context, delivered_monotonic + ): + return + self.progress.record_delivery(client, context, delivered_monotonic) + self.progress.reconcile_eligibility() + + +def apply_stratum_send_timeout(sock: socket.socket, timeout_seconds: float) -> None: + """Apply a send-only timeout without changing blocking receive behavior.""" + if timeout_seconds <= 0: + return + seconds = int(timeout_seconds) + microseconds = int((timeout_seconds - seconds) * 1_000_000) + try: + sock.setsockopt( + socket.SOL_SOCKET, + socket.SO_SNDTIMEO, + struct.pack("ll", seconds, microseconds), + ) + except (AttributeError, OSError, struct.error): + return diff --git a/lab/prism/template_artifacts.py b/lab/prism/template_artifacts.py new file mode 100644 index 0000000..1c2f3a3 --- /dev/null +++ b/lab/prism/template_artifacts.py @@ -0,0 +1,455 @@ +"""Immutable qbit template observations and their derived artifacts. + +The repository owns template observation ordering and derivation caching. It +does not know about the coordinator: tip publication and refresh scheduling are +supplied through narrow callbacks so detection can remain separate from R1's +published share-validation authority. +""" + +from __future__ import annotations + +import copy +from contextlib import contextmanager +from dataclasses import dataclass, field +import hashlib +import json +import threading +import time +from typing import Any, Callable, Iterator, Mapping, Sequence + +from lab.prism import direct_stratum +from lab.prism.payout_state import ( + TemplateRefreshBlocked, + TemplateRefreshSuperseded, +) + + +PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS = frozenset( + { + "curtime", + "longpollid", + "mintime", + } +) + + +def qbit_template_fingerprint(template: dict[str, Any]) -> str: + stable_template = { + key: value + for key, value in template.items() + if key not in PRISM_TEMPLATE_FINGERPRINT_VOLATILE_KEYS + } + encoded = json.dumps( + stable_template, + sort_keys=True, + separators=(",", ":"), + default=str, + ).encode() + return hashlib.sha256(encoded).hexdigest() + + +class FrozenJsonDict(dict[str, Any]): + """JSON-compatible mapping that fails closed on mutation.""" + + __slots__ = () + + @staticmethod + def _immutable(*_args: object, **_kwargs: object) -> None: + raise TypeError("template artifact JSON is immutable") + + __setitem__ = _immutable + __delitem__ = _immutable + clear = _immutable + pop = _immutable + popitem = _immutable + setdefault = _immutable + update = _immutable + __ior__ = _immutable + + def __copy__(self) -> FrozenJsonDict: + return self + + def __deepcopy__(self, _memo: dict[int, object]) -> FrozenJsonDict: + return self + + +class FrozenJsonList(list[Any]): + """JSON-compatible sequence that fails closed on mutation.""" + + __slots__ = () + + @staticmethod + def _immutable(*_args: object, **_kwargs: object) -> None: + raise TypeError("template artifact JSON is immutable") + + __setitem__ = _immutable + __delitem__ = _immutable + __iadd__ = _immutable + __imul__ = _immutable + append = _immutable + clear = _immutable + extend = _immutable + insert = _immutable + pop = _immutable + remove = _immutable + reverse = _immutable + sort = _immutable + + def __copy__(self) -> FrozenJsonList: + return self + + def __deepcopy__(self, _memo: dict[int, object]) -> FrozenJsonList: + return self + + +def freeze_json(value: Any) -> Any: + """Detach and recursively freeze a JSON-like value without changing shape.""" + if isinstance(value, (FrozenJsonDict, FrozenJsonList)): + return value + if isinstance(value, Mapping): + frozen = FrozenJsonDict() + dict.update( + frozen, + ((str(key), freeze_json(item)) for key, item in value.items()), + ) + return frozen + if isinstance(value, (list, tuple)): + frozen = FrozenJsonList() + list.extend(frozen, (freeze_json(item) for item in value)) + return frozen + return value + + +def freeze_json_rows( + rows: Sequence[Mapping[str, Any]], +) -> FrozenJsonList: + frozen = freeze_json(rows) + if not isinstance(frozen, FrozenJsonList): + raise TypeError("JSON rows must be a sequence") + return frozen + + +@dataclass(frozen=True) +class CachedTemplateArtifacts: + """One exact template observation plus template-only derivations.""" + + template: dict[str, Any] + fingerprint: str + previousblockhash: str + transaction_hexes: tuple[str, ...] + witness_merkle_leaves_hex: tuple[str, ...] + network_difficulty: int + fetched_monotonic: float + generation: int = 0 + + def __post_init__(self) -> None: + frozen = freeze_json(self.template) + if not isinstance(frozen, dict): + raise TypeError("template artifact must be a JSON object") + object.__setattr__(self, "template", frozen) + + +@dataclass(frozen=True) +class QbitTipTemplateSnapshot: + bestblockhash: str + previousblockhash: str + template_fingerprint: str + template_generation: int = 0 + template_artifacts: CachedTemplateArtifacts | None = field( + default=None, + compare=False, + repr=False, + ) + + +@dataclass(frozen=True) +class TemplateArtifactPorts: + fetch_template: Callable[[], object] + fetch_bestblockhash: Callable[[], str] + newest_observed_tip: Callable[[], str | None] + observe_tip: Callable[[str], object] + schedule_refresh_retry: Callable[[], None] + pinned_issuance_artifacts: Callable[[], CachedTemplateArtifacts | None] + repinned_issuance_artifacts: Callable[ + [CachedTemplateArtifacts], CachedTemplateArtifacts | None + ] + record_tip: Callable[[str], object] | None = None + + +@dataclass(frozen=True) +class TemplateArtifactEventSink: + record_cache_event: Callable[[bool], None] + record_build_phase: Callable[[str, float], None] + artifacts_changed: Callable[[CachedTemplateArtifacts, bool], None] + artifacts_cleared: Callable[[CachedTemplateArtifacts], None] + + +class TemplateArtifactRepository: + """Sole owner of template artifact state and observation generations.""" + + def __init__( + self, + ports: TemplateArtifactPorts, + *, + cache_seconds: float, + scale_network_difficulty: Callable[[str], int], + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self._ports = ports + self._cache_seconds = float(cache_seconds) + self._scale_network_difficulty = scale_network_difficulty + self._monotonic = monotonic + self._lock = threading.Lock() + # Serialize accepting an observation with all side effects caused by + # that acceptance. The state lock is intentionally released around + # the event sink so it may query the repository, but a newer + # observation cannot become current until older effects finish. + self._publication_lock = threading.Lock() + self._current: CachedTemplateArtifacts | None = None + self._generation = 0 + self._event_sink: TemplateArtifactEventSink | None = None + + def bind_event_sink(self, event_sink: TemplateArtifactEventSink) -> None: + """Bind the owning service after both repository and service exist.""" + with self._lock: + if self._event_sink is not None: + raise RuntimeError("template artifact event sink is already bound") + self._event_sink = event_sink + + def _required_event_sink(self) -> TemplateArtifactEventSink: + with self._lock: + event_sink = self._event_sink + if event_sink is None: + raise RuntimeError("template artifact event sink is not bound") + return event_sink + + @contextmanager + def publication_admission(self) -> Iterator[None]: + """Fence cache admission against observation publication effects.""" + with self._publication_lock: + yield + + def reserve_generation(self) -> int: + """Reserve ordering when a fetch starts, not when it finishes.""" + with self._lock: + self._generation += 1 + return self._generation + + def derive( + self, + template: dict[str, Any], + *, + generation: int, + ) -> CachedTemplateArtifacts: + event_sink = self._required_event_sink() + template = copy.deepcopy(template) + fingerprint = qbit_template_fingerprint(template) + with self._lock: + previous = self._current + if previous is not None and previous.fingerprint == fingerprint: + return CachedTemplateArtifacts( + template=template, + fingerprint=fingerprint, + previousblockhash=str(template.get("previousblockhash", "")), + transaction_hexes=previous.transaction_hexes, + witness_merkle_leaves_hex=previous.witness_merkle_leaves_hex, + network_difficulty=previous.network_difficulty, + fetched_monotonic=self._monotonic(), + generation=generation, + ) + started = self._monotonic() + transaction_hexes = direct_stratum.transaction_hexes_from_template(template) + witness_leaves = tuple( + direct_stratum.witness_merkle_leaves_hex(transaction_hexes) + ) + network_difficulty = self._scale_network_difficulty(str(template["bits"])) + event_sink.record_build_phase( + "merkle", + self._monotonic() - started, + ) + return CachedTemplateArtifacts( + template=template, + fingerprint=fingerprint, + previousblockhash=str(template.get("previousblockhash", "")), + transaction_hexes=transaction_hexes, + witness_merkle_leaves_hex=witness_leaves, + network_difficulty=network_difficulty, + fetched_monotonic=self._monotonic(), + generation=generation, + ) + + def store_artifacts(self, artifacts: CachedTemplateArtifacts) -> bool: + event_sink = self._required_event_sink() + with self._publication_lock: + with self._lock: + previous = self._current + if ( + previous is not None + and artifacts.generation < previous.generation + ): + return False + self._current = artifacts + observation_changed = bool( + previous is not None + and ( + previous.generation != artifacts.generation + or previous.fingerprint != artifacts.fingerprint + ) + ) + fingerprint_changed = bool( + previous is not None + and previous.fingerprint != artifacts.fingerprint + ) + if observation_changed: + event_sink.artifacts_changed(artifacts, fingerprint_changed) + return True + + def store( + self, + template: dict[str, Any], + *, + generation: int | None = None, + ) -> CachedTemplateArtifacts | None: + if generation is None: + generation = self.reserve_generation() + try: + artifacts = self.derive(template, generation=generation) + except Exception: + return None + self.store_artifacts(artifacts) + return artifacts + + def current(self) -> CachedTemplateArtifacts: + event_sink = self._required_event_sink() + now = self._monotonic() + with self._lock: + cached = self._current + observed_tip = self._ports.newest_observed_tip() + cached_tip_current = ( + observed_tip is None + or cached is None + or cached.previousblockhash == observed_tip + ) + if ( + cached is not None + and cached_tip_current + and self._cache_seconds > 0 + and now - cached.fetched_monotonic <= self._cache_seconds + ): + event_sink.record_cache_event(True) + return cached + event_sink.record_cache_event(False) + generation = self.reserve_generation() + started = self._monotonic() + template = self._ports.fetch_template() + if not isinstance(template, dict): + raise RuntimeError("getblocktemplate returned non-object") + event_sink.record_build_phase("template", self._monotonic() - started) + artifacts = self.derive(template, generation=generation) + if self.store_artifacts(artifacts): + self._ports.observe_tip(artifacts.previousblockhash) + return artifacts + current = self.current_artifacts() + if current is None: + raise RuntimeError( + "newer template artifacts disappeared after cache race" + ) + self._ports.observe_tip(current.previousblockhash) + return current + + def issuance(self) -> CachedTemplateArtifacts: + pinned = self._ports.pinned_issuance_artifacts() + if pinned is not None: + return pinned + artifacts = self.current() + return self._ports.repinned_issuance_artifacts(artifacts) or artifacts + + def fetch_coherent_snapshot( + self, + observed_best_tip: str | None = None, + ) -> QbitTipTemplateSnapshot: + event_sink = self._required_event_sink() + if observed_best_tip is not None: + now = self._monotonic() + with self._lock: + cached = self._current + if ( + cached is not None + and self._cache_seconds > 0 + and now - cached.fetched_monotonic <= self._cache_seconds + and cached.previousblockhash == observed_best_tip + ): + event_sink.record_cache_event(True) + return QbitTipTemplateSnapshot( + bestblockhash=cached.previousblockhash, + previousblockhash=cached.previousblockhash, + template_fingerprint=cached.fingerprint, + template_generation=cached.generation, + template_artifacts=cached, + ) + event_sink.record_cache_event(False) + generation = self.reserve_generation() + template = self._ports.fetch_template() + if not isinstance(template, dict): + raise RuntimeError("getblocktemplate returned non-object") + previousblockhash = str(template.get("previousblockhash", "") or "") + if not previousblockhash: + raise RuntimeError("getblocktemplate omitted previousblockhash") + bestblockhash = str(self._ports.fetch_bestblockhash()) + if bestblockhash != previousblockhash: + record_tip = self._ports.record_tip or self._ports.observe_tip + record_tip(bestblockhash) + self._ports.schedule_refresh_retry() + raise TemplateRefreshSuperseded( + "qbit tip changed while fetching block template " + f"template_parent={previousblockhash} current={bestblockhash}" + ) + artifacts = self.store(template, generation=generation) + if artifacts is None: + raise TemplateRefreshBlocked( + "unable to derive exact artifacts for observed qbit template" + ) + return QbitTipTemplateSnapshot( + bestblockhash=bestblockhash, + previousblockhash=artifacts.previousblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + template_artifacts=artifacts, + ) + + def current_artifacts(self) -> CachedTemplateArtifacts | None: + with self._lock: + return self._current + + def is_current(self, artifacts: CachedTemplateArtifacts) -> bool: + with self._lock: + return self._current is artifacts + + def clear_if_current(self, artifacts: CachedTemplateArtifacts) -> bool: + event_sink = self._required_event_sink() + with self._publication_lock: + with self._lock: + if self._current is not artifacts: + return False + self._current = None + event_sink.artifacts_cleared(artifacts) + return True + + def replace_for_test( + self, + artifacts: CachedTemplateArtifacts | None, + ) -> None: + # Test-only state injection intentionally bypasses event-sink effects, + # but still participates in the production publication fence. + with self._publication_lock: + with self._lock: + self._current = artifacts + if artifacts is not None: + self._generation = max(self._generation, artifacts.generation) + + def set_cache_seconds_for_test(self, seconds: float) -> None: + self._cache_seconds = float(seconds) + + def generation_for_test(self) -> int: + with self._lock: + return self._generation diff --git a/lab/prism/tip_refresh.py b/lab/prism/tip_refresh.py new file mode 100644 index 0000000..b18e4d7 --- /dev/null +++ b/lab/prism/tip_refresh.py @@ -0,0 +1,4081 @@ +"""PRISM tip observation, publication, and bounded refresh ownership. + +The service deliberately has no dependency on :mod:`prism_coordinator`. +Construction wires narrow ports for qbit RPC, payout/job-bundle invalidation, +progress-health events, and the temporary client-delivery boundary that S2 +will replace. +""" + +from __future__ import annotations + +from concurrent.futures import FIRST_COMPLETED, Future, wait +from contextlib import contextmanager +from dataclasses import dataclass, field, replace as dataclass_replace +import random +import threading +import time +import traceback +from typing import Any, Callable, Mapping, Protocol + +from lab.prism.bounded_executor import _BoundedPriorityExecutor +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.job_bundle import CachedJobBundle, JobBuildKey, JobBuildSuperseded +from lab.prism.payout_state import ( + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES, + PayoutStatePublicationBlocked, + PayoutStateSnapshot, + TemplateRefreshBlocked, + TemplateRefreshSuperseded, +) +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + QbitTipTemplateSnapshot, + qbit_template_fingerprint, +) + + +PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS = 0.05 +PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION = 0.25 +PRISM_TIP_REFRESH_SECONDS_BUCKETS = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, +) +PRISM_TIP_REFRESH_BUILD_PHASES = ( + "ledger_snapshot", + "payout_state_derivation", + "ctv_manifest_construction", + "coinbase_bundle_construction", + "signing_verification", + "serialization_copy", + "singleflight_wait", +) +PRISM_TIP_REFRESH_RESULTS = ("sent", "skipped", "disconnected", "failed") +PRISM_TIP_REFRESH_CANCELLATION_STAGES = ( + "executor_queue", + "client_lock", + "payout_gate", +) +PRISM_TIP_REFRESH_TRIGGER_REASONS = ( + "blockpoll", + "blockwait", + "payout", + "post_accept", + "readiness", + "retained_collection", + "template", +) +PRISM_TIP_REFRESH_TRIGGER_PENDING_CAPACITY = 1 + +_UNSET = object() + + +class RefreshActivityPort(Protocol): + def note_activity(self, observed_monotonic: float | None = None) -> None: ... + + def finish(self) -> None: ... + + +class DeliveryAdmissionPort(Protocol): + def __bool__(self) -> bool: ... + + +class PayoutDeliveryGatePort(Protocol): + def delivery_cancelable( + self, + cancelled: Callable[[], bool], + *, + generation: int, + priority: bool = False, + ) -> Any: ... + + +class PayoutStatePort(Protocol): + @property + def delivery_gate(self) -> PayoutDeliveryGatePort: ... + + def snapshot(self) -> PayoutStateSnapshot: ... + + def reserve_source_for_tip_change( + self, + tip_hash: str, + *, + cause: str, + invalidated_monotonic: float, + ) -> int: ... + + +class JobBundlePort(Protocol): + def begin_priority_preparation( + self, + requested_monotonic: float | None = None, + ) -> tuple[int, float]: ... + + def finish_priority_preparation(self, token: int) -> None: ... + + def ready_latched(self) -> bool: ... + + def clear_prepared_ready(self) -> None: ... + + def record_failure(self) -> None: ... + + def pool_readiness_latched(self) -> bool: ... + + def shared_job_bundle( + self, + artifacts: CachedTemplateArtifacts, + worker: object | None = None, + *, + mode: str | None = None, + retry_superseded: bool = True, + publication_critical: bool = False, + request_source: str = "routine", + priority_requested_monotonic: float | None = None, + ) -> CachedJobBundle: ... + + def set_preparation_pending(self, pending: bool) -> None: ... + + def set_prepared_ready( + self, + snapshot: QbitTipTemplateSnapshot | None, + bundle: CachedJobBundle | None, + ) -> None: ... + + +@dataclass(frozen=True) +class RefreshClientTarget: + client: object = field(repr=False) + expected_active_job: object | None = field(default=None, repr=False) + + +class TipRefreshDeliveryPort(Protocol): + """Temporary R1-to-S2 boundary; S2 replaces this adapter outright.""" + + def eligible_clients(self) -> tuple[object, ...]: ... + + def client_can_receive_jobs(self, client: object) -> bool: ... + + def client_needs_refresh( + self, + client: object, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: ... + + def active_job(self, client: object) -> object | None: ... + + def connection_id(self, client: object) -> int: ... + + def delivery_priority( + self, + client: object, + snapshot: QbitTipTemplateSnapshot, + expected_active_job: object | None, + ) -> int: ... + + def submit_task( + self, + executor: object, + fn: Callable[..., RefreshResult], + *args: object, + priority: int, + ) -> Future[RefreshResult]: ... + + def send_prepared_job( + self, + client: object, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + validation_token: TipRefreshValidationToken, + expected_connection_id: int, + expected_active_job: object | None, + cancel_event: FanoutCancellation, + submitted_monotonic: float, + ) -> RefreshResult: ... + + def disconnect(self, client: object) -> None: ... + + def log_identity(self, client: object) -> str: ... + + def select_targets( + self, + snapshot: QbitTipTemplateSnapshot, + *, + refresh_all: bool, + ) -> tuple[RefreshClientTarget, ...]: ... + + def merge_poll_start_targets( + self, + targets: tuple[RefreshClientTarget, ...], + poll_start_clients: tuple[object, ...], + snapshot: QbitTipTemplateSnapshot, + *, + refresh_all: bool, + ) -> tuple[RefreshClientTarget, ...]: ... + + def revalidate_targets( + self, + targets: tuple[RefreshClientTarget, ...], + snapshot: QbitTipTemplateSnapshot, + ) -> tuple[tuple[RefreshClientTarget, ...], tuple[str, ...]]: ... + + def deliver_collection( + self, + client: object, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> RefreshResult: ... + + def take_post_accept_refresh( + self, + client: object, + ) -> tuple[int, str] | None: ... + + +@dataclass(frozen=True) +class TipRefreshConfig: + blockpoll_seconds: float + blockwait_timeout_seconds: float + failure_holdoff_seconds: float + max_workers: int + submit_tip_max_age_seconds: float + failure_exit_seconds: float + watchdog_timeout_seconds: float + payout_reconcile_supersession_retries: int = ( + DEFAULT_PRISM_PAYOUT_RECONCILE_SUPERSESSION_RETRIES + ) + + +@dataclass(frozen=True) +class PublishedTipSnapshot: + """Atomic share-validation authority and its exact template artifact.""" + + first_seen: tuple[str, float | None] | None + parent: tuple[str, str] | None + observation_sequence: int + observed_monotonic: float | None + template: QbitTipTemplateSnapshot | None = field(repr=False) + + @property + def tip_hash(self) -> str | None: + return None if self.first_seen is None else self.first_seen[0] + + +@dataclass(frozen=True) +class TipRefreshStateSnapshot: + published: PublishedTipSnapshot + latest_detected_tip: tuple[str, int] | None + divergence_started_monotonic: float | None + observation_sequence: int + pending: bool + pending_counter: int + pending_token: int | None + retry_requested: bool + last_successful_refresh_monotonic: float | None + failure_started_monotonic: float | None + refresh_job_count: int + post_accept_refresh_failure_count: int + + +@dataclass(frozen=True) +class TipRefreshTrigger: + """Immutable refresh authority submitted to the latest-wins scheduler.""" + + admission_sequence: int + observation_sequence: int + tip_hash: str | None + template_fingerprint: str | None + template_generation: int | None + payout_state_generation: int + ready_required: bool + reasons: tuple[str, ...] + submitted_monotonic: float + pending_signal_token: int | None + snapshot: QbitTipTemplateSnapshot | None = field(default=None, repr=False) + poll_start_clients: tuple[object, ...] = field(default=(), repr=False) + initial_targets: tuple[RefreshClientTarget, ...] = field(default=(), repr=False) + snapshot_changed: bool = False + post_accept_block: tuple[int, str] | None = None + post_accept_admission_sequence: int | None = None + fresh_capture_required: bool = False + + +@dataclass(frozen=True) +class TipObservationAdmission: + """Result of admitting one live tip observation to the scheduler.""" + + accepted: bool + refresh_needed: bool + observation_sequence: int + completion: Future[int] | None = field(default=None, repr=False) + + +@dataclass(frozen=True) +class TipRefreshSchedulerSnapshot: + admission_open: bool + worker_alive: bool + active: TipRefreshTrigger | None + pending: TipRefreshTrigger | None + pending_capacity: int + + +@dataclass +class _ScheduledTipRefresh: + trigger: TipRefreshTrigger + completion: Future[int] + reporting_trigger: TipRefreshTrigger | None = None + + +@dataclass(frozen=True) +class RetainedCollectionRefresh: + snapshot: QbitTipTemplateSnapshot + observation_sequence: int + payout_state_generation: int + + +@dataclass(frozen=True) +class RefreshResult: + result: str + delivered_monotonic: float | None = None + + +@dataclass(frozen=True, eq=False) +class TipRefreshValidationToken: + """Immutable proof that one prepared refresh passed its expensive guard.""" + + tip_hash: str + template_fingerprint: str + template_generation: int + payout_state_generation: int + observation_sequence: int + build_key: JobBuildKey + snapshot: QbitTipTemplateSnapshot = field(repr=False) + + +@dataclass(frozen=True) +class ActiveRefreshSnapshot: + token: TipRefreshValidationToken + cancelling: bool + + +class FanoutCancellation: + """Close fanout admission, then drain already-admitted deliveries.""" + + def __init__(self) -> None: + self._condition = threading.Condition() + self._cancelling = False + self._active_deliveries = 0 + + def is_set(self) -> bool: + with self._condition: + return self._cancelling + + def begin_delivery(self) -> bool: + with self._condition: + if self._cancelling: + return False + self._active_deliveries += 1 + return True + + def end_delivery(self) -> None: + with self._condition: + if self._active_deliveries <= 0: + raise RuntimeError("fanout delivery gate released without admission") + self._active_deliveries -= 1 + if self._active_deliveries == 0: + self._condition.notify_all() + + def cancel(self) -> None: + with self._condition: + self._cancelling = True + + def set(self) -> None: + self.cancel() + with self._condition: + while self._active_deliveries: + self._condition.wait() + + +@dataclass(frozen=True) +class TipRefreshPorts: + """Narrow cross-domain operations used by tip refresh ownership.""" + + rpc_call: Callable[[str, list[object] | None], object] + rpc_call_with_timeout: Callable[[str, list[object] | None, float], object] + payout_state: Callable[[], PayoutStatePort] + job_bundles: Callable[[], JobBundlePort] + delivery: TipRefreshDeliveryPort + mark_progress_pending: Callable[[float | None], None] + observe_progress_tip_poll: Callable[[QbitTipTemplateSnapshot], None] + publish_progress_work: Callable[[QbitTipTemplateSnapshot, int], None] + start_progress_refresh: Callable[[], RefreshActivityPort] + cancel_obsolete_bundle_builds: Callable[[str | None, int | None], None] + cancel_obsolete_job_builds: Callable[[str], None] + prune_evicted_jobs: Callable[[float | None, bool], None] + delivery_queue_limit: Callable[[], int] + stop_requested: Callable[[], bool] + heartbeat: Callable[[str], None] + remove_heartbeat: Callable[[str], None] + chain_view_untrusted: Callable[[], bool] + ensure_reorg_current: Callable[[str], bool] + observe_job_build_elapsed: Callable[[float, Mapping[str, float]], None] + fetch_snapshot: Callable[[], QbitTipTemplateSnapshot] + ensure_reorg_tip: Callable[[str], bool] + wait_for_execution_permit: Callable[[float], bool] + wait_for_stop: Callable[[float], bool] + hard_exit: Callable[[int], None] + fetch_snapshot_for_tip: ( + Callable[[str], QbitTipTemplateSnapshot] | None + ) = None + + +class TipRefreshService: + """Own tip detection/publication state, pending work, and refresh runtime.""" + + def __init__( + self, + config: TipRefreshConfig, + ports: TipRefreshPorts, + *, + monotonic: Callable[[], float] = time.monotonic, + state_lock: threading.RLock | None = None, + ) -> None: + self.config = config + self._ports = ports + self._monotonic = monotonic + # Observation state and its cross-domain consequences must retain one + # total order. The RLock lets us reject accidental synchronous port + # reentry explicitly instead of deadlocking; production ports never + # reenter tip observation. + self._observation_effects_lock = threading.RLock() + self._observation_effects_active = False + # The coordinator injects the S1 registry RLock so R1 publication + # authority and delivery proof share one atomic boundary. Standalone + # R1 users retain a private lock. + self._state_lock = state_lock or threading.RLock() + self._publication_lock = threading.Lock() + self._refresh_lock = threading.Lock() + self._executor_lock = threading.Lock() + self._metrics_lock = threading.Lock() + self._scheduler_condition = threading.Condition(threading.Lock()) + self._scheduler_admission_sequence = 0 + self._scheduler_admission_open = True + self._scheduler_active: _ScheduledTipRefresh | None = None + self._scheduler_pending: _ScheduledTipRefresh | None = None + self._scheduler_worker: threading.Thread | None = None + self._scheduler_cancel_active = False + self._trigger_capture_local = threading.local() + self._executor: _BoundedPriorityExecutor | None = None + self._executor_shutdown = False + self._pending_event = threading.Event() + self._retry_event = threading.Event() + self._retry_counter = 0 + self._retry_consumed = 0 + self._failure_holdoff_until: float | None = None + self._failure_tip: str | None = None + self._pending_counter = 0 + self._pending_token: int | None = None + self._active_refresh: tuple[ + TipRefreshValidationToken, + FanoutCancellation, + ] | None = None + self._observation_sequence = 0 + self._latest_detected_tip: tuple[str, int] | None = None + self._published = PublishedTipSnapshot(None, None, 0, None, None) + self._divergence_started_monotonic: float | None = None + self._retained_collection_refresh: RetainedCollectionRefresh | None = None + self._last_successful_refresh_monotonic: float | None = None + self._failure_started_monotonic: float | None = None + self._refresh_job_count = 0 + self._post_accept_refresh_failure_count = 0 + self._histograms = { + name: { + "buckets": {bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS}, + "sum": 0.0, + "count": 0, + } + for name in ("refresh", "bundle_build", "first_delivery", "last_delivery") + } + self._phase_histograms = { + phase: { + "buckets": {bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS}, + "sum": 0.0, + "count": 0, + } + for phase in PRISM_TIP_REFRESH_BUILD_PHASES + } + self._client_counts = {result: 0 for result in PRISM_TIP_REFRESH_RESULTS} + self._cancellation_counts = { + stage: 0 for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES + } + self._inflight = 0 + self._build_inflight = 0 + self._build_queue_depth = 0 + self._singleflight_hits = 0 + self._superseded_results = 0 + self._worker_failures = 0 + self._worker_restarts = 0 + self._ipc_bytes = {"input": 0, "output": 0} + self._trigger_coalesces = 0 + self._trigger_supersessions = 0 + self._trigger_latency = { + "buckets": {bucket: 0 for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS}, + "sum": 0.0, + "count": 0, + } + + def reconfigure_for_test( + self, + *, + blockpoll_seconds: float | None = None, + blockwait_timeout_seconds: float | None = None, + failure_holdoff_seconds: float | None = None, + max_workers: int | None = None, + submit_tip_max_age_seconds: float | None = None, + failure_exit_seconds: float | None = None, + ) -> None: + """Explicit fixture hook; production configuration stays immutable.""" + updates: dict[str, object] = {} + if blockpoll_seconds is not None: + updates["blockpoll_seconds"] = float(blockpoll_seconds) + if blockwait_timeout_seconds is not None: + updates["blockwait_timeout_seconds"] = float(blockwait_timeout_seconds) + if failure_holdoff_seconds is not None: + updates["failure_holdoff_seconds"] = float(failure_holdoff_seconds) + if max_workers is not None: + updates["max_workers"] = int(max_workers) + if submit_tip_max_age_seconds is not None: + updates["submit_tip_max_age_seconds"] = float( + submit_tip_max_age_seconds + ) + if failure_exit_seconds is not None: + updates["failure_exit_seconds"] = float(failure_exit_seconds) + self.config = dataclass_replace(self.config, **updates) + + def reconfigure_ports_for_test( + self, + *, + rpc_call: Callable[[str, list[object] | None], object] | None = None, + rpc_call_with_timeout: ( + Callable[[str, list[object] | None, float], object] | None + ) = None, + fetch_snapshot: Callable[[], QbitTipTemplateSnapshot] | None = None, + heartbeat: Callable[[str], None] | None = None, + remove_heartbeat: Callable[[str], None] | None = None, + wait_for_stop: Callable[[float], bool] | None = None, + wait_for_execution_permit: Callable[[float], bool] | None = None, + stop_requested: Callable[[], bool] | None = None, + ) -> None: + """Replace explicit runtime seams in deterministic fixtures only.""" + updates: dict[str, object] = {} + if rpc_call is not None: + updates["rpc_call"] = rpc_call + if rpc_call_with_timeout is not None: + updates["rpc_call_with_timeout"] = rpc_call_with_timeout + if fetch_snapshot is not None: + updates["fetch_snapshot"] = fetch_snapshot + updates["fetch_snapshot_for_tip"] = lambda _tip: fetch_snapshot() + if heartbeat is not None: + updates["heartbeat"] = heartbeat + if remove_heartbeat is not None: + updates["remove_heartbeat"] = remove_heartbeat + if wait_for_stop is not None: + updates["wait_for_stop"] = wait_for_stop + if wait_for_execution_permit is not None: + updates["wait_for_execution_permit"] = wait_for_execution_permit + if stop_requested is not None: + updates["stop_requested"] = stop_requested + self._ports = dataclass_replace(self._ports, **updates) + + @staticmethod + def _trigger_axis_key(trigger: TipRefreshTrigger) -> tuple[int, int]: + return ( + trigger.observation_sequence, + -1 if trigger.template_generation is None else trigger.template_generation, + ) + + @staticmethod + def _trigger_authority_is_older( + candidate: TipRefreshTrigger, + current: TipRefreshTrigger, + ) -> bool: + """Compare only authority axes actually carried by the candidate.""" + if candidate.observation_sequence != current.observation_sequence: + return candidate.observation_sequence < current.observation_sequence + return bool( + candidate.template_generation is not None + and current.template_generation is not None + and candidate.template_generation < current.template_generation + ) + + @classmethod + def _trigger_requirement_supersedes( + cls, + current: TipRefreshTrigger, + candidate: TipRefreshTrigger, + ) -> bool: + return bool( + cls._trigger_axis_key(candidate) > cls._trigger_axis_key(current) + or candidate.payout_state_generation > current.payout_state_generation + or (candidate.ready_required and not current.ready_required) + or ( + candidate.fresh_capture_required + and not current.fresh_capture_required + ) + or ( + candidate.pending_signal_token is not None + and not cls._trigger_authority_is_older(candidate, current) + and ( + current.pending_signal_token is None + or candidate.pending_signal_token > current.pending_signal_token + ) + ) + ) + + @classmethod + def _merge_triggers( + cls, + current: TipRefreshTrigger, + candidate: TipRefreshTrigger, + ) -> TipRefreshTrigger: + current_axis = cls._trigger_axis_key(current) + candidate_axis = cls._trigger_axis_key(candidate) + latest = current + if candidate_axis > current_axis or ( + candidate_axis == current_axis + and candidate.snapshot is not None + and ( + current.snapshot is None + or ( + candidate.pending_signal_token is not None + and ( + current.pending_signal_token is None + or candidate.pending_signal_token + > current.pending_signal_token + ) + ) + ) + ): + latest = candidate + current_post_accept_sequence = ( + -1 + if current.post_accept_admission_sequence is None + else current.post_accept_admission_sequence + ) + candidate_post_accept_sequence = ( + -1 + if candidate.post_accept_admission_sequence is None + else candidate.post_accept_admission_sequence + ) + candidate_owns_post_accept = bool( + candidate.post_accept_block is not None + and candidate_post_accept_sequence >= current_post_accept_sequence + ) + newest_post_accept = ( + candidate.post_accept_block + if candidate_owns_post_accept + else current.post_accept_block + ) + newest_post_accept_sequence = ( + candidate.post_accept_admission_sequence + if candidate_owns_post_accept + else current.post_accept_admission_sequence + ) + merged = dataclass_replace( + latest, + payout_state_generation=max( + current.payout_state_generation, + candidate.payout_state_generation, + ), + ready_required=current.ready_required or candidate.ready_required, + reasons=tuple( + reason + for reason in PRISM_TIP_REFRESH_TRIGGER_REASONS + if reason in current.reasons or reason in candidate.reasons + ), + submitted_monotonic=min( + current.submitted_monotonic, + candidate.submitted_monotonic, + ), + pending_signal_token=( + candidate.pending_signal_token + if candidate.pending_signal_token is not None + and not cls._trigger_authority_is_older(candidate, current) + and ( + current.pending_signal_token is None + or candidate.pending_signal_token > current.pending_signal_token + ) + else current.pending_signal_token + ), + post_accept_block=newest_post_accept, + post_accept_admission_sequence=newest_post_accept_sequence, + fresh_capture_required=( + current.fresh_capture_required + or candidate.fresh_capture_required + ), + ) + if not merged.fresh_capture_required: + return merged + return dataclass_replace( + merged, + template_fingerprint=None, + template_generation=None, + snapshot=None, + poll_start_clients=(), + initial_targets=(), + snapshot_changed=False, + ) + + @staticmethod + def _trigger_cancels_active( + current: TipRefreshTrigger, + candidate: TipRefreshTrigger, + ) -> bool: + return bool( + ( + current.tip_hash is not None + and candidate.tip_hash is not None + and current.tip_hash != candidate.tip_hash + ) + or candidate.payout_state_generation > current.payout_state_generation + or (candidate.ready_required and not current.ready_required) + ) + + def _new_trigger( + self, + *, + observation_sequence: int, + tip_hash: str | None, + payout_state_generation: int, + ready_required: bool, + reasons: tuple[str, ...], + pending_signal_token: int | None, + snapshot: QbitTipTemplateSnapshot | None = None, + template_fingerprint: str | None = None, + template_generation: int | None = None, + poll_start_clients: tuple[object, ...] = (), + initial_targets: tuple[RefreshClientTarget, ...] = (), + snapshot_changed: bool = False, + post_accept_block: tuple[int, str] | None = None, + fresh_capture_required: bool = False, + ) -> TipRefreshTrigger: + if not reasons or any( + reason not in PRISM_TIP_REFRESH_TRIGGER_REASONS for reason in reasons + ): + raise ValueError("tip refresh trigger has an unknown reason") + with self._scheduler_condition: + self._scheduler_admission_sequence += 1 + admission_sequence = self._scheduler_admission_sequence + return TipRefreshTrigger( + admission_sequence=admission_sequence, + observation_sequence=observation_sequence, + tip_hash=tip_hash, + template_fingerprint=( + template_fingerprint + if snapshot is None + else snapshot.template_fingerprint + ), + template_generation=( + template_generation if snapshot is None else snapshot.template_generation + ), + payout_state_generation=int(payout_state_generation), + ready_required=bool(ready_required), + reasons=tuple( + reason for reason in PRISM_TIP_REFRESH_TRIGGER_REASONS if reason in reasons + ), + submitted_monotonic=self._monotonic(), + pending_signal_token=pending_signal_token, + snapshot=snapshot, + poll_start_clients=poll_start_clients, + initial_targets=initial_targets, + snapshot_changed=snapshot_changed, + post_accept_block=post_accept_block, + post_accept_admission_sequence=( + admission_sequence if post_accept_block is not None else None + ), + fresh_capture_required=fresh_capture_required, + ) + + def _ensure_scheduler_worker_locked(self) -> None: + worker = self._scheduler_worker + if worker is not None and worker.is_alive(): + return + worker = threading.Thread( + target=self._scheduler_loop, + name="prism-tip-refresh-scheduler", + daemon=True, + ) + self._scheduler_worker = worker + worker.start() + + def submit_trigger(self, trigger: TipRefreshTrigger) -> Future[int]: + """Admit one immutable observation into active plus one pending slot.""" + cancel_active = False + coalesces = 0 + supersessions = 0 + with self._scheduler_condition: + if not self._scheduler_admission_open: + raise ShutdownInProgress("tip refresh trigger admission is closed") + active = self._scheduler_active + pending = self._scheduler_pending + force_fresh_followup = bool( + active is not None + and trigger.fresh_capture_required + ) + if ( + active is not None + and not force_fresh_followup + and not self._trigger_requirement_supersedes( + active.trigger, + trigger, + ) + ): + active.reporting_trigger = self._merge_triggers( + active.reporting_trigger or active.trigger, + trigger, + ) + completion = active.completion + coalesces = 1 + else: + pending_advanced = pending is None + if pending is None: + pending_trigger = ( + trigger + if active is None + else dataclass_replace( + self._merge_triggers(active.trigger, trigger), + reasons=trigger.reasons, + submitted_monotonic=trigger.submitted_monotonic, + ) + ) + pending = _ScheduledTipRefresh( + pending_trigger, + Future(), + pending_trigger, + ) + self._scheduler_pending = pending + else: + if self._trigger_requirement_supersedes( + pending.trigger, + trigger, + ): + supersessions = 1 + pending_advanced = True + pending.trigger = self._merge_triggers(pending.trigger, trigger) + pending.reporting_trigger = self._merge_triggers( + pending.reporting_trigger or pending.trigger, + trigger, + ) + coalesces = 1 + if ( + pending_advanced + and active is not None + and self._trigger_cancels_active( + active.trigger, + pending.trigger, + ) + ): + cancel_active = True + supersessions = 1 + active_reporting = active.reporting_trigger or active.trigger + pending_reporting = ( + pending.reporting_trigger or pending.trigger + ) + active_post_accept_sequence = ( + -1 + if active_reporting.post_accept_admission_sequence is None + else active_reporting.post_accept_admission_sequence + ) + pending_post_accept_sequence = ( + -1 + if pending_reporting.post_accept_admission_sequence is None + else pending_reporting.post_accept_admission_sequence + ) + if ( + active_reporting.post_accept_block is not None + and pending_reporting.post_accept_block is not None + and pending_post_accept_sequence + >= active_post_accept_sequence + ): + # Expected supersession is not a post-accept failure + # once the canceled pass's reporting context belongs + # to an equal-or-newer pending successor. Same-tip + # draining work is not canceled and retains ownership. + active.reporting_trigger = dataclass_replace( + active_reporting, + post_accept_block=None, + post_accept_admission_sequence=None, + ) + self._ensure_scheduler_worker_locked() + self._scheduler_condition.notify() + completion = pending.completion + if coalesces or supersessions: + with self._metrics_lock: + self._trigger_coalesces += coalesces + self._trigger_supersessions += supersessions + if cancel_active: + self._cancel_active_fanout() + return completion + + def scheduler_snapshot(self) -> TipRefreshSchedulerSnapshot: + with self._scheduler_condition: + worker = self._scheduler_worker + return TipRefreshSchedulerSnapshot( + admission_open=self._scheduler_admission_open, + worker_alive=bool(worker is not None and worker.is_alive()), + active=( + None + if self._scheduler_active is None + else self._scheduler_active.trigger + ), + pending=( + None + if self._scheduler_pending is None + else self._scheduler_pending.trigger + ), + pending_capacity=PRISM_TIP_REFRESH_TRIGGER_PENDING_CAPACITY, + ) + + def wait_for_scheduler_idle_for_test(self, timeout_seconds: float = 1.0) -> bool: + """Wait for active and pending scheduler slots in deterministic tests.""" + deadline = time.monotonic() + max(0.0, timeout_seconds) + with self._scheduler_condition: + while ( + self._scheduler_active is not None + or self._scheduler_pending is not None + ): + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._scheduler_condition.wait(remaining) + return True + + def _cancel_active_fanout(self) -> None: + with self._state_lock: + active = self._active_refresh + if active is not None: + active[1].cancel() + + def _scheduler_owns_current_thread(self) -> bool: + with self._scheduler_condition: + return bool( + self._scheduler_worker is threading.current_thread() + and self._scheduler_active is not None + ) + + def _scheduler_trigger_current(self, trigger: TipRefreshTrigger) -> bool: + with self._scheduler_condition: + active = self._scheduler_active + pending = self._scheduler_pending + return bool( + active is not None + and active.trigger is trigger + and not self._scheduler_cancel_active + and ( + pending is None + or not self._trigger_cancels_active( + trigger, + pending.trigger, + ) + ) + ) + + def _scheduler_has_followup(self, trigger: TipRefreshTrigger) -> bool: + with self._scheduler_condition: + active = self._scheduler_active + pending = self._scheduler_pending + return bool( + active is not None + and active.trigger is trigger + and pending is not None + and not self._trigger_cancels_active(trigger, pending.trigger) + ) + + def _replace_active_trigger( + self, + current: TipRefreshTrigger, + replacement: TipRefreshTrigger, + ) -> TipRefreshTrigger: + with self._scheduler_condition: + active = self._scheduler_active + if active is not None and active.trigger is current: + active.trigger = replacement + pending = self._scheduler_pending + if ( + pending is not None + and pending.trigger.fresh_capture_required + ): + pending_trigger = pending.trigger + pending_reporting = ( + pending.reporting_trigger or pending_trigger + ) + pending.trigger = self._merge_triggers( + pending_trigger, + replacement, + ) + pending.reporting_trigger = self._merge_triggers( + pending_reporting, + replacement, + ) + return replacement + + def _raise_if_scheduler_superseded(self, trigger: TipRefreshTrigger) -> None: + if not self._scheduler_trigger_current(trigger): + raise TemplateRefreshSuperseded( + "refresh trigger was superseded by newer queued authority" + ) + + def _observe_trigger_latency(self, elapsed_seconds: float) -> None: + elapsed_seconds = max(0.0, elapsed_seconds) + with self._metrics_lock: + self._trigger_latency["count"] = int(self._trigger_latency["count"]) + 1 + self._trigger_latency["sum"] = ( + float(self._trigger_latency["sum"]) + elapsed_seconds + ) + buckets = self._trigger_latency["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def _scheduler_loop(self) -> None: + while True: + with self._scheduler_condition: + if self._scheduler_pending is None: + scheduled = None + else: + scheduled = self._scheduler_pending + self._scheduler_pending = None + self._scheduler_active = scheduled + self._scheduler_cancel_active = False + trigger = scheduled.trigger + if scheduled is None: + self._ports.remove_heartbeat("tip_refresh_scheduler") + with self._scheduler_condition: + # Admission may have arrived while the external heartbeat + # port ran. This worker remains published until that port + # returns, so it can consume the new pending slot without + # racing a second scheduler thread. + if self._scheduler_pending is not None: + continue + if self._scheduler_worker is threading.current_thread(): + self._scheduler_worker = None + self._scheduler_condition.notify_all() + return + self._observe_trigger_latency( + self._monotonic() - trigger.submitted_monotonic + ) + try: + result = self._execute_refresh_trigger(trigger) + except BaseException as exc: + if not scheduled.completion.done(): + scheduled.completion.set_exception(exc) + self._handle_scheduled_failure( + scheduled.reporting_trigger or trigger, + exc, + ) + else: + if not scheduled.completion.done(): + scheduled.completion.set_result(result) + self._handle_scheduled_success( + scheduled.reporting_trigger or trigger, + result, + ) + finally: + with self._scheduler_condition: + if self._scheduler_active is scheduled: + self._scheduler_active = None + self._scheduler_cancel_active = False + self._scheduler_condition.notify_all() + + def _handle_scheduled_failure( + self, + trigger: TipRefreshTrigger, + exc: BaseException, + ) -> None: + if isinstance(exc, ShutdownInProgress) or self._ports.stop_requested(): + return + if trigger.post_accept_block is not None: + self.schedule_retry() + if isinstance( + exc, + (TemplateRefreshSuperseded, PayoutStatePublicationBlocked), + ): + return + block_height, block_hash = trigger.post_accept_block + with self._state_lock: + self._post_accept_refresh_failure_count += 1 + print( + "prism coordinator: post-accept clean job refresh failed after " + "direct PRISM block " + f"height={block_height} hash={block_hash}", + flush=True, + ) + traceback.print_exception(exc) + return + if isinstance(exc, (TemplateRefreshSuperseded, PayoutStatePublicationBlocked)): + print( + f"prism coordinator: tip/template refresh superseded; retrying: {exc}", + flush=True, + ) + return + print("prism coordinator: qbit tip/template poll failed", flush=True) + traceback.print_exception(exc) + + @staticmethod + def _handle_scheduled_success(trigger: TipRefreshTrigger, refreshed: int) -> None: + if trigger.post_accept_block is not None: + block_height, block_hash = trigger.post_accept_block + if refreshed: + print( + "prism coordinator: refreshed " + f"{refreshed} client job(s) after direct PRISM block " + f"height={block_height} hash={block_hash}", + flush=True, + ) + elif refreshed: + print( + f"prism coordinator: refreshed {refreshed} client job(s) " + "after qbit tip/template change", + flush=True, + ) + + def snapshot(self) -> TipRefreshStateSnapshot: + with self._state_lock: + return TipRefreshStateSnapshot( + published=self._published, + latest_detected_tip=self._latest_detected_tip, + divergence_started_monotonic=self._divergence_started_monotonic, + observation_sequence=self._observation_sequence, + pending=self._pending_event.is_set(), + pending_counter=self._pending_counter, + pending_token=self._pending_token, + retry_requested=self._retry_event.is_set(), + last_successful_refresh_monotonic=( + self._last_successful_refresh_monotonic + ), + failure_started_monotonic=self._failure_started_monotonic, + refresh_job_count=self._refresh_job_count, + post_accept_refresh_failure_count=( + self._post_accept_refresh_failure_count + ), + ) + + def seed_state_for_test( + self, + *, + published: PublishedTipSnapshot | object = _UNSET, + latest_detected_tip: tuple[str, int] | None | object = _UNSET, + divergence_started_monotonic: float | None | object = _UNSET, + observation_sequence: int | object = _UNSET, + last_successful_refresh_monotonic: float | None | object = _UNSET, + failure_started_monotonic: float | None | object = _UNSET, + refresh_job_count: int | object = _UNSET, + post_accept_refresh_failure_count: int | object = _UNSET, + ) -> None: + """Seed explicitly named state for focused deterministic fixtures.""" + with self._state_lock: + if published is not _UNSET: + self._published = published # type: ignore[assignment] + if latest_detected_tip is not _UNSET: + self._latest_detected_tip = latest_detected_tip # type: ignore[assignment] + if divergence_started_monotonic is not _UNSET: + self._divergence_started_monotonic = divergence_started_monotonic # type: ignore[assignment] + if observation_sequence is not _UNSET: + self._observation_sequence = int(observation_sequence) + if last_successful_refresh_monotonic is not _UNSET: + self._last_successful_refresh_monotonic = last_successful_refresh_monotonic # type: ignore[assignment] + if failure_started_monotonic is not _UNSET: + self._failure_started_monotonic = failure_started_monotonic # type: ignore[assignment] + if refresh_job_count is not _UNSET: + self._refresh_job_count = int(refresh_job_count) + if post_accept_refresh_failure_count is not _UNSET: + self._post_accept_refresh_failure_count = int( + post_accept_refresh_failure_count + ) + + def seed_published_for_test( + self, + *, + first_seen: tuple[str, float | None] | None | object = _UNSET, + parent: tuple[str, str] | None | object = _UNSET, + observation_sequence: int | object = _UNSET, + observed_monotonic: float | None | object = _UNSET, + template: QbitTipTemplateSnapshot | None | object = _UNSET, + ) -> None: + with self._state_lock: + current = self._published + self._published = PublishedTipSnapshot( + current.first_seen if first_seen is _UNSET else first_seen, # type: ignore[arg-type] + current.parent if parent is _UNSET else parent, # type: ignore[arg-type] + ( + current.observation_sequence + if observation_sequence is _UNSET + else int(observation_sequence) + ), + ( + current.observed_monotonic + if observed_monotonic is _UNSET + else observed_monotonic + ), # type: ignore[arg-type] + current.template if template is _UNSET else template, # type: ignore[arg-type] + ) + + def replace_executor_for_test(self, executor: object) -> None: + with self._executor_lock: + self._executor = executor # type: ignore[assignment] + + def replace_refresh_lock_for_test(self, lock: threading.Lock) -> None: + self._refresh_lock = lock + + def clear_retry_for_test(self) -> None: + """Clear a scheduled retry in deterministic fixtures only.""" + with self._state_lock: + self._retry_consumed = self._retry_counter + self._retry_event.clear() + + @contextmanager + def suppress_trigger_callbacks_for_test(self) -> Any: + """Suppress synchronous producer callbacks during manual test setup.""" + previous = bool(getattr(self._trigger_capture_local, "active", False)) + self._trigger_capture_local.active = True + try: + yield + finally: + self._trigger_capture_local.active = previous + + def retained_collection_refresh_snapshot( + self, + ) -> RetainedCollectionRefresh | None: + with self._state_lock: + return self._retained_collection_refresh + + def active_refresh_snapshot(self) -> ActiveRefreshSnapshot | None: + with self._state_lock: + active = self._active_refresh + if active is None: + return None + return ActiveRefreshSnapshot(active[0], active[1].is_set()) + + def seed_active_refresh_for_test( + self, + token: TipRefreshValidationToken, + cancellation: FanoutCancellation, + ) -> None: + with self._state_lock: + self._active_refresh = (token, cancellation) + + def executor_stats(self) -> tuple[int, int]: + with self._executor_lock: + executor = self._executor + return (0, 0) if executor is None else executor.stats() + + def _submit_requirement_trigger( + self, + reason: str, + *, + payout_state_generation: int | None = None, + ready_required: bool | None = None, + pending_signal_token: int | None = None, + tip_hash: str | None = None, + template_fingerprint: str | None = None, + template_generation: int | None = None, + post_accept_block: tuple[int, str] | None = None, + observation_sequence: int | None = None, + fresh_capture_required: bool = False, + ) -> Future[int]: + authority_tip, authority_sequence = self._current_observation_authority() + if tip_hash is None: + tip_hash = authority_tip + if observation_sequence is None: + observation_sequence = authority_sequence + if payout_state_generation is None: + payout_state_generation = int( + self._ports.payout_state().snapshot().generation + ) + if ready_required is None: + ready_required = self._ports.job_bundles().ready_latched() + trigger = self._new_trigger( + observation_sequence=observation_sequence, + tip_hash=tip_hash, + template_fingerprint=template_fingerprint, + template_generation=template_generation, + payout_state_generation=payout_state_generation, + ready_required=ready_required, + reasons=(reason,), + pending_signal_token=pending_signal_token, + post_accept_block=post_accept_block, + fresh_capture_required=fresh_capture_required, + ) + return self.submit_trigger(trigger) + + def submit_post_accept_trigger( + self, + *, + block_height: int, + block_hash: str, + ) -> Future[int]: + # The accepted candidate hash is reporting context, not a validated + # qbit best-tip observation. The scheduler's coherent live capture + # establishes authority for this post-accept refresh. + tip_hash, observation_sequence = self._current_observation_authority() + return self._submit_requirement_trigger( + "post_accept", + tip_hash=tip_hash, + pending_signal_token=self.claim_pending(), + post_accept_block=(block_height, block_hash), + observation_sequence=observation_sequence, + fresh_capture_required=True, + ) + + def submit_tip_observation_admission( + self, + tip_hash: str, + *, + reason: str, + ) -> TipObservationAdmission: + """Record one live observation and return its exact scheduler future.""" + observation_sequence = self.reserve_observation_sequence() + before = self.newest_observed_tip() + accepted = self.observe_tip( + tip_hash, + observation_sequence=observation_sequence, + mark_pending=False, + ) + published = self.published_snapshot() + published_tip = published.tip_hash + completion: Future[int] | None = None + if ( + accepted + and published.template is not None + and (before != tip_hash or published_tip != tip_hash) + ): + pending_signal_token = self.claim_pending() + if before != tip_hash or pending_signal_token is None: + pending_signal_token = self.mark_pending(observation_sequence) + trigger = self._new_trigger( + observation_sequence=observation_sequence, + tip_hash=tip_hash, + payout_state_generation=int( + self._ports.payout_state().snapshot().generation + ), + ready_required=self._ports.job_bundles().ready_latched(), + reasons=(reason,), + pending_signal_token=pending_signal_token, + ) + completion = self.submit_trigger(trigger) + return TipObservationAdmission( + accepted=accepted, + refresh_needed=completion is not None, + observation_sequence=observation_sequence, + completion=completion, + ) + + def submit_tip_observation(self, tip_hash: str, *, reason: str) -> bool: + return self.submit_tip_observation_admission( + tip_hash, + reason=reason, + ).accepted + + def payout_generation_invalidated(self, generation: int) -> None: + """Fence active work now; publication admits the runnable trigger.""" + if self._scheduler_owns_current_thread() or bool( + getattr(self._trigger_capture_local, "active", False) + ): + return + with self._state_lock: + active = self._active_refresh + if active is not None and active[0].payout_state_generation >= generation: + return + if active is not None: + active[1].cancel() + self.mark_pending(generation) + + def payout_generation_changed(self, generation: int) -> None: + """Admit a runnable payout requirement after atomic publication.""" + # Reconciliation/publication performed by the active scheduler pass is + # re-snapshotted before target selection and publication below. A + # recursive payout trigger would only obsolete its own owner. + if self._scheduler_owns_current_thread() or bool( + getattr(self._trigger_capture_local, "active", False) + ): + return + # Direct block acceptance guarantees a fresh post-accept capture after + # its writer scope closes. Keep the invalidation token pending until + # that marker is admitted instead of racing a separate payout pass + # that can deliver duplicate clean work. If acceptance aborts before + # producing the marker, the periodic poll still consumes the token. + if not self._ports.wait_for_execution_permit(0.0): + return + # Startup consumes the first published payout state together with its + # initial coherent tip/template authority. + if self.published_snapshot().template is None: + return + with self._state_lock: + active = self._active_refresh + if active is not None and active[0].payout_state_generation >= generation: + return + if active is not None: + active[1].cancel() + pending_token = self.claim_pending() + if pending_token is None: + pending_token = self.mark_pending(generation) + try: + self._submit_requirement_trigger( + "payout", + payout_state_generation=generation, + pending_signal_token=pending_token, + ) + except ShutdownInProgress: + if not self._ports.stop_requested(): + raise + return + + def readiness_promoted(self) -> None: + # The active scheduler pass called the one-way readiness latch before + # selecting/building work and therefore already owns this requirement. + # Re-enqueueing synchronously from that callback would obsolete the + # pass that is about to satisfy it. + if self._scheduler_owns_current_thread() or bool( + getattr(self._trigger_capture_local, "active", False) + ): + return + # Startup owns readiness promotion until it has published the initial + # coherent tip/template authority. + if self.published_snapshot().template is None: + return + pending_token = self.mark_pending("readiness") + try: + self._submit_requirement_trigger( + "readiness", + ready_required=True, + pending_signal_token=pending_token, + ) + except ShutdownInProgress: + if not self._ports.stop_requested(): + raise + return + + def template_artifacts_changed( + self, + artifacts: CachedTemplateArtifacts, + ) -> None: + # A scheduler fetch owns the exact returned artifacts. Its immutable + # trigger is rebound immediately after the callback returns, so a + # recursive trigger would only supersede identical in-flight work. + if self._scheduler_owns_current_thread() or bool( + getattr(self._trigger_capture_local, "active", False) + ): + return + # The initial repository fill happens before the tip/template authority + # is published. Startup owns that first delivery; admitting a scheduler + # pass here would race the caller that is still assembling it. + with self._state_lock: + published = self._published + if published.template is None: + return + latest = self._latest_detected_tip + if latest is not None and latest[1] >= published.observation_sequence: + authority_tip, observation_sequence = latest + else: + authority_tip = published.tip_hash + observation_sequence = published.observation_sequence + # Repository callback order is not chain-tip authority. A template + # is admitted only after a live observation owns its parent axis. + if artifacts.previousblockhash != authority_tip: + return + pending_token = self.mark_pending(artifacts.generation) + snapshot = QbitTipTemplateSnapshot( + bestblockhash=artifacts.previousblockhash, + previousblockhash=artifacts.previousblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + template_artifacts=artifacts, + ) + snapshot_changed = bool( + published.template.bestblockhash != snapshot.bestblockhash + or published.template.previousblockhash != snapshot.previousblockhash + or published.template.template_fingerprint + != snapshot.template_fingerprint + ) + poll_start_clients = self._ports.delivery.eligible_clients() + targets = self._ports.delivery.select_targets( + snapshot, + refresh_all=snapshot_changed, + ) + try: + self.submit_trigger( + self._new_trigger( + observation_sequence=observation_sequence, + tip_hash=snapshot.bestblockhash, + payout_state_generation=int( + self._ports.payout_state().snapshot().generation + ), + ready_required=self._ports.job_bundles().ready_latched(), + reasons=("template",), + snapshot=snapshot, + poll_start_clients=poll_start_clients, + initial_targets=targets, + snapshot_changed=snapshot_changed, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + pending_signal_token=pending_token, + ) + ) + except ShutdownInProgress: + if not self._ports.stop_requested(): + raise + return + + def record_successful_refresh(self, observed_monotonic: float) -> None: + with self._state_lock: + self._last_successful_refresh_monotonic = observed_monotonic + self._failure_started_monotonic = None + + def published_snapshot(self) -> PublishedTipSnapshot: + with self._state_lock: + return self._published + + def artifacts_parent_current_locked( + self, + artifacts: CachedTemplateArtifacts, + *, + now: float, + ) -> bool: + """Validate repository artifacts against R1's newest chain axis. + + The caller owns ``_state_lock``. R1 deliberately validates only the + immutable parent here: a repository callback may advance the exact + same-tip template generation without first replacing the published + refresh snapshot. S2 separately proves the artifact object, + fingerprint, generation, and payout generation at delivery commit. + + A different parent is admitted only after a live observation advances + ``_latest_detected_tip``. The sole fallback is the exact artifact + owned by the published snapshot while its bounded authority lease is + open. Repository callback order therefore cannot manufacture chain + authority, reuse a generation, or revive arbitrary older artifacts. + """ + published = self._published + latest = self._latest_detected_tip + published_snapshot = published.template + published_artifacts = ( + None + if published_snapshot is None + else published_snapshot.template_artifacts + ) + latest_is_newest = bool( + latest is not None + and latest[1] >= published.observation_sequence + ) + newest_tip = latest[0] if latest_is_newest else published.tip_hash + if artifacts.previousblockhash == newest_tip: + if published.tip_hash != newest_tip or published_snapshot is None: + return True + # On the published parent, repository generations may advance but + # cannot move backward or reuse the published generation with a + # different artifact identity. + return bool( + artifacts is published_artifacts + or artifacts.generation > published_snapshot.template_generation + ) + return bool( + artifacts is published_artifacts + and artifacts.previousblockhash == published.tip_hash + and self._published_tip_authoritative_locked(now) + ) + + def ensure_artifacts_parent_observed( + self, + artifacts: CachedTemplateArtifacts, + ) -> bool: + """Bootstrap missing R1 chain authority from one live observation. + + Existing observed/published authority is never replaced to make an + artifact fit. When authority is wholly absent, R1 releases its state + lock, fetches the live tip, records that observation through the normal + R2 admission path, and then rechecks the persisted state. RPC and + observation effects therefore run without the shared S1/R1 lock. + """ + now = self._monotonic() + with self._state_lock: + if self.artifacts_parent_current_locked(artifacts, now=now): + return True + if ( + self._latest_detected_tip is not None + or self._published.tip_hash is not None + ): + return False + live_tip = str(self._ports.rpc_call("getbestblockhash", None)) + if not self.submit_tip_observation(live_tip, reason="template"): + return False + now = self._monotonic() + with self._state_lock: + return self.artifacts_parent_current_locked(artifacts, now=now) + + @staticmethod + def artifacts( + snapshot: QbitTipTemplateSnapshot, + ) -> CachedTemplateArtifacts: + artifacts = snapshot.template_artifacts + if ( + artifacts is None + or artifacts.fingerprint != snapshot.template_fingerprint + or artifacts.previousblockhash != snapshot.previousblockhash + or artifacts.generation != snapshot.template_generation + or snapshot.bestblockhash != snapshot.previousblockhash + or qbit_template_fingerprint(artifacts.template) != artifacts.fingerprint + or str(artifacts.template.get("previousblockhash", "")) + != artifacts.previousblockhash + ): + raise TemplateRefreshBlocked( + "tip/template snapshot does not own matching exact artifacts" + ) + return artifacts + + def prepare_bundle( + self, + snapshot: QbitTipTemplateSnapshot, + *, + priority_requested_monotonic: float | None = None, + ) -> CachedJobBundle: + artifacts = self.artifacts(snapshot) + build_started = self._monotonic() + job_bundles = self._ports.job_bundles() + priority_token, priority_requested_monotonic = ( + job_bundles.begin_priority_preparation( + priority_requested_monotonic + ) + ) + try: + max_retries = max( + 0, + int(self.config.payout_reconcile_supersession_retries), + ) + for attempt in range(max_retries + 1): + payout_before = self._ports.payout_state().snapshot() + try: + bundle = job_bundles.shared_job_bundle( + artifacts, + mode="ready", + retry_superseded=False, + publication_critical=True, + request_source="tip_refresh", + priority_requested_monotonic=( + priority_requested_monotonic + ), + ) + break + except JobBuildSuperseded: + payout_after = self._ports.payout_state().snapshot() + if ( + attempt >= max_retries + or payout_after.publication_blocked + or payout_after.generation == payout_before.generation + ): + raise + else: # pragma: no cover - range always runs at least once + raise TemplateRefreshBlocked( + "payout generation did not stabilize during preparation" + ) + except TemplateRefreshBlocked: + raise + except Exception as exc: + self._ports.job_bundles().record_failure() + raise TemplateRefreshBlocked("prepared refresh bundle build failed") from exc + finally: + job_bundles.finish_priority_preparation(priority_token) + self.observe_seconds("bundle_build", self._monotonic() - build_started) + artifacts = self.artifacts(snapshot) + if ( + bundle.template is not artifacts.template + or bundle.template_fingerprint != artifacts.fingerprint + or bundle.template_generation != artifacts.generation + or str(bundle.template.get("previousblockhash", "")) + != artifacts.previousblockhash + ): + raise TemplateRefreshBlocked( + "prepared refresh bundle does not match exact template artifacts" + ) + if bundle.collection_only: + raise TemplateRefreshBlocked( + "ready-pool prepared refresh unexpectedly produced a collection bundle" + ) + return bundle + + def prewarm_current_tip_ready_bundle(self) -> CachedJobBundle | None: + job_bundles = self._ports.job_bundles() + job_bundles.set_preparation_pending(True) + try: + observation_sequence = self.reserve_observation_sequence() + snapshot = self._ports.fetch_snapshot() + self._ports.observe_progress_tip_poll(snapshot) + try: + reconciled = self._ports.ensure_reorg_tip(snapshot.bestblockhash) + except Exception as exc: + raise TemplateRefreshBlocked( + "startup reorg reconciliation failed before job preparation" + ) from exc + if not reconciled: + raise TemplateRefreshBlocked( + "startup chain view remained untrusted during job preparation" + ) + ready = job_bundles.pool_readiness_latched() + bundle: CachedJobBundle | None = None + if ready: + bundle = job_bundles.shared_job_bundle( + self.artifacts(snapshot), + None, + publication_critical=True, + request_source="tip_refresh", + ) + if bundle.collection_only: + raise TemplateRefreshBlocked( + "startup ready preparation produced collection work" + ) + if bundle.payout_state_generation != int( + self._ports.payout_state().snapshot().generation + ): + raise TemplateRefreshSuperseded( + "payout state changed during startup job preparation" + ) + if str(self._ports.rpc_call("getbestblockhash", None)) != snapshot.bestblockhash: + raise TemplateRefreshSuperseded( + "qbit tip changed during startup job preparation" + ) + if not self.publish_tip( + snapshot.bestblockhash, + observation_sequence=observation_sequence, + publish_refresh_observation=True, + published_snapshot=snapshot, + ): + raise TemplateRefreshSuperseded( + "startup job preparation was superseded before publication" + ) + job_bundles.set_prepared_ready( + snapshot if bundle is not None else None, + bundle, + ) + payout_generation = ( + bundle.payout_state_generation + if bundle is not None + else int(self._ports.payout_state().snapshot().generation) + ) + self._ports.publish_progress_work(snapshot, payout_generation) + with self._state_lock: + self._last_successful_refresh_monotonic = self._monotonic() + self._ports.observe_progress_tip_poll(snapshot) + return bundle + finally: + job_bundles.set_preparation_pending(False) + + def prewarm_startup_jobs(self) -> CachedJobBundle | None: + try: + return self.prewarm_current_tip_ready_bundle() + except TemplateRefreshBlocked as exc: + self.schedule_retry() + print( + "prism coordinator: startup job preparation deferred " + f"reason={exc}", + flush=True, + ) + return None + + def snapshot_current( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> bool: + with self._state_lock: + return self._snapshot_current_locked(snapshot, observation_sequence) + + def _snapshot_current_locked( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> bool: + return bool( + self._published.template is snapshot + and self._published.tip_hash == snapshot.bestblockhash + and self._published.observation_sequence == observation_sequence + and not self._detected_tip_supersedes_locked( + snapshot.bestblockhash, + observation_sequence, + ) + ) + + def retain_collection_refresh( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + payout_state_generation: int, + ) -> None: + retained = RetainedCollectionRefresh( + snapshot, + observation_sequence, + payout_state_generation, + ) + should_log = False + has_eligible_clients = bool(self._ports.delivery.eligible_clients()) + with self._state_lock: + if not self.snapshot_current(snapshot, observation_sequence): + return + if has_eligible_clients: + return + previous = self._retained_collection_refresh + self._retained_collection_refresh = retained + should_log = previous is None or ( + previous.snapshot.bestblockhash != snapshot.bestblockhash + or previous.payout_state_generation != payout_state_generation + ) + if should_log: + print( + "prism coordinator: collection refresh retained while no " + "authorized worker identity is available", + flush=True, + ) + + def retained_collection_artifacts(self) -> CachedTemplateArtifacts | None: + payout_generation = self._ports.payout_state().snapshot().generation + with self._state_lock: + retained = self._retained_collection_refresh + published = self._published + if retained is None or retained.payout_state_generation != payout_generation: + return None + if ( + published.template is None + or published.tip_hash != published.template.bestblockhash + ): + return None + return self.artifacts(published.template) + + def retain_current_collection_refresh_if_unrepresented(self) -> None: + if self._ports.job_bundles().ready_latched(): + return + if self._ports.delivery.eligible_clients(): + return + with self._state_lock: + snapshot = self._published.template + observation_sequence = self._published.observation_sequence + if snapshot is None: + return + self.retain_collection_refresh( + snapshot, + observation_sequence, + self._ports.payout_state().snapshot().generation, + ) + + def note_collection_identity_available(self, client: object) -> None: + if not self._ports.delivery.client_can_receive_jobs(client): + return + retained_artifacts = self.retained_collection_artifacts() + if retained_artifacts is None: + return + with self._state_lock: + published = self._published + snapshot = published.template + latest = self._latest_detected_tip + if ( + snapshot is None + or snapshot.template_artifacts is not retained_artifacts + or ( + latest is not None + and ( + latest[1] > published.observation_sequence + or latest[0] != published.tip_hash + ) + ) + ): + return + observation_sequence = published.observation_sequence + pending_token = self.mark_pending( + getattr(client, "connection_id", None) + ) + poll_start_clients = self._ports.delivery.eligible_clients() + targets = self._ports.delivery.select_targets( + snapshot, + refresh_all=False, + ) + try: + self.submit_trigger( + self._new_trigger( + observation_sequence=observation_sequence, + tip_hash=snapshot.bestblockhash, + payout_state_generation=int( + self._ports.payout_state().snapshot().generation + ), + ready_required=self._ports.job_bundles().ready_latched(), + reasons=("retained_collection",), + pending_signal_token=pending_token, + snapshot=snapshot, + poll_start_clients=poll_start_clients, + initial_targets=targets, + snapshot_changed=False, + ) + ) + except ShutdownInProgress: + if not self._ports.stop_requested(): + raise + + def consume_retained_collection_refresh(self, context: object) -> None: + if not bool(getattr(context, "collection_only", False)): + return + with self._state_lock: + retained = self._retained_collection_refresh + snapshot = self._published.template + artifacts = None if snapshot is None else snapshot.template_artifacts + if ( + retained is not None + and retained.payout_state_generation + == int(getattr(context, "payout_state_generation")) + and artifacts is not None + and getattr(context, "template") is artifacts.template + and getattr(context, "template_fingerprint") == artifacts.fingerprint + and int(getattr(context, "template_generation")) == artifacts.generation + ): + self._retained_collection_refresh = None + + def clear_retained_collection_refresh(self) -> None: + with self._state_lock: + self._retained_collection_refresh = None + + def newest_observed_tip(self) -> str | None: + with self._state_lock: + if self._latest_detected_tip is not None: + return self._latest_detected_tip[0] + return self._published.tip_hash + + def _current_observation_authority(self) -> tuple[str | None, int]: + """Snapshot the newest live-detected or published chain authority.""" + with self._state_lock: + latest = self._latest_detected_tip + published = self._published + if latest is not None and latest[1] >= published.observation_sequence: + return latest + return published.tip_hash, published.observation_sequence + + def reserve_observation_sequence(self) -> int: + with self._state_lock: + self._observation_sequence = max( + self._observation_sequence, + self._published.observation_sequence, + ) + 1 + return self._observation_sequence + + def observation_sequence(self) -> int: + with self._state_lock: + return self._observation_sequence + + def pending(self) -> bool: + return self._pending_event.is_set() + + def mark_pending(self, _observation: object = None) -> int: + with self._state_lock: + self._pending_counter += 1 + token = self._pending_counter + self._pending_token = token + self._pending_event.set() + return token + + def claim_pending(self) -> int | None: + with self._state_lock: + return self._pending_token if self._pending_event.is_set() else None + + def mark_pending_for_poll( + self, + owned_token: int | None, + _observation: object = None, + ) -> int | None: + with self._state_lock: + if self._pending_token != owned_token: + return owned_token + if owned_token is not None: + self._pending_event.set() + return owned_token + self._pending_counter += 1 + token = self._pending_counter + self._pending_token = token + self._pending_event.set() + return token + + def clear_pending(self, token: int) -> None: + with self._state_lock: + if self._pending_token == token: + self._pending_token = None + self._pending_event.clear() + + def schedule_retry(self) -> None: + # Pair the event with a monotonic generation so a producer cannot set + # it between a waiter's wake and clear and lose the newest retry. + with self._state_lock: + self._retry_counter += 1 + self._retry_event.set() + + def consume_retry(self) -> bool: + """Consume all retry signals visible at one atomic wake boundary.""" + with self._state_lock: + generation = self._retry_counter + if generation == self._retry_consumed: + return False + self._retry_consumed = generation + self._retry_event.clear() + return True + + def note_attempt_failed(self, observed_tip: str | None = None) -> None: + """Space another failed attempt while its observed tip is unchanged.""" + holdoff = float(self.config.failure_holdoff_seconds) + if holdoff <= 0: + return + holdoff += random.uniform( + 0.0, + holdoff * PRISM_TIP_REFRESH_FAILURE_HOLDOFF_JITTER_FRACTION, + ) + with self._state_lock: + if observed_tip is None: + observed_tip = ( + self._latest_detected_tip[0] + if self._latest_detected_tip is not None + else self._published.tip_hash + ) + self._failure_tip = observed_tip + self._failure_holdoff_until = self._monotonic() + holdoff + + def clear_failure_holdoff(self) -> None: + with self._state_lock: + self._failure_holdoff_until = None + self._failure_tip = None + + def failure_holdoff_remaining(self) -> float: + """Return zero immediately when a newer observed tip re-arms refresh.""" + with self._state_lock: + deadline = self._failure_holdoff_until + failed_tip = self._failure_tip + current_tip = ( + self._latest_detected_tip[0] + if self._latest_detected_tip is not None + else self._published.tip_hash + ) + if deadline is None or current_tip != failed_tip: + return 0.0 + return max(0.0, deadline - self._monotonic()) + + def _detected_tip_supersedes_locked( + self, + tip_hash: str, + observation_sequence: int, + ) -> bool: + latest = self._latest_detected_tip + return bool( + latest is not None + and latest[0] != tip_hash + and latest[1] > observation_sequence + ) + + def observe_tip( + self, + tip_hash: str, + *, + observation_sequence: int | None = None, + mark_pending: bool = True, + ) -> bool: + """Record detection without moving published submit authority.""" + if observation_sequence is None: + observation_sequence = self.reserve_observation_sequence() + with self._observation_effects_lock: + if self._observation_effects_active: + raise RuntimeError( + "tip observation ports must not synchronously reenter observation" + ) + self._observation_effects_active = True + try: + now = self._monotonic() + active_to_cancel: FanoutCancellation | None = None + with self._state_lock: + latest = self._latest_detected_tip + if latest is not None and observation_sequence < latest[1]: + return latest[0] == tip_hash + published_tip = self._published.tip_hash + prior_detected_hash = ( + latest[0] + if latest is not None + else self._published.tip_hash + ) + detection_changed = ( + prior_detected_hash is not None + and prior_detected_hash != tip_hash + ) + self._latest_detected_tip = (tip_hash, observation_sequence) + replacement_needed = ( + published_tip is None or published_tip != tip_hash + ) + if published_tip == tip_hash: + self._published = PublishedTipSnapshot( + self._published.first_seen, + self._published.parent, + self._published.observation_sequence, + now, + self._published.template, + ) + elif ( + published_tip is not None + and self._divergence_started_monotonic is None + ): + # First departure owns the bounded lease; later churn + # cannot renew it. + self._divergence_started_monotonic = now + active = self._active_refresh + if ( + active is not None + and active[0].tip_hash != tip_hash + and active[0].observation_sequence < observation_sequence + ): + active_to_cancel = active[1] + should_mark = bool( + mark_pending + and ( + detection_changed + or ( + replacement_needed + and not self._pending_event.is_set() + ) + ) + ) + + if detection_changed: + # The effects lock preserves observation order while the + # R1 state lock stays released across domain callbacks. + self._ports.payout_state().reserve_source_for_tip_change( + tip_hash, + cause="external_tip", + invalidated_monotonic=now, + ) + if active_to_cancel is not None: + active_to_cancel.cancel() + if detection_changed: + self._ports.mark_progress_pending(now) + self._ports.cancel_obsolete_bundle_builds(tip_hash, None) + self._ports.cancel_obsolete_job_builds("chain tip superseded") + if should_mark: + self.mark_pending(observation_sequence) + self.schedule_retry() + return True + finally: + self._observation_effects_active = False + + def publication_failure_expired( + self, + now: float | None = None, + *, + budget_seconds: float | None = None, + ) -> bool: + """Return whether coherent publication has diverged past its budget.""" + current = self._monotonic() if now is None else now + budget = ( + self.config.failure_exit_seconds + if budget_seconds is None + else float(budget_seconds) + ) + if budget <= 0: + return False + with self._state_lock: + started = self._divergence_started_monotonic + return bool(started is not None and current - started >= budget) + + def _fetch_parent_hash(self, tip_hash: str) -> str | None: + block = self._ports.rpc_call("getblock", [tip_hash]) + if not isinstance(block, Mapping): + return None + parent = str(block.get("previousblockhash", "") or "") + return parent or None + + def publish_tip( + self, + tip_hash: str, + *, + observation_sequence: int | None = None, + publish_refresh_observation: bool = False, + published_snapshot: QbitTipTemplateSnapshot | None = None, + ) -> bool: + """Publish exact coherent work only after caller validation.""" + if published_snapshot is not None and published_snapshot.bestblockhash != tip_hash: + raise ValueError("published snapshot does not match tip hash") + if observation_sequence is None: + observation_sequence = self.reserve_observation_sequence() + if not self.observe_tip( + tip_hash, + observation_sequence=observation_sequence, + mark_pending=False, + ): + return False + now = self._monotonic() + with self._state_lock: + if ( + observation_sequence < self._published.observation_sequence + or self._detected_tip_supersedes_locked(tip_hash, observation_sequence) + ): + return False + if self._published.tip_hash == tip_hash: + active = self._active_refresh + sequence = self._published.observation_sequence + if publish_refresh_observation and ( + active is None or active[0].tip_hash != tip_hash + ): + sequence = observation_sequence + self._published = PublishedTipSnapshot( + self._published.first_seen, + self._published.parent, + sequence, + now, + published_snapshot or self._published.template, + ) + self._divergence_started_monotonic = None + return True + + try: + parent_hash = self._fetch_parent_hash(tip_hash) + except Exception: + parent_hash = None + + with self._state_lock: + if ( + observation_sequence < self._published.observation_sequence + or self._detected_tip_supersedes_locked(tip_hash, observation_sequence) + ): + return False + if self._published.tip_hash == tip_hash: + active = self._active_refresh + sequence = self._published.observation_sequence + if publish_refresh_observation and ( + active is None or active[0].tip_hash != tip_hash + ): + sequence = observation_sequence + self._published = PublishedTipSnapshot( + self._published.first_seen, + self._published.parent, + sequence, + now, + published_snapshot or self._published.template, + ) + self._divergence_started_monotonic = None + return True + tip_changed = self._published.first_seen is not None + self._published = PublishedTipSnapshot( + (tip_hash, now if tip_changed else None), + None if parent_hash is None else (tip_hash, parent_hash), + observation_sequence, + now, + published_snapshot, + ) + self._divergence_started_monotonic = None + self._retained_collection_refresh = None + + self._ports.prune_evicted_jobs(now, True) + if tip_changed: + self._ports.job_bundles().clear_prepared_ready() + return True + + def current_tip_parent_hash(self, tip_hash: str) -> str | None: + with self._state_lock: + published = self._published + if published.parent is not None and published.parent[0] == tip_hash: + return published.parent[1] + observed_sequence = ( + published.observation_sequence if published.tip_hash == tip_hash else None + ) + parent = self._fetch_parent_hash(tip_hash) + if parent is None: + return None + with self._state_lock: + published = self._published + if ( + observed_sequence is not None + and published.tip_hash == tip_hash + and published.observation_sequence == observed_sequence + ): + self._published = PublishedTipSnapshot( + published.first_seen, + (tip_hash, parent), + published.observation_sequence, + published.observed_monotonic, + published.template, + ) + return parent + + def published_tip_authoritative(self, now: float | None = None) -> bool: + now = self._monotonic() if now is None else now + with self._state_lock: + return self._published_tip_authoritative_locked(now) + + def _published_tip_authoritative_locked(self, now: float) -> bool: + published = self._published + if self.config.submit_tip_max_age_seconds <= 0 or published.first_seen is None: + return False + if ( + published.observed_monotonic is not None + and now - published.observed_monotonic + <= self.config.submit_tip_max_age_seconds + ): + return True + return bool( + self._latest_detected_tip is not None + and self._latest_detected_tip[0] != published.first_seen[0] + and self._divergence_started_monotonic is not None + and self.config.failure_exit_seconds > 0 + and now - self._divergence_started_monotonic + <= self.config.failure_exit_seconds + ) + + def submit_authority(self) -> str: + with self._state_lock: + if self.published_tip_authoritative(self._monotonic()): + assert self._published.first_seen is not None + return self._published.first_seen[0] + return str(self._ports.rpc_call("getbestblockhash", None)) + + def clear_pending_for_completed_refresh( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + payout_state_generation: int, + pending_signal_token: int | None, + ) -> bool: + payout = self._ports.payout_state() + with payout.delivery_gate.delivery_cancelable( + lambda: payout.snapshot().generation != payout_state_generation, + generation=payout_state_generation, + priority=True, + ) as admission: + if not admission: + return False + if payout.snapshot().generation != payout_state_generation: + return False + with self._state_lock: + published = self._published + current = bool( + published.template is snapshot + and published.tip_hash == snapshot.bestblockhash + and published.observation_sequence == observation_sequence + and not self._detected_tip_supersedes_locked( + snapshot.bestblockhash, + observation_sequence, + ) + ) + if not current or self._pending_token != pending_signal_token: + return False + self._pending_token = None + self._pending_event.clear() + return True + + def token_prepublication_current( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + payout = self._ports.payout_state().snapshot() + with self._state_lock: + return self._token_prepublication_current_locked( + token, + bundle, + snapshot, + payout, + ) + + def _token_prepublication_current_locked( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + payout: PayoutStateSnapshot, + ) -> bool: + published_payout = payout.published + return bool( + token.snapshot is snapshot + and token.tip_hash == snapshot.bestblockhash + and token.template_fingerprint == snapshot.template_fingerprint + and token.template_generation == snapshot.template_generation + and bundle.template_fingerprint == token.template_fingerprint + and bundle.template_generation == token.template_generation + and bundle.payout_state_generation == token.payout_state_generation + and bundle.build_key is token.build_key + and token.build_key.best_tip_hash == snapshot.bestblockhash + and token.build_key.previous_block_hash == snapshot.previousblockhash + and token.build_key.template_fingerprint == snapshot.template_fingerprint + and token.build_key.template_generation == snapshot.template_generation + and token.build_key.payout_state_generation == token.payout_state_generation + and token.payout_state_generation == int(payout.generation) + and published_payout is not None + and published_payout.artifact is not None + and token.build_key.payout_artifact_sha256 + == published_payout.artifact.prior_balances_sha256 + and snapshot.template_artifacts is not None + and bundle.template is snapshot.template_artifacts.template + and not self._detected_tip_supersedes_locked( + snapshot.bestblockhash, + token.observation_sequence, + ) + ) + + def token_current( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + ) -> bool: + payout = self._ports.payout_state().snapshot() + with self._state_lock: + return bool( + self._token_prepublication_current_locked( + token, + bundle, + snapshot, + payout, + ) + and self._snapshot_current_locked( + snapshot, + token.observation_sequence, + ) + ) + + def token_current_for_payout_snapshot( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + payout: PayoutStateSnapshot, + ) -> bool: + """Validate a delivery token without acquiring the P1 state lock. + + S2 captures ``payout`` while its P1 delivery admission is active, then + invokes this method under the shared S1/R1 authority lock. + """ + with self._state_lock: + return bool( + self._token_prepublication_current_locked( + token, + bundle, + snapshot, + payout, + ) + and self._snapshot_current_locked( + snapshot, + token.observation_sequence, + ) + ) + + def validate_prepared( + self, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> TipRefreshValidationToken: + artifacts = self.artifacts(snapshot) + if ( + bundle.template is not artifacts.template + or bundle.template_fingerprint != artifacts.fingerprint + or bundle.template_generation != artifacts.generation + or bundle.build_key is None + or bundle.build_key.best_tip_hash != snapshot.bestblockhash + or bundle.build_key.previous_block_hash != snapshot.previousblockhash + or bundle.build_key.template_fingerprint != artifacts.fingerprint + or bundle.build_key.template_generation != artifacts.generation + or bundle.build_key.mode != "ready" + ): + raise TemplateRefreshBlocked( + "prepared refresh bundle changed before final validation" + ) + try: + current_tip = str(self._ports.rpc_call("getbestblockhash", None)) + except Exception as exc: + self.schedule_retry() + raise TemplateRefreshBlocked( + "qbit tip validation failed before prepared fanout" + ) from exc + if current_tip != snapshot.bestblockhash: + self.schedule_retry() + raise TemplateRefreshSuperseded( + "qbit tip changed before prepared fanout " + f"expected={snapshot.bestblockhash} current={current_tip}" + ) + try: + chain_view_untrusted = self._ports.chain_view_untrusted() + except Exception as exc: + self.schedule_retry() + raise TemplateRefreshBlocked( + "qbit chain trust check failed before prepared fanout" + ) from exc + if chain_view_untrusted: + self.schedule_retry() + raise TemplateRefreshBlocked( + "qbit chain view became untrusted before prepared fanout" + ) + token = TipRefreshValidationToken( + tip_hash=snapshot.bestblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + payout_state_generation=bundle.payout_state_generation, + observation_sequence=observation_sequence, + build_key=bundle.build_key, + snapshot=snapshot, + ) + if not self.token_prepublication_current(token, bundle, snapshot): + self.schedule_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded before tip publication" + ) + return token + + def activate( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + cancel_event: FanoutCancellation, + ) -> None: + payout = self._ports.payout_state().snapshot() + prior_cancel: FanoutCancellation | None = None + with self._state_lock: + if not ( + self._token_prepublication_current_locked( + token, + bundle, + snapshot, + payout, + ) + and self._snapshot_current_locked( + snapshot, + token.observation_sequence, + ) + ): + self.schedule_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded before cancellation registration" + ) + active = self._active_refresh + if active is not None: + prior_cancel = active[1] + self._active_refresh = (token, cancel_event) + if prior_cancel is not None: + prior_cancel.cancel() + + def publish_prepared( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + *, + parent_hash: str | None, + ) -> FanoutCancellation: + now = self._monotonic() + cancel_event = FanoutCancellation() + payout = self._ports.payout_state() + tip_changed = False + prior_cancel: FanoutCancellation | None = None + with payout.delivery_gate.delivery_cancelable( + lambda: False, + generation=token.payout_state_generation, + priority=True, + ) as admitted: + if not admitted: + self.schedule_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded before atomic publication" + ) + with self._publication_lock: + payout_snapshot = payout.snapshot() + with self._state_lock: + if ( + payout_snapshot.publication_blocked + or self._published.observation_sequence > token.observation_sequence + or not self._token_prepublication_current_locked( + token, + bundle, + snapshot, + payout_snapshot, + ) + ): + self.schedule_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded before atomic publication" + ) + first_seen = self._published.first_seen + tip_changed = first_seen is not None and first_seen[0] != token.tip_hash + flip_stamp = ( + now + if tip_changed + else first_seen[1] + if first_seen is not None + else None + ) + prior_parent = self._published.parent + published_parent = ( + (token.tip_hash, parent_hash) + if parent_hash is not None + else prior_parent + if prior_parent is not None and prior_parent[0] == token.tip_hash + else None + ) + self._published = PublishedTipSnapshot( + (token.tip_hash, flip_stamp), + published_parent, + token.observation_sequence, + now, + snapshot, + ) + self._divergence_started_monotonic = None + if tip_changed: + self._retained_collection_refresh = None + if not ( + self._token_prepublication_current_locked( + token, + bundle, + snapshot, + payout_snapshot, + ) + and self._snapshot_current_locked( + snapshot, + token.observation_sequence, + ) + ): + raise TemplateRefreshBlocked( + "prepared refresh publication did not produce a current token" + ) + active = self._active_refresh + if active is not None: + prior_cancel = active[1] + self._active_refresh = (token, cancel_event) + if prior_cancel is not None: + prior_cancel.cancel() + if tip_changed: + self._ports.job_bundles().clear_prepared_ready() + self._ports.prune_evicted_jobs(now, True) + return cancel_event + + def clear_active( + self, + token: TipRefreshValidationToken, + cancel_event: FanoutCancellation, + ) -> None: + with self._state_lock: + active = self._active_refresh + if active is not None and active[0] is token and active[1] is cancel_event: + self._active_refresh = None + + def prepared_obsolete( + self, + token: TipRefreshValidationToken, + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + cancel_event: FanoutCancellation | None, + ) -> bool: + if self._ports.stop_requested() or ( + cancel_event is not None and cancel_event.is_set() + ): + return True + current = self.token_current(token, bundle, snapshot) + if not current and cancel_event is not None: + cancel_event.cancel() + return not current + + def fanout_prepared( + self, + clients: list[object], + bundle: CachedJobBundle, + snapshot: QbitTipTemplateSnapshot, + *, + observation_sequence: int | None = None, + validation_token: TipRefreshValidationToken | None = None, + preactivated_cancel_event: FanoutCancellation | None = None, + executor: object | None = None, + expected_active_jobs: dict[object, object | None] | None = None, + heartbeat_name: str, + ) -> tuple[int, float | None, float | None, int]: + executor = executor or self.executor() + cancel_event = preactivated_cancel_event or FanoutCancellation() + if observation_sequence is None: + observation_sequence = self.published_snapshot().observation_sequence + if validation_token is None: + validation_token = self.validate_prepared( + bundle, + snapshot, + observation_sequence, + ) + if preactivated_cancel_event is None: + self.activate(validation_token, bundle, snapshot, cancel_event) + futures: dict[Future[RefreshResult], object] = {} + submitted_at: dict[Future[RefreshResult], float] = {} + queued_cancellations: set[Future[RefreshResult]] = set() + if expected_active_jobs is None: + expected_active_jobs = { + client: self._ports.delivery.active_job(client) for client in clients + } + clients_iter = iter(clients) + max_inflight = max(1, self.config.max_workers) + + def record_queued_cancellation(future: Future[RefreshResult]) -> None: + if future in queued_cancellations: + return + queued_cancellations.add(future) + elapsed = max(0.0, self._monotonic() - submitted_at[future]) + self._ports.observe_job_build_elapsed( + elapsed, + {"executor_queue": elapsed}, + ) + self.record_cancellation("executor_queue") + + def cancel_pending_futures(pending: set[Future[RefreshResult]]) -> None: + cancel_event.cancel() + for future in pending: + if future.cancel(): + record_queued_cancellation(future) + + def submit_available(pending: set[Future[RefreshResult]]) -> None: + while ( + len(pending) < max_inflight + and not self._ports.stop_requested() + and not cancel_event.is_set() + ): + if not self.token_current(validation_token, bundle, snapshot): + cancel_event.cancel() + return + try: + client = next(clients_iter) + except StopIteration: + return + submitted = self._monotonic() + expected = expected_active_jobs.get(client) + future = self._ports.delivery.submit_task( + executor, + self._ports.delivery.send_prepared_job, + client, + bundle, + snapshot, + validation_token, + self._ports.delivery.connection_id(client), + expected, + cancel_event, + submitted, + priority=self._ports.delivery.delivery_priority( + client, + snapshot, + expected, + ), + ) + self.future_started() + future.add_done_callback(self.future_finished) + futures[future] = client + submitted_at[future] = submitted + pending.add(future) + + pending: set[Future[RefreshResult]] = set() + try: + sent = 0 + failed = 0 + first_delivery: float | None = None + last_delivery: float | None = None + invalidation: TemplateRefreshBlocked | None = None + last_live_trust_check = self._monotonic() + try: + submit_available(pending) + except RuntimeError: + cancel_pending_futures(pending) + cancel_event.set() + if pending: + wait(pending) + if not self._ports.stop_requested(): + self.schedule_retry() + raise + while pending: + self._ports.heartbeat(heartbeat_name) + if self._ports.stop_requested() or cancel_event.is_set(): + cancel_pending_futures(pending) + done, pending = wait( + pending, + timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS, + return_when=FIRST_COMPLETED, + ) + for future in done: + client = futures[future] + if future.cancelled(): + if future not in queued_cancellations: + record_queued_cancellation(future) + self.record_client_result("skipped") + continue + try: + result = future.result() + except OSError: + self.record_client_result("disconnected") + self._ports.delivery.disconnect(client) + continue + except TemplateRefreshBlocked as exc: + self.record_client_result("skipped") + invalidation = exc + cancel_pending_futures(pending) + continue + except Exception: + failed += 1 + self.record_client_result("failed") + self._ports.job_bundles().record_failure() + print( + "prism coordinator: prepared job fanout failed " + + self._ports.delivery.log_identity(client), + flush=True, + ) + import traceback + + traceback.print_exc() + continue + self.record_client_result(result.result) + if result.result == "sent": + sent += 1 + delivered = result.delivered_monotonic + if delivered is not None: + first_delivery = ( + delivered + if first_delivery is None + else min(first_delivery, delivered) + ) + last_delivery = ( + delivered + if last_delivery is None + else max(last_delivery, delivered) + ) + if ( + pending + and invalidation is None + and not self._ports.stop_requested() + and self._monotonic() - last_live_trust_check >= 1.0 + ): + try: + if not self._ports.ensure_reorg_current(snapshot.bestblockhash): + raise TemplateRefreshBlocked( + "qbit chain view became untrusted during prepared fanout" + ) + last_live_trust_check = self._monotonic() + except ShutdownInProgress: + cancel_pending_futures(pending) + raise + except TemplateRefreshBlocked as exc: + invalidation = exc + except Exception as exc: + invalidation = TemplateRefreshBlocked( + "qbit chain trust check failed during prepared fanout" + ) + invalidation.__cause__ = exc + if invalidation is not None: + cancel_pending_futures(pending) + if invalidation is None: + try: + submit_available(pending) + except RuntimeError: + cancel_pending_futures(pending) + cancel_event.set() + if pending: + wait(pending) + if not self._ports.stop_requested(): + self.schedule_retry() + raise + if invalidation is not None: + cancel_event.set() + self.schedule_retry() + raise invalidation + if not self.token_current(validation_token, bundle, snapshot): + self.schedule_retry() + raise TemplateRefreshSuperseded( + "prepared refresh was superseded during fanout; immediate retry scheduled" + ) + try: + post_fanout_tip = str(self._ports.rpc_call("getbestblockhash", None)) + except Exception as exc: + self.schedule_retry() + raise TemplateRefreshBlocked( + "qbit tip validation failed after prepared fanout; " + "immediate retry scheduled" + ) from exc + if post_fanout_tip != snapshot.bestblockhash: + cancel_event.set() + self.schedule_retry() + raise TemplateRefreshSuperseded( + "qbit tip changed during prepared fanout; immediate retry scheduled " + f"expected={snapshot.bestblockhash} current={post_fanout_tip}" + ) + try: + post_fanout_untrusted = self._ports.chain_view_untrusted() + except Exception as exc: + cancel_event.set() + self.schedule_retry() + raise TemplateRefreshBlocked( + "qbit chain trust check failed after prepared fanout; " + "immediate retry scheduled" + ) from exc + if post_fanout_untrusted: + cancel_event.set() + self.schedule_retry() + raise TemplateRefreshBlocked( + "qbit chain view became untrusted during prepared fanout; " + "immediate retry scheduled" + ) + if not self.token_current(validation_token, bundle, snapshot): + cancel_event.set() + self.schedule_retry() + raise TemplateRefreshSuperseded( + "prepared refresh payout state changed during post-fanout " + "validation; immediate retry scheduled" + ) + return sent, first_delivery, last_delivery, failed + finally: + self.clear_active(validation_token, cancel_event) + + def _raise_if_superseded( + self, + snapshot: QbitTipTemplateSnapshot, + observation_sequence: int, + ) -> None: + with self._state_lock: + superseded = self._detected_tip_supersedes_locked( + snapshot.bestblockhash, + observation_sequence, + ) + if superseded: + self.schedule_retry() + raise TemplateRefreshSuperseded( + "tip/template poll was superseded by a newer tip observation " + "before refresh preparation" + ) + + def _probe_tip_while_waiting(self) -> None: + observation_sequence = self.reserve_observation_sequence() + try: + observed_tip = str(self._ports.rpc_call("getbestblockhash", None)) + except Exception: + return + self.observe_tip(observed_tip, observation_sequence=observation_sequence) + + def _capture_poll_trigger( + self, + *, + observation_sequence: int, + reasons: tuple[str, ...], + pending_signal_token: int | None, + post_accept_block: tuple[int, str] | None = None, + existing: TipRefreshTrigger | None = None, + ) -> TipRefreshTrigger: + poll_start_clients = self._ports.delivery.eligible_clients() + observed_best_tip = str(self._ports.rpc_call("getbestblockhash", None)) + if not self.observe_tip( + observed_best_tip, + observation_sequence=observation_sequence, + mark_pending=False, + ): + self.schedule_retry() + raise TemplateRefreshSuperseded( + "tip/template poll was superseded before template fetch" + ) + published_tip = self.published_snapshot().tip_hash + if published_tip is not None and published_tip != observed_best_tip: + pending_signal_token = self.mark_pending_for_poll( + pending_signal_token, + observation_sequence, + ) + capture_was_active = bool( + getattr(self._trigger_capture_local, "active", False) + ) + self._trigger_capture_local.active = True + try: + fetch_for_tip = self._ports.fetch_snapshot_for_tip + snapshot = ( + self._ports.fetch_snapshot() + if fetch_for_tip is None + else fetch_for_tip(observed_best_tip) + ) + except Exception: + self.note_attempt_failed(observed_best_tip) + raise + finally: + self._trigger_capture_local.active = capture_was_active + if not self.observe_tip( + snapshot.bestblockhash, + observation_sequence=observation_sequence, + mark_pending=False, + ): + self.schedule_retry() + raise TemplateRefreshSuperseded( + "tip/template poll was superseded during template fetch" + ) + published_after_fetch = self.published_snapshot().tip_hash + if ( + published_after_fetch is not None + and published_after_fetch != snapshot.bestblockhash + ): + pending_signal_token = self.mark_pending_for_poll( + pending_signal_token, + observation_sequence, + ) + self._ports.observe_progress_tip_poll(snapshot) + job_bundles = self._ports.job_bundles() + ready_required = job_bundles.ready_latched() + payout_generation = int(self._ports.payout_state().snapshot().generation) + previous_snapshot = self.published_snapshot().template + snapshot_changed = previous_snapshot is not None and ( + previous_snapshot.bestblockhash != snapshot.bestblockhash + or previous_snapshot.previousblockhash != snapshot.previousblockhash + or previous_snapshot.template_fingerprint != snapshot.template_fingerprint + ) + targets = self._ports.delivery.select_targets( + snapshot, + refresh_all=snapshot_changed, + ) + if targets and snapshot_changed: + pending_signal_token = self.mark_pending_for_poll( + pending_signal_token, + observation_sequence, + ) + if existing is None: + return self._new_trigger( + observation_sequence=observation_sequence, + tip_hash=snapshot.bestblockhash, + payout_state_generation=payout_generation, + ready_required=ready_required, + reasons=reasons, + pending_signal_token=pending_signal_token, + snapshot=snapshot, + poll_start_clients=poll_start_clients, + initial_targets=targets, + snapshot_changed=snapshot_changed, + post_accept_block=post_accept_block, + ) + return dataclass_replace( + existing, + observation_sequence=observation_sequence, + tip_hash=snapshot.bestblockhash, + template_fingerprint=snapshot.template_fingerprint, + template_generation=snapshot.template_generation, + payout_state_generation=max( + existing.payout_state_generation, + payout_generation, + ), + ready_required=existing.ready_required or ready_required, + pending_signal_token=( + pending_signal_token + if pending_signal_token is not None + else existing.pending_signal_token + ), + snapshot=snapshot, + poll_start_clients=poll_start_clients, + initial_targets=targets, + snapshot_changed=snapshot_changed, + fresh_capture_required=False, + ) + + def detect_poll_trigger( + self, + *, + reason: str = "blockpoll", + post_accept_block: tuple[int, str] | None = None, + ) -> TipRefreshTrigger: + observation_sequence = self.reserve_observation_sequence() + return self._capture_poll_trigger( + observation_sequence=observation_sequence, + reasons=(reason,), + pending_signal_token=self.claim_pending(), + post_accept_block=post_accept_block, + ) + + def submit_poll_trigger( + self, + *, + reason: str = "blockpoll", + post_accept_block: tuple[int, str] | None = None, + ) -> Future[int]: + return self.submit_trigger( + self.detect_poll_trigger( + reason=reason, + post_accept_block=post_accept_block, + ) + ) + + def poll_once(self, *, heartbeat_name: str = "qbit_blockpoll") -> int: + reason = "blockwait" if heartbeat_name == "qbit_blockwait" else "blockpoll" + with self._scheduler_condition: + owner_already_active = self._scheduler_active is not None + try: + completion = self.submit_poll_trigger(reason=reason) + except (TemplateRefreshSuperseded, PayoutStatePublicationBlocked): + raise + except Exception: + self.record_template_refresh_failure(self._monotonic()) + raise + if owner_already_active: + # Contending producers only contribute their newest immutable + # requirement. The scheduler owner drains that coalesced follow-up + # without tying another poll/blockwait caller to the heavy lane. + return 0 + return self._await_scheduler_result(completion, heartbeat_name) + + def _await_scheduler_result( + self, + completion: Future[int], + heartbeat_name: str, + *, + stop_result: int | None = None, + ) -> int: + try: + while True: + self._ports.heartbeat(heartbeat_name) + try: + return completion.result( + timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS + ) + except TimeoutError: + if stop_result is not None and self._ports.stop_requested(): + return stop_result + finally: + self._wait_for_scheduler_completion(completion, heartbeat_name) + + def _wait_for_scheduler_completion( + self, + completion: Future[int], + heartbeat_name: str, + ) -> None: + while True: + self._ports.heartbeat(heartbeat_name) + with self._scheduler_condition: + if ( + self._scheduler_active is None + or self._scheduler_active.completion is not completion + ): + return + self._scheduler_condition.wait( + PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS + ) + + def _execute_refresh_trigger(self, trigger: TipRefreshTrigger) -> int: + while True: + self._raise_if_scheduler_superseded(trigger) + self._ports.heartbeat("tip_refresh_scheduler") + if self._ports.wait_for_execution_permit( + PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS + ): + break + if self._ports.stop_requested(): + raise ShutdownInProgress( + "tip refresh stopped while awaiting writer quiescence" + ) + self._raise_if_scheduler_superseded(trigger) + try: + if trigger.snapshot is None: + original_trigger = trigger + capture_sequence = ( + self.reserve_observation_sequence() + if trigger.fresh_capture_required + else trigger.observation_sequence + ) + trigger = self._capture_poll_trigger( + observation_sequence=capture_sequence, + reasons=trigger.reasons, + pending_signal_token=trigger.pending_signal_token, + post_accept_block=trigger.post_accept_block, + existing=trigger, + ) + trigger = self._replace_active_trigger(original_trigger, trigger) + except ( + ShutdownInProgress, + TemplateRefreshSuperseded, + PayoutStatePublicationBlocked, + ): + raise + except Exception: + self.record_template_refresh_failure(self._monotonic()) + raise + self._raise_if_scheduler_superseded(trigger) + refresh_started = self._monotonic() + publication_lock_acquired = False + progress_refresh: RefreshActivityPort | None = None + heartbeat_name = "tip_refresh_scheduler" + observation_sequence = trigger.observation_sequence + pending_signal_token = trigger.pending_signal_token + snapshot = trigger.snapshot + assert snapshot is not None + poll_start_clients = trigger.poll_start_clients + targets = trigger.initial_targets + snapshot_changed = trigger.snapshot_changed + try: + progress_refresh = self._ports.start_progress_refresh() + job_bundles = self._ports.job_bundles() + ready_required = job_bundles.pool_readiness_latched() + if ready_required and not trigger.ready_required: + trigger = self._replace_active_trigger( + trigger, + dataclass_replace(trigger, ready_required=True), + ) + payout = self._ports.payout_state() + payout_generation_before = int(payout.snapshot().generation) + refreshed = 0 + build_failures = 0 + first_delivery: float | None = None + last_delivery: float | None = None + self._raise_if_scheduler_superseded(trigger) + self._raise_if_superseded(snapshot, observation_sequence) + try: + payout_only_same_tip = bool( + trigger.reasons == ("payout",) + and self.published_snapshot().tip_hash + == snapshot.bestblockhash + ) + reconciled = ( + not self._ports.chain_view_untrusted() + if payout_only_same_tip + else self._ports.ensure_reorg_tip(snapshot.bestblockhash) + ) + except ShutdownInProgress: + return 0 + except Exception as exc: + raise TemplateRefreshBlocked( + "qbit reorg reconciliation failed before refresh preparation" + ) from exc + if not reconciled: + raise TemplateRefreshBlocked( + "qbit chain view remained untrusted after reorg reconciliation" + ) + payout_generation_after = int(payout.snapshot().generation) + if payout_generation_after > trigger.payout_state_generation: + trigger = self._replace_active_trigger( + trigger, + dataclass_replace( + trigger, + payout_state_generation=payout_generation_after, + ), + ) + if payout_generation_after != payout_generation_before: + targets = self._ports.delivery.select_targets( + snapshot, + refresh_all=False, + ) + pending_signal_token = self.claim_pending() + targets = self._ports.delivery.merge_poll_start_targets( + targets, + poll_start_clients, + snapshot, + refresh_all=snapshot_changed, + ) + selected_clients = [target.client for target in targets] + expected_active_jobs = { + target.client: target.expected_active_job for target in targets + } + use_prepared_fanout = bool( + selected_clients and job_bundles.ready_latched() + ) + ready_mode = job_bundles.ready_latched() + bundle: CachedJobBundle | None = None + validation_token: TipRefreshValidationToken | None = None + preactivated_cancel_event: FanoutCancellation | None = None + prepared_executor: object | None = None + if use_prepared_fanout: + self._raise_if_scheduler_superseded(trigger) + self._raise_if_superseded(snapshot, observation_sequence) + try: + bundle = self.prepare_bundle( + snapshot, + priority_requested_monotonic=( + trigger.submitted_monotonic + ), + ) + except PayoutStatePublicationBlocked: + for _client in selected_clients: + self.record_client_result("skipped") + self.schedule_retry() + raise + except TemplateRefreshBlocked: + for _client in selected_clients: + self.record_client_result("failed") + raise + if bundle.payout_state_generation != payout_generation_after: + latest_payout = payout.snapshot() + if ( + latest_payout.publication_blocked + or bundle.payout_state_generation + != latest_payout.generation + ): + self.schedule_retry() + raise TemplateRefreshSuperseded( + "payout state changed after refresh client selection; " + "immediate retry scheduled" + ) + payout_generation_after = int(latest_payout.generation) + trigger = self._replace_active_trigger( + trigger, + dataclass_replace( + trigger, + payout_state_generation=payout_generation_after, + ), + ) + targets = self._ports.delivery.select_targets( + snapshot, + refresh_all=False, + ) + targets = self._ports.delivery.merge_poll_start_targets( + targets, + poll_start_clients, + snapshot, + refresh_all=snapshot_changed, + ) + selected_clients = [target.client for target in targets] + expected_active_jobs = { + target.client: target.expected_active_job + for target in targets + } + pending_signal_token = self.claim_pending() + + self._raise_if_scheduler_superseded(trigger) + self._raise_if_superseded(snapshot, observation_sequence) + while not self._refresh_lock.acquire( + timeout=PRISM_TIP_REFRESH_ADMISSION_POLL_SECONDS + ): + self._ports.heartbeat(heartbeat_name) + if self._ports.stop_requested(): + return 0 + self._probe_tip_while_waiting() + self._raise_if_scheduler_superseded(trigger) + self._raise_if_superseded(snapshot, observation_sequence) + publication_lock_acquired = True + payout_snapshot = payout.snapshot() + current_payout_artifact = payout_snapshot.published.artifact + if ( + payout_snapshot.generation != payout_generation_after + or ( + bundle is not None + and ( + current_payout_artifact is None + or bundle.build_key is None + or bundle.build_key.payout_artifact_sha256 + != current_payout_artifact.prior_balances_sha256 + ) + ) + ): + self.schedule_retry() + raise TemplateRefreshBlocked( + "complete build key changed before refresh publication" + ) + + if use_prepared_fanout: + assert bundle is not None + self._raise_if_scheduler_superseded(trigger) + prepared_executor = self.executor() + try: + parent_hash = self._fetch_parent_hash(snapshot.bestblockhash) + except Exception: + parent_hash = None + validation_token = self.validate_prepared( + bundle, + snapshot, + observation_sequence, + ) + preactivated_cancel_event = self.publish_prepared( + validation_token, + bundle, + snapshot, + parent_hash=parent_hash, + ) + else: + if not self.publish_tip( + snapshot.bestblockhash, + observation_sequence=observation_sequence, + publish_refresh_observation=True, + published_snapshot=snapshot, + ): + raise TemplateRefreshSuperseded( + "tip/template poll was superseded by a newer tip observation" + ) + self._ports.prune_evicted_jobs(None, False) + if not self.snapshot_current(snapshot, observation_sequence): + raise TemplateRefreshSuperseded( + "tip/template poll was superseded before snapshot publication" + ) + + targets, dropped = self._ports.delivery.revalidate_targets( + targets, + snapshot, + ) + selected_clients = [target.client for target in targets] + for result in dropped: + self.record_client_result(result) + + if bundle is not None and not bundle.collection_only: + if bundle.payout_state_generation == payout.snapshot().generation: + job_bundles.set_prepared_ready(snapshot, bundle) + + if ( + not use_prepared_fanout + and selected_clients + and job_bundles.ready_latched() + ): + self.mark_pending(observation_sequence) + self.schedule_retry() + raise TemplateRefreshBlocked( + "current clients appeared after build selection; retry scheduled" + ) + self._refresh_lock.release() + publication_lock_acquired = False + + progress_eligible_client = bool(self._ports.delivery.eligible_clients()) + if use_prepared_fanout or not progress_eligible_client: + self._ports.publish_progress_work(snapshot, payout_generation_after) + + if not ready_mode and not progress_eligible_client: + self.retain_collection_refresh( + snapshot, + observation_sequence, + payout_generation_after, + ) + + if use_prepared_fanout: + assert bundle is not None + ( + refreshed, + first_delivery, + last_delivery, + build_failures, + ) = self.fanout_prepared( + selected_clients, + bundle, + snapshot, + observation_sequence=observation_sequence, + validation_token=validation_token, + preactivated_cancel_event=preactivated_cancel_event, + executor=prepared_executor, + expected_active_jobs=expected_active_jobs, + heartbeat_name=heartbeat_name, + ) + else: + for client in selected_clients: + if self._ports.stop_requested(): + break + self._ports.heartbeat(heartbeat_name) + result = self._ports.delivery.deliver_collection( + client, + snapshot, + observation_sequence, + ) + self.record_client_result(result.result) + if result.result == "sent": + refreshed += 1 + assert result.delivered_monotonic is not None + first_delivery = ( + result.delivered_monotonic + if first_delivery is None + else min(first_delivery, result.delivered_monotonic) + ) + last_delivery = ( + result.delivered_monotonic + if last_delivery is None + else max(last_delivery, result.delivered_monotonic) + ) + elif result.result == "failed": + build_failures += 1 + + if not ready_mode and not self._ports.delivery.eligible_clients(): + self.retain_collection_refresh( + snapshot, + observation_sequence, + payout_generation_after, + ) + self._ports.publish_progress_work( + snapshot, + payout_generation_after, + ) + + if selected_clients: + try: + post_fanout_tip = str( + self._ports.rpc_call("getbestblockhash", None) + ) + except Exception as exc: + self.schedule_retry() + raise TemplateRefreshBlocked( + "qbit tip validation failed after sequential refresh; " + "immediate retry scheduled" + ) from exc + if post_fanout_tip != snapshot.bestblockhash: + self.schedule_retry() + raise TemplateRefreshSuperseded( + "qbit tip changed during sequential refresh; " + "immediate retry scheduled " + f"expected={snapshot.bestblockhash} current={post_fanout_tip}" + ) + if int(payout.snapshot().generation) != payout_generation_after: + self.schedule_retry() + raise TemplateRefreshSuperseded( + "payout state changed during sequential refresh; " + "immediate retry scheduled" + ) + + if refreshed == 0 and build_failures: + raise TemplateRefreshBlocked( + f"job builds failed for {build_failures} client(s); " + "no refreshed work was issued" + ) + if refreshed: + self.add_refresh_jobs(refreshed) + assert first_delivery is not None and last_delivery is not None + self.observe_seconds( + "first_delivery", + first_delivery - refresh_started, + ) + self.observe_seconds( + "last_delivery", + last_delivery - refresh_started, + ) + has_followup = self._scheduler_has_followup(trigger) + pending_cleared = has_followup or self.clear_pending_for_completed_refresh( + snapshot, + observation_sequence, + payout_generation_after, + pending_signal_token, + ) + if not pending_cleared: + pending_signal_token = None + self.schedule_retry() + raise TemplateRefreshSuperseded( + "tip or payout state changed before refresh completion; " + "immediate retry scheduled" + ) + with self._state_lock: + self._last_successful_refresh_monotonic = self._monotonic() + self._failure_started_monotonic = None + self.clear_failure_holdoff() + self._ports.observe_progress_tip_poll(snapshot) + return refreshed + except (TemplateRefreshSuperseded, PayoutStatePublicationBlocked): + self.note_attempt_failed(snapshot.bestblockhash) + raise + except Exception: + self.record_template_refresh_failure(self._monotonic()) + self.note_attempt_failed(snapshot.bestblockhash) + raise + finally: + if publication_lock_acquired: + self._refresh_lock.release() + if progress_refresh is not None: + progress_refresh.finish() + self.observe_seconds("refresh", self._monotonic() - refresh_started) + + def refresh_after_pending_accepted_block( + self, + client: object, + *, + heartbeat_name: str = "qbit_blockpoll", + ) -> int: + block = self._ports.delivery.take_post_accept_refresh(client) + if block is None: + return 0 + block_height, block_hash = block + return self.refresh_after_accepted_block( + block_height=block_height, + block_hash=block_hash, + heartbeat_name=heartbeat_name, + ) + + def refresh_after_accepted_block( + self, + *, + block_height: int, + block_hash: str, + heartbeat_name: str = "qbit_blockpoll", + ) -> int: + try: + self._ports.heartbeat(heartbeat_name) + completion = self.submit_post_accept_trigger( + block_height=block_height, + block_hash=block_hash, + ) + except (TemplateRefreshSuperseded, PayoutStatePublicationBlocked): + self.schedule_retry() + return 0 + except Exception: + self.schedule_retry() + with self._state_lock: + self._post_accept_refresh_failure_count += 1 + print( + "prism coordinator: post-accept clean job refresh failed after " + f"direct PRISM block height={block_height} hash={block_hash}", + flush=True, + ) + traceback.print_exc() + return 0 + try: + return self._await_scheduler_result( + completion, + heartbeat_name, + stop_result=0, + ) + except ( + ShutdownInProgress, + TemplateRefreshSuperseded, + PayoutStatePublicationBlocked, + ): + # The scheduler owns retry and failure classification after + # admission; coordination churn is not a failed notification. + return 0 + except Exception: + # The scheduler already recorded and logged this post-accept + # failure against the merged reporting trigger. + return 0 + + def template_refresh_failure_expired(self, now: float) -> bool: + if self.config.failure_exit_seconds <= 0: + return False + with self._state_lock: + started = self._failure_started_monotonic + return started is not None and now - started >= self.config.failure_exit_seconds + + def record_template_refresh_failure(self, now: float) -> None: + if self.config.failure_exit_seconds <= 0: + return + with self._state_lock: + if self._failure_started_monotonic is None: + self._failure_started_monotonic = now + + def blockwait_once(self, known_tip: str) -> str: + max_rpc_timeout = max(1.0, self.config.watchdog_timeout_seconds * 0.8) + timeout_seconds = min( + self.config.blockwait_timeout_seconds, + max(1.0, max_rpc_timeout - 1.0), + ) + result = self._ports.rpc_call_with_timeout( + "waitfornewblock", + [max(1, int(timeout_seconds * 1000)), known_tip], + timeout_seconds + 10.0, + ) + if isinstance(result, Mapping): + new_tip = str(result.get("hash", "") or "") + if new_tip: + return new_tip + return known_tip + + def wait_for_blockpoll_trigger(self) -> bool: + remaining = self.config.blockpoll_seconds + while remaining > 0: + if self._ports.stop_requested(): + return False + holdoff = self.failure_holdoff_remaining() + if holdoff <= 0 and self.consume_retry(): + return not self._ports.stop_requested() + wait_seconds = min(remaining, 0.25) + if holdoff > 0: + self._ports.heartbeat("qbit_blockpoll") + wait_seconds = min(wait_seconds, holdoff, 0.05) + self._ports.wait_for_stop(wait_seconds) + else: + self._retry_event.wait(wait_seconds) + remaining -= wait_seconds + while not self._ports.stop_requested(): + holdoff = self.failure_holdoff_remaining() + if holdoff <= 0: + break + self._ports.heartbeat("qbit_blockpoll") + self._ports.wait_for_stop(min(holdoff, 0.05)) + self.consume_retry() + return not self._ports.stop_requested() + + def blockpoll_loop(self) -> None: + while self.wait_for_blockpoll_trigger(): + self._ports.heartbeat("qbit_blockpoll") + try: + self.poll_once() + except ShutdownInProgress: + return + except (TemplateRefreshSuperseded, PayoutStatePublicationBlocked) as exc: + print( + f"prism coordinator: tip/template refresh superseded; retrying: {exc}", + flush=True, + ) + except Exception: + print("prism coordinator: qbit tip/template poll failed", flush=True) + traceback.print_exc() + if self.template_refresh_failure_expired(self._monotonic()): + print( + "prism coordinator: template refresh failure budget exhausted; " + "exiting non-zero so the restart policy recovers the process", + flush=True, + ) + self._ports.hard_exit(1) + + def blockwait_loop(self) -> None: + known_tip: str | None = None + while not self._ports.stop_requested(): + self._ports.heartbeat("qbit_blockwait") + try: + if known_tip is None: + observed_tip = str( + self._ports.rpc_call("getbestblockhash", None) + ) + self.observe_tip( + observed_tip, + ) + known_tip = observed_tip + new_tip = self.blockwait_once(known_tip) + if new_tip == known_tip: + if self._ports.wait_for_stop(0.25): + return + continue + # Advance the cursor before notification. A bookkeeping + # failure must not rediscover this transition in a loop. + known_tip = new_tip + try: + self.observe_tip(new_tip) + finally: + self.schedule_retry() + print( + f"prism coordinator: blockwait saw new tip {new_tip}; " + "single-flight refresh scheduled", + flush=True, + ) + except Exception as exc: + if known_tip is not None and self.blockwait_unsupported(exc): + print( + "prism coordinator: waitfornewblock unavailable on this qbitd; " + "tip detection falls back to blockpoll only", + flush=True, + ) + self._ports.remove_heartbeat("qbit_blockwait") + return + print("prism coordinator: blockwait pass failed", flush=True) + traceback.print_exc() + if self._ports.wait_for_stop( + min(5.0, self.config.blockpoll_seconds) + ): + return + + @staticmethod + def blockwait_unsupported(exc: Exception) -> bool: + detail = str(exc).lower() + return any( + marker in detail + for marker in ( + "-32601", + "-32602", + "method not found", + "unknown method", + "invalid params", + "invalid parameter", + "wrong number of", + "too many parameters", + "incorrect number of", + ) + ) + + def executor(self) -> _BoundedPriorityExecutor: + with self._executor_lock: + if self._executor_shutdown: + raise RuntimeError("tip refresh executor is shut down") + if self._executor is None: + self._executor = _BoundedPriorityExecutor( + max_workers=self.config.max_workers, + max_queue_size=self._ports.delivery_queue_limit(), + ) + return self._executor + + def cancel_active(self) -> None: + pending: _ScheduledTipRefresh | None + with self._scheduler_condition: + self._scheduler_admission_open = False + self._scheduler_cancel_active = True + pending = self._scheduler_pending + self._scheduler_pending = None + self._scheduler_condition.notify_all() + if pending is not None and not pending.completion.done(): + pending.completion.set_exception( + ShutdownInProgress("pending tip refresh cancelled by shutdown") + ) + self._cancel_active_fanout() + + def shutdown(self) -> bool: + self.cancel_active() + with self._scheduler_condition: + worker = self._scheduler_worker + if worker is not None and worker is not threading.current_thread(): + worker.join(timeout=1.0) + if worker is not None and worker.is_alive(): + return False + with self._executor_lock: + executor = self._executor + self._executor = None + self._executor_shutdown = True + if executor is not None: + executor.shutdown(wait=True, cancel_futures=True) + return True + + def observe_seconds(self, name: str, elapsed_seconds: float) -> None: + with self._metrics_lock: + histogram = self._histograms[name] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + elapsed_seconds + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def observe_build_phase(self, phase: str, elapsed_seconds: float) -> None: + if phase not in PRISM_TIP_REFRESH_BUILD_PHASES: + raise ValueError(f"unknown tip refresh build phase: {phase}") + with self._metrics_lock: + histogram = self._phase_histograms[phase] + histogram["count"] = int(histogram["count"]) + 1 + histogram["sum"] = float(histogram["sum"]) + max(0.0, elapsed_seconds) + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS: + if elapsed_seconds <= bucket: + buckets[bucket] = int(buckets.get(bucket, 0)) + 1 + + def record_ipc_bytes(self, direction: str, byte_count: int) -> None: + if direction not in self._ipc_bytes: + raise ValueError(f"unknown tip refresh IPC direction: {direction}") + with self._metrics_lock: + self._ipc_bytes[direction] += max(0, int(byte_count)) + + def record_client_result(self, result: str) -> None: + if result not in PRISM_TIP_REFRESH_RESULTS: + raise ValueError(f"unknown tip refresh result: {result}") + with self._metrics_lock: + self._client_counts[result] += 1 + + def record_cancellation(self, stage: str) -> None: + if stage not in PRISM_TIP_REFRESH_CANCELLATION_STAGES: + raise ValueError(f"unknown tip refresh cancellation stage: {stage}") + with self._metrics_lock: + self._cancellation_counts[stage] += 1 + + def future_started(self) -> None: + with self._metrics_lock: + self._inflight += 1 + + def future_finished(self, _future: Future[RefreshResult] | None = None) -> None: + with self._metrics_lock: + self._inflight = max(0, self._inflight - 1) + + def record_superseded_result(self) -> None: + with self._metrics_lock: + self._superseded_results += 1 + + def record_worker_failure(self) -> None: + with self._metrics_lock: + self._worker_failures += 1 + + def record_worker_restart(self) -> None: + with self._metrics_lock: + self._worker_restarts += 1 + + def record_singleflight_hit(self) -> None: + with self._metrics_lock: + self._singleflight_hits += 1 + + def add_refresh_jobs(self, count: int) -> None: + with self._state_lock: + self._refresh_job_count += max(0, int(count)) + + def set_build_gauges(self, *, inflight: int, queue_depth: int) -> None: + with self._metrics_lock: + self._build_inflight = int(inflight) + self._build_queue_depth = int(queue_depth) + + def metrics_snapshot(self) -> dict[str, object]: + with self._scheduler_condition: + trigger_queue_depth = int(self._scheduler_pending is not None) + with self._executor_lock: + executor_workers = self.config.max_workers if self._executor is not None else 0 + with self._metrics_lock: + return { + "histograms": { + name: { + "buckets": dict(value["buckets"]), + "sum": float(value["sum"]), + "count": int(value["count"]), + } + for name, value in self._histograms.items() + }, + "phase_histograms": { + name: { + "buckets": dict(value["buckets"]), + "sum": float(value["sum"]), + "count": int(value["count"]), + } + for name, value in self._phase_histograms.items() + }, + "client_counts": dict(self._client_counts), + "cancellation_counts": dict(self._cancellation_counts), + "inflight": self._inflight, + "executor_workers": executor_workers, + "build_inflight": self._build_inflight, + "build_queue_depth": self._build_queue_depth, + "singleflight_hits": self._singleflight_hits, + "superseded_results": self._superseded_results, + "worker_failures": self._worker_failures, + "worker_restarts": self._worker_restarts, + "ipc_bytes": dict(self._ipc_bytes), + "trigger_queue_depth": trigger_queue_depth, + "trigger_queue_capacity": PRISM_TIP_REFRESH_TRIGGER_PENDING_CAPACITY, + "trigger_coalesces": self._trigger_coalesces, + "trigger_supersessions": self._trigger_supersessions, + "trigger_latency": { + "buckets": dict(self._trigger_latency["buckets"]), + "sum": float(self._trigger_latency["sum"]), + "count": int(self._trigger_latency["count"]), + }, + } + + def metrics_lines(self) -> list[str]: + snapshot = self.metrics_snapshot() + histograms = snapshot["histograms"] + phase_histograms = snapshot["phase_histograms"] + client_counts = snapshot["client_counts"] + cancellation_counts = snapshot["cancellation_counts"] + ipc_bytes = snapshot["ipc_bytes"] + trigger_latency = snapshot["trigger_latency"] + assert isinstance(histograms, dict) + assert isinstance(phase_histograms, dict) + assert isinstance(client_counts, dict) + assert isinstance(cancellation_counts, dict) + assert isinstance(ipc_bytes, dict) + assert isinstance(trigger_latency, dict) + + metric_names = { + "refresh": "qbit_prism_tip_refresh_seconds", + "bundle_build": "qbit_prism_tip_refresh_bundle_build_seconds", + "first_delivery": "qbit_prism_tip_refresh_first_delivery_seconds", + "last_delivery": "qbit_prism_tip_refresh_last_delivery_seconds", + } + descriptions = { + "refresh": "Full qbit tip/template refresh pass wall time.", + "bundle_build": "Shared ready-pool refresh bundle preparation wall time.", + "first_delivery": "Tip observation to first successful client delivery.", + "last_delivery": "Tip observation to last successful client delivery.", + } + lines: list[str] = [] + for name, metric_name in metric_names.items(): + histogram = histograms[name] + assert isinstance(histogram, dict) + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + f"# HELP {metric_name} {descriptions[name]}", + f"# TYPE {metric_name} histogram", + *[ + f'{metric_name}_bucket{{le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS + ], + f'{metric_name}_bucket{{le="+Inf"}} {histogram["count"]}', + f'{metric_name}_sum {float(histogram["sum"]):.6f}', + f'{metric_name}_count {histogram["count"]}', + ] + ) + phase_metric_name = "qbit_prism_tip_refresh_bundle_phase_seconds" + lines.extend( + [ + "# HELP qbit_prism_tip_refresh_bundle_phase_seconds Shared bundle-build phase wall time.", + "# TYPE qbit_prism_tip_refresh_bundle_phase_seconds histogram", + ] + ) + for phase in PRISM_TIP_REFRESH_BUILD_PHASES: + histogram = phase_histograms[phase] + assert isinstance(histogram, dict) + buckets = histogram["buckets"] + assert isinstance(buckets, dict) + lines.extend( + [ + *[ + f'{phase_metric_name}_bucket{{phase="{phase}",le="{bucket:g}"}} {int(buckets.get(bucket, 0))}' + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS + ], + f'{phase_metric_name}_bucket{{phase="{phase}",le="+Inf"}} {histogram["count"]}', + f'{phase_metric_name}_sum{{phase="{phase}"}} {float(histogram["sum"]):.6f}', + f'{phase_metric_name}_count{{phase="{phase}"}} {histogram["count"]}', + ] + ) + lines.extend( + [ + "# HELP qbit_prism_tip_refresh_clients_total Client outcomes from tip/template refresh passes.", + "# TYPE qbit_prism_tip_refresh_clients_total counter", + *[ + f'qbit_prism_tip_refresh_clients_total{{result="{result}"}} {int(client_counts.get(result, 0))}' + for result in PRISM_TIP_REFRESH_RESULTS + ], + "# HELP qbit_prism_tip_refresh_cancellations_total Obsolete prepared refresh tasks canceled before delivery admission.", + "# TYPE qbit_prism_tip_refresh_cancellations_total counter", + *[ + f'qbit_prism_tip_refresh_cancellations_total{{stage="{stage}"}} {int(cancellation_counts.get(stage, 0))}' + for stage in PRISM_TIP_REFRESH_CANCELLATION_STAGES + ], + "# HELP qbit_prism_tip_refresh_inflight Prepared refresh client tasks currently queued or running.", + "# TYPE qbit_prism_tip_refresh_inflight gauge", + f'qbit_prism_tip_refresh_inflight {int(snapshot["inflight"])}', + "# HELP qbit_prism_tip_refresh_executor_workers Configured persistent refresh executor workers, or zero before creation.", + "# TYPE qbit_prism_tip_refresh_executor_workers gauge", + f'qbit_prism_tip_refresh_executor_workers {int(snapshot["executor_workers"])}', + "# HELP qbit_prism_tip_refresh_bundle_inflight Shared bundle builds currently running.", + "# TYPE qbit_prism_tip_refresh_bundle_inflight gauge", + f'qbit_prism_tip_refresh_bundle_inflight {int(snapshot["build_inflight"])}', + "# HELP qbit_prism_tip_refresh_bundle_queue_depth Shared bundle callers waiting on bounded build admission or an identical single-flight.", + "# TYPE qbit_prism_tip_refresh_bundle_queue_depth gauge", + f'qbit_prism_tip_refresh_bundle_queue_depth {int(snapshot["build_queue_depth"])}', + "# HELP qbit_prism_tip_refresh_bundle_singleflight_hits_total Shared bundle callers coalesced behind an identical build.", + "# TYPE qbit_prism_tip_refresh_bundle_singleflight_hits_total counter", + f'qbit_prism_tip_refresh_bundle_singleflight_hits_total {int(snapshot["singleflight_hits"])}', + "# HELP qbit_prism_tip_refresh_bundle_superseded_results_total Completed or canceled shared bundles discarded after supersession.", + "# TYPE qbit_prism_tip_refresh_bundle_superseded_results_total counter", + f'qbit_prism_tip_refresh_bundle_superseded_results_total {int(snapshot["superseded_results"])}', + "# HELP qbit_prism_tip_refresh_builder_worker_failures_total Audit-builder subprocess failures.", + "# TYPE qbit_prism_tip_refresh_builder_worker_failures_total counter", + f'qbit_prism_tip_refresh_builder_worker_failures_total {int(snapshot["worker_failures"])}', + "# HELP qbit_prism_tip_refresh_builder_worker_restarts_total Long-lived builder worker restarts; zero for the inline subprocess design.", + "# TYPE qbit_prism_tip_refresh_builder_worker_restarts_total counter", + f'qbit_prism_tip_refresh_builder_worker_restarts_total {int(snapshot["worker_restarts"])}', + "# HELP qbit_prism_tip_refresh_builder_ipc_bytes_total Bytes copied across audit-builder subprocess IPC.", + "# TYPE qbit_prism_tip_refresh_builder_ipc_bytes_total counter", + *[ + f'qbit_prism_tip_refresh_builder_ipc_bytes_total{{direction="{direction}"}} {int(ipc_bytes.get(direction, 0))}' + for direction in ("input", "output") + ], + ] + ) + trigger_latency_buckets = trigger_latency["buckets"] + assert isinstance(trigger_latency_buckets, dict) + lines.extend( + [ + "# HELP qbit_prism_tip_refresh_trigger_queue_depth Immutable refresh triggers waiting in the fixed latest-wins pending slot.", + "# TYPE qbit_prism_tip_refresh_trigger_queue_depth gauge", + f'qbit_prism_tip_refresh_trigger_queue_depth {int(snapshot["trigger_queue_depth"])}', + "# HELP qbit_prism_tip_refresh_trigger_queue_capacity Fixed pending trigger capacity, excluding the active refresh.", + "# TYPE qbit_prism_tip_refresh_trigger_queue_capacity gauge", + f'qbit_prism_tip_refresh_trigger_queue_capacity {int(snapshot["trigger_queue_capacity"])}', + "# HELP qbit_prism_tip_refresh_trigger_coalesces_total Trigger admissions merged with already active or pending work.", + "# TYPE qbit_prism_tip_refresh_trigger_coalesces_total counter", + f'qbit_prism_tip_refresh_trigger_coalesces_total {int(snapshot["trigger_coalesces"])}', + "# HELP qbit_prism_tip_refresh_trigger_supersessions_total Trigger admissions that replaced older pending or active requirements.", + "# TYPE qbit_prism_tip_refresh_trigger_supersessions_total counter", + f'qbit_prism_tip_refresh_trigger_supersessions_total {int(snapshot["trigger_supersessions"])}', + "# HELP qbit_prism_tip_refresh_trigger_latency_seconds Trigger admission to scheduler execution latency.", + "# TYPE qbit_prism_tip_refresh_trigger_latency_seconds histogram", + *[ + 'qbit_prism_tip_refresh_trigger_latency_seconds_bucket' + f'{{le="{bucket:g}"}} {int(trigger_latency_buckets.get(bucket, 0))}' + for bucket in PRISM_TIP_REFRESH_SECONDS_BUCKETS + ], + "qbit_prism_tip_refresh_trigger_latency_seconds_bucket" + f'{{le="+Inf"}} {trigger_latency["count"]}', + "qbit_prism_tip_refresh_trigger_latency_seconds_sum " + f'{float(trigger_latency["sum"]):.6f}', + "qbit_prism_tip_refresh_trigger_latency_seconds_count " + f'{trigger_latency["count"]}', + ] + ) + return lines diff --git a/test/test-prism-postgres-ledger.sh b/test/test-prism-postgres-ledger.sh index c66e9c4..eb8990c 100644 --- a/test/test-prism-postgres-ledger.sh +++ b/test/test-prism-postgres-ledger.sh @@ -72,7 +72,7 @@ import os import tempfile from pathlib import Path -from lab.prism.share_ledger import PendingShare, PsqlShareLedger +from lab.prism.share_ledger import PendingShare, PsqlShareLedger, ShareReplayConflict def pending( @@ -164,6 +164,16 @@ candidate_intent = { candidate_row = ledger.append_batch([(batch_c, candidate_intent)])[0] assert_equal(candidate_row.share_seq, 3, "candidate share sequence") assert_equal(ledger.append_batch([(batch_c, candidate_intent)])[0].share_seq, 3, "exact replay") +recovery_exact = ledger.append_recovered_share(batch_c) +assert_equal(recovery_exact.disposition, "exact_existing", "typed recovery exact replay") +try: + ledger.append_recovered_share( + PendingShare(**{**batch_c.__dict__, "ntime": batch_c.ntime + 1}) + ) +except ShareReplayConflict: + pass +else: + raise SystemExit("typed recovery payload conflict was not rejected") assert_equal(ledger.pending_block_candidates(), [candidate_intent], "pending candidate replay") assert_equal( ledger.pending_block_candidate_rows(), diff --git a/tests/prism_coordinator_test_support.py b/tests/prism_coordinator_test_support.py index 3502b5b..b368d87 100644 --- a/tests/prism_coordinator_test_support.py +++ b/tests/prism_coordinator_test_support.py @@ -20,6 +20,7 @@ from lab.auxpow import vardiff from lab.prism import direct_stratum +from lab.prism.coordinator_shutdown import ShutdownInProgress from lab.prism.prism_coordinator import ( ClientState, JobBuildSuperseded, @@ -27,7 +28,6 @@ PRISM_JOB_EXTRANONCE1_PLACEHOLDER_HEX, PRISM_REJECTION_REASON_IDS, PrismCoordinator, - ShutdownInProgress, TemplateRefreshBlocked, WorkerIdentity, canonical_json_sha256, @@ -231,7 +231,6 @@ def coordinator(*, ledger: object | None = None, template: dict[str, object] | N server.duplicate_share_count = 0 server.low_difficulty_share_count = 0 server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - server.job_build_failure_count = 0 server.tip_refresh_job_count = 0 server.post_accept_refresh_failure_count = 0 server.reorg_reconciler_enabled = False @@ -266,10 +265,10 @@ def coordinator(*, ledger: object | None = None, template: dict[str, object] | N server.min_ready_miners = 3 server.ledger = ledger if ledger is not None else FakeLedger() server.blockpoll_seconds = 2.0 - # Failed-refresh spacing is opt-in per test: its holdoff waits on real - # time, which deadlocks tests that freeze time.monotonic around failing - # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. - server.tip_refresh_failure_holdoff_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds, + failure_holdoff_seconds=0.0, + ) server.job_bundle_cache_seconds = 10.0 server.template_cache_seconds = 2.0 server.reorg_reconcile_cache_seconds = 5.0 @@ -300,6 +299,9 @@ def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: } server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + fake_build_audit_bundle + ) return recorded @@ -339,7 +341,7 @@ def mark_progress_healthy(server: PrismCoordinator) -> None: server._record_progress_tip_poll(snapshot) server._record_progress_publication( snapshot, - int(getattr(server, "_payout_state_generation", 0)), + int(server._payout_state_service._generation), ) diff --git a/tests/prism_vardiff_test_support.py b/tests/prism_vardiff_test_support.py index 2a071b6..0b21f10 100644 --- a/tests/prism_vardiff_test_support.py +++ b/tests/prism_vardiff_test_support.py @@ -23,7 +23,12 @@ from lab.auxpow import vardiff from lab.prism import direct_stratum -from lab.prism.share_ledger import PendingShare, SingleWriterShareLedger +from lab.prism.share_ledger import ( + PendingShare, + ShareReplayConflict, + ShareReplayResult, + SingleWriterShareLedger, +) from lab.prism.prism_coordinator import ( CachedJobBundle, CachedTemplateArtifacts, @@ -190,6 +195,18 @@ def append(self, pending: object) -> object: self.shares += 1 return SimpleNamespace(share_seq=self.shares, miner_id=getattr(pending, "miner_id", "miner-a")) + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + for index, existing in enumerate(self.pending, start=1): + if getattr(existing, "share_id", None) != pending.share_id: + continue + if existing != pending: + raise ShareReplayConflict(pending.share_id) + return ShareReplayResult( + "exact_existing", + SimpleNamespace(share_seq=index, miner_id=pending.miner_id), + ) + return ShareReplayResult("inserted", self.append(pending)) + def persist_accepted_block(self, **kwargs: object) -> dict[str, object]: self.persisted.append({**kwargs, "submit_seen_at_persist": self.submit_seen}) return { @@ -590,8 +607,9 @@ def install_idle_job_cache( *, tip: str = "00" * 32, ) -> CachedJobBundle: - server._pool_ready_latched = True + server._ensure_job_bundle_service()._ready_latched = True server.job_bundle_cache_seconds = 60.0 + server._ensure_job_bundle_service().set_cache_seconds_for_test(60.0) server.job_counter = 0 server.share_weights_by_username = {} server.default_share_weight = 1 @@ -611,7 +629,7 @@ def install_idle_job_cache( key = server._job_bundle_key( artifacts, mode="ready", - payout_state_generation=server._payout_state_generation, + payout_state_generation=server._payout_state_service._generation, payout_artifact_generation=0, worker=None, ) @@ -639,7 +657,7 @@ def install_idle_job_cache( ) bundle = CachedJobBundle( key=key, - template=template, + template=artifacts.template, template_fingerprint=fingerprint, coinbase_manifest={}, shares_json=[], @@ -655,20 +673,39 @@ def install_idle_job_cache( payout_artifact_sha256=payout_artifact_sha256, ), ) - with server._job_cache_lock: - server._template_artifacts = artifacts - server._published_payout_state = dataclass_replace( - server._published_payout_state, + service = server._ensure_job_bundle_service() + service.template_repository.replace_for_test(artifacts) + now = time.monotonic() + tip_service = server._ensure_tip_refresh_service() + tip_service.seed_state_for_test( + latest_detected_tip=None, + observation_sequence=1, + ) + tip_service.seed_published_for_test( + first_seen=(tip, now), + observation_sequence=1, + observed_monotonic=now, + template=QbitTipTemplateSnapshot( + bestblockhash=tip, + previousblockhash=tip, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + template_artifacts=artifacts, + ), + ) + with service._cache_lock: + server._payout_state_service._published = dataclass_replace( + server._payout_state_service._published, artifact=PayoutStateArtifact( - generation=server._payout_state_generation, + generation=server._payout_state_service._generation, source_generation=0, prior_balances_json="[]", prior_balances_sha256=payout_artifact_sha256, prepared_monotonic=time.monotonic(), ), ) - server._job_bundle_cache.clear() - server._job_bundle_cache[bundle.key] = bundle + service._bundle_cache.clear() + service._bundle_cache[bundle.key] = bundle return bundle @@ -718,7 +755,7 @@ def coordinator() -> PrismCoordinator: server.duplicate_share_count = 0 server.low_difficulty_share_count = 0 server.collection_block_submission_count = 0 - server._pool_ready_latched = False + server._ensure_job_bundle_service()._ready_latched = False server.grace_credited_share_count = 0 server.idle_retarget_count = 0 server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} @@ -742,8 +779,11 @@ def coordinator() -> PrismCoordinator: server.stale_grace_seconds = 3.0 server.blockwait_enabled = True server.blockwait_timeout_seconds = 5.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockwait_timeout_seconds=server.blockwait_timeout_seconds, + failure_holdoff_seconds=0.0, + ) server.vardiff_idle_sweep_seconds = 15.0 - server.job_build_failure_count = 0 server.tip_refresh_job_count = 0 server.post_accept_refresh_failure_count = 0 server.reorg_reconciler_enabled = False @@ -758,13 +798,13 @@ def coordinator() -> PrismCoordinator: server.started_monotonic = time.monotonic() - 10 server.ledger = FakeLedger(shares=5) server.latest_coinbase_size_bytes = 250 - server.rpc = FakeRpc() + server.rpc = TipRpc("00" * 32) server.qbit_chain = "regtest" server.blockpoll_seconds = 2.0 - # Failed-refresh spacing is opt-in per test: its holdoff waits on real - # time, which deadlocks tests that freeze time.monotonic around failing - # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. - server.tip_refresh_failure_holdoff_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds, + failure_holdoff_seconds=0.0, + ) server.ctv_broadcaster_enabled = False server.ctv_broadcaster_wallet = None server.ctv_broadcaster_fee_sats = 0 @@ -773,7 +813,9 @@ def coordinator() -> PrismCoordinator: server.ctv_fanout_broadcast_daemon = None server._ctv_fanout_market_fee_rate_cache = {} server.tip_template_snapshot = None - server._tip_refresh_lock = threading.Lock() + server._ensure_tip_refresh_service().replace_refresh_lock_for_test( + threading.Lock() + ) server.extranonce2_size = 8 server.coinbase_tag_hex = default_prism_coinbase_tag_hex() server.version_mask = direct_stratum.QBIT_VERSION_ROLLING_MASK diff --git a/tests/test_prism_background_services.py b/tests/test_prism_background_services.py new file mode 100644 index 0000000..67d7e52 --- /dev/null +++ b/tests/test_prism_background_services.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import threading +from types import SimpleNamespace +import unittest + +from lab.prism.background_services import ( + BackgroundServiceRegistry, + BackgroundServiceSpec, +) +from lab.prism.prism_coordinator import PrismCoordinator + + +class DormantThread: + def __init__( + self, + *, + target: object, + name: str, + daemon: bool, + ) -> None: + self.target = target + self.name = name + self.daemon = daemon + self.start_count = 0 + self.join_timeouts: list[float | None] = [] + + def start(self) -> None: + self.start_count += 1 + + def join(self, timeout: float | None = None) -> None: + self.join_timeouts.append(timeout) + + +class FlakyThreadFactory: + def __init__(self) -> None: + self.attempts = 0 + self.threads: list[DormantThread] = [] + + def __call__(self, **kwargs: object) -> DormantThread: + factory = self + + class FlakyDormantThread(DormantThread): + def start(self) -> None: + factory.attempts += 1 + if factory.attempts == 1: + raise RuntimeError("thread start failed") + super().start() + + thread = FlakyDormantThread(**kwargs) # type: ignore[arg-type] + self.threads.append(thread) + return thread + + +class ContentionObservedLock: + """Lock that exposes an acquire attempt made while another caller owns it.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self.contended = threading.Event() + + def __enter__(self) -> ContentionObservedLock: + if self._lock.locked(): + self.contended.set() + self._lock.acquire() + return self + + def __exit__(self, *_args: object) -> None: + self._lock.release() + + +def specification( + name: str, + *, + join_timeout: float = 1.0, + watchdog_monitored: bool = False, +) -> BackgroundServiceSpec: + return BackgroundServiceSpec( + name=name, + thread_name=f"prism-{name}", + target=lambda: None, + daemon=True, + join_timeout=join_timeout, + watchdog_monitored=watchdog_monitored, + ) + + +class BackgroundServiceRegistryTests(unittest.TestCase): + def test_named_start_is_idempotent_and_retains_exact_thread(self) -> None: + registry = BackgroundServiceRegistry( + [specification("poll", watchdog_monitored=True)], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + + first = registry.start("poll") + second = registry.start("poll") + snapshot = registry.snapshot("poll") + + self.assertIs(first, second) + self.assertIs(snapshot.thread, first) + self.assertTrue(snapshot.started) + self.assertEqual(first.start_count, 1) # type: ignore[attr-defined] + self.assertEqual(first.name, "prism-poll") + self.assertTrue(first.daemon) + + def test_drain_threads_are_only_started_services_in_registration_order(self) -> None: + registry = BackgroundServiceRegistry( + [ + specification("poll", join_timeout=1.0), + specification("writer", join_timeout=5.0), + specification("optional", join_timeout=2.0), + ], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + poll = registry.start("poll") + writer = registry.start("writer") + + self.assertEqual( + registry.threads_to_drain(), + ((poll, 1.0), (writer, 5.0)), + ) + + def test_watchdog_names_derive_from_the_same_service_records(self) -> None: + registry = BackgroundServiceRegistry( + [ + specification("poll", watchdog_monitored=True), + specification("health"), + specification("writer", watchdog_monitored=True), + ], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + + registry.start("writer") + + self.assertEqual(registry.watchdog_service_names(), ("poll", "writer")) + self.assertEqual( + registry.watchdog_service_names(started_only=True), + ("writer",), + ) + + def test_duplicate_service_or_thread_names_are_rejected(self) -> None: + registry = BackgroundServiceRegistry([specification("poll")]) + + with self.assertRaisesRegex(ValueError, "already registered: poll"): + registry.register(specification("poll")) + with self.assertRaisesRegex(ValueError, "thread name is already registered"): + registry.register( + BackgroundServiceSpec( + name="other", + thread_name="prism-poll", + target=lambda: None, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + ) + ) + + def test_start_failure_rolls_back_and_retry_runs_start_hook_once(self) -> None: + factory = FlakyThreadFactory() + registry = BackgroundServiceRegistry( + [specification("poll", watchdog_monitored=True)], + thread_factory=factory, + ) + started: list[str] = [] + + with self.assertRaisesRegex(RuntimeError, "thread start failed"): + registry.start( + "poll", + on_started=lambda service: started.append(service.name), + ) + + failed = registry.snapshot("poll") + self.assertFalse(failed.started) + self.assertIsNone(failed.thread) + self.assertEqual(started, []) + + thread = registry.start( + "poll", + on_started=lambda service: started.append(service.name), + ) + + self.assertEqual(factory.attempts, 2) + self.assertIs(thread, factory.threads[1]) + self.assertEqual(started, ["poll"]) + + def test_started_thread_retries_a_failed_start_hook_without_restarting(self) -> None: + registry = BackgroundServiceRegistry( + [specification("poll", watchdog_monitored=True)], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + hook_attempts = 0 + + def flaky_hook(_service: BackgroundServiceSpec) -> None: + nonlocal hook_attempts + hook_attempts += 1 + if hook_attempts == 1: + raise RuntimeError("start hook failed") + + with self.assertRaisesRegex(RuntimeError, "start hook failed"): + registry.start("poll", on_started=flaky_hook) + + started = registry.snapshot("poll") + self.assertTrue(started.started) + self.assertIsNotNone(started.thread) + + retried = registry.start("poll", on_started=flaky_hook) + + self.assertIs(retried, started.thread) + self.assertEqual(retried.start_count, 1) # type: ignore[attr-defined] + self.assertEqual(hook_attempts, 2) + + def test_dynamic_registration_is_equivalent_or_fails_clearly(self) -> None: + def target() -> None: + return None + + registered = specification("dynamic") + registered = BackgroundServiceSpec( + name=registered.name, + thread_name=registered.thread_name, + target=target, + daemon=registered.daemon, + join_timeout=registered.join_timeout, + watchdog_monitored=registered.watchdog_monitored, + registration_identity=("dynamic", 1), + ) + registry = BackgroundServiceRegistry() + + self.assertTrue(registry.register_if_absent(registered)) + self.assertFalse( + registry.register_if_absent( + BackgroundServiceSpec( + name="dynamic", + thread_name="prism-dynamic", + target=lambda: None, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("dynamic", 1), + ) + ) + ) + with self.assertRaisesRegex(ValueError, "incompatible.*dynamic"): + registry.register_if_absent( + BackgroundServiceSpec( + name="dynamic", + thread_name="prism-dynamic", + target=target, + daemon=True, + join_timeout=1.0, + watchdog_monitored=False, + registration_identity=("dynamic", 2), + ) + ) + + def test_post_start_registration_keeps_registration_order_for_drain(self) -> None: + registry = BackgroundServiceRegistry( + [specification("first")], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + first = registry.start("first") + registry.register(specification("second", join_timeout=2.0)) + registry.register(specification("third", join_timeout=3.0)) + third = registry.start("third") + second = registry.start("second") + + self.assertEqual( + registry.threads_to_drain(), + ((first, 1.0), (second, 2.0), (third, 3.0)), + ) + + +class CoordinatorBackgroundServiceIntegrationTests(unittest.TestCase): + @staticmethod + def coordinator_with_optional_services(enabled: bool) -> PrismCoordinator: + server = PrismCoordinator.__new__(PrismCoordinator) + server.blockwait_enabled = enabled + server.vardiff_idle_sweep_seconds = 1.0 if enabled else 0.0 + server.stratum_initial_job_timeout_seconds = 1.0 if enabled else 0.0 + server.ctv_broadcaster_enabled = enabled + server.watchdog_enabled = enabled + server.audit_bind = "127.0.0.1" if enabled else None + server.audit_port = 8080 if enabled else 0 + return server + + def test_optional_process_services_are_absent_when_disabled(self) -> None: + server = self.coordinator_with_optional_services(False) + + registry = server._make_background_service_registry() + + self.assertEqual( + registry.service_names(), + ("qbit_blockpoll", "block_submitter", "share_writer"), + ) + + def test_process_service_specs_preserve_names_and_join_order(self) -> None: + server = self.coordinator_with_optional_services(True) + + registry = server._make_background_service_registry() + + self.assertEqual( + registry.service_names(), + ( + "qbit_blockpoll", + "block_submitter", + "qbit_blockwait", + "vardiff_idle_sweep", + "initial_job_timeout_sweep", + "share_writer", + "ctv_fanout_broadcaster", + "watchdog", + "health_snapshot_refresher", + ), + ) + expected = { + "qbit_blockpoll": ("prism-qbit-block-poll", 1.0, True), + "block_submitter": ("prism-block-submitter", 1.0, True), + "qbit_blockwait": ("prism-qbit-block-wait", 1.0, True), + "vardiff_idle_sweep": ("prism-vardiff-idle-sweep", 1.0, True), + "initial_job_timeout_sweep": ("prism-initial-job-timeouts", 1.0, False), + "share_writer": ("prism-share-writer", 5.0, True), + "ctv_fanout_broadcaster": ( + "prism-ctv-fanout-broadcaster", + 1.0, + True, + ), + "watchdog": ("prism-watchdog", 1.0, False), + "health_snapshot_refresher": ( + "prism-health-snapshot-refresher", + 1.0, + False, + ), + } + for name, properties in expected.items(): + service = registry.snapshot(name).specification + self.assertEqual( + (service.thread_name, service.join_timeout, service.watchdog_monitored), + properties, + ) + + def test_monitored_service_start_seeds_its_own_watchdog_key(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server._background_services = BackgroundServiceRegistry( + [specification("tracked", watchdog_monitored=True)], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + + server._start_background_service("tracked") + + self.assertEqual(tuple(server._heartbeats), ("tracked",)) + server._heartbeats["tracked"] = 1.0 + server._start_background_service("tracked") + self.assertEqual(server._heartbeats["tracked"], 1.0) + + def test_concurrent_wrapper_starts_seed_one_heartbeat_and_one_thread(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server._background_services = BackgroundServiceRegistry( + [specification("tracked", watchdog_monitored=True)], + thread_factory=DormantThread, # type: ignore[arg-type] + ) + observed_registry_lock = ContentionObservedLock() + server._background_services._lock = observed_registry_lock # type: ignore[assignment] + heartbeat_entered = threading.Event() + release_heartbeat = threading.Event() + heartbeat_names: list[str] = [] + results: list[object] = [] + errors: list[BaseException] = [] + + def blocked_heartbeat(name: str) -> None: + heartbeat_names.append(name) + heartbeat_entered.set() + if not release_heartbeat.wait(5): + raise AssertionError("heartbeat test interleaving timed out") + + server._record_heartbeat = blocked_heartbeat # type: ignore[method-assign] + + def start() -> None: + try: + results.append(server._start_background_service("tracked")) + except BaseException as exc: + errors.append(exc) + + first = threading.Thread(target=start) + second = threading.Thread(target=start) + first.start() + self.assertTrue(heartbeat_entered.wait(5)) + second.start() + second_contended_inside_start = observed_registry_lock.contended.wait(5) + release_heartbeat.set() + first.join(5) + second.join(5) + + self.assertFalse(first.is_alive()) + self.assertFalse(second.is_alive()) + self.assertTrue(second_contended_inside_start) + self.assertEqual(errors, []) + self.assertEqual(heartbeat_names, ["tracked"]) + self.assertEqual(len(results), 2) + self.assertIs(results[0], results[1]) + self.assertEqual(results[0].start_count, 1) # type: ignore[union-attr] + + def test_wrapper_start_failure_rolls_back_heartbeat_and_retries_cleanly(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + factory = FlakyThreadFactory() + server._background_services = BackgroundServiceRegistry( + [specification("tracked", watchdog_monitored=True)], + thread_factory=factory, + ) + + with self.assertRaisesRegex(RuntimeError, "thread start failed"): + server._start_background_service("tracked") + + self.assertEqual(server._heartbeats, {}) + self.assertFalse(server._background_services.snapshot("tracked").started) + + thread = server._start_background_service("tracked") + + self.assertIs(thread, factory.threads[1]) + self.assertEqual(tuple(server._heartbeats), ("tracked",)) + self.assertTrue(server._background_services.snapshot("tracked").started) + + def test_concurrent_secondary_starts_register_and_start_once(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server._background_services = BackgroundServiceRegistry( + thread_factory=DormantThread, # type: ignore[arg-type] + ) + profile = SimpleNamespace( + heartbeat_name="stratum_accept_highdiff", + name="highdiff", + ) + listener = SimpleNamespace() + registration_barrier = threading.Barrier(2) + original_register = server._background_services.register_if_absent + + def synchronized_register(service: BackgroundServiceSpec) -> bool: + registration_barrier.wait(timeout=5) + return original_register(service) + + server._background_services.register_if_absent = synchronized_register # type: ignore[method-assign] + heartbeat_names: list[str] = [] + server._record_heartbeat = heartbeat_names.append # type: ignore[method-assign] + results: list[object] = [] + errors: list[BaseException] = [] + + def start() -> None: + try: + results.append( + server._start_secondary_accept_service( # type: ignore[arg-type] + listener, + profile, + ) + ) + except BaseException as exc: + errors.append(exc) + + callers = [threading.Thread(target=start) for _ in range(2)] + for caller in callers: + caller.start() + for caller in callers: + caller.join(5) + + self.assertTrue(all(not caller.is_alive() for caller in callers)) + self.assertEqual(errors, []) + self.assertEqual(heartbeat_names, ["stratum_accept_highdiff"]) + self.assertEqual(len(results), 2) + self.assertIs(results[0], results[1]) + self.assertEqual(results[0].start_count, 1) # type: ignore[union-attr] + self.assertEqual( + server._background_services.service_names(), + ("stratum_accept_highdiff",), + ) + + def test_secondary_listener_is_named_monitored_and_bounded_for_drain(self) -> None: + server = self.coordinator_with_optional_services(False) + server._heartbeats = {} + server._watchdog_pauses = {} + server._heartbeats_lock = threading.Lock() + server._background_services = BackgroundServiceRegistry( + thread_factory=DormantThread, # type: ignore[arg-type] + ) + profile = SimpleNamespace( + heartbeat_name="stratum_accept_highdiff", + name="highdiff", + ) + + thread = server._start_secondary_accept_service( # type: ignore[arg-type] + SimpleNamespace(), + profile, + ) + + snapshot = server._background_services.snapshot("stratum_accept_highdiff") + self.assertEqual(thread.name, "prism-stratum-accept-highdiff") + self.assertTrue(snapshot.specification.watchdog_monitored) + self.assertEqual(snapshot.specification.join_timeout, 1.0) + self.assertEqual(tuple(server._heartbeats), ("stratum_accept_highdiff",)) + self.assertEqual( + server._background_services.threads_to_drain(), + ((thread, 1.0),), + ) + + def test_health_loop_always_clears_running_flag(self) -> None: + server = self.coordinator_with_optional_services(False) + server.stop_event = threading.Event() + server.stop_event.set() + server._health_snapshot_lock = threading.RLock() + server._health_refresh_loop_running = True + + server.health_snapshot_loop() + + self.assertFalse(server._health_refresh_loop_running) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_block_candidates.py b/tests/test_prism_block_candidates.py index 25e2a58..0c3d62e 100644 --- a/tests/test_prism_block_candidates.py +++ b/tests/test_prism_block_candidates.py @@ -4,6 +4,8 @@ from __future__ import annotations +import contextlib +from dataclasses import replace as dataclass_replace import unittest from tests.prism_vardiff_test_support import * @@ -25,7 +27,7 @@ def test_build_audit_bundle_passes_pool_fee_policy_to_cli_payload(self) -> None: }, clear=True, ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", fake_audit_bundle_popen(captured), ): bundle = server.build_audit_bundle( @@ -60,7 +62,7 @@ def test_build_audit_bundle_passes_ctv_settlement_config_to_cli_payload(self) -> }, clear=True, ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", fake_audit_bundle_popen(captured), ): bundle = server.build_audit_bundle( @@ -95,7 +97,7 @@ def test_build_audit_bundle_preserves_exact_canonical_output_file(self) -> None: captured: dict[str, object] = {} with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", fake_audit_bundle_popen( captured, output_text=canonical_bytes.decode("utf-8"), @@ -131,7 +133,7 @@ def test_build_audit_bundle_summary_only_requests_and_parses_job_summary(self) - captured: dict[str, object] = {} with patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", fake_audit_bundle_popen( captured, output_text=json.dumps(summary, separators=(",", ":")), @@ -153,6 +155,53 @@ def test_build_audit_bundle_summary_only_requests_and_parses_job_summary(self) - self.assertEqual(set(bundle), {"found_block", "signed_coinbase_manifest"}) self.assertIn("--job-summary-output", captured["cmd"]) self.assertNotIn("--canonical-output", captured["cmd"]) + + def test_summary_build_records_serialization_phase_once(self) -> None: + server = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + observed: list[tuple[str, float]] = [] + service = server._ensure_job_bundle_service() + service._phase_local.tip_refresh_metrics = True + compiler = server._ensure_bundle_compiler() + compiler._ports = dataclass_replace( + compiler._ports, + record_tip_refresh_phase=( + lambda phase, elapsed: observed.append((phase, elapsed)) + ), + ) + phase_metrics = { + "phases_seconds": {}, + "input_deserialization_seconds": 0.25, + "output_serialization_seconds": 0.5, + } + + with patch( + "lab.prism.bundle_compiler.subprocess.Popen", + fake_audit_bundle_popen( + {}, + stderr_text=( + "qbit-prism-build-phase-metrics " + + json.dumps(phase_metrics, separators=(",", ":")) + ), + ), + ): + server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + summary_only=True, + ) + + serialization = [ + elapsed for phase, elapsed in observed if phase == "serialization_copy" + ] + self.assertEqual(len(serialization), 1) + self.assertGreaterEqual(serialization[0], 0.75) def test_build_audit_bundle_removes_partial_output_after_builder_failure(self) -> None: server = coordinator() server.signing_seed_hex = "42" * 32 @@ -162,7 +211,7 @@ def test_build_audit_bundle_removes_partial_output_after_builder_failure(self) - with tempfile.TemporaryDirectory() as tmp: output_path = Path(tmp) / "candidate.audit.json" with patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", fake_audit_bundle_popen( captured, output_text='{"partial":', @@ -179,10 +228,10 @@ def test_build_audit_bundle_removes_partial_output_after_builder_failure(self) - canonical_output_path=output_path, ) self.assertFalse(output_path.exists()) - self.assertEqual(server.job_build_worker_counts["crashes"], 1) + self.assertEqual(server._ensure_job_bundle_service()._worker_counts["crashes"], 1) with patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", fake_audit_bundle_popen(captured), ): recovered = server.build_audit_bundle( @@ -193,7 +242,7 @@ def test_build_audit_bundle_removes_partial_output_after_builder_failure(self) - ) self.assertEqual(recovered, {"ok": True}) - self.assertEqual(server.job_build_worker_counts["restarts"], 1) + self.assertEqual(server._ensure_job_bundle_service()._worker_counts["restarts"], 1) def test_build_audit_bundle_recovers_after_cancelled_worker_timeout(self) -> None: server = coordinator() server.signing_seed_hex = "42" * 32 @@ -252,7 +301,7 @@ def wait(self, timeout: float | None = None) -> int: "coinbase_script_sig_suffix_hex": "00", } with patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", HungThenHealthyPopen, ): with self.assertRaisesRegex(JobBuildCancelled, "timeout"): @@ -267,15 +316,15 @@ def wait(self, timeout: float | None = None) -> int: self.assertEqual(recovered, {"ok": True}) self.assertEqual(process_calls, 2) - self.assertEqual(server.job_build_worker_counts["terminations"], 1) - self.assertEqual(server.job_build_worker_counts["restarts"], 1) + self.assertEqual(server._ensure_job_bundle_service()._worker_counts["terminations"], 1) + self.assertEqual(server._ensure_job_bundle_service()._worker_counts["restarts"], 1) def test_build_audit_bundle_does_not_unlink_preexisting_output_path(self) -> None: server = coordinator() server.signing_seed_hex = "42" * 32 server.ledger_attestation_signing_seed_hex = "43" * 32 with tempfile.TemporaryDirectory() as tmp, patch( - "lab.prism.prism_coordinator.subprocess.Popen" + "lab.prism.bundle_compiler.subprocess.Popen" ) as popen: output_path = Path(tmp) / "preexisting-candidate.audit.json" output_path.write_bytes(b"do-not-clobber") @@ -445,7 +494,7 @@ def active_descendant_call( server._block_candidate_outcome.reason, PRISM_REJECTION_LEDGER_CONFIRMATION_FAILED, ) - transition = server._accepted_block_payout_previews[descendant_hash] + transition = server._payout_state_service._previews[descendant_hash] self.assertTrue(transition.landed) self.assertIsNone(transition.preview) self.assertFalse(server.stop_event.is_set()) @@ -898,11 +947,11 @@ def terminal(_candidate: PrismBlockCandidate) -> bool: self.assertEqual(ledger.pending_block_candidates(), []) self.assertNotIn( candidate.submission.block_hash_hex, - server._accepted_block_payout_previews, + server._payout_state_service._previews, ) self.assertNotIn( candidate.submission.block_hash_hex, - server._invalidated_accepted_block_payout_previews, + server._payout_state_service._invalidated_previews, ) def test_terminal_abandonment_keeps_tombstone_when_outbox_update_fails( self, @@ -911,21 +960,25 @@ def test_terminal_abandonment_keeps_tombstone_when_outbox_update_fails( ledger = SingleWriterShareLedger() server.ledger = ledger pending = self._pending_append("terminal-outbox-failure").pending_share - candidate = block_candidate( - server, - state, - SimpleNamespace( - coinbase_tx_hex="00", - block_hash_hex="a3" * 32, - block_hex="00", - share_pass=True, - block_pass=True, + candidate = dataclass_replace( + block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="a3" * 32, + block_hex="00", + share_pass=False, + block_pass=True, + ), + pending_share=pending, ), - pending_share=pending, + credit_share_on_accept=True, ) block_hash = candidate.submission.block_hash_hex block_height = int(candidate.context.template["height"]) ledger.append_batch([(pending, server.block_candidate_intent(candidate))]) + server._ensure_share_writer_service().adopt_pending_share(pending) def terminal(_candidate: PrismBlockCandidate) -> bool: server._begin_accepted_block_payout_preview( @@ -952,12 +1005,21 @@ def terminal(_candidate: PrismBlockCandidate) -> bool: ): self.assertTrue(server.submit_next_block_candidate()) - self.assertNotIn(block_hash, server._accepted_block_payout_previews) - self.assertIn(block_hash, server._invalidated_accepted_block_payout_previews) + self.assertNotIn(block_hash, server._payout_state_service._previews) + self.assertIn(block_hash, server._payout_state_service._invalidated_previews) self.assertEqual( [intent["block_hash_hex"] for intent in ledger.pending_block_candidates()], [block_hash], ) + self.assertEqual(server._pending_share_commit_floor, {}) + + # Once the same durable row terminalizes normally, the reconstructed + # credit source is no longer reachable and the S3 floor is released. + server.enqueue_block_candidate(candidate) + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertEqual(server._pending_share_commit_floor, {}) + def test_finalize_failure_replays_with_candidate_backoff(self) -> None: server, state, _recording = submit_coordinator() ledger = SingleWriterShareLedger() @@ -979,9 +1041,6 @@ def test_finalize_failure_replays_with_candidate_backoff(self) -> None: ledger.append_batch([(pending, candidate_intent)]) server.block_candidate_retry_initial_seconds = 0.1 server.block_candidate_retry_max_seconds = 0.4 - # The block itself lands once; only the terminal outbox update fails. - # Replays must pace like any other candidate retry AND run finalize- - # only, never re-entering submission. submit_calls = 0 def accepting_submit(_candidate: PrismBlockCandidate) -> bool: @@ -1026,7 +1085,10 @@ def flaky_finish(*, block_hash: str) -> bool: self.assertAlmostEqual(observed, expected) self.assertEqual(finish_attempts, 5) self.assertEqual(submit_calls, 1) - self.assertNotIn(candidate.submission.block_hash_hex, server.block_candidate_retry_delays) + self.assertNotIn( + candidate.submission.block_hash_hex, + server.block_candidate_retry_delays, + ) self.assertEqual(server.block_candidate_abandoned_counts, {}) self.assertEqual(ledger.pending_block_candidates(), []) self.assertEqual(server._block_candidate_finalize_retries, {}) @@ -1092,7 +1154,6 @@ def flaky_finish(*, block_hash: str, error: str) -> bool: self.assertEqual(waits, [0.1, 0.2]) self.assertEqual(submit_calls, 1) self.assertEqual(finish_attempts, 3) - # Finalize-only replays must not recount the terminal abandonment. self.assertEqual( server.block_candidate_abandoned_counts, {PRISM_REJECTION_STALE_JOB: 1}, @@ -1131,13 +1192,13 @@ def failing_finish(*, block_hash: str) -> bool: lambda client, **_kwargs: refreshed_clients.append(client) or 0 ) released_shares: list[object] = [] - original_release = server._finish_pending_share_commit + original_release = server._finish_pending_share_candidate def recording_release(pending_share: object) -> None: released_shares.append(pending_share) original_release(pending_share) - server._finish_pending_share_commit = recording_release # type: ignore[method-assign] + server._finish_pending_share_candidate = recording_release # type: ignore[method-assign] waits: list[float] = [] with patch.object( server.stop_event, @@ -1146,17 +1207,12 @@ def recording_release(pending_share: object) -> None: ): server.enqueue_block_candidate(candidate) self.assertTrue(server.submit_next_block_candidate()) - # The block is active on-chain; the fleet refresh fires on the - # first finalize failure without waiting out a backoff, and the - # snapshot anchor floor is released despite the pending outbox - # mark. self.assertEqual(refreshed_clients, [state]) self.assertEqual(waits, []) self.assertIn(pending, released_shares) self.assertTrue(server.submit_next_block_candidate()) self.assertEqual(refreshed_clients, [state]) self.assertEqual(waits, [0.1]) - def test_invalid_durable_candidate_is_quarantined_by_outbox_row_key(self) -> None: for payload_hash in (None, "ff" * 32): with self.subTest(payload_hash=payload_hash): @@ -1185,6 +1241,158 @@ def test_invalid_durable_candidate_is_quarantined_by_outbox_row_key(self) -> Non "qbit_prism_block_candidate_poisoned_total 1", server.metrics_payload(), ) + + def test_credit_replay_preview_failure_releases_floor_only_after_quarantine(self) -> None: + for quarantine_fails in (False, True): + with self.subTest(quarantine_fails=quarantine_fails): + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + block_hash = "93" * 32 + pending = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=123, + ntime=1, + ) + candidate = dataclass_replace( + block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=False, + block_pass=True, + ), + pending_share=pending, + ), + credit_share_on_accept=True, + ) + ledger.persist_block_candidate_intent( + server.block_candidate_intent(candidate) + ) + + quarantine = ( + patch.object( + ledger, + "mark_block_candidate_abandoned", + side_effect=RuntimeError("terminal update failed"), + ) + if quarantine_fails + else contextlib.nullcontext() + ) + with patch.object( + server, + "_begin_accepted_block_payout_preview", + side_effect=RuntimeError("preview failed"), + ), quarantine: + self.assertEqual(server.replay_pending_block_candidates(), 0) + + if quarantine_fails: + self.assertEqual(len(ledger.pending_block_candidates()), 1) + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 122) + else: + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertEqual(server._pending_share_commit_floor, {}) + + def test_credit_append_failure_never_drops_durable_floor_before_retry_adoption( + self, + ) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + pending = PendingShare( + share_id="miner-a:" + "94" * 32, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=123, + ntime=1, + ) + candidate = dataclass_replace( + block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="94" * 32, + block_hex="00", + share_pass=False, + block_pass=True, + ), + pending_share=pending, + ), + credit_share_on_accept=True, + ) + ledger.persist_block_candidate_intent(server.block_candidate_intent(candidate)) + writer = server._ensure_share_writer_service() + writer.adopt_pending_share(pending) + + append_entered = threading.Event() + release_append = threading.Event() + readoption_entered = threading.Event() + release_readoption = threading.Event() + + def failing_append(_entries: object) -> list[object]: + append_entered.set() + release_append.wait(timeout=2) + raise RuntimeError("share commit failed") + + original_adopt = writer.adopt_pending_share + + def blocking_readopt(value: PendingShare) -> None: + readoption_entered.set() + release_readoption.wait(timeout=2) + original_adopt(value) + + def credit_then_fail(value: PrismBlockCandidate) -> bool: + server.append_accepted_share( + value.client, + value.context, + value.submission, + value.pending_share, + candidate_intent=server.block_candidate_intent(value), + ) + return True + + ledger.append_batch = failing_append # type: ignore[method-assign] + writer.adopt_pending_share = blocking_readopt # type: ignore[method-assign] + server.submit_block_candidate = credit_then_fail # type: ignore[method-assign] + thread = threading.Thread( + target=server._submit_next_block_candidate_writer, + args=(candidate,), + ) + thread.start() + self.assertTrue(append_entered.wait(timeout=1)) + self.assertIn(pending.share_id, server._pending_share_commit_floor) + release_append.set() + self.assertTrue(readoption_entered.wait(timeout=1)) + + # S3's failed append has completed its attempt-only cleanup, while the + # caller is deliberately stopped before retry adoption. The preexisting + # durable holder must still cover this entire interval. + self.assertIn(pending.share_id, server._pending_share_commit_floor) + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 122) + release_readoption.set() + thread.join(timeout=2) + self.assertFalse(thread.is_alive()) + self.assertIs(server._retry_block_candidate, candidate) + server._finish_pending_share_commit(pending) + self.assertEqual(server._pending_share_commit_floor, {}) def test_block_submitter_drops_candidate_when_pool_closed(self) -> None: server, state, ledger = submit_coordinator() server.accepted_block_count = 1 @@ -1235,9 +1443,9 @@ def test_block_candidate_submits_before_full_audit_persistence(self) -> None: server._ensure_job_cache_state() server._ensure_tip_refresh_state() stale_bundle_key = ("stale-payout-bundle",) - server._job_bundle_cache[stale_bundle_key] = object() # type: ignore[assignment] + server._ensure_job_bundle_service()._bundle_cache[stale_bundle_key] = object() # type: ignore[assignment] active_fanout = _FanoutCancellation() - server._active_tip_refresh = ( # type: ignore[assignment] + server._ensure_tip_refresh_service().seed_active_refresh_for_test( SimpleNamespace(payout_state_generation=0), active_fanout, ) @@ -1354,10 +1562,10 @@ def persist_with_canonicalization(**kwargs: object) -> dict[str, object]: # path. The verified content is valid, but the path is never claimed as # a byte-canonical artifact for ledger persistence. self.assertIsNone(ledger.persisted[0]["canonical_bundle_path"]) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._job_bundle_cache, {}) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertEqual(server._ensure_job_bundle_service()._bundle_cache, {}) self.assertTrue(active_fanout.is_set()) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_issued_preview_is_invalidated_when_final_coinbase_mismatches(self) -> None: server, state, ledger = submit_coordinator() server._ensure_job_cache_state() @@ -1378,8 +1586,8 @@ def test_issued_preview_is_invalidated_when_final_coinbase_mismatches(self) -> N server.audit_dir = Path(tempdir) self.assertFalse(server.submit_block_candidate(candidate)) - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._accepted_block_payout_previews, {}) + self.assertEqual(server._payout_state_service._generation, 2) + self.assertEqual(server._payout_state_service._previews, {}) self.assertEqual(ledger.persisted, []) self.assertEqual( server._block_candidate_outcome.reason, @@ -1443,7 +1651,7 @@ def mark_pool_block_inactive(**_kwargs: object) -> dict[str, object]: delivery_admitted = threading.Event() delivery_thread: threading.Thread | None = None reconcile_thread: threading.Thread | None = None - mutation_lock = server._payout_balance_mutation_lock + mutation_lock = server._payout_state_service._balance_mutation_lock class ObservedBalanceLock: def __enter__(self) -> ObservedBalanceLock: @@ -1455,7 +1663,7 @@ def __enter__(self) -> ObservedBalanceLock: def __exit__(self, *_args: object) -> None: mutation_lock.release() - server._payout_balance_mutation_lock = ObservedBalanceLock() # type: ignore[assignment] + server._payout_state_service._balance_mutation_lock = ObservedBalanceLock() # type: ignore[assignment] with tempfile.TemporaryDirectory() as tempdir: server.audit_dir = Path(tempdir) @@ -1488,7 +1696,7 @@ def submit() -> None: def deliver() -> None: try: - with server._payout_state_delivery_gate.delivery(): + with server._payout_state_service._delivery_gate.delivery(): note("replacement-delivery") delivery_admitted.set() if not release_delivery.wait(15): @@ -1549,7 +1757,7 @@ def reconcile() -> None: self.assertEqual(accepted, [True]) self.assertEqual(len(reconcile_results), 1) self.assertEqual(reconcile_results[0]["inactive_blocks"], 1) - self.assertEqual(server._payout_state_generation, 2) + self.assertEqual(server._payout_state_service._generation, 2) self.assertLess(events.index("persist-start"), events.index("replacement-delivery")) self.assertLess(events.index("replacement-delivery"), events.index("persist-end")) self.assertLess(events.index("persist-end"), events.index("confirm")) @@ -1565,7 +1773,7 @@ def test_next_tip_preview_job_lands_after_parent_persistence(self) -> None: server.stop_after_block = False server.max_blocks = 10 server.clients = {state} - server.refresh_jobs_after_accepted_block = ( # type: ignore[method-assign] + server._ensure_tip_refresh_service().refresh_after_accepted_block = ( # type: ignore[method-assign] lambda **_kwargs: None ) build_started = threading.Event() @@ -1681,7 +1889,7 @@ def build_child_job(client: ClientState, *, clean_jobs: bool) -> object: ) context.template["height"] = 11 context.prior_balances = server._prior_balances_for_job_parent(parent_hash) - context.payout_state_generation = server._payout_state_generation + context.payout_state_generation = server._payout_state_service._generation return context server.build_job_for_client = build_child_job # type: ignore[method-assign] @@ -1720,9 +1928,9 @@ def submit_parent() -> None: self.assertEqual(child_context.prior_balances, preview) self.assertEqual( child_context.payout_state_generation, - server._payout_state_generation, + server._payout_state_service._generation, ) - preview_generation = server._payout_state_generation + preview_generation = server._payout_state_service._generation self.assertEqual(sent[-1]["method"], "mining.notify") self.assertTrue(sent[-1]["params"][8]) # type: ignore[index] @@ -1758,7 +1966,7 @@ def submit_parent() -> None: raise parent_errors[0] self.assertEqual(parent_results, [True]) self.assertEqual(ledger.current_prior_balances(), preview) - self.assertEqual(server._payout_state_generation, preview_generation) + self.assertEqual(server._payout_state_service._generation, preview_generation) self.assertTrue(server.submit_next_block_candidate()) self.assertEqual(rpc.submitted, [parent_hash, child_hash]) @@ -1769,7 +1977,7 @@ def submit_parent() -> None: server.block_candidate_abandoned_counts.get(PRISM_REJECTION_STALE_JOB, 0), 0, ) - self.assertEqual(server._accepted_block_payout_previews, {}) + self.assertEqual(server._payout_state_service._previews, {}) def test_direct_block_preparation_does_not_hold_delivery_gate(self) -> None: server, state, ledger = submit_coordinator() server._ensure_job_cache_state() @@ -1782,7 +1990,7 @@ def test_direct_block_preparation_does_not_hold_delivery_gate(self) -> None: def blocking_persist(**kwargs: object) -> dict[str, object]: self.assertIsNone( - server._payout_state_delivery_gate._mutation_owner + server._payout_state_service._delivery_gate._mutation_owner ) entered.set() if not release.wait(5): @@ -1825,9 +2033,9 @@ def submit() -> None: thread.start() try: self.assertTrue(entered.wait(5)) - with server._payout_state_delivery_gate.delivery_cancelable( + with server._payout_state_service._delivery_gate.delivery_cancelable( lambda: False, - generation=server._payout_state_generation, + generation=server._payout_state_service._generation, priority=True, ) as admission: self.assertTrue(admission) @@ -1835,9 +2043,9 @@ def submit() -> None: thread.join(5) self.assertFalse(thread.is_alive()) self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) + server._payout_state_service._prepare_lock.acquire(timeout=1) ) - server._payout_state_prepare_lock.release() + server._payout_state_service._prepare_lock.release() admission.mark_delivered() finally: release.set() @@ -1846,14 +2054,14 @@ def submit() -> None: self.assertFalse(thread.is_alive()) self.assertEqual(errors, []) self.assertEqual(accepted, [True]) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service._generation, 1) def test_failed_direct_block_audit_does_not_reserve_payout_source(self) -> None: for failure_phase in ("build", "verify"): with self.subTest(failure_phase=failure_phase): server, state, ledger = submit_coordinator() server._ensure_job_cache_state() - initial_source = server._payout_state_source - initial_published = server._published_payout_state + initial_source = server._payout_state_service._source + initial_published = server._payout_state_service._published block_hash = "ce" * 32 server.rpc = SubmitRpc( tip="00" * 32, @@ -1893,9 +2101,9 @@ def test_failed_direct_block_audit_does_not_reserve_payout_source(self) -> None: block_candidate(server, state, submission) ) - self.assertEqual(server._payout_state_source, initial_source) - self.assertEqual(server._published_payout_state, initial_published) - self.assertEqual(server._payout_state_generation, 0) + self.assertEqual(server._payout_state_service._source, initial_source) + self.assertEqual(server._payout_state_service._published, initial_published) + self.assertEqual(server._payout_state_service._generation, 0) self.assertEqual(ledger.persisted, []) def test_uncertain_direct_block_ledger_commit_fences_delivery(self) -> None: server, state, ledger = submit_coordinator() @@ -1937,11 +2145,11 @@ def test_uncertain_direct_block_ledger_commit_fences_delivery(self) -> None: self.assertEqual(len(ledger.persisted), 1) # The prospective preview was source/generation 1; the uncertain # durable commit supersedes it with fenced source 2. - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 2) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertEqual(server._payout_state_service._source[0], 2) + self.assertEqual(server._payout_state_service._published.source_generation, 1) + self.assertTrue(server._payout_state_service._publication_blocked) + self.assertTrue(server._payout_state_service._delivery_gate._delivery_blocked) def test_uncertain_commit_supersedes_concurrently_published_source(self) -> None: server, state, ledger = submit_coordinator() server._ensure_job_cache_state() @@ -1990,16 +2198,16 @@ def publish_newer_source_then_fail(**_kwargs: object) -> dict[str, object]: block_candidate(server, state, submission) ) - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._published_payout_state.source_generation, 2) - self.assertEqual(server._payout_state_source[0], 3) - self.assertEqual(server._payout_state_source[1], newer_tip) + self.assertEqual(server._payout_state_service._generation, 2) + self.assertEqual(server._payout_state_service._published.source_generation, 2) + self.assertEqual(server._payout_state_service._source[0], 3) + self.assertEqual(server._payout_state_service._source[1], newer_tip) self.assertEqual( - server._payout_state_source[2], + server._payout_state_service._source[2], "direct_block_uncertain", ) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertTrue(server._payout_state_service._publication_blocked) + self.assertTrue(server._payout_state_service._delivery_gate._delivery_blocked) def test_post_confirm_publication_loss_completes_candidate_and_fences(self) -> None: # Once persist + confirm are durable, losing the forced payout # publication must not abort the candidate: the outbox row is marked @@ -2036,7 +2244,8 @@ def confirm_then_reserve_newer_source(**kwargs: object) -> dict[str, object]: return result ledger.confirm_accepted_block = confirm_then_reserve_newer_source # type: ignore[method-assign] - server._publish_current_payout_state_with_retry_budget = ( # type: ignore[method-assign] + service = server._payout_state_service + service.publish_current_with_retry_budget = ( # type: ignore[method-assign] lambda **_kwargs: None ) submission = SimpleNamespace( @@ -2061,16 +2270,16 @@ def confirm_then_reserve_newer_source(**kwargs: object) -> dict[str, object]: self.assertEqual(submitted[0]["block_hash"], block_hash) self.assertEqual(server.accepted_block_count, 1) self.assertIn(block_hash, server._accounted_accepted_block_hashes) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertTrue(server._payout_state_service._publication_blocked) + self.assertTrue(server._payout_state_service._delivery_gate._delivery_blocked) self.assertTrue(server.tip_refresh_is_pending()) # The scheduled refresh publishes the pending source and reopens # delivery without any candidate replay. - del server._publish_current_payout_state_with_retry_budget + del service.publish_current_with_retry_budget self.assertIsNotNone(server._publish_current_payout_state_with_retry_budget()) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) + self.assertFalse(server._payout_state_service._publication_blocked) + self.assertFalse(server._payout_state_service._delivery_gate._delivery_blocked) def test_idempotent_direct_block_replay_skips_publication(self) -> None: server, state, ledger = submit_coordinator() server._ensure_job_cache_state() @@ -2104,12 +2313,12 @@ def test_idempotent_direct_block_replay_skips_publication(self) -> None: block_candidate(server, state, submission) ) ) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service._generation, 1) self.assertEqual( - server._published_payout_state.source_tip_hash, + server._payout_state_service._published.source_tip_hash, block_hash, ) - self.assertEqual(server._payout_state_source[0], 1) + self.assertEqual(server._payout_state_service._source[0], 1) # Replay the durable candidate after its block landed, its # confirmation committed, and the network built on top of it. @@ -2160,9 +2369,9 @@ def count_retry() -> None: server._schedule_tip_refresh_retry = count_retry # type: ignore[method-assign] cache_key = ("sentinel",) - server._job_bundle_cache[cache_key] = object() - discarded_before = server.payout_state_candidates_discarded - pending_marks_before = server._tip_refresh_pending_counter + server._ensure_job_bundle_service()._bundle_cache[cache_key] = object() + discarded_before = server._payout_state_service.metrics_snapshot()["discarded_candidates"] + pending_marks_before = server._ensure_tip_refresh_service().snapshot().pending_counter self.assertTrue( server.submit_block_candidate( @@ -2170,22 +2379,22 @@ def count_retry() -> None: ) ) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, block_hash) - self.assertEqual(server._payout_state_source[0], 1) - self.assertIn(cache_key, server._job_bundle_cache) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertEqual(server._payout_state_service._published.source_generation, 1) + self.assertEqual(server._payout_state_service._published.source_tip_hash, block_hash) + self.assertEqual(server._payout_state_service._source[0], 1) + self.assertIn(cache_key, server._ensure_job_bundle_service()._bundle_cache) self.assertEqual(retry_calls, 0) self.assertEqual( - server._tip_refresh_pending_counter, + server._ensure_tip_refresh_service().snapshot().pending_counter, pending_marks_before, ) self.assertEqual( - server.payout_state_candidates_discarded, + server._payout_state_service.metrics_snapshot()["discarded_candidates"], discarded_before, ) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) + self.assertFalse(server._payout_state_service._publication_blocked) + self.assertFalse(server._payout_state_service._delivery_gate._delivery_blocked) def test_leaked_publication_fence_replay_republishes(self) -> None: server, state, ledger = submit_coordinator() server._ensure_job_cache_state() @@ -2218,7 +2427,7 @@ def test_leaked_publication_fence_replay_republishes(self) -> None: block_candidate(server, state, submission) ) ) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service._generation, 1) # Simulate the exception tail a replay must heal: a prior attempt # force-blocked delivery while its source generation already @@ -2227,10 +2436,10 @@ def test_leaked_publication_fence_replay_republishes(self) -> None: # lands, so a confirmed_count=0 replay must not take the covered # skip here. server._block_payout_state_publication(force=True) - self.assertTrue(server._payout_state_publication_blocked) + self.assertTrue(server._payout_state_service._publication_blocked) self.assertEqual( - server._payout_state_source[0], - server._published_payout_state.source_generation, + server._payout_state_service._source[0], + server._payout_state_service._published.source_generation, ) child_tip = "d9" * 32 @@ -2271,19 +2480,20 @@ def call(self, method: str, params: object = None) -> object: ) ) - self.assertEqual(server._payout_state_generation, 2) + self.assertEqual(server._payout_state_service._generation, 2) # Fence healing republishes identical covered state. It advances the # delivery generation without inventing a logical invalidation source. - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) - self.assertFalse(server._payout_state_publication_blocked) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) + self.assertEqual(server._payout_state_service._published.source_generation, 1) + self.assertEqual(server._payout_state_service._source[0], 1) + self.assertFalse(server._payout_state_service._publication_blocked) + self.assertFalse(server._payout_state_service._delivery_gate._delivery_blocked) def test_direct_block_disabled_reconciler_bounds_publish_supersession(self) -> None: server, state, ledger = submit_coordinator() server._ensure_job_cache_state() - server.payout_reconcile_supersession_retries = 2 + service = server._payout_state_service + service.set_reconcile_retries_for_test(2) block_hash = "d3" * 32 - real_publish = server._publish_payout_state_candidate + real_publish = service.publish_candidate publish_attempts = 0 def supersede_before_publish(candidate: object) -> int | None: @@ -2295,7 +2505,7 @@ def supersede_before_publish(candidate: object) -> int | None: ) return real_publish(candidate) # type: ignore[arg-type] - server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] + service.publish_candidate = supersede_before_publish # type: ignore[method-assign] with tempfile.TemporaryDirectory() as tempdir: server.audit_dir = Path(tempdir) server.evidence_path = Path(tempdir) / "evidence.json" @@ -2323,9 +2533,9 @@ def supersede_before_publish(candidate: object) -> int | None: self.assertTrue(accepted) self.assertEqual(publish_attempts, 3) - self.assertEqual(server._payout_state_generation, 0) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertEqual(server._payout_state_service._generation, 0) + self.assertTrue(server._payout_state_service._publication_blocked) + self.assertTrue(server._payout_state_service._delivery_gate._delivery_blocked) self.assertTrue(server.tip_refresh_is_pending()) def test_untrusted_direct_block_reconcile_publishes_newer_source_once(self) -> None: server, state, ledger = submit_coordinator() @@ -2370,9 +2580,9 @@ def superseding_noop_confirmation(**_kwargs: object) -> dict[str, object]: ) self.assertTrue(accepted) - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._published_payout_state.source_tip_hash, newer_tip) - self.assertEqual(server.payout_state_candidates_discarded, 0) + self.assertEqual(server._payout_state_service._generation, 2) + self.assertEqual(server._payout_state_service._published.source_tip_hash, newer_tip) + self.assertEqual(server._payout_state_service.metrics_snapshot()["discarded_candidates"], 0) self.assertEqual(server.reorg_reconcile_skip_count, 1) def test_verified_canonical_bundle_path_requires_exact_bytes(self) -> None: with tempfile.TemporaryDirectory() as tempdir: @@ -2399,7 +2609,7 @@ def test_active_ancestor_candidate_resumes_full_finalization_without_resubmit(se server.max_blocks = 10 server.stop_after_block = False refreshes: list[str] = [] - server.refresh_jobs_after_accepted_block = ( # type: ignore[method-assign] + server._ensure_tip_refresh_service().refresh_after_accepted_block = ( # type: ignore[method-assign] lambda **kwargs: refreshes.append(str(kwargs["block_hash"])) ) with tempfile.TemporaryDirectory() as tempdir: @@ -2623,10 +2833,10 @@ def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: }, ) # The ack goes out before the block path runs; draining the - # submitter queue lands the block and pushes fresh work. + # submitter queue lands the block and its admitted scheduler + # trigger pushes fresh work before returning. self.assertEqual(sent, [{"id": "submit-1", "result": True, "error": None}]) self.assertTrue(server.submit_next_block_candidate()) - self.assertEqual(server.poll_qbit_tip_template_once(), 1) self.assertEqual(sent[0], {"id": "submit-1", "result": True, "error": None}) self.assertEqual([payload.get("method") for payload in sent[1:]], ["mining.set_difficulty", "mining.notify"]) @@ -2666,7 +2876,7 @@ def build_fresh_job(client: ClientState, *, clean_jobs: bool) -> object: self.assertFalse(should_close) self.assertEqual(ledger.pending[-1].job_id, "fresh-job") - def test_post_accept_notification_does_not_run_failing_template_build(self) -> None: + def test_post_accept_scheduler_reports_failing_template_build(self) -> None: old_tip = "00" * 32 block_hash = "ad" * 32 server, state, ledger = submit_coordinator(tip=old_tip) @@ -2722,12 +2932,14 @@ def unexpected_build(client: ClientState, *, clean_jobs: bool) -> object: self.assertEqual(len(ledger.confirmed), 1) self.assertEqual(len(ledger.pending), 1) self.assertEqual(server.tip_refresh_job_count, 0) - self.assertEqual(server.post_accept_refresh_failure_count, 0) + self.assertEqual(server.post_accept_refresh_failure_count, 1) self.assertEqual(state.active_job_ids, {"job-1"}) self.assertIn("job-1", server.jobs) self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertIn("qbit_prism_post_accept_refresh_failures_total 0", server.metrics_payload()) + self.assertTrue( + server._ensure_tip_refresh_service().snapshot().retry_requested + ) + self.assertIn("qbit_prism_post_accept_refresh_failures_total 1", server.metrics_payload()) def test_post_accept_refresh_preserves_pending_vardiff_difficulty_pair(self) -> None: old_tip = "00" * 32 block_hash = "ae" * 32 @@ -3020,6 +3232,49 @@ def unsafe_submit(_candidate: PrismBlockCandidate) -> bool: self.assertEqual(submit_calls, 0) self.assertIsNone(getattr(server, "_retry_block_candidate", None)) self.assertEqual(ledger.pending_block_candidates(), []) + + def test_durable_intent_promotion_failure_keeps_attempt_floor_for_replay(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + coinbase_tx_hex="00", + block_hex="00", + block_hash_hex="bf" * 32, + share_pass=False, + block_pass=True, + ) + submit_calls = 0 + + def unsafe_submit(_candidate: PrismBlockCandidate) -> bool: + nonlocal submit_calls + submit_calls += 1 + return True + + server.submit_block_candidate = unsafe_submit # type: ignore[method-assign] + writer = server._ensure_share_writer_service() + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ), patch.object( + writer, + "begin_candidate_actor", + side_effect=RuntimeError("promotion wiring failed"), + ): + with self.assertRaisesRegex(RuntimeError, "promotion wiring failed"): + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + self.assertEqual(submit_calls, 0) + self.assertEqual(len(ledger.pending_block_candidates()), 1) + self.assertNotEqual(server._pending_share_commit_floor, {}) + pending = next(iter(server._pending_share_commit_floor.values()))[0] + self.assertEqual(pending.share_id, "miner-a:" + submission.block_hash_hex) + server._finish_pending_share_commit(PendingShare(**pending.__dict__)) + self.assertEqual(server._pending_share_commit_floor, {}) def test_block_worthy_below_target_rejects_low_difficulty_when_block_fails(self) -> None: # If the block does not land, the below-share-target hash earns nothing # and the miner is rejected as low-difficulty -- never acked accepted @@ -3057,6 +3312,54 @@ def reject_candidate(_candidate: PrismBlockCandidate) -> bool: # block-abandonment reason -- this synchronous path used to skip it. self.assertEqual(server.rejection_counts_by_reason[PRISM_REJECTION_LOW_DIFFICULTY], 1) self.assertEqual(server.low_difficulty_share_count, 1) + + def test_below_target_terminal_update_failure_retains_floor_until_replay(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + coinbase_tx_hex="00", + block_hex="00", + block_hash_hex="b9" * 32, + share_pass=False, + block_pass=True, + ) + + def reject_candidate(_candidate: PrismBlockCandidate) -> bool: + server._abandon_block_candidate( + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + "rejected", + worker="miner-a", + ) + return False + + server.submit_block_candidate = reject_candidate # type: ignore[method-assign] + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ), patch.object( + ledger, + "mark_block_candidate_abandoned", + side_effect=RuntimeError("terminal update failed"), + ): + with self.assertRaisesRegex(RuntimeError, "terminal update failed"): + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + + (intent,) = ledger.pending_block_candidates() + self.assertEqual(intent["block_hash_hex"], submission.block_hash_hex) + self.assertNotEqual(server._pending_share_commit_floor, {}) + + # Exact durable replay reconstructs the candidate, reaches a successful + # terminal outbox return, and releases the same stable share-ID floor. + replayed = server.block_candidate_from_intent(intent) + server.enqueue_block_candidate(replayed) + self.assertTrue(server.submit_next_block_candidate()) + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertEqual(server._pending_share_commit_floor, {}) def test_below_target_transient_outcome_closes_without_definitive_reject(self) -> None: server, state, _recording = submit_coordinator() ledger = SingleWriterShareLedger() @@ -3099,6 +3402,460 @@ def test_below_target_transient_outcome_closes_without_definitive_reject(self) - server._retry_block_candidate.submission.block_hash_hex, submission.block_hash_hex, ) + # Same-hash retries have different acceptance stamps but one durable + # share identity. The stable S3 lease keeps the earliest stamp and a + # reconstructed terminal can release it by share_id. + (floor_entry,) = server._pending_share_commit_floor.values() + self.assertEqual(len(floor_entry), 3) + self.assertEqual(server._job_snapshot_anchor_ms(1_800_000_000_000), 1_699_999_999_999) + server._finish_pending_share_commit( + PendingShare(**server._retry_block_candidate.pending_share.__dict__) + ) + self.assertEqual(server._pending_share_commit_floor, {}) + + def test_retry_slot_is_not_published_before_floor_adoption(self) -> None: + server, state, _ledger = submit_coordinator() + pending = PendingShare( + share_id="miner-a:" + "91" * 32, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=2, + ntime=1, + ) + candidate = dataclass_replace( + block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="91" * 32, + block_hex="00", + share_pass=False, + block_pass=True, + ), + pending_share=pending, + ), + credit_share_on_accept=True, + ) + writer = server._ensure_share_writer_service() + original_adopt = writer.adopt_pending_share + adoption_entered = threading.Event() + release_adoption = threading.Event() + + def blocking_adopt(value: PendingShare) -> None: + adoption_entered.set() + release_adoption.wait(timeout=2) + original_adopt(value) + + writer.adopt_pending_share = blocking_adopt # type: ignore[method-assign] + thread = threading.Thread( + target=server._retain_block_candidate_for_retry, + args=(candidate,), + ) + thread.start() + self.assertTrue(adoption_entered.wait(timeout=1)) + self.assertIsNone(getattr(server, "_retry_block_candidate", None)) + release_adoption.set() + thread.join(timeout=2) + self.assertFalse(thread.is_alive()) + self.assertIs(server._retry_block_candidate, candidate) + + # Simulate the submitter winning immediately after publication. No + # delayed post-publication adoption remains to resurrect the floor. + server._finish_pending_share_commit(candidate.pending_share) + self.assertEqual(server._pending_share_commit_floor, {}) + + def test_failed_same_hash_attempt_cannot_release_older_durable_holder(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + submission = SimpleNamespace( + header_hex="aa" * 80, + coinbase_tx_hex="00", + block_hex="00", + block_hash_hex="95" * 32, + share_pass=False, + block_pass=True, + ) + server.submit_block_candidate = lambda _candidate: False # type: ignore[method-assign] + submit_params = ["miner-a", "job-1", "00" * 8, "00000001", "00000002"] + persist_entered = threading.Event() + release_persist = threading.Event() + errors: list[BaseException] = [] + clock_ms = iter([100, 200]) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ), patch( + "lab.prism.prism_coordinator.now_ms", + side_effect=clock_ms.__next__, + ): + with self.assertRaisesRegex(RuntimeError, "pending durable retry"): + server.handle_submit(state, submit_params) + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 99) + + def fail_second_persist(_intent: dict[str, object]) -> bool: + persist_entered.set() + release_persist.wait(timeout=2) + raise RuntimeError("outbox unavailable") + + def second_attempt() -> None: + try: + server.handle_submit(state, submit_params) + except BaseException as exc: + errors.append(exc) + + with patch.object( + ledger, + "persist_block_candidate_intent", + side_effect=fail_second_persist, + ): + thread = threading.Thread(target=second_attempt) + thread.start() + self.assertTrue(persist_entered.wait(timeout=1)) + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 99) + release_persist.set() + thread.join(timeout=2) + + self.assertFalse(thread.is_alive()) + self.assertEqual(len(errors), 1) + self.assertRegex(str(errors[0]), "outbox unavailable") + # The failed 200ms attempt released only its object holder. The first + # durable 100ms candidate remains reachable without re-adoption. + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 99) + self.assertEqual( + set(server._pending_share_commit_floor), + {server._retry_block_candidate.pending_share.share_id}, + ) + server._finish_pending_share_commit( + PendingShare(**server._retry_block_candidate.pending_share.__dict__) + ) + self.assertEqual(server._pending_share_commit_floor, {}) + + def test_same_hash_terminal_actor_cannot_drop_live_actor_or_failure_retry_floor( + self, + ) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + block_hash = "96" * 32 + share_id = f"miner-a:{block_hash}" + + def pending(accepted_at_ms: int) -> PendingShare: + return PendingShare( + share_id=share_id, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1, + ) + + submission = SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=False, + block_pass=True, + ) + actor_a = dataclass_replace( + block_candidate( + server, + state, + submission, + pending_share=pending(100), + ), + credit_share_on_accept=True, + ) + actor_b = dataclass_replace( + block_candidate( + server, + state, + submission, + pending_share=pending(200), + ), + credit_share_on_accept=True, + ) + self.assertTrue( + ledger.persist_block_candidate_intent(server.block_candidate_intent(actor_a)) + ) + self.assertFalse( + ledger.persist_block_candidate_intent(server.block_candidate_intent(actor_b)) + ) + a_entered = threading.Event() + release_a = threading.Event() + append_failed = threading.Event() + a_results: list[bool] = [] + + def controlled_submit(candidate: PrismBlockCandidate) -> bool: + if candidate is actor_a: + a_entered.set() + release_a.wait(timeout=2) + server.append_accepted_share( + candidate.client, + candidate.context, + candidate.submission, + candidate.pending_share, + candidate_intent=server.block_candidate_intent(candidate), + ) + return True + server._abandon_block_candidate( + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + "same-hash retry rejected", + worker="miner-a", + ) + return False + + server.submit_block_candidate = controlled_submit # type: ignore[method-assign] + server._next_block_candidate_retry_delay = ( # type: ignore[method-assign] + lambda _block_hash: 0.0 + ) + actor_a_thread = threading.Thread( + target=lambda: a_results.append( + server._submit_next_block_candidate_writer(actor_a) + ) + ) + actor_a_thread.start() + self.assertTrue(a_entered.wait(timeout=1)) + + # B can terminalize the one durable row, but its terminal holder and + # actor release must not erase A's independent active acceptance floor. + self.assertTrue(server._submit_next_block_candidate_writer(actor_b)) + self.assertTrue(actor_a_thread.is_alive()) + self.assertEqual(len(ledger), 0) + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertEqual(set(server._pending_share_commit_floor), {share_id}) + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 99) + writer = server._ensure_share_writer_service() + self.assertIn(id(actor_a.pending_share), writer._candidate_actor_holders) + self.assertNotIn(id(actor_b.pending_share), writer._candidate_actor_holders) + + def fail_actor_a_append( + _entries: list[tuple[PendingShare, dict[str, object] | None]], + ) -> list[object]: + append_failed.set() + raise RuntimeError("share commit failed after competing terminal") + + ledger.append_batch = fail_actor_a_append # type: ignore[method-assign] + release_a.set() + actor_a_thread.join(timeout=2) + + self.assertFalse(actor_a_thread.is_alive()) + self.assertTrue(append_failed.is_set()) + self.assertEqual(a_results, [True]) + self.assertEqual(len(ledger), 0) + self.assertIs(server._retry_block_candidate, actor_a) + self.assertEqual(set(server._pending_share_commit_floor), {share_id}) + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 99) + self.assertEqual(writer._candidate_actor_holders, {}) + server._finish_pending_share_commit(actor_a.pending_share) + self.assertEqual(server._pending_share_commit_floor, {}) + + def test_sync_same_hash_terminal_cannot_drop_async_actor_before_credit(self) -> None: + server, state, _recording = submit_coordinator() + ledger = SingleWriterShareLedger() + server.ledger = ledger + block_hash = "97" * 32 + pending_a = PendingShare( + share_id=f"miner-a:{block_hash}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=7, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=12_345, + accepted_at_ms=100, + ntime=1, + ) + submission = SimpleNamespace( + header_hex="aa" * 80, + coinbase_tx_hex="00", + block_hash_hex=block_hash, + block_hex="00", + share_pass=False, + block_pass=True, + ) + actor_a = dataclass_replace( + block_candidate( + server, + state, + submission, + pending_share=pending_a, + ), + credit_share_on_accept=True, + ) + self.assertTrue( + ledger.persist_block_candidate_intent(server.block_candidate_intent(actor_a)) + ) + a_entered = threading.Event() + release_a = threading.Event() + a_results: list[bool] = [] + + def controlled_submit(candidate: PrismBlockCandidate) -> bool: + if candidate is actor_a: + a_entered.set() + release_a.wait(timeout=2) + server.append_accepted_share( + candidate.client, + candidate.context, + candidate.submission, + candidate.pending_share, + candidate_intent=server.block_candidate_intent(candidate), + ) + return True + server._abandon_block_candidate( + PRISM_REJECTION_SUBMITBLOCK_REJECTED, + "synchronous same-hash retry rejected", + worker="miner-a", + ) + return False + + server.submit_block_candidate = controlled_submit # type: ignore[method-assign] + actor_a_thread = threading.Thread( + target=lambda: a_results.append( + server._submit_next_block_candidate_writer(actor_a) + ) + ) + actor_a_thread.start() + self.assertTrue(a_entered.wait(timeout=1)) + + with patch( + "lab.prism.prism_coordinator.direct_stratum.assemble_submission", + return_value=submission, + ), patch("lab.prism.prism_coordinator.now_ms", return_value=200): + with self.assertRaises(StratumError) as raised: + server.handle_submit( + state, + ["miner-a", "job-1", "00" * 8, "00000001", "00000002"], + ) + self.assertEqual(raised.exception.reason, PRISM_REJECTION_LOW_DIFFICULTY) + self.assertTrue(actor_a_thread.is_alive()) + self.assertEqual(len(ledger), 0) + self.assertEqual(ledger.pending_block_candidates(), []) + self.assertEqual( + server._job_snapshot_anchor_ms(10_000), + pending_a.accepted_at_ms - 1, + ) + writer = server._ensure_share_writer_service() + self.assertIn(id(pending_a), writer._candidate_actor_holders) + + release_a.set() + actor_a_thread.join(timeout=2) + + self.assertFalse(actor_a_thread.is_alive()) + self.assertEqual(a_results, [True]) + self.assertEqual(len(ledger), 1) + self.assertEqual(ledger.all_shares()[0].accepted_at_ms, 100) + self.assertEqual(server._pending_share_commit_floor, {}) + self.assertEqual(writer._candidate_actor_holders, {}) + + def test_lower_parent_retry_replacement_keeps_both_durable_floors(self) -> None: + server, state, _ledger = submit_coordinator() + + def candidate(tag: str, height: int, accepted_at_ms: int) -> PrismBlockCandidate: + context = SimpleNamespace(**vars(server.jobs["job-1"])) + context.template = {**context.template, "height": height} + pending = PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=height, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1, + ) + return dataclass_replace( + block_candidate( + server, + state, + SimpleNamespace(block_hash_hex=tag, block_hex="00"), + pending_share=pending, + ), + context=context, + credit_share_on_accept=True, + ) + + descendant = candidate("d1" * 32, 11, 200) + parent = candidate("a1" * 32, 10, 100) + + server._retain_block_candidate_for_retry(descendant) + server._retain_block_candidate_for_retry(parent) + + self.assertIs(server._retry_block_candidate, parent) + self.assertEqual( + set(server._pending_share_commit_floor), + {descendant.pending_share.share_id, parent.pending_share.share_id}, + ) + for retained in (parent, descendant): + server._finish_pending_share_commit( + PendingShare(**retained.pending_share.__dict__) + ) + self.assertEqual(server._pending_share_commit_floor, {}) + + def test_equal_height_nonselection_keeps_competitor_durable_floor(self) -> None: + server, state, _ledger = submit_coordinator() + + def candidate(tag: str, accepted_at_ms: int) -> PrismBlockCandidate: + context = SimpleNamespace(**vars(server.jobs["job-1"])) + context.template = {**context.template, "height": 10} + pending = PendingShare( + share_id=f"miner-a:{tag}", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=10, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=accepted_at_ms, + ntime=1, + ) + return dataclass_replace( + block_candidate( + server, + state, + SimpleNamespace(block_hash_hex=tag, block_hex="00"), + pending_share=pending, + ), + context=context, + credit_share_on_accept=True, + ) + + selected = candidate("b1" * 32, 100) + competitor = candidate("c1" * 32, 200) + + server._retain_block_candidate_for_retry(selected) + server._retain_block_candidate_for_retry(competitor) + + self.assertIs(server._retry_block_candidate, selected) + self.assertEqual( + set(server._pending_share_commit_floor), + {selected.pending_share.share_id, competitor.pending_share.share_id}, + ) + for retained in (selected, competitor): + server._finish_pending_share_commit( + PendingShare(**retained.pending_share.__dict__) + ) + self.assertEqual(server._pending_share_commit_floor, {}) def test_low_difficulty_submission_without_block_solve_is_rejected(self) -> None: server, state, ledger = submit_coordinator() submission = SimpleNamespace( @@ -3226,6 +3983,13 @@ def test_block_candidate_intent_round_trips_compact_payout_state(self) -> None: (("miner-a", "miner-a", "11" * 32, 25),), ) + static_replayed = PrismCoordinator.block_candidate_from_intent(intent) + self.assertTrue(static_replayed.context.collection_only) + self.assertEqual( + static_replayed.context.prospective_prior_balances, + (("miner-a", "miner-a", "11" * 32, 25),), + ) + # Intents persisted before the flag existed replay as ready-window # candidates, which is all the outbox could ever have contained then. intent.pop("collection_only") @@ -3233,3 +3997,44 @@ def test_block_candidate_intent_round_trips_compact_payout_state(self) -> None: replayed = server.block_candidate_from_intent(intent) self.assertFalse(replayed.context.collection_only) self.assertIsNone(replayed.context.prospective_prior_balances) + + def test_replayed_credit_candidate_adopts_floor_before_job_anchor(self) -> None: + server, state, _ledger = submit_coordinator() + pending = PendingShare( + share_id="miner-a:" + "92" * 32, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=123, + ntime=1, + ) + candidate = dataclass_replace( + block_candidate( + server, + state, + SimpleNamespace( + coinbase_tx_hex="00", + block_hash_hex="92" * 32, + block_hex="00", + share_pass=False, + block_pass=True, + ), + pending_share=pending, + ), + credit_share_on_accept=True, + ) + + replayed = server.block_candidate_from_intent( + server.block_candidate_intent(candidate) + ) + + self.assertEqual(server._job_snapshot_anchor_ms(10_000), 122) + server._finish_pending_share_commit( + PendingShare(**replayed.pending_share.__dict__) + ) + self.assertEqual(server._pending_share_commit_floor, {}) diff --git a/tests/test_prism_bounded_executor.py b/tests/test_prism_bounded_executor.py new file mode 100644 index 0000000..22c1dc4 --- /dev/null +++ b/tests/test_prism_bounded_executor.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Focused tests for PRISM's bounded priority executor.""" + +from __future__ import annotations + +import threading +import unittest + +from lab.prism import prism_coordinator +from lab.prism.bounded_executor import ( + _BoundedPriorityExecutor, + _DeliveryQueueFull, +) + + +class BoundedPriorityExecutorTests(unittest.TestCase): + def test_compatibility_reexports_reference_executor_owner(self) -> None: + self.assertIs( + prism_coordinator._BoundedPriorityExecutor, + _BoundedPriorityExecutor, + ) + self.assertIs(prism_coordinator._DeliveryQueueFull, _DeliveryQueueFull) + + def test_queue_bound_excludes_active_worker_and_reports_active_count(self) -> None: + executor = _BoundedPriorityExecutor(max_workers=1, max_queue_size=1) + blocker_started = threading.Event() + release = threading.Event() + + def blocker() -> None: + blocker_started.set() + release.wait(5) + + try: + executor.submit(blocker) + self.assertTrue(blocker_started.wait(5)) + queued = executor.submit(lambda: None) + self.assertEqual(executor.stats(), (1, 1)) + with self.assertRaisesRegex(_DeliveryQueueFull, "queue is full"): + executor.submit(lambda: None) + release.set() + queued.result(5) + finally: + release.set() + executor.shutdown(wait=True, cancel_futures=True) + + def test_shutdown_cancels_queued_future_and_joins_named_workers(self) -> None: + executor = _BoundedPriorityExecutor(max_workers=2, max_queue_size=2) + blocker_started = [threading.Event(), threading.Event()] + release = threading.Event() + + def blocker(index: int) -> None: + blocker_started[index].set() + release.wait(5) + + executor.submit(blocker, 0) + executor.submit(blocker, 1) + self.assertTrue(all(started.wait(5) for started in blocker_started)) + queued = executor.submit(lambda: None) + + executor.shutdown(wait=False, cancel_futures=True) + self.assertTrue(queued.cancelled()) + self.assertEqual( + [thread.name for thread in executor._threads], + ["prism-job-delivery-1", "prism-job-delivery-2"], + ) + release.set() + executor.shutdown(wait=True) + + self.assertTrue(all(not thread.is_alive() for thread in executor._threads)) + + def test_wait_false_shutdown_does_not_block_when_queue_is_smaller_than_pool( + self, + ) -> None: + executor = _BoundedPriorityExecutor(max_workers=4, max_queue_size=1) + blocker_started = [threading.Event() for _index in range(4)] + release = threading.Event() + + def blocker(index: int) -> None: + blocker_started[index].set() + release.wait(5) + + for index in range(4): + executor.submit(blocker, index) + self.assertTrue(blocker_started[index].wait(5)) + queued = executor.submit(lambda: "drained") + + shutdown_returned = threading.Event() + shutdown = threading.Thread( + target=lambda: ( + executor.shutdown(wait=False), + shutdown_returned.set(), + ) + ) + shutdown.start() + self.assertTrue(shutdown_returned.wait(0.5)) + shutdown.join(0.5) + self.assertFalse(shutdown.is_alive()) + + release.set() + self.assertEqual(queued.result(5), "drained") + executor.shutdown(wait=True) + self.assertTrue(all(not thread.is_alive() for thread in executor._threads)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_coordinator_config.py b/tests/test_prism_coordinator_config.py index 0e92847..40fe474 100644 --- a/tests/test_prism_coordinator_config.py +++ b/tests/test_prism_coordinator_config.py @@ -5,6 +5,8 @@ from __future__ import annotations import unittest + +from lab.prism.coordinator_config import env_decimal from tests.prism_vardiff_test_support import * @@ -862,6 +864,7 @@ def test_fixed_ledger_session_token_requires_explicit_opt_in(self) -> None: env = { "PRISM_POSTGRES_PSQL_COMMAND": "psql postgresql://example.invalid/qbit", "PRISM_LEDGER_WRITER_SESSION_TOKEN": "fixed-session", + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS": "47", } with patch.dict(os.environ, env, clear=True): @@ -875,6 +878,10 @@ def test_fixed_ledger_session_token_requires_explicit_opt_in(self) -> None: self.assertEqual(ledger.backend_name, "postgres-psql") self.assertEqual(fake_ledger.call_args.kwargs["writer_session_token"], "fixed-session") + self.assertEqual( + fake_ledger.call_args.kwargs["reward_window_cache_seconds"], + 47.0, + ) def test_same_tip_retention_requires_connection_derived_production_bound(self) -> None: with self.assertRaisesRegex( SystemExit, @@ -1242,7 +1249,6 @@ def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppe class PrismCoordinatorReliabilityTests(unittest.TestCase): def _bare_coordinator(self) -> PrismCoordinator: server = PrismCoordinator.__new__(PrismCoordinator) - server.lock = threading.RLock() server.stop_event = threading.Event() server._heartbeats = {} server._watchdog_pauses = {} @@ -1258,6 +1264,16 @@ def test_positive_float_env_rejects_non_finite_values(self) -> None: with self.assertRaisesRegex(SystemExit, "PRISM_WATCHDOG_TIMEOUT_SECONDS must be finite"): env_positive_float("PRISM_WATCHDOG_TIMEOUT_SECONDS", 120.0) + def test_decimal_env_rejects_invalid_and_non_finite_values(self) -> None: + with self.assertRaisesRegex(SystemExit, "TEST_DECIMAL must be a decimal number"): + env_decimal("TEST_DECIMAL", "1", environ={"TEST_DECIMAL": "not-decimal"}) + for raw in ("NaN", "sNaN", "Infinity", "-Infinity"): + with self.subTest(raw=raw), self.assertRaisesRegex( + SystemExit, + "TEST_DECIMAL must be finite", + ): + env_decimal("TEST_DECIMAL", "1", environ={"TEST_DECIMAL": raw}) + class PrismListenerProfileTests(unittest.TestCase): def test_highdiff_listener_disabled_without_port(self) -> None: with patch.dict(os.environ, {}, clear=False): diff --git a/tests/test_prism_coordinator_config_loading.py b/tests/test_prism_coordinator_config_loading.py new file mode 100644 index 0000000..bf685f4 --- /dev/null +++ b/tests/test_prism_coordinator_config_loading.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Direct tests for the extracted coordinator configuration and RPC seams.""" + +from __future__ import annotations + +import dataclasses +import subprocess +import sys +import tempfile +import time +import unittest +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from lab.prism.coordinator_config import ( + CoordinatorConfig, + LifecycleConfig, + load_coordinator_config, +) +from lab.prism.prism_coordinator import JsonRpc as CompatibilityJsonRpc +from lab.prism.prism_coordinator import PrismCoordinator +from lab.prism.rpc import JsonRpc +from lab.prism.run_ctv_broadcaster_daemon import JsonRpc as DaemonJsonRpc + + +def minimal_environment(root: Path) -> dict[str, str]: + return { + "QBIT_RPC_HOST": "qbit.example", + "QBIT_RPC_USER": "rpc-user", + "QBIT_RPC_PASSWORD": "rpc-password", + "PRISM_ALLOW_MEMORY_LEDGER": "1", + "PRISM_ALLOW_TEST_SIGNING_SEEDS": "1", + "PRISM_ALLOW_BUNDLE_EMBEDDED_LEDGER_KEY": "1", + "PRISM_AUDIT_DIR": str(root), + "PRISM_EVIDENCE_PATH": str(root / "evidence.json"), + } + + +class CoordinatorConfigLoadingTests(unittest.TestCase): + def construct(self, source: dict[str, str]) -> PrismCoordinator: + config = load_coordinator_config(source) + with patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")): + return PrismCoordinator(config) + + def test_loader_accepts_mapping_without_reading_process_environment(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = minimal_environment(Path(temp_dir)) + with patch.dict("os.environ", {}, clear=True): + config = load_coordinator_config(source) + + self.assertIsInstance(config, CoordinatorConfig) + self.assertEqual(config.rpc.host, "qbit.example") + self.assertEqual(config.rpc.port, 18452) + self.assertEqual(config.stratum.bind, "127.0.0.1") + self.assertEqual(config.lifecycle.pending_refresh_health_deadline_seconds, 15.0) + self.assertEqual(config.lifecycle.coherent_tip_poll_health_deadline_seconds, 15.0) + + def test_loader_captures_pr54_health_deadlines_in_frozen_lifecycle_config(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_HEALTH_REFRESH_SECONDS": "7", + "PRISM_HEALTH_PENDING_REFRESH_MAX_AGE_SECONDS": "23", + "PRISM_HEALTH_TIP_POLL_MAX_AGE_SECONDS": "29", + "PRISM_MINING_HEALTH_STARTUP_GRACE_SECONDS": "31", + } + config = load_coordinator_config(source) + + self.assertEqual( + config.lifecycle, + replace( + config.lifecycle, + health_refresh_seconds=7.0, + pending_refresh_health_deadline_seconds=23.0, + coherent_tip_poll_health_deadline_seconds=29.0, + mining_health_startup_grace_seconds=31.0, + ), + ) + self.assertIsInstance(config.lifecycle, LifecycleConfig) + with self.assertRaises(dataclasses.FrozenInstanceError): + config.lifecycle.health_refresh_seconds = 1.0 # type: ignore[misc] + + def test_loader_owns_public_reward_window_cache_interval(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_PUBLIC_REWARD_WINDOW_CACHE_SECONDS": "47", + } + config = load_coordinator_config(source) + + self.assertEqual(config.ledger.reward_window_cache_seconds, 47.0) + + def test_coordinator_uses_supplied_snapshot_without_reloading_environment(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + config = load_coordinator_config(minimal_environment(Path(temp_dir))) + with ( + patch( + "lab.prism.prism_coordinator.load_coordinator_config", + side_effect=AssertionError("unexpected environment reload"), + ), + patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")), + ): + coordinator = PrismCoordinator(config) + + self.assertIs(coordinator.config, config) + self.assertEqual(coordinator.bind, config.stratum.bind) + self.assertEqual( + coordinator.health_pending_refresh_max_age_seconds, + config.lifecycle.pending_refresh_health_deadline_seconds, + ) + + def test_zero_argument_coordinator_still_loads_environment(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = minimal_environment(Path(temp_dir)) + with ( + patch.dict("os.environ", source, clear=True), + patch.object(JsonRpc, "call", side_effect=RuntimeError("offline")), + ): + coordinator = PrismCoordinator() + + self.assertEqual(coordinator.rpc.host, "qbit.example") + self.assertEqual(coordinator.config.rpc.user, "rpc-user") + + def test_dormant_live_chain_validation_is_deferred_until_use(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_MIN_PEERS": "0", + "PRISM_TEMPLATE_MAX_AGE_SECONDS": "not-an-int", + } + coordinator = self.construct(source) + + class PublicChainRpc: + def call(self, method: str, params: list[object] | None = None) -> object: + if method == "getblockchaininfo": + return { + "chain": "testnet4", + "initialblockdownload": False, + "blocks": 10, + "headers": 10, + } + if method == "getnetworkinfo": + return {"connections": 1} + raise RuntimeError(method) + + coordinator.qbit_chain = "testnet4" + coordinator.rpc = PublicChainRpc() # type: ignore[assignment] + with self.assertRaisesRegex(SystemExit, "PRISM_MIN_PEERS must be positive"): + coordinator.validate_live_chain_identity() + + coordinator.current_template_artifacts = lambda: SimpleNamespace( # type: ignore[method-assign] + template={"curtime": int(time.time())}, + previousblockhash="11" * 32, + ) + with self.assertRaisesRegex( + SystemExit, "PRISM_TEMPLATE_MAX_AGE_SECONDS must be an integer" + ): + coordinator.validate_live_template_and_fee_policy() + + def test_disabled_ctv_settlement_ignores_invalid_settlement_only_values(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_CTV_SETTLEMENT_ENABLED": "0", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", + "PRISM_RESERVED_COINBASE_OUTPUTS": "also-not-an-int", + } + coordinator = self.construct(source) + + self.assertIsNone(coordinator.prism_ctv_settlement_config()) + + def test_enabled_ctv_settlement_validates_at_consuming_boundary(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_CTV_SETTLEMENT_ENABLED": "1", + "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS": "not-an-int", + } + coordinator = self.construct(source) + + with self.assertRaisesRegex( + SystemExit, "PRISM_DIRECT_COINBASE_PAYOUT_FLOOR_BITS must be an integer" + ): + coordinator.prism_ctv_settlement_config() + + def test_payout_validation_is_deferred_until_policy_is_consumed(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES": "not-an-int", + "PRISM_POOL_FEE_ENABLED": "", + } + coordinator = self.construct(source) + + with self.assertRaisesRegex( + SystemExit, "PRISM_PAYOUT_P2MR_SPEND_INPUT_BYTES must be an integer" + ): + coordinator.prism_payout_policy() + + def test_explicit_disabled_broadcaster_defers_empty_settlement_flag(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_CTV_BROADCASTER_ENABLED": "0", + "PRISM_CTV_SETTLEMENT_ENABLED": "", + } + coordinator = self.construct(source) + + self.assertFalse(coordinator.ctv_broadcaster_enabled) + with self.assertRaisesRegex( + SystemExit, "PRISM_CTV_SETTLEMENT_ENABLED is required" + ): + coordinator.prism_ctv_settlement_config() + + def test_broadcaster_default_still_consumes_settlement_flag_during_construction(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + source = { + **minimal_environment(Path(temp_dir)), + "PRISM_CTV_SETTLEMENT_ENABLED": "", + } + with self.assertRaisesRegex( + SystemExit, "PRISM_CTV_SETTLEMENT_ENABLED is required" + ): + load_coordinator_config(source) + + def test_rpc_compatibility_reexport_points_to_leaf_class(self) -> None: + self.assertIs(CompatibilityJsonRpc, JsonRpc) + self.assertIs(DaemonJsonRpc, JsonRpc) + + def test_importing_rpc_leaf_does_not_import_coordinator(self) -> None: + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; import lab.prism.rpc; " + "assert 'lab.prism.prism_coordinator' not in sys.modules" + ), + ], + check=False, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_coordinator_job_cache.py b/tests/test_prism_coordinator_job_cache.py new file mode 100644 index 0000000..cfd60a4 --- /dev/null +++ b/tests/test_prism_coordinator_job_cache.py @@ -0,0 +1,1240 @@ +#!/usr/bin/env python3 +"""Direct ownership tests for extracted PRISM template and job services.""" + +from __future__ import annotations + +import copy +from collections import OrderedDict +from concurrent.futures import Future +from contextlib import contextmanager +from dataclasses import replace as dataclass_replace +import json +import threading +import time +from types import SimpleNamespace +import unittest + +from lab.prism.job_bundle import ( + CachedJobBundle, + JobBuildFlight, + JobBuildKey, + JobBuildSuperseded, +) +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + TemplateArtifactEventSink, + TemplateArtifactPorts, + TemplateArtifactRepository, + qbit_template_fingerprint, +) +from tests.prism_coordinator_test_support import ( + base_template, + coordinator, + install_fake_bundle_builder, + worker, +) + + +def cached_bundle( + artifacts: CachedTemplateArtifacts, + *, + payout_generation: int, + payout_artifact_sha256: str, + key: tuple[object, ...] = ("candidate",), + collection_only: bool = True, +) -> CachedJobBundle: + build_key = JobBuildKey( + best_tip_hash=artifacts.previousblockhash, + previous_block_hash=artifacts.previousblockhash, + template_fingerprint=artifacts.fingerprint, + template_generation=artifacts.generation, + payout_state_generation=payout_generation, + payout_artifact_sha256=payout_artifact_sha256, + mode="collection" if collection_only else "ready", + collection_identity=("miner", "22" * 32) if collection_only else None, + block_height=int(artifacts.template["height"]), + coinbase_value_sats=int(artifacts.template["coinbasevalue"]), + network_difficulty=artifacts.network_difficulty, + issued_at_ms=1, + payout_policy_sha256="payout", + ctv_settlement_sha256=None, + witness_merkle_sha256="witness", + transaction_set_sha256="transactions", + coinbase_suffix_hex="00", + signing_key_sha256="signing", + ledger_signing_key_sha256="ledger", + numeric_context_sha256="numeric", + ) + return CachedJobBundle( + key=key, + template=artifacts.template, + template_fingerprint=artifacts.fingerprint, + coinbase_manifest={"coinbase_tx_hex": "00"}, + shares_json=[], + prior_balances=[], + found_block={"network_difficulty": artifacts.network_difficulty}, + collection_only=collection_only, + issued_at_ms=1, + base_job=SimpleNamespace(), # type: ignore[arg-type] + built_monotonic=1.0, + template_generation=artifacts.generation, + payout_state_generation=payout_generation, + collection_identity=build_key.collection_identity, + build_key=build_key, + ) + + +def template_repository() -> TemplateArtifactRepository: + repository = TemplateArtifactRepository( + TemplateArtifactPorts( + fetch_template=lambda: base_template(), + fetch_bestblockhash=lambda: "11" * 32, + newest_observed_tip=lambda: None, + observe_tip=lambda _tip: None, + schedule_refresh_retry=lambda: None, + pinned_issuance_artifacts=lambda: None, + repinned_issuance_artifacts=lambda _artifacts: None, + ), + cache_seconds=2.0, + scale_network_difficulty=lambda _bits: 1, + ) + repository.bind_event_sink( + TemplateArtifactEventSink( + record_cache_event=lambda _hit: None, + record_build_phase=lambda _phase, _elapsed: None, + artifacts_changed=lambda _artifacts, _fingerprint_changed: None, + artifacts_cleared=lambda _artifacts: None, + ) + ) + return repository + + +class ImmutableArtifactOwnershipTests(unittest.TestCase): + def test_repository_event_sink_fails_fast_and_binds_once(self) -> None: + repository = TemplateArtifactRepository( + TemplateArtifactPorts( + fetch_template=lambda: base_template(), + fetch_bestblockhash=lambda: "11" * 32, + newest_observed_tip=lambda: None, + observe_tip=lambda _tip: None, + schedule_refresh_retry=lambda: None, + pinned_issuance_artifacts=lambda: None, + repinned_issuance_artifacts=lambda _artifacts: None, + ), + cache_seconds=2.0, + scale_network_difficulty=lambda _bits: 1, + ) + sink = TemplateArtifactEventSink( + record_cache_event=lambda _hit: None, + record_build_phase=lambda _phase, _elapsed: None, + artifacts_changed=lambda _artifacts, _changed: None, + artifacts_cleared=lambda _artifacts: None, + ) + + with self.assertRaisesRegex(RuntimeError, "event sink is not bound"): + repository.derive(base_template(), generation=1) + repository.bind_event_sink(sink) + with self.assertRaisesRegex(RuntimeError, "event sink is already bound"): + repository.bind_event_sink(sink) + + def test_template_artifact_detaches_and_recursively_freezes_json(self) -> None: + source = base_template() + source["extension"] = {"rows": [{"value": 1}]} + expected = copy.deepcopy(source) + + artifacts = CachedTemplateArtifacts( + template=source, + fingerprint=qbit_template_fingerprint(source), + previousblockhash=str(source["previousblockhash"]), + transaction_hexes=(), + witness_merkle_leaves_hex=(), + network_difficulty=1, + fetched_monotonic=1.0, + generation=1, + ) + + source["extension"]["rows"][0]["value"] = 2 # type: ignore[index] + self.assertEqual(artifacts.template, expected) + self.assertEqual(json.loads(json.dumps(artifacts.template)), expected) + self.assertIs(copy.deepcopy(artifacts.template), artifacts.template) + with self.assertRaises(TypeError): + artifacts.template["height"] = 11 + with self.assertRaises(TypeError): + artifacts.template["extension"]["rows"].append({}) # type: ignore[index,union-attr] + with self.assertRaises(TypeError): + artifacts.template["extension"]["rows"][0]["value"] = 3 # type: ignore[index] + + def test_bundle_detaches_and_recursively_freezes_owned_json(self) -> None: + template = base_template() + manifest = {"coinbase_tx_hex": "00", "nested": {"values": [1]}} + shares = [{"miner_id": "miner-a", "proof": {"path": ["aa"]}}] + balances = [{"miner_id": "miner-a", "balance_sats": 1}] + found_block = {"network_difficulty": 1, "nested": {"values": [2]}} + + bundle = CachedJobBundle( + key=("bundle",), + template=template, + template_fingerprint="fingerprint", + coinbase_manifest=manifest, + shares_json=shares, + prior_balances=balances, + found_block=found_block, + collection_only=False, + issued_at_ms=1, + base_job=SimpleNamespace(), # type: ignore[arg-type] + built_monotonic=1.0, + ) + + manifest["nested"]["values"].append(9) # type: ignore[index,union-attr] + shares[0]["proof"]["path"].append("bb") # type: ignore[index,union-attr] + balances[0]["balance_sats"] = 2 + found_block["nested"]["values"].append(3) # type: ignore[index,union-attr] + self.assertEqual(bundle.coinbase_manifest["nested"]["values"], [1]) # type: ignore[index] + self.assertEqual(bundle.shares_json[0]["proof"]["path"], ["aa"]) # type: ignore[index] + self.assertEqual(bundle.prior_balances[0]["balance_sats"], 1) + self.assertEqual(bundle.found_block["nested"]["values"], [2]) # type: ignore[index] + self.assertIs(copy.deepcopy(bundle.shares_json), bundle.shares_json) + with self.assertRaises(TypeError): + bundle.shares_json.append({}) + with self.assertRaises(TypeError): + bundle.coinbase_manifest["nested"]["values"].append(4) # type: ignore[index,union-attr] + + def test_repository_derives_from_frozen_artifact_and_orders_stores(self) -> None: + repository = template_repository() + source = base_template() + first = repository.derive(source, generation=1) + self.assertTrue(repository.store_artifacts(first)) + + second = repository.derive(first.template, generation=2) + + self.assertEqual(second.template, first.template) + self.assertEqual(second.transaction_hexes, first.transaction_hexes) + self.assertEqual( + second.witness_merkle_leaves_hex, + first.witness_merkle_leaves_hex, + ) + self.assertTrue(repository.store_artifacts(second)) + self.assertFalse(repository.store_artifacts(first)) + self.assertIs(repository.current_artifacts(), second) + + def test_fingerprint_callback_finishes_before_newer_generation_wins( + self, + ) -> None: + second_callback_started = threading.Event() + release_second_callback = threading.Event() + third_store_started = threading.Event() + third_store_finished = threading.Event() + callback_observations: list[tuple[int, int | None]] = [] + callback_wait_timed_out: list[int] = [] + repository: TemplateArtifactRepository + + def artifacts_changed( + artifacts: CachedTemplateArtifacts, + _fingerprint_changed: bool, + ) -> None: + if artifacts.generation == 2: + second_callback_started.set() + if not release_second_callback.wait(2.0): + callback_wait_timed_out.append(artifacts.generation) + current = repository.current_artifacts() + callback_observations.append( + ( + artifacts.generation, + None if current is None else current.generation, + ) + ) + + repository = TemplateArtifactRepository( + TemplateArtifactPorts( + fetch_template=lambda: base_template(), + fetch_bestblockhash=lambda: "11" * 32, + newest_observed_tip=lambda: None, + observe_tip=lambda _tip: None, + schedule_refresh_retry=lambda: None, + pinned_issuance_artifacts=lambda: None, + repinned_issuance_artifacts=lambda _artifacts: None, + ), + cache_seconds=2.0, + scale_network_difficulty=lambda _bits: 1, + ) + repository.bind_event_sink( + TemplateArtifactEventSink( + record_cache_event=lambda _hit: None, + record_build_phase=lambda _phase, _elapsed: None, + artifacts_changed=artifacts_changed, + artifacts_cleared=lambda _artifacts: None, + ) + ) + templates = [base_template() for _generation in range(3)] + for generation, template in enumerate(templates, start=1): + template["height"] = generation + artifacts = [ + repository.derive(template, generation=generation) + for generation, template in enumerate(templates, start=1) + ] + self.assertTrue(repository.store_artifacts(artifacts[0])) + + second_thread = threading.Thread( + target=repository.store_artifacts, + args=(artifacts[1],), + ) + + def store_third() -> None: + third_store_started.set() + repository.store_artifacts(artifacts[2]) + third_store_finished.set() + + third_thread = threading.Thread(target=store_third) + second_thread.start() + self.assertTrue(second_callback_started.wait(2.0)) + third_thread.start() + self.assertTrue(third_store_started.wait(2.0)) + self.assertFalse(third_store_finished.wait(0.1)) + self.assertIs(repository.current_artifacts(), artifacts[1]) + + release_second_callback.set() + second_thread.join(2.0) + third_thread.join(2.0) + + self.assertFalse(second_thread.is_alive()) + self.assertFalse(third_thread.is_alive()) + self.assertTrue(third_store_finished.is_set()) + self.assertIs(repository.current_artifacts(), artifacts[2]) + self.assertEqual(callback_wait_timed_out, []) + self.assertEqual(callback_observations, [(2, 2), (3, 3)]) + + +class JobBundleServiceOwnershipAndPriorityTests(unittest.TestCase): + def test_coordinator_binds_leaf_callbacks_without_lazy_reentry(self) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + repository = service.template_repository + compiler = server._ensure_bundle_compiler() + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + callback_finished = threading.Event() + callback_errors: list[BaseException] = [] + init_lock = server._job_bundle_service_init_lock + saved_service = server._job_bundle_service + + def use_bound_callback() -> None: + try: + repository.current() + except BaseException as exc: # noqa: BLE001 - surface thread errors + callback_errors.append(exc) + finally: + callback_finished.set() + + server._job_bundle_service = None + init_lock.acquire() + try: + callback_thread = threading.Thread(target=use_bound_callback) + callback_thread.start() + self.assertTrue(callback_finished.wait(2.0)) + callback_thread.join(2.0) + finally: + init_lock.release() + server._job_bundle_service = saved_service + + self.assertEqual(callback_errors, []) + self.assertIs(service.bundle_compiler(), compiler) + with self.assertRaisesRegex(RuntimeError, "compiler is already bound"): + service.bind_bundle_compiler(compiler) + @staticmethod + def request_for( + server: object, + artifacts: CachedTemplateArtifacts, + *, + mode: str = "ready", + identity: object | None = None, + publication_critical: bool = False, + request_source: str = "initial", + priority_requested_monotonic: float | None = None, + ) -> object: + service = server._ensure_job_bundle_service() # type: ignore[attr-defined] + payout_generation = ( + server._ensure_payout_state_service().snapshot().generation # type: ignore[attr-defined] + ) + cache_key = service.job_bundle_key( + artifacts, + mode=mode, + payout_state_generation=payout_generation, + worker=identity, + ) + return service.new_build_request( + artifacts, + identity, + mode=mode, + payout_state_generation=payout_generation, + cache_key=cache_key, + publication_critical=publication_critical, + request_source=request_source, + priority_requested_monotonic=priority_requested_monotonic, + ) + + def test_publication_critical_build_cannot_be_displaced_by_initial_work( + self, + ) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + old_artifacts = server.store_template_artifacts(dict(rpc.template)) + new_artifacts = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert old_artifacts is not None and new_artifacts is not None + latest = self.request_for( + server, + new_artifacts, + publication_critical=True, + request_source="tip_refresh", + ) + reconnect = self.request_for(server, old_artifacts) + same_tip_reconnect = self.request_for(server, new_artifacts) + latest_flight = JobBuildFlight(latest) # type: ignore[arg-type] + service._active = latest_flight + + deferred = service.request_build(reconnect) # type: ignore[arg-type] + coalesced = service.request_build( # type: ignore[arg-type] + same_tip_reconnect + ) + + self.assertIs(service._active, latest_flight) + self.assertIs(coalesced, latest.promise) # type: ignore[union-attr] + self.assertFalse(latest.cancellation.is_set()) # type: ignore[union-attr] + self.assertFalse(deferred.done()) + self.assertEqual(service._priority_counts["routine_deferred"], 1) + self.assertEqual(service._initial_prepared_work_counts["deferred"], 1) + self.assertEqual( + service._initial_prepared_work_counts["singleflight"], + 1, + ) + latest.promise.set_result(object()) # type: ignore[union-attr] + with self.assertRaises(JobBuildSuperseded): + deferred.result() + metrics = "\n".join(server.job_build_metrics_lines()) + self.assertIn( + 'qbit_prism_job_build_priority_events_total{result="routine_deferred"} 1', + metrics, + ) + self.assertIn( + 'qbit_prism_initial_job_prepared_work_total{result="singleflight"} 1', + metrics, + ) + + def test_publication_critical_build_preempts_routine_builder_capacity( + self, + ) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + old_artifacts = server.store_template_artifacts(dict(rpc.template)) + new_artifacts = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert old_artifacts is not None and new_artifacts is not None + routine = self.request_for(server, old_artifacts) + latest = self.request_for( + server, + new_artifacts, + publication_critical=True, + request_source="tip_refresh", + ) + routine_flight = JobBuildFlight(routine) # type: ignore[arg-type] + service._active = routine_flight + service._start_locked = ( # type: ignore[method-assign] + lambda request: JobBuildFlight(request) + ) + service._arm_locked = lambda _flight: None # type: ignore[method-assign] + + promise = service.request_build(latest) # type: ignore[arg-type] + + self.assertIs(promise, latest.promise) # type: ignore[union-attr] + self.assertTrue(routine.cancellation.is_set()) # type: ignore[union-attr] + self.assertIs(service._retiring, routine_flight) + assert service._active is not None + self.assertIs(service._active.request, latest) + self.assertEqual(service._priority_counts["routine_preempted"], 1) + + def test_publication_critical_build_restarts_unhealthy_exact_flight( + self, + ) -> None: + for unhealthy in ("almost_expired", "stalled"): + with self.subTest(unhealthy=unhealthy): + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + routine = self.request_for(server, artifacts) + now = time.monotonic() + if unhealthy == "almost_expired": + routine.cancellation.started_monotonic = now - 59.99 + routine.cancellation.deadline_monotonic = now + 0.01 + routine.cancellation.last_checkpoint_monotonic = now + else: + routine.cancellation.started_monotonic = now - 1.0 + routine.cancellation.deadline_monotonic = now + 59.0 + routine.cancellation.last_checkpoint_monotonic = ( + now - service._config.cancel_grace_seconds - 0.01 + ) + priority_requested = now - 59.0 + latest = self.request_for( + server, + artifacts, + publication_critical=True, + request_source="tip_refresh", + priority_requested_monotonic=priority_requested, + ) + routine_flight = JobBuildFlight(routine) # type: ignore[arg-type] + service._active = routine_flight + service._start_locked = ( # type: ignore[method-assign] + lambda request: JobBuildFlight(request) + ) + service._arm_locked = lambda _flight: None # type: ignore[method-assign] + + promise = service.request_build(latest) # type: ignore[arg-type] + + self.assertIs(promise, latest.promise) # type: ignore[union-attr] + self.assertIsNot(promise, routine.promise) # type: ignore[union-attr] + self.assertTrue(routine.cancellation.is_set()) # type: ignore[union-attr] + self.assertIs(service._retiring, routine_flight) + assert service._active is not None + self.assertIs(service._active.request, latest) + self.assertEqual( + latest.requested_monotonic, # type: ignore[union-attr] + priority_requested, + ) + self.assertEqual(service._priority_counts["routine_preempted"], 1) + + def test_publication_priority_precedes_immutable_request_preparation( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + service = server._ensure_job_bundle_service() + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + critical_entered = threading.Event() + release_critical = threading.Event() + initial_entered = threading.Event() + original_new_request = service.new_build_request + + def observed_new_request(*args: object, **kwargs: object) -> object: + request_source = str(kwargs.get("request_source", "routine")) + if request_source == "tip_refresh": + critical_entered.set() + if not release_critical.wait(5): + raise AssertionError("test did not release priority preparation") + elif request_source == "initial": + initial_entered.set() + return original_new_request(*args, **kwargs) # type: ignore[arg-type] + + service.new_build_request = observed_new_request # type: ignore[method-assign] + results: dict[str, list[object]] = {"critical": [], "initial": []} + errors: list[BaseException] = [] + + def build(label: str, *, publication_critical: bool) -> None: + try: + results[label].append( + service.shared_job_bundle( + artifacts, + mode="ready", + publication_critical=publication_critical, + request_source=( + "tip_refresh" if publication_critical else "initial" + ), + ) + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + critical_thread = threading.Thread( + target=build, + args=("critical",), + kwargs={"publication_critical": True}, + ) + initial_thread = threading.Thread( + target=build, + args=("initial",), + kwargs={"publication_critical": False}, + ) + critical_thread.start() + try: + self.assertTrue(critical_entered.wait(5)) + initial_thread.start() + self.assertFalse(initial_entered.wait(0.1)) + self.assertIn( + "qbit_prism_job_build_priority_active 1", + "\n".join(server.job_build_metrics_lines()), + ) + finally: + release_critical.set() + critical_thread.join(5) + initial_thread.join(5) + service.shutdown() + self.assertEqual(errors, []) + self.assertEqual([len(results[label]) for label in results], [1, 1]) + self.assertIs(results["critical"][0], results["initial"][0]) + self.assertFalse(initial_entered.is_set()) + self.assertEqual(service._initial_prepared_work_counts["deferred"], 1) + self.assertEqual(service._initial_prepared_work_counts["cache_hit"], 1) + self.assertEqual(service._priority_admission_seconds["count"], 1) + self.assertGreaterEqual(service._priority_admission_seconds["sum"], 0.1) + + def test_priority_reservation_cancels_admitted_routine_preparation( + self, + ) -> None: + server, rpc = coordinator() + install_fake_bundle_builder(server) + service = server._ensure_job_bundle_service() + payout = server._ensure_payout_state_service() + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + routine_in_lookup = threading.Event() + release_routine = threading.Event() + routine_constructed = threading.Event() + original_usable = payout.usable_ledger_artifact + original_new_request = service.new_build_request + routine_thread: threading.Thread + + def blocked_usable(*args: object, **kwargs: object) -> object: + if threading.current_thread() is routine_thread: + routine_in_lookup.set() + if not release_routine.wait(5): + raise AssertionError("test did not release payout lookup") + return original_usable(*args, **kwargs) # type: ignore[arg-type] + + def observed_new_request(*args: object, **kwargs: object) -> object: + if kwargs.get("request_source") == "initial": + routine_constructed.set() + return original_new_request(*args, **kwargs) # type: ignore[arg-type] + + payout.usable_ledger_artifact = blocked_usable # type: ignore[method-assign] + service.new_build_request = observed_new_request # type: ignore[method-assign] + errors: list[BaseException] = [] + + def build_routine() -> None: + try: + service.shared_job_bundle( + artifacts, + mode="ready", + retry_superseded=False, + request_source="initial", + ) + except BaseException as exc: # noqa: BLE001 - surfaced below + errors.append(exc) + + routine_thread = threading.Thread(target=build_routine) + routine_thread.start() + priority_token: int | None = None + try: + self.assertTrue(routine_in_lookup.wait(5)) + with service._scheduler_lock: + cancellations = tuple( + cancellation_ref() + for cancellation_ref in service._routine_preparations.values() + ) + self.assertEqual(len(cancellations), 1) + self.assertIsNotNone(cancellations[0]) + priority_token, _requested = service.begin_priority_preparation() + assert cancellations[0] is not None + self.assertTrue(cancellations[0].is_set()) + finally: + release_routine.set() + routine_thread.join(5) + if priority_token is not None: + service.finish_priority_preparation(priority_token) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], JobBuildSuperseded) + self.assertFalse(routine_constructed.is_set()) + with service._scheduler_lock: + self.assertEqual(len(service._routine_preparations), 0) + + def test_publication_critical_collection_promotes_past_ready_work(self) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + old_artifacts = server.store_template_artifacts(dict(rpc.template)) + new_artifacts = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert old_artifacts is not None and new_artifacts is not None + ready = self.request_for(server, old_artifacts) + latest = self.request_for( + server, + new_artifacts, + mode="collection", + identity=worker("tq1latest", "tq1latest.rig"), + publication_critical=True, + request_source="tip_refresh", + ) + ready_flight = JobBuildFlight(ready) # type: ignore[arg-type] + service._active = ready_flight + service._pending = latest # type: ignore[assignment] + armed: list[object] = [] + service._start_locked = ( # type: ignore[method-assign] + lambda request: JobBuildFlight(request) + ) + service._arm_locked = armed.append # type: ignore[method-assign] + + service._promote_pending_locked() + + self.assertTrue(ready.cancellation.is_set()) # type: ignore[union-attr] + self.assertIs(service._retiring, ready_flight) + self.assertIsNone(service._pending) + assert service._active is not None + self.assertIs(service._active.request, latest) + self.assertEqual(armed, [service._active]) + self.assertEqual(service._priority_counts["routine_preempted"], 1) + + def test_routine_pending_never_promotes_over_publication_critical_flight( + self, + ) -> None: + for placement in ("active", "retiring"): + with self.subTest(placement=placement): + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + critical_artifacts = server.store_template_artifacts( + dict(rpc.template) + ) + routine_artifacts = server.store_template_artifacts( + base_template(height=11, prevhash="22" * 32) + ) + assert critical_artifacts is not None + assert routine_artifacts is not None + critical = self.request_for( + server, + critical_artifacts, + publication_critical=True, + request_source="tip_refresh", + ) + routine = self.request_for(server, routine_artifacts) + critical_flight = JobBuildFlight(critical) # type: ignore[arg-type] + service._active = ( + critical_flight if placement == "active" else None + ) + service._retiring = ( + critical_flight if placement == "retiring" else None + ) + service._pending = routine # type: ignore[assignment] + service._start_locked = ( # type: ignore[method-assign] + lambda _request: self.fail( + "routine pending work displaced publication-critical work" + ) + ) + + service._promote_pending_locked() + + self.assertIs(service._pending, routine) + self.assertFalse(critical.cancellation.is_set()) # type: ignore[union-attr] + self.assertIs( + service._active if placement == "active" else service._retiring, + critical_flight, + ) + + def test_service_owns_bundle_build_token_success_lifetime(self) -> None: + server, _rpc = coordinator() + service = server._ensure_job_bundle_service() + events: list[str] = [] + expected = SimpleNamespace() + + @contextmanager + def build_token(): + events.append("start") + try: + yield object() + finally: + events.append("finish") + + service._ports = dataclass_replace( # type: ignore[assignment] + service._ports, + start_bundle_build=build_token, + ) + service._shared_job_bundle = lambda *_args, **_kwargs: expected # type: ignore[method-assign] + + actual = service.shared_job_bundle(SimpleNamespace()) # type: ignore[arg-type] + + self.assertIs(actual, expected) + self.assertEqual(events, ["start", "finish"]) + + def test_service_finishes_bundle_build_token_on_exception(self) -> None: + server, _rpc = coordinator() + service = server._ensure_job_bundle_service() + events: list[str] = [] + + @contextmanager + def build_token(): + events.append("start") + try: + yield object() + finally: + events.append("finish") + + def fail(*_args: object, **_kwargs: object) -> CachedJobBundle: + raise RuntimeError("build failed") + + service._ports = dataclass_replace( # type: ignore[assignment] + service._ports, + start_bundle_build=build_token, + ) + service._shared_job_bundle = fail # type: ignore[method-assign] + + with self.assertRaisesRegex(RuntimeError, "build failed"): + service.shared_job_bundle(SimpleNamespace()) # type: ignore[arg-type] + + self.assertEqual(events, ["start", "finish"]) + + def test_build_done_rechecks_cancellation_under_scheduler_lock(self) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + payout_state = server._ensure_payout_state_service() + payout_artifact = payout_state.current_artifact() + request = service.new_build_request( + artifacts, + None, + mode="ready", + payout_state_generation=payout_artifact.generation, + cache_key=("build-done-race",), + ) + result = cached_bundle( + artifacts, + payout_generation=payout_artifact.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + collection_only=False, + ) + completed: Future[CachedJobBundle] = Future() + completed.set_result(result) + flight = JobBuildFlight(request=request, future=completed) + lock_attempted = threading.Event() + original_lock = service._scheduler_lock + + class InstrumentedSchedulerLock: + def __init__(self) -> None: + self.entries = 0 + + def __enter__(self) -> None: + self.entries += 1 + if self.entries > 1: + lock_attempted.set() + original_lock.acquire() + + def __exit__( + self, + _exc_type: object, + _exc_value: object, + _traceback: object, + ) -> None: + original_lock.release() + + instrumented_lock = InstrumentedSchedulerLock() + service._scheduler_lock = instrumented_lock # type: ignore[assignment] + before = dict(service._scheduler_counts) + shared_before = dict(service._shared_build_counts) + try: + with instrumented_lock: + done_thread = threading.Thread( + target=service._build_done, + args=(flight, completed), + ) + done_thread.start() + self.assertTrue(lock_attempted.wait(2.0)) + self.assertTrue(request.cancellation.cancel("shutdown")) + done_thread.join(2.0) + finally: + service._scheduler_lock = original_lock + + self.assertFalse(done_thread.is_alive()) + with self.assertRaises(JobBuildSuperseded): + request.promise.result() + self.assertEqual( + service._scheduler_counts["completions"], + before["completions"] + 1, + ) + self.assertEqual( + service._scheduler_counts["obsolete_results"], + before["obsolete_results"] + 1, + ) + self.assertEqual( + service._shared_build_counts["superseded"], + shared_before["superseded"] + 1, + ) + self.assertEqual( + service._shared_build_counts["completed"], + shared_before["completed"], + ) + with service._cache_lock: + self.assertNotIn(result.key, service._bundle_cache) + + def test_readiness_promotion_is_one_way_and_emits_once(self) -> None: + server, _rpc = coordinator() + service = server._ensure_job_bundle_service() + ready_miners = 0 + events: list[str] = [] + service.set_ready_for_test(False) + service._ports = dataclass_replace( # type: ignore[assignment] + service._ports, + accepted_share_stats=lambda: (0, ready_miners), + clear_retained_collection_refresh=lambda: events.append("clear"), + readiness_promoted=lambda: events.append("pending"), + ) + + self.assertFalse(service.pool_readiness_latched()) + ready_miners = 3 + self.assertTrue(service.pool_readiness_latched()) + self.assertTrue(service.pool_readiness_latched()) + self.assertEqual(events, ["clear", "pending"]) + + +class JobBundleCacheAdmissionTests(unittest.TestCase): + def test_collection_pinning_requires_exact_published_generation(self) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + repository = service.template_repository + published = server.store_template_artifacts(dict(rpc.template)) + assert published is not None + stale = repository.derive( + published.template, + generation=repository.reserve_generation(), + ) + self.assertTrue(repository.store_artifacts(stale)) + current = repository.derive( + stale.template, + generation=repository.reserve_generation(), + ) + self.assertTrue(repository.store_artifacts(current)) + payout_state = server._ensure_payout_state_service() + payout_artifact = payout_state.current_artifact() + service._ports = dataclass_replace( # type: ignore[assignment] + service._ports, + artifacts_buildable=lambda _artifacts: True, + published_snapshot_artifacts=lambda artifacts: ( + artifacts.fingerprint == published.fingerprint + and artifacts.previousblockhash == published.previousblockhash + ), + published_artifacts=lambda: published, + ) + stale_collection = cached_bundle( + stale, + payout_generation=payout_artifact.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + key=("stale-collection",), + ) + exact_published_collection = cached_bundle( + published, + payout_generation=payout_artifact.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + key=("published-collection",), + ) + reusable_ready = cached_bundle( + stale, + payout_generation=payout_artifact.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + key=("reusable-ready",), + collection_only=False, + ) + sentinel = cached_bundle( + current, + payout_generation=payout_artifact.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + collection_only=False, + ) + with service._cache_lock: + service._bundle_cache = OrderedDict( + (("sentinel", index), sentinel) + for index in range(128) + ) + + self.assertFalse( + service.cache_bundle_if_current(stale_collection, stale) + ) + with service._cache_lock: + self.assertEqual(len(service._bundle_cache), 128) + self.assertIn(("sentinel", 0), service._bundle_cache) + self.assertNotIn(stale_collection.key, service._bundle_cache) + service._bundle_cache.clear() + + self.assertTrue( + service.cache_bundle_if_current( + exact_published_collection, + published, + ) + ) + self.assertTrue(service.cache_bundle_if_current(reusable_ready, stale)) + + def test_clear_cannot_linearize_inside_final_cache_admission(self) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + repository = service.template_repository + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + payout_state = server._ensure_payout_state_service() + payout_artifact = payout_state.current_artifact() + built = cached_bundle( + artifacts, + payout_generation=payout_artifact.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + ) + final_validation_started = threading.Event() + release_final_validation = threading.Event() + validation_calls = 0 + + def artifacts_buildable(_artifacts: CachedTemplateArtifacts) -> bool: + nonlocal validation_calls + validation_calls += 1 + if validation_calls == 2: + final_validation_started.set() + release_final_validation.wait(2.0) + return True + + service._ports = dataclass_replace( # type: ignore[assignment] + service._ports, + artifacts_buildable=artifacts_buildable, + published_snapshot_artifacts=lambda _artifacts: False, + published_artifacts=lambda: None, + ) + cache_results: list[bool] = [] + clear_results: list[bool] = [] + clear_finished = threading.Event() + cache_thread = threading.Thread( + target=lambda: cache_results.append( + service.cache_bundle_if_current(built, artifacts) + ) + ) + + def clear_current() -> None: + clear_results.append(repository.clear_if_current(artifacts)) + clear_finished.set() + + cache_thread.start() + self.assertTrue(final_validation_started.wait(2.0)) + clear_thread = threading.Thread(target=clear_current) + clear_thread.start() + self.assertFalse(clear_finished.wait(0.1)) + release_final_validation.set() + cache_thread.join(2.0) + clear_thread.join(2.0) + + self.assertFalse(cache_thread.is_alive()) + self.assertFalse(clear_thread.is_alive()) + self.assertEqual(cache_results, [True]) + self.assertEqual(clear_results, [True]) + self.assertIsNone(repository.current_artifacts()) + with service._cache_lock: + self.assertNotIn(built.key, service._bundle_cache) + + def _template_race(self, *, same_fingerprint: bool) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + repository = service.template_repository + old = server.store_template_artifacts(dict(rpc.template)) + assert old is not None + payout_state = server._ensure_payout_state_service() + payout_state.current_artifact() + payout = payout_state.snapshot() + payout_artifact = payout.published.artifact + assert payout_artifact is not None + new_template = json.loads(json.dumps(old.template)) + if not same_fingerprint: + new_template["height"] = int(new_template["height"]) + 1 + new = repository.derive( + new_template, + generation=repository.reserve_generation(), + ) + old_bundle = cached_bundle( + old, + payout_generation=payout.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + ) + sentinel = cached_bundle( + new, + payout_generation=payout.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + collection_only=False, + ) + with service._cache_lock: + service._bundle_cache = OrderedDict( + (("sentinel", index), sentinel) + for index in range(128) + ) + first_validation = threading.Event() + release_validation = threading.Event() + validation_calls = 0 + validation_lock = threading.Lock() + + def artifacts_buildable(_artifacts: CachedTemplateArtifacts) -> bool: + nonlocal validation_calls + with validation_lock: + validation_calls += 1 + block = validation_calls == 1 + if block: + first_validation.set() + release_validation.wait(2.0) + return True + + service._ports = dataclass_replace( # type: ignore[assignment] + service._ports, + artifacts_buildable=artifacts_buildable, + published_snapshot_artifacts=lambda _artifacts: False, + published_artifacts=lambda: None, + ) + cache_results: list[bool] = [] + cache_thread = threading.Thread( + target=lambda: cache_results.append( + service.cache_bundle_if_current(old_bundle, old) + ) + ) + cache_thread.start() + self.assertTrue(first_validation.wait(2.0)) + + self.assertTrue(repository.store_artifacts(new)) + self.assertIs(repository.current_artifacts(), new) + release_validation.set() + cache_thread.join(2.0) + + self.assertFalse(cache_thread.is_alive()) + self.assertEqual(cache_results, [False]) + with service._cache_lock: + self.assertNotIn(old_bundle.key, service._bundle_cache) + self.assertEqual(len(service._bundle_cache), 128) + self.assertIn(("sentinel", 0), service._bundle_cache) + + def test_old_completion_cannot_cache_after_new_fingerprint_publication( + self, + ) -> None: + self._template_race(same_fingerprint=False) + + def test_old_collection_cannot_cache_after_same_fingerprint_generation( + self, + ) -> None: + self._template_race(same_fingerprint=True) + + def test_old_completion_cannot_cache_after_payout_invalidation(self) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + old = server.store_template_artifacts(dict(rpc.template)) + assert old is not None + payout_state = server._ensure_payout_state_service() + payout_state.current_artifact() + payout = payout_state.snapshot() + payout_artifact = payout.published.artifact + assert payout_artifact is not None + old_bundle = cached_bundle( + old, + payout_generation=payout.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + ) + service._ports = dataclass_replace( # type: ignore[assignment] + service._ports, + artifacts_buildable=lambda _artifacts: True, + published_snapshot_artifacts=lambda _artifacts: False, + ) + payout_validated = threading.Event() + release_validation = threading.Event() + original_snapshot = payout_state.snapshot + snapshot_calls = 0 + + def blocking_snapshot(): # type: ignore[no-untyped-def] + nonlocal snapshot_calls + snapshot = original_snapshot() + snapshot_calls += 1 + if snapshot_calls == 1: + payout_validated.set() + release_validation.wait(2.0) + return snapshot + + payout_state.snapshot = blocking_snapshot # type: ignore[method-assign] + cache_results: list[bool] = [] + cache_thread = threading.Thread( + target=lambda: cache_results.append( + service.cache_bundle_if_current(old_bundle, old) + ) + ) + try: + cache_thread.start() + self.assertTrue(payout_validated.wait(2.0)) + payout_state.block_publication(force=True) + release_validation.set() + cache_thread.join(2.0) + finally: + payout_state.snapshot = original_snapshot # type: ignore[method-assign] + release_validation.set() + + self.assertFalse(cache_thread.is_alive()) + self.assertEqual(cache_results, [False]) + with service._cache_lock: + self.assertNotIn(old_bundle.key, service._bundle_cache) + + def test_payout_fence_wait_does_not_block_template_publication(self) -> None: + server, rpc = coordinator() + service = server._ensure_job_bundle_service() + repository = service.template_repository + old = server.store_template_artifacts(dict(rpc.template)) + assert old is not None + payout_state = server._ensure_payout_state_service() + payout_artifact = payout_state.current_artifact() + payout = payout_state.snapshot() + new_template = json.loads(json.dumps(old.template)) + new_template["height"] = int(new_template["height"]) + 1 + new = repository.derive( + new_template, + generation=repository.reserve_generation(), + ) + old_bundle = cached_bundle( + old, + payout_generation=payout.generation, + payout_artifact_sha256=payout_artifact.prior_balances_sha256, + ) + service._ports = dataclass_replace( # type: ignore[assignment] + service._ports, + artifacts_buildable=lambda _artifacts: True, + published_snapshot_artifacts=lambda _artifacts: False, + published_artifacts=lambda: None, + ) + cache_results: list[bool] = [] + cache_admission_attempted = threading.Event() + original_admission = payout_state.cache_publication_admission + + @contextmanager + def signaling_admission(): # type: ignore[no-untyped-def] + cache_admission_attempted.set() + with original_admission(): + yield + + def cache_old() -> None: + cache_results.append(service.cache_bundle_if_current(old_bundle, old)) + + cache_thread = threading.Thread( + target=cache_old, + ) + new_store_finished = threading.Event() + + def store_new() -> None: + repository.store_artifacts(new) + new_store_finished.set() + + try: + with original_admission(): + payout_state.cache_publication_admission = ( # type: ignore[method-assign] + signaling_admission + ) + cache_thread.start() + self.assertTrue(cache_admission_attempted.wait(2.0)) + store_thread = threading.Thread(target=store_new) + store_thread.start() + self.assertTrue(new_store_finished.wait(2.0)) + store_thread.join(2.0) + finally: + payout_state.cache_publication_admission = ( # type: ignore[method-assign] + original_admission + ) + + cache_thread.join(2.0) + self.assertFalse(cache_thread.is_alive()) + self.assertFalse(store_thread.is_alive()) + self.assertEqual(cache_results, [False]) + self.assertIs(repository.current_artifacts(), new) + with service._cache_lock: + self.assertNotIn(old_bundle.key, service._bundle_cache) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_coordinator_metrics.py b/tests/test_prism_coordinator_metrics.py index 20266f4..144dfb9 100644 --- a/tests/test_prism_coordinator_metrics.py +++ b/tests/test_prism_coordinator_metrics.py @@ -604,19 +604,69 @@ def highdiff_client(self) -> ClientState: state.minimum_advertised_difficulty = Decimal("500000") state.share_difficulty = Decimal("500000") return state - def test_post_block_notification_stamps_caller_heartbeat(self) -> None: + def test_post_block_refresh_runs_on_scheduler_worker(self) -> None: server, _state, _ledger = submit_coordinator() - seen: list[str] = [] - server._record_heartbeat = seen.append # type: ignore[method-assign] - server.refresh_jobs_after_accepted_block( - block_height=10, block_hash="bb" * 32, heartbeat_name="block_submitter" + server.watchdog_timeout_seconds = 0.12 + seen: list[tuple[str, tuple[int, str] | None]] = [] + service = server._ensure_tip_refresh_service() + execute_started = threading.Event() + release_execute = threading.Event() + repeated_heartbeat = threading.Event() + caller_heartbeats: list[float] = [] + + def heartbeat(name: str) -> None: + server._record_heartbeat(name) + if name == "block_submitter": + caller_heartbeats.append(time.monotonic()) + if len(caller_heartbeats) >= 5: + repeated_heartbeat.set() + + def fake_execute(trigger: object) -> int: + seen.append( + ( + threading.current_thread().name, + getattr(trigger, "post_accept_block"), + ) + ) + execute_started.set() + self.assertTrue(release_execute.wait(2.0)) + return 0 + + service.reconfigure_ports_for_test(heartbeat=heartbeat) + service._execute_refresh_trigger = fake_execute # type: ignore[method-assign] + results: list[int] = [] + caller = threading.Thread( + target=lambda: results.append( + server.refresh_jobs_after_accepted_block( + block_height=10, + block_hash="bb" * 32, + heartbeat_name="block_submitter", + ) + ) ) - self.assertEqual(seen, ["block_submitter"]) - - # The client-thread pending refresh keeps the default poller heartbeat. - seen.clear() + caller.start() + self.assertTrue(execute_started.wait(1.0)) + self.assertTrue(repeated_heartbeat.wait(1.0)) + self.assertGreaterEqual( + caller_heartbeats[-1] - caller_heartbeats[0], + server.watchdog_timeout_seconds, + ) + self.assertNotIn( + "block_submitter", + server._overdue_heartbeats(time.monotonic()), + ) + release_execute.set() + caller.join(1.0) + self.assertFalse(caller.is_alive()) + self.assertEqual(results, [0]) server.refresh_jobs_after_accepted_block(block_height=11, block_hash="cc" * 32) - self.assertEqual(seen, ["qbit_blockpoll"]) + self.assertEqual( + seen, + [ + ("prism-tip-refresh-scheduler", (10, "bb" * 32)), + ("prism-tip-refresh-scheduler", (11, "cc" * 32)), + ], + ) class HealthSnapshotTests(_JobSupportTestCase): def test_health_payload_uses_aggregate_stats_not_all_shares(self) -> None: diff --git a/tests/test_prism_coordinator_shutdown.py b/tests/test_prism_coordinator_shutdown.py index eaa35bc..e3e7593 100644 --- a/tests/test_prism_coordinator_shutdown.py +++ b/tests/test_prism_coordinator_shutdown.py @@ -11,14 +11,17 @@ from unittest.mock import patch from lab.prism import prism_coordinator -from lab.prism.prism_coordinator import ( +from lab.prism.coordinator_shutdown import ( CoordinatorShutdownController, + ShutdownInProgress, +) +from lab.prism.prism_coordinator import ( PendingShareAppend, PRISM_REJECTION_POOL_CLOSED, PrismCoordinator, - ShutdownInProgress, StratumError, ) +from lab.prism.share_writer import PendingShareInput class RecordingLeaseLedger: @@ -48,7 +51,68 @@ def coordinator( return server +class CoordinatorShutdownControllerTests(unittest.TestCase): + def test_compatibility_reexports_reference_shutdown_owner(self) -> None: + self.assertIs( + prism_coordinator.CoordinatorShutdownController, + CoordinatorShutdownController, + ) + self.assertIs(prism_coordinator.ShutdownInProgress, ShutdownInProgress) + + def test_nested_writer_inherits_admission_after_shutdown_request(self) -> None: + controller = CoordinatorShutdownController(0.5) + outer = controller.enter_writer("outer") + controller.request_shutdown(signal.SIGTERM) + inner = controller.enter_writer("inner") + + controller.exit_writer(inner) + controller.exit_writer(outer) + + self.assertEqual(controller.snapshot()["active_writers"], {}) + with self.assertRaisesRegex(ShutdownInProgress, "coordinator is shutting down"): + controller.enter_writer("late") + + def test_transferable_writer_token_finishes_idempotently_on_another_thread( + self, + ) -> None: + controller = CoordinatorShutdownController(0.5) + token = controller.reserve_writer("share_persistence") + + finisher = threading.Thread(target=lambda: (token.finish(), token.finish())) + finisher.start() + finisher.join(1) + + self.assertFalse(finisher.is_alive()) + self.assertTrue(token.finished) + self.assertEqual(controller.snapshot()["active_writers"], {}) + + class PrismCoordinatorShutdownTests(unittest.TestCase): + def test_refresh_timeout_still_drains_build_executors(self) -> None: + server = coordinator() + calls: list[str] = [] + server._ensure_job_delivery_service = ( # type: ignore[method-assign] + lambda: SimpleNamespace( + shutdown_initial_executor=lambda: calls.append("initial") + ) + ) + server._ensure_tip_refresh_service = ( # type: ignore[method-assign] + lambda: SimpleNamespace(shutdown=lambda: calls.append("refresh") or False) + ) + server.shutdown_job_build_executor = ( # type: ignore[method-assign] + lambda: calls.append("job_build") + ) + server.shutdown_payout_artifact_executor = ( # type: ignore[method-assign] + lambda: calls.append("payout_artifact") + ) + + server.shutdown_tip_refresh_executor() + + self.assertEqual( + calls, + ["initial", "refresh", "job_build", "payout_artifact"], + ) + def test_normal_shutdown_releases_lease_promptly_and_exports_metrics(self) -> None: ledger = RecordingLeaseLedger() server = coordinator(ledger) @@ -198,6 +262,9 @@ def producer() -> None: writer_thread.join(1) shutdown_thread.join(1) + self.assertFalse(producer_thread.is_alive(), "producer thread leaked") + self.assertFalse(writer_thread.is_alive(), "share writer thread leaked") + self.assertFalse(shutdown_thread.is_alive(), "shutdown thread leaked") self.assertTrue(entry.committed.is_set()) self.assertEqual(ledger.release_calls, 1) @@ -333,13 +400,17 @@ def rejected_replay() -> int: def test_blockpoll_shutdown_race_does_not_take_hard_exit_path(self) -> None: server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] def rejected_poll() -> int: server.request_shutdown(signal.SIGTERM) raise ShutdownInProgress("PRISM coordinator is shutting down") - server.poll_qbit_tip_template_once = rejected_poll # type: ignore[method-assign] + tip_refresh = server._ensure_tip_refresh_service() + tip_refresh.poll_once = rejected_poll # type: ignore[method-assign] with patch("lab.prism.prism_coordinator.os._exit") as hard_exit: server.blockpoll_loop() @@ -400,6 +471,96 @@ def test_startup_replays_stop_cleanly_after_shutdown_admission_closes(self) -> N ], ) + def test_startup_replay_shutdown_cancels_gated_candidate_before_quiescence( + self, + ) -> None: + class StartupRaceLedger(RecordingLeaseLedger): + def __init__(self) -> None: + super().__init__() + self.append_calls = 0 + + def append(self, pending: object) -> object: + self.append_calls += 1 + return pending + + ledger = StartupRaceLedger() + server = coordinator(ledger, timeout=0.5) + share_writer = server._ensure_share_writer_service() + pending = share_writer.make_pending_share( + PendingShareInput( + share_id="miner-a:startup-race", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + ntime=1_700_000_000, + ) + ) + entry = PendingShareAppend( + pending_share=pending, + username="miner-a", + job_id="job-1", + block_hash_hex="aa" * 32, + collection_only=False, + credit_policy=None, + ) + share_writer.begin_startup_recovery() + entered = threading.Event() + errors: list[BaseException] = [] + + def gated_candidate() -> None: + try: + with server._writer_operation("accepted_block_handling"): + entered.set() + share_writer.append_and_wait(entry) + except BaseException as exc: + errors.append(exc) + + writer = threading.Thread(target=gated_candidate) + writer.start() + self.assertTrue(entered.wait(0.2)) + self.assertEqual( + server._ensure_shutdown_controller().snapshot()["active_writers"], + {"accepted_block_handling": 1}, + ) + + def rejected_replay() -> int: + server.request_shutdown(signal.SIGTERM) + raise ShutdownInProgress("PRISM coordinator is shutting down") + + started = time.monotonic() + with patch("builtins.print"): + self.assertFalse( + server._run_startup_writer_replay( + rejected_replay, + drain_threads=[], + before_shutdown=share_writer.cancel_startup_recovery, + ) + ) + elapsed = time.monotonic() - started + # Match serve()'s unconditional finally without erasing cancellation. + share_writer.finish_startup_recovery() + writer.join(1) + + self.assertLess(elapsed, 0.25) + self.assertFalse(writer.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], ShutdownInProgress) + self.assertEqual(ledger.append_calls, 0) + self.assertEqual(ledger.release_calls, 1) + snapshot = server._ensure_shutdown_controller().snapshot() + self.assertEqual(snapshot["active_writers"], {}) + self.assertFalse(snapshot["lease_release_withheld"]) + self.assertEqual(snapshot["release_withheld_total"], 0) + self.assertEqual(snapshot["lease_release_outcomes"]["success"], 1) + with patch("builtins.print"): + self.assertTrue(server.shutdown(reason="main_finally")) + self.assertEqual(ledger.release_calls, 1) + def test_replacement_can_acquire_immediately_after_graceful_release(self) -> None: lease_lock = threading.Lock() holder: list[str | None] = [None] diff --git a/tests/test_prism_ctv_refresh_priority.py b/tests/test_prism_ctv_refresh_priority.py index 3ad28dc..37ad8e8 100644 --- a/tests/test_prism_ctv_refresh_priority.py +++ b/tests/test_prism_ctv_refresh_priority.py @@ -201,13 +201,13 @@ def test_poll_local_mark_cannot_replace_newer_pending_work(self) -> None: ) self.assertIsNone(retained_token) - self.assertEqual(server._tip_refresh_pending_token, newer_token) + self.assertEqual(server._ensure_tip_refresh_service().snapshot().pending_token, newer_token) self.assertTrue(server._tip_refresh_pending()) def test_routine_same_tip_observation_does_not_starve_ctv(self) -> None: server = coordinator() cancellations: list[bool] = [] - server._active_tip_refresh = ( # type: ignore[assignment] + server._ensure_tip_refresh_service().seed_active_refresh_for_test( SimpleNamespace(tip_hash=OLD_TIP, observation_sequence=1), SimpleNamespace(cancel=lambda: cancellations.append(True)), ) diff --git a/tests/test_prism_ctv_runtime.py b/tests/test_prism_ctv_runtime.py new file mode 100644 index 0000000..c751027 --- /dev/null +++ b/tests/test_prism_ctv_runtime.py @@ -0,0 +1,544 @@ +"""Direct tests for the coordinator-facing CTV runtime service.""" + +from __future__ import annotations + +from contextlib import contextmanager +import threading +import unittest +from unittest.mock import patch + +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.ctv_broadcaster_daemon import ( + CtvFanoutChunkResult, + CtvFanoutDaemonResult, +) +from lab.prism.ctv_runtime import ( + CTV_BROADCAST_STATE_COMPONENT, + CtvRuntimeConfig, + CtvRuntimeService, +) +from lab.prism.prism_coordinator import PrismCoordinator + + +class StopAfterOnePass: + def is_set(self) -> bool: + return False + + def wait(self, _timeout: float) -> bool: + return True + + +class PrismCtvRuntimeTests(unittest.TestCase): + @staticmethod + def config(**overrides: object) -> CtvRuntimeConfig: + values: dict[str, object] = { + "enabled": True, + "wallet": None, + "fee_sats": 0, + "limit": 7, + "chunk_size": 2, + "interval_seconds": 30.0, + } + values.update(overrides) + return CtvRuntimeConfig(**values) # type: ignore[arg-type] + + def test_run_once_constructs_daemon_and_holds_exact_writer_admission(self) -> None: + ledger = object() + admission: list[str] = [] + captured: dict[str, object] = {} + heartbeats: list[bool] = [] + + def rpc_call(*_args: object, **_kwargs: object) -> None: + return None + + class FakeBroadcaster: + def __init__(self, call: object, *, funding_wallet: str | None) -> None: + captured["rpc_call"] = call + captured["wallet"] = funding_wallet + + class FakeDaemon: + def __init__( + self, + daemon_ledger: object, + broadcaster: object, + *, + fee_sats: int, + ) -> None: + captured["ledger"] = daemon_ledger + captured["broadcaster"] = broadcaster + captured["fee_sats"] = fee_sats + + def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: + captured.update(kwargs) + progress_callback = kwargs["progress_callback"] + chunk_callback = kwargs["chunk_callback"] + tip_refresh_pending = kwargs["tip_refresh_pending"] + assert callable(progress_callback) + assert callable(chunk_callback) + assert callable(tip_refresh_pending) + progress_callback() + chunk_callback( + CtvFanoutChunkResult(processed_count=1, elapsed_seconds=0.25) + ) + captured["tip_pending"] = tip_refresh_pending() + return CtvFanoutDaemonResult(1, 0, 1, 0) + + @contextmanager + def writer_admission(component: str): + admission.append(f"enter:{component}") + try: + yield + finally: + admission.append(f"exit:{component}") + + service = CtvRuntimeService( + rpc_call=rpc_call, + ledger=ledger, + writer_admission=writer_admission, + tip_refresh_pending=lambda: False, + heartbeat=lambda: heartbeats.append(True), + stop_event=threading.Event(), + config=self.config(wallet="fee-wallet", fee_sats=900), + daemon_type=FakeDaemon, # type: ignore[arg-type] + broadcaster_type=FakeBroadcaster, # type: ignore[arg-type] + ) + + result = service.run_once(progress_callback=service.record_progress) + + self.assertEqual(result.updated_count, 1) + self.assertEqual( + admission, + [ + f"enter:{CTV_BROADCAST_STATE_COMPONENT}", + f"exit:{CTV_BROADCAST_STATE_COMPONENT}", + ], + ) + self.assertIs(captured["ledger"], ledger) + self.assertIs(captured["rpc_call"], rpc_call) + self.assertEqual(captured["wallet"], "fee-wallet") + self.assertEqual(captured["fee_sats"], 900) + self.assertEqual(captured["limit"], 7) + self.assertEqual(captured["chunk_size"], 2) + self.assertFalse(captured["tip_pending"]) + self.assertEqual(len(heartbeats), 2) + metrics = "\n".join(service.metrics_lines()) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_processed_rows_total 1", + metrics, + ) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_chunk_seconds_sum 0.250000", + metrics, + ) + + def test_closed_writer_admission_does_not_construct_daemon(self) -> None: + constructed: list[bool] = [] + + class RejectingAdmission: + def __enter__(self) -> None: + raise ShutdownInProgress("closed") + + def __exit__(self, *_args: object) -> None: + return None + + class UnexpectedDaemon: + def __init__(self, *_args: object, **_kwargs: object) -> None: + constructed.append(True) + + service = CtvRuntimeService( + rpc_call=lambda *_args, **_kwargs: None, + ledger=object(), + writer_admission=lambda _component: RejectingAdmission(), + tip_refresh_pending=lambda: False, + heartbeat=lambda: None, + stop_event=threading.Event(), + config=self.config(), + daemon_type=UnexpectedDaemon, # type: ignore[arg-type] + ) + + with self.assertRaises(ShutdownInProgress): + service.run_once() + + self.assertEqual(constructed, []) + + def test_loop_records_completion_before_wait_and_preserves_summary_and_spec(self) -> None: + clock_values = iter((10.0, 112.0)) + heartbeats: list[float] = [] + wait_observation: dict[str, object] = {} + + class ObservedStop(StopAfterOnePass): + def wait(self, timeout: float) -> bool: + wait_observation["timeout"] = timeout + wait_observation["pass_count"] = service.pass_count + return True + + @contextmanager + def writer_admission(_component: str): + yield + + class OnePassDaemon: + def run_once(self, **_kwargs: object) -> CtvFanoutDaemonResult: + return CtvFanoutDaemonResult(0, 0, 0, 0, True) + + service = CtvRuntimeService( + rpc_call=lambda *_args, **_kwargs: None, + ledger=object(), + writer_admission=writer_admission, + tip_refresh_pending=lambda: False, + heartbeat=lambda: heartbeats.append(1.0), + stop_event=ObservedStop(), + config=self.config(), + monotonic=lambda: next(clock_values), + ) + service.daemon = OnePassDaemon() # type: ignore[assignment] + + service.loop() + + self.assertEqual(wait_observation, {"timeout": 30.0, "pass_count": 1}) + self.assertEqual(len(heartbeats), 2) + metrics = "\n".join(service.metrics_lines()) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_pass_seconds_sum 102.000000", + metrics, + ) + self.assertIn( + "qbit_prism_ctv_fanout_broadcaster_tip_refresh_yields_total 1", + metrics, + ) + specification = service.background_service_spec() + self.assertEqual(specification.name, "ctv_fanout_broadcaster") + self.assertEqual(specification.thread_name, "prism-ctv-fanout-broadcaster") + self.assertEqual(specification.join_timeout, 1.0) + self.assertTrue(specification.watchdog_monitored) + self.assertEqual( + service.startup_summary(), + "prism coordinator: CTV fanout broadcaster enabled " + "mode=direct fee_bits=0 wallet=none interval=30s limit=7 chunk_size=2", + ) + + def test_cpfp_wallet_validation_remains_at_daemon_construction(self) -> None: + @contextmanager + def writer_admission(_component: str): + yield + + service = CtvRuntimeService( + rpc_call=lambda *_args, **_kwargs: None, + ledger=object(), + writer_admission=writer_admission, + tip_refresh_pending=lambda: False, + heartbeat=lambda: None, + stop_event=threading.Event(), + config=self.config(wallet=None, fee_sats=1), + ) + + with self.assertRaisesRegex( + ValueError, + "ctv_broadcaster_wallet is required", + ): + service.make_daemon() + + def test_lazy_handoff_consumes_overrides_and_updates_only_live_config(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.ctv_broadcaster_limit = 7 + + self.assertEqual( + server.__dict__["_ctv_runtime_compat_config"], + {"limit": 7}, + ) + + runtime = server._ensure_ctv_runtime() + + self.assertEqual(runtime.config.limit, 7) + self.assertNotIn("_ctv_runtime_compat_config", server.__dict__) + + server.ctv_broadcaster_limit = 11 + + self.assertIs(server._ensure_ctv_runtime(), runtime) + self.assertEqual(runtime.config.limit, 11) + self.assertNotIn("_ctv_runtime_compat_config", server.__dict__) + self.assertEqual(server._legacy_ctv_runtime_config().limit, 100) + + def test_wallet_or_fee_config_change_rebuilds_daemon_for_next_pass(self) -> None: + constructions: list[tuple[str | None, int]] = [] + runs: list[tuple[int, int]] = [] + + class FakeBroadcaster: + def __init__( + self, + _rpc_call: object, + *, + funding_wallet: str | None, + ) -> None: + self.wallet = funding_wallet + + class FakeDaemon: + def __init__( + self, + _ledger: object, + broadcaster: FakeBroadcaster, + *, + fee_sats: int, + ) -> None: + constructions.append((broadcaster.wallet, fee_sats)) + + def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: + runs.append((int(kwargs["limit"]), int(kwargs["chunk_size"]))) + return CtvFanoutDaemonResult(0, 0, 0, 0) + + @contextmanager + def writer_admission(_component: str): + yield + + service = CtvRuntimeService( + rpc_call=lambda *_args, **_kwargs: None, + ledger=object(), + writer_admission=writer_admission, + tip_refresh_pending=lambda: False, + heartbeat=lambda: None, + stop_event=threading.Event(), + config=self.config(), + daemon_type=FakeDaemon, # type: ignore[arg-type] + broadcaster_type=FakeBroadcaster, # type: ignore[arg-type] + ) + + service.run_once() + first = service.daemon + service.replace_config(limit=11, chunk_size=4) + service.run_once() + self.assertIs(service.daemon, first) + + service.replace_config(wallet="fee-wallet", fee_sats=900) + service.run_once() + + self.assertIsNot(service.daemon, first) + self.assertEqual(constructions, [(None, 0), ("fee-wallet", 900)]) + self.assertEqual(runs, [(7, 2), (11, 4), (11, 4)]) + + def test_concurrent_live_config_updates_retain_both_frozen_fields(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + runtime = server._ensure_ctv_runtime() + start = threading.Barrier(3) + errors: list[BaseException] = [] + + def update_limit() -> None: + try: + start.wait(timeout=2.0) + server.ctv_broadcaster_limit = 17 + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def update_chunk_size() -> None: + try: + start.wait(timeout=2.0) + server.ctv_broadcaster_chunk_size = 3 + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + threads = [ + threading.Thread(target=update_limit), + threading.Thread(target=update_chunk_size), + ] + for thread in threads: + thread.start() + start.wait(timeout=2.0) + for thread in threads: + thread.join(timeout=2.0) + + self.assertTrue(all(not thread.is_alive() for thread in threads)) + self.assertEqual(errors, []) + snapshot = runtime.config + self.assertEqual(snapshot.limit, 17) + self.assertEqual(snapshot.chunk_size, 3) + + def test_lazy_handoff_is_singleton_under_deterministic_contention(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + arrivals = threading.Barrier(2) + underlying_lock = threading.Lock() + candidates: list[CtvRuntimeService] = [] + results: list[CtvRuntimeService] = [] + errors: list[BaseException] = [] + + class CoordinatedInitLock: + def __enter__(self) -> None: + arrivals.wait(timeout=2.0) + underlying_lock.acquire() + + def __exit__(self, *_args: object) -> None: + underlying_lock.release() + + server._ctv_runtime_init_lock = CoordinatedInitLock() # type: ignore[assignment] + original_make = server._make_ctv_runtime_service + + def counted_make( + config: CtvRuntimeConfig | None = None, + ) -> CtvRuntimeService: + candidate = original_make(config) + candidates.append(candidate) + return candidate + + server._make_ctv_runtime_service = counted_make # type: ignore[method-assign] + + def ensure_runtime() -> None: + try: + results.append(server._ensure_ctv_runtime()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + threads = [threading.Thread(target=ensure_runtime) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=2.0) + + self.assertTrue(all(not thread.is_alive() for thread in threads)) + self.assertEqual(errors, []) + self.assertEqual(len(candidates), 1) + self.assertEqual(len(results), 2) + self.assertIs(results[0], candidates[0]) + self.assertIs(results[1], candidates[0]) + marker = object() + results[0].daemon = marker # type: ignore[assignment] + self.assertIs(results[1].daemon, marker) + results[0].record_progress() + results[1].record_progress() + self.assertEqual(results[0].processed_rows_total, 2) + + def test_daemon_getter_waits_for_paused_lazy_handoff_publish(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + daemon_marker = object() + server.ctv_fanout_broadcast_daemon = daemon_marker # type: ignore[assignment] + underlying_lock = threading.Lock() + getter_attempted = threading.Event() + factory_popped_compat = threading.Event() + release_factory = threading.Event() + getter_returned = threading.Event() + runtime_results: list[CtvRuntimeService] = [] + daemon_results: list[object | None] = [] + errors: list[BaseException] = [] + + class ObservedInitLock: + def __enter__(self) -> None: + if threading.current_thread().name == "ctv-daemon-getter": + getter_attempted.set() + underlying_lock.acquire() + + def __exit__(self, *_args: object) -> None: + underlying_lock.release() + + server._ctv_runtime_init_lock = ObservedInitLock() # type: ignore[assignment] + original_make = server._make_ctv_runtime_service + + def paused_make( + config: CtvRuntimeConfig | None = None, + ) -> CtvRuntimeService: + runtime = original_make(config) + if "_ctv_runtime_compat_daemon" in server.__dict__: + raise AssertionError("compat daemon was not consumed") + factory_popped_compat.set() + if not release_factory.wait(timeout=2.0): + raise AssertionError("test did not release runtime publication") + return runtime + + server._make_ctv_runtime_service = paused_make # type: ignore[method-assign] + + def initialize() -> None: + try: + runtime_results.append(server._ensure_ctv_runtime()) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + + def read_daemon() -> None: + try: + daemon_results.append(server.ctv_fanout_broadcast_daemon) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + finally: + getter_returned.set() + + initializer = threading.Thread(target=initialize, name="ctv-runtime-initializer") + getter = threading.Thread(target=read_daemon, name="ctv-daemon-getter") + initializer.start() + self.assertTrue(factory_popped_compat.wait(timeout=2.0)) + getter.start() + self.assertTrue(getter_attempted.wait(timeout=2.0)) + self.assertFalse(getter_returned.is_set()) + + release_factory.set() + initializer.join(timeout=2.0) + getter.join(timeout=2.0) + + self.assertFalse(initializer.is_alive()) + self.assertFalse(getter.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(len(runtime_results), 1) + self.assertEqual(daemon_results, [daemon_marker]) + self.assertIs(runtime_results[0].daemon, daemon_marker) + self.assertNotIn("_ctv_runtime_compat_daemon", server.__dict__) + + def test_coordinator_facade_routes_every_retained_loop_patch_seam(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + server.stop_event = StopAfterOnePass() # type: ignore[assignment] + server.ctv_broadcaster_limit = 7 + server.ctv_broadcaster_chunk_size = 2 + server.ctv_broadcaster_interval_seconds = 30.0 + server.tip_refresh_is_pending = lambda: False # type: ignore[method-assign] + server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + events: list[object] = [] + + @contextmanager + def writer_admission(component: str): + events.append(("admit", component)) + yield + + server._writer_operation = writer_admission # type: ignore[method-assign] + + class FacadeDaemon: + def run_once(self, **kwargs: object) -> CtvFanoutDaemonResult: + progress_callback = kwargs["progress_callback"] + chunk_callback = kwargs["chunk_callback"] + assert callable(progress_callback) + assert callable(chunk_callback) + progress_callback() + chunk_callback( + CtvFanoutChunkResult(processed_count=2, elapsed_seconds=0.25) + ) + return CtvFanoutDaemonResult(2, 0, 2, 0, True) + + server.ctv_fanout_broadcast_daemon = FacadeDaemon() # type: ignore[assignment] + server._record_ctv_fanout_broadcaster_progress = ( # type: ignore[method-assign] + lambda: events.append("progress") + ) + server.observe_ctv_fanout_broadcaster_chunk = ( # type: ignore[method-assign] + lambda result: events.append(("chunk", result.processed_count)) + ) + server.observe_ctv_fanout_broadcaster_pass = ( # type: ignore[method-assign] + lambda elapsed: events.append(("pass", elapsed)) + ) + server._record_ctv_fanout_broadcaster_yield = ( # type: ignore[method-assign] + lambda: events.append("yield") + ) + original_run_once = server.run_ctv_fanout_broadcaster_once + + def patched_run_once(**kwargs: object) -> CtvFanoutDaemonResult: + events.append("run_once") + return original_run_once(**kwargs) # type: ignore[arg-type] + + server.run_ctv_fanout_broadcaster_once = patched_run_once # type: ignore[method-assign] + + with patch("builtins.print"): + server.ctv_fanout_broadcaster_loop() + + self.assertEqual( + [event if isinstance(event, str) else event[0] for event in events], + ["run_once", "admit", "progress", "chunk", "pass", "yield"], + ) + self.assertEqual(events[1], ("admit", CTV_BROADCAST_STATE_COMPONENT)) + runtime = server._ensure_ctv_runtime() + specification = runtime.background_service_spec() + self.assertIs(getattr(specification.target, "__self__", None), runtime) + self.assertIs(getattr(specification.target, "__func__", None), CtvRuntimeService.loop) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_hot_path.py b/tests/test_prism_hot_path.py index 92ae853..20fbde3 100644 --- a/tests/test_prism_hot_path.py +++ b/tests/test_prism_hot_path.py @@ -290,7 +290,6 @@ def coordinator( server.duplicate_share_count = 0 server.low_difficulty_share_count = 0 server.rejection_counts_by_reason = {reason: 0 for reason in PRISM_REJECTION_REASON_IDS} - server.job_build_failure_count = 0 server.tip_refresh_job_count = 0 server.post_accept_refresh_failure_count = 0 server.reorg_reconciler_enabled = False @@ -326,10 +325,10 @@ def coordinator( server.min_ready_miners = 3 server.ledger = FakeAppendLedger() server.blockpoll_seconds = 2.0 - # Failed-refresh spacing is opt-in per test: its holdoff waits on real - # time, which deadlocks tests that freeze time.monotonic around failing - # polls. Pacing behavior is covered by test_prism_refresh_retry_pacing. - server.tip_refresh_failure_holdoff_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds, + failure_holdoff_seconds=0.0, + ) server.job_bundle_cache_seconds = 10.0 server.template_cache_seconds = 2.0 server.reorg_reconcile_cache_seconds = 5.0 @@ -377,6 +376,9 @@ def fake_build_audit_bundle(**kwargs: object) -> dict[str, object]: } server.build_audit_bundle = fake_build_audit_bundle # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + fake_build_audit_bundle + ) return recorded @@ -533,6 +535,9 @@ def test_submit_falls_back_to_rpc_when_observation_goes_stale(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.submit_tip_max_age_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + submit_tip_max_age_seconds=server.submit_tip_max_age_seconds + ) state = client(1) context = register_job(server, state) params = submit_params(state, context) @@ -553,7 +558,13 @@ def test_submit_keeps_published_tip_during_bounded_replacement_build(self) -> No server, rpc = coordinator() install_fake_bundle_builder(server) server.submit_tip_max_age_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + submit_tip_max_age_seconds=server.submit_tip_max_age_seconds + ) server.template_refresh_failure_exit_seconds = 120.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) state = client(1) context = register_job(server, state) params = submit_params(state, context) @@ -574,7 +585,13 @@ def test_submit_keeps_published_tip_during_bounded_replacement_build(self) -> No def test_unpublished_divergence_lease_does_not_renew_and_expires(self) -> None: server, rpc = coordinator() server.submit_tip_max_age_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + submit_tip_max_age_seconds=server.submit_tip_max_age_seconds + ) server.template_refresh_failure_exit_seconds = 120.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) observe_tip(server, OLD_TIP, age_seconds=11.0) rpc.tip = NEW_TIP self.assertTrue(server.observe_tip_for_refresh(NEW_TIP)) @@ -601,6 +618,9 @@ def test_submit_tip_max_age_zero_restores_per_share_rpc(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.submit_tip_max_age_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + submit_tip_max_age_seconds=server.submit_tip_max_age_seconds + ) state = client(1) context = register_job(server, state) params = submit_params(state, context) @@ -909,6 +929,9 @@ def test_driver_live_trust_check_aborts_queued_delivery(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) server.reorg_reconciler_enabled = True def reconcile_live_chain_view(tip_hash: str) -> bool: diff --git a/tests/test_prism_immutable_refresh_artifacts.py b/tests/test_prism_immutable_refresh_artifacts.py index f6af607..05df268 100644 --- a/tests/test_prism_immutable_refresh_artifacts.py +++ b/tests/test_prism_immutable_refresh_artifacts.py @@ -26,7 +26,8 @@ class ImmutableRefreshArtifactTests(unittest.TestCase): def test_fetched_snapshot_owns_exact_derived_artifacts(self) -> None: server, _ = coordinator() - original_store = server.store_template_artifacts + repository = server._ensure_job_bundle_service().template_repository + original_store = repository.store stored: list[CachedTemplateArtifacts | None] = [] def recording_store( @@ -38,7 +39,7 @@ def recording_store( stored.append(artifacts) return artifacts - server.store_template_artifacts = recording_store # type: ignore[method-assign] + repository.store = recording_store # type: ignore[method-assign] snapshot = server.fetch_qbit_tip_template_snapshot() @@ -65,7 +66,8 @@ def recording_store( def test_fetch_fails_closed_when_exact_artifacts_cannot_be_derived(self) -> None: server, _ = coordinator() - server.store_template_artifacts = ( # type: ignore[method-assign] + repository = server._ensure_job_bundle_service().template_repository + repository.store = ( # type: ignore[method-assign] lambda _template, *, generation=None: None ) @@ -73,7 +75,9 @@ def test_fetch_fails_closed_when_exact_artifacts_cannot_be_derived(self) -> None server.fetch_qbit_tip_template_snapshot() self.assertIsNone(server.tip_template_snapshot) - self.assertIsNone(server._template_artifacts) + self.assertIsNone( + server._ensure_job_bundle_service().template_repository.current_artifacts() + ) def test_prepare_rejects_mismatched_snapshot_artifact_invariants(self) -> None: server, _ = coordinator() @@ -144,7 +148,8 @@ def test_concurrent_global_cache_replacement_does_not_abort_snapshot_artifacts( build_entered = threading.Event() release_build = threading.Event() - original_shared_job_bundle = server.shared_job_bundle + job_bundles = server._ensure_job_bundle_service() + original_shared_job_bundle = job_bundles.shared_job_bundle bundles: list[CachedJobBundle] = [] errors: list[BaseException] = [] @@ -165,7 +170,7 @@ def prepare() -> None: except BaseException as exc: # noqa: BLE001 - surfaced by the test errors.append(exc) - server.shared_job_bundle = blocked_shared_job_bundle # type: ignore[method-assign] + job_bundles.shared_job_bundle = blocked_shared_job_bundle # type: ignore[method-assign] thread = threading.Thread(target=prepare) thread.start() try: @@ -176,7 +181,10 @@ def prepare() -> None: self.assertIsNotNone(artifacts_b) assert artifacts_b is not None self.assertNotEqual(artifacts_a.fingerprint, artifacts_b.fingerprint) - self.assertIs(server._template_artifacts, artifacts_b) + self.assertIs( + server._ensure_job_bundle_service().template_repository.current_artifacts(), + artifacts_b, + ) finally: release_build.set() thread.join(5) @@ -210,7 +218,7 @@ def test_newer_observation_supersedes_old_build_before_send_and_runs_promptly( template_b = base_template(height=11, prevhash=tip_b) server, rpc = coordinator(template=template_a) install_fake_bundle_builder(server) - server._pool_ready_latched = True + server._ensure_job_bundle_service().set_ready_for_test(True) states = [client(1), client(2)] server.clients = set(states) sent_fingerprints: list[str] = [] @@ -231,7 +239,8 @@ def record_send(state: ClientState, payload: dict[str, object]) -> None: first_build_entered = threading.Event() release_first_build = threading.Event() - original_shared_job_bundle = server.shared_job_bundle + job_bundles = server._ensure_job_bundle_service() + original_shared_job_bundle = job_bundles.shared_job_bundle build_count = 0 build_count_lock = threading.Lock() @@ -250,7 +259,7 @@ def block_first_shared_job_bundle( raise AssertionError("superseded artifact build was not released") return original_shared_job_bundle(artifacts, identity, **kwargs) - server.shared_job_bundle = block_first_shared_job_bundle # type: ignore[method-assign] + job_bundles.shared_job_bundle = block_first_shared_job_bundle # type: ignore[method-assign] old_results: list[int] = [] old_errors: list[BaseException] = [] new_results: list[int] = [] @@ -284,11 +293,19 @@ def poll(results: list[int], errors: list[BaseException]) -> None: self.assertEqual(len(old_errors), 1) self.assertIsInstance(old_errors[0], TemplateRefreshBlocked) self.assertEqual(new_errors, []) + # R2 owns the single heavy lane. The competing caller contributes the + # newer immutable requirement and returns without becoming another + # refresh owner; the scheduler drains that follow-up in the background. self.assertEqual(new_results, [0]) + self.assertTrue( + server._ensure_tip_refresh_service().wait_for_scheduler_idle_for_test( + 5.0 + ) + ) try: self.assertEqual( server.poll_qbit_tip_template_once(), - len(states), + 0, ) finally: server.shutdown_tip_refresh_executor() diff --git a/tests/test_prism_initial_job_delivery.py b/tests/test_prism_initial_job_delivery.py index 9b3d70b..48458c8 100644 --- a/tests/test_prism_initial_job_delivery.py +++ b/tests/test_prism_initial_job_delivery.py @@ -35,16 +35,14 @@ def wait_until(predicate: object, *, timeout: float = 10.0) -> None: class PrismInitialJobDeliveryTests(unittest.TestCase): def test_blocked_startup_prewarm_defers_to_background_refresh(self) -> None: server, _rpc = coordinator() - retry_scheduled = threading.Event() - def blocked() -> object: raise TemplateRefreshBlocked("template raced startup") - server.prewarm_current_tip_ready_bundle = blocked # type: ignore[method-assign] - server._schedule_tip_refresh_retry = retry_scheduled.set # type: ignore[method-assign] + tip_refresh = server._ensure_tip_refresh_service() + tip_refresh.prewarm_current_tip_ready_bundle = blocked # type: ignore[method-assign] self.assertIsNone(server.prewarm_startup_jobs()) - self.assertTrue(retry_scheduled.is_set()) + self.assertTrue(tip_refresh.snapshot().retry_requested) def test_startup_prewarm_builds_worker_independent_ready_bundle(self) -> None: server, rpc = coordinator() @@ -68,7 +66,7 @@ def test_startup_prewarm_builds_worker_independent_ready_bundle(self) -> None: self.assertEqual(recorded["calls"], 1) self.assertEqual(server.ledger.snapshot_calls, 1) self.assertEqual(server.tip_template_snapshot.bestblockhash, rpc.tip) - self.assertIs(server._prepared_ready_bundle, bundle) + self.assertIs(server._ensure_job_bundle_service()._prepared_ready_bundle, bundle) self.assertEqual( recorded["suffixes"], [ @@ -142,7 +140,8 @@ def transient_reorg() -> bool: return reorg_attempts > 1 artifact_attempts = 0 - original_artifacts = server.current_template_artifacts + repository = server._ensure_job_bundle_service().template_repository + original_artifacts = repository.current def transient_artifacts() -> object: nonlocal artifact_attempts @@ -152,7 +151,7 @@ def transient_artifacts() -> object: return original_artifacts() server.ensure_reorg_reconciled_for_current_tip = transient_reorg # type: ignore[method-assign] - server.current_template_artifacts = transient_artifacts # type: ignore[method-assign] + repository.current = transient_artifacts # type: ignore[method-assign] server.request_initial_job_delivery(state) try: @@ -162,7 +161,7 @@ def transient_artifacts() -> object: self.assertEqual(reorg_attempts, 3) self.assertEqual(artifact_attempts, 2) - self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 1) self.assertIn(state, server.clients) self.assertEqual( server.progress_health_snapshot()["eligible_clients_requiring_refresh"], @@ -188,6 +187,9 @@ def test_initial_delivery_backs_off_after_superseded_work(self) -> None: - 1.0 ) server.template_refresh_failure_exit_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) request = PendingInitialJob( client=state, authorization_generation=1, @@ -220,8 +222,9 @@ def tip_churn_once( deliveries = iter((None, True)) rpc.call = tip_churn_once # type: ignore[method-assign] - server.current_template_artifacts = lambda: artifacts # type: ignore[method-assign] - server.shared_job_bundle = lambda *_args, **_kwargs: bundle # type: ignore[method-assign] + service = server._ensure_job_bundle_service() + service.template_repository.current = lambda: artifacts # type: ignore[method-assign] + service.shared_job_bundle = lambda *_args, **_kwargs: bundle # type: ignore[method-assign] server._deliver_initial_bundle = ( # type: ignore[method-assign] lambda *_args: next(deliveries) ) @@ -269,8 +272,9 @@ def churned_tip( return original_call(method, params) rpc.call = churned_tip # type: ignore[method-assign] - server.current_template_artifacts = lambda: artifacts # type: ignore[method-assign] - server.shared_job_bundle = lambda *_args, **_kwargs: bundle # type: ignore[method-assign] + service = server._ensure_job_bundle_service() + service.template_repository.current = lambda: artifacts # type: ignore[method-assign] + service.shared_job_bundle = lambda *_args, **_kwargs: bundle # type: ignore[method-assign] server._deliver_initial_bundle = ( # type: ignore[method-assign] lambda *_args: True ) @@ -290,16 +294,19 @@ def test_250_client_reconnect_storm_uses_one_build_without_client_locks(self) -> server, _rpc = coordinator() recorded = install_fake_bundle_builder(server) server.tip_refresh_max_workers = 8 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) server.stratum_max_pending_initial_jobs = 250 server.prewarm_current_tip_ready_bundle() - with server._job_cache_lock: - server._job_bundle_cache.clear() - server._prepared_ready_bundle = None - server._prepared_ready_snapshot = None + service = server._ensure_job_bundle_service() + with service._cache_lock: + service._bundle_cache.clear() + service.clear_prepared_ready() build_entered = threading.Event() release_build = threading.Event() - original_build = server.build_shared_job_bundle + original_build = service.build_shared_job_bundle build_calls = 0 build_calls_lock = threading.Lock() @@ -311,7 +318,7 @@ def blocked_build(*args: object, **kwargs: object) -> object: self.assertTrue(release_build.wait(10)) return original_build(*args, **kwargs) # type: ignore[arg-type] - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] + service.build_shared_job_bundle = blocked_build # type: ignore[method-assign] clients = [client(index + 1) for index in range(250)] now = time.monotonic() for state in clients: @@ -375,19 +382,20 @@ def test_reauthorization_supersedes_pending_identity(self) -> None: server, _rpc = coordinator() install_fake_bundle_builder(server) server.prewarm_current_tip_ready_bundle() - with server._job_cache_lock: - server._job_bundle_cache.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() build_entered = threading.Event() release_build = threading.Event() - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle def blocked_build(*args: object, **kwargs: object) -> object: build_entered.set() self.assertTrue(release_build.wait(5)) return original_build(*args, **kwargs) # type: ignore[arg-type] - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] + service.build_shared_job_bundle = blocked_build # type: ignore[method-assign] state = client(1, worker(username="old-worker")) state.authorization_generation = 1 state.authorized_monotonic = time.monotonic() @@ -419,12 +427,13 @@ def test_payout_generation_supersedes_pending_initial_bundle(self) -> None: server, _rpc = coordinator() install_fake_bundle_builder(server) server.prewarm_current_tip_ready_bundle() - with server._job_cache_lock: - server._job_bundle_cache.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() build_entered = threading.Event() release_build = threading.Event() - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle build_calls = 0 def blocked_build(*args: object, **kwargs: object) -> object: @@ -435,7 +444,7 @@ def blocked_build(*args: object, **kwargs: object) -> object: self.assertTrue(release_build.wait(5)) return original_build(*args, **kwargs) # type: ignore[arg-type] - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] + service.build_shared_job_bundle = blocked_build # type: ignore[method-assign] state = client(1) state.authorization_generation = 1 state.authorized_monotonic = time.monotonic() @@ -486,7 +495,7 @@ def test_collection_mode_initial_bundles_remain_identity_specific(self) -> None: self.assertNotEqual(first.active_job.worker, second.active_job.worker) collection_keys = { key - for key in server._job_bundle_cache + for key in server._ensure_job_bundle_service()._bundle_cache if len(key) >= 8 and key[2] == "collection" } self.assertEqual( @@ -500,12 +509,13 @@ def test_new_tip_supersedes_blocked_build_without_duplicate_client_task(self) -> server, rpc = coordinator(template=base_template(prevhash=tip_a)) install_fake_bundle_builder(server) server.prewarm_current_tip_ready_bundle() - with server._job_cache_lock: - server._job_bundle_cache.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() first_build_entered = threading.Event() release_first_build = threading.Event() - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle build_count = 0 build_lock = threading.Lock() @@ -519,7 +529,7 @@ def block_first_build(*args: object, **kwargs: object) -> object: self.assertTrue(release_first_build.wait(10)) return original_build(*args, **kwargs) # type: ignore[arg-type] - server.build_shared_job_bundle = block_first_build # type: ignore[method-assign] + service.build_shared_job_bundle = block_first_build # type: ignore[method-assign] state = client(1) state.authorization_generation = 1 state.authorized_monotonic = time.monotonic() @@ -546,7 +556,10 @@ def block_first_build(*args: object, **kwargs: object) -> object: self.assertGreaterEqual(build_count, 2) self.assertEqual(state.active_job.template["previousblockhash"], tip_b) - self.assertGreaterEqual(server.shared_bundle_build_counts["superseded"], 1) + build_counts = server._ensure_job_bundle_service().shared_preparation_metrics()[ + "build_counts" + ] + self.assertGreaterEqual(build_counts["superseded"], 1) def test_health_turns_non_green_after_stalled_delivery_deadline(self) -> None: server, _rpc = coordinator() diff --git a/tests/test_prism_job_builder.py b/tests/test_prism_job_builder.py index 6956097..c0cafea 100644 --- a/tests/test_prism_job_builder.py +++ b/tests/test_prism_job_builder.py @@ -44,8 +44,8 @@ def test_job_bundle_anchor_clamps_below_pending_share_commit(self) -> None: # The issued time is frozen per template generation; drop the frozen # entry so the rebuild stamps a fresh anchor now that no commit is # pending. - with server._job_cache_lock: - server._job_build_issued_at_ms.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._issued_at_ms.clear() rebuilt = server.build_shared_job_bundle( server.current_template_artifacts(), worker(), @@ -79,8 +79,8 @@ def test_payout_artifact_declares_its_own_snapshot_anchor(self) -> None: server._finish_pending_share_commit(share) # Construction re-validates that the passed artifact is the installed # current one. - with server._job_cache_lock: - server._payout_ledger_artifact = artifact + with server._payout_state_service._lock: + server._payout_state_service._ledger_artifact = artifact bundle = server.build_shared_job_bundle( artifacts, worker(), @@ -153,7 +153,7 @@ def test_one_heavy_build_shared_across_clients_with_per_client_stamping(self) -> self.assertTrue( all(context.prospective_prior_balances == () for context in contexts) ) - cached = next(iter(server._job_bundle_cache.values())) + cached = next(iter(server._ensure_job_bundle_service()._bundle_cache.values())) self.assertFalse(hasattr(cached, "bundle")) self.assertEqual(cached.prospective_prior_balances, ()) self.assertEqual( @@ -250,13 +250,14 @@ def test_template_fingerprint_change_invalidates_bundle_cache(self) -> None: self.assertEqual(context.template_fingerprint, qbit_template_fingerprint(new_template)) # Bundles for the old fingerprint are evicted. self.assertEqual( - {entry.template_fingerprint for entry in server._job_bundle_cache.values()}, + {entry.template_fingerprint for entry in server._ensure_job_bundle_service()._bundle_cache.values()}, {qbit_template_fingerprint(new_template)}, ) def test_bundle_cache_ttl_expiry_rebuilds(self) -> None: server, _ = coordinator() recorded = install_fake_bundle_builder(server) server.job_bundle_cache_seconds = 0.05 + server._ensure_job_bundle_service().set_cache_seconds_for_test(0.05) server.build_job_for_client(client(1), clean_jobs=True) time.sleep(0.06) @@ -276,17 +277,18 @@ def test_bundle_cache_lookup_prunes_every_expired_snapshot(self) -> None: template_fingerprint="expired-template", built_monotonic=time.monotonic() - 60, ) - server._job_bundle_cache[expired_key] = expired + server._ensure_job_bundle_service()._bundle_cache[expired_key] = expired looked_up = server._lookup_job_bundle(current.key) self.assertIs(looked_up, current) - self.assertNotIn(expired_key, server._job_bundle_cache) - self.assertEqual(list(server._job_bundle_cache.values()), [current]) + self.assertNotIn(expired_key, server._ensure_job_bundle_service()._bundle_cache) + self.assertEqual(list(server._ensure_job_bundle_service()._bundle_cache.values()), [current]) def test_zero_ttl_disables_bundle_cache(self) -> None: server, _ = coordinator() recorded = install_fake_bundle_builder(server) server.job_bundle_cache_seconds = 0.0 + server._ensure_job_bundle_service().set_cache_seconds_for_test(0.0) server.build_job_for_client(client(1), clean_jobs=True) server.build_job_for_client(client(2), clean_jobs=True) @@ -299,7 +301,8 @@ def test_payout_state_change_during_build_retries_before_cache_or_return(self) - self.assertIsNotNone(artifacts) assert artifacts is not None identity = worker() - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle built_generations: list[int] = [] def mutate_after_first_build( @@ -317,7 +320,7 @@ def mutate_after_first_build( server._advance_payout_state_generation() return bundle - server.build_shared_job_bundle = mutate_after_first_build # type: ignore[method-assign] + service.build_shared_job_bundle = mutate_after_first_build # type: ignore[method-assign] bundle = server.shared_job_bundle(artifacts, identity) cached = server.shared_job_bundle(artifacts, identity) @@ -349,8 +352,8 @@ def advance_after_bundle(*args: object, **kwargs: object) -> object: self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) self.assertEqual(sent, []) self.assertIsNone(state.active_job) - self.assertEqual(server._payout_state_generation, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) self.assertIsNotNone(state.active_job) @@ -372,7 +375,7 @@ def test_priority_decision_uses_one_publication_snapshot(self) -> None: server.build_job_for_client = ( # type: ignore[method-assign] lambda *_args, **_kwargs: context ) - original_lock = server._job_cache_lock + original_lock = server._payout_state_service._lock class PublishAfterPrioritySnapshot: advanced = False @@ -390,7 +393,7 @@ def __exit__( original_lock.release() if not self.advanced: self.advanced = True - server._payout_state_generation = 1 + server._payout_state_service._generation = 1 priorities: list[bool] = [] @@ -406,16 +409,17 @@ def delivery_cancelable( priorities.append(priority) yield False - server._job_cache_lock = PublishAfterPrioritySnapshot() # type: ignore[assignment] - server._payout_state_delivery_gate = RecordingGate() # type: ignore[assignment] + server._payout_state_service._lock = PublishAfterPrioritySnapshot() # type: ignore[assignment] + server._payout_state_service._delivery_gate = RecordingGate() # type: ignore[assignment] self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) self.assertEqual(priorities, [True]) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service._generation, 1) def test_zero_template_ttl_fetches_template_per_build(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.template_cache_seconds = 0.0 + server._ensure_job_bundle_service().template_repository.set_cache_seconds_for_test(0.0) server.build_job_for_client(client(1), clean_jobs=True) server.build_job_for_client(client(2), clean_jobs=True) @@ -424,6 +428,7 @@ def test_zero_template_ttl_fetches_template_per_build(self) -> None: def test_late_stale_template_fetch_cannot_replace_newer_artifacts(self) -> None: server, rpc = coordinator() server.template_cache_seconds = 0.0 + server._ensure_job_bundle_service().template_repository.set_cache_seconds_for_test(0.0) stale_template = dict(rpc.template) current_template = base_template(height=11, prevhash="22" * 32) fetch_started = threading.Event() @@ -466,7 +471,10 @@ def fetch_stale_artifacts() -> None: self.assertFalse(thread.is_alive()) self.assertEqual(errors, []) self.assertEqual(results, [current_artifacts]) - self.assertIs(server._template_artifacts, current_artifacts) + self.assertIs( + server._ensure_job_bundle_service().template_repository.current_artifacts(), + current_artifacts, + ) self.assertEqual( current_artifacts.fingerprint, qbit_template_fingerprint(current_template), @@ -475,6 +483,7 @@ def test_collection_mode_bundles_are_keyed_per_worker(self) -> None: server, _ = coordinator(ledger=FakeLedger(miners=["solo"])) recorded = install_fake_bundle_builder(server) server.min_ready_miners = 3 + server._ensure_job_bundle_service().set_min_ready_miners_for_test(3) worker_a = worker(payout="tq1worker-a") worker_b = worker(payout="tq1worker-b") @@ -616,6 +625,9 @@ def slow_builder(**kwargs: object) -> dict[str, object]: return original_builder(**kwargs) server.build_audit_bundle = slow_builder # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + slow_builder + ) errors: list[BaseException] = [] def build(connection_id: int) -> None: @@ -677,11 +689,11 @@ def blocked_snapshot(*args: object, **kwargs: object) -> object: build_thread.start() try: self.assertTrue(entered_snapshot.wait(2)) - mutation_acquired = server._payout_state_prepare_lock.acquire( + mutation_acquired = server._payout_state_service._prepare_lock.acquire( blocking=False ) if mutation_acquired: - server._payout_state_prepare_lock.release() + server._payout_state_service._prepare_lock.release() self.assertFalse(mutation_acquired) finally: release_snapshot.set() @@ -698,7 +710,7 @@ def test_ready_build_identity_separates_clock_only_generations(self) -> None: assert first is not None and second is not None self.assertEqual(first.fingerprint, second.fingerprint) self.assertNotEqual(first.generation, second.generation) - payout_generation = server._payout_state_generation + payout_generation = server._payout_state_service._generation with patch( "lab.prism.prism_coordinator.now_ms", @@ -745,10 +757,10 @@ def test_precomputed_payout_artifact_matches_inline_output_exactly(self) -> None with patch("lab.prism.prism_coordinator.now_ms", return_value=1_700_000_123_000): inline = server.shared_job_bundle(artifacts, mode="ready") - with server._job_cache_lock: - server._job_bundle_cache.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() server._prepare_payout_ledger_artifact( - server._payout_state_generation, + server._payout_state_service._generation, artifacts.network_difficulty, ) prepared = server.shared_job_bundle(artifacts, mode="ready") @@ -782,7 +794,7 @@ def current_prior_balances(self) -> list[dict[str, object]]: recorded = install_fake_bundle_builder(server) artifacts = server.store_template_artifacts(dict(rpc.template)) assert artifacts is not None - server._pool_ready_latched = True + server._ensure_job_bundle_service().set_ready_for_test(True) parent_hash = str(rpc.template["previousblockhash"]) parent_height = int(rpc.template["height"]) - 1 preview = [ @@ -800,13 +812,13 @@ def current_prior_balances(self) -> list[dict[str, object]]: ) server._publish_accepted_block_payout_preview(parent_hash, preview) - artifact = server._payout_ledger_artifact + artifact = server._payout_state_service._ledger_artifact self.assertIsNotNone(artifact) assert artifact is not None self.assertEqual(artifact.prior_balances, tuple(preview)) self.assertEqual( artifact.payout_state_generation, - server._payout_state_generation, + server._payout_state_service._generation, ) self.assertGreater(artifact.generation, 0) @@ -821,18 +833,18 @@ def current_prior_balances(self) -> list[dict[str, object]]: ) self.assertEqual(ledger.prior_balance_reads, 0) - payout_generation = server._payout_state_generation + payout_generation = server._payout_state_service._generation ledger.database_balances = [dict(balance) for balance in preview] server._clear_accepted_block_payout_preview(parent_hash) - self.assertEqual(server._payout_state_generation, payout_generation) - self.assertIs(server._payout_ledger_artifact, artifact) - self.assertNotIn(parent_hash, server._accepted_block_payout_previews) + self.assertEqual(server._payout_state_service._generation, payout_generation) + self.assertIs(server._payout_state_service._ledger_artifact, artifact) + self.assertNotIn(parent_hash, server._payout_state_service._previews) self.assertNotIn( parent_hash, - server._invalidated_accepted_block_payout_previews, + server._payout_state_service._invalidated_previews, ) - with server._job_cache_lock: - server._job_bundle_cache.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() post_clear_bundle = server.shared_job_bundle(artifacts, mode="ready") @@ -849,7 +861,7 @@ def test_valid_precomputed_artifact_skips_tip_path_ledger_snapshot(self) -> None snapshot = server.fetch_qbit_tip_template_snapshot() assert snapshot.template_artifacts is not None server._prepare_payout_ledger_artifact( - server._payout_state_generation, + server._payout_state_service._generation, snapshot.template_artifacts.network_difficulty, ) server.ledger.snapshot_calls = 0 @@ -866,14 +878,14 @@ def test_mismatched_precomputed_artifact_falls_back_to_inline_snapshot(self) -> assert artifacts is not None server._current_payout_state_artifact() server._prepare_payout_ledger_artifact( - server._payout_state_generation, + server._payout_state_service._generation, artifacts.network_difficulty, ) - with server._job_cache_lock: - published = server._published_payout_state + with server._payout_state_service._lock: + published = server._payout_state_service._published assert published.artifact is not None changed_balances = [{"miner_id": "miner-z", "balance_sats": 1}] - server._published_payout_state = dataclass_replace( + server._payout_state_service._published = dataclass_replace( published, artifact=dataclass_replace( published.artifact, @@ -884,11 +896,11 @@ def test_mismatched_precomputed_artifact_falls_back_to_inline_snapshot(self) -> self.assertIsNone( server._usable_payout_ledger_artifact( - server._payout_state_generation, + server._payout_state_service._generation, artifacts.network_difficulty, ) ) - self.assertIsNone(server._payout_ledger_artifact) + self.assertIsNone(server._payout_state_service._ledger_artifact) server.ledger.snapshot_calls = 0 bundle = server.shared_job_bundle(artifacts, mode="ready") @@ -904,12 +916,15 @@ def test_new_tip_cancels_blocked_old_bundle_without_publication(self) -> None: build_started = threading.Event() def cancelable_builder(**kwargs: object) -> dict[str, object]: - control = server._job_build_phase_local.bundle_build_control + control = server._ensure_job_bundle_service()._phase_local.bundle_build_control build_started.set() self.assertTrue(control.cancel_event.wait(2)) return original_builder(**kwargs) server.build_audit_bundle = cancelable_builder # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + cancelable_builder + ) artifacts = server.store_template_artifacts(dict(rpc.template)) assert artifacts is not None server.observe_tip_first_seen(old_tip, observation_sequence=1) @@ -930,13 +945,13 @@ def cancelable_builder(**kwargs: object) -> dict[str, object]: self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0], TemplateRefreshBlocked) self.assertEqual(recorded["calls"], 1) - self.assertEqual(server._active_job_bundle_builds, {}) - self.assertEqual(server.tip_refresh_build_inflight, 0) + self.assertEqual(server._ensure_job_bundle_service()._active_bundle_builds, {}) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["build_inflight"], 0) self.assertFalse(any( entry.template_fingerprint == artifacts.fingerprint - for entry in server._job_bundle_cache.values() + for entry in server._ensure_job_bundle_service()._bundle_cache.values() )) - self.assertEqual(server.tip_refresh_superseded_results, 1) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["superseded_results"], 1) def test_builder_crash_and_timeout_fail_closed_then_recover(self) -> None: server, _rpc = coordinator() server.prism_ctv_settlement_config = lambda **_kwargs: None # type: ignore[method-assign] @@ -955,7 +970,7 @@ def test_builder_crash_and_timeout_fail_closed_then_recover(self) -> None: } with patch( - "lab.prism.prism_coordinator.prism_tool_command", + "lab.prism.bundle_compiler.prism_tool_command", return_value=[sys.executable, "-c", "raise SystemExit(7)"], ): with self.assertRaisesRegex(RuntimeError, "failed"): @@ -963,7 +978,7 @@ def test_builder_crash_and_timeout_fail_closed_then_recover(self) -> None: server.bundle_build_timeout_seconds = 0.01 with patch( - "lab.prism.prism_coordinator.prism_tool_command", + "lab.prism.bundle_compiler.prism_tool_command", return_value=[ sys.executable, "-c", @@ -979,7 +994,7 @@ def test_builder_crash_and_timeout_fail_closed_then_recover(self) -> None: "json.dump({'recovered': True}, sys.stdout)" ) with patch( - "lab.prism.prism_coordinator.prism_tool_command", + "lab.prism.bundle_compiler.prism_tool_command", return_value=[sys.executable, "-c", recovery_script], ): recovered = server.build_audit_bundle(**build_kwargs) @@ -999,7 +1014,7 @@ def test_audit_builder_child_does_not_inherit_open_socket(self) -> None: with socket.socket() as parent_socket: parent_socket.set_inheritable(True) with patch( - "lab.prism.prism_coordinator.prism_tool_command", + "lab.prism.bundle_compiler.prism_tool_command", return_value=[ sys.executable, "-c", @@ -1026,12 +1041,15 @@ def test_repeated_superseded_builds_leave_state_bounded(self) -> None: starts: queue.Queue[None] = queue.Queue() def cancelable_builder(**kwargs: object) -> dict[str, object]: - control = server._job_build_phase_local.bundle_build_control + control = server._ensure_job_bundle_service()._phase_local.bundle_build_control starts.put(None) self.assertTrue(control.cancel_event.wait(2)) return original_builder(**kwargs) server.build_audit_bundle = cancelable_builder # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + cancelable_builder + ) current_tip = str(rpc.tip) server.observe_tip_first_seen(current_tip, observation_sequence=1) errors: list[BaseException] = [] @@ -1058,14 +1076,14 @@ def cancelable_builder(**kwargs: object) -> dict[str, object]: self.assertEqual(len(errors), 8) self.assertTrue(all(isinstance(exc, TemplateRefreshBlocked) for exc in errors)) - self.assertEqual(server._active_job_bundle_builds, {}) - self.assertEqual(server.tip_refresh_build_inflight, 0) - self.assertEqual(server.tip_refresh_build_queue_depth, 0) + self.assertEqual(server._ensure_job_bundle_service()._active_bundle_builds, {}) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["build_inflight"], 0) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["build_queue_depth"], 0) self.assertLessEqual( - len(server._job_bundle_cache), + len(server._ensure_job_bundle_service()._bundle_cache), MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, ) - self.assertEqual(server.tip_refresh_superseded_results, 8) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["superseded_results"], 8) @staticmethod def _capture_error( errors: list[BaseException], @@ -1101,7 +1119,8 @@ def test_clock_only_refresh_does_not_discard_inflight_ready_build(self) -> None: assert first is not None build_entered = threading.Event() release_build = threading.Event() - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle build_calls = 0 build_calls_lock = threading.Lock() @@ -1113,7 +1132,7 @@ def blocking_build(*args: object, **kwargs: object) -> object: self.assertTrue(release_build.wait(5)) return original_build(*args, **kwargs) # type: ignore[arg-type] - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + service.build_shared_job_bundle = blocking_build # type: ignore[method-assign] results: list[list[object]] = [[], []] errors: list[list[BaseException]] = [[], []] @@ -1195,12 +1214,12 @@ def test_job_bundle_cache_is_bounded(self) -> None: server._cache_job_bundle_if_current(candidate, artifacts) self.assertEqual( - len(server._job_bundle_cache), + len(server._ensure_job_bundle_service()._bundle_cache), MAX_PRISM_JOB_BUNDLE_CACHE_ENTRIES, ) self.assertNotIn( (artifacts.fingerprint, "test", 0), - server._job_bundle_cache, + server._ensure_job_bundle_service()._bundle_cache, ) def test_job_bundle_cache_preserves_coordinator_lock_order(self) -> None: server, _ = coordinator() @@ -1209,7 +1228,7 @@ def test_job_bundle_cache_preserves_coordinator_lock_order(self) -> None: assert artifacts is not None bundle = server.shared_job_bundle(artifacts, mode="ready") observed_cache_lock = ObservedRLock() - server._job_cache_lock = observed_cache_lock # type: ignore[assignment] + server._ensure_job_bundle_service()._cache_lock = observed_cache_lock # type: ignore[assignment] errors: list[BaseException] = [] def cache_bundle() -> None: @@ -1242,7 +1261,7 @@ def test_active_gap_replaces_older_pending_job_build(self) -> None: second_template["coinbasevalue"] = int(second_template["coinbasevalue"]) + 1 second = server.store_template_artifacts(second_template) assert first is not None and second is not None - payout_generation = server._payout_state_generation + payout_generation = server._payout_state_service._generation def request_for(artifacts: object) -> object: return server._new_job_build_request( @@ -1260,564 +1279,32 @@ def request_for(artifacts: object) -> object: pending = request_for(first) newest = request_for(second) - server._job_build_active = None - server._job_build_retiring = SimpleNamespace(request=pending) - server._job_build_pending = pending - server._start_job_build_locked = ( # type: ignore[method-assign] + server._ensure_job_bundle_service()._active = None + server._ensure_job_bundle_service()._retiring = SimpleNamespace(request=pending) + server._ensure_job_bundle_service()._pending = pending + server._ensure_job_bundle_service()._start_locked = ( # type: ignore[method-assign] lambda request: SimpleNamespace(request=request, future=None) ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] + server._ensure_job_bundle_service()._arm_locked = lambda _flight: None # type: ignore[method-assign] promise = server._request_job_build(newest) # type: ignore[arg-type] self.assertIs(promise, newest.promise) # type: ignore[union-attr] - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, newest) + self.assertIsNone(server._ensure_job_bundle_service()._pending) + assert server._ensure_job_bundle_service()._active is not None + self.assertIs(server._ensure_job_bundle_service()._active.request, newest) self.assertTrue(pending.promise.done()) # type: ignore[union-attr] self.assertIsInstance( pending.promise.exception(), # type: ignore[union-attr] JobBuildSuperseded, ) - - def test_publication_critical_build_cannot_be_displaced_by_initial_work( - self, - ) -> None: - server, rpc = coordinator() - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - request_source: str, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=request_source, - ) - - latest = request_for( - new_artifacts, - publication_critical=True, - request_source="tip_refresh", - ) - reconnect = request_for( - old_artifacts, - publication_critical=False, - request_source="initial", - ) - same_tip_reconnect = request_for( - new_artifacts, - publication_critical=False, - request_source="initial", - ) - latest_flight = SimpleNamespace(request=latest) - server._job_build_active = latest_flight - server._job_build_retiring = None - server._job_build_pending = None - - deferred = server._request_job_build(reconnect) # type: ignore[arg-type] - coalesced = server._request_job_build( # type: ignore[arg-type] - same_tip_reconnect - ) - - self.assertIs(server._job_build_active, latest_flight) - self.assertIs(coalesced, latest.promise) # type: ignore[union-attr] - self.assertFalse(latest.cancellation.is_set()) # type: ignore[union-attr] - self.assertFalse(deferred.done()) - self.assertEqual( - server.job_build_priority_counts["routine_deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["singleflight"], - 1, - ) - - latest.promise.set_result(object()) # type: ignore[union-attr] - with self.assertRaises(JobBuildSuperseded): - deferred.result() - - metrics = "\n".join(server.job_build_metrics_lines()) - self.assertIn( - 'qbit_prism_job_build_priority_events_total{result="routine_deferred"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_initial_job_prepared_work_total{result="deferred"} 1', - metrics, - ) - self.assertIn( - 'qbit_prism_initial_job_prepared_work_total{result="singleflight"} 1', - metrics, - ) - - def test_publication_critical_build_preempts_routine_builder_capacity( - self, - ) -> None: - server, rpc = coordinator() - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - request_source: str, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=request_source, - ) - - routine = request_for( - old_artifacts, - publication_critical=False, - request_source="initial", - ) - latest = request_for( - new_artifacts, - publication_critical=True, - request_source="tip_refresh", - ) - routine_flight = SimpleNamespace(request=routine) - server._job_build_active = routine_flight - server._job_build_retiring = None - server._job_build_pending = None - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - - promise = server._request_job_build(latest) # type: ignore[arg-type] - - self.assertIs(promise, latest.promise) # type: ignore[union-attr] - self.assertTrue(routine.cancellation.is_set()) # type: ignore[union-attr] - self.assertIs(server._job_build_retiring, routine_flight) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_publication_critical_build_restarts_unhealthy_exact_flight( - self, - ) -> None: - for unhealthy in ("almost_expired", "stalled"): - with self.subTest(unhealthy=unhealthy): - server, rpc = coordinator() - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - *, - publication_critical: bool, - priority_requested_monotonic: float | None = None, - ) -> object: - return server._new_job_build_request( - artifacts, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=( - "tip_refresh" - if publication_critical - else "initial" - ), - priority_requested_monotonic=( - priority_requested_monotonic - ), - ) - - routine = request_for(publication_critical=False) - now = time.monotonic() - if unhealthy == "almost_expired": - routine.cancellation.started_monotonic = now - 59.99 - routine.cancellation.deadline_monotonic = now + 0.01 - routine.cancellation.last_checkpoint_monotonic = now - else: - routine.cancellation.started_monotonic = now - 1.0 - routine.cancellation.deadline_monotonic = now + 59.0 - routine.cancellation.last_checkpoint_monotonic = ( - now - server.job_build_cancel_grace_seconds - 0.01 - ) - priority_requested = now - 59.0 - latest = request_for( - publication_critical=True, - priority_requested_monotonic=priority_requested, - ) - routine_flight = SimpleNamespace(request=routine) - server._job_build_active = routine_flight - server._job_build_retiring = None - server._job_build_pending = None - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] - - promise = server._request_job_build(latest) # type: ignore[arg-type] - - self.assertIs(promise, latest.promise) - self.assertIsNot(promise, routine.promise) - self.assertTrue(routine.cancellation.is_set()) - self.assertIs(server._job_build_retiring, routine_flight) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual( - latest.requested_monotonic, - priority_requested, - ) - self.assertGreater( - latest.cancellation.deadline_monotonic - - time.monotonic(), - server.job_build_timeout_seconds - 1.0, - ) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_publication_priority_precedes_immutable_request_preparation( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - critical_preparation_entered = threading.Event() - release_critical_preparation = threading.Event() - initial_preparation_entered = threading.Event() - original_new_request = server._new_job_build_request - - def observed_new_request(*args: object, **kwargs: object) -> object: - request_source = str(kwargs.get("request_source", "routine")) - if request_source == "tip_refresh": - critical_preparation_entered.set() - if not release_critical_preparation.wait(5): - raise AssertionError("test did not release priority preparation") - elif request_source == "initial": - initial_preparation_entered.set() - return original_new_request(*args, **kwargs) # type: ignore[arg-type] - - server._new_job_build_request = observed_new_request # type: ignore[method-assign] - results: dict[str, list[object]] = {"critical": [], "initial": []} - errors: list[BaseException] = [] - - def build(label: str, *, publication_critical: bool) -> None: - try: - results[label].append( - server.shared_job_bundle( - artifacts, - mode="ready", - publication_critical=publication_critical, - request_source=( - "tip_refresh" if publication_critical else "initial" - ), - ) - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - critical_thread = threading.Thread( - target=build, - args=("critical",), - kwargs={"publication_critical": True}, - ) - initial_thread = threading.Thread( - target=build, - args=("initial",), - kwargs={"publication_critical": False}, - ) - critical_thread.start() - try: - self.assertTrue(critical_preparation_entered.wait(5)) - initial_thread.start() - self.assertFalse(initial_preparation_entered.wait(0.1)) - metrics = "\n".join(server.job_build_metrics_lines()) - self.assertIn("qbit_prism_job_build_priority_active 1", metrics) - finally: - release_critical_preparation.set() - critical_thread.join(5) - initial_thread.join(5) - - self.assertFalse(critical_thread.is_alive()) - self.assertFalse(initial_thread.is_alive()) - self.assertEqual(errors, []) - self.assertEqual([len(results[label]) for label in results], [1, 1]) - self.assertIs(results["critical"][0], results["initial"][0]) - self.assertFalse(initial_preparation_entered.is_set()) - self.assertEqual( - server.initial_job_prepared_work_counts["deferred"], - 1, - ) - self.assertEqual( - server.initial_job_prepared_work_counts["cache_hit"], - 1, - ) - self.assertEqual( - server.job_build_priority_admission_seconds["count"], - 1, - ) - self.assertGreaterEqual( - server.job_build_priority_admission_seconds["sum"], - 0.1, - ) - - def test_priority_reservation_cancels_admitted_routine_preparation( - self, - ) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - artifacts = server.store_template_artifacts(dict(rpc.template)) - assert artifacts is not None - routine_in_payout_lookup = threading.Event() - release_routine_lookup = threading.Event() - routine_request_constructed = threading.Event() - original_usable_artifact = server._usable_payout_ledger_artifact - original_new_request = server._new_job_build_request - routine_thread: threading.Thread - - def blocked_usable_artifact(*args: object, **kwargs: object) -> object: - if threading.current_thread() is routine_thread: - routine_in_payout_lookup.set() - if not release_routine_lookup.wait(5): - raise AssertionError("test did not release payout lookup") - return original_usable_artifact(*args, **kwargs) # type: ignore[arg-type] - - def observed_new_request(*args: object, **kwargs: object) -> object: - if kwargs.get("request_source") == "initial": - routine_request_constructed.set() - return original_new_request(*args, **kwargs) # type: ignore[arg-type] - - server._usable_payout_ledger_artifact = blocked_usable_artifact # type: ignore[method-assign] - server._new_job_build_request = observed_new_request # type: ignore[method-assign] - errors: list[BaseException] = [] - - def build_routine() -> None: - try: - server.shared_job_bundle( - artifacts, - mode="ready", - retry_superseded=False, - request_source="initial", - ) - except BaseException as exc: # noqa: BLE001 - surfaced below - errors.append(exc) - - routine_thread = threading.Thread(target=build_routine) - routine_thread.start() - priority_token: int | None = None - try: - self.assertTrue(routine_in_payout_lookup.wait(5)) - with server._job_build_scheduler_lock: - routine_cancellations = tuple( - cancellation_ref() - for cancellation_ref in ( - server._job_build_routine_preparations.values() - ) - ) - self.assertEqual(len(routine_cancellations), 1) - self.assertIsNotNone(routine_cancellations[0]) - priority_token, _requested = ( - server._begin_job_build_priority_preparation() - ) - assert routine_cancellations[0] is not None - self.assertTrue(routine_cancellations[0].is_set()) - finally: - release_routine_lookup.set() - routine_thread.join(5) - if priority_token is not None: - server._finish_job_build_priority_preparation(priority_token) - - self.assertFalse(routine_thread.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], JobBuildSuperseded) - self.assertFalse(routine_request_constructed.is_set()) - with server._job_build_scheduler_lock: - self.assertEqual(len(server._job_build_routine_preparations), 0) - - def test_publication_critical_collection_promotes_past_ready_work(self) -> None: - server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) - old_artifacts = server.store_template_artifacts(dict(rpc.template)) - new_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert old_artifacts is not None and new_artifacts is not None - payout_generation = server._payout_state_generation - ready = server._new_job_build_request( - old_artifacts, - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - old_artifacts, - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - request_source="initial", - ) - latest_identity = worker("tq1latest", "tq1latest.rig") - latest = server._new_job_build_request( - new_artifacts, - latest_identity, - mode="collection", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - new_artifacts, - mode="collection", - payout_state_generation=payout_generation, - worker=latest_identity, - ), - publication_critical=True, - request_source="tip_refresh", - ) - ready_flight = SimpleNamespace(request=ready) - server._job_build_active = ready_flight - server._job_build_retiring = None - server._job_build_pending = latest - armed: list[object] = [] - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda request: SimpleNamespace(request=request, future=None) - ) - server._arm_job_build_locked = armed.append # type: ignore[method-assign] - - server._promote_pending_job_build_locked() - - self.assertTrue(ready.cancellation.is_set()) - self.assertIs(server._job_build_retiring, ready_flight) - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) - self.assertEqual(armed, [server._job_build_active]) - self.assertEqual( - server.job_build_priority_counts["routine_preempted"], - 1, - ) - - def test_routine_pending_never_promotes_over_publication_critical_flight( - self, - ) -> None: - for placement in ("active", "retiring"): - with self.subTest(placement=placement): - server, rpc = coordinator() - critical_artifacts = server.store_template_artifacts( - dict(rpc.template) - ) - routine_artifacts = server.store_template_artifacts( - base_template(height=11, prevhash="22" * 32) - ) - assert critical_artifacts is not None - assert routine_artifacts is not None - payout_generation = server._payout_state_generation - - def request_for( - artifacts: object, - *, - publication_critical: bool, - ) -> object: - return server._new_job_build_request( - artifacts, # type: ignore[arg-type] - None, - mode="ready", - payout_state_generation=payout_generation, - cache_key=server._job_bundle_key( - artifacts, # type: ignore[arg-type] - mode="ready", - payout_state_generation=payout_generation, - worker=None, - ), - publication_critical=publication_critical, - request_source=( - "tip_refresh" if publication_critical else "initial" - ), - ) - - critical = request_for( - critical_artifacts, - publication_critical=True, - ) - routine = request_for( - routine_artifacts, - publication_critical=False, - ) - critical_flight = SimpleNamespace(request=critical) - server._job_build_active = ( - critical_flight if placement == "active" else None - ) - server._job_build_retiring = ( - critical_flight if placement == "retiring" else None - ) - server._job_build_pending = routine - server._start_job_build_locked = ( # type: ignore[method-assign] - lambda _request: self.fail( - "routine pending work displaced publication-critical work" - ) - ) - - server._promote_pending_job_build_locked() - - self.assertIs(server._job_build_pending, routine) - self.assertFalse(critical.cancellation.is_set()) - self.assertIs( - ( - server._job_build_active - if placement == "active" - else server._job_build_retiring - ), - critical_flight, - ) - def test_cancelled_ready_does_not_block_collection_promotion(self) -> None: for placement in ("active", "retiring"): with self.subTest(placement=placement): server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) artifacts = server.store_template_artifacts(dict(rpc.template)) assert artifacts is not None - payout_generation = server._payout_state_generation + payout_generation = server._payout_state_service._generation def request_for( mode: str, @@ -1846,30 +1333,30 @@ def request_for( ) ready_flight = SimpleNamespace(request=ready) if placement == "active": - server._job_build_active = ready_flight - server._job_build_retiring = None + server._ensure_job_bundle_service()._active = ready_flight + server._ensure_job_bundle_service()._retiring = None else: - server._job_build_active = None - server._job_build_retiring = ready_flight - server._job_build_pending = collection + server._ensure_job_bundle_service()._active = None + server._ensure_job_bundle_service()._retiring = ready_flight + server._ensure_job_bundle_service()._pending = collection armed: list[object] = [] - server._start_job_build_locked = ( # type: ignore[method-assign] + server._ensure_job_bundle_service()._start_locked = ( # type: ignore[method-assign] lambda request: SimpleNamespace(request=request, future=None) ) - server._arm_job_build_locked = armed.append # type: ignore[method-assign] + server._ensure_job_bundle_service()._arm_locked = armed.append # type: ignore[method-assign] server._promote_pending_job_build_locked() - self.assertIsNone(server._job_build_pending) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, collection) - self.assertIs(server._job_build_retiring, ready_flight) - self.assertEqual(armed, [server._job_build_active]) + self.assertIsNone(server._ensure_job_bundle_service()._pending) + assert server._ensure_job_bundle_service()._active is not None + self.assertIs(server._ensure_job_bundle_service()._active.request, collection) + self.assertIs(server._ensure_job_bundle_service()._retiring, ready_flight) + self.assertEqual(armed, [server._ensure_job_bundle_service()._active]) def test_immediate_collection_completion_does_not_reoccupy_slot(self) -> None: server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) artifacts = server.store_template_artifacts(dict(rpc.template)) assert artifacts is not None - payout_generation = server._payout_state_generation + payout_generation = server._payout_state_service._generation identities = [ worker(f"tq1worker-{index}", f"tq1worker-{index}.rig") for index in range(2) @@ -1900,8 +1387,8 @@ def completed_flight(request: object) -> object: future.set_result(result) return SimpleNamespace(request=request, future=future) - server._job_build_pending = pending - server._start_job_build_locked = completed_flight # type: ignore[method-assign] + server._ensure_job_bundle_service()._pending = pending + server._ensure_job_bundle_service()._start_locked = completed_flight # type: ignore[method-assign] promise = server._request_job_build(incoming) # type: ignore[arg-type] @@ -1910,9 +1397,9 @@ def completed_flight(request: object) -> object: pending.promise.result(), results[id(pending)], ) - self.assertIsNone(server._job_build_active) - self.assertIsNone(server._job_build_retiring) - self.assertIsNone(server._job_build_pending) + self.assertIsNone(server._ensure_job_bundle_service()._active) + self.assertIsNone(server._ensure_job_bundle_service()._retiring) + self.assertIsNone(server._ensure_job_bundle_service()._pending) def test_independent_collection_workers_do_not_supersede_each_other(self) -> None: server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) recorded = install_fake_bundle_builder(server) @@ -1924,7 +1411,8 @@ def test_independent_collection_workers_do_not_supersede_each_other(self) -> Non ] entered = [threading.Event() for _identity in identities] releases = [threading.Event() for _identity in identities] - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle active_builds = 0 max_active_builds = 0 active_lock = threading.Lock() @@ -1951,7 +1439,7 @@ def blocking_build( with active_lock: active_builds -= 1 - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + service.build_shared_job_bundle = blocking_build # type: ignore[method-assign] results: list[list[object]] = [[] for _identity in identities] errors: list[list[BaseException]] = [[] for _identity in identities] @@ -1979,20 +1467,20 @@ def build(index: int) -> None: threads[2].start() pending_deadline = time.monotonic() + 5 while time.monotonic() < pending_deadline: - with server._job_build_scheduler_lock: - pending = server._job_build_pending + with server._ensure_job_bundle_service()._scheduler_lock: + pending = server._ensure_job_bundle_service()._pending if pending is not None and pending.worker == identities[2]: break time.sleep(0.01) else: self.fail("third collection build was not queued") threads[3].start() - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) + self.assertEqual(server._ensure_job_bundle_service()._scheduler_counts["starts"], 2) releases[0].set() self.assertTrue(entered[2].wait(5)) releases[1].set() self.assertTrue(entered[3].wait(5)) - self.assertEqual(server.job_build_scheduler_counts["starts"], 4) + self.assertEqual(server._ensure_job_bundle_service()._scheduler_counts["starts"], 4) finally: for release in releases: release.set() @@ -2004,7 +1492,7 @@ def build(index: int) -> None: self.assertEqual([len(items) for items in results], [1, 1, 1, 1]) self.assertEqual(recorded["calls"], 4) self.assertLessEqual(max_active_builds, 2) - self.assertEqual(server.job_build_scheduler_counts["supersessions"], 0) + self.assertEqual(server._ensure_job_bundle_service()._scheduler_counts["supersessions"], 0) def test_collection_independence_requires_one_immutable_cohort(self) -> None: server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) first = server.store_template_artifacts(dict(rpc.template)) @@ -2013,7 +1501,7 @@ def test_collection_independence_requires_one_immutable_cohort(self) -> None: second_template["curtime"] = int(second_template["curtime"]) + 1 second = server.store_template_artifacts(second_template) assert second is not None - payout_generation = server._payout_state_generation + payout_generation = server._payout_state_service._generation first_worker = worker("tq1worker-1", "tq1worker-1.rig") second_worker = worker("tq1worker-2", "tq1worker-2.rig") @@ -2063,7 +1551,8 @@ def test_ready_build_cancels_both_live_collection_flights(self) -> None: entered = [threading.Event(), threading.Event()] cancellation_observed = [threading.Event(), threading.Event()] release_cancelled = threading.Event() - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle def blocking_build( build_artifacts: object, @@ -2088,8 +1577,8 @@ def blocking_build( **kwargs, ) - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - payout_generation = server._payout_state_generation + service.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + payout_generation = server._payout_state_service._generation def request_for( mode: str, @@ -2132,7 +1621,7 @@ def request_for( self.assertFalse(collection_promises[0].done()) self.assertFalse(collection_promises[1].done()) self.assertFalse(ready_promise.done()) - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) + self.assertEqual(server._ensure_job_bundle_service()._scheduler_counts["starts"], 2) finally: release_cancelled.set() @@ -2145,7 +1634,7 @@ def request_for( promise.exception(timeout=5), JobBuildSuperseded, ) - self.assertEqual(server.job_build_scheduler_counts["starts"], 3) + self.assertEqual(server._ensure_job_bundle_service()._scheduler_counts["starts"], 3) def test_ready_build_cancels_retiring_only_collection_flight(self) -> None: server, rpc = coordinator(ledger=FakeLedger(miners=["miner-a"])) server._ensure_tip_refresh_state() @@ -2160,7 +1649,8 @@ def test_ready_build_cancels_retiring_only_collection_flight(self) -> None: release_active = threading.Event() retiring_cancelled = threading.Event() release_retiring = threading.Event() - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle def blocking_build( build_artifacts: object, @@ -2187,8 +1677,8 @@ def blocking_build( **kwargs, ) - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] - payout_generation = server._payout_state_generation + service.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + payout_generation = server._payout_state_service._generation def request_for( mode: str, @@ -2221,11 +1711,11 @@ def request_for( release_active.set() second_bundle = second_promise.result(timeout=5) self.assertTrue(second_bundle.collection_only) - with server._job_build_scheduler_lock: - self.assertIsNone(server._job_build_active) - assert server._job_build_retiring is not None + with server._ensure_job_bundle_service()._scheduler_lock: + self.assertIsNone(server._ensure_job_bundle_service()._active) + assert server._ensure_job_bundle_service()._retiring is not None self.assertIs( - server._job_build_retiring.request, + server._ensure_job_bundle_service()._retiring.request, first_request, ) @@ -2263,7 +1753,8 @@ def test_collection_retries_do_not_supersede_ready_build(self) -> None: stop_collections = threading.Event() ready_requests: list[object] = [] collection_requests: list[object | None] = [None, None] - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle def blocking_build( build_artifacts: object, @@ -2293,7 +1784,7 @@ def blocking_build( **kwargs, ) - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + service.build_shared_job_bundle = blocking_build # type: ignore[method-assign] collection_results: list[list[object]] = [[], []] collection_errors: list[list[BaseException]] = [[], []] ready_results: list[object] = [] @@ -2337,21 +1828,21 @@ def build_ready() -> None: ready_thread.start() self.assertTrue(collection_cancelled[0].wait(5)) self.assertTrue(collection_cancelled[1].wait(5)) - self.assertEqual(server.job_build_scheduler_counts["starts"], 2) + self.assertEqual(server._ensure_job_bundle_service()._scheduler_counts["starts"], 2) release_cancelled.set() self.assertTrue(ready_entered.wait(5)) retry_deadline = time.monotonic() + 5 while ( - server.job_build_scheduler_counts["requests"] < 5 + server._ensure_job_bundle_service()._scheduler_counts["requests"] < 5 and time.monotonic() < retry_deadline ): time.sleep(0.01) self.assertGreaterEqual( - server.job_build_scheduler_counts["requests"], + server._ensure_job_bundle_service()._scheduler_counts["requests"], 5, ) - self.assertEqual(server.job_build_scheduler_counts["starts"], 3) + self.assertEqual(server._ensure_job_bundle_service()._scheduler_counts["starts"], 3) self.assertEqual(len(ready_requests), 1) self.assertFalse( ready_requests[0].cancellation.is_set() # type: ignore[union-attr] @@ -2388,7 +1879,7 @@ def test_shutdown_cancels_builder_with_full_helper_input_pipe(self) -> None: server.ledger_attestation_signing_seed_hex = "43" * 32 artifacts = server.store_template_artifacts(dict(rpc.template)) assert artifacts is not None - payout_generation = server._payout_state_generation + payout_generation = server._payout_state_service._generation request = server._new_job_build_request( artifacts, None, @@ -2432,7 +1923,7 @@ def fill_helper_pipe(*_args: object, **kwargs: object) -> object: cancellation=build_request.cancellation, # type: ignore[union-attr] ) - server.build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] + server._ensure_job_bundle_service().build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] shutdown_finished = threading.Event() shutdown_errors: list[BaseException] = [] @@ -2445,10 +1936,10 @@ def shutdown() -> None: shutdown_finished.set() with patch( - "lab.prism.prism_coordinator.prism_tool_command", + "lab.prism.bundle_compiler.prism_tool_command", return_value=[sys.executable, "-c", "import time; time.sleep(30)"], ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", side_effect=capture_popen, ): promise = server._request_job_build(request) @@ -2508,7 +1999,7 @@ def fill_helper_pipe(*_args: object, **kwargs: object) -> object: cancellation=build_request.cancellation, # type: ignore[union-attr] ) - server.build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] + server._ensure_job_bundle_service().build_shared_job_bundle = fill_helper_pipe # type: ignore[method-assign] errors: list[BaseException] = [] def build() -> None: @@ -2522,17 +2013,17 @@ def build() -> None: errors.append(exc) with patch( - "lab.prism.prism_coordinator.prism_tool_command", + "lab.prism.bundle_compiler.prism_tool_command", return_value=[sys.executable, "-c", "import time; time.sleep(30)"], ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", side_effect=capture_popen, ): build_thread = threading.Thread(target=build) build_thread.start() self.assertTrue(helper_started.wait(5)) - with server._job_cache_lock: - controls = list(server._active_job_bundle_builds.values()) + with server._ensure_job_bundle_service()._cache_lock: + controls = list(server._ensure_job_bundle_service()._active_bundle_builds.values()) self.assertEqual(len(controls), 1) controls[0].cancel_event.set() build_thread.join(2) @@ -2545,9 +2036,87 @@ def build() -> None: self.assertFalse(build_thread.is_alive()) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0], JobBuildSuperseded) - self.assertEqual(server.shared_bundle_build_counts["superseded"], 1) - self.assertEqual(server.shared_bundle_build_counts["failed"], 0) - self.assertEqual(server.tip_refresh_superseded_results, 1) + build_counts = server._ensure_job_bundle_service().shared_preparation_metrics()[ + "build_counts" + ] + self.assertEqual(build_counts["superseded"], 1) + self.assertEqual(build_counts["failed"], 0) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["superseded_results"], 1) + + def test_externally_terminated_superseded_builder_is_not_a_crash(self) -> None: + server, rpc = coordinator() + server.signing_seed_hex = "42" * 32 + server.ledger_attestation_signing_seed_hex = "43" * 32 + artifacts = server.store_template_artifacts(dict(rpc.template)) + assert artifacts is not None + real_popen = subprocess.Popen + + def build_with_control(*_args: object, **kwargs: object) -> object: + build_request = kwargs["build_request"] + return server.build_audit_bundle( + shares=[], + found_block={ + "block_height": 10, + "coinbase_value_sats": 50_00000000, + "network_difficulty": 1, + "anchor_job_issued_at_ms": 1_700_000_000_000, + }, + prior_balances=[], + coinbase_script_sig_suffix_hex="00", + cancellation=build_request.cancellation, # type: ignore[union-attr] + ) + + server._ensure_job_bundle_service().build_shared_job_bundle = ( # type: ignore[method-assign] + build_with_control + ) + + def cancel_then_exit( + *args: object, + **kwargs: object, + ) -> subprocess.Popen[str]: + process = real_popen(*args, **kwargs) # type: ignore[arg-type] + original_poll = process.poll + original_wait = process.wait + first_poll = True + + def poll() -> int | None: + nonlocal first_poll + if not first_poll: + return original_poll() + first_poll = False + service = server._ensure_job_bundle_service() + with service._cache_lock: + controls = list(service._active_bundle_builds.values()) + self.assertEqual(len(controls), 1) + controls[0].cancel_event.set() + process.kill() + return original_wait() + + process.poll = poll # type: ignore[method-assign] + return process + + with patch( + "lab.prism.bundle_compiler.prism_tool_command", + return_value=[sys.executable, "-c", "import time; time.sleep(30)"], + ), patch( + "lab.prism.bundle_compiler.subprocess.Popen", + side_effect=cancel_then_exit, + ): + with self.assertRaises(JobBuildSuperseded): + server.shared_job_bundle( + artifacts, + mode="ready", + retry_superseded=False, + ) + + metrics = server._ensure_job_bundle_service().shared_preparation_metrics() + self.assertEqual(metrics["build_counts"]["superseded"], 1) + self.assertEqual(metrics["build_counts"]["failed"], 0) + worker_counts = server._ensure_job_bundle_service().metrics_snapshot()[ + "worker_counts" + ] + self.assertEqual(worker_counts["crashes"], 0) + def test_full_helper_input_pipe_obeys_builder_timeout(self) -> None: server, _rpc = coordinator() server.signing_seed_hex = "42" * 32 @@ -2588,10 +2157,10 @@ def build() -> None: errors.append(exc) with patch( - "lab.prism.prism_coordinator.prism_tool_command", + "lab.prism.bundle_compiler.prism_tool_command", return_value=[sys.executable, "-c", "import time; time.sleep(30)"], ), patch( - "lab.prism.prism_coordinator.subprocess.Popen", + "lab.prism.bundle_compiler.subprocess.Popen", side_effect=capture_popen, ): build_thread = threading.Thread(target=build) diff --git a/tests/test_prism_job_delivery.py b/tests/test_prism_job_delivery.py new file mode 100644 index 0000000..3c1b4b9 --- /dev/null +++ b/tests/test_prism_job_delivery.py @@ -0,0 +1,1663 @@ +#!/usr/bin/env python3 +"""Direct contract tests for the coordinator-free S2 boundary.""" + +from __future__ import annotations + +from collections import OrderedDict +from concurrent.futures import Future +from dataclasses import replace +from decimal import Decimal +import inspect +import threading +import time +import unittest +from unittest.mock import Mock +from types import SimpleNamespace + +from lab.auxpow import vardiff +from lab.prism.direct_stratum import DirectQbitStratumJob +from lab.prism.job_delivery import ( + AdmittedIdleBundleSource, + DeliveryCompatibilityHooks, + DeliverySourceAuthority, + EvictedJobEntry, + IdleDeliveryAuthority, + InitialJobRuntimePort, + InitialJobState, + JobDeliveryRuntimePort, + JobPreparationPort, + JobDeliveryService, + PayoutDeliveryPort, + PendingInitialJob, + PrismJobContext, + ProgressDeliveryPort, + RetainedJobIndex, + TipAuthorityPort, +) +import lab.prism.job_delivery as job_delivery_module +from lab.prism.prism_coordinator import ( + EvictedJobEntry as FacadeEvictedJobEntry, + PendingInitialJob as FacadePendingInitialJob, + PrismCoordinator, + PrismJobContext as FacadePrismJobContext, +) +from lab.prism.stratum_session import ClientState, SessionRegistry, WorkerIdentity +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + QbitTipTemplateSnapshot, +) +from tests.prism_vardiff_test_support import ( + client as compatibility_client, + coordinator as compatibility_coordinator, + install_idle_job_cache, + prepare_idle_client, +) + + +TIP_A = "11" * 32 +TIP_B = "22" * 32 + + +def worker(name: str = "miner-a") -> WorkerIdentity: + return WorkerIdentity( + username=name, + payout_address=name, + worker_name=None, + script_pubkey_hex="5220" + "33" * 32, + p2mr_program_hex="33" * 32, + ) + + +def client(connection_id: int = 1) -> ClientState: + state = ClientState( + sock=object(), + address=("127.0.0.1", 1), + connection_id=connection_id, + extranonce1_hex=f"{connection_id:08x}", + share_difficulty=Decimal("1"), + ) + state.subscribed = True + state.authorized = True + state.worker = worker() + state.username = state.worker.username + return state + + +def job(job_id: str, *, clean_jobs: bool = True) -> DirectQbitStratumJob: + return DirectQbitStratumJob( + job_id=job_id, + previousblockhash_display=TIP_A, + prevhash=TIP_A, + coinb1="", + coinb2="", + full_coinbase_prefix="", + full_coinbase_suffix="", + merkle_branch=(), + transaction_hexes=(), + version="20000000", + nbits="207fffff", + ntime="6553f100", + qbit_target=(1 << 255), + share_target=(1 << 255), + share_difficulty=Decimal("1"), + extranonce1_hex="00000001", + extranonce2_size=8, + clean_jobs=clean_jobs, + ) + + +def context( + job_id: str, + *, + parent: str = TIP_A, + owner: WorkerIdentity | None = None, + collection_only: bool = False, + template_generation: int = 3, + payout_generation: int = 7, +) -> PrismJobContext: + return PrismJobContext( + job=job(job_id), + template={"previousblockhash": parent}, + shares_json=[], + prior_balances=[], + found_block={}, + share_weight=1, + collection_only=collection_only, + worker=owner or worker(), + issued_at_ms=1, + template_fingerprint="fingerprint", + template_generation=template_generation, + payout_state_generation=payout_generation, + connection_id=1, + authorization_generation=0, + difficulty_generation=0, + ) + + +def tip_snapshot( + tip_hash: str = TIP_A, + *, + generation: int = 3, + fingerprint: str = "fingerprint", +) -> QbitTipTemplateSnapshot: + artifacts = CachedTemplateArtifacts( + template={"previousblockhash": tip_hash}, + fingerprint=fingerprint, + previousblockhash=tip_hash, + transaction_hexes=(), + witness_merkle_leaves_hex=(), + network_difficulty=1, + fetched_monotonic=time.monotonic(), + generation=generation, + ) + return QbitTipTemplateSnapshot( + bestblockhash=tip_hash, + previousblockhash=tip_hash, + template_fingerprint=fingerprint, + template_generation=generation, + template_artifacts=artifacts, + ) + + +class Runtime: + def __init__(self) -> None: + self.counter = 0 + self.payout_generation = 7 + self.ready = False + self.events: list[str] = [] + + def next_job_id(self) -> str: + self.counter += 1 + return f"stamped-{self.counter}" + + def collection_identity(self, owner: WorkerIdentity) -> object: + return (owner.payout_address, owner.script_pubkey_hex) + + def desired_share_difficulty(self, state: ClientState) -> Decimal: + return state.pending_share_difficulty or state.share_difficulty + + def minimum_advertised_difficulty(self, _state: ClientState) -> Decimal: + return Decimal("0") + + def share_weight(self, _owner: WorkerIdentity) -> int: + return 5 + + def current_payout_generation(self) -> int: + return self.payout_generation + + def ready_latched(self) -> bool: + return self.ready + + def template_fingerprint(self, _template: object) -> str: + return "fingerprint" + + def send_difficulty(self, _state: ClientState, _job: object) -> None: + self.events.append("difficulty") + + def send_job(self, _state: ClientState, _job: object) -> None: + self.events.append("notify") + + def send_job_batch(self, _state: ClientState, _job: object) -> None: + self.events.extend(("difficulty", "notify")) + + +def service( + state: ClientState | None = None, +) -> tuple[JobDeliveryService, SessionRegistry, Runtime]: + state = state or client() + registry = SessionRegistry( + lock=threading.RLock(), + clients={state}, + connection_generation=state.connection_id, + rejection_counts={"global": 0, "username": 0}, + ) + runtime = Runtime() + delivery = JobDeliveryService( + registry=registry, + runtime=runtime, + jobs={}, + retained=RetainedJobIndex(), + preparation=SimpleNamespace( + collection_identity=runtime.collection_identity, + ready_latched=runtime.ready_latched, + template_fingerprint=runtime.template_fingerprint, + ), # type: ignore[arg-type] + payout=SimpleNamespace( + generation=runtime.current_payout_generation, + ), # type: ignore[arg-type] + ) + return delivery, registry, runtime + + +class PrismJobDeliveryTests(unittest.TestCase): + def test_s2_ports_are_capability_scoped_and_hide_coordinator_context(self) -> None: + self.assertFalse(hasattr(job_delivery_module, "JobDeliveryOperationsPort")) + expected_methods = { + JobDeliveryRuntimePort: { + "desired_share_difficulty", + "minimum_advertised_difficulty", + "share_weight", + "vardiff_config", + "send_difficulty", + "send_job", + "send_job_batch", + }, + JobPreparationPort: { + "ensure_reorg_current", + "issuance_artifacts", + "shared_bundle", + "artifacts_current", + "clear_artifacts", + "record_failure", + "phases", + "retained_artifacts", + "chain_view_untrusted", + "admit_idle_bundle_source", + "observe_elapsed", + "collection_identity", + "ready_latched", + "template_fingerprint", + }, + TipAuthorityPort: { + "live_tip", + "observe_tip", + "published_authority", + "published_authoritative", + "current_tip_locked", + "published_template_locked", + "snapshot_current_locked", + "artifacts_parent_current_locked", + "ensure_artifacts_parent_observed", + "schedule_retry", + "prepared_obsolete", + "prepared_token_current_locked", + "record_cancellation", + "retention_authority_locked", + "consume_retained_refresh", + "published_current_locked", + }, + PayoutDeliveryPort: { + "snapshot", + "generation", + "initial_admission", + "admission", + "observe_admission", + "record_first_delivery", + }, + InitialJobRuntimePort: { + "stopping", + "wait", + "disconnect", + "submit_initial", + }, + ProgressDeliveryPort: { + "record_health_delivery", + "reconcile_health_eligibility", + }, + } + for port, methods in expected_methods.items(): + with self.subTest(port=port.__name__): + public = { + name + for name, value in port.__dict__.items() + if not name.startswith("_") and inspect.isfunction(value) + } + self.assertEqual(public, methods) + + server = compatibility_coordinator() + delivery = server._ensure_job_delivery_service() + for capability in ( + delivery.preparation, + delivery.tip_authority, + delivery.payout, + delivery.initial_runtime, + delivery.progress, + ): + self.assertFalse(hasattr(capability, "coordinator")) + self.assertFalse(hasattr(delivery.initial_runtime, "executor")) + self.assertFalse(hasattr(delivery.initial_runtime, "submit")) + + def test_compatibility_hooks_resolve_only_instance_overrides(self) -> None: + server = compatibility_coordinator() + hooks = server._ensure_job_delivery_service().hooks + self.assertIsInstance(hooks, DeliveryCompatibilityHooks) + assert hooks is not None + self.assertIsNone(hooks.build_job_override()) + + def replacement(*_args: object, **_kwargs: object) -> PrismJobContext: + return context("replacement") + + server.build_job_for_client = replacement # type: ignore[method-assign] + self.assertIs(hooks.build_job_override(), replacement) + + def test_r1_prune_resolves_late_override_or_calls_s2_directly(self) -> None: + server = compatibility_coordinator() + tip_refresh = server._ensure_tip_refresh_service() + delivery = server._ensure_job_delivery_service() + direct_calls: list[tuple[float | None, bool]] = [] + + def direct_prune( + *, + now: float | None = None, + force: bool = True, + ) -> None: + direct_calls.append((now, force)) + + delivery.prune_retained = direct_prune # type: ignore[method-assign] + tip_refresh._ports.prune_evicted_jobs(1.25, False) # type: ignore[attr-defined] + self.assertEqual(direct_calls, [(1.25, False)]) + + saved_original = server.prune_evicted_job_graveyard + override_calls: list[tuple[float | None, bool]] = [] + + def wrapper( + *, + now: float | None = None, + force: bool = True, + ) -> None: + override_calls.append((now, force)) + saved_original(now=now, force=force) + + server.prune_evicted_job_graveyard = wrapper # type: ignore[method-assign] + tip_refresh._ports.prune_evicted_jobs(2.5, True) # type: ignore[attr-defined] + self.assertEqual(override_calls, [(2.5, True)]) + self.assertEqual(direct_calls, [(1.25, False), (2.5, True)]) + + def test_post_construction_send_update_override_is_late_and_nonrecursive( + self, + ) -> None: + server = compatibility_coordinator() + delivery = server._ensure_job_delivery_service() + state = compatibility_client() + events: list[tuple[ClientState, str]] = [] + + def replacement(client_state: ClientState, candidate: object) -> None: + events.append((client_state, candidate.job_id)) + + server.send_job_update = replacement # type: ignore[method-assign] + delivery.send_update(state, job("late-override"), split_send=False) + self.assertEqual(events, [(state, "late-override")]) + + def test_send_update_override_can_call_saved_original_once(self) -> None: + server = compatibility_coordinator() + delivery = server._ensure_job_delivery_service() + state = compatibility_client() + events: list[str] = [] + original = server.send_job_update + state.send_batch = lambda _payloads: events.append("batch") # type: ignore[method-assign] + + def wrapper(client_state: ClientState, candidate: object) -> None: + events.append("wrapper") + original(client_state, candidate) + + server.send_job_update = wrapper # type: ignore[method-assign] + delivery.send_update(state, job("saved-original"), split_send=False) + self.assertEqual(events, ["wrapper", "batch"]) + + def test_facade_reexports_exact_s2_identities(self) -> None: + self.assertIs(FacadePrismJobContext, PrismJobContext) + self.assertIs(FacadeEvictedJobEntry, EvictedJobEntry) + self.assertIs(FacadePendingInitialJob, PendingInitialJob) + + def test_stamp_freezes_worker_and_all_delivery_generations(self) -> None: + state = client() + delivery, _registry, runtime = service(state) + base = job("shared") + bundle = SimpleNamespace( + collection_only=False, + collection_identity=None, + base_job=base, + template={"previousblockhash": TIP_A}, + shares_json=[], + prior_balances=[], + found_block={}, + issued_at_ms=1, + template_fingerprint="fingerprint", + template_generation=3, + payout_state_generation=7, + prospective_prior_balances=None, + payout_artifact_generation=4, + ) + state.authorization_generation = 8 + state.difficulty_generation = 9 + + stamped = delivery.stamp(state, bundle, clean_jobs=False) + + self.assertEqual(stamped.job.job_id, "prism-1") + self.assertEqual(stamped.worker, state.worker) + self.assertEqual(stamped.share_weight, 5) + self.assertEqual(stamped.authorization_generation, 8) + self.assertEqual(stamped.difficulty_generation, 9) + self.assertFalse(stamped.job.clean_jobs) + self.assertEqual(runtime.counter, 0) + + def test_difficulty_and_notify_are_adjacent_for_both_send_seams(self) -> None: + state = client() + delivery, _registry, runtime = service(state) + delivery.send_update(state, job("one"), split_send=True) + delivery.send_update(state, job("two"), split_send=False) + self.assertEqual( + runtime.events, + ["difficulty", "notify", "difficulty", "notify"], + ) + + def test_send_update_holds_no_registry_or_retained_lock(self) -> None: + state = client() + delivery, registry, _runtime = service(state) + checks: list[bool] = [] + + def check_locks(_state: ClientState, _job: object) -> None: + acquired_registry = registry.lock.acquire(blocking=False) + if acquired_registry: + registry.lock.release() + acquired_retained = delivery.retained.lock.acquire(blocking=False) + if acquired_retained: + delivery.retained.lock.release() + checks.append(acquired_registry and acquired_retained) + + delivery.runtime.send_job_batch = check_locks # type: ignore[method-assign] + delivery.send_update(state, job("one"), split_send=False) + self.assertEqual(checks, [True]) + + def test_every_final_guard_dimension_rejects_stale_delivery(self) -> None: + mutators = { + "connection": lambda state: setattr(state, "connection_id", 2), + "authorization": lambda state: setattr( + state, "authorization_generation", 1 + ), + "difficulty": lambda state: setattr(state, "difficulty_generation", 1), + "worker": lambda state: setattr(state, "worker", worker("miner-b")), + "subscription": lambda state: setattr(state, "subscribed", False), + "authorization-state": lambda state: setattr(state, "authorized", False), + "closing": lambda state: setattr(state, "closing", True), + "active": lambda state: setattr(state, "active_job", context("other")), + } + for name, mutate in mutators.items(): + with self.subTest(name=name): + state = client() + delivery, registry, _runtime = service(state) + ready = context("ready") + authority = delivery.capture_authority( + state, ready, expected_active_job=None + ) + with registry.lock: + delivery.register_locked( + state, ready, clean_jobs=True, current_tip=TIP_A + ) + mutate(state) + self.assertFalse( + delivery.record_successful_delivery(state, authority, ready, 10.0) + ) + + def test_template_and_payout_identity_are_final_guards(self) -> None: + state = client() + delivery, registry, _runtime = service(state) + ready = context("ready") + authority = delivery.capture_authority(state, ready, expected_active_job=None) + with registry.lock: + delivery.register_locked(state, ready, clean_jobs=True, current_tip=TIP_A) + changed = replace(ready, payout_state_generation=8) + state.active_job = changed + self.assertFalse( + delivery.record_successful_delivery(state, authority, changed, 10.0) + ) + + def test_disconnect_during_successful_blocking_send_emits_no_proof(self) -> None: + state = client() + delivery, registry, _runtime = service(state) + ready = context("ready") + authority = delivery.capture_authority(state, ready, expected_active_job=None) + with registry.lock: + delivery.register_locked(state, ready, clean_jobs=True, current_tip=TIP_A) + send_started = threading.Event() + release_send = threading.Event() + + def blocked_send(_state: ClientState, _job: object) -> None: + send_started.set() + release_send.wait(2) + + delivery.runtime.send_job_batch = blocked_send # type: ignore[method-assign] + result: list[bool] = [] + + def run() -> None: + delivery.send_update(state, ready.job, split_send=False) + result.append( + delivery.record_successful_delivery(state, authority, ready, 10.0) + ) + + thread = threading.Thread(target=run) + thread.start() + self.assertTrue(send_started.wait(1)) + with registry.lock: + registry.begin_retirement_locked(state) + release_send.set() + thread.join(2) + self.assertEqual(result, [False]) + self.assertIsNone(state._progress_delivered_context) + + def test_delivery_proof_is_exactly_once(self) -> None: + state = client() + delivery, registry, _runtime = service(state) + ready = context("ready") + authority = delivery.capture_authority(state, ready, expected_active_job=None) + with registry.lock: + delivery.register_locked(state, ready, clean_jobs=True, current_tip=TIP_A) + self.assertTrue( + delivery.record_successful_delivery(state, authority, ready, 10.0) + ) + self.assertFalse( + delivery.record_successful_delivery(state, authority, ready, 11.0) + ) + + def test_complete_delivery_commits_proof_before_unlocked_g1_effects(self) -> None: + state = client() + registry = SessionRegistry( + lock=threading.RLock(), + clients={state}, + connection_generation=state.connection_id, + rejection_counts={"global": 0, "username": 0}, + ) + events: list[str] = [] + + class Progress: + def record_health_delivery( + self, + delivered_client: ClientState, + delivered_context: PrismJobContext, + _delivered_monotonic: float, + ) -> None: + self.assert_unlocked() + proof = registry.eligible_snapshot()[delivered_client.connection_id] + assert proof.delivered is not None + self_outer.assertIs(proof.delivered.context, delivered_context) + events.append("record") + + def reconcile_health_eligibility(self) -> None: + self.assert_unlocked() + events.append("reconcile") + + def assert_unlocked(self) -> None: + is_owned = getattr(registry.lock, "_is_owned") + self_outer.assertFalse(is_owned()) + + self_outer = self + def source_current(*_args: object, **_kwargs: object) -> bool: + self.assertTrue(getattr(registry.lock, "_is_owned")()) + events.append("source") + return True + + delivery = JobDeliveryService( + registry=registry, + runtime=Runtime(), + jobs={}, + retained=RetainedJobIndex(), + progress=Progress(), + tip_authority=SimpleNamespace( + published_current_locked=source_current, + ), # type: ignore[arg-type] + ) + ready = context("ready") + authority = delivery.capture_authority(state, ready, expected_active_job=None) + with registry.lock: + delivery.register_locked(state, ready, clean_jobs=True, current_tip=TIP_A) + original_record = registry.record_delivery_locked + + def record_after_source(*args: object, **kwargs: object) -> bool: + self.assertEqual(events, ["source"]) + events.append("proof") + return original_record(*args, **kwargs) # type: ignore[arg-type] + + registry.record_delivery_locked = record_after_source # type: ignore[method-assign] + source = DeliverySourceAuthority( + kind="published_tip", + payout_generation=ready.payout_state_generation, + template_generation=ready.template_generation, + observation_sequence=1, + template_fingerprint=ready.template_fingerprint, + context_parent=TIP_A, + ) + + self.assertTrue( + delivery.complete_delivery( + state, + authority, + ready, + 10.0, + source_authorities=(source,), + ) + ) + self.assertEqual(events, ["source", "proof", "record", "reconcile"]) + self.assertFalse( + delivery.complete_delivery(state, authority, ready, 11.0) + ) + self.assertEqual(events, ["source", "proof", "record", "reconcile"]) + + def test_stale_source_guard_rejects_proof_and_g1_effects(self) -> None: + state = client() + registry = SessionRegistry( + lock=threading.RLock(), + clients={state}, + connection_generation=state.connection_id, + rejection_counts={"global": 0, "username": 0}, + ) + progress = SimpleNamespace( + record_health_delivery=Mock(), + reconcile_health_eligibility=Mock(), + ) + delivery = JobDeliveryService( + registry=registry, + runtime=Runtime(), + jobs={}, + retained=RetainedJobIndex(), + progress=progress, + tip_authority=SimpleNamespace( + published_current_locked=Mock(return_value=False), + ), # type: ignore[arg-type] + ) + ready = context("ready") + authority = delivery.capture_authority( + state, + ready, + expected_active_job=None, + ) + with registry.lock: + delivery.register_locked( + state, + ready, + clean_jobs=True, + current_tip=TIP_A, + ) + + stale_source = DeliverySourceAuthority( + kind="published_tip", + payout_generation=ready.payout_state_generation, + template_generation=ready.template_generation, + observation_sequence=2, + template_fingerprint=ready.template_fingerprint, + context_parent=TIP_A, + ) + + self.assertFalse( + delivery.complete_delivery( + state, + authority, + ready, + 10.0, + source_authorities=(stale_source,), + ) + ) + proof = registry.eligible_snapshot()[state.connection_id] + self.assertIsNone(proof.delivered) + progress.record_health_delivery.assert_not_called() + progress.reconcile_health_eligibility.assert_not_called() + + def test_artifact_source_accepts_new_same_tip_generation(self) -> None: + server = compatibility_coordinator() + delivery = server._ensure_job_delivery_service() + registry = server._ensure_session_registry() + tip = server._ensure_tip_refresh_service() + published = tip_snapshot(TIP_A, generation=3, fingerprint="published") + same_tip = tip_snapshot(TIP_A, generation=4, fingerprint="repository") + artifacts = same_tip.template_artifacts + assert artifacts is not None + now = time.monotonic() + tip.seed_state_for_test(latest_detected_tip=None, observation_sequence=1) + tip.seed_published_for_test( + first_seen=(TIP_A, now), + observation_sequence=1, + observed_monotonic=now, + template=published, + ) + ready = replace( + context("same-tip", template_generation=artifacts.generation), + template=artifacts.template, + template_fingerprint=artifacts.fingerprint, + ) + source = DeliverySourceAuthority( + kind="artifacts", + payout_generation=ready.payout_state_generation, + template_generation=artifacts.generation, + observation_sequence=0, + template_fingerprint=artifacts.fingerprint, + artifacts=artifacts, + ) + + with registry.lock: + self.assertTrue( + delivery.source_authority_current_locked(source, ready) + ) + + def test_artifact_source_rejects_arbitrary_older_published_parent(self) -> None: + server = compatibility_coordinator() + delivery = server._ensure_job_delivery_service() + registry = server._ensure_session_registry() + tip = server._ensure_tip_refresh_service() + published = tip_snapshot(TIP_A, generation=3, fingerprint="published") + stale = tip_snapshot(TIP_A, generation=2, fingerprint="stale") + artifacts = stale.template_artifacts + assert artifacts is not None + now = time.monotonic() + tip.seed_state_for_test( + latest_detected_tip=(TIP_B, 2), + observation_sequence=2, + divergence_started_monotonic=now, + ) + tip.seed_published_for_test( + first_seen=(TIP_A, now), + observation_sequence=1, + observed_monotonic=now, + template=published, + ) + ready = replace( + context("stale-parent", template_generation=artifacts.generation), + template=artifacts.template, + template_fingerprint=artifacts.fingerprint, + ) + source = DeliverySourceAuthority( + kind="artifacts", + payout_generation=ready.payout_state_generation, + template_generation=artifacts.generation, + observation_sequence=0, + template_fingerprint=artifacts.fingerprint, + artifacts=artifacts, + ) + + with registry.lock: + self.assertFalse( + delivery.source_authority_current_locked(source, ready) + ) + + def test_artifact_source_accepts_exact_pinned_published_lease(self) -> None: + server = compatibility_coordinator() + delivery = server._ensure_job_delivery_service() + registry = server._ensure_session_registry() + tip = server._ensure_tip_refresh_service() + published = tip_snapshot(TIP_A, generation=3, fingerprint="published") + artifacts = published.template_artifacts + assert artifacts is not None + now = time.monotonic() + tip.seed_state_for_test( + latest_detected_tip=(TIP_B, 2), + observation_sequence=2, + divergence_started_monotonic=now, + ) + tip.seed_published_for_test( + first_seen=(TIP_A, now), + observation_sequence=1, + observed_monotonic=( + now - tip.config.submit_tip_max_age_seconds - 1.0 + ), + template=published, + ) + ready = replace( + context("pinned", template_generation=artifacts.generation), + template=artifacts.template, + template_fingerprint=artifacts.fingerprint, + ) + source = DeliverySourceAuthority( + kind="artifacts", + payout_generation=ready.payout_state_generation, + template_generation=artifacts.generation, + observation_sequence=0, + template_fingerprint=artifacts.fingerprint, + artifacts=artifacts, + ) + + with registry.lock: + self.assertTrue( + delivery.source_authority_current_locked(source, ready) + ) + + def test_artifact_source_rejects_expired_pinned_published_lease(self) -> None: + server = compatibility_coordinator() + delivery = server._ensure_job_delivery_service() + registry = server._ensure_session_registry() + tip = server._ensure_tip_refresh_service() + published = tip_snapshot(TIP_A, generation=3, fingerprint="published") + artifacts = published.template_artifacts + assert artifacts is not None + now = time.monotonic() + tip.seed_state_for_test( + latest_detected_tip=(TIP_B, 2), + observation_sequence=2, + divergence_started_monotonic=( + now - tip.config.failure_exit_seconds - 1.0 + ), + ) + tip.seed_published_for_test( + first_seen=(TIP_A, now), + observation_sequence=1, + observed_monotonic=( + now - tip.config.submit_tip_max_age_seconds - 1.0 + ), + template=published, + ) + ready = replace( + context("expired", template_generation=artifacts.generation), + template=artifacts.template, + template_fingerprint=artifacts.fingerprint, + ) + source = DeliverySourceAuthority( + kind="artifacts", + payout_generation=ready.payout_state_generation, + template_generation=artifacts.generation, + observation_sequence=0, + template_fingerprint=artifacts.fingerprint, + artifacts=artifacts, + ) + + with registry.lock: + self.assertFalse( + delivery.source_authority_current_locked(source, ready) + ) + + def test_prepared_idle_delivery_carries_exact_artifact_lease(self) -> None: + tip_hash = "00" * 32 + server = compatibility_coordinator() + state = compatibility_client() + prepare_idle_client(server, state, tip=tip_hash) + bundle = install_idle_job_cache(server, tip=tip_hash) + delivery = server._ensure_job_delivery_service() + registry = server._ensure_session_registry() + tip = server._ensure_tip_refresh_service() + cache_lock = server._ensure_job_bundle_service()._cache_lock + artifacts = ( + server._ensure_job_bundle_service() + .template_repository.current_artifacts() + ) + assert artifacts is not None + tip.seed_state_for_test( + latest_detected_tip=None, + divergence_started_monotonic=None, + observation_sequence=0, + ) + tip.seed_published_for_test( + first_seen=None, + parent=None, + observation_sequence=0, + observed_monotonic=None, + template=None, + ) + prior_active = state.active_job + prior_window = state.vardiff_window_started_monotonic + state.pending_share_difficulty = Decimal("4") + idle_authority = IdleDeliveryAuthority( + connection_id=state.connection_id, + worker=state.worker, + expected_active_job=prior_active, + expected_window_started=prior_window, + pending_difficulty=Decimal("4"), + ) + leases: list[AdmittedIdleBundleSource] = [] + bootstrap_events: list[str] = [] + original_rpc_call = server.rpc.call + + def observe_live_rpc( + method: str, + params: list[object] | None = None, + ) -> object: + if method == "getbestblockhash": + self.assertFalse(bool(getattr(registry.lock, "_is_owned")())) + acquired = cache_lock.acquire(blocking=False) + self.assertTrue(acquired) + if acquired: + cache_lock.release() + bootstrap_events.append("live") + return original_rpc_call(method, params) + + server.rpc.call = observe_live_rpc # type: ignore[method-assign] + original_observe_tip = tip.observe_tip + + def observe_tip_unlocked(*args: object, **kwargs: object) -> bool: + self.assertFalse(bool(getattr(registry.lock, "_is_owned")())) + acquired = cache_lock.acquire(blocking=False) + self.assertTrue(acquired) + if acquired: + cache_lock.release() + bootstrap_events.append("observe") + return original_observe_tip(*args, **kwargs) # type: ignore[arg-type] + + tip.observe_tip = observe_tip_unlocked # type: ignore[method-assign] + original_admit = server._admit_idle_bundle_source + + def observe_admission( + admitted_client: ClientState, + admitted_bundle: object, + *, + allow_uncached: bool, + ) -> AdmittedIdleBundleSource | None: + self.assertFalse(bool(getattr(registry.lock, "_is_owned")())) + admitted = original_admit( + admitted_client, + admitted_bundle, # type: ignore[arg-type] + allow_uncached=allow_uncached, + ) + if admitted is not None: + leases.append(admitted) + return admitted + + server._admit_idle_bundle_source = observe_admission # type: ignore[method-assign] + original_parent_current = tip.artifacts_parent_current_locked + + def source_current( + source_artifacts: CachedTemplateArtifacts, + *, + now: float, + ) -> bool: + acquired = cache_lock.acquire(blocking=False) + self.assertTrue(acquired) + if acquired: + cache_lock.release() + return original_parent_current(source_artifacts, now=now) + + tip.artifacts_parent_current_locked = source_current # type: ignore[method-assign] + sent: list[dict[str, object]] = [] + state.send = sent.append # type: ignore[method-assign] + + with state.job_update_lock: + delivered = delivery.maybe_send_job_locked( + state, + clean_jobs=True, + raise_on_build_failure=True, + prepared_bundle=bundle, + idle_authority=idle_authority, + prepared_bundle_allow_uncached=True, + ) + + self.assertTrue(delivered) + self.assertEqual(len(leases), 1) + self.assertEqual(bootstrap_events, ["live", "observe"]) + self.assertIs(leases[0].artifacts, artifacts) + self.assertIs(leases[0].bundle, bundle) + self.assertEqual(leases[0].cache_identity, bundle.key) + self.assertIsNotNone(state.active_job) + self.assertIs(state.active_job.template, artifacts.template) + self.assertEqual( + [payload["method"] for payload in sent], + ["mining.set_difficulty", "mining.notify"], + ) + + def test_prepared_idle_delivery_rejects_live_tip_mismatch_without_send( + self, + ) -> None: + server = compatibility_coordinator() + state = compatibility_client() + prepare_idle_client(server, state, tip=TIP_A) + bundle = install_idle_job_cache(server, tip=TIP_A) + delivery = server._ensure_job_delivery_service() + tip = server._ensure_tip_refresh_service() + tip.seed_state_for_test( + latest_detected_tip=None, + divergence_started_monotonic=None, + observation_sequence=0, + ) + tip.seed_published_for_test( + first_seen=None, + parent=None, + observation_sequence=0, + observed_monotonic=None, + template=None, + ) + self.assertIsNone(tip.newest_observed_tip()) + prior_active = state.active_job + prior_window = state.vardiff_window_started_monotonic + state.pending_share_difficulty = Decimal("4") + idle_authority = IdleDeliveryAuthority( + connection_id=state.connection_id, + worker=state.worker, + expected_active_job=prior_active, + expected_window_started=prior_window, + pending_difficulty=Decimal("4"), + ) + sent: list[dict[str, object]] = [] + state.send = sent.append # type: ignore[method-assign] + + with state.job_update_lock: + delivered = delivery.maybe_send_job_locked( + state, + clean_jobs=True, + raise_on_build_failure=True, + prepared_bundle=bundle, + idle_authority=idle_authority, + prepared_bundle_allow_uncached=True, + ) + + self.assertFalse(delivered) + self.assertEqual(sent, []) + self.assertEqual(tip.newest_observed_tip(), "00" * 32) + self.assertIs(state.active_job, prior_active) + + def test_real_r1_source_flip_waits_for_atomic_s1_proof(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + state.worker = worker() + state.username = state.worker.username + server.clients = {state} + delivery = server._ensure_job_delivery_service() + registry = server._ensure_session_registry() + tip = server._ensure_tip_refresh_service() + self.assertIs(tip._state_lock, registry.lock) # type: ignore[attr-defined] + + published = tip_snapshot() + now = time.monotonic() + tip.seed_state_for_test(latest_detected_tip=None, observation_sequence=1) + tip.seed_published_for_test( + first_seen=(TIP_A, now), + observation_sequence=1, + observed_monotonic=now, + template=published, + ) + ready = context("atomic-proof", owner=state.worker, payout_generation=0) + authority = delivery.capture_authority( + state, + ready, + expected_active_job=None, + ) + with registry.lock: + delivery.register_locked( + state, + ready, + clean_jobs=True, + current_tip=TIP_A, + ) + source = DeliverySourceAuthority( + kind="published_tip", + payout_generation=ready.payout_state_generation, + template_generation=ready.template_generation, + observation_sequence=1, + template_fingerprint=ready.template_fingerprint, + context_parent=TIP_A, + ) + assert delivery.progress is not None + delivery.progress._record_health_delivery = lambda *_args: None # type: ignore[attr-defined] + delivery.progress._reconcile_health_eligibility = lambda: None # type: ignore[attr-defined] + + proof_entered = threading.Event() + release_proof = threading.Event() + mutation_done = threading.Event() + original_record = registry.record_delivery_locked + + def blocking_record(*args: object, **kwargs: object) -> bool: + proof_entered.set() + self.assertTrue(release_proof.wait(2)) + return original_record(*args, **kwargs) # type: ignore[arg-type] + + registry.record_delivery_locked = blocking_record # type: ignore[method-assign] + results: list[bool] = [] + delivery_thread = threading.Thread( + target=lambda: results.append( + delivery.complete_delivery( + state, + authority, + ready, + 10.0, + source_authorities=(source,), + ) + ) + ) + + replacement = tip_snapshot(TIP_B, generation=4, fingerprint="new") + + def flip_source() -> None: + tip.seed_published_for_test( + first_seen=(TIP_B, time.monotonic()), + observation_sequence=2, + observed_monotonic=time.monotonic(), + template=replacement, + ) + mutation_done.set() + + delivery_thread.start() + self.assertTrue(proof_entered.wait(1)) + mutation_thread = threading.Thread(target=flip_source) + mutation_thread.start() + self.assertFalse(mutation_done.wait(0.05)) + release_proof.set() + delivery_thread.join(2) + mutation_thread.join(2) + self.assertFalse(delivery_thread.is_alive()) + self.assertFalse(mutation_thread.is_alive()) + self.assertEqual(results, [True]) + self.assertTrue(mutation_done.is_set()) + + def test_real_p1_mutation_waits_for_delivery_admission_not_registry(self) -> None: + server = compatibility_coordinator() + delivery = server._ensure_job_delivery_service() + registry = server._ensure_session_registry() + payout = server._ensure_payout_state_service() + generation = payout.snapshot().generation + admitted_registry = threading.Event() + release_delivery = threading.Event() + mutation_done = threading.Event() + + def hold_delivery() -> None: + assert delivery.payout is not None + with delivery.payout.admission( + lambda: False, + generation=generation, + priority=True, + ) as admitted: + self.assertTrue(admitted) + with registry.lock: + admitted_registry.set() + self.assertTrue(release_delivery.wait(2)) + + def mutate() -> None: + with payout.delivery_gate.mutation(): + mutation_done.set() + + delivery_thread = threading.Thread(target=hold_delivery) + delivery_thread.start() + self.assertTrue(admitted_registry.wait(1)) + mutation_thread = threading.Thread(target=mutate) + mutation_thread.start() + self.assertFalse(mutation_done.wait(0.05)) + release_delivery.set() + delivery_thread.join(2) + mutation_thread.join(2) + self.assertFalse(delivery_thread.is_alive()) + self.assertFalse(mutation_thread.is_alive()) + self.assertTrue(mutation_done.is_set()) + + def test_empty_registry_never_admits_or_records_nonmember_delivery(self) -> None: + state = client() + registry = SessionRegistry( + lock=threading.RLock(), + clients=set(), + connection_generation=state.connection_id, + rejection_counts={"global": 0, "username": 0}, + ) + runtime = Runtime() + delivery = JobDeliveryService( + registry=registry, + runtime=runtime, + jobs={}, + retained=RetainedJobIndex(), + ) + ready = context("ready") + authority = delivery.capture_authority( + state, ready, expected_active_job=None + ) + self.assertFalse( + delivery.authority_current_locked( + state, authority, expected_active_job=None + ) + ) + state.active_job = ready + self.assertFalse( + delivery.record_successful_delivery(state, authority, ready, 10.0) + ) + self.assertIsNone(state._progress_delivered_context) + + def test_collection_context_becomes_refresh_needed_after_ready_latch(self) -> None: + state = client() + delivery, _registry, runtime = service(state) + state.active_job = context("collection", collection_only=True) + snapshot = QbitTipTemplateSnapshot( + bestblockhash=TIP_A, + previousblockhash=TIP_A, + template_fingerprint="fingerprint", + template_generation=3, + ) + self.assertFalse(delivery.client_needs_refresh(state, snapshot)) + runtime.ready = True + self.assertTrue(delivery.client_needs_refresh(state, snapshot)) + + def test_clean_registration_retains_frozen_original_context(self) -> None: + state = client() + delivery, registry, _runtime = service(state) + original_owner = state.worker + old = context("old", owner=original_owner) + new = context("new", owner=original_owner) + with registry.lock: + delivery.register_locked(state, old, clean_jobs=False, current_tip=TIP_A) + delivery.register_locked(state, new, clean_jobs=True, current_tip=TIP_A) + state.worker = worker("miner-b") + state.share_difficulty = Decimal("32") + retained = delivery.retained.lookup( + state, + "old", + current_tip=TIP_A, + current_tip_first_delivery=None, + cached_parent=None, + now=1.0, + ) + self.assertIsNotNone(retained) + assert retained is not None + self.assertEqual(retained.context.worker, original_owner) + self.assertEqual(retained.context.job.share_difficulty, Decimal("1")) + + def test_same_tip_capacity_is_per_connection(self) -> None: + index = RetainedJobIndex(same_tip_per_connection=1) + first = client(1) + second = client(2) + index.retain(first, "a1", context("a1"), current_tip=TIP_A, now=1) + index.retain(second, "b1", context("b1"), current_tip=TIP_A, now=1) + index.retain(first, "a2", context("a2"), current_tip=TIP_A, now=2) + self.assertNotIn("a1", index.graveyard) + self.assertIn("a2", index.graveyard) + self.assertIn("b1", index.graveyard) + + def test_stale_grace_begins_at_first_replacement_delivery(self) -> None: + state = client() + index = RetainedJobIndex(stale_grace_seconds=3) + index.retain(state, "old", context("old"), current_tip=TIP_A, now=1) + index.prune( + current_tip=TIP_B, + current_tip_first_delivery=2, + cached_parent=TIP_A, + now=20, + ) + self.assertIn("old", index.graveyard) + state.tip_work_delivered = (TIP_B, 20) + index.prune( + current_tip=TIP_B, + current_tip_first_delivery=2, + cached_parent=TIP_A, + now=22.9, + ) + self.assertIn("old", index.graveyard) + index.prune( + current_tip=TIP_B, + current_tip_first_delivery=2, + cached_parent=TIP_A, + now=23.1, + ) + self.assertNotIn("old", index.graveyard) + + def test_prior_tip_does_not_consume_same_tip_capacity(self) -> None: + state = client() + index = RetainedJobIndex(same_tip_per_connection=1) + index.retain(state, "prior", context("prior", parent=TIP_A), current_tip=TIP_B) + index.retain(state, "same-1", context("same-1", parent=TIP_B), current_tip=TIP_B) + index.retain(state, "same-2", context("same-2", parent=TIP_B), current_tip=TIP_B) + self.assertIn("prior", index.graveyard) + self.assertNotIn("same-1", index.graveyard) + self.assertIn("same-2", index.graveyard) + + def test_disconnect_retires_active_and_retained_indexes(self) -> None: + state = client() + delivery, registry, _runtime = service(state) + old = context("old") + new = context("new") + with registry.lock: + delivery.register_locked(state, old, clean_jobs=False, current_tip=TIP_A) + delivery.register_locked(state, new, clean_jobs=True, current_tip=TIP_A) + delivery.retire_client_locked(state) + self.assertEqual(delivery.jobs, {}) + self.assertEqual(delivery.retained.graveyard, {}) + self.assertEqual(state.active_job_ids, set()) + + def test_map_adoption_rebuilds_indexes_and_converts_legacy_entries(self) -> None: + state = client() + frozen = context("legacy") + replacement = {"legacy": (frozen, state.connection_id, 1.0)} + index = RetainedJobIndex() + index.adopt( + graveyard=replacement, # type: ignore[arg-type] + by_connection={}, + same_tip_by_connection={}, + same_tip_job_ids=OrderedDict(), + current_tip=TIP_A, + ) + self.assertIsInstance(index.graveyard["legacy"], EvictedJobEntry) + self.assertIn("legacy", index.by_connection[state.connection_id]) + + def test_graveyard_only_replacement_rebuilds_disconnect_index(self) -> None: + state = client() + index = RetainedJobIndex() + index.retain(state, "old", context("old"), current_tip=TIP_A) + replacement = OrderedDict( + [("new", EvictedJobEntry(context("new"), state.connection_id, 1, TIP_A))] + ) + index.adopt( + graveyard=replacement, + by_connection=index.by_connection, + same_tip_by_connection=index.same_tip_by_connection, + same_tip_job_ids=index.same_tip_job_ids, + current_tip=TIP_A, + ) + self.assertEqual(tuple(index.by_connection[state.connection_id]), ("new",)) + self.assertEqual(index.retire_connection(state.connection_id), ("new",)) + self.assertEqual(index.graveyard, {}) + + def test_difficulty_transition_clears_only_matching_pending_value(self) -> None: + state = client() + config = vardiff.VardiffConfig( + enabled=True, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("1"), + max_difficulty=Decimal("1024"), + retarget_interval_seconds=Decimal("90"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("1"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("0.4"), + retarget_tolerance=Decimal("0.25"), + ) + state.pending_share_difficulty = Decimal("2") + JobDeliveryService.apply_job_difficulty( + state, + replace(job("job"), share_difficulty=Decimal("2")), + config=config, + ) + self.assertIsNone(state.pending_share_difficulty) + + def test_initial_request_coalesces_one_identity(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + state.worker = worker() + state.username = state.worker.username + server.clients = {state} + submitted: list[PendingInitialJob] = [] + + def submit(request: PendingInitialJob) -> bool: + submitted.append(request) + return True + + server._submit_initial_job_request = submit # type: ignore[method-assign] + self.assertTrue(server.request_initial_job_delivery(state)) + first = server.pending_initial_jobs[state] + self.assertEqual(submitted, [first]) + self.assertTrue(server.request_initial_job_delivery(state)) + self.assertIs(server.pending_initial_jobs[state], first) + self.assertEqual(submitted, [first]) + self.assertEqual(server.initial_job_coalesced_count, 1) + + def test_initial_reauthorization_replaces_cancelled_predecessor(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + state.worker = worker() + state.username = state.worker.username + server.clients = {state} + server._submit_initial_job_request = lambda _request: True # type: ignore[method-assign] + self.assertTrue(server.request_initial_job_delivery(state)) + first = server.pending_initial_jobs[state] + state.authorization_generation += 1 + self.assertTrue(server.request_initial_job_delivery(state)) + replacement = server.pending_initial_jobs[state] + self.assertIsNot(replacement, first) + self.assertTrue(first.cancelled.is_set()) + self.assertEqual(server.initial_job_superseded_count, 1) + + def test_initial_timeout_marks_closing_before_disconnect(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + state.worker = worker() + state.username = state.worker.username + server.clients = {state} + server._submit_initial_job_request = lambda _request: True # type: ignore[method-assign] + server.stratum_initial_job_timeout_seconds = 1 + self.assertTrue(server.request_initial_job_delivery(state)) + request = server.pending_initial_jobs[state] + observed: list[bool] = [] + server.disconnect_client = lambda candidate: observed.append(candidate.closing) # type: ignore[method-assign] + assert request.deadline_monotonic is not None + self.assertEqual( + server.sweep_initial_job_timeouts(now=request.deadline_monotonic), + 1, + ) + self.assertEqual(observed, [True]) + + def test_initial_future_cancellation_callbacks_run_outside_registry(self) -> None: + for operation in ("cancel", "expire", "shutdown"): + with self.subTest(operation=operation): + state = client() + delivery, registry, _runtime = service(state) + future: Future[bool] = Future() + request = PendingInitialJob( + client=state, + connection_id=state.connection_id, + authorization_generation=state.authorization_generation, + difficulty_generation=state.difficulty_generation, + worker=state.worker, + requested_monotonic=0.0, + deadline_monotonic=0.0, + future=future, + ) + delivery.initial_state.pending[state] = request + lock_states: list[bool] = [] + future.add_done_callback( + lambda _future: lock_states.append( + bool(getattr(registry.lock, "_is_owned")()) + ) + ) + + if operation == "cancel": + delivery.cancel_initial_job(state, count=True) + elif operation == "expire": + delivery.initial_runtime = SimpleNamespace( + disconnect=lambda _client: None, + ) # type: ignore[assignment] + delivery.sweep_initial_job_timeouts(now=1.0) + else: + delivery.shutdown_initial_jobs() + + self.assertTrue(future.cancelled()) + self.assertEqual(lock_states, [False]) + + def test_coordinator_adopts_replaced_jobs_mapping(self) -> None: + server = PrismCoordinator.__new__(PrismCoordinator) + seeded: dict[str, PrismJobContext] = {"seeded": context("seeded")} + server.jobs = seeded + first_service = server._ensure_job_delivery_service() + self.assertIs(first_service.jobs, seeded) + replacement: dict[str, PrismJobContext] = { + "replacement": context("replacement") + } + server.jobs = replacement + self.assertIs(first_service.jobs, replacement) + self.assertIs(server.jobs, replacement) + + def test_coordinator_adopts_replaced_pending_initial_mapping(self) -> None: + server = compatibility_coordinator() + server._ensure_initial_job_state() + replacement: dict[ClientState, PendingInitialJob] = {} + server.pending_initial_jobs = replacement + server._ensure_initial_job_state() + self.assertIs(server._initial_job_tracker.pending, replacement) + + def test_coordinator_compatibility_aliases_adopt_s2_owned_state(self) -> None: + server = compatibility_coordinator() + server.stratum_max_pending_initial_jobs = 3 + server.initial_job_sent_count = 4 + server.job_counter = 7 + delivery = server._ensure_job_delivery_service() + self.assertIsInstance(delivery.initial_state, InitialJobState) + self.assertEqual(delivery.initial_state.config.max_pending, 3) + self.assertEqual(delivery.initial_state.sent_count, 4) + self.assertEqual(delivery.next_job_id(), "prism-8") + self.assertEqual(server.job_counter, 8) + + server.stratum_max_pending_initial_jobs = 5 + server.initial_job_sent_count = 6 + self.assertEqual(delivery.initial_state.config.max_pending, 5) + self.assertEqual(delivery.initial_state.sent_count, 6) + + retained = OrderedDict() + by_connection: dict[int, OrderedDict[str, None]] = {} + server.evicted_job_graveyard = retained + server.evicted_jobs_by_connection = by_connection + self.assertIs(delivery.retained.graveyard, retained) + self.assertIs(delivery.retained.by_connection, by_connection) + + def test_direct_build_membership_loss_never_registers_or_sends(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + state.worker = worker() + state.username = state.worker.username + events: list[str] = [] + + def remove_membership( + _client: ClientState, + *, + clean_jobs: bool, + ) -> PrismJobContext: + registry = server._ensure_session_registry() + with registry.lock: + registry._discard_client_locked(state) + return context( + "lost-membership", + owner=state.worker, + payout_generation=0, + ) + + server.build_job_for_client = remove_membership # type: ignore[method-assign] + server.send_difficulty = lambda *_args: events.append("difficulty") # type: ignore[method-assign] + server.send_job = lambda *_args: events.append("notify") # type: ignore[method-assign] + server.apply_job_difficulty = lambda *_args: None # type: ignore[method-assign] + + self.assertFalse(server.maybe_send_job(state, clean_jobs=True)) + self.assertEqual(events, []) + self.assertIsNone(state.active_job) + self.assertEqual(state.active_job_ids, set()) + self.assertNotIn("lost-membership", server.jobs) + self.assertIsNone(state._progress_delivered_context) + self.assertNotIn( + state.connection_id, + server._ensure_session_registry().eligible_snapshot(), + ) + + def test_coordinator_send_monkeypatches_remain_dynamic_and_adjacent(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + events: list[str] = [] + server.send_difficulty = lambda *_args: events.append("difficulty") # type: ignore[method-assign] + server.send_job = lambda *_args: events.append("notify") # type: ignore[method-assign] + server.send_job_update(state, job("patched")) + self.assertEqual(events, ["difficulty", "notify"]) + + def test_coordinator_stamp_dependencies_remain_dynamic_after_construction(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + state.worker = worker() + server._ensure_job_delivery_service() + server.desired_client_share_difficulty = lambda _client: Decimal("4") # type: ignore[method-assign] + server.client_minimum_advertised_difficulty = lambda _client: Decimal("0") # type: ignore[method-assign] + server.share_weight_for_worker = lambda _worker: 17 # type: ignore[method-assign] + bundle = SimpleNamespace( + collection_only=False, + collection_identity=None, + base_job=replace(job("shared"), qbit_target=1), + template={"previousblockhash": TIP_A}, + shares_json=[], + prior_balances=[], + found_block={}, + issued_at_ms=1, + template_fingerprint="fingerprint", + template_generation=3, + payout_state_generation=7, + prospective_prior_balances=None, + payout_artifact_generation=4, + ) + stamped = server.stamp_job_for_client(state, bundle, clean_jobs=True) + self.assertEqual(stamped.share_weight, 17) + self.assertEqual(stamped.job.share_difficulty, Decimal("4")) + + def test_collection_identity_override_remains_dynamic_after_construction(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + state.worker = worker() + delivery = server._ensure_job_delivery_service() + replacement_identity = ("replacement", "5220" + "44" * 32) + server._collection_bundle_identity = lambda _worker: replacement_identity # type: ignore[method-assign] + server.share_weight_for_worker = lambda _worker: 1 # type: ignore[method-assign] + server.desired_client_share_difficulty = lambda _client: Decimal("1") # type: ignore[method-assign] + server.client_minimum_advertised_difficulty = lambda _client: Decimal("0") # type: ignore[method-assign] + bundle = SimpleNamespace( + collection_only=True, + collection_identity=replacement_identity, + base_job=replace(job("shared"), qbit_target=1), + template={"previousblockhash": TIP_A}, + shares_json=[], + prior_balances=[], + found_block={}, + issued_at_ms=1, + template_fingerprint="fingerprint", + template_generation=3, + payout_state_generation=0, + prospective_prior_balances=None, + payout_artifact_generation=4, + ) + + stamped = delivery.stamp(state, bundle, clean_jobs=True) + + self.assertTrue(stamped.collection_only) + self.assertEqual(stamped.worker, state.worker) + + def test_coordinator_g1_adapter_never_recommits_s1_delivery_proof(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + state.worker = worker() + state.username = state.worker.username + server.clients = {state} + events: list[str] = [] + delivery = server._ensure_job_delivery_service() + assert delivery.progress is not None + delivery.progress._record_health_delivery = ( # type: ignore[attr-defined] + lambda *_args: events.append("health") + ) + delivery.progress._reconcile_health_eligibility = ( # type: ignore[attr-defined] + lambda: events.append("reconcile") + ) + registry = server._ensure_session_registry() + ready = context("ready", owner=state.worker, payout_generation=0) + authority = delivery.capture_authority(state, ready, expected_active_job=None) + with registry.lock: + delivery.register_locked(state, ready, clean_jobs=True, current_tip=TIP_A) + record_calls = 0 + original_record = registry.record_delivery_locked + + def counted_record(*args: object, **kwargs: object) -> bool: + nonlocal record_calls + record_calls += 1 + return original_record(*args, **kwargs) # type: ignore[arg-type] + + registry.record_delivery_locked = counted_record # type: ignore[method-assign] + + self.assertTrue(delivery.complete_delivery(state, authority, ready, 10.0)) + self.assertFalse(delivery.complete_delivery(state, authority, ready, 11.0)) + self.assertEqual(record_calls, 1) + self.assertEqual(events, ["health", "reconcile"]) + self.assertIs(state._progress_delivered_context, ready) + + def test_coordinator_delivery_facades_execute_service_owned_state_machines(self) -> None: + server = compatibility_coordinator() + state = compatibility_client() + service = server._ensure_job_delivery_service() + schedule = Mock(return_value=True) + maybe_send = Mock(return_value=True) + prepared = Mock(return_value=SimpleNamespace(status="sent")) + advertise = Mock(return_value=True) + service.schedule_initial_job = schedule # type: ignore[method-assign] + service.maybe_send_job = maybe_send # type: ignore[method-assign] + service.send_prepared_job = prepared # type: ignore[method-assign] + service.advertise_client_difficulty = advertise # type: ignore[method-assign] + + self.assertTrue(server.schedule_initial_job(state)) + self.assertTrue(server.maybe_send_job(state, clean_jobs=True)) + sent = server.send_prepared_job( + state, + SimpleNamespace(), # type: ignore[arg-type] + SimpleNamespace(), # type: ignore[arg-type] + SimpleNamespace(), # type: ignore[arg-type] + state.connection_id, + None, + ) + self.assertEqual(sent.status, "sent") + self.assertTrue(server.advertise_client_difficulty(state, Decimal("4"))) + schedule.assert_called_once_with(state) + maybe_send.assert_called_once_with( + state, + clean_jobs=True, + raise_on_reorg_failure=False, + raise_on_build_failure=False, + tip_refresh_snapshot=None, + tip_refresh_observation_sequence=None, + ) + prepared.assert_called_once() + advertise.assert_called_once_with(state, Decimal("4")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_payout_state.py b/tests/test_prism_payout_state.py index 72a5a20..547c3b5 100644 --- a/tests/test_prism_payout_state.py +++ b/tests/test_prism_payout_state.py @@ -38,8 +38,8 @@ def wait(self, timeout: float | None = None) -> bool: assert artifacts is not None parent_hash = str(rpc.template["previousblockhash"]) preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 + server._payout_state_service.preview_condition = preview_condition + server._payout_state_service.set_preview_wait_seconds_for_test(10) preview = [ { "recipient_id": "miner-a", @@ -85,7 +85,7 @@ def build_child_bundle() -> None: self.assertEqual(ledger.current_balance_reads, 1) self.assertEqual( # type: ignore[union-attr] bundle.payout_state_generation, - server._payout_state_generation, + server._payout_state_service.snapshot().generation, ) self.assertEqual(server._prior_balances_for_job_parent(parent_hash), []) @@ -109,13 +109,13 @@ def test_parent_preview_publication_is_idempotent_and_withdrawal_invalidates( server._publish_accepted_block_payout_preview(parent_hash, preview), preview, ) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service.snapshot().generation, 1) self.assertEqual( server._publish_accepted_block_payout_preview(parent_hash, preview), preview, ) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service.snapshot().generation, 1) with self.assertRaisesRegex(RuntimeError, "changed during retry"): server._publish_accepted_block_payout_preview( parent_hash, @@ -126,21 +126,21 @@ def test_parent_preview_publication_is_idempotent_and_withdrawal_invalidates( parent_hash, invalidate_published=True, ) - self.assertEqual(server._payout_state_generation, 2) - self.assertEqual(server._accepted_block_payout_previews, {}) + self.assertEqual(server._payout_state_service.snapshot().generation, 2) + self.assertEqual(server._payout_state_service.previews, {}) self.assertEqual( - server._invalidated_accepted_block_payout_previews, + server._payout_state_service.invalidated_previews, {parent_hash: None}, ) with self.assertRaisesRegex(TemplateRefreshBlocked, "was withdrawn"): server._prior_balances_for_job_parent(parent_hash) server._begin_accepted_block_payout_preview(parent_hash) - self.assertEqual(server._invalidated_accepted_block_payout_previews, {}) + self.assertEqual(server._payout_state_service.invalidated_previews, {}) server._clear_accepted_block_payout_preview(parent_hash) def test_unpublished_parent_preview_retries_and_reopens_delivery(self) -> None: server, _rpc = coordinator() - server.payout_reconcile_supersession_retries = 2 + server._payout_state_service.set_reconcile_retries_for_test(2) parent_hash = "ac" * 32 preview = [ { @@ -153,8 +153,8 @@ def test_unpublished_parent_preview_retries_and_reopens_delivery(self) -> None: server._begin_accepted_block_payout_preview(parent_hash) with patch.object( - server, - "_publish_payout_state_candidate", + server._payout_state_service, + "publish_candidate", return_value=None, ) as publish_candidate: self.assertEqual( @@ -162,24 +162,24 @@ def test_unpublished_parent_preview_retries_and_reopens_delivery(self) -> None: preview, ) - transition = server._accepted_block_payout_previews[parent_hash] + transition = server._payout_state_service.previews[parent_hash] self.assertEqual(publish_candidate.call_count, 3) self.assertIsNotNone(transition.preview) self.assertIsNone(transition.published_generation) - self.assertEqual(server._payout_state_generation, 0) + self.assertEqual(server._payout_state_service.snapshot().generation, 0) self.assertTrue(server._payout_state_publication_fenced()) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertTrue(server._payout_state_service.delivery_gate._delivery_blocked) self.assertEqual( server._publish_accepted_block_payout_preview(parent_hash, preview), preview, ) - transition = server._accepted_block_payout_previews[parent_hash] + transition = server._payout_state_service.previews[parent_hash] self.assertEqual(transition.published_generation, 1) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service.snapshot().generation, 1) self.assertFalse(server._payout_state_publication_fenced()) - self.assertFalse(server._payout_state_delivery_gate._delivery_blocked) + self.assertFalse(server._payout_state_service.delivery_gate._delivery_blocked) def test_withdrawn_landed_transition_blocks_active_descendant_fallback( self, ) -> None: @@ -227,7 +227,7 @@ def active_chain_call( parent_height=11, ) self.assertEqual(ledger.current_balance_reads, 0) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_inactive_landed_ancestor_rejects_preview_patched_artifact( self, ) -> None: @@ -257,8 +257,8 @@ def test_inactive_landed_ancestor_rejects_preview_patched_artifact( block_height=10, ) server._publish_accepted_block_payout_preview(accepted_hash, preview) - with server._job_cache_lock: - artifact = server._payout_ledger_artifact + with server._ensure_job_bundle_service()._cache_lock: + artifact = server._payout_state_service.snapshot().ledger_artifact self.assertIsNotNone(artifact) assert artifact is not None self.assertEqual(list(artifact.prior_balances), preview) @@ -283,7 +283,7 @@ def alternate_chain_call( server.shared_job_bundle(artifacts, mode="ready") self.assertEqual(recorded["calls"], 0) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_waiting_child_does_not_fall_back_after_transition_withdrawal( self, ) -> None: @@ -309,8 +309,8 @@ def wait(self, timeout: float | None = None) -> bool: server, _rpc = coordinator(ledger=ledger) parent_hash = "be" * 32 preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 + server._payout_state_service.preview_condition = preview_condition + server._payout_state_service.set_preview_wait_seconds_for_test(10) server._begin_accepted_block_payout_preview( parent_hash, block_height=10, @@ -350,13 +350,13 @@ def read_parent_balances() -> None: def test_pending_parent_preview_wait_is_bounded_and_retryable(self) -> None: server, _rpc = coordinator() parent_hash = "ac" * 32 - server.accepted_block_payout_preview_wait_seconds = 0.01 + server._payout_state_service.set_preview_wait_seconds_for_test(0.01) server._begin_accepted_block_payout_preview(parent_hash) with self.assertRaisesRegex(TemplateRefreshBlocked, "not ready"): server._prior_balances_for_job_parent(parent_hash) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) server._clear_accepted_block_payout_preview(parent_hash) def test_replayed_active_ancestor_blocks_descendant_until_preview(self) -> None: class PreviewLedger(FakeLedger): @@ -402,8 +402,8 @@ def active_chain_call( rpc.call = active_chain_call # type: ignore[method-assign] preview_condition = ObservedCondition() - server._accepted_block_payout_preview_condition = preview_condition - server.accepted_block_payout_preview_wait_seconds = 10 + server._payout_state_service.preview_condition = preview_condition + server._payout_state_service.set_preview_wait_seconds_for_test(10) server._begin_accepted_block_payout_preview(accepted_hash, block_height=10) balances: list[list[dict[str, object]]] = [] errors: list[BaseException] = [] @@ -456,7 +456,8 @@ def test_readiness_latch_during_preparation_admission_reselects_ready_mode(self) assert artifacts is not None first_lookup_entered = threading.Event() release_first_lookup = threading.Event() - original_lookup = server._lookup_job_bundle + service = server._ensure_job_bundle_service() + original_lookup = service.lookup_bundle lookup_calls = 0 def block_first_lookup(key: tuple[object, ...]) -> object: @@ -467,7 +468,7 @@ def block_first_lookup(key: tuple[object, ...]) -> object: self.assertTrue(release_first_lookup.wait(2.0)) return original_lookup(key) - server._lookup_job_bundle = block_first_lookup # type: ignore[method-assign] + service.lookup_bundle = block_first_lookup # type: ignore[method-assign] bundles: list[object] = [] errors: list[BaseException] = [] @@ -504,23 +505,18 @@ class Cancellation: def cancel(self) -> None: cancellations.append("cancelled") - class InjectCurrentFanoutLock: - def __enter__(self) -> object: - server._active_tip_refresh = (CurrentToken(), Cancellation()) - return self - - def __exit__(self, *_args: object) -> None: - return None - - # Model a new-generation refresh registering after the cache-state - # increment but before the invalidator reaches the coordinator lock. - server.lock = InjectCurrentFanoutLock() # type: ignore[assignment] + # Model a new-generation refresh already registered when the payout + # invalidation callback reaches the R1 owner. + server._ensure_tip_refresh_service().seed_active_refresh_for_test( + CurrentToken(), # type: ignore[arg-type] + Cancellation(), # type: ignore[arg-type] + ) self.assertEqual(server._advance_payout_state_generation(), 1) self.assertEqual(cancellations, []) self.assertFalse(server.tip_refresh_is_pending()) - self.assertFalse(server._tip_refresh_retry.is_set()) - def test_payout_generation_retry_marks_tip_refresh_pending(self) -> None: + self.assertFalse(server._ensure_tip_refresh_service().snapshot().retry_requested) + def test_payout_generation_marks_pending_without_duplicate_retry(self) -> None: server, _rpc = coordinator() server._ensure_tip_refresh_state() @@ -528,14 +524,18 @@ def test_payout_generation_retry_marks_tip_refresh_pending(self) -> None: self.assertEqual(server._advance_payout_state_generation(), 1) self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertFalse(server._ensure_tip_refresh_service().snapshot().retry_requested) + scheduler = server._ensure_tip_refresh_service().scheduler_snapshot() + self.assertIsNone(scheduler.active) + self.assertIsNone(scheduler.pending) self.assertEqual(server.poll_qbit_tip_template_once(), 0) self.assertFalse(server.tip_refresh_is_pending()) def test_payout_only_advance_bounds_publish_supersession(self) -> None: server, _rpc = coordinator() - server.payout_reconcile_supersession_retries = 2 - real_publish = server._publish_payout_state_candidate + service = server._payout_state_service + service.set_reconcile_retries_for_test(2) + real_publish = service.publish_candidate publish_attempts = 0 def supersede_before_publish(candidate: object) -> int | None: @@ -547,7 +547,7 @@ def supersede_before_publish(candidate: object) -> int | None: ) return real_publish(candidate) # type: ignore[arg-type] - server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] + service.publish_candidate = supersede_before_publish # type: ignore[method-assign] with self.assertRaisesRegex( TemplateRefreshBlocked, @@ -556,9 +556,9 @@ def supersede_before_publish(candidate: object) -> int | None: server._advance_payout_state_generation() self.assertEqual(publish_attempts, 3) - self.assertEqual(server._payout_state_generation, 0) - self.assertTrue(server._payout_state_publication_blocked) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertEqual(server._payout_state_service.snapshot().generation, 0) + self.assertTrue(server._payout_state_service.snapshot().publication_blocked) + self.assertTrue(server._payout_state_service.delivery_gate._delivery_blocked) self.assertTrue(server.tip_refresh_is_pending()) def test_payout_publication_fence_is_not_a_job_build_failure(self) -> None: server, _rpc = coordinator() @@ -566,7 +566,7 @@ def test_payout_publication_fence_is_not_a_job_build_failure(self) -> None: state = client(1) state.send = lambda _payload: None # type: ignore[method-assign] server.clients = {state} - server._pool_ready_latched = True + server._ensure_job_bundle_service().set_ready_for_test(True) server._reserve_payout_state_source("payout_only") server._block_payout_state_publication() @@ -574,9 +574,9 @@ def test_payout_publication_fence_is_not_a_job_build_failure(self) -> None: with self.assertRaisesRegex(TemplateRefreshBlocked, "pending publication"): server.poll_qbit_tip_template_once() - self.assertEqual(server.job_build_failure_count, 0) - self.assertEqual(server.tip_refresh_client_counts["failed"], 0) - self.assertEqual(server.tip_refresh_client_counts["skipped"], 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 0) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["client_counts"]["failed"], 0) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["client_counts"]["skipped"], 1) def test_successful_poll_clears_payout_pending_created_during_reconcile(self) -> None: server, _rpc = coordinator() server._ensure_tip_refresh_state() @@ -611,7 +611,7 @@ def poll() -> None: self.assertFalse(poll_thread.is_alive()) self.assertEqual(errors, []) self.assertEqual(results, [0]) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service.snapshot().generation, 1) self.assertIsNotNone(server.last_successful_template_refresh_monotonic) self.assertFalse(server.tip_refresh_is_pending()) def test_failed_poll_preserves_pending_signal_until_successful_retry(self) -> None: @@ -631,12 +631,12 @@ def fail_reconciliation(_tip_hash: str) -> bool: ): server.poll_qbit_tip_template_once() - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) self.assertTrue(server.tip_refresh_is_pending()) - self.assertEqual(server._tip_refresh_pending_token, pending_token) + self.assertEqual(server._ensure_tip_refresh_service().snapshot().pending_token, pending_token) # Model blockpoll claiming the immediate wake and completing the retry. - server._tip_refresh_retry.clear() + server._ensure_tip_refresh_service().clear_retry_for_test() server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] self.assertEqual(server.poll_qbit_tip_template_once(), 0) @@ -669,9 +669,12 @@ def test_completed_refresh_cannot_clear_newer_payout_pending(self) -> None: ) with server.lock: server.tip_template_snapshot = snapshot - completed_generation = server._payout_state_generation - self.assertEqual(server._advance_payout_state_generation(), 1) - newer_token = server._tip_refresh_pending_token + completed_generation = server._payout_state_service.snapshot().generation + tip_refresh = server._ensure_tip_refresh_service() + with tip_refresh.suppress_trigger_callbacks_for_test(): + self.assertEqual(server._advance_payout_state_generation(), 1) + server._mark_tip_refresh_pending(1) + newer_token = server._ensure_tip_refresh_service().snapshot().pending_token self.assertFalse( server._clear_tip_refresh_pending_for_completed_refresh( @@ -680,7 +683,7 @@ def test_completed_refresh_cannot_clear_newer_payout_pending(self) -> None: completed_generation, ) ) - self.assertEqual(server._tip_refresh_pending_token, newer_token) + self.assertEqual(server._ensure_tip_refresh_service().snapshot().pending_token, newer_token) self.assertTrue(server.tip_refresh_is_pending()) @staticmethod def _capture_error( diff --git a/tests/test_prism_progress_health.py b/tests/test_prism_progress_health.py index d3ef9b2..ba4be7f 100644 --- a/tests/test_prism_progress_health.py +++ b/tests/test_prism_progress_health.py @@ -4,10 +4,18 @@ from __future__ import annotations import unittest +from concurrent.futures import CancelledError from types import SimpleNamespace from unittest.mock import patch from lab.prism.prism_coordinator import QbitTipTemplateSnapshot +from lab.prism.progress_health import ( + DeliveryProof, + EligibilitySnapshot, + ProgressHealthConfig, + ProgressHealthService, + WorkGeneration, +) from tests.prism_coordinator_test_support import client, coordinator @@ -22,6 +30,77 @@ def advance(self, seconds: float) -> None: self.now += seconds +def work( + *, generation: int, fingerprint: str, payout_generation: int = 0 +) -> WorkGeneration: + return WorkGeneration(generation, fingerprint, payout_generation) + + +def progress_service( + *, bundle_build_deadline_seconds: float = 60.0 +) -> tuple[ProgressHealthService, FakeMonotonicClock]: + clock = FakeMonotonicClock() + return ( + ProgressHealthService( + ProgressHealthConfig( + pending_refresh_deadline_seconds=15.0, + tip_poll_deadline_seconds=15.0, + bundle_build_deadline_seconds=bundle_build_deadline_seconds, + ), + started_monotonic=clock.now, + monotonic=clock, + ), + clock, + ) + + +def proof( + connection_id: int, + delivered_work: WorkGeneration, + delivered_monotonic: float, + *, + collection_only: bool = False, +) -> DeliveryProof: + return DeliveryProof( + connection_id=connection_id, + delivered_work=delivered_work, + collection_only=collection_only, + delivered_monotonic=delivered_monotonic, + ) + + +def eligibility( + *connection_ids: int, + proofs: tuple[DeliveryProof, ...] = (), + ready_mode_required: bool = False, +) -> EligibilitySnapshot: + return EligibilitySnapshot( + eligible_connection_ids=connection_ids, + delivery_proofs=proofs, + ready_mode_required=ready_mode_required, + ) + + +def service_publish( + service: ProgressHealthService, + current_work: WorkGeneration, +) -> None: + service.observe_tip(current_work) + service.publish_work(current_work) + + +def service_health( + service: ProgressHealthService, + clients: EligibilitySnapshot | None = None, + *, + payout_generation: int = 0, +) -> dict[str, object]: + return service.snapshot( + clients or eligibility(), + payout_generation, + ).as_mapping() + + def snapshot( *, generation: int, fingerprint: str, tip: str = "11" * 32 ) -> QbitTipTemplateSnapshot: @@ -51,30 +130,18 @@ def context_for( def progress_coordinator() -> tuple[object, FakeMonotonicClock]: server, _ = coordinator() clock = FakeMonotonicClock() - server._progress_monotonic = clock server.started_monotonic = clock.now server.health_pending_refresh_max_age_seconds = 15.0 server.health_tip_poll_max_age_seconds = 15.0 - with server._progress_health_lock: - server._progress_current_template_generation = 0 - server._progress_current_template_fingerprint = None - server._progress_current_payout_generation = 0 - server._progress_published_template_generation = 0 - server._progress_published_template_fingerprint = None - server._progress_published_payout_generation = 0 - server._progress_has_published_work = False - server._progress_last_tip_poll_monotonic = None - server._progress_last_delivery_template_generation = 0 - server._progress_last_delivery_template_fingerprint = None - server._progress_last_delivery_payout_generation = 0 - server._progress_last_delivery_monotonic = None - server._progress_pending_since_monotonic = clock.now - server._progress_publication_divergence_since_monotonic = clock.now - server._progress_refresh_signal_pending = False - server._progress_active_refresh_count = 0 - server._progress_last_refresh_activity_monotonic = None - server._progress_bundle_build_counter = 0 - server._progress_bundle_builds.clear() + server.progress_health_service = ProgressHealthService( + ProgressHealthConfig( + pending_refresh_deadline_seconds=15.0, + tip_poll_deadline_seconds=15.0, + bundle_build_deadline_seconds=60.0, + ), + started_monotonic=clock.now, + monotonic=clock, + ) server._health_snapshot = None server._health_snapshot_monotonic = None server._health_refresh_loop_running = False @@ -91,168 +158,13 @@ def publish( class ProgressHealthTests(unittest.TestCase): - def test_publication_progress_uses_template_failure_budget(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - current = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, current) - replacement = snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - server._record_progress_tip_poll(replacement) - - clock.advance(9.999) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(0.001) - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - - server._record_progress_publication(replacement, 0) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - - def test_publication_watchdog_does_not_inherit_client_delivery_age(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - current = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, current) - miner = client(1) - server.clients.add(miner) - server._progress_reconcile_pending(now=clock.now) - - clock.advance(20.0) - self.assertEqual(server._progress_pending_since_monotonic, 100.0) - self.assertIsNone( - server._progress_publication_divergence_since_monotonic - ) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - - replacement = snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - with server.lock: - server.latest_detected_tip = (replacement.bestblockhash, 1) - server._progress_note_refresh_pending(clock.now) - - # A delayed delivery of the still-published tip can clear the broader - # client health condition, but it must not clear or age the newer - # publication-divergence deadline. - server._record_progress_delivery( - miner, - context_for(current, 0), - clock.now, - ) - self.assertIsNone(server._progress_pending_since_monotonic) - self.assertEqual( - server._progress_publication_divergence_since_monotonic, - clock.now, - ) - - server._record_progress_tip_poll(replacement, clock.now) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(9.999) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(0.001) - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - - def test_publication_divergence_survives_old_tip_delivery(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - current = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, current) - miner = client(1) - server.clients.add(miner) - replacement = snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - with server.lock: - server.latest_detected_tip = (replacement.bestblockhash, 1) - server._progress_note_refresh_pending(clock.now) - - clock.advance(6.0) - server._record_progress_delivery( - miner, - context_for(current, 0), - clock.now, - ) - - self.assertEqual( - server._progress_publication_divergence_since_monotonic, - 100.0, - ) - clock.advance(3.999) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(0.001) - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - - def test_publication_divergence_churn_does_not_renew_deadline(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - latest = None - first_replacement = None - - for generation, marker in ((2, "bb"), (3, "cc"), (4, "dd")): - latest = snapshot( - generation=generation, - fingerprint=marker * 32, - tip=marker * 32, - ) - if first_replacement is None: - first_replacement = latest - with server.lock: - server.latest_detected_tip = (latest.bestblockhash, generation) - server._progress_note_refresh_pending(clock.now) - server._record_progress_tip_poll(latest, clock.now) - self.assertEqual( - server._progress_publication_divergence_since_monotonic, - 100.0, - ) - clock.advance(3.0) - - self.assertIsNotNone(latest) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - clock.advance(1.0) - self.assertTrue(server.publication_progress_failure_expired(clock.now)) - - assert first_replacement is not None - server._record_progress_publication(first_replacement, 0) - self.assertEqual( - server._progress_publication_divergence_since_monotonic, - 100.0, - ) - - assert latest is not None - server._record_progress_publication(latest, 0) - self.assertIsNone( - server._progress_publication_divergence_since_monotonic - ) - self.assertFalse(server.publication_progress_failure_expired(clock.now)) - def test_publication_watchdog_fires_with_heartbeat_watchdog_disabled(self) -> None: - server, clock = progress_coordinator() - server.template_refresh_failure_exit_seconds = 10.0 + server, _clock = progress_coordinator() server.watchdog_enabled = False server.watchdog_interval_seconds = 0.001 - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - server._record_progress_tip_poll( - snapshot( - generation=2, - fingerprint="bb" * 32, - tip="22" * 32, - ) - ) - clock.advance(10.0) + server.publication_progress_failure_expired = lambda _now: True # type: ignore[method-assign] with ( - patch( - "lab.prism.prism_coordinator.time.monotonic", - return_value=clock.now, - ), patch( "lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1), @@ -265,35 +177,43 @@ def test_publication_watchdog_fires_with_heartbeat_watchdog_disabled(self) -> No exit_process.assert_called_once_with(1) def test_unchanged_tip_for_hours_with_valid_work_stays_healthy(self) -> None: - server, clock = progress_coordinator() - original = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, original) - miner = client(1) - delivered = context_for(original, 0) - miner.active_job = delivered - server.clients.add(miner) - server._record_progress_delivery(miner, delivered, clock.now) + service, clock = progress_service() + original = work(generation=1, fingerprint="aa" * 32) + service_publish(service, original) + delivered = proof(1, original, clock.now) + service.record_delivery(delivered, ready_mode_required=False) + clients = eligibility(1, proofs=(delivered,)) clock.advance(6 * 60 * 60) - same_work = snapshot(generation=2, fingerprint=original.template_fingerprint) - server._record_progress_tip_poll(same_work) - health = server.progress_health_snapshot() + same_work = work(generation=2, fingerprint=original.template_fingerprint or "") + service.observe_tip(same_work) + health = service_health(service, clients) self.assertTrue(health["ok"]) self.assertEqual(health["published_template_generation"], 2) self.assertGreater(health["last_valid_delivery_age_seconds"], 21_000) def test_repeated_successful_same_tip_polls_stay_healthy(self) -> None: - server, clock = progress_coordinator() + service, clock = progress_service() fingerprint = "aa" * 32 - publish(server, snapshot(generation=1, fingerprint=fingerprint)) + service_publish(service, work(generation=1, fingerprint=fingerprint)) for generation in range(2, 20): clock.advance(10) - server._record_progress_tip_poll( - snapshot(generation=generation, fingerprint=fingerprint) + service.observe_tip( + work(generation=generation, fingerprint=fingerprint) ) - self.assertTrue(server.progress_health_snapshot()["ok"]) + self.assertTrue(service_health(service)["ok"]) + + def test_publication_resolves_pending_before_a_later_signal(self) -> None: + server, clock = progress_coordinator() + publish(server, snapshot(generation=1, fingerprint="aa" * 32)) + clock.advance(60) + + server._progress_note_refresh_pending() + health = server.progress_health_snapshot() + + self.assertEqual(health["pending_refresh_age_seconds"], 0.0) def test_new_tip_without_publication_exceeds_deadline_and_returns_503(self) -> None: server, clock = progress_coordinator() @@ -317,18 +237,21 @@ def test_new_tip_without_publication_exceeds_deadline_and_returns_503(self) -> N def test_payout_change_without_replacement_delivery_returns_503(self) -> None: server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) + current_work = snapshot(generation=1, fingerprint="aa" * 32) + publish(server, current_work) miner = client(1) - old_delivery = context_for(work, 0) + old_delivery = context_for(current_work, 0) miner.active_job = old_delivery server.clients.add(miner) server._record_progress_delivery(miner, old_delivery, clock.now) server._record_progress_payout_generation(1, clock.now) - server._record_progress_publication(work, 1) + server._record_progress_publication(current_work, 1) clock.advance(16) - same_work = snapshot(generation=2, fingerprint=work.template_fingerprint) + same_work = snapshot( + generation=2, + fingerprint=current_work.template_fingerprint, + ) server._record_progress_tip_poll(same_work) status, health = server.cached_health_payload() @@ -340,20 +263,23 @@ def test_payout_change_without_replacement_delivery_returns_503(self) -> None: def test_current_generation_delivery_clears_failure_immediately(self) -> None: server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) + current_work = snapshot(generation=1, fingerprint="aa" * 32) + publish(server, current_work) miner = client(1) - miner.active_job = context_for(work, 0) + miner.active_job = context_for(current_work, 0) server.clients.add(miner) server._record_progress_payout_generation(1, clock.now) - server._record_progress_publication(work, 1) + server._record_progress_publication(current_work, 1) clock.advance(16) server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) + snapshot( + generation=2, + fingerprint=current_work.template_fingerprint, + ) ) self.assertFalse(server.progress_health_snapshot()["ok"]) - current_delivery = context_for(work, 1) + current_delivery = context_for(current_work, 1) miner.active_job = current_delivery server._record_progress_delivery(miner, current_delivery, clock.now) @@ -363,58 +289,59 @@ def test_current_generation_delivery_clears_failure_immediately(self) -> None: self.assertFalse(health["pending_refresh"]) def test_blocked_bundle_build_becomes_unhealthy(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - server.bundle_build_timeout_seconds = 60.0 - token = server._progress_bundle_build_started() + service, clock = progress_service(bundle_build_deadline_seconds=60.0) + original = work(generation=1, fingerprint="aa" * 32) + service_publish(service, original) + token = service.start_bundle_build() clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) + service.observe_tip( + work(generation=2, fingerprint=original.template_fingerprint or "") ) - within_build_timeout = server.progress_health_snapshot() + within_build_timeout = service_health(service) self.assertTrue(within_build_timeout["ok"]) self.assertNotIn("bundle_build_stuck", within_build_timeout["reasons"]) clock.advance(45) - server._record_progress_tip_poll( - snapshot(generation=3, fingerprint=work.template_fingerprint) + service.observe_tip( + work(generation=3, fingerprint=original.template_fingerprint or "") ) - health = server.progress_health_snapshot() + health = service_health(service) self.assertFalse(health["ok"]) self.assertIn("bundle_build_stuck", health["reasons"]) self.assertEqual(health["bundle_build_oldest_age_seconds"], 61.0) - server._progress_bundle_build_finished(token) + token.finish() def test_no_eligible_miners_need_no_socket_delivery_after_publication(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - server._record_progress_tip_poll(work) - server._record_progress_payout_generation(1, clock.now) - server._record_progress_publication(work, 1) + service, clock = progress_service() + current = work( + generation=1, + fingerprint="aa" * 32, + payout_generation=1, + ) + service.observe_tip(work(generation=1, fingerprint="aa" * 32)) + service.observe_payout_generation(1, clock.now) + service.publish_work(current) - health = server.progress_health_snapshot() + health = service_health(service, payout_generation=1) self.assertTrue(health["ok"]) self.assertEqual(health["eligible_client_count"], 0) self.assertIsNone(health["last_valid_delivery_age_seconds"]) def test_eligible_miners_require_current_generation_delivery(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - miner = client(1) - server.clients.add(miner) - server.progress_health_snapshot() + service, clock = progress_service() + original = work(generation=1, fingerprint="aa" * 32) + service_publish(service, original) + service_health(service, eligibility(1)) clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) + service.observe_tip( + work(generation=2, fingerprint=original.template_fingerprint or "") ) - health = server.progress_health_snapshot() + health = service_health(service, eligibility(1)) self.assertFalse(health["ok"]) self.assertIn("current_generation_not_delivered", health["reasons"]) @@ -422,30 +349,32 @@ def test_eligible_miners_require_current_generation_delivery(self) -> None: self.assertEqual(health["eligible_clients_requiring_refresh"], 1) def test_partial_fanout_stays_pending_until_every_client_is_current(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - delivered = client(1) - missing = client(2) - current_context = context_for(work, 0) - delivered.active_job = current_context - server.clients.update((delivered, missing)) - server._record_progress_delivery(delivered, current_context, clock.now) + service, clock = progress_service() + original = work(generation=1, fingerprint="aa" * 32) + service_publish(service, original) + delivered = proof(1, original, clock.now) + service.record_delivery(delivered, ready_mode_required=False) clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=2, fingerprint=work.template_fingerprint) + service.observe_tip( + work(generation=2, fingerprint=original.template_fingerprint or "") ) - health = server.progress_health_snapshot() + clients = eligibility(1, 2, proofs=(delivered,)) + health = service_health(service, clients) self.assertFalse(health["ok"]) self.assertTrue(health["pending_refresh"]) self.assertEqual(health["eligible_clients_requiring_refresh"], 1) self.assertIn("current_generation_not_delivered", health["reasons"]) - missing.active_job = current_context - server._record_progress_delivery(missing, current_context, clock.now) - self.assertTrue(server.progress_health_snapshot()["ok"]) + missing = proof(2, original, clock.now) + service.record_delivery(missing, ready_mode_required=False) + self.assertTrue( + service_health( + service, + eligibility(1, 2, proofs=(delivered, missing)), + )["ok"] + ) def test_registered_job_is_not_delivery_proof_before_socket_send(self) -> None: server, clock = progress_coordinator() @@ -550,35 +479,33 @@ def test_stale_cached_ok_cannot_mask_a_progress_failure(self) -> None: self.assertLess(health["snapshot_age_seconds"], server.health_refresh_seconds) def test_wall_clock_changes_do_not_affect_health_decisions(self) -> None: - server, clock = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) + service, clock = progress_service() + service_publish(service, work(generation=1, fingerprint="aa" * 32)) with patch("lab.prism.prism_coordinator.time.time", return_value=-10**12): - self.assertTrue(server.progress_health_snapshot()["ok"]) + self.assertTrue(service_health(service)["ok"]) with patch("lab.prism.prism_coordinator.time.time", return_value=10**12): - self.assertTrue(server.progress_health_snapshot()["ok"]) + self.assertTrue(service_health(service)["ok"]) self.assertEqual(clock.now, 100.0) def test_tip_poll_freshness_has_an_independent_deadline(self) -> None: - server, clock = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) + service, clock = progress_service() + service_publish(service, work(generation=1, fingerprint="aa" * 32)) clock.advance(16) - health = server.progress_health_snapshot() + health = service_health(service) self.assertFalse(health["ok"]) self.assertEqual(health["reasons"], ["tip_poll_stale"]) def test_older_poll_cannot_renew_current_generation_freshness(self) -> None: - server, clock = progress_coordinator() - current = snapshot(generation=2, fingerprint="bb" * 32, tip="22" * 32) - publish(server, current) + service, clock = progress_service() + current = work(generation=2, fingerprint="bb" * 32) + service_publish(service, current) clock.advance(16) - server._record_progress_tip_poll( - snapshot(generation=1, fingerprint="aa" * 32) - ) - health = server.progress_health_snapshot() + service.observe_tip(work(generation=1, fingerprint="aa" * 32)) + health = service_health(service) self.assertFalse(health["ok"]) self.assertEqual(health["current_template_generation"], 2) @@ -586,34 +513,120 @@ def test_older_poll_cannot_renew_current_generation_freshness(self) -> None: self.assertIn("tip_poll_stale", health["reasons"]) def test_progressing_refresh_does_not_report_tip_poll_stale(self) -> None: - server, clock = progress_coordinator() - work = snapshot(generation=1, fingerprint="aa" * 32) - publish(server, work) - server._progress_refresh_started() + service, clock = progress_service() + current = work(generation=1, fingerprint="aa" * 32) + service_publish(service, current) + refresh = service.start_refresh() clock.advance(10) - server._record_progress_publication(work, 0) + service.publish_work(current) clock.advance(10) - health = server.progress_health_snapshot() + health = service_health(service) self.assertTrue(health["ok"]) self.assertTrue(health["tip_refresh_in_progress"]) self.assertEqual(health["tip_poll_age_seconds"], 20.0) self.assertEqual(health["tip_refresh_progress_age_seconds"], 10.0) self.assertNotIn("tip_poll_stale", health["reasons"]) - server._progress_refresh_finished() + refresh.finish() def test_stalled_active_refresh_still_reports_tip_poll_stale(self) -> None: - server, clock = progress_coordinator() - publish(server, snapshot(generation=1, fingerprint="aa" * 32)) - server._progress_refresh_started() + service, clock = progress_service() + service_publish(service, work(generation=1, fingerprint="aa" * 32)) + refresh = service.start_refresh() clock.advance(16) - health = server.progress_health_snapshot() + health = service_health(service) self.assertFalse(health["ok"]) self.assertIn("tip_poll_stale", health["reasons"]) - server._progress_refresh_finished() + refresh.finish() + + def test_refresh_token_finishes_on_exception(self) -> None: + service, _ = progress_service() + + with self.assertRaisesRegex(RuntimeError, "boom"): + with service.start_refresh(): + raise RuntimeError("boom") + + self.assertFalse(service_health(service)["tip_refresh_in_progress"]) + + def test_bundle_token_finishes_on_cancellation(self) -> None: + service, clock = progress_service(bundle_build_deadline_seconds=1.0) + + with self.assertRaises(CancelledError): + with service.start_bundle_build(): + raise CancelledError() + clock.advance(2) + + self.assertEqual( + service_health(service)["bundle_build_oldest_age_seconds"], + 0.0, + ) + + def test_token_finish_is_idempotent(self) -> None: + service, _ = progress_service() + refresh = service.start_refresh() + build = service.start_bundle_build() + + refresh.finish() + refresh.finish() + build.finish() + build.finish() + + health = service_health(service) + self.assertFalse(health["tip_refresh_in_progress"]) + self.assertEqual(health["bundle_build_oldest_age_seconds"], 0.0) + + def test_overlapping_refresh_tokens_finish_independently(self) -> None: + service, clock = progress_service() + first = service.start_refresh() + clock.advance(5) + second = service.start_refresh() + + second.finish() + self.assertTrue(service_health(service)["tip_refresh_in_progress"]) + first.note_activity() + first.finish() + + self.assertFalse(service_health(service)["tip_refresh_in_progress"]) + + def test_oldest_overlapping_bundle_controls_health(self) -> None: + service, clock = progress_service(bundle_build_deadline_seconds=60.0) + current = work(generation=1, fingerprint="aa" * 32) + service_publish(service, current) + first = service.start_bundle_build() + clock.advance(10) + second = service.start_bundle_build() + clock.advance(51) + service.observe_tip(work(generation=2, fingerprint="aa" * 32)) + + health = service_health(service) + self.assertIn("bundle_build_stuck", health["reasons"]) + self.assertEqual(health["bundle_build_oldest_age_seconds"], 61.0) + + first.finish() + health = service_health(service) + self.assertNotIn("bundle_build_stuck", health["reasons"]) + self.assertEqual(health["bundle_build_oldest_age_seconds"], 51.0) + second.finish() + + def test_multiple_failures_keep_the_fixed_reason_order(self) -> None: + service, clock = progress_service(bundle_build_deadline_seconds=60.0) + build = service.start_bundle_build() + clock.advance(61) + + health = service_health(service) + + self.assertEqual( + health["reasons"], + [ + "tip_poll_stale", + "bundle_build_stuck", + "current_generation_not_published", + ], + ) + build.finish() def test_progress_health_cannot_mask_base_mining_failure(self) -> None: server, _ = progress_coordinator() diff --git a/tests/test_prism_reconnect_backpressure.py b/tests/test_prism_reconnect_backpressure.py index 0562312..893c591 100644 --- a/tests/test_prism_reconnect_backpressure.py +++ b/tests/test_prism_reconnect_backpressure.py @@ -119,6 +119,9 @@ def coordinator(*, connection_limit: int = 3, pending_limit: int = 2) -> PrismCo server.stratum_initial_job_timeout_seconds = 30.0 server.mining_health_startup_grace_seconds = 30.0 server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) server.tip_template_snapshot = None server.started_monotonic = 0.0 server.submitted_share_count = 0 @@ -884,6 +887,7 @@ def test_640_client_storm_cannot_displace_latest_tip_builder(self) -> None: any_worker_passed_priority = threading.Event() contend = threading.Event() release_workers = threading.Event() + job_bundles = server._ensure_job_bundle_service() def build_request( key: str, @@ -922,7 +926,7 @@ def contending_initial(_request: PendingInitialJob) -> bool: if workers_contending == server.initial_job_max_workers: all_workers_contending.set() preparation_token, _preparation_cancellation = ( - server._begin_routine_job_build_preparation( + job_bundles.begin_routine_preparation( request_source="initial", cancelled=_request.cancelled.is_set, ) @@ -933,9 +937,7 @@ def contending_initial(_request: PendingInitialJob) -> bool: any_worker_passed_priority.set() release_workers.wait(5) finally: - server._finish_routine_job_build_preparation( - preparation_token - ) + job_bundles.finish_routine_preparation(preparation_token) return False server._run_initial_job = contending_initial # type: ignore[method-assign] @@ -950,18 +952,18 @@ def contending_initial(_request: PendingInitialJob) -> bool: self.assertEqual(server.initial_job_executor().stats(), (124, 4)) def start_without_execution(request: object) -> SimpleNamespace: - server.job_build_scheduler_counts["starts"] += 1 - server._record_priority_admission_locked( # type: ignore[arg-type] + job_bundles._scheduler_counts["starts"] += 1 + job_bundles.record_priority_admission_locked( # type: ignore[arg-type] request, "started", ) return SimpleNamespace(request=request, future=Future()) - server._start_job_build_locked = start_without_execution # type: ignore[method-assign] - server._arm_job_build_locked = lambda _flight: None # type: ignore[method-assign] + job_bundles._start_locked = start_without_execution # type: ignore[method-assign] + job_bundles._arm_locked = lambda _flight: None # type: ignore[method-assign] admission_started = time.monotonic() priority_token, priority_requested = ( - server._begin_job_build_priority_preparation() + job_bundles.begin_priority_preparation() ) latest = build_request( "latest-tip", @@ -970,9 +972,9 @@ def start_without_execution(request: object) -> SimpleNamespace: requested_monotonic=priority_requested, ) try: - latest_promise = server._request_job_build(latest) # type: ignore[arg-type] + latest_promise = job_bundles.request_build(latest) # type: ignore[arg-type] finally: - server._finish_job_build_priority_preparation(priority_token) + job_bundles.finish_priority_preparation(priority_token) admission_elapsed = time.monotonic() - admission_started contend.set() @@ -982,24 +984,24 @@ def start_without_execution(request: object) -> SimpleNamespace: self.assertLess(admission_elapsed, 0.1) self.assertIs(latest_promise, latest.promise) self.assertFalse(latest.cancellation.is_set()) - assert server._job_build_active is not None - self.assertIs(server._job_build_active.request, latest) + assert job_bundles._active is not None + self.assertIs(job_bundles._active.request, latest) self.assertEqual( - server.job_build_priority_counts["routine_deferred"], + job_bundles._priority_counts["routine_deferred"], 4, ) self.assertEqual( - server.initial_job_prepared_work_counts["deferred"], + job_bundles._initial_prepared_work_counts["deferred"], 4, ) self.assertEqual( - server.job_build_priority_admission_seconds["count"], + job_bundles._priority_admission_seconds["count"], 1, ) finally: - with server._job_build_scheduler_lock: - server._job_build_active = None - server._job_build_priority_changed.set() + with job_bundles._scheduler_lock: + job_bundles._active = None + job_bundles._priority_changed.set() release_workers.set() server.shutdown_initial_job_executor() self.assertEqual(workers_passed_priority, 4) diff --git a/tests/test_prism_refresh_retry_pacing.py b/tests/test_prism_refresh_retry_pacing.py index 383a881..66e459c 100644 --- a/tests/test_prism_refresh_retry_pacing.py +++ b/tests/test_prism_refresh_retry_pacing.py @@ -44,23 +44,28 @@ def test_failed_pass_arms_holdoff_and_gates_the_trigger(self) -> None: server.clients = set() server.blockpoll_seconds = 30.0 server.tip_refresh_failure_holdoff_seconds = 0.3 + service = server._ensure_tip_refresh_service() + service.reconfigure_for_test( + blockpoll_seconds=30.0, + failure_holdoff_seconds=0.3, + ) self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) + self.assertEqual(service.failure_holdoff_remaining(), 0.0) _block_refreshes(server) - with patch("lab.prism.prism_coordinator.random.uniform", return_value=0.0): + with patch("lab.prism.tip_refresh.random.uniform", return_value=0.0): self._fail_one_poll(server) - deadline = server._tip_refresh_failure_holdoff_until + deadline = service._failure_holdoff_until self.assertIsNotNone(deadline) assert deadline is not None - self.assertGreater(server._tip_refresh_failure_holdoff_remaining(), 0.0) + self.assertGreater(service.failure_holdoff_remaining(), 0.0) server._schedule_tip_refresh_retry() self.assertTrue(server._wait_for_blockpoll_trigger()) # The trigger only released after the spacing window, never before. self.assertGreaterEqual(time.monotonic(), deadline) - self.assertFalse(server._tip_refresh_retry.is_set()) + self.assertFalse(service._retry_event.is_set()) def test_holdoff_wait_keeps_blockpoll_heartbeat_alive(self) -> None: server, _rpc = coordinator() @@ -68,10 +73,14 @@ def test_holdoff_wait_keeps_blockpoll_heartbeat_alive(self) -> None: server.clients = set() server.blockpoll_seconds = 30.0 server.tip_refresh_failure_holdoff_seconds = 0.4 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=30.0, + failure_holdoff_seconds=0.4, + ) self.assertEqual(server.poll_qbit_tip_template_once(), 0) _block_refreshes(server) - with patch("lab.prism.prism_coordinator.random.uniform", return_value=0.0): + with patch("lab.prism.tip_refresh.random.uniform", return_value=0.0): self._fail_one_poll(server) server._schedule_tip_refresh_retry() @@ -88,6 +97,9 @@ def test_holdoff_wait_keeps_blockpoll_heartbeat_alive(self) -> None: def test_trigger_without_prior_failure_stays_immediate(self) -> None: server, _rpc = coordinator() server.blockpoll_seconds = 30.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=30.0 + ) server._schedule_tip_refresh_retry() started = time.monotonic() @@ -100,17 +112,22 @@ def test_new_tip_observation_releases_the_holdoff(self) -> None: server.clients = set() server.blockpoll_seconds = 30.0 server.tip_refresh_failure_holdoff_seconds = 30.0 + service = server._ensure_tip_refresh_service() + service.reconfigure_for_test( + blockpoll_seconds=30.0, + failure_holdoff_seconds=30.0, + ) self.assertEqual(server.poll_qbit_tip_template_once(), 0) _block_refreshes(server) self._fail_one_poll(server) - self.assertGreater(server._tip_refresh_failure_holdoff_remaining(), 0.0) + self.assertGreater(service.failure_holdoff_remaining(), 0.0) # Blockwait's push path: record the detection, then arm the poller # trigger. The armed holdoff must not delay that wake even though the # newer tip is not yet published as share-validation authority. self.assertTrue(server.observe_tip_for_refresh(NEW_TIP)) - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) + self.assertEqual(service.failure_holdoff_remaining(), 0.0) server._schedule_tip_refresh_retry() started = time.monotonic() self.assertTrue(server._wait_for_blockpoll_trigger()) @@ -123,6 +140,14 @@ def test_mid_fetch_tip_discovery_releases_the_holdoff(self) -> None: server.blockpoll_seconds = 30.0 server.tip_refresh_failure_holdoff_seconds = 30.0 server.template_cache_seconds = 0.0 + server._ensure_job_bundle_service().template_repository.set_cache_seconds_for_test( + 0.0 + ) + service = server._ensure_tip_refresh_service() + service.reconfigure_for_test( + blockpoll_seconds=30.0, + failure_holdoff_seconds=30.0, + ) self.assertEqual(server.poll_qbit_tip_template_once(), 0) @@ -148,45 +173,51 @@ def advance_tip_after_template( # The failed pass recorded its discovery, so the newer tip's retry # is not held for the spacing window. - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) + self.assertEqual(service.failure_holdoff_remaining(), 0.0) def test_successful_pass_clears_the_holdoff(self) -> None: server, _rpc = coordinator() install_fake_bundle_builder(server) server.clients = set() server.tip_refresh_failure_holdoff_seconds = 30.0 + service = server._ensure_tip_refresh_service() + service.reconfigure_for_test(failure_holdoff_seconds=30.0) original_reconcile = server.ensure_reorg_reconciled_for_tip self.assertEqual(server.poll_qbit_tip_template_once(), 0) _block_refreshes(server) self._fail_one_poll(server) - self.assertGreater(server._tip_refresh_failure_holdoff_remaining(), 0.0) + self.assertGreater(service.failure_holdoff_remaining(), 0.0) server.ensure_reorg_reconciled_for_tip = original_reconcile # type: ignore[method-assign] self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertIsNone(server._tip_refresh_failure_holdoff_until) - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) + self.assertIsNone(service._failure_holdoff_until) + self.assertEqual(service.failure_holdoff_remaining(), 0.0) def test_zero_holdoff_restores_unspaced_retries(self) -> None: server, _rpc = coordinator() install_fake_bundle_builder(server) server.clients = set() server.tip_refresh_failure_holdoff_seconds = 0.0 + service = server._ensure_tip_refresh_service() + service.reconfigure_for_test(failure_holdoff_seconds=0.0) self.assertEqual(server.poll_qbit_tip_template_once(), 0) _block_refreshes(server) self._fail_one_poll(server) - self.assertIsNone(server._tip_refresh_failure_holdoff_until) - self.assertEqual(server._tip_refresh_failure_holdoff_remaining(), 0.0) + self.assertIsNone(service._failure_holdoff_until) + self.assertEqual(service.failure_holdoff_remaining(), 0.0) def test_holdoff_includes_bounded_jitter(self) -> None: server, _rpc = coordinator() server.tip_refresh_failure_holdoff_seconds = 1.0 + service = server._ensure_tip_refresh_service() + service.reconfigure_for_test(failure_holdoff_seconds=1.0) before = time.monotonic() - server._note_tip_refresh_attempt_failed() - deadline = server._tip_refresh_failure_holdoff_until + service.note_attempt_failed() + deadline = service._failure_holdoff_until self.assertIsNotNone(deadline) assert deadline is not None @@ -212,7 +243,8 @@ def test_same_tip_poll_within_window_skips_getblocktemplate(self) -> None: server.shutdown_tip_refresh_executor() self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(server.job_cache_hit_counts["template"], 1) + metrics = server._ensure_job_bundle_service().metrics_snapshot() + self.assertEqual(metrics["hit_counts"]["template"], 1) # The reused pass rebuilt nothing: the client's job is still the one # delivered from the originally fetched template. self.assertEqual( @@ -234,7 +266,8 @@ def test_blocked_passes_cost_one_template_per_tip(self) -> None: server.poll_qbit_tip_template_once() self.assertEqual(rpc.count("getblocktemplate"), 1) - self.assertEqual(server.job_cache_hit_counts["template"], 5) + metrics = server._ensure_job_bundle_service().metrics_snapshot() + self.assertEqual(metrics["hit_counts"]["template"], 5) def test_tip_change_bypasses_template_reuse(self) -> None: server, rpc = coordinator() @@ -257,18 +290,25 @@ def test_zero_window_disables_template_reuse(self) -> None: install_fake_bundle_builder(server) server.clients = set() server.template_cache_seconds = 0.0 + server._ensure_job_bundle_service().template_repository.set_cache_seconds_for_test( + 0.0 + ) self.assertEqual(server.poll_qbit_tip_template_once(), 0) self.assertEqual(server.poll_qbit_tip_template_once(), 0) self.assertEqual(rpc.count("getblocktemplate"), 2) - self.assertEqual(server.job_cache_hit_counts["template"], 0) + metrics = server._ensure_job_bundle_service().metrics_snapshot() + self.assertEqual(metrics["hit_counts"]["template"], 0) def test_expired_window_refetches_the_template(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.clients = set() server.template_cache_seconds = 0.05 + server._ensure_job_bundle_service().template_repository.set_cache_seconds_for_test( + 0.05 + ) self.assertEqual(server.poll_qbit_tip_template_once(), 0) time.sleep(0.06) diff --git a/tests/test_prism_representative_independence.py b/tests/test_prism_representative_independence.py index 43ef86a..b1433e5 100644 --- a/tests/test_prism_representative_independence.py +++ b/tests/test_prism_representative_independence.py @@ -12,7 +12,6 @@ CachedTemplateArtifacts, CollectionIdentityUnavailable, StratumError, - TemplateRefreshBlocked, WorkerIdentity, ) from tests.prism_coordinator_test_support import ( @@ -28,6 +27,7 @@ class RepresentativeIndependentRefreshTests(unittest.TestCase): def _run_ready_disconnect_race(self, stage: str) -> None: server, rpc = coordinator() + tip_refresh = server._ensure_tip_refresh_service() recorded = install_fake_bundle_builder(server) disconnected = client(1) survivor = client(2) @@ -43,17 +43,20 @@ def remove_target() -> None: server.clients.discard(disconnected) if stage == "before_build": - original_shared_job_bundle = server.shared_job_bundle + original_prepare_bundle = tip_refresh.prepare_bundle def disconnect_before_build( - artifacts: CachedTemplateArtifacts, - identity: WorkerIdentity | None = None, - **kwargs: object, + snapshot: object, + *, + priority_requested_monotonic: float | None = None, ) -> CachedJobBundle: remove_target() - return original_shared_job_bundle(artifacts, identity, **kwargs) + return original_prepare_bundle( # type: ignore[arg-type] + snapshot, + priority_requested_monotonic=priority_requested_monotonic, + ) - server.shared_job_bundle = disconnect_before_build # type: ignore[method-assign] + tip_refresh.prepare_bundle = disconnect_before_build # type: ignore[method-assign] elif stage == "during_build": original_build_audit_bundle = server.build_audit_bundle @@ -62,14 +65,17 @@ def disconnect_during_build(**kwargs: object) -> dict[str, object]: return original_build_audit_bundle(**kwargs) server.build_audit_bundle = disconnect_during_build # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + disconnect_during_build + ) elif stage == "before_fanout": - original_fanout = server._fanout_prepared_tip_refresh + original_fanout = tip_refresh.fanout_prepared def disconnect_before_fanout(*args: object, **kwargs: object) -> object: remove_target() return original_fanout(*args, **kwargs) - server._fanout_prepared_tip_refresh = disconnect_before_fanout # type: ignore[method-assign] + tip_refresh.fanout_prepared = disconnect_before_fanout # type: ignore[method-assign] else: # pragma: no cover - test helper guard raise AssertionError(f"unknown race stage {stage}") @@ -86,7 +92,10 @@ def disconnect_before_fanout(*args: object, **kwargs: object) -> object: [payload["method"] for payload in survivor_payloads], ["mining.set_difficulty", "mining.notify"], ) - self.assertEqual(server.tip_refresh_client_counts["disconnected"], 1) + self.assertEqual( + tip_refresh.metrics_snapshot()["client_counts"]["disconnected"], + 1, + ) self.assertIsNotNone(survivor.active_job) assert survivor.active_job is not None self.assertFalse(survivor.active_job.collection_only) @@ -102,6 +111,7 @@ def test_target_disconnect_after_ready_construction_before_fanout(self) -> None: def test_reselected_poll_start_target_keeps_expected_job_snapshot(self) -> None: server, _rpc = coordinator() + tip_refresh = server._ensure_tip_refresh_service() install_fake_bundle_builder(server) stable = client(1) reselected = client(2) @@ -129,9 +139,11 @@ def capture_fanout( delivered = time.monotonic() return len(clients), delivered, delivered, 0 - server.fetch_qbit_tip_template_snapshot = make_target_temporarily_ineligible # type: ignore[method-assign] + tip_refresh.reconfigure_ports_for_test( + fetch_snapshot=make_target_temporarily_ineligible, + ) server.ensure_reorg_reconciled_for_tip = restore_target # type: ignore[method-assign] - server._fanout_prepared_tip_refresh = capture_fanout # type: ignore[method-assign] + tip_refresh.fanout_prepared = capture_fanout # type: ignore[method-assign] try: refreshed = server.poll_qbit_tip_template_once() finally: @@ -158,16 +170,18 @@ def test_ready_bundle_builds_without_clients_or_worker_identity(self) -> None: def test_collection_ineligible_connected_target_is_skipped(self) -> None: server, _rpc = coordinator(ledger=FakeLedger(miners=["solo"])) + tip_refresh = server._ensure_tip_refresh_service() state = client(1) server.clients = {state} - original_observe_tip = server.observe_tip_first_seen + original_publish_tip = tip_refresh.publish_tip def make_target_ineligible(*args: object, **kwargs: object) -> bool: - observed = original_observe_tip(*args, **kwargs) - state.authorized = False - return observed + published = original_publish_tip(*args, **kwargs) + if kwargs.get("publish_refresh_observation"): + state.authorized = False + return published - server.observe_tip_first_seen = make_target_ineligible # type: ignore[method-assign] + tip_refresh.publish_tip = make_target_ineligible # type: ignore[method-assign] server.maybe_send_job = lambda *_args, **_kwargs: self.fail( # type: ignore[method-assign] "ineligible collection target received work" ) @@ -178,8 +192,8 @@ def make_target_ineligible(*args: object, **kwargs: object) -> bool: server.shutdown_tip_refresh_executor() self.assertEqual(refreshed, 0) - self.assertEqual(server.tip_refresh_client_counts["skipped"], 1) - self.assertEqual(server.tip_refresh_client_counts["disconnected"], 0) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["client_counts"]["skipped"], 1) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["client_counts"]["disconnected"], 0) def test_collection_identity_absence_has_a_distinct_temporary_result(self) -> None: server, _rpc = coordinator(ledger=FakeLedger(miners=["solo"])) @@ -197,15 +211,16 @@ def test_all_collection_identities_disappear_then_authorization_reuses_artifacts server, rpc = coordinator(ledger=FakeLedger(miners=["solo"])) recorded = install_fake_bundle_builder(server) server.template_cache_seconds = 0.0 + server._ensure_job_bundle_service().template_repository.set_cache_seconds_for_test(0.0) original = client(1) original.send = lambda _payload: None # type: ignore[method-assign] original.close = lambda: None # type: ignore[method-assign] server.clients = {original} self.assertEqual(server.poll_qbit_tip_template_once(), 1) - self.assertIsNone(server._retained_collection_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().retained_collection_refresh_snapshot()) server.disconnect_client(original) - retained = server._retained_collection_refresh + retained = server._ensure_tip_refresh_service().retained_collection_refresh_snapshot() self.assertIsNotNone(retained) self.assertEqual(rpc.count("getblocktemplate"), 1) self.assertEqual(recorded["calls"], 1) @@ -218,9 +233,10 @@ def test_all_collection_identities_disappear_then_authorization_reuses_artifacts authorized.authorized = True server._note_collection_identity_available(authorized) - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertTrue(server.maybe_send_job(authorized, clean_jobs=True)) + tip_refresh = server._ensure_tip_refresh_service() + self.assertTrue(tip_refresh.wait_for_scheduler_idle_for_test()) + self.assertFalse(server.tip_refresh_is_pending()) + self.assertFalse(tip_refresh.snapshot().retry_requested) self.assertEqual(rpc.count("getblocktemplate"), 1) self.assertEqual(recorded["calls"], 1) self.assertEqual( @@ -234,16 +250,13 @@ def test_all_collection_identities_disappear_then_authorization_reuses_artifacts authorized.active_job.template, retained.snapshot.template_artifacts.template, ) - self.assertIsNone(server._retained_collection_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().retained_collection_refresh_snapshot()) - pending_token = server._tip_refresh_pending_token - if pending_token is not None: - server._clear_tip_refresh_pending(pending_token) - server._tip_refresh_retry.clear() server._note_collection_identity_available(authorized) + self.assertTrue(tip_refresh.wait_for_scheduler_idle_for_test()) self.assertFalse(server.tip_refresh_is_pending()) - self.assertFalse(server._tip_refresh_retry.is_set()) + self.assertFalse(tip_refresh.snapshot().retry_requested) def test_authorization_during_same_tip_publication_keeps_retained_wake( self, @@ -251,29 +264,31 @@ def test_authorization_during_same_tip_publication_keeps_retained_wake( server, rpc = coordinator(ledger=FakeLedger(miners=["solo"])) recorded = install_fake_bundle_builder(server) server.template_cache_seconds = 0.0 + server._ensure_job_bundle_service().template_repository.set_cache_seconds_for_test(0.0) server.clients = set() self.assertEqual(server.poll_qbit_tip_template_once(), 0) - retained = server._retained_collection_refresh + retained = server._ensure_tip_refresh_service().retained_collection_refresh_snapshot() self.assertIsNotNone(retained) self.assertEqual(rpc.count("getblocktemplate"), 1) - server._tip_refresh_retry.clear() + server._ensure_tip_refresh_service().clear_retry_for_test() publication_window = threading.Event() release_publication = threading.Event() - original_observe_tip = server.observe_tip_first_seen + tip_refresh = server._ensure_tip_refresh_service() + original_publish_tip = tip_refresh.publish_tip def pause_after_same_tip_observation( *args: object, **kwargs: object, ) -> bool: - observed = original_observe_tip(*args, **kwargs) + published = original_publish_tip(*args, **kwargs) if kwargs.get("publish_refresh_observation"): publication_window.set() self.assertTrue(release_publication.wait(5.0)) - return observed + return published - server.observe_tip_first_seen = pause_after_same_tip_observation # type: ignore[method-assign] + tip_refresh.publish_tip = pause_after_same_tip_observation # type: ignore[method-assign] poll_results: list[int] = [] poll_errors: list[BaseException] = [] @@ -296,42 +311,37 @@ def poll_same_tip() -> None: server._note_collection_identity_available(authorized) self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertTrue(server.maybe_send_job(authorized, clean_jobs=True)) + self.assertFalse(server._ensure_tip_refresh_service().snapshot().retry_requested) self.assertEqual(rpc.count("getblocktemplate"), 2) - self.assertEqual(recorded["calls"], 1) - self.assertIsNotNone(authorized.active_job) - with server.lock: - published_snapshot = server.tip_template_snapshot - assert ( - authorized.active_job is not None - and published_snapshot is not None - and published_snapshot.template_artifacts is not None - ) - self.assertIs( - authorized.active_job.template, - published_snapshot.template_artifacts.template, - ) finally: release_publication.set() poll_thread.join(5.0) self.assertFalse(poll_thread.is_alive()) - self.assertEqual(poll_results, []) - self.assertEqual(len(poll_errors), 1) - self.assertIsInstance(poll_errors[0], TemplateRefreshBlocked) - # Authorization minted a newer wake token while this poll owned - # the previous state. The first poll must not clear that work. - self.assertTrue(server.tip_refresh_is_pending()) - self.assertTrue(server._tip_refresh_retry.is_set()) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) + self.assertTrue(tip_refresh.wait_for_scheduler_idle_for_test()) + self.assertEqual(poll_results, [0]) + self.assertEqual(poll_errors, []) self.assertFalse(server.tip_refresh_is_pending()) - self.assertIsNone(server._retained_collection_refresh) + self.assertFalse(tip_refresh.snapshot().retry_requested) + self.assertIsNone(server._ensure_tip_refresh_service().retained_collection_refresh_snapshot()) + self.assertEqual(rpc.count("getblocktemplate"), 2) + self.assertEqual(recorded["calls"], 1) self.assertEqual( [payload["method"] for payload in sent], ["mining.set_difficulty", "mining.notify"], ) self.assertIsNotNone(authorized.active_job) + with server.lock: + published_snapshot = server.tip_template_snapshot + assert ( + authorized.active_job is not None + and published_snapshot is not None + and published_snapshot.template_artifacts is not None + ) + self.assertIs( + authorized.active_job.template, + published_snapshot.template_artifacts.template, + ) finally: release_publication.set() server.shutdown_tip_refresh_executor() @@ -342,11 +352,11 @@ def test_ready_latch_discards_retained_collection_marker(self) -> None: server.clients = set() self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertIsNotNone(server._retained_collection_refresh) + self.assertIsNotNone(server._ensure_tip_refresh_service().retained_collection_refresh_snapshot()) ledger.miners = ["miner-a", "miner-b", "miner-c"] self.assertTrue(server.pool_readiness_latched()) - self.assertIsNone(server._retained_collection_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().retained_collection_refresh_snapshot()) def test_collection_reauthorization_reselects_identity_without_template_refetch( self, @@ -424,16 +434,16 @@ def test_new_tip_supersedes_retained_collection_preparation(self) -> None: server.clients = set() self.assertEqual(server.poll_qbit_tip_template_once(), 0) - old_retained = server._retained_collection_refresh + old_retained = server._ensure_tip_refresh_service().retained_collection_refresh_snapshot() self.assertIsNotNone(old_retained) rpc.tip = new_tip rpc.template = base_template(height=11, prevhash=new_tip) self.assertTrue(server.observe_tip_first_seen(new_tip)) - self.assertIsNone(server._retained_collection_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().retained_collection_refresh_snapshot()) self.assertEqual(server.poll_qbit_tip_template_once(), 0) - current = server._retained_collection_refresh + current = server._ensure_tip_refresh_service().retained_collection_refresh_snapshot() self.assertIsNotNone(current) assert current is not None and old_retained is not None self.assertIsNot(current.snapshot, old_retained.snapshot) diff --git a/tests/test_prism_retained_jobs.py b/tests/test_prism_retained_jobs.py index b1a50f7..32bc6c8 100644 --- a/tests/test_prism_retained_jobs.py +++ b/tests/test_prism_retained_jobs.py @@ -480,7 +480,13 @@ def overtake_parent_lookup(tip_hash: str) -> str: server.current_tip_parent = (new_tip, new_parent) return old_parent - server._fetch_tip_parent_hash = overtake_parent_lookup # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + rpc_call=lambda method, params: { + "previousblockhash": overtake_parent_lookup(str(params[0])) + } + if method == "getblock" and params + else server.rpc.call(method, params) + ) self.assertEqual(server.current_tip_parent_hash(old_tip), old_parent) self.assertEqual(server.current_tip_parent, (new_tip, new_parent)) diff --git a/tests/test_prism_share_ledger.py b/tests/test_prism_share_ledger.py index e12c477..40dc576 100644 --- a/tests/test_prism_share_ledger.py +++ b/tests/test_prism_share_ledger.py @@ -29,6 +29,8 @@ AUDIT_BUNDLE_V2_SCHEMA, PendingShare, PsqlShareLedger, + ShareReplayConflict, + ShareReplayResult, AUDIT_WINDOW_COMPLETENESS_PROOF_SCHEMA, SingleWriterShareLedger, _NativePostgresClient, @@ -214,6 +216,20 @@ def release_reads(self) -> None: class PrismShareLedgerTests(unittest.TestCase): + def test_memory_recovery_append_distinguishes_insert_exact_and_conflict(self) -> None: + ledger = SingleWriterShareLedger() + share = pending_share(1) + + inserted = ledger.append_recovered_share(share) + exact = ledger.append_recovered_share(share) + + self.assertIsInstance(inserted, ShareReplayResult) + self.assertEqual(inserted.disposition, "inserted") + self.assertEqual(exact.disposition, "exact_existing") + with self.assertRaises(ShareReplayConflict): + ledger.append_recovered_share( + PendingShare(**{**share.__dict__, "ntime": share.ntime + 1}) + ) def test_single_writer_assigns_contiguous_sequence_numbers(self) -> None: ledger = SingleWriterShareLedger() @@ -870,6 +886,48 @@ def test_postgres_batch_sql_fences_share_and_candidate_in_one_statement(self) -> self.assertIn("duplicate share_id payload mismatch", query) self.assertEqual(query.count("SELECT CASE"), 1) + def test_postgres_recovery_append_returns_exact_existing_from_batch_comparator(self) -> None: + share = pending_share(2) + record = { + "share_seq": 8, + "share_id": share.share_id, + "miner_id": share.miner_id, + "order_key": share.order_key, + "p2mr_program_hex": share.p2mr_program_hex, + "share_difficulty": str(share.share_difficulty), + "network_difficulty": str(share.network_difficulty), + "template_height": share.template_height, + "job_id": share.job_id, + "job_issued_at_ms": share.job_issued_at_ms, + "accepted_at_ms": share.accepted_at_ms, + "ntime": share.ntime, + "credit_policy": share.credit_policy, + "newly_inserted": False, + } + ledger = FakeLeasePsqlShareLedger( + [acquired_lease(), {"records": [record]}] + ) + + outcome = ledger.append_recovered_share(share) + + self.assertEqual(outcome.disposition, "exact_existing") + self.assertEqual(outcome.record.share_seq, 8) + self.assertIn("share_mismatch AS", ledger.lease_queries[-1]) + + def test_postgres_recovery_append_raises_typed_payload_conflict(self) -> None: + ledger = FakeLeasePsqlShareLedger( + [ + acquired_lease(), + { + "error": "duplicate share_id payload mismatch", + "error_kind": "share_replay_conflict", + }, + ] + ) + + with self.assertRaises(ShareReplayConflict): + ledger.append_recovered_share(pending_share(3)) + def test_postgres_candidate_only_intent_forces_durable_fenced_commit(self) -> None: ledger = FakeLeasePsqlShareLedger( [acquired_lease(), {"inserted": 1}] diff --git a/tests/test_prism_share_writer.py b/tests/test_prism_share_writer.py index 588f6ff..4754542 100644 --- a/tests/test_prism_share_writer.py +++ b/tests/test_prism_share_writer.py @@ -223,6 +223,10 @@ def append(self, pending: object) -> object: self.assertEqual(len(flaky.pending), 1) self.assertEqual(server.share_append_failure_count, 2) + self.assertIn( + "qbit_prism_share_append_failures_total 2", + server.metrics_payload(), + ) self.assertEqual(waited.call_count, 2) def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: from lab.prism.share_ledger import PendingShare @@ -340,8 +344,10 @@ def test_replay_skips_torn_line_and_keeps_file(self) -> None: ) # File kept because a line could not be parsed. self.assertTrue(server.share_recovery_path.exists()) - # Re-running dedups the good shares (ledger is idempotent by id). - self.assertEqual(server.replay_recovered_shares(), 2) + # Re-running classifies the intact rows as exact-existing. They are + # not inserted or counted again, while the torn line keeps the + # journal for inspection. + self.assertEqual(server.replay_recovered_shares(), 0) def test_replay_is_idempotent_across_partial_replay(self) -> None: # Finding: a partial replay (A commits, B fails transiently) kept the # whole file; on retry, replay hit A's duplicate and stopped, stranding @@ -355,16 +361,28 @@ def test_replay_is_idempotent_across_partial_replay(self) -> None: class DedupLedger: def __init__(self) -> None: self.ids: list[str] = [] + self.pending_by_id: dict[str, object] = {} self.fail_b_once = True - def append(self, pending: object) -> object: + def append_recovered_share(self, pending: object) -> object: if pending.share_id == "miner-a:B" and self.fail_b_once: self.fail_b_once = False raise RuntimeError("postgres unavailable") if pending.share_id in self.ids: - raise RuntimeError("duplicate share_id") + if self.pending_by_id[pending.share_id] != pending: + raise ShareReplayConflict(pending.share_id) + return ShareReplayResult( + "exact_existing", + SimpleNamespace( + share_seq=self.ids.index(pending.share_id) + 1 + ), + ) self.ids.append(pending.share_id) - return SimpleNamespace(share_seq=len(self.ids)) + self.pending_by_id[pending.share_id] = pending + return ShareReplayResult( + "inserted", + SimpleNamespace(share_seq=len(self.ids)), + ) server.ledger = DedupLedger() diff --git a/tests/test_prism_share_writer_service.py b/tests/test_prism_share_writer_service.py new file mode 100644 index 0000000..5e3163a --- /dev/null +++ b/tests/test_prism_share_writer_service.py @@ -0,0 +1,567 @@ +#!/usr/bin/env python3 +"""Direct tests for the coordinator-free S3 share writer.""" + +from __future__ import annotations + +from contextlib import contextmanager +import queue +from pathlib import Path +import tempfile +import threading +from types import SimpleNamespace +import unittest + +from lab.prism.coordinator_shutdown import ( + CoordinatorShutdownController, + ShutdownInProgress, +) +from lab.prism.share_ledger import ( + PendingShare, + ShareReplayResult, + SingleWriterShareLedger, +) +from lab.prism.share_writer import ( + PendingShareAppend, + PendingShareInput, + ShareWriter, + ShareWriterConfig, + ShareWriterPorts, +) + + +class ShareWriterServiceTests(unittest.TestCase): + def test_coordinator_reexports_exact_pending_append_identity(self) -> None: + from lab.prism.prism_coordinator import PendingShareAppend as compatibility + + self.assertIs(compatibility, PendingShareAppend) + + def test_append_failure_property_and_metrics_snapshot_are_readable(self) -> None: + service, _ledger, _controller, _stop = self._service() + + service.append_failures = 7 + + self.assertEqual(service.append_failures, 7) + self.assertEqual(service.metrics_snapshot().append_failures, 7) + + def _service( + self, + *, + ledger: object | None = None, + wall_times: list[int] | None = None, + recovery_path: Path | None = None, + controller: CoordinatorShutdownController | None = None, + reserve_error: BaseException | None = None, + floor: dict[object, list[object]] | None = None, + ) -> tuple[ShareWriter, object, CoordinatorShutdownController, threading.Event]: + ledger = ledger or SingleWriterShareLedger() + controller = controller or CoordinatorShutdownController(1.0) + stop = threading.Event() + times = list(wall_times or [100]) + last = times[-1] + + def wall_time_ms() -> int: + return times.pop(0) if times else last + + @contextmanager + def writer_operation(component: str): + token = controller.enter_writer(component) + try: + yield + finally: + controller.exit_writer(token) + + def reserve_writer(component: str): + if reserve_error is not None: + raise reserve_error + return controller.reserve_writer(component) + + service = ShareWriter( + ShareWriterConfig( + batch_size=8, + linger_seconds=0, + enqueue_timeout_seconds=0.01, + recovery_path=recovery_path, + ), + ShareWriterPorts( + ledger=lambda: ledger, + writer_operation=writer_operation, + reserve_writer=reserve_writer, + writer_admission_closed=controller.writer_admission_closed, + has_active_writer=controller.has_active_writer, + heartbeat=lambda _name: None, + monotonic=lambda: 10.0, + wall_time_ms=wall_time_ms, + stop_is_set=stop.is_set, + stop_wait=stop.wait, + log=lambda _message: None, + log_exception=lambda: None, + hot_path_log_enabled=lambda: False, + ), + append_queue=queue.Queue(maxsize=4), + floor=floor, + ) + return service, ledger, controller, stop + + @staticmethod + def _input(share_id: str) -> PendingShareInput: + return PendingShareInput( + share_id=share_id, + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + ntime=1_700_000_000, + ) + + @staticmethod + def _entry(pending: PendingShare) -> PendingShareAppend: + return PendingShareAppend( + pending_share=pending, + username="miner-a", + job_id="job-1", + block_hash_hex=pending.share_id.rsplit(":", 1)[-1], + collection_only=False, + credit_policy=None, + ) + + def test_floor_uses_stable_share_id_and_preserves_three_slot_compatibility(self) -> None: + service, _ledger, _controller, _stop = self._service(wall_times=[100, 250]) + service.make_pending_share(self._input("miner-a:same")) + second = service.make_pending_share(self._input("miner-a:same")) + + self.assertEqual(list(service.floor), ["miner-a:same"]) + self.assertEqual(len(service.floor["miner-a:same"]), 3) + self.assertIs(service.floor["miner-a:same"][0], second) + self.assertEqual(service.snapshot_anchor_ms(1_000), 99) + + # A reconstructed object with the same durable identity can finish the + # logical lease even though neither original Python identity survives. + reconstructed = PendingShare(**{**second.__dict__, "accepted_at_ms": 80}) + service.adopt_pending_share(reconstructed) + self.assertEqual(service.snapshot_anchor_ms(1_000), 79) + service.finish_pending_share(reconstructed) + self.assertEqual(service.floor, {}) + + def test_attempt_promotion_is_atomic_and_owner_specific(self) -> None: + service, _ledger, _controller, _stop = self._service(wall_times=[100]) + pending = service.make_pending_share(self._input("miner-a:promoted")) + + service.adopt_pending_share(pending) + + self.assertNotIn(id(pending), service._attempt_holders) + self.assertIn(pending.share_id, service._candidate_holders) + service.finish_pending_attempt(pending) + self.assertIn(pending.share_id, service.floor) + service.finish_pending_candidate(PendingShare(**pending.__dict__)) + self.assertEqual(service.floor, {}) + + def test_same_id_candidate_actors_survive_each_others_terminal_changes( + self, + ) -> None: + service, _ledger, _controller, _stop = self._service(wall_times=[100, 200]) + first = service.make_pending_share(self._input("miner-a:actor")) + entry = service.floor[first.share_id] + service.begin_candidate_actor(first) + second = service.make_pending_share(self._input("miner-a:actor")) + service.begin_candidate_actor(second) + + # Actor B terminalizes the shared durable outbox source. Its own actor + # can leave, but actor A independently preserves the older floor. + service.finish_pending_candidate(second) + service.finish_candidate_actor(second) + self.assertIs(service.floor[first.share_id], entry) + self.assertEqual(service.snapshot_anchor_ms(1_000), 99) + + # If A's credit append fails, retry adoption is established while A is + # still held. Releasing the actor then leaves the durable retry source. + service.adopt_pending_share(first) + service.finish_candidate_actor(first) + self.assertIs(service.floor[first.share_id], entry) + self.assertEqual(service.snapshot_anchor_ms(1_000), 99) + service.finish_pending_candidate(first) + self.assertEqual(service.floor, {}) + + def test_floor_entry_list_identity_survives_holder_transitions_and_warning( + self, + ) -> None: + service, _ledger, _controller, _stop = self._service(wall_times=[100, 200]) + first = service.make_pending_share(self._input("miner-a:identity")) + entry = service.floor[first.share_id] + + service.adopt_pending_share(first) + self.assertIs(service.floor[first.share_id], entry) + second = service.make_pending_share(self._input("miner-a:identity")) + self.assertIs(service.floor[first.share_id], entry) + service.config.pending_floor_warn_seconds = -1 + service.snapshot_anchor_ms(1_000) + self.assertIs(service.floor[first.share_id], entry) + self.assertTrue(entry[2]) + service.finish_pending_attempt(second) + self.assertIs(service.floor[first.share_id], entry) + self.assertEqual(len(entry), 3) + + service.finish_pending_candidate(first) + self.assertEqual(service.floor, {}) + + def test_nonempty_compatibility_floor_is_reachable_before_and_after_adoption( + self, + ) -> None: + first = PendingShare( + share_id="miner-a:first", + miner_id="miner-a", + order_key="miner-a", + p2mr_program_hex="11" * 32, + share_difficulty=1, + network_difficulty=1, + template_height=9, + job_id="job-1", + job_issued_at_ms=1, + accepted_at_ms=100, + ntime=1_700_000_000, + ) + second = PendingShare( + **{ + **first.__dict__, + "share_id": "miner-a:second", + "accepted_at_ms": 200, + } + ) + first_floor = {id(first): [first, 1.0, False]} + first_entry = first_floor[id(first)] + service, _ledger, _controller, _stop = self._service(floor=first_floor) + + service.adopt_pending_share(first) + self.assertIs(service.floor[first.share_id], first_entry) + service.finish_pending_candidate(first) + self.assertEqual(service.floor, {}) + + second_floor = {id(second): [second, 2.0, True]} + second_entry = second_floor[id(second)] + service.adopt_floor(second_floor) + self.assertIs(service.floor, second_floor) + service.adopt_pending_share(second) + self.assertIs(service.floor[second.share_id], second_entry) + service.finish_pending_candidate(second) + self.assertEqual(service.floor, {}) + + def test_parent_descendant_and_nonselected_candidates_keep_reachable_leases(self) -> None: + service, _ledger, _controller, _stop = self._service(wall_times=[200, 150, 175]) + descendant = service.make_pending_share(self._input("miner-a:descendant")) + parent = service.make_pending_share(self._input("miner-a:parent")) + competitor = service.make_pending_share(self._input("miner-a:competitor")) + + # Retry selection may replace a descendant with its parent or decline + # a competitor. All three durable identities remain terminally + # reachable rather than being tied to the selected in-memory object. + service.transfer_pending_floor(descendant, parent) + self.assertEqual( + set(service.floor), + {"miner-a:descendant", "miner-a:parent", "miner-a:competitor"}, + ) + self.assertEqual(service.snapshot_anchor_ms(1_000), 149) + + for pending in (parent, competitor, descendant): + reconstructed = PendingShare(**pending.__dict__) + service.finish_pending_share(reconstructed) + self.assertEqual(service.floor, {}) + + def test_direct_enqueue_admission_refusal_releases_registered_floor(self) -> None: + service, _ledger, _controller, _stop = self._service( + reserve_error=RuntimeError("closed") + ) + pending = service.make_pending_share(self._input("miner-a:closed")) + + with self.assertRaisesRegex(RuntimeError, "closed"): + service.enqueue(self._entry(pending)) + + self.assertEqual(service.floor, {}) + + def test_append_admission_refusal_releases_unhanded_attempt(self) -> None: + controller = CoordinatorShutdownController(1.0) + service, _ledger, _controller, _stop = self._service(controller=controller) + pending = service.make_pending_share(self._input("miner-a:closed-append")) + controller.request_shutdown(None) + + with self.assertRaises(ShutdownInProgress): + service.append_and_wait(self._entry(pending)) + + self.assertEqual(service.floor, {}) + + def test_invisible_queue_exception_releases_token_and_floor(self) -> None: + class ExplodingQueue(queue.Queue): + def put_nowait(self, _item: object) -> None: + raise OSError("queue transport failed") + + service, _ledger, controller, _stop = self._service() + service.adopt_queue(ExplodingQueue(maxsize=1)) + pending = service.make_pending_share(self._input("miner-a:invisible")) + entry = self._entry(pending) + + with self.assertRaisesRegex(OSError, "transport failed"): + service.enqueue(entry) + + self.assertIsNone(entry.writer_token) + self.assertEqual(service.floor, {}) + self.assertEqual(controller.snapshot()["active_writers"], {}) + + def test_interrupted_visible_wait_leaves_attempt_owned_by_writer(self) -> None: + class InterruptingEvent: + def __init__(self) -> None: + self.was_set = False + + def wait(self) -> None: + raise RuntimeError("wait interrupted") + + def set(self) -> None: + self.was_set = True + + service, ledger, controller, _stop = self._service() + service.active = True + pending = service.make_pending_share(self._input("miner-a:visible")) + committed = InterruptingEvent() + entry = PendingShareAppend( + **{ + **self._entry(pending).__dict__, + "committed": committed, + } + ) + + with self.assertRaisesRegex(RuntimeError, "wait interrupted"): + service.append_and_wait(entry) + + self.assertIn(pending.share_id, service.floor) + self.assertIsNotNone(entry.writer_token) + service.append_batch([service.append_queue.get_nowait()]) + self.assertEqual(len(ledger), 1) + self.assertTrue(committed.was_set) + self.assertEqual(service.floor, {}) + self.assertEqual(controller.snapshot()["active_writers"], {}) + + def test_nested_append_inherits_outer_submit_after_shutdown_closes(self) -> None: + controller = CoordinatorShutdownController(1.0) + service, ledger, _controller, _stop = self._service(controller=controller) + outer = controller.enter_writer("share_submission") + try: + pending = service.make_pending_share(self._input("miner-a:inherited")) + controller.request_shutdown(None) + service.append_and_wait(self._entry(pending)) + finally: + controller.exit_writer(outer) + + self.assertEqual(len(ledger), 1) + self.assertEqual(service.floor, {}) + + def test_startup_gate_orders_legacy_ack_replay_before_candidate_credit(self) -> None: + service, ledger, _controller, _stop = self._service(wall_times=[200]) + candidate = service.make_pending_share(self._input("miner-a:candidate")) + legacy = PendingShare( + **{ + **candidate.__dict__, + "share_id": "miner-a:legacy", + "accepted_at_ms": 300, + } + ) + service.begin_startup_recovery() + errors: list[BaseException] = [] + + def append_candidate() -> None: + try: + service.append_and_wait(self._entry(candidate)) + except BaseException as exc: + errors.append(exc) + + append = threading.Thread( + target=append_candidate, + ) + append.start() + append.join(timeout=0.05) + self.assertTrue(append.is_alive()) + self.assertEqual(len(ledger), 0) + + replay = ledger.append_recovered_share(legacy) + service.finish_startup_recovery() + append.join(timeout=2) + + self.assertFalse(append.is_alive()) + self.assertEqual(errors, []) + self.assertEqual(replay.record.share_seq, 1) + self.assertEqual( + [record.share_id for record in ledger.all_shares()], + ["miner-a:legacy", "miner-a:candidate"], + ) + + def test_startup_gate_cancellation_aborts_waiter_and_late_caller(self) -> None: + service, ledger, _controller, _stop = self._service(wall_times=[200, 300]) + waiting = service.make_pending_share(self._input("miner-a:waiting")) + service.begin_startup_recovery() + errors: list[BaseException] = [] + + def append_waiting() -> None: + try: + service.append_and_wait(self._entry(waiting)) + except BaseException as exc: + errors.append(exc) + + append = threading.Thread(target=append_waiting) + append.start() + append.join(timeout=0.05) + self.assertTrue(append.is_alive()) + + service.cancel_startup_recovery() + append.join(timeout=1) + + self.assertFalse(append.is_alive()) + self.assertEqual(len(errors), 1) + self.assertIsInstance(errors[0], ShutdownInProgress) + self.assertEqual(len(ledger), 0) + self.assertEqual(service.floor, {}) + + late = service.make_pending_share(self._input("miner-a:late")) + with self.assertRaisesRegex(ShutdownInProgress, "recovery was cancelled"): + service.append_and_wait(self._entry(late)) + self.assertEqual(len(ledger), 0) + self.assertEqual(service.floor, {}) + + def test_startup_gate_fast_path_rechecks_interleaved_cancellation(self) -> None: + service, ledger, _controller, _stop = self._service(wall_times=[200]) + pending = service.make_pending_share(self._input("miner-a:fast-cancel")) + + class CancellationInterleavingEvent: + def __init__(self) -> None: + self.opened = False + self.interleaved = False + + def is_set(self) -> bool: + if not self.interleaved: + self.interleaved = True + service.cancel_startup_recovery() + return self.opened + + def set(self) -> None: + self.opened = True + + def clear(self) -> None: + self.opened = False + + def wait(self, _timeout: float | None = None) -> bool: + return self.opened + + gate = CancellationInterleavingEvent() + service._startup_recovery_complete = gate # type: ignore[assignment] + + with self.assertRaisesRegex(ShutdownInProgress, "recovery was cancelled"): + service.append_and_wait(self._entry(pending)) + + self.assertTrue(gate.interleaved) + self.assertEqual(len(ledger), 0) + self.assertEqual(service.floor, {}) + + def test_recovery_exact_existing_is_typed_and_clears_clean_journal(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, ledger, _controller, _stop = self._service(recovery_path=path) + pending = service.make_pending_share(self._input("miner-a:exact")) + entry = self._entry(pending) + service.recover_to_disk(entry, "test") + ledger.append_recovered_share(pending) + + self.assertEqual(service.replay_recovery_file(), 0) + self.assertFalse(path.exists()) + self.assertEqual(service.metrics_snapshot().replay_exact_existing, 1) + + def test_recovery_payload_conflict_is_typed_and_retains_journal(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, ledger, _controller, _stop = self._service(recovery_path=path) + pending = service.make_pending_share(self._input("miner-a:conflict")) + service.recover_to_disk(self._entry(pending), "test") + ledger.append_recovered_share( + PendingShare(**{**pending.__dict__, "ntime": pending.ntime + 1}) + ) + + self.assertEqual(service.replay_recovery_file(), 0) + self.assertTrue(path.exists()) + self.assertEqual(service.metrics_snapshot().replay_conflicts, 1) + + def test_unknown_recovery_disposition_is_conservative_and_retains_journal( + self, + ) -> None: + class FutureLedger: + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + return ShareReplayResult( + "future-disposition", + SimpleNamespace(share_seq=1, share_id=pending.share_id), + ) + + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, _ledger, _controller, _stop = self._service( + ledger=FutureLedger(), + recovery_path=path, + ) + pending = service.make_pending_share(self._input("miner-a:future")) + service.recover_to_disk(self._entry(pending), "test") + + self.assertEqual(service.replay_recovery_file(), 0) + self.assertTrue(path.exists()) + + def test_recovery_identity_cannot_change_after_replay_begins(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, _ledger, _controller, _stop = self._service(recovery_path=path) + self.assertEqual(service.replay_recovery_file(), 0) + + with self.assertRaisesRegex(RuntimeError, "after replay starts"): + service.set_recovery_path(path.with_name("other.jsonl")) + with self.assertRaisesRegex(RuntimeError, "after replay starts"): + service.adopt_recovery_lock(threading.Lock()) + + def test_recovery_append_serializes_with_replay_and_clean_unlink(self) -> None: + entered = threading.Event() + release = threading.Event() + + class BlockingReplayLedger: + def append_recovered_share(self, pending: PendingShare) -> ShareReplayResult: + entered.set() + release.wait(timeout=2) + return ShareReplayResult( + "inserted", + SimpleNamespace(share_seq=1, share_id=pending.share_id), + ) + + with tempfile.TemporaryDirectory() as tempdir: + path = Path(tempdir) / "recovery.jsonl" + service, _ledger, _controller, _stop = self._service( + ledger=BlockingReplayLedger(), + recovery_path=path, + ) + first = service.make_pending_share(self._input("miner-a:first")) + second = PendingShare(**{**first.__dict__, "share_id": "miner-a:second"}) + service.recover_to_disk(self._entry(first), "first") + replay = threading.Thread(target=service.replay_recovery_file) + replay.start() + self.assertTrue(entered.wait(timeout=1)) + + append = threading.Thread( + target=service.recover_to_disk, + args=(self._entry(second), "second"), + ) + append.start() + append.join(timeout=0.05) + self.assertTrue(append.is_alive()) + + release.set() + replay.join(timeout=2) + append.join(timeout=2) + self.assertFalse(replay.is_alive()) + self.assertFalse(append.is_alive()) + self.assertIn("miner-a:second", path.read_text(encoding="utf-8")) + self.assertNotIn("miner-a:first", path.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_stratum_restart_bind.py b/tests/test_prism_stratum_restart_bind.py index df98368..439f872 100644 --- a/tests/test_prism_stratum_restart_bind.py +++ b/tests/test_prism_stratum_restart_bind.py @@ -148,6 +148,9 @@ def test_listeners_accept_tcp_connections_during_block_work_recovery(self) -> No server.vardiff_config = make_vardiff_config() server.max_blocks = 1 server.blockpoll_seconds = 0.1 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.version_mask = 0x1FFFE000 server.version_mask_selection = types.SimpleNamespace( source="test", detail="fixed" diff --git a/tests/test_prism_stratum_session.py b/tests/test_prism_stratum_session.py new file mode 100644 index 0000000..5643348 --- /dev/null +++ b/tests/test_prism_stratum_session.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +"""Deterministic tests for the extracted PRISM Stratum session boundary.""" + +from __future__ import annotations + +import socket +import threading +import unittest +from collections import OrderedDict +from decimal import Decimal +from types import SimpleNamespace + +from lab.auxpow import vardiff +from lab.prism import stratum_session +from lab.prism.prism_coordinator import ( + ClientState as FacadeClientState, + PrismCoordinator, + StratumError as FacadeStratumError, + WorkerIdentity as FacadeWorkerIdentity, +) +from lab.prism.stratum_session import ( + ClientState, + P2mrAddressValidator, + SessionRegistry, + StratumError, + StratumListenerProfile, + StratumSessionService, + WorkerIdentity, + client_can_receive_jobs, + error_payload, + parse_stratum_password_options, + result_payload, +) + + +def listener() -> StratumListenerProfile: + config = vardiff.VardiffConfig( + enabled=False, + target_share_interval_seconds=Decimal("15"), + min_difficulty=Decimal("1"), + max_difficulty=Decimal("1024"), + retarget_interval_seconds=Decimal("90"), + max_step_factor=Decimal("4"), + startup_difficulty=Decimal("1"), + max_step_down_factor=Decimal("4"), + ewma_alpha=Decimal("0.4"), + retarget_tolerance=Decimal("0.25"), + ) + return StratumListenerProfile( + name="default", + bind="127.0.0.1", + port=3340, + share_difficulty=Decimal("1"), + vardiff_config=config, + heartbeat_name="stratum_accept", + ) + + +def worker(username: str) -> WorkerIdentity: + return WorkerIdentity( + username=username, + payout_address=username, + worker_name=None, + script_pubkey_hex="5220" + "11" * 32, + p2mr_program_hex="11" * 32, + ) + + +class FakeSocket: + def __init__(self) -> None: + self.closed = threading.Event() + self.sent: list[dict[str, object]] = [] + + def settimeout(self, _timeout: object) -> None: + return + + def setsockopt(self, *_args: object) -> None: + return + + def shutdown(self, _how: object) -> None: + self.closed.set() + + def close(self) -> None: + self.closed.set() + + def sendall(self, _data: bytes) -> None: + return + + +class FakeJobs: + def __init__(self, registry: SessionRegistry) -> None: + self.registry = registry + self.cancelled: list[ClientState] = [] + self.cleaned: list[ClientState] = [] + self.retained = 0 + + def note_collection_identity_available(self, _client: ClientState) -> None: + return + + def request_initial_job_delivery(self, _client: ClientState) -> None: + return + + def apply_client_difficulty_requests(self, _client: ClientState) -> Decimal | None: + return None + + def advertise_client_difficulty(self, _client: ClientState, _target: Decimal) -> bool: + return False + + def handle_submit(self, _client: ClientState, _params: list[object]) -> bool: + return False + + def refresh_jobs_after_pending_accepted_block(self, _client: ClientState) -> None: + return + + def cancel_pending_initial_job_locked(self, client: ClientState) -> None: + self.cancelled.append(client) + + def cleanup_disconnected_client(self, client: ClientState) -> None: + self.cleaned.append(client) + with self.registry.lock: + self.registry.clear_active_jobs_locked(client) + client.authorized = False + client.worker = None + client.username = "" + + def retain_current_collection_refresh_if_unrepresented(self) -> None: + self.retained += 1 + + +class FakeProgress: + def __init__(self, registry: SessionRegistry) -> None: + self.registry = registry + self.deliveries: list[tuple[int, object, float]] = [] + self.registry_had_proof_at_callback: list[bool] = [] + self.reconciles = 0 + + def record_delivery( + self, client: ClientState, context: object, delivered_monotonic: float + ) -> None: + snapshot = self.registry.eligible_snapshot() + delivered = snapshot.get(client.connection_id) + self.registry_had_proof_at_callback.append( + delivered is not None + and delivered.delivered is not None + and delivered.delivered.context is context + ) + self.deliveries.append( + (client.connection_id, context, delivered_monotonic) + ) + + def reconcile_eligibility(self) -> None: + self.reconciles += 1 + + +class FailingThread: + def start(self) -> None: + raise RuntimeError("thread unavailable") + + +class FakeRuntime: + def __init__(self) -> None: + self.is_running = True + self.setup_failures = 0 + self.registry_metrics: list[tuple[int, int]] = [] + + def running(self) -> bool: + return self.is_running + + def record_heartbeat(self, _name: str) -> None: + return + + def wait_after_resource_failure(self, _heartbeat_name: str) -> None: + return + + def record_resource_exhaustion(self, **_kwargs: object) -> None: + return + + def record_setup_failure(self) -> int: + self.setup_failures += 1 + return self.setup_failures + + def sync_registry_metrics(self, registry: SessionRegistry) -> None: + self.registry_metrics.append( + (len(registry.clients), registry.handler_thread_count) # type: ignore[arg-type] + ) + + def max_connections(self) -> int: + return 8 + + def max_connections_per_username(self) -> int: + return 1 + + def client_startup_difficulty(self, _profile: StratumListenerProfile) -> Decimal: + return Decimal("1") + + def apply_send_timeout(self, _sock: socket.socket) -> None: + return + + def make_client_thread(self, _client: ClientState) -> FailingThread: + return FailingThread() + + def extranonce2_size(self) -> int: + return 8 + + def version_mask(self) -> int: + return 0x1FFFE000 + + def username_fallback_address(self) -> str | None: + return None + + def resolve_worker(self, _username: str, fallback: object) -> WorkerIdentity: + return fallback() # type: ignore[operator] + + def reserve_client_username( + self, + _client: ClientState, + _worker: WorkerIdentity, + fallback: object, + ) -> bool: + return bool(fallback()) # type: ignore[operator] + + def send_result( + self, client: ClientState, request_id: object, result: object + ) -> None: + client.send(result_payload(request_id, result)) + + def send_error( + self, + client: ClientState, + request_id: object, + code: int, + message: str, + *, + reason: str | None, + ) -> None: + client.send(error_payload(request_id, code, message, reason=reason)) + + def disconnect_client(self, _client: ClientState, fallback: object) -> None: + fallback() # type: ignore[operator] + + +def service_fixture( + clients: object | None = None, +) -> tuple[StratumSessionService, SessionRegistry, FakeRuntime, FakeJobs, FakeProgress]: + registry = SessionRegistry( + lock=threading.RLock(), + clients=set() if clients is None else clients, + ) + runtime = FakeRuntime() + jobs = FakeJobs(registry) + progress = FakeProgress(registry) + validator = P2mrAddressValidator( + rpc_call=lambda _method, _params: { + "isvalid": True, + "scriptPubKey": "5220" + "11" * 32, + }, + max_entries=lambda: 16, + ttl_seconds=lambda: 60.0, + cache=OrderedDict(), + ) + service = StratumSessionService( + registry=registry, + runtime=runtime, + jobs=jobs, + progress=progress, + address_validator=validator, + pool_closed_reason="pool-closed", + ) + return service, registry, runtime, jobs, progress + + +class SessionRegistryTests(unittest.TestCase): + def test_admission_generation_and_global_capacity_are_one_atomic_registry_step(self) -> None: + registry = SessionRegistry(lock=threading.RLock()) + profile = listener() + + first, first_rejection = registry.admit( + sock=FakeSocket(), # type: ignore[arg-type] + address=("127.0.0.1", 1), + profile=profile, + share_difficulty=Decimal("1"), + max_connections=1, + ) + second, second_rejection = registry.admit( + sock=FakeSocket(), # type: ignore[arg-type] + address=("127.0.0.1", 2), + profile=profile, + share_difficulty=Decimal("1"), + max_connections=1, + ) + + assert first is not None + self.assertEqual(first.connection_id, 1) + self.assertEqual(first_rejection, 0) + self.assertIsNone(second) + self.assertEqual(second_rejection, 1) + self.assertEqual(registry.connection_generation, 1) + self.assertEqual(registry.clients, {first}) + + def test_reauthorization_limit_preserves_prior_live_identity(self) -> None: + first = ClientState(FakeSocket(), ("127.0.0.1", 1), 1, "00000001") # type: ignore[arg-type] + occupant = ClientState(FakeSocket(), ("127.0.0.1", 2), 2, "00000002") # type: ignore[arg-type] + registry = SessionRegistry(lock=threading.RLock(), clients={first, occupant}) + original = worker("original") + full = worker("full") + self.assertEqual( + registry.reserve_username( + first, original, max_connections_per_username=1 + ), + (True, 0), + ) + self.assertEqual( + registry.reserve_username( + occupant, full, max_connections_per_username=1 + ), + (True, 0), + ) + + accepted, count = registry.reserve_username( + first, full, max_connections_per_username=1 + ) + + self.assertFalse(accepted) + self.assertEqual(count, 1) + self.assertIs(first.worker, original) + self.assertEqual(first.username, "original") + + def test_eligibility_snapshot_is_immutable_exact_and_uses_delivered_context(self) -> None: + current = ClientState(FakeSocket(), ("127.0.0.1", 1), 1, "00000001") # type: ignore[arg-type] + current.subscribed = current.authorized = True + current.worker = worker("miner") + current.username = "miner" + idle = ClientState(FakeSocket(), ("127.0.0.1", 2), 2, "00000002") # type: ignore[arg-type] + clients = [current, idle] + registry = SessionRegistry(lock=threading.RLock(), clients=clients) + delivered = SimpleNamespace(name="delivered") + current.active_job = SimpleNamespace(name="registered-before-send") + registry.record_delivery(current, delivered, 42.0) + current.active_job = SimpleNamespace(name="newer-unsent") + + snapshot = registry.eligible_snapshot() + + self.assertEqual(tuple(snapshot), (1,)) + self.assertIs(snapshot[1].delivered.context, delivered) # type: ignore[union-attr] + self.assertTrue(client_can_receive_jobs(current)) + with self.assertRaises(TypeError): + snapshot[3] = snapshot[1] # type: ignore[index] + + # Reauthorization does not erase a valid socket-delivery proof. + current.worker = worker("replacement") + self.assertIs( + registry.eligible_snapshot()[1].delivered.context, # type: ignore[union-attr] + delivered, + ) + with registry.lock: + self.assertTrue(registry.begin_retirement_locked(current)) + self.assertFalse(registry.eligible_snapshot()) + + def test_active_job_registration_is_registry_owned(self) -> None: + state = ClientState(FakeSocket(), ("127.0.0.1", 1), 1, "00000001") # type: ignore[arg-type] + state.active_job_ids.update({"old-a", "old-b"}) + registry = SessionRegistry(lock=threading.RLock(), clients={state}) + context = SimpleNamespace(job=SimpleNamespace(job_id="new")) + + with registry.lock: + retired = registry.register_active_job_locked( + state, context, job_id="new", clean_jobs=True + ) + + self.assertEqual(set(retired), {"old-a", "old-b"}) + self.assertIs(state.active_job, context) + self.assertEqual(state.active_job_ids, {"new"}) + + def test_ordered_membership_can_be_adopted_before_and_after_use(self) -> None: + first = ClientState(FakeSocket(), ("127.0.0.1", 1), 1, "00000001") # type: ignore[arg-type] + second = ClientState(FakeSocket(), ("127.0.0.1", 2), 2, "00000002") # type: ignore[arg-type] + registry = SessionRegistry(lock=threading.RLock(), clients=[first, second]) + self.assertIsInstance(registry.clients, list) + self.assertEqual(registry.clients, [first, second]) + + replacement = [second, first] + registry.adopt_clients(replacement) + + self.assertIs(registry.clients, replacement) + self.assertEqual(registry.clients, [second, first]) + + def test_coordinator_registry_adopts_ordered_membership_replacement(self) -> None: + first = ClientState(FakeSocket(), ("127.0.0.1", 1), 1, "00000001") # type: ignore[arg-type] + second = ClientState(FakeSocket(), ("127.0.0.1", 2), 2, "00000002") # type: ignore[arg-type] + coordinator = PrismCoordinator.__new__(PrismCoordinator) + coordinator.lock = threading.RLock() + coordinator.clients = [first, second] # type: ignore[assignment] + coordinator.connection_counter = 0 + + registry = coordinator._ensure_session_registry() + self.assertIs(registry.clients, coordinator.clients) + self.assertEqual(registry.connection_generation, 2) + first.handler_thread_registered = True + replacement = [second, first] + coordinator.clients = replacement # type: ignore[assignment] + + self.assertIs(coordinator._ensure_session_registry(), registry) + self.assertIs(registry.clients, replacement) + self.assertEqual(registry.clients, [second, first]) + self.assertEqual(registry.handler_thread_count, 1) + self.assertEqual(coordinator.connection_counter, 2) + self.assertEqual(coordinator.handler_thread_count, 1) + + +class SessionLifecycleTests(unittest.TestCase): + def test_handler_thread_failure_rolls_back_membership_and_socket(self) -> None: + service, registry, runtime, _jobs, _progress = service_fixture() + accepted = FakeSocket() + + class Listener: + calls = 0 + + def accept(self) -> tuple[FakeSocket, tuple[str, int]]: + self.calls += 1 + if self.calls == 1: + return accepted, ("127.0.0.1", 1) + runtime.is_running = False + raise socket.timeout + + service.accept_loop(Listener(), listener()) # type: ignore[arg-type] + + self.assertTrue(accepted.closed.is_set()) + self.assertFalse(registry.clients) + self.assertEqual(registry.handler_thread_count, 0) + self.assertEqual(runtime.setup_failures, 1) + + def test_disconnect_closes_socket_before_waiting_for_job_update_lock(self) -> None: + service, registry, _runtime, jobs, progress = service_fixture() + sock = FakeSocket() + state = ClientState(sock, ("127.0.0.1", 1), 1, "00000001") # type: ignore[arg-type] + registry._add_client_locked(state) + state.job_update_lock.acquire() + finished = threading.Event() + + def disconnect() -> None: + service.disconnect_client(state) + finished.set() + + thread = threading.Thread(target=disconnect) + thread.start() + self.assertTrue(sock.closed.wait(1.0)) + self.assertFalse(finished.is_set()) + self.assertNotIn(state, registry.clients) + state.job_update_lock.release() + thread.join(1.0) + + self.assertTrue(finished.is_set()) + self.assertEqual(jobs.cancelled, [state]) + self.assertEqual(jobs.cleaned, [state]) + self.assertEqual(progress.reconciles, 1) + service.disconnect_client(state) + self.assertEqual(jobs.cleaned, [state]) + + def test_successful_delivery_records_registry_proof_before_health_callback(self) -> None: + service, registry, _runtime, _jobs, progress = service_fixture() + state = ClientState(FakeSocket(), ("127.0.0.1", 1), 1, "00000001") # type: ignore[arg-type] + state.subscribed = state.authorized = True + state.worker = worker("miner") + registry._add_client_locked(state) + context = SimpleNamespace(name="sent") + + service.record_successful_delivery(state, context, 12.5) + + self.assertIs( + registry.eligible_snapshot()[1].delivered.context, # type: ignore[union-attr] + context, + ) + self.assertEqual(progress.deliveries, [(1, context, 12.5)]) + self.assertEqual(progress.registry_had_proof_at_callback, [True]) + self.assertEqual(progress.reconciles, 1) + + def test_retired_client_delivery_cannot_reach_progress_health(self) -> None: + service, registry, _runtime, _jobs, progress = service_fixture() + state = ClientState(FakeSocket(), ("127.0.0.1", 1), 1, "00000001") # type: ignore[arg-type] + state.subscribed = state.authorized = True + state.worker = worker("miner") + registry._add_client_locked(state) + with registry.lock: + self.assertTrue(registry.begin_retirement_locked(state)) + + service.record_successful_delivery( + state, + SimpleNamespace(name="sent-before-retirement-won"), + 12.5, + ) + + self.assertFalse(registry.eligible_snapshot()) + self.assertEqual(progress.deliveries, []) + self.assertEqual(progress.registry_had_proof_at_callback, []) + self.assertEqual(progress.reconciles, 0) + + +class CompatibilityTests(unittest.TestCase): + def test_coordinator_reexports_exact_session_model_identities(self) -> None: + self.assertIs(FacadeClientState, ClientState) + self.assertIs(FacadeWorkerIdentity, WorkerIdentity) + self.assertIs(FacadeStratumError, StratumError) + self.assertFalse(hasattr(stratum_session, "PrismCoordinator")) + + def test_protocol_helpers_preserve_payload_and_password_behavior(self) -> None: + self.assertEqual(result_payload(1, True), {"id": 1, "result": True, "error": None}) + self.assertEqual( + error_payload(2, 21, "stale", reason="stale-job"), + { + "id": 2, + "result": None, + "error": [21, "stale", {"reason_id": "stale-job"}], + }, + ) + self.assertEqual( + parse_stratum_password_options("x,md=4,d=8,bad=1"), + (Decimal("8"), Decimal("4")), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_tip_publication_boundary.py b/tests/test_prism_tip_publication_boundary.py index 3b7bcc8..e0c3f47 100644 --- a/tests/test_prism_tip_publication_boundary.py +++ b/tests/test_prism_tip_publication_boundary.py @@ -45,7 +45,9 @@ def fail_final_validation(*_args: object, **_kwargs: object) -> object: validation_reached = True raise TemplateRefreshBlocked("forced final validation failure") - server._validate_prepared_tip_refresh = fail_final_validation # type: ignore[method-assign] + server._ensure_tip_refresh_service().validate_prepared = ( # type: ignore[method-assign] + fail_final_validation + ) with self.assertRaisesRegex( TemplateRefreshBlocked, @@ -58,7 +60,7 @@ def fail_final_validation(*_args: object, **_kwargs: object) -> object: self.assertEqual(server.current_tip_first_seen, published_tip) self.assertIs(server.tip_template_snapshot, published_snapshot) self.assertEqual(server.latest_detected_tip[0], next_tip) - self.assertIsNone(server._active_tip_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().active_refresh_snapshot()) finally: server.shutdown_tip_refresh_executor() @@ -96,7 +98,8 @@ def record_send(payload: dict[str, object]) -> None: mark_pending=False, ) ) - snapshot = server.fetch_qbit_tip_template_snapshot() + with server._ensure_tip_refresh_service().suppress_trigger_callbacks_for_test(): + snapshot = server.fetch_qbit_tip_template_snapshot() bundle = server.prepare_tip_refresh_bundle(snapshot) token = server._validate_prepared_tip_refresh( bundle, @@ -124,7 +127,7 @@ def record_send(payload: dict[str, object]) -> None: self.assertIs(server.tip_template_snapshot, published_snapshot) self.assertEqual(server.latest_detected_tip[0], winning_tip) self.assertEqual(len(sent_tips), 1) - self.assertIsNone(server._active_tip_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().active_refresh_snapshot()) finally: server.shutdown_tip_refresh_executor() @@ -148,14 +151,14 @@ def test_executor_failure_happens_before_prepared_publication(self) -> None: def fail_executor() -> object: raise RuntimeError("executor unavailable") - server.tip_refresh_executor = fail_executor # type: ignore[method-assign] + server._ensure_tip_refresh_service().executor = fail_executor # type: ignore[method-assign] with self.assertRaisesRegex(RuntimeError, "executor unavailable"): server.poll_qbit_tip_template_once() self.assertEqual(server.current_tip_first_seen, published_tip) self.assertIs(server.tip_template_snapshot, published_snapshot) self.assertEqual(server.latest_detected_tip[0], next_tip) - self.assertIsNone(server._active_tip_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().active_refresh_snapshot()) finally: server.shutdown_tip_refresh_executor() @@ -183,7 +186,8 @@ def test_payout_mutation_before_publication_keeps_previous_authority(self) -> No mark_pending=False, ) ) - snapshot = server.fetch_qbit_tip_template_snapshot() + with server._ensure_tip_refresh_service().suppress_trigger_callbacks_for_test(): + snapshot = server.fetch_qbit_tip_template_snapshot() bundle = server.prepare_tip_refresh_bundle(snapshot) token = server._validate_prepared_tip_refresh( bundle, @@ -205,7 +209,7 @@ def test_payout_mutation_before_publication_keeps_previous_authority(self) -> No self.assertEqual(server.current_tip_first_seen, published_tip) self.assertIs(server.tip_template_snapshot, published_snapshot) - self.assertIsNone(server._active_tip_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().active_refresh_snapshot()) finally: server.shutdown_tip_refresh_executor() @@ -233,7 +237,8 @@ def test_payout_publication_block_keeps_previous_authority(self) -> None: mark_pending=False, ) ) - snapshot = server.fetch_qbit_tip_template_snapshot() + with server._ensure_tip_refresh_service().suppress_trigger_callbacks_for_test(): + snapshot = server.fetch_qbit_tip_template_snapshot() bundle = server.prepare_tip_refresh_bundle(snapshot) token = server._validate_prepared_tip_refresh( bundle, @@ -242,7 +247,7 @@ def test_payout_publication_block_keeps_previous_authority(self) -> None: ) server._block_payout_state_publication() - self.assertTrue(server._payout_state_publication_blocked) + self.assertTrue(server._payout_state_service._publication_blocked) with self.assertRaisesRegex( TemplateRefreshBlocked, "superseded before atomic publication", @@ -256,7 +261,7 @@ def test_payout_publication_block_keeps_previous_authority(self) -> None: self.assertEqual(server.current_tip_first_seen, published_tip) self.assertIs(server.tip_template_snapshot, published_snapshot) - self.assertIsNone(server._active_tip_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().active_refresh_snapshot()) finally: server.shutdown_tip_refresh_executor() @@ -271,14 +276,19 @@ def test_tip_publication_preserves_first_payout_delivery_priority(self) -> None: try: self.assertEqual(server.poll_qbit_tip_template_once(), 1) server._reserve_payout_state_source("payout_only") + # This test manually drives publication below. Suppress the normal + # payout callback so its scheduler cannot legitimately consume the + # priority delivery while the manual token is being assembled. + tip_refresh = server._ensure_tip_refresh_service() + with tip_refresh.suppress_trigger_callbacks_for_test(): + self.assertEqual( + server._publish_payout_state_candidate( + server._current_payout_state_candidate() + ), + 1, + ) self.assertEqual( - server._publish_payout_state_candidate( - server._current_payout_state_candidate() - ), - 1, - ) - self.assertEqual( - server._payout_state_delivery_gate._priority_generation, + server._payout_state_service._delivery_gate._priority_generation, 1, ) @@ -305,10 +315,10 @@ def test_tip_publication_preserves_first_payout_delivery_priority(self) -> None: ) try: self.assertEqual( - server._payout_state_delivery_gate._priority_generation, + server._payout_state_service._delivery_gate._priority_generation, 1, ) - with server._payout_state_delivery_gate.delivery_cancelable( + with server._payout_state_service._delivery_gate.delivery_cancelable( lambda: False, generation=1, priority=False, @@ -382,12 +392,14 @@ def test_direct_issuance_pins_published_snapshot_during_unpublished_window(self) mark_pending=False, ) ) - server.fetch_qbit_tip_template_snapshot() - with server._job_cache_lock: - self.assertEqual( - server._template_artifacts.previousblockhash, - next_tip, - ) + with server._ensure_tip_refresh_service().suppress_trigger_callbacks_for_test(): + server.fetch_qbit_tip_template_snapshot() + current = ( + server._ensure_job_bundle_service() + .template_repository.current_artifacts() + ) + assert current is not None + self.assertEqual(current.previousblockhash, next_tip) # Direct issuance stays pinned to the published snapshot while the # published tip still owns share classification, so the issued job @@ -409,8 +421,8 @@ def test_direct_issuance_pins_published_snapshot_during_unpublished_window(self) # A pruned bundle cache (payout mutation, LRU pressure) must not # strand pinned issuance either: published-parent work stays # buildable and cacheable while its authority holds. - with server._job_cache_lock: - server._job_bundle_cache.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() rebuilt = server.build_job_for_client(state, clean_jobs=False) self.assertEqual( str(rebuilt.template["previousblockhash"]), @@ -424,7 +436,9 @@ def test_direct_issuance_pins_published_snapshot_during_unpublished_window(self) - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - 1.0 ) - server.template_refresh_failure_exit_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=0.0 + ) lapsed = server.job_issuance_template_artifacts() self.assertEqual(lapsed.previousblockhash, next_tip) finally: @@ -483,13 +497,14 @@ def test_pinned_published_rebuild_recaches_for_repeat_issuance(self) -> None: rpc.tip = next_tip rpc.template = base_template(height=11, prevhash=next_tip) self.assertTrue(server.observe_tip_for_refresh(next_tip)) - server.fetch_qbit_tip_template_snapshot() + with server._ensure_tip_refresh_service().suppress_trigger_callbacks_for_test(): + server.fetch_qbit_tip_template_snapshot() # Simulate cache pressure dropping the retained published bundle, # then issue twice: the first pinned rebuild re-enters the cache # so the second issuance is a hit instead of another heavy build. - with server._job_cache_lock: - server._job_bundle_cache.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() first = server.build_job_for_client(state, clean_jobs=False) builds_after_first = int(builds["calls"]) second = server.build_job_for_client(state, clean_jobs=False) @@ -618,7 +633,9 @@ def build_then_lapse( - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - 1.0 ) - server.template_refresh_failure_exit_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=0.0 + ) return context server.build_job_for_client = build_then_lapse # type: ignore[method-assign] @@ -666,7 +683,9 @@ def record_send(payload: dict[str, object]) -> None: - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - 1.0 ) - server.template_refresh_failure_exit_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=0.0 + ) state.active_job = None state.active_job_ids = set() @@ -716,7 +735,9 @@ def lapse_during_gate(*args: object, **kwargs: object) -> None: - float(getattr(server, "submit_tip_max_age_seconds", 10.0)) - 1.0 ) - server.template_refresh_failure_exit_seconds = 0.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=0.0 + ) original_observe_admission(*args, **kwargs) server._observe_payout_gate_admission = lapse_during_gate # type: ignore[method-assign] @@ -760,8 +781,8 @@ def test_fingerprint_cancel_keeps_pinned_published_build(self) -> None: superseded_monotonic=None, ) ) - with server._job_build_scheduler_lock: - server._job_build_active = pinned_flight # type: ignore[assignment] + with server._ensure_job_bundle_service()._scheduler_lock: + server._ensure_job_bundle_service()._active = pinned_flight # type: ignore[assignment] try: # The detected-tip store prunes fingerprints and sweeps stale # builds, but an in-flight build for exactly the published @@ -769,8 +790,8 @@ def test_fingerprint_cancel_keeps_pinned_published_build(self) -> None: server.fetch_qbit_tip_template_snapshot() self.assertFalse(pinned_cancellation.is_set()) finally: - with server._job_build_scheduler_lock: - server._job_build_active = None + with server._ensure_job_bundle_service()._scheduler_lock: + server._ensure_job_bundle_service()._active = None finally: server.shutdown_tip_refresh_executor() @@ -793,8 +814,9 @@ def test_issuance_fetch_racing_ahead_of_detection_repins_published(self) -> None next_tip = "cc" * 32 rpc.tip = next_tip rpc.template = base_template(height=11, prevhash=next_tip) - with server._job_cache_lock: - server._template_artifacts = None + server._ensure_job_bundle_service().template_repository.replace_for_test( + None + ) artifacts = server.job_issuance_template_artifacts() # The fetch is recorded as a detection and issuance stays on the @@ -846,14 +868,14 @@ def record_send(payload: dict[str, object]) -> None: # gate's first-delivery reservation exactly as a fresh payout # publication does, so a non-priority same-generation delivery # would be rejected rather than admitted. - with server._job_cache_lock: - server._published_payout_state = dataclasses.replace( - server._published_payout_state, + with server._payout_state_service._lock: + server._payout_state_service._published = dataclasses.replace( + server._payout_state_service._published, source_tip_hash=next_tip, ) - gate = server._payout_state_delivery_gate + gate = server._payout_state_service._delivery_gate with gate._condition: - gate._priority_generation = int(server._payout_state_generation) + gate._priority_generation = int(server._payout_state_service._generation) state.active_job = None state.active_job_ids = set() @@ -912,7 +934,7 @@ def needs_after_initial_selection( self.assertGreaterEqual(calls["count"], 2) self.assertEqual(sent_notifies[-1], published_tip[0]) self.assertEqual(server.current_tip_first_seen[0], published_tip[0]) - self.assertIsNone(server._active_tip_refresh) + self.assertIsNone(server._ensure_tip_refresh_service().active_refresh_snapshot()) finally: server.shutdown_tip_refresh_executor() @@ -942,7 +964,8 @@ def test_tip_flip_publication_clears_mismatched_parent_on_lookup_failure(self) - mark_pending=False, ) ) - snapshot = server.fetch_qbit_tip_template_snapshot() + with server._ensure_tip_refresh_service().suppress_trigger_callbacks_for_test(): + snapshot = server.fetch_qbit_tip_template_snapshot() bundle = server.prepare_tip_refresh_bundle(snapshot) token = server._validate_prepared_tip_refresh(bundle, snapshot, sequence) cancellation = server._publish_prepared_tip_refresh( diff --git a/tests/test_prism_tip_refresh.py b/tests/test_prism_tip_refresh.py new file mode 100644 index 0000000..a6bd54f --- /dev/null +++ b/tests/test_prism_tip_refresh.py @@ -0,0 +1,2194 @@ +#!/usr/bin/env python3 +"""Direct payout-state service and port-contract tests for the tip pipeline.""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, replace as dataclass_replace +import json +from types import SimpleNamespace +import threading +import unittest +from unittest.mock import patch + +from lab.prism.payout_state import ( + PayoutStateConfig, + PayoutStatePorts, + PayoutStateService, + TemplateRefreshBlocked, + TemplateRefreshSuperseded, +) +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.template_artifacts import QbitTipTemplateSnapshot +from lab.prism.template_artifacts import ( + CachedTemplateArtifacts, + qbit_template_fingerprint, +) +from lab.prism.tip_refresh import ( + FanoutCancellation, + RefreshClientTarget, + RefreshResult, + TipRefreshConfig, + TipRefreshPorts, + TipRefreshService, +) + + +@dataclass(frozen=True) +class _Share: + sequence: int + + def to_prism_json(self) -> dict[str, object]: + return {"sequence": self.sequence} + + +class _ServiceFixture: + def __init__(self) -> None: + self.now = 100.0 + self.accepted_count = 1 + self.invalidated: list[tuple[int, float]] = [] + self.published: list[tuple[int, float]] = [] + self.cache_invalidations = 0 + self.refresh_retries = 0 + self.phases: list[tuple[str, float]] = [] + ports = PayoutStatePorts( + accepted_share_stats=lambda: (self.accepted_count, 1), + snapshot_at_job_issue=lambda _anchor, _window: [_Share(1)], + current_prior_balances=lambda: [ + { + "recipient_id": "miner-a", + "order_key": "miner-a", + "p2mr_program_hex": "11" * 32, + "balance_sats": 5, + "metadata": {"durability": ["accepted", "fenced"]}, + } + ], + snapshot_anchor_ms=lambda value: value - 1, + current_template_network_difficulty=lambda: None, + pool_ready=lambda: False, + record_build_phase=lambda phase, elapsed: self.phases.append( + (phase, elapsed) + ), + invalidate_job_cache=self._invalidate_cache, + clear_retained_collection_refresh=lambda: None, + cancel_obsolete_job_builds=lambda _reason: None, + cancel_obsolete_bundle_builds=lambda _generation: None, + payout_invalidated=lambda generation, stamp: self.invalidated.append( + (generation, stamp) + ), + payout_published=lambda generation, stamp: self.published.append( + (generation, stamp) + ), + schedule_refresh_retry=self._schedule_retry, + chain_block_hash=lambda _height: "aa" * 32, + stop_requested=lambda: False, + ) + self.service = PayoutStateService( + ports, + monotonic=lambda: self.now, + wall_time_ms=lambda: 1_700_000_000_000, + config=PayoutStateConfig( + accepted_block_preview_wait_seconds=0.0, + reconcile_supersession_retries=2, + ), + ) + + def _invalidate_cache(self) -> None: + self.cache_invalidations += 1 + + def _schedule_retry(self) -> None: + self.refresh_retries += 1 + + +class PayoutStateServiceTests(unittest.TestCase): + def test_invalidation_and_publication_emit_exact_generation_boundary( + self, + ) -> None: + fixture = _ServiceFixture() + service = fixture.service + service.reserve_source( + "payout_only", + invalidated_monotonic=42.5, + ) + service.block_publication(force=True) + published = service.publish_candidate(service.current_candidate()) + + self.assertEqual(published, 1) + self.assertEqual(fixture.invalidated, [(1, 42.5)]) + self.assertEqual(fixture.published, [(1, 42.5)]) + snapshot = service.snapshot() + self.assertEqual(snapshot.generation, 1) + self.assertFalse(snapshot.publication_blocked) + self.assertIsNotNone(snapshot.published.artifact) + self.assertEqual(fixture.cache_invalidations, 2) + + def test_stale_source_candidate_cannot_publish(self) -> None: + fixture = _ServiceFixture() + service = fixture.service + service.reserve_source("first", invalidated_monotonic=1.0) + stale = service.current_candidate() + service.reserve_source("second", invalidated_monotonic=2.0) + + self.assertIsNone(service.publish_candidate(stale)) + self.assertEqual(service.snapshot().generation, 0) + self.assertEqual( + service.metrics_snapshot()["discarded_candidates"], + 1, + ) + + def test_first_delivery_priority_survives_publication(self) -> None: + fixture = _ServiceFixture() + service = fixture.service + service.reserve_source("payout", invalidated_monotonic=1.0) + service.block_publication(force=True) + self.assertEqual(service.publish_candidate(service.current_candidate()), 1) + + with service.delivery( + 1, + cancelled=lambda: False, + priority=False, + ) as routine: + self.assertFalse(routine) + with service.delivery( + 1, + cancelled=lambda: False, + priority=True, + ) as first: + self.assertTrue(first) + first.mark_delivered() + with service.delivery( + 1, + cancelled=lambda: False, + priority=False, + ) as routine_after_first: + self.assertTrue(routine_after_first) + + def test_landed_preview_withdrawal_leaves_fail_closed_tombstone(self) -> None: + fixture = _ServiceFixture() + service = fixture.service + block_hash = "aa" * 32 + service.begin_accepted_block_preview(block_hash, block_height=10) + service.mark_accepted_block_landed(block_hash, block_height=10) + service.clear_accepted_block_preview( + block_hash, + invalidate_published=True, + ) + + self.assertNotIn(block_hash, service.previews) + self.assertEqual(service.invalidated_previews, {block_hash: 10}) + with self.assertRaisesRegex(TemplateRefreshBlocked, "was withdrawn"): + service.prior_balances_for_parent(block_hash, parent_height=10) + self.assertEqual(fixture.refresh_retries, 1) + + def test_ledger_artifact_uses_pending_share_anchor_port(self) -> None: + fixture = _ServiceFixture() + artifact = fixture.service.build_ledger_artifact(0, 0, 100) + + self.assertIsNotNone(artifact) + assert artifact is not None + self.assertEqual(artifact.snapshot_anchor_ms, 1_699_999_999_999) + self.assertEqual(artifact.accepted_share_count, 1) + self.assertEqual(artifact.shares_json, ({"sequence": 1},)) + self.assertEqual( + [phase for phase, _elapsed in fixture.phases], + ["ledger_snapshot", "serialization_copy"], + ) + + def test_ledger_artifact_snapshot_rejects_deep_mutation(self) -> None: + fixture = _ServiceFixture() + service = fixture.service + service.prepare_ledger_artifact(0, 100) + + first = service.snapshot().ledger_artifact + assert first is not None + with self.assertRaisesRegex(TypeError, "immutable"): + first.shares_json[0]["sequence"] = 99 + with self.assertRaisesRegex(TypeError, "immutable"): + first.prior_balances[0]["balance_sats"] = 99 + metadata = first.prior_balances[0]["metadata"] + assert isinstance(metadata, dict) + durability = metadata["durability"] + assert isinstance(durability, list) + with self.assertRaisesRegex(TypeError, "immutable"): + durability.append("corrupt") + + second = service.snapshot().ledger_artifact + assert second is not None + self.assertEqual(second.shares_json[0]["sequence"], 1) + self.assertEqual(second.prior_balances[0]["balance_sats"], 5) + self.assertEqual( + second.prior_balances[0]["metadata"], + {"durability": ["accepted", "fenced"]}, + ) + self.assertEqual( + json.loads( + json.dumps( + { + "shares": second.shares_json, + "prior_balances": second.prior_balances, + } + ) + )["prior_balances"][0]["metadata"], + {"durability": ["accepted", "fenced"]}, + ) + + +class _TipPayoutGate: + @contextmanager + def delivery_cancelable( + self, + _cancelled: object, + *, + generation: int, + priority: bool = False, + ) -> object: + del generation, priority + yield True + + +class _TipPayout: + def __init__(self) -> None: + self.generation = 0 + self.delivery_gate = _TipPayoutGate() + self.reservations: list[tuple[str, str, float]] = [] + + def snapshot(self) -> object: + return SimpleNamespace( + generation=self.generation, + publication_blocked=False, + published=SimpleNamespace(artifact=None), + ) + + def reserve_source_for_tip_change( + self, + tip_hash: str, + *, + cause: str, + invalidated_monotonic: float, + ) -> int: + self.reservations.append((tip_hash, cause, invalidated_monotonic)) + return self.generation + + +class _TipJobs: + def __init__(self, events: list[str]) -> None: + self.events = events + self.ready = False + self.readiness_checks = 0 + self.prepared_clears = 0 + + def begin_priority_preparation( + self, + requested_monotonic: float | None = None, + ) -> tuple[int, float]: + return 1, 0.0 if requested_monotonic is None else requested_monotonic + + def finish_priority_preparation(self, _token: int) -> None: + return None + + def ready_latched(self) -> bool: + return self.ready + + def clear_prepared_ready(self) -> None: + self.prepared_clears += 1 + + def record_failure(self) -> None: + return None + + def pool_readiness_latched(self) -> bool: + self.events.append("readiness") + self.readiness_checks += 1 + return self.ready + + def set_preparation_pending(self, _pending: bool) -> None: + return None + + def set_prepared_ready(self, _snapshot: object, _bundle: object) -> None: + return None + + +class _TipDelivery: + def eligible_clients(self) -> tuple[object, ...]: + return () + + def client_can_receive_jobs(self, _client: object) -> bool: + return False + + def client_needs_refresh( + self, + _client: object, + _snapshot: QbitTipTemplateSnapshot, + ) -> bool: + return False + + def active_job(self, _client: object) -> object | None: + return None + + def connection_id(self, _client: object) -> int: + return 0 + + def delivery_priority( + self, + _client: object, + _snapshot: QbitTipTemplateSnapshot, + _expected_active_job: object | None, + ) -> int: + return 0 + + def select_targets( + self, + _snapshot: QbitTipTemplateSnapshot, + *, + refresh_all: bool, + ) -> tuple[RefreshClientTarget, ...]: + del refresh_all + return () + + def merge_poll_start_targets( + self, + targets: tuple[RefreshClientTarget, ...], + _poll_start_clients: tuple[object, ...], + _snapshot: QbitTipTemplateSnapshot, + *, + refresh_all: bool, + ) -> tuple[RefreshClientTarget, ...]: + del refresh_all + return targets + + def revalidate_targets( + self, + targets: tuple[RefreshClientTarget, ...], + _snapshot: QbitTipTemplateSnapshot, + ) -> tuple[tuple[RefreshClientTarget, ...], tuple[str, ...]]: + return targets, () + + def deliver_collection( + self, + _client: object, + _snapshot: QbitTipTemplateSnapshot, + _observation_sequence: int, + ) -> RefreshResult: + return RefreshResult("skipped") + + def take_post_accept_refresh(self, _client: object) -> tuple[int, str] | None: + return None + + +class _RefreshActivity: + def __init__(self, events: list[str]) -> None: + self.events = events + + def note_activity(self, _observed_monotonic: float | None = None) -> None: + self.events.append("activity") + + def finish(self) -> None: + self.events.append("finish") + + +class _TipRefreshFixture: + TIP_A = "aa" * 32 + TIP_B = "bb" * 32 + TIP_C = "cc" * 32 + + def __init__(self) -> None: + self.now = 100.0 + self.tip = self.TIP_A + self.events: list[str] = [] + self.removed_heartbeats: list[str] = [] + self.bundle_cancellations: list[tuple[str | None, int | None]] = [] + self.payout = _TipPayout() + self.jobs = _TipJobs(self.events) + self.delivery = _TipDelivery() + self.snapshot = self.make_snapshot(self.tip, "11" * 32) + self.fetch_error: Exception | None = None + self.service = TipRefreshService( + TipRefreshConfig( + blockpoll_seconds=1.0, + blockwait_timeout_seconds=5.0, + failure_holdoff_seconds=0.0, + max_workers=2, + submit_tip_max_age_seconds=10.0, + failure_exit_seconds=30.0, + watchdog_timeout_seconds=120.0, + ), + TipRefreshPorts( + rpc_call=self.rpc_call, + rpc_call_with_timeout=lambda method, params, timeout: self.rpc_call( + method, params + ), + payout_state=lambda: self.payout, + job_bundles=lambda: self.jobs, + delivery=self.delivery, + mark_progress_pending=lambda _stamp: self.events.append("pending"), + observe_progress_tip_poll=lambda _snapshot: self.events.append( + "coherent" + ), + publish_progress_work=lambda _snapshot, _generation: self.events.append( + "publish" + ), + start_progress_refresh=self.start_refresh, + cancel_obsolete_bundle_builds=self.cancel_bundles, + cancel_obsolete_job_builds=lambda _reason: self.events.append( + "cancel-jobs" + ), + prune_evicted_jobs=lambda _now, _force: None, + delivery_queue_limit=lambda: 4, + stop_requested=lambda: False, + heartbeat=lambda _name: None, + remove_heartbeat=self.removed_heartbeats.append, + chain_view_untrusted=lambda: False, + ensure_reorg_current=lambda _tip: True, + observe_job_build_elapsed=lambda _elapsed, _phases: None, + fetch_snapshot=self.fetch_snapshot, + ensure_reorg_tip=lambda _tip: True, + wait_for_execution_permit=lambda _timeout: True, + wait_for_stop=lambda _seconds: False, + hard_exit=lambda code: (_ for _ in ()).throw(SystemExit(code)), + ), + monotonic=lambda: self.now, + ) + + @staticmethod + def make_snapshot(tip: str, fingerprint: str) -> QbitTipTemplateSnapshot: + return QbitTipTemplateSnapshot( + bestblockhash=tip, + previousblockhash=tip, + template_fingerprint=fingerprint, + ) + + def rpc_call(self, method: str, _params: list[object] | None) -> object: + if method == "getbestblockhash": + return self.tip + if method == "getblock": + return {"previousblockhash": "00" * 32} + raise AssertionError(method) + + def fetch_snapshot(self) -> QbitTipTemplateSnapshot: + if self.fetch_error is not None: + raise self.fetch_error + return self.snapshot + + def start_refresh(self) -> _RefreshActivity: + self.events.append("start") + return _RefreshActivity(self.events) + + def cancel_bundles( + self, + tip_hash: str | None, + payout_generation: int | None, + ) -> None: + self.bundle_cancellations.append((tip_hash, payout_generation)) + self.events.append("cancel-bundles") + + +class TipRefreshServiceTests(unittest.TestCase): + @staticmethod + def _scheduler_trigger( + fixture: _TipRefreshFixture, + *, + sequence: int, + tip: str, + payout_generation: int = 0, + ready_required: bool = False, + reason: str = "blockpoll", + pending_token: int | None = None, + ) -> object: + return fixture.service._new_trigger( + observation_sequence=sequence, + tip_hash=tip, + payout_state_generation=payout_generation, + ready_required=ready_required, + reasons=(reason,), + pending_signal_token=pending_token, + ) + + def test_observation_effects_cannot_reorder_behind_a_newer_tip(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue(service.publish_tip(fixture.TIP_A, observation_sequence=1)) + + older_effect_started = threading.Event() + release_older_effect = threading.Event() + newer_call_started = threading.Event() + newer_call_finished = threading.Event() + original_reserve = fixture.payout.reserve_source_for_tip_change + + def pause_older_reservation( + tip_hash: str, + *, + cause: str, + invalidated_monotonic: float, + ) -> int: + if tip_hash == fixture.TIP_B: + older_effect_started.set() + self.assertTrue(release_older_effect.wait(2.0)) + return original_reserve( + tip_hash, + cause=cause, + invalidated_monotonic=invalidated_monotonic, + ) + + fixture.payout.reserve_source_for_tip_change = pause_older_reservation # type: ignore[method-assign] + results: list[tuple[str, bool]] = [] + errors: list[BaseException] = [] + + def observe(tip_hash: str, sequence: int) -> None: + try: + if tip_hash == fixture.TIP_C: + newer_call_started.set() + results.append( + (tip_hash, service.observe_tip(tip_hash, observation_sequence=sequence)) + ) + except BaseException as exc: # pragma: no cover - asserted below + errors.append(exc) + finally: + if tip_hash == fixture.TIP_C: + newer_call_finished.set() + + older = threading.Thread(target=observe, args=(fixture.TIP_B, 2)) + newer = threading.Thread(target=observe, args=(fixture.TIP_C, 3)) + older.start() + self.assertTrue(older_effect_started.wait(2.0)) + newer.start() + self.assertTrue(newer_call_started.wait(2.0)) + self.assertFalse(newer_call_finished.wait(0.05)) + release_older_effect.set() + older.join(2.0) + newer.join(2.0) + + self.assertFalse(older.is_alive()) + self.assertFalse(newer.is_alive()) + self.assertEqual(errors, []) + self.assertCountEqual( + results, + [(fixture.TIP_B, True), (fixture.TIP_C, True)], + ) + self.assertEqual( + [tip_hash for tip_hash, _cause, _stamp in fixture.payout.reservations], + [fixture.TIP_B, fixture.TIP_C], + ) + self.assertEqual( + fixture.bundle_cancellations, + [(fixture.TIP_B, None), (fixture.TIP_C, None)], + ) + self.assertEqual( + service.snapshot().latest_detected_tip, + (fixture.TIP_C, 3), + ) + + def test_monotonic_observation_rejects_reordered_tip(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + + self.assertTrue(service.observe_tip(fixture.TIP_A, observation_sequence=2)) + self.assertFalse(service.observe_tip(fixture.TIP_B, observation_sequence=1)) + self.assertTrue(service.observe_tip(fixture.TIP_A, observation_sequence=1)) + self.assertEqual(service.snapshot().latest_detected_tip, (fixture.TIP_A, 2)) + + def test_pending_clear_requires_exact_completion_token(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + snapshot = fixture.snapshot + self.assertTrue( + service.publish_tip( + fixture.tip, + observation_sequence=1, + published_snapshot=snapshot, + ) + ) + stale_token = service.mark_pending() + current_token = service.mark_pending() + + self.assertFalse( + service.clear_pending_for_completed_refresh( + snapshot, + 1, + fixture.payout.generation, + stale_token, + ) + ) + self.assertEqual(service.snapshot().pending_token, current_token) + self.assertTrue( + service.clear_pending_for_completed_refresh( + snapshot, + 1, + fixture.payout.generation, + current_token, + ) + ) + self.assertFalse(service.snapshot().pending) + + def test_divergence_lease_is_anchored_to_first_departure(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue(service.publish_tip(fixture.TIP_A, observation_sequence=1)) + + fixture.now = 110.0 + self.assertTrue(service.observe_tip(fixture.TIP_B, observation_sequence=2)) + self.assertEqual(service.snapshot().divergence_started_monotonic, 110.0) + fixture.now = 120.0 + self.assertTrue(service.observe_tip(fixture.TIP_C, observation_sequence=3)) + self.assertEqual(service.snapshot().divergence_started_monotonic, 110.0) + fixture.now = 130.0 + self.assertTrue(service.observe_tip(fixture.TIP_A, observation_sequence=4)) + self.assertEqual(service.snapshot().divergence_started_monotonic, 110.0) + self.assertTrue(service.publication_failure_expired(140.0)) + self.assertTrue(service.publish_tip(fixture.TIP_A, observation_sequence=4)) + self.assertIsNone(service.snapshot().divergence_started_monotonic) + self.assertFalse(service.publication_failure_expired(10_000.0)) + + def test_post_accept_failure_wakes_retry_without_counting_supersession( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + trigger = dataclass_replace( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A), + post_accept_block=(10, fixture.TIP_B), + post_accept_admission_sequence=1, + ) + + with patch("lab.prism.tip_refresh.traceback.print_exception"): + service._handle_scheduled_failure(trigger, RuntimeError("rpc failed")) + self.assertTrue(service.snapshot().retry_requested) + self.assertEqual(service.snapshot().post_accept_refresh_failure_count, 1) + + service.clear_retry_for_test() + service._handle_scheduled_failure( + trigger, + TemplateRefreshSuperseded("newer observation"), + ) + self.assertTrue(service.snapshot().retry_requested) + self.assertEqual(service.snapshot().post_accept_refresh_failure_count, 1) + + def test_post_accept_notification_stamps_heartbeat_and_runs_scheduler( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + heartbeats: list[str] = [] + service.reconfigure_ports_for_test(heartbeat=heartbeats.append) + + self.assertEqual( + service.refresh_after_accepted_block( + block_height=10, + block_hash=fixture.TIP_B, + heartbeat_name="block_submitter", + ), + 0, + ) + + self.assertEqual(heartbeats[0], "block_submitter") + self.assertGreaterEqual(heartbeats.count("block_submitter"), 2) + self.assertIn("tip_refresh_scheduler", heartbeats) + self.assertFalse(service.snapshot().retry_requested) + self.assertEqual(service.snapshot().post_accept_refresh_failure_count, 0) + + def test_post_accept_rpc_failure_wakes_driver_and_counts_failure(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + service.reconfigure_ports_for_test( + rpc_call=lambda _method, _params: (_ for _ in ()).throw( + RuntimeError("best-tip unavailable") + ) + ) + + with patch("lab.prism.tip_refresh.traceback.print_exc"): + self.assertEqual( + service.refresh_after_accepted_block( + block_height=10, + block_hash=fixture.TIP_B, + ), + 0, + ) + + self.assertTrue(service.snapshot().retry_requested) + self.assertEqual(service.snapshot().post_accept_refresh_failure_count, 1) + + def test_post_accept_supersession_wakes_driver_without_counting_failure( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + service.observe_tip = lambda *_args, **_kwargs: False # type: ignore[method-assign] + + self.assertEqual( + service.refresh_after_accepted_block( + block_height=10, + block_hash=fixture.TIP_B, + ), + 0, + ) + + self.assertTrue(service.snapshot().retry_requested) + self.assertEqual(service.snapshot().post_accept_refresh_failure_count, 0) + + def test_retry_generation_preserves_each_visible_wake(self) -> None: + service = _TipRefreshFixture().service + + self.assertFalse(service.consume_retry()) + service.schedule_retry() + self.assertTrue(service.consume_retry()) + self.assertFalse(service.consume_retry()) + service.schedule_retry() + self.assertTrue(service.consume_retry()) + self.assertFalse(service.consume_retry()) + + def test_fanout_cancellation_closes_admission_then_drains(self) -> None: + cancellation = FanoutCancellation() + self.assertTrue(cancellation.begin_delivery()) + drained = threading.Event() + + def cancel_and_drain() -> None: + cancellation.set() + drained.set() + + thread = threading.Thread(target=cancel_and_drain) + thread.start() + self.assertFalse(drained.wait(0.05)) + self.assertFalse(cancellation.begin_delivery()) + cancellation.end_delivery() + self.assertTrue(drained.wait(1.0)) + thread.join(1.0) + self.assertFalse(thread.is_alive()) + + def test_failure_budget_excludes_coordination_supersession(self) -> None: + coordination = _TipRefreshFixture() + coordination.fetch_error = TemplateRefreshSuperseded("newer observation") + with self.assertRaises(TemplateRefreshSuperseded): + coordination.service.poll_once() + self.assertIsNone( + coordination.service.snapshot().failure_started_monotonic + ) + + unhealthy = _TipRefreshFixture() + unhealthy.fetch_error = TemplateRefreshBlocked("invalid template") + with self.assertRaises(TemplateRefreshBlocked): + unhealthy.service.poll_once() + self.assertEqual( + unhealthy.service.snapshot().failure_started_monotonic, + unhealthy.now, + ) + + def test_same_tip_refresh_rechecks_readiness_without_pending_work(self) -> None: + fixture = _TipRefreshFixture() + fixture.jobs.ready = True + self.assertEqual(fixture.service.poll_once(), 0) + fixture.snapshot = fixture.make_snapshot(fixture.tip, "22" * 32) + self.assertEqual(fixture.service.poll_once(), 0) + + state = fixture.service.snapshot() + self.assertFalse(state.pending) + self.assertFalse(state.retry_requested) + self.assertEqual(fixture.jobs.readiness_checks, 2) + self.assertEqual(fixture.jobs.prepared_clears, 0) + self.assertIs( + fixture.service.published_snapshot().template, + fixture.snapshot, + ) + + def test_g1_progress_order_follows_coherence_and_publication(self) -> None: + fixture = _TipRefreshFixture() + fixture.jobs.ready = True + + self.assertEqual(fixture.service.poll_once(), 0) + + self.assertEqual( + fixture.events, + ["coherent", "start", "readiness", "publish", "coherent", "finish"], + ) + + def test_detector_thread_only_enqueues_scheduler_refresh_work(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + executed_by: list[str] = [] + results: list[int] = [] + + def execute(_trigger: object) -> int: + executed_by.append(threading.current_thread().name) + return 0 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + detector = threading.Thread( + target=lambda: results.append(service.poll_once()), + name="test-blockpoll-detector", + ) + detector.start() + detector.join(1.0) + + self.assertFalse(detector.is_alive()) + self.assertEqual(results, [0]) + self.assertEqual(executed_by, ["prism-tip-refresh-scheduler"]) + self.assertTrue(service.shutdown()) + + def test_blockpoll_await_renews_caller_heartbeat_while_worker_is_blocked( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + execute_started = threading.Event() + release_execute = threading.Event() + repeated_heartbeat = threading.Event() + heartbeats: list[str] = [] + executed_by: list[str] = [] + + def heartbeat(name: str) -> None: + heartbeats.append(name) + if heartbeats.count("qbit_blockpoll") >= 3: + repeated_heartbeat.set() + + def execute(_trigger: object) -> int: + executed_by.append(threading.current_thread().name) + execute_started.set() + self.assertTrue(release_execute.wait(2.0)) + return 0 + + service.reconfigure_ports_for_test(heartbeat=heartbeat) + service._execute_refresh_trigger = execute # type: ignore[method-assign] + results: list[int] = [] + caller = threading.Thread(target=lambda: results.append(service.poll_once())) + caller.start() + self.assertTrue(execute_started.wait(1.0)) + self.assertTrue(repeated_heartbeat.wait(1.0)) + release_execute.set() + caller.join(1.0) + + self.assertFalse(caller.is_alive()) + self.assertEqual(results, [0]) + self.assertEqual(executed_by, ["prism-tip-refresh-scheduler"]) + self.assertGreaterEqual(heartbeats.count("qbit_blockpoll"), 3) + self.assertTrue(service.shutdown()) + + def test_blockwait_changed_tip_only_notifies_refresh_driver(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + stop = threading.Event() + heartbeats: list[str] = [] + executions: list[object] = [] + + def heartbeat(name: str) -> None: + heartbeats.append(name) + + def blockwait_once(_known_tip: str) -> str: + fixture.tip = fixture.TIP_B + fixture.snapshot = fixture.make_snapshot(fixture.TIP_B, "22" * 32) + stop.set() + return fixture.TIP_B + + def execute(trigger: object) -> int: + executions.append(trigger) + return 0 + + service.blockwait_once = blockwait_once # type: ignore[method-assign] + service._execute_refresh_trigger = execute # type: ignore[method-assign] + service.reconfigure_ports_for_test( + heartbeat=heartbeat, + stop_requested=stop.is_set, + wait_for_stop=stop.wait, + ) + caller = threading.Thread(target=service.blockwait_loop) + caller.start() + caller.join(1.0) + + self.assertFalse(caller.is_alive()) + self.assertEqual(executions, []) + self.assertEqual(service.newest_observed_tip(), fixture.TIP_B) + self.assertTrue(service.snapshot().pending) + self.assertTrue(service.snapshot().retry_requested) + self.assertEqual(heartbeats, ["qbit_blockwait"]) + metrics = service.metrics_snapshot() + self.assertEqual(metrics["trigger_latency"]["count"], 0) # type: ignore[index] + self.assertEqual(metrics["trigger_coalesces"], 0) + self.assertEqual(metrics["trigger_supersessions"], 0) + self.assertTrue(service.shutdown()) + + def test_scheduler_waits_for_refresh_execution_permit(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + permit_wait_started = threading.Event() + permit = threading.Event() + + def wait_for_permit(timeout_seconds: float) -> bool: + permit_wait_started.set() + return permit.wait(timeout_seconds) + + service.reconfigure_ports_for_test( + wait_for_execution_permit=wait_for_permit, + ) + completion = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + + self.assertTrue(permit_wait_started.wait(1.0)) + self.assertFalse(completion.done()) + self.assertEqual(fixture.events, []) + permit.set() + self.assertEqual(completion.result(timeout=1.0), 0) + self.assertTrue(service.shutdown()) + self.assertIn("tip_refresh_scheduler", fixture.removed_heartbeats) + + def test_newer_trigger_supersedes_active_while_execution_permit_is_blocked( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + permit_wait_started = threading.Event() + permit = threading.Event() + + def wait_for_permit(timeout_seconds: float) -> bool: + permit_wait_started.set() + return permit.wait(timeout_seconds) + + service.reconfigure_ports_for_test( + wait_for_execution_permit=wait_for_permit, + ) + first = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(permit_wait_started.wait(1.0)) + + fixture.tip = fixture.TIP_B + fixture.snapshot = fixture.make_snapshot(fixture.TIP_B, "22" * 32) + second = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=2, tip=fixture.TIP_B) # type: ignore[arg-type] + ) + scheduler = service.scheduler_snapshot() + self.assertEqual(scheduler.active.tip_hash, fixture.TIP_A) # type: ignore[union-attr] + self.assertEqual(scheduler.pending.tip_hash, fixture.TIP_B) # type: ignore[union-attr] + + with self.assertRaises(TemplateRefreshSuperseded): + first.result(timeout=1.0) + self.assertFalse(second.done()) + self.assertEqual(fixture.events, []) + + permit.set() + self.assertEqual(second.result(timeout=1.0), 0) + self.assertEqual(service.published_snapshot().tip_hash, fixture.TIP_B) + self.assertEqual(fixture.events.count("coherent"), 2) + self.assertTrue(service.shutdown()) + + def test_stale_retained_wake_cannot_roll_back_blocked_live_tip(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + template = {"previousblockhash": fixture.TIP_A, "transactions": []} + fingerprint = qbit_template_fingerprint(template) + artifacts = CachedTemplateArtifacts( + template=template, + fingerprint=fingerprint, + previousblockhash=fixture.TIP_A, + transaction_hexes=(), + witness_merkle_leaves_hex=(), + network_difficulty=1, + fetched_monotonic=fixture.now, + generation=4, + ) + retained_snapshot = QbitTipTemplateSnapshot( + bestblockhash=fixture.TIP_A, + previousblockhash=fixture.TIP_A, + template_fingerprint=fingerprint, + template_generation=4, + template_artifacts=artifacts, + ) + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=retained_snapshot, + ) + ) + service.retain_collection_refresh(retained_snapshot, 1, 0) + self.assertIsNotNone(service.retained_collection_refresh_snapshot()) + + permit_wait_started = threading.Event() + permit = threading.Event() + executed: list[str | None] = [] + original_execute = service._execute_refresh_trigger + + def wait_for_permit(timeout_seconds: float) -> bool: + permit_wait_started.set() + return permit.wait(timeout_seconds) + + def record_execute(trigger: object) -> int: + executed.append(getattr(trigger, "tip_hash")) + return original_execute(trigger) # type: ignore[arg-type] + + service.reconfigure_ports_for_test(wait_for_execution_permit=wait_for_permit) + service._execute_refresh_trigger = record_execute # type: ignore[method-assign] + fixture.tip = fixture.TIP_B + fixture.snapshot = fixture.make_snapshot(fixture.TIP_B, "22" * 32) + admission = service.submit_tip_observation_admission( + fixture.TIP_B, + reason="blockwait", + ) + assert admission.completion is not None + self.assertTrue(permit_wait_started.wait(1.0)) + sequence_before_wake = service.observation_sequence() + + client = object() + fixture.delivery.client_can_receive_jobs = lambda _client: True # type: ignore[method-assign] + fixture.delivery.eligible_clients = lambda: (client,) # type: ignore[method-assign] + service.note_collection_identity_available(client) + + scheduler = service.scheduler_snapshot() + self.assertEqual(scheduler.active.tip_hash, fixture.TIP_B) # type: ignore[union-attr] + self.assertIsNone(scheduler.pending) + self.assertEqual(service.observation_sequence(), sequence_before_wake) + permit.set() + self.assertEqual(admission.completion.result(timeout=1.0), 0) + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + self.assertEqual(executed, [fixture.TIP_B]) + self.assertEqual(service.published_snapshot().tip_hash, fixture.TIP_B) + self.assertTrue(service.shutdown()) + + def test_exact_template_and_payout_merge_on_current_chain_axis(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + active_started = threading.Event() + release_active = threading.Event() + + def execute(trigger: object) -> int: + if int(getattr(trigger, "observation_sequence")) == 1: + active_started.set() + self.assertTrue(release_active.wait(2.0)) + service._raise_if_scheduler_superseded(trigger) # type: ignore[arg-type] + return 0 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + first = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + self.assertTrue( + service.observe_tip( + fixture.TIP_C, + observation_sequence=3, + mark_pending=False, + ) + ) + exact_snapshot = QbitTipTemplateSnapshot( + bestblockhash=fixture.TIP_C, + previousblockhash=fixture.TIP_C, + template_fingerprint="99" * 32, + template_generation=9, + ) + exact_token = service.mark_pending(9) + exact = service.submit_trigger( + service._new_trigger( + observation_sequence=3, + tip_hash=fixture.TIP_C, + payout_state_generation=2, + ready_required=False, + reasons=("template",), + pending_signal_token=exact_token, + snapshot=exact_snapshot, + ) + ) + fixture.payout.generation = 7 + service.payout_generation_changed(7) + pending = service.scheduler_snapshot().pending + assert pending is not None + self.assertEqual(pending.tip_hash, fixture.TIP_C) + self.assertEqual(pending.observation_sequence, 3) + self.assertEqual(pending.template_generation, 9) + self.assertIs(pending.snapshot, exact_snapshot) + self.assertEqual(pending.payout_state_generation, 7) + release_active.set() + with self.assertRaises(TemplateRefreshSuperseded): + first.result(timeout=1.0) + self.assertEqual(exact.result(timeout=1.0), 0) + self.assertTrue(service.shutdown()) + + def test_payout_admitted_between_live_observation_and_exact_template_submit( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + active_started = threading.Event() + release_active = threading.Event() + + def execute(trigger: object) -> int: + if int(getattr(trigger, "observation_sequence")) == 1: + active_started.set() + self.assertTrue(release_active.wait(2.0)) + service._raise_if_scheduler_superseded(trigger) # type: ignore[arg-type] + return 0 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + first = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + self.assertTrue( + service.observe_tip( + fixture.TIP_C, + observation_sequence=3, + mark_pending=False, + ) + ) + fixture.payout.generation = 8 + service.payout_generation_changed(8) + payout_pending = service.scheduler_snapshot().pending + assert payout_pending is not None + self.assertEqual(payout_pending.tip_hash, fixture.TIP_C) + self.assertEqual(payout_pending.observation_sequence, 3) + + exact_snapshot = QbitTipTemplateSnapshot( + bestblockhash=fixture.TIP_C, + previousblockhash=fixture.TIP_C, + template_fingerprint="99" * 32, + template_generation=9, + ) + exact_token = service.mark_pending(9) + exact = service.submit_trigger( + service._new_trigger( + observation_sequence=3, + tip_hash=fixture.TIP_C, + payout_state_generation=3, + ready_required=False, + reasons=("template",), + pending_signal_token=exact_token, + snapshot=exact_snapshot, + ) + ) + pending = service.scheduler_snapshot().pending + assert pending is not None + self.assertEqual(pending.tip_hash, fixture.TIP_C) + self.assertEqual(pending.observation_sequence, 3) + self.assertEqual(pending.template_generation, 9) + self.assertIs(pending.snapshot, exact_snapshot) + self.assertEqual(pending.payout_state_generation, 8) + release_active.set() + with self.assertRaises(TemplateRefreshSuperseded): + first.result(timeout=1.0) + self.assertEqual(exact.result(timeout=1.0), 0) + self.assertTrue(service.shutdown()) + + def test_admission_during_idle_heartbeat_removal_reuses_scheduler_worker( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + heartbeat_removal_started = threading.Event() + release_heartbeat_removal = threading.Event() + executed: list[tuple[str | None, int | None]] = [] + + def remove_heartbeat(name: str) -> None: + if name == "tip_refresh_scheduler" and not heartbeat_removal_started.is_set(): + heartbeat_removal_started.set() + self.assertTrue(release_heartbeat_removal.wait(2.0)) + + def execute(trigger: object) -> int: + executed.append( + ( + getattr(trigger, "tip_hash"), + threading.current_thread().ident, + ) + ) + return 0 + + service.reconfigure_ports_for_test(remove_heartbeat=remove_heartbeat) + service._execute_refresh_trigger = execute # type: ignore[method-assign] + first = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertEqual(first.result(timeout=1.0), 0) + self.assertTrue(heartbeat_removal_started.wait(1.0)) + + second = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=2, tip=fixture.TIP_B) # type: ignore[arg-type] + ) + scheduler = service.scheduler_snapshot() + self.assertTrue(scheduler.worker_alive) + self.assertIsNone(scheduler.active) + self.assertEqual(scheduler.pending.tip_hash, fixture.TIP_B) # type: ignore[union-attr] + release_heartbeat_removal.set() + + self.assertEqual(second.result(timeout=1.0), 0) + self.assertEqual([tip for tip, _ident in executed], [fixture.TIP_A, fixture.TIP_B]) + self.assertEqual(len({ident for _tip, ident in executed}), 1) + self.assertTrue(service.shutdown()) + + def test_nested_callback_suppression_survives_coherent_capture(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + + with service.suppress_trigger_callbacks_for_test(): + self.assertTrue(service._trigger_capture_local.active) + service.detect_poll_trigger() + self.assertTrue(service._trigger_capture_local.active) + + self.assertFalse(service._trigger_capture_local.active) + self.assertTrue(service.shutdown()) + + def test_scheduler_replaces_blocked_active_with_newer_tip(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + active_started = threading.Event() + release_active = threading.Event() + executed: list[str | None] = [] + + def execute(trigger: object) -> int: + tip_hash = getattr(trigger, "tip_hash") + executed.append(tip_hash) + if tip_hash == fixture.TIP_A: + active_started.set() + self.assertTrue(release_active.wait(2.0)) + service._raise_if_scheduler_superseded(trigger) # type: ignore[arg-type] + return 1 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + first = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + second = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=2, tip=fixture.TIP_B) # type: ignore[arg-type] + ) + snapshot = service.scheduler_snapshot() + self.assertEqual(snapshot.active.tip_hash, fixture.TIP_A) # type: ignore[union-attr] + self.assertEqual(snapshot.pending.tip_hash, fixture.TIP_B) # type: ignore[union-attr] + release_active.set() + with self.assertRaises(TemplateRefreshSuperseded): + first.result(timeout=1.0) + self.assertEqual(second.result(timeout=1.0), 1) + self.assertEqual(executed, [fixture.TIP_A, fixture.TIP_B]) + self.assertTrue(service.shutdown()) + + def test_same_tip_newer_payout_generation_survives_coalescing(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + active_started = threading.Event() + release_active = threading.Event() + executed: list[int] = [] + + def execute(trigger: object) -> int: + generation = int(getattr(trigger, "payout_state_generation")) + executed.append(generation) + if generation == 1: + active_started.set() + self.assertTrue(release_active.wait(2.0)) + service._raise_if_scheduler_superseded(trigger) # type: ignore[arg-type] + return generation + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + first = service.submit_trigger( + self._scheduler_trigger( + fixture, + sequence=4, + tip=fixture.TIP_A, + payout_generation=1, + ) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + second = service.submit_trigger( + self._scheduler_trigger( + fixture, + sequence=4, + tip=fixture.TIP_A, + payout_generation=2, + reason="payout", + ) # type: ignore[arg-type] + ) + release_active.set() + with self.assertRaises(TemplateRefreshSuperseded): + first.result(timeout=1.0) + self.assertEqual(second.result(timeout=1.0), 2) + self.assertEqual(executed, [1, 2]) + self.assertTrue(service.shutdown()) + + def test_payout_invalidation_defers_one_trigger_until_publication(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + trigger_started = threading.Event() + release_trigger = threading.Event() + executed: list[object] = [] + + def execute(trigger: object) -> int: + executed.append(trigger) + trigger_started.set() + self.assertTrue(release_trigger.wait(2.0)) + return 0 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + service.payout_generation_invalidated(1) + invalidated = service.snapshot() + invalidated_token = invalidated.pending_token + self.assertIsNotNone(invalidated_token) + self.assertFalse(invalidated.retry_requested) + self.assertIsNone(service.scheduler_snapshot().active) + self.assertIsNone(service.scheduler_snapshot().pending) + + fixture.payout.generation = 1 + service.payout_generation_changed(1) + self.assertTrue(trigger_started.wait(1.0)) + self.assertEqual(len(executed), 1) + trigger = executed[0] + self.assertEqual(getattr(trigger, "payout_state_generation"), 1) + self.assertEqual(getattr(trigger, "pending_signal_token"), invalidated_token) + self.assertEqual(getattr(trigger, "reasons"), ("payout",)) + self.assertEqual(service.snapshot().pending_token, invalidated_token) + self.assertFalse(service.snapshot().retry_requested) + + release_trigger.set() + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + self.assertEqual(service.metrics_snapshot()["trigger_latency"]["count"], 1) # type: ignore[index] + self.assertTrue(service.shutdown()) + + def test_accepted_writer_defers_payout_trigger_to_post_accept_marker(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + writer_active = True + service.reconfigure_ports_for_test( + wait_for_execution_permit=lambda _timeout: not writer_active, + ) + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + executed: list[object] = [] + service._execute_refresh_trigger = lambda trigger: executed.append(trigger) or 0 # type: ignore[method-assign] + + service.payout_generation_invalidated(1) + invalidated_token = service.snapshot().pending_token + fixture.payout.generation = 1 + service.payout_generation_changed(1) + + self.assertIsNotNone(invalidated_token) + self.assertEqual(executed, []) + self.assertIsNone(service.scheduler_snapshot().active) + self.assertIsNone(service.scheduler_snapshot().pending) + self.assertEqual(service.snapshot().pending_token, invalidated_token) + + writer_active = False + completion = service.submit_post_accept_trigger( + block_height=2, + block_hash=fixture.TIP_B, + ) + self.assertEqual(completion.result(timeout=1.0), 0) + self.assertEqual(len(executed), 1) + trigger = executed[0] + self.assertEqual(getattr(trigger, "reasons"), ("post_accept",)) + self.assertEqual(getattr(trigger, "payout_state_generation"), 1) + self.assertEqual(getattr(trigger, "pending_signal_token"), invalidated_token) + self.assertTrue(getattr(trigger, "fresh_capture_required")) + self.assertTrue(service.shutdown()) + + def test_immediate_producers_do_not_duplicate_live_blockpoll_work(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + stopped = False + detector_waiting = threading.Event() + detector_heartbeats: list[str] = [] + + def stop_requested() -> bool: + detector_waiting.set() + return stopped + + service.reconfigure_for_test(blockpoll_seconds=60.0) + service.reconfigure_ports_for_test( + stop_requested=stop_requested, + heartbeat=detector_heartbeats.append, + ) + executed: list[object] = [] + service._execute_refresh_trigger = lambda trigger: executed.append(trigger) or 0 # type: ignore[method-assign] + detector = threading.Thread( + target=service.blockpoll_loop, + name="test-live-blockpoll", + ) + detector.start() + self.assertTrue(detector_waiting.wait(1.0)) + + service.payout_generation_invalidated(1) + self.assertEqual(executed, []) + fixture.payout.generation = 1 + service.payout_generation_changed(1) + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + + artifacts = SimpleNamespace( + generation=2, + previousblockhash=fixture.TIP_A, + fingerprint="22" * 32, + ) + service.template_artifacts_changed(artifacts) # type: ignore[arg-type] + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + service.submit_tip_observation(fixture.TIP_B, reason="blockwait") + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + + self.assertEqual( + [getattr(trigger, "reasons") for trigger in executed], + [("payout",), ("template",), ("blockwait",)], + ) + self.assertIs(getattr(executed[1], "snapshot").template_artifacts, artifacts) + metrics = service.metrics_snapshot() + self.assertEqual(metrics["trigger_supersessions"], 0) + self.assertEqual(metrics["trigger_coalesces"], 0) + self.assertEqual(metrics["trigger_latency"]["count"], 3) # type: ignore[index] + self.assertEqual(detector_heartbeats, []) + + stopped = True + service.schedule_retry() + detector.join(1.0) + self.assertFalse(detector.is_alive()) + self.assertTrue(service.shutdown()) + + def test_async_capture_failure_arms_budget_but_supersession_does_not(self) -> None: + failed = _TipRefreshFixture() + failed_service = failed.service + self.assertTrue( + failed_service.publish_tip( + failed.TIP_A, + observation_sequence=1, + published_snapshot=failed.snapshot, + ) + ) + failed_service.reconfigure_ports_for_test( + fetch_snapshot=lambda: (_ for _ in ()).throw( + RuntimeError("template RPC unavailable") + ) + ) + failed_service.payout_generation_invalidated(1) + failed.payout.generation = 1 + failed_service.payout_generation_changed(1) + self.assertTrue(failed_service.wait_for_scheduler_idle_for_test()) + self.assertEqual(failed_service.snapshot().failure_started_monotonic, failed.now) + self.assertTrue(failed_service.shutdown()) + + superseded = _TipRefreshFixture() + superseded_service = superseded.service + self.assertTrue( + superseded_service.publish_tip( + superseded.TIP_A, + observation_sequence=1, + published_snapshot=superseded.snapshot, + ) + ) + superseded_service.reconfigure_ports_for_test( + fetch_snapshot=lambda: (_ for _ in ()).throw( + TemplateRefreshSuperseded("newer capture") + ) + ) + superseded_service.payout_generation_invalidated(1) + superseded.payout.generation = 1 + superseded_service.payout_generation_changed(1) + self.assertTrue(superseded_service.wait_for_scheduler_idle_for_test()) + self.assertIsNone( + superseded_service.snapshot().failure_started_monotonic + ) + self.assertTrue(superseded_service.shutdown()) + + def test_post_accept_context_never_becomes_detected_tip_authority(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + accepted_hash = fixture.TIP_C + fixture.tip = fixture.TIP_B + fixture.snapshot = fixture.make_snapshot(fixture.TIP_B, "22" * 32) + observed: list[str] = [] + original_observe = service.observe_tip + + def record_observe(tip_hash: str, **kwargs: object) -> bool: + observed.append(tip_hash) + return original_observe(tip_hash, **kwargs) # type: ignore[arg-type] + + service.observe_tip = record_observe # type: ignore[method-assign] + completion = service.submit_post_accept_trigger( + block_height=11, + block_hash=accepted_hash, + ) + + self.assertEqual(completion.result(timeout=1.0), 0) + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + self.assertNotIn(accepted_hash, observed) + self.assertEqual( + service.snapshot().latest_detected_tip, + (fixture.TIP_B, 2), + ) + self.assertEqual( + [tip_hash for tip_hash, _cause, _stamp in fixture.payout.reservations], + [fixture.TIP_B], + ) + published = service.published_snapshot() + self.assertEqual(published.tip_hash, fixture.TIP_B) + self.assertEqual(published.observation_sequence, 2) + self.assertTrue(service.shutdown()) + + def test_post_accept_after_captured_blockpoll_gets_fresh_followup(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + permit_wait_started = threading.Event() + permit = threading.Event() + executions: list[object] = [] + reports: list[tuple[tuple[int, str] | None, int]] = [] + original_execute = service._execute_refresh_trigger + + def wait_for_permit(timeout_seconds: float) -> bool: + permit_wait_started.set() + return permit.wait(timeout_seconds) + + def execute(trigger: object) -> int: + executions.append(trigger) + return original_execute(trigger) # type: ignore[arg-type] + + service.reconfigure_ports_for_test(wait_for_execution_permit=wait_for_permit) + service._execute_refresh_trigger = execute # type: ignore[method-assign] + service._handle_scheduled_success = ( # type: ignore[method-assign] + lambda trigger, result: reports.append( + (getattr(trigger, "post_accept_block"), result) + ) + ) + blockpoll = service.submit_trigger( + service._new_trigger( + observation_sequence=1, + tip_hash=fixture.TIP_A, + payout_state_generation=0, + ready_required=False, + reasons=("blockpoll",), + pending_signal_token=None, + snapshot=fixture.snapshot, + ) + ) + self.assertTrue(permit_wait_started.wait(1.0)) + post_accept = service.submit_post_accept_trigger( + block_height=10, + block_hash=fixture.TIP_C, + ) + + self.assertIsNot(post_accept, blockpoll) + scheduler = service.scheduler_snapshot() + pending = scheduler.pending + assert pending is not None + self.assertTrue(pending.fresh_capture_required) + self.assertIsNone(pending.snapshot) + self.assertIsNone(pending.template_generation) + self.assertEqual(pending.post_accept_block, (10, fixture.TIP_C)) + fixture.tip = fixture.TIP_B + fixture.snapshot = fixture.make_snapshot(fixture.TIP_B, "22" * 32) + permit.set() + + self.assertEqual(blockpoll.result(timeout=1.0), 0) + self.assertEqual(post_accept.result(timeout=1.0), 0) + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + self.assertEqual(len(executions), 2) + self.assertIsNotNone(getattr(executions[0], "snapshot")) + self.assertIsNone(getattr(executions[0], "post_accept_block")) + self.assertTrue(getattr(executions[1], "fresh_capture_required")) + self.assertEqual( + getattr(executions[1], "post_accept_block"), + (10, fixture.TIP_C), + ) + self.assertEqual(reports, [(None, 0), ((10, fixture.TIP_C), 0)]) + self.assertEqual(service.snapshot().latest_detected_tip, (fixture.TIP_B, 2)) + published = service.published_snapshot() + self.assertEqual(published.tip_hash, fixture.TIP_B) + self.assertEqual(published.observation_sequence, 2) + self.assertTrue(service.shutdown()) + + def test_post_accept_after_capture_start_uses_pending_latest_context( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + capture_port_entered = threading.Event() + release_capture_port = threading.Event() + getbest_calls = 0 + reports: list[tuple[int, str] | None] = [] + + def rpc_call(method: str, params: list[object] | None) -> object: + nonlocal getbest_calls + if method == "getbestblockhash": + getbest_calls += 1 + if getbest_calls == 1: + capture_port_entered.set() + self.assertTrue(release_capture_port.wait(2.0)) + return fixture.TIP_A + return fixture.rpc_call(method, params) + + service.reconfigure_ports_for_test(rpc_call=rpc_call) + service._handle_scheduled_success = ( # type: ignore[method-assign] + lambda trigger, _result: reports.append( + getattr(trigger, "post_accept_block") + ) + ) + first = service.submit_post_accept_trigger( + block_height=10, + block_hash=fixture.TIP_A, + ) + self.assertTrue(capture_port_entered.wait(1.0)) + second = service.submit_post_accept_trigger( + block_height=11, + block_hash=fixture.TIP_B, + ) + latest = service.submit_post_accept_trigger( + block_height=12, + block_hash=fixture.TIP_C, + ) + + self.assertIsNot(first, second) + self.assertIs(latest, second) + pending = service.scheduler_snapshot().pending + assert pending is not None + self.assertTrue(pending.fresh_capture_required) + self.assertEqual(pending.post_accept_block, (12, fixture.TIP_C)) + self.assertIsNone(pending.snapshot) + release_capture_port.set() + + self.assertEqual(first.result(timeout=1.0), 0) + self.assertEqual(second.result(timeout=1.0), 0) + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + self.assertEqual(getbest_calls, 2) + self.assertEqual( + reports, + [(10, fixture.TIP_A), (12, fixture.TIP_C)], + ) + self.assertEqual(service.snapshot().latest_detected_tip, (fixture.TIP_A, 3)) + self.assertEqual(service.published_snapshot().observation_sequence, 3) + self.assertEqual(service.snapshot().post_accept_refresh_failure_count, 0) + self.assertTrue(service.shutdown()) + + def test_post_accept_before_capture_uses_pending_latest_context( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + permit_wait_started = threading.Event() + permit = threading.Event() + getbest_calls = 0 + reports: list[tuple[int, str] | None] = [] + + def wait_for_permit(timeout_seconds: float) -> bool: + permit_wait_started.set() + return permit.wait(timeout_seconds) + + def rpc_call(method: str, params: list[object] | None) -> object: + nonlocal getbest_calls + if method == "getbestblockhash": + getbest_calls += 1 + return fixture.rpc_call(method, params) + + service.reconfigure_ports_for_test( + rpc_call=rpc_call, + wait_for_execution_permit=wait_for_permit, + ) + service._handle_scheduled_success = ( # type: ignore[method-assign] + lambda trigger, _result: reports.append( + getattr(trigger, "post_accept_block") + ) + ) + active = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(permit_wait_started.wait(1.0)) + first = service.submit_post_accept_trigger( + block_height=10, + block_hash=fixture.TIP_B, + ) + latest = service.submit_post_accept_trigger( + block_height=11, + block_hash=fixture.TIP_C, + ) + + self.assertIsNot(first, active) + self.assertIs(latest, first) + pending = service.scheduler_snapshot().pending + assert pending is not None + self.assertTrue(pending.fresh_capture_required) + self.assertEqual(pending.post_accept_block, (11, fixture.TIP_C)) + fixture.tip = fixture.TIP_B + fixture.snapshot = fixture.make_snapshot(fixture.TIP_B, "22" * 32) + permit.set() + + self.assertEqual(active.result(timeout=1.0), 0) + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + self.assertEqual(getbest_calls, 2) + self.assertEqual(reports, [None, (11, fixture.TIP_C)]) + self.assertEqual(service.snapshot().latest_detected_tip, (fixture.TIP_B, 2)) + self.assertEqual(service.published_snapshot().observation_sequence, 2) + self.assertTrue(service.shutdown()) + + def test_live_tip_supersession_preserves_pending_post_accept_future( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + permit_wait_started = threading.Event() + permit = threading.Event() + failures: list[tuple[tuple[int, str] | None, type[BaseException]]] = [] + successes: list[tuple[int, str] | None] = [] + + def wait_for_permit(timeout_seconds: float) -> bool: + permit_wait_started.set() + return permit.wait(timeout_seconds) + + service.reconfigure_ports_for_test(wait_for_execution_permit=wait_for_permit) + service._handle_scheduled_failure = ( # type: ignore[method-assign] + lambda trigger, exc: failures.append( + (getattr(trigger, "post_accept_block"), type(exc)) + ) + ) + service._handle_scheduled_success = ( # type: ignore[method-assign] + lambda trigger, _result: successes.append( + getattr(trigger, "post_accept_block") + ) + ) + active = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(permit_wait_started.wait(1.0)) + post_accept = service.submit_post_accept_trigger( + block_height=10, + block_hash=fixture.TIP_C, + ) + self.assertIsNot(post_accept, active) + + fixture.tip = fixture.TIP_B + fixture.snapshot = fixture.make_snapshot(fixture.TIP_B, "22" * 32) + live = service.submit_tip_observation_admission( + fixture.TIP_B, + reason="blockwait", + ) + self.assertIs(live.completion, post_accept) + pending = service.scheduler_snapshot().pending + assert pending is not None + self.assertEqual(pending.tip_hash, fixture.TIP_B) + self.assertTrue(pending.fresh_capture_required) + self.assertIsNone(pending.snapshot) + self.assertEqual(pending.post_accept_block, (10, fixture.TIP_C)) + + with self.assertRaises(TemplateRefreshSuperseded): + active.result(timeout=1.0) + permit.set() + self.assertEqual(post_accept.result(timeout=1.0), 0) + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + self.assertEqual(failures, [(None, TemplateRefreshSuperseded)]) + self.assertEqual(successes, [(10, fixture.TIP_C)]) + self.assertEqual(service.snapshot().latest_detected_tip, (fixture.TIP_B, 3)) + published = service.published_snapshot() + self.assertEqual(published.tip_hash, fixture.TIP_B) + self.assertEqual(published.observation_sequence, 3) + self.assertTrue(service.shutdown()) + + def test_live_tip_superseding_active_post_accept_transfers_reporting( + self, + ) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + permit_wait_started = threading.Event() + permit = threading.Event() + successes: list[tuple[int, str] | None] = [] + + def wait_for_permit(timeout_seconds: float) -> bool: + permit_wait_started.set() + return permit.wait(timeout_seconds) + + service.reconfigure_ports_for_test(wait_for_execution_permit=wait_for_permit) + service._handle_scheduled_success = ( # type: ignore[method-assign] + lambda trigger, _result: successes.append( + getattr(trigger, "post_accept_block") + ) + ) + active_post_accept = service.submit_post_accept_trigger( + block_height=10, + block_hash=fixture.TIP_C, + ) + self.assertTrue(permit_wait_started.wait(1.0)) + + fixture.tip = fixture.TIP_B + fixture.snapshot = fixture.make_snapshot(fixture.TIP_B, "22" * 32) + live = service.submit_tip_observation_admission( + fixture.TIP_B, + reason="blockwait", + ) + assert live.completion is not None + self.assertIsNot(live.completion, active_post_accept) + pending = service.scheduler_snapshot().pending + assert pending is not None + self.assertEqual(pending.tip_hash, fixture.TIP_B) + self.assertTrue(pending.fresh_capture_required) + self.assertEqual(pending.post_accept_block, (10, fixture.TIP_C)) + + with self.assertRaises(TemplateRefreshSuperseded): + active_post_accept.result(timeout=1.0) + self.assertEqual( + service.snapshot().post_accept_refresh_failure_count, + 0, + ) + permit.set() + self.assertEqual(live.completion.result(timeout=1.0), 0) + self.assertTrue(service.wait_for_scheduler_idle_for_test()) + self.assertEqual(successes, [(10, fixture.TIP_C)]) + self.assertEqual( + service.snapshot().post_accept_refresh_failure_count, + 0, + ) + self.assertEqual(service.snapshot().latest_detected_tip, (fixture.TIP_B, 3)) + published = service.published_snapshot() + self.assertEqual(published.tip_hash, fixture.TIP_B) + self.assertEqual(published.observation_sequence, 3) + self.assertTrue(service.shutdown()) + + def test_stale_slow_trigger_cannot_replace_newer_pending_authority(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + active_started = threading.Event() + release_active = threading.Event() + + def execute(trigger: object) -> int: + sequence = int(getattr(trigger, "observation_sequence")) + if sequence == 5: + active_started.set() + self.assertTrue(release_active.wait(2.0)) + service._raise_if_scheduler_superseded(trigger) # type: ignore[arg-type] + return sequence + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + service.submit_trigger( + self._scheduler_trigger(fixture, sequence=5, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + newest = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=7, tip=fixture.TIP_C) # type: ignore[arg-type] + ) + stale = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=6, tip=fixture.TIP_B) # type: ignore[arg-type] + ) + self.assertIs(stale, newest) + self.assertEqual( + service.scheduler_snapshot().pending.tip_hash, # type: ignore[union-attr] + fixture.TIP_C, + ) + release_active.set() + self.assertEqual(newest.result(timeout=1.0), 7) + self.assertTrue(service.shutdown()) + + def test_old_tip_template_event_cannot_reclaim_newer_tip_authority(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + self.assertTrue( + service.publish_tip( + fixture.TIP_A, + observation_sequence=1, + published_snapshot=fixture.snapshot, + ) + ) + self.assertTrue( + service.observe_tip( + fixture.TIP_B, + observation_sequence=2, + mark_pending=False, + ) + ) + executed: list[object] = [] + service._execute_refresh_trigger = lambda trigger: executed.append(trigger) or 0 # type: ignore[method-assign] + sequence_before_callback = service.observation_sequence() + + service.template_artifacts_changed( + SimpleNamespace( + generation=99, + previousblockhash=fixture.TIP_A, + fingerprint="99" * 32, + ) + ) + + scheduler = service.scheduler_snapshot() + self.assertIsNone(scheduler.active) + self.assertIsNone(scheduler.pending) + self.assertFalse(scheduler.worker_alive) + self.assertEqual(executed, []) + self.assertEqual(service.snapshot().latest_detected_tip, (fixture.TIP_B, 2)) + self.assertEqual(service.published_snapshot().tip_hash, fixture.TIP_A) + self.assertEqual(service.observation_sequence(), sequence_before_callback) + self.assertTrue(service.shutdown()) + + def test_equivalent_blockwait_trigger_coalesces_with_active_blockpoll(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + active_started = threading.Event() + release_active = threading.Event() + executions = 0 + + def execute(_trigger: object) -> int: + nonlocal executions + executions += 1 + active_started.set() + self.assertTrue(release_active.wait(2.0)) + return 0 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + active = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=3, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + duplicate = service.submit_trigger( + self._scheduler_trigger( + fixture, + sequence=3, + tip=fixture.TIP_A, + reason="blockwait", + ) # type: ignore[arg-type] + ) + self.assertIs(duplicate, active) + self.assertIsNone(service.scheduler_snapshot().pending) + release_active.set() + self.assertEqual(active.result(timeout=1.0), 0) + self.assertEqual(executions, 1) + metrics = service.metrics_snapshot() + self.assertEqual(metrics["trigger_coalesces"], 1) + self.assertEqual(metrics["trigger_supersessions"], 0) + self.assertTrue(service.shutdown()) + + def test_same_tip_template_and_readiness_requirements_are_not_dropped(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + active_started = threading.Event() + release_active = threading.Event() + + def execute(trigger: object) -> int: + if int(getattr(trigger, "observation_sequence")) == 1: + active_started.set() + self.assertTrue(release_active.wait(2.0)) + service._raise_if_scheduler_superseded(trigger) # type: ignore[arg-type] + return 0 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + service.submit_trigger( + self._scheduler_trigger( + fixture, + sequence=2, + tip=fixture.TIP_A, + payout_generation=4, + reason="template", + pending_token=8, + ) # type: ignore[arg-type] + ) + completion = service.submit_trigger( + self._scheduler_trigger( + fixture, + sequence=2, + tip=fixture.TIP_A, + payout_generation=5, + ready_required=True, + reason="readiness", + pending_token=9, + ) # type: ignore[arg-type] + ) + pending = service.scheduler_snapshot().pending + assert pending is not None + self.assertEqual(pending.tip_hash, fixture.TIP_A) + self.assertEqual(pending.payout_state_generation, 5) + self.assertTrue(pending.ready_required) + self.assertEqual(pending.pending_signal_token, 9) + self.assertEqual(pending.reasons, ("readiness", "template")) + release_active.set() + self.assertEqual(completion.result(timeout=1.0), 0) + self.assertTrue(service.shutdown()) + + def test_scheduler_metrics_are_exact_and_snapshot_lock_order_is_safe(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + active_started = threading.Event() + release_active = threading.Event() + + def execute(trigger: object) -> int: + if int(getattr(trigger, "observation_sequence")) == 1: + active_started.set() + self.assertTrue(release_active.wait(2.0)) + service._raise_if_scheduler_superseded(trigger) # type: ignore[arg-type] + return 0 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + fixture.now = 100.25 + pending = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=2, tip=fixture.TIP_B) # type: ignore[arg-type] + ) + duplicate = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=2, tip=fixture.TIP_B) # type: ignore[arg-type] + ) + self.assertIs(duplicate, pending) + + snapshots: list[dict[str, object]] = [] + snapshot_thread = threading.Thread( + target=lambda: snapshots.append(service.metrics_snapshot()) + ) + snapshot_thread.start() + snapshot_thread.join(1.0) + self.assertFalse(snapshot_thread.is_alive()) + self.assertEqual(snapshots[0]["trigger_queue_depth"], 1) + self.assertEqual(snapshots[0]["trigger_queue_capacity"], 1) + self.assertEqual(snapshots[0]["trigger_coalesces"], 1) + self.assertEqual(snapshots[0]["trigger_supersessions"], 1) + release_active.set() + self.assertEqual(pending.result(timeout=1.0), 0) + final = service.metrics_snapshot() + latency = final["trigger_latency"] + assert isinstance(latency, dict) + self.assertEqual(final["trigger_queue_depth"], 0) + self.assertEqual(latency["count"], 2) + self.assertEqual(latency["sum"], 0.0) + self.assertEqual(latency["buckets"][0.01], 2) # type: ignore[index] + self.assertTrue(service.shutdown()) + + def test_scheduler_callbacks_converge_without_recursive_trigger(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + readiness_callbacks = 0 + template_callbacks = 0 + original_snapshot = fixture.snapshot + + def latch_readiness() -> bool: + nonlocal readiness_callbacks + readiness_callbacks += 1 + service.readiness_promoted() + return True + + def fetch_with_template_callback() -> QbitTipTemplateSnapshot: + nonlocal template_callbacks + template_callbacks += 1 + service.template_artifacts_changed( + SimpleNamespace( + generation=1, + previousblockhash=fixture.tip, + fingerprint=original_snapshot.template_fingerprint, + ) + ) + return original_snapshot + + fixture.jobs.pool_readiness_latched = latch_readiness # type: ignore[method-assign] + service.reconfigure_ports_for_test(fetch_snapshot=fetch_with_template_callback) + self.assertEqual(service.poll_once(), 0) + self.assertEqual(readiness_callbacks, 1) + self.assertEqual(template_callbacks, 1) + self.assertIsNone(service.scheduler_snapshot().pending) + self.assertIsNone(service.snapshot().pending_token) + self.assertEqual(service.metrics_snapshot()["trigger_supersessions"], 0) + self.assertTrue(service.shutdown()) + + def test_shutdown_closes_admission_cancels_pending_and_joins_worker(self) -> None: + fixture = _TipRefreshFixture() + service = fixture.service + active_started = threading.Event() + release_active = threading.Event() + + def execute(trigger: object) -> int: + active_started.set() + self.assertTrue(release_active.wait(2.0)) + service._raise_if_scheduler_superseded(trigger) # type: ignore[arg-type] + return 0 + + service._execute_refresh_trigger = execute # type: ignore[method-assign] + active = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=1, tip=fixture.TIP_A) # type: ignore[arg-type] + ) + self.assertTrue(active_started.wait(1.0)) + pending = service.submit_trigger( + self._scheduler_trigger(fixture, sequence=2, tip=fixture.TIP_B) # type: ignore[arg-type] + ) + shutdown_results: list[bool] = [] + shutdown_thread = threading.Thread( + target=lambda: shutdown_results.append(service.shutdown()) + ) + shutdown_thread.start() + while service.scheduler_snapshot().admission_open: + self.assertTrue(shutdown_thread.is_alive()) + release_active.set() + shutdown_thread.join(1.0) + self.assertFalse(shutdown_thread.is_alive()) + self.assertEqual(shutdown_results, [True]) + with self.assertRaises(ShutdownInProgress): + pending.result(timeout=1.0) + with self.assertRaises(TemplateRefreshSuperseded): + active.result(timeout=1.0) + with self.assertRaises(ShutdownInProgress): + service.submit_trigger( + self._scheduler_trigger(fixture, sequence=3, tip=fixture.TIP_C) # type: ignore[arg-type] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prism_tip_refresh_delivery.py b/tests/test_prism_tip_refresh_delivery.py index ac03b20..746b297 100644 --- a/tests/test_prism_tip_refresh_delivery.py +++ b/tests/test_prism_tip_refresh_delivery.py @@ -152,7 +152,9 @@ def test_tip_refresh_rpc_race_blocks_mismatched_tip_template_snapshot(self) -> N # maps, or the published tip state. self.assertEqual(server.latest_detected_tip[0], old_tip) self.assertIsNone(server.current_tip_first_seen) - self.assertIsNone(server._template_artifacts) + self.assertIsNone( + server._ensure_job_bundle_service().template_repository.current_artifacts() + ) def test_slow_tip_poll_cannot_regress_newer_blockwait_observation(self) -> None: old_tip = "00" * 32 new_tip = "11" * 32 @@ -169,10 +171,12 @@ def overtake_poll() -> QbitTipTemplateSnapshot: self.assertTrue(server.observe_tip_for_refresh(new_tip)) return old_snapshot - server.fetch_qbit_tip_template_snapshot = overtake_poll # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=overtake_poll + ) with self.assertRaisesRegex( - TemplateRefreshSuperseded, + TemplateRefreshBlocked, "tip/template poll was superseded during template fetch", ): server.poll_qbit_tip_template_once() @@ -180,11 +184,6 @@ def overtake_poll() -> QbitTipTemplateSnapshot: self.assertEqual(server.latest_detected_tip[0], new_tip) self.assertEqual(server.current_tip_first_seen[0], old_tip) self.assertIsNone(server.tip_template_snapshot) - self.assertIsNone( - getattr(server, "template_refresh_failure_started_monotonic", None) - ) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) def test_same_tip_template_refresh_sends_non_clean_job_and_keeps_old_job_submittable(self) -> None: tip = "00" * 32 server = coordinator() @@ -345,7 +344,7 @@ def failing_build(client: ClientState, *, clean_jobs: bool) -> object: with self.assertRaisesRegex(TemplateRefreshBlocked, "no refreshed work was issued"): server.poll_qbit_tip_template_once() - self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 1) self.assertEqual(server.tip_refresh_job_count, 0) self.assertIn(state, server.clients) self.assertEqual(state.active_job_ids, {"old-job"}) @@ -407,7 +406,7 @@ def mixed_build(state: ClientState, *, clean_jobs: bool) -> object: with self.assertRaisesRegex(TemplateRefreshBlocked, "no refreshed work was issued"): server.poll_qbit_tip_template_once() - self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 1) self.assertEqual(disconnected_clients, [disconnected]) self.assertIn(build_failed, server.clients) def test_tip_reconciliation_quarantines_disconnected_block_before_refresh_job(self) -> None: @@ -565,7 +564,7 @@ def reorg_watch_blocks(self, *, active_tip_height: int) -> list[dict[str, object self.assertFalse(sent_job) self.assertEqual(server.reorg_reconcile_error_count, 1) - self.assertEqual(server.job_build_failure_count, 0) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 0) def test_reconciliation_runs_again_for_same_tip_hash(self) -> None: tip = "5a" * 32 server = coordinator() @@ -583,8 +582,8 @@ def test_reconciliation_runs_again_for_same_tip_hash(self) -> None: self.assertTrue(server.ensure_reorg_reconciled_for_tip(tip)) self.assertEqual(ledger.events, [("watch", 10), ("mature", 10), ("watch", 10), ("mature", 10)]) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertEqual(server._payout_state_service._source[0], 1) def test_reconciliation_reactivates_inactive_block_that_returns_to_active_chain(self) -> None: pool_block_hash = "cc" * 32 server = coordinator() @@ -662,6 +661,9 @@ def build_direct_job(client: ClientState, *, clean_jobs: bool) -> object: def test_template_refresh_failure_budget_starts_at_first_failure(self) -> None: server = PrismCoordinator.__new__(PrismCoordinator) server.template_refresh_failure_exit_seconds = 120 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server.last_successful_template_refresh_monotonic = 100.0 server._record_template_refresh_failure(500.0) @@ -672,10 +674,15 @@ def test_template_refresh_failure_budget_starts_at_first_failure(self) -> None: def test_disabled_template_refresh_failure_budget_does_not_start_or_expire(self) -> None: server = PrismCoordinator.__new__(PrismCoordinator) server.template_refresh_failure_exit_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_template_refresh_failure(500.0) self.assertFalse(server.template_refresh_failure_expired(500.0)) - self.assertFalse(hasattr(server, "template_refresh_failure_started_monotonic")) + self.assertIsNone( + getattr(server, "template_refresh_failure_started_monotonic", None) + ) def test_coordination_blocked_refresh_does_not_start_failure_budget(self) -> None: for blocked_error in ( TemplateRefreshSuperseded("qbit tip changed during sequential refresh"), @@ -684,13 +691,21 @@ def test_coordination_blocked_refresh_does_not_start_failure_budget(self) -> Non with self.subTest(blocked=type(blocked_error).__name__): server = coordinator() server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) server.rpc = TipRpc("11" * 32) def raise_blocked(error: Exception = blocked_error) -> QbitTipTemplateSnapshot: raise error - server.fetch_qbit_tip_template_snapshot = raise_blocked # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=raise_blocked + ) with self.assertRaises(type(blocked_error)): server.poll_qbit_tip_template_once() @@ -704,7 +719,13 @@ def test_non_coordination_blocked_refresh_still_starts_failure_budget(self) -> N # the TemplateRefreshSuperseded/payout-fence subclasses are exempt. server = coordinator() server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) server.rpc = TipRpc("11" * 32) def raise_blocked() -> QbitTipTemplateSnapshot: @@ -712,7 +733,9 @@ def raise_blocked() -> QbitTipTemplateSnapshot: "unable to derive exact artifacts for observed qbit template" ) - server.fetch_qbit_tip_template_snapshot = raise_blocked # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=raise_blocked + ) with ( patch("lab.prism.prism_coordinator.time.monotonic", return_value=100.0), self.assertRaises(TemplateRefreshBlocked), @@ -724,8 +747,17 @@ def raise_blocked() -> QbitTipTemplateSnapshot: def test_sustained_blocked_refresh_storm_never_exhausts_failure_budget(self) -> None: server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) server.rpc = TipRpc("11" * 32) clock = {"now": 100.0} blocked_polls = 0 @@ -744,13 +776,15 @@ def blocked_fetch() -> QbitTipTemplateSnapshot: "payout state invalidation is pending publication" ) - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=blocked_fetch + ) with ( patch( "lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"], ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.tip_refresh.traceback.print_exc"), patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, ): server.blockpoll_loop() @@ -768,9 +802,18 @@ def test_armed_budget_is_not_fired_by_coordination_blocked_refresh(self) -> None # budgeted failure, and the clock clears on the next completed refresh. server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server.template_refresh_failure_started_monotonic = 100.0 server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) server.rpc = TipRpc("11" * 32) clock = {"now": 100.0} blocked_polls = 0 @@ -785,13 +828,15 @@ def blocked_fetch() -> QbitTipTemplateSnapshot: "qbit tip changed during sequential refresh; immediate retry scheduled" ) - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=blocked_fetch + ) with ( patch( "lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"], ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.tip_refresh.traceback.print_exc"), patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, ): server.blockpoll_loop() @@ -807,8 +852,17 @@ def blocked_fetch() -> QbitTipTemplateSnapshot: def test_rpc_outage_arms_and_exhausts_failure_budget(self) -> None: server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) clock = {"now": 100.0} class OutageRpc: @@ -822,7 +876,7 @@ def call(self, method: str, params: list[object] | None = None, **_kwargs: objec "lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"], ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.tip_refresh.traceback.print_exc"), patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, self.assertRaises(SystemExit), ): @@ -837,8 +891,17 @@ def test_persistent_blocked_template_derivation_arms_and_exhausts_failure_budget # first such failure and take the budgeted restart path. server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) server.rpc = TipRpc("11" * 32) clock = {"now": 100.0} @@ -848,13 +911,15 @@ def blocked_fetch() -> QbitTipTemplateSnapshot: "unable to derive exact artifacts for observed qbit template" ) - server.fetch_qbit_tip_template_snapshot = blocked_fetch # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=blocked_fetch + ) with ( patch( "lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"], ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.tip_refresh.traceback.print_exc"), patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, self.assertRaises(SystemExit), ): @@ -865,10 +930,19 @@ def blocked_fetch() -> QbitTipTemplateSnapshot: def test_healthy_noop_template_poll_resets_refresh_failure_clock(self) -> None: server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.last_successful_template_refresh_monotonic = 100.0 server.template_refresh_failure_started_monotonic = 190.0 server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) snapshot = QbitTipTemplateSnapshot( bestblockhash="11" * 32, previousblockhash="11" * 32, @@ -876,7 +950,9 @@ def test_healthy_noop_template_poll_resets_refresh_failure_clock(self) -> None: ) server.tip_template_snapshot = snapshot server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=lambda: snapshot + ) def trusted_chain_view(_tip: str) -> bool: server.stop_event.set() @@ -901,7 +977,9 @@ def test_shared_template_poll_records_success_for_blockwait_callers(self) -> Non template_fingerprint="22" * 32, ) server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=lambda: snapshot + ) server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] with patch("lab.prism.prism_coordinator.time.monotonic", return_value=200.0): @@ -912,22 +990,33 @@ def test_shared_template_poll_records_success_for_blockwait_callers(self) -> Non def test_untrusted_reconciliation_exhausts_template_refresh_failure_budget(self) -> None: server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.last_successful_template_refresh_monotonic = 100.0 server.template_refresh_failure_started_monotonic = 100.0 server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) snapshot = QbitTipTemplateSnapshot( bestblockhash="11" * 32, previousblockhash="11" * 32, template_fingerprint="22" * 32, ) server.rpc = TipRpc(snapshot.bestblockhash) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=lambda: snapshot + ) server.ensure_reorg_reconciled_for_tip = lambda _tip: False # type: ignore[method-assign] with ( patch("lab.prism.prism_coordinator.time.monotonic", return_value=110.0), - patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.tip_refresh.traceback.print_exc"), patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, self.assertRaises(SystemExit), ): @@ -940,10 +1029,19 @@ def test_all_refresh_job_builds_failing_exhausts_failure_budget(self) -> None: new_tip = "33" * 32 server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.last_successful_template_refresh_monotonic = 100.0 server.template_refresh_failure_started_monotonic = 100.0 server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) state = client() state.username = "miner-a" state.worker = worker_identity() @@ -962,21 +1060,23 @@ def test_all_refresh_job_builds_failing_exhausts_failure_budget(self) -> None: template_fingerprint="44" * 32, ) server.rpc = TipRpc(new_tip) - server.fetch_qbit_tip_template_snapshot = lambda: snapshot # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + fetch_snapshot=lambda: snapshot + ) server.build_job_for_client = lambda *_args, **_kwargs: (_ for _ in ()).throw( # type: ignore[method-assign] RuntimeError("template build unavailable") ) with ( patch("lab.prism.prism_coordinator.time.monotonic", return_value=110.0), - patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.tip_refresh.traceback.print_exc"), patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, self.assertRaises(SystemExit), ): server.blockpoll_loop() exit_process.assert_called_once_with(1) - self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 1) self.assertEqual(server.last_successful_template_refresh_monotonic, 100.0) def test_guarded_sequential_build_supersession_does_not_arm_failure_budget(self) -> None: # Non-ready/collection mode: the sequential loop's guarded client @@ -988,8 +1088,17 @@ def test_guarded_sequential_build_supersession_does_not_arm_failure_budget(self) new_tip = "33" * 32 server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) state = client() state.username = "miner-a" state.worker = worker_identity() @@ -1019,9 +1128,10 @@ def fetch_snapshot() -> QbitTipTemplateSnapshot: server.stop_event.set() return snapshot - server.fetch_qbit_tip_template_snapshot = fetch_snapshot # type: ignore[method-assign] + tip_refresh = server._ensure_tip_refresh_service() + tip_refresh.reconfigure_ports_for_test(fetch_snapshot=fetch_snapshot) # The guarded pre-build currency check loses the race on every pass. - server._tip_refresh_snapshot_current_locked = ( # type: ignore[method-assign] + tip_refresh.snapshot_current = ( # type: ignore[method-assign] lambda *_args, **_kwargs: False ) @@ -1030,7 +1140,7 @@ def fetch_snapshot() -> QbitTipTemplateSnapshot: "lab.prism.prism_coordinator.time.monotonic", side_effect=lambda: clock["now"], ), - patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.tip_refresh.traceback.print_exc"), patch("lab.prism.prism_coordinator.os._exit", side_effect=SystemExit(1)) as exit_process, ): server.blockpoll_loop() @@ -1044,9 +1154,18 @@ def fetch_snapshot() -> QbitTipTemplateSnapshot: def test_transient_template_refresh_failure_recovers_on_healthy_noop(self) -> None: server = coordinator() server.blockpoll_seconds = 0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) server.last_successful_template_refresh_monotonic = 100.0 server.template_refresh_failure_exit_seconds = 10.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + failure_exit_seconds=server.template_refresh_failure_exit_seconds + ) server._record_heartbeat = lambda _name: None # type: ignore[method-assign] + server._ensure_tip_refresh_service().reconfigure_ports_for_test( + heartbeat=server._record_heartbeat + ) poll_count = 0 def fail_then_noop() -> int: @@ -1058,10 +1177,11 @@ def fail_then_noop() -> int: server.stop_event.set() return 0 - server.poll_qbit_tip_template_once = fail_then_noop # type: ignore[method-assign] + tip_refresh = server._ensure_tip_refresh_service() + tip_refresh.poll_once = fail_then_noop # type: ignore[method-assign] with ( patch("lab.prism.prism_coordinator.time.monotonic", side_effect=[105.0, 106.0]), - patch("lab.prism.prism_coordinator.traceback.print_exc"), + patch("lab.prism.tip_refresh.traceback.print_exc"), ): server.blockpoll_loop() @@ -1113,6 +1233,9 @@ def test_blockwait_advances_known_tip_before_notification_failure(self) -> None: tip_b = "bb" * 32 server.rpc = SimpleNamespace(call=lambda method: tip_a) server.blockpoll_seconds = 1.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) known_tips: list[str] = [] detected_tips: list[str] = [] @@ -1122,7 +1245,8 @@ def blockwait_once(known_tip: str) -> str: server.stop_event.set() return tip_b - server.blockwait_once = blockwait_once # type: ignore[method-assign] + tip_refresh = server._ensure_tip_refresh_service() + tip_refresh.blockwait_once = blockwait_once # type: ignore[method-assign] def observe_tip_for_refresh(tip_hash: str, **_kwargs: object) -> bool: detected_tips.append(tip_hash) @@ -1130,14 +1254,10 @@ def observe_tip_for_refresh(tip_hash: str, **_kwargs: object) -> bool: raise TemplateRefreshBlocked("notification failed") return True - def reject_premature_publication(*_args: object, **_kwargs: object) -> bool: - raise AssertionError("blockwait must not publish submit authority") - - server.observe_tip_for_refresh = observe_tip_for_refresh # type: ignore[method-assign] - server.observe_tip_first_seen = reject_premature_publication # type: ignore[method-assign] + tip_refresh.observe_tip = observe_tip_for_refresh # type: ignore[method-assign] with patch("builtins.print"), patch( - "lab.prism.prism_coordinator.traceback.print_exc" + "lab.prism.tip_refresh.traceback.print_exc" ), patch.object( server.stop_event, "wait", @@ -1147,6 +1267,38 @@ def reject_premature_publication(*_args: object, **_kwargs: object) -> bool: self.assertEqual(known_tips, [tip_a, tip_b]) self.assertEqual(detected_tips, [tip_a, tip_b]) + def test_blockwait_failure_loop_reuses_existing_stop_event(self) -> None: + server = self._bare_coordinator() + tip = "aa" * 32 + server.rpc = SimpleNamespace(call=lambda method: tip) + server.blockpoll_seconds = 1.0 + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) + attempts = 0 + tip_refresh = server._ensure_tip_refresh_service() + + def fail_blockwait(_known_tip: str) -> str: + nonlocal attempts + attempts += 1 + if attempts == 3: + server.stop_event.set() + raise RuntimeError("transient waitfornewblock failure") + + tip_refresh.blockwait_once = fail_blockwait # type: ignore[method-assign] + tip_refresh.reconfigure_ports_for_test(wait_for_stop=lambda _seconds: False) + + with ( + patch("builtins.print"), + patch("lab.prism.tip_refresh.traceback.print_exc"), + patch( + "lab.prism.prism_coordinator.threading.Event", + side_effect=AssertionError("allocated fallback stop event"), + ), + ): + server.blockwait_loop() + + self.assertEqual(attempts, 3) def test_blockwait_unsupported_removes_watchdog_heartbeat(self) -> None: server = coordinator() server.rpc = UnsupportedBlockwaitRpc("00" * 32) @@ -1235,6 +1387,7 @@ def test_ready_pool_refreshes_clients_left_on_collection_jobs(self) -> None: state.active_job = context server.min_ready_miners = 1 + server._ensure_job_bundle_service().set_min_ready_miners_for_test(1) server.accepted_share_stats = lambda: (0, 0) # type: ignore[method-assign] self.assertFalse(server.pool_readiness_latched()) self.assertFalse(server.client_needs_tip_template_refresh(state, snapshot)) @@ -1334,7 +1487,7 @@ def chain_view_untrusted() -> bool: self.assertEqual(refreshed, 250) self.assertEqual(recorded["calls"], 1) - cached = next(iter(server._job_bundle_cache.values())) + cached = next(iter(server._ensure_job_bundle_service()._bundle_cache.values())) self.assertEqual( len({id(state.active_job.shares_json) for state in clients}), 1, @@ -1354,7 +1507,10 @@ def chain_view_untrusted() -> bool: def test_supersession_retry_wakes_blockpoll_without_full_interval(self) -> None: server, _ = coordinator() server.blockpoll_seconds = 60.0 - server._ensure_tip_refresh_state() + server._ensure_tip_refresh_service().reconfigure_for_test( + blockpoll_seconds=server.blockpoll_seconds + ) + tip_refresh = server._ensure_tip_refresh_service() poll_called = threading.Event() def poll_once() -> int: @@ -1362,7 +1518,7 @@ def poll_once() -> int: server.stop_event.set() return 0 - server.poll_qbit_tip_template_once = poll_once # type: ignore[method-assign] + tip_refresh.poll_once = poll_once # type: ignore[method-assign] thread = threading.Thread(target=server.blockpoll_loop) thread.start() try: @@ -1378,6 +1534,9 @@ def test_ready_tip_refresh_respects_executor_bound(self) -> None: server, _ = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 2 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) clients = [client(index + 1) for index in range(6)] server.clients = set(clients) release = threading.Event() @@ -1421,6 +1580,9 @@ def test_blocked_socket_does_not_delay_another_client(self) -> None: server, _ = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 2 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) blocked = client(1) healthy = client(2) server.clients = {blocked, healthy} @@ -1455,6 +1617,9 @@ def test_shutdown_drains_inflight_tip_refresh_worker(self) -> None: server, _ = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) state = client(1) server.clients = {state} worker_started = threading.Event() @@ -1502,7 +1667,7 @@ def shutdown() -> None: self.assertTrue(shutdown_complete.is_set()) self.assertTrue(worker_send_finished.is_set()) self.assertEqual(poll_errors, []) - self.assertEqual(server.tip_refresh_inflight, 0) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["inflight"], 0) with self.assertRaisesRegex(RuntimeError, "executor is shut down"): server.tip_refresh_executor() def test_queued_fanout_stops_when_chain_view_becomes_untrusted(self) -> None: @@ -1538,6 +1703,9 @@ def test_queued_fanout_stops_when_live_tip_changes(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first = client(1) second = client(2) server.clients = [first, second] # type: ignore[assignment] @@ -1576,7 +1744,7 @@ def poll() -> None: self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0], TemplateRefreshBlocked) self.assertIn("immediate retry scheduled", str(errors[0])) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) self.assertIsNotNone(first.active_job) self.assertIsNotNone(second.active_job) self.assertEqual( @@ -1587,6 +1755,9 @@ def test_multiworker_cancel_releases_client_lock_while_draining_peer(self) -> No server, _ = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) admitted = client(1) queued = client(2) server.clients = [admitted, queued] # type: ignore[assignment] @@ -1639,6 +1810,9 @@ def test_same_tip_cache_refresh_during_fanout_does_not_abort(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first = client(1) second = client(2) # Preserve task order so the cache replacement happens after one @@ -1657,6 +1831,7 @@ def first_send(payload: dict[str, object]) -> None: second.send = second_sent.append # type: ignore[method-assign] refreshed: list[int] = [] errors: list[BaseException] = [] + followup_drained = False def poll() -> None: try: @@ -1679,20 +1854,44 @@ def poll() -> None: finally: release_first.set() thread.join(5) + followup_drained = ( + server._ensure_tip_refresh_service().wait_for_scheduler_idle_for_test() + ) server.shutdown_tip_refresh_executor() self.assertFalse(thread.is_alive()) + self.assertTrue(followup_drained) self.assertEqual(errors, []) self.assertEqual(refreshed, [2]) self.assertIsNotNone(server.last_successful_template_refresh_monotonic) self.assertEqual( [payload["method"] for payload in second_sent], - ["mining.set_difficulty", "mining.notify"], - ) + [ + "mining.set_difficulty", + "mining.notify", + "mining.set_difficulty", + "mining.notify", + ], + ) + assert replacement_artifacts is not None + for state in (first, second): + self.assertIsNotNone(state.active_job) + self.assertIs(state.active_job.template, replacement_artifacts.template) + self.assertEqual( + state.active_job.template_fingerprint, + replacement_artifacts.fingerprint, + ) + self.assertEqual( + state.active_job.template_generation, + replacement_artifacts.generation, + ) def test_queued_fanout_does_not_overwrite_intervening_job(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first = client(1) second = client(2) server.clients = [first, second] # type: ignore[assignment] @@ -1751,10 +1950,14 @@ def test_queued_fanout_replaces_stale_intervening_job(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 - # The poll must observe the same-tip template rotation immediately for - # the queued-fanout replacement race below; disable the same-tip - # template reuse window so the rotation is fetched, not deferred. + # This race requires the same-tip template rotation to be fetched now. server.template_cache_seconds = 0.0 + server._ensure_job_bundle_service().template_repository.set_cache_seconds_for_test( + 0.0 + ) + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first = client(1) second = client(2) server.clients = [first, second] # type: ignore[assignment] @@ -1850,7 +2053,7 @@ def test_newer_template_does_not_supersede_current_payout_refresh(self) -> None: ) current_intervening_job = dataclass_replace( stale_intervening_job, - payout_state_generation=server._payout_state_generation, + payout_state_generation=server._payout_state_service._generation, ) self.assertTrue( server.intervening_job_supersedes_snapshot( @@ -1919,6 +2122,9 @@ def test_client_removed_before_pending_task_runs_is_skipped(self) -> None: server, _ = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first = client(1) removed = client(2) clients = [first, removed] @@ -1970,7 +2176,8 @@ def test_template_fingerprint_race_uses_snapshot_owned_artifacts(self) -> None: for state in states: state.send = sent.append # type: ignore[method-assign] server.clients = set(states) - original_shared_job_bundle = server.shared_job_bundle + job_bundles = server._ensure_job_bundle_service() + original_shared_job_bundle = job_bundles.shared_job_bundle race_calls = 0 def race_artifacts( @@ -1981,14 +2188,15 @@ def race_artifacts( nonlocal race_calls race_calls += 1 bundle = original_shared_job_bundle(artifacts, identity, **kwargs) - with server._job_cache_lock: - server._template_artifacts = dataclass_replace( - server._template_artifacts, - fingerprint="ff" * 32, - ) + repository = server._ensure_job_bundle_service().template_repository + current = repository.current_artifacts() + assert current is not None + repository.replace_for_test( + dataclass_replace(current, fingerprint="ff" * 32) + ) return bundle - server.shared_job_bundle = race_artifacts # type: ignore[method-assign] + job_bundles.shared_job_bundle = race_artifacts # type: ignore[method-assign] refreshed = server.poll_qbit_tip_template_once() diff --git a/tests/test_prism_tip_refresh_validation.py b/tests/test_prism_tip_refresh_validation.py index 4200141..620d6ce 100644 --- a/tests/test_prism_tip_refresh_validation.py +++ b/tests/test_prism_tip_refresh_validation.py @@ -10,8 +10,9 @@ from dataclasses import FrozenInstanceError from unittest.mock import patch +from lab.prism.coordinator_shutdown import ShutdownInProgress +from lab.prism.job_bundle import JobBuildCancellation from lab.prism.prism_coordinator import ( - ShutdownInProgress, TemplateRefreshBlocked, TipRefreshValidationToken, _FanoutCancellation, @@ -201,7 +202,7 @@ def mark_mature_pool_payouts( server.reorg_reconciler_enabled = True server._ensure_job_cache_state() ledger.assert_not_atomic = lambda: self.assertIsNone( # type: ignore[attr-defined] - server._payout_state_delivery_gate._mutation_owner + server._payout_state_service._delivery_gate._mutation_owner ) results: list[dict[str, object]] = [] errors: list[BaseException] = [] @@ -218,25 +219,25 @@ def reconcile() -> None: thread.start() try: self.assertTrue(entered.wait(5)) - with server._payout_state_delivery_gate.delivery_cancelable( + with server._payout_state_service._delivery_gate.delivery_cancelable( lambda: False, generation=0, ) as admission: self.assertTrue(admission) release.set() - with server._payout_state_delivery_gate._condition: + with server._payout_state_service._delivery_gate._condition: self.assertTrue( - server._payout_state_delivery_gate._condition.wait_for( + server._payout_state_service._delivery_gate._condition.wait_for( lambda: ( - server._payout_state_delivery_gate._publisher_waiting + server._payout_state_service._delivery_gate._publisher_waiting ), timeout=5, ) ) self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) + server._payout_state_service._prepare_lock.acquire(timeout=1) ) - server._payout_state_prepare_lock.release() + server._payout_state_service._prepare_lock.release() finally: release.set() thread.join(5) @@ -244,7 +245,7 @@ def reconcile() -> None: self.assertFalse(thread.is_alive()) self.assertEqual(errors, []) self.assertEqual(results[0]["inactive_blocks"], 1) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service._generation, 1) def test_payout_only_preparation_does_not_hold_delivery_gate(self) -> None: entered = threading.Event() @@ -274,7 +275,7 @@ def mark_mature_pool_payouts( server.reorg_reconciler_enabled = True server._ensure_job_cache_state() ledger.assert_not_atomic = lambda: self.assertIsNone( # type: ignore[attr-defined] - server._payout_state_delivery_gate._mutation_owner + server._payout_state_service._delivery_gate._mutation_owner ) results: list[dict[str, object]] = [] @@ -286,32 +287,32 @@ def mark_mature_pool_payouts( thread.start() try: self.assertTrue(entered.wait(5)) - with server._payout_state_delivery_gate.delivery_cancelable( + with server._payout_state_service._delivery_gate.delivery_cancelable( lambda: False, generation=0, ) as admission: self.assertTrue(admission) release.set() - with server._payout_state_delivery_gate._condition: + with server._payout_state_service._delivery_gate._condition: self.assertTrue( - server._payout_state_delivery_gate._condition.wait_for( + server._payout_state_service._delivery_gate._condition.wait_for( lambda: ( - server._payout_state_delivery_gate._publisher_waiting + server._payout_state_service._delivery_gate._publisher_waiting ), timeout=5, ) ) self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) + server._payout_state_service._prepare_lock.acquire(timeout=1) ) - server._payout_state_prepare_lock.release() + server._payout_state_service._prepare_lock.release() finally: release.set() thread.join(5) self.assertFalse(thread.is_alive()) self.assertEqual(results[0]["matured_payouts"], 1) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service._generation, 1) def test_tip_poll_reuses_reconciled_external_source(self) -> None: server, rpc = coordinator() @@ -332,15 +333,15 @@ def test_tip_poll_reuses_reconciled_external_source(self) -> None: _advance_fake_tip(rpc, next_tip, 11) self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) - self.assertEqual(server._published_payout_state.source_generation, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, next_tip) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertEqual(server._payout_state_service._source[0], 1) + self.assertEqual(server._payout_state_service._published.source_generation, 1) + self.assertEqual(server._payout_state_service._published.source_tip_hash, next_tip) self.assertFalse(server.tip_refresh_is_pending()) self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server._payout_state_source[0], 1) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertEqual(server._payout_state_service._source[0], 1) def test_newer_tip_discards_in_progress_payout_candidate(self) -> None: entered = threading.Event() @@ -411,9 +412,9 @@ def reconcile() -> None: self.assertFalse(thread.is_alive()) self.assertEqual(errors, []) - self.assertEqual(server._payout_state_generation, 1) - self.assertEqual(server.payout_state_candidates_discarded, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, new_tip) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertEqual(server._payout_state_service.metrics_snapshot()["discarded_candidates"], 1) + self.assertEqual(server._payout_state_service._published.source_tip_hash, new_tip) self.assertEqual(server.current_tip_first_seen, published_tip) def test_supersession_retry_preserves_durable_reorg_counts(self) -> None: @@ -478,8 +479,8 @@ def mark_mature_pool_payouts( self.assertEqual(summary["inactive_blocks"], 1) self.assertEqual(server.reorg_inactive_block_count, 1) self.assertEqual(summary["published_generation"], 1) - self.assertEqual(server.payout_state_candidates_discarded, 1) - self.assertEqual(server._published_payout_state.source_tip_hash, new_tip) + self.assertEqual(server._payout_state_service.metrics_snapshot()["discarded_candidates"], 1) + self.assertEqual(server._payout_state_service._published.source_tip_hash, new_tip) def test_failed_superseded_reconcile_does_not_publish_latest_source(self) -> None: newer_tip = "79" * 32 @@ -530,19 +531,20 @@ def mark_mature_pool_payouts( with self.assertRaisesRegex(RuntimeError, "maturity RPC failed"): server.reconcile_prism_pool_blocks_once(tip_hash=rpc.tip) - self.assertEqual(server._payout_state_generation, 0) - self.assertIsNone(server._published_payout_state.source_tip_hash) - self.assertTrue(server._payout_state_publication_blocked) - self.assertEqual(server.payout_state_candidates_discarded, 1) + self.assertEqual(server._payout_state_service._generation, 0) + self.assertIsNone(server._payout_state_service._published.source_tip_hash) + self.assertTrue(server._payout_state_service._publication_blocked) + self.assertEqual(server._payout_state_service.metrics_snapshot()["discarded_candidates"], 1) self.assertEqual(server.reorg_inactive_block_count, 1) self.assertTrue(server.tip_refresh_is_pending()) def test_tip_churn_bounds_reconcile_retries_and_fences_job_builds(self) -> None: server, rpc = coordinator() server.reorg_reconciler_enabled = True - server.payout_reconcile_supersession_retries = 2 + service = server._payout_state_service + service.set_reconcile_retries_for_test(2) server.qbit_chain_view_untrusted = lambda: True # type: ignore[method-assign] - real_publish = server._publish_payout_state_candidate + real_publish = service.publish_candidate publish_attempts = 0 def supersede_before_publish(candidate: object) -> int | None: @@ -555,7 +557,7 @@ def supersede_before_publish(candidate: object) -> int | None: ) return real_publish(candidate) # type: ignore[arg-type] - server._publish_payout_state_candidate = supersede_before_publish # type: ignore[method-assign] + service.publish_candidate = supersede_before_publish # type: ignore[method-assign] summary = server.reconcile_prism_pool_blocks_once( tip_hash=rpc.tip, @@ -565,9 +567,9 @@ def supersede_before_publish(candidate: object) -> int | None: self.assertTrue(summary["superseded"]) self.assertEqual(publish_attempts, 3) self.assertEqual(server.reorg_reconcile_skip_count, 1) - self.assertEqual(server._payout_state_generation, 0) - self.assertEqual(server.payout_state_candidates_discarded, 3) - self.assertTrue(server._payout_state_publication_blocked) + self.assertEqual(server._payout_state_service._generation, 0) + self.assertEqual(server._payout_state_service.metrics_snapshot()["discarded_candidates"], 3) + self.assertTrue(server._payout_state_service._publication_blocked) state = client(1) assert state.worker is not None with self.assertRaisesRegex( @@ -692,7 +694,7 @@ def test_publication_updates_gate_without_coordinator_locks(self) -> None: server, _rpc = coordinator() server._reserve_payout_state_source("payout_only") original_publish_generation = ( - server._payout_state_delivery_gate.publish_generation + server._payout_state_service._delivery_gate.publish_generation ) lock_snapshots: list[tuple[bool, bool]] = [] @@ -703,7 +705,7 @@ def observe_locks( ) -> None: lock_snapshots.append( ( - server._job_cache_lock.locked(), + server._ensure_job_bundle_service()._cache_lock.locked(), server.lock._is_owned(), # type: ignore[attr-defined] ) ) @@ -712,7 +714,7 @@ def observe_locks( prioritize_delivery=prioritize_delivery, ) - server._payout_state_delivery_gate.publish_generation = observe_locks # type: ignore[method-assign] + server._payout_state_service._delivery_gate.publish_generation = observe_locks # type: ignore[method-assign] self.assertEqual( server._publish_payout_state_candidate( @@ -736,12 +738,14 @@ def test_pending_clear_does_not_consume_first_delivery_priority(self) -> None: with server.lock: server.tip_template_snapshot = snapshot server._reserve_payout_state_source("payout_only") - self.assertEqual( - server._publish_payout_state_candidate( - server._current_payout_state_candidate() - ), - 1, - ) + with server._ensure_tip_refresh_service().suppress_trigger_callbacks_for_test(): + self.assertEqual( + server._publish_payout_state_candidate( + server._current_payout_state_candidate() + ), + 1, + ) + server._mark_tip_refresh_pending(1) pending_token = server._claim_tip_refresh_pending() self.assertIsNotNone(pending_token) @@ -754,16 +758,16 @@ def test_pending_clear_does_not_consume_first_delivery_priority(self) -> None: ) ) self.assertEqual( - server._payout_state_delivery_gate._priority_generation, + server._payout_state_service._delivery_gate._priority_generation, 1, ) - with server._payout_state_delivery_gate.delivery_cancelable( + with server._payout_state_service._delivery_gate.delivery_cancelable( lambda: False, generation=1, priority=False, ) as routine_admission: self.assertFalse(routine_admission) - with server._payout_state_delivery_gate.delivery_cancelable( + with server._payout_state_service._delivery_gate.delivery_cancelable( lambda: False, generation=1, priority=True, @@ -771,7 +775,7 @@ def test_pending_clear_does_not_consume_first_delivery_priority(self) -> None: self.assertTrue(first_delivery) first_delivery.mark_delivered() self.assertIsNone( - server._payout_state_delivery_gate._priority_generation + server._payout_state_service._delivery_gate._priority_generation ) def test_collection_refresh_stops_when_chain_becomes_untrusted(self) -> None: @@ -806,7 +810,7 @@ def chain_view_untrusted() -> bool: self.assertEqual(notifications, [first.connection_id]) self.assertIsNotNone(first.active_job) self.assertIsNone(second.active_job) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_orphan_reconciliation_rebuilds_bundle_from_post_reorg_balances(self) -> None: orphaned_balance = { @@ -891,6 +895,9 @@ def record_signed_balances(**kwargs: object) -> dict[str, object]: return bundle server.build_audit_bundle = record_signed_balances # type: ignore[method-assign] + server._ensure_bundle_compiler().build_audit_bundle = ( # type: ignore[method-assign] + record_signed_balances + ) artifacts = server.store_template_artifacts(dict(rpc.template)) self.assertIsNotNone(artifacts) @@ -932,12 +939,12 @@ def record_send(payload: dict[str, object]) -> None: self.assertEqual(state.active_job.prior_balances, []) self.assertFalse(hasattr(state.active_job, "bundle")) self.assertFalse(hasattr(stale_bundle, "bundle")) - refreshed_bundle = next(iter(server._job_bundle_cache.values())) + refreshed_bundle = next(iter(server._ensure_job_bundle_service()._bundle_cache.values())) self.assertEqual(refreshed_bundle.prior_balances, []) self.assertIsNot(refreshed_bundle.coinbase_manifest, stale_bundle.coinbase_manifest) self.assertIn("coinbase_tx_hex", refreshed_bundle.coinbase_manifest) self.assertEqual(stale_bundle.payout_state_generation, 0) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service._generation, 1) self.assertEqual(state.active_job.payout_state_generation, 1) self.assertEqual(clean_notifications, [True]) self.assertNotIn(stale_context.job.job_id, state.active_job_ids) @@ -947,6 +954,9 @@ def test_hundred_client_refresh_validates_chain_once(self) -> None: install_fake_bundle_builder(server) server.reorg_reconciler_enabled = True server.tip_refresh_max_workers = 4 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) clients = [client(index + 1) for index in range(100)] notifications: set[int] = set() notifications_lock = threading.Lock() @@ -991,7 +1001,8 @@ def reject_per_client_validation( f"for {expected_tip_hash}" ) - original_validate = server._validate_prepared_tip_refresh + tip_refresh = server._ensure_tip_refresh_service() + original_validate = tip_refresh.validate_prepared def record_validation( bundle: object, @@ -1007,7 +1018,7 @@ def record_validation( server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] reject_per_client_validation ) - server._validate_prepared_tip_refresh = record_validation # type: ignore[method-assign] + tip_refresh.validate_prepared = record_validation # type: ignore[method-assign] try: refreshed = server.poll_qbit_tip_template_once() @@ -1034,7 +1045,7 @@ def record_validation( ) self.assertEqual( token.payout_state_generation, - server._payout_state_generation, + server._payout_state_service._generation, ) self.assertEqual( token.observation_sequence, @@ -1084,7 +1095,7 @@ def untrusted() -> bool: self.assertEqual(chain_view_checks, 1) self.assertEqual(sent, []) self.assertTrue(all(state.active_job is None for state in clients)) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_untrusted_post_fanout_does_not_report_refresh_success(self) -> None: server, _rpc = coordinator() @@ -1125,7 +1136,7 @@ def becomes_untrusted() -> bool: set(notifications), {state.connection_id for state in clients}, ) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) self.assertIsNone( getattr(server, "last_successful_template_refresh_monotonic", None) ) @@ -1160,11 +1171,7 @@ def shutdown_during_reconciliation(**_kwargs: object) -> bool: with self.assertRaises(ShutdownInProgress): server.poll_qbit_tip_template_once() self.assertTrue(notification_started.is_set()) - self.assertFalse(server._tip_refresh_retry.is_set()) - self.assertTrue( - server._tip_refresh_singleflight_lock.acquire(blocking=False) - ) - server._tip_refresh_singleflight_lock.release() + self.assertFalse(server._ensure_tip_refresh_service().snapshot().retry_requested) finally: release_notification.set() server.shutdown_tip_refresh_executor() @@ -1194,7 +1201,7 @@ def fail_tip_validation( ): server._validate_prepared_tip_refresh(bundle, snapshot, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_prevalidation_trust_check_failure_schedules_immediate_retry(self) -> None: server, _rpc = coordinator() @@ -1216,7 +1223,7 @@ def fail_chain_trust_check() -> bool: ): server._validate_prepared_tip_refresh(bundle, snapshot, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_reconciliation_failure_before_fanout_sends_zero_jobs(self) -> None: server, _rpc = coordinator() @@ -1257,6 +1264,9 @@ def test_newer_observation_cancels_pending_fanout_tasks(self) -> None: install_fake_bundle_builder(server) server.reorg_reconciler_enabled = True server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first, second, third = client(1), client(2), client(3) clients = [first, second, third] server.clients = clients # type: ignore[assignment] @@ -1316,11 +1326,14 @@ def poll() -> None: errors, ) - def test_waiting_poll_observes_new_tip_then_retries_after_owner_exits(self) -> None: + def test_waiting_poll_observes_new_tip_and_supersedes_slow_fanout(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.reorg_reconciler_enabled = True server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first, second = client(1), client(2) server.clients = [first, second] # type: ignore[assignment] server._ensure_tip_refresh_state() @@ -1330,7 +1343,7 @@ def test_waiting_poll_observes_new_tip_then_retries_after_owner_exits(self) -> N lambda **_kwargs: True ) self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() + server._ensure_tip_refresh_service().clear_retry_for_test() tip_a = rpc.tip tip_b = "33" * 32 @@ -1359,7 +1372,8 @@ def record_second_send(payload: dict[str, object]) -> None: first.send = record_first_send # type: ignore[method-assign] second.send = record_second_send # type: ignore[method-assign] - original_observe = server.observe_tip_for_refresh + tip_refresh = server._ensure_tip_refresh_service() + original_observe = tip_refresh.observe_tip def record_observation(*args: object, **kwargs: object) -> bool: observed = original_observe(*args, **kwargs) # type: ignore[arg-type] @@ -1367,13 +1381,14 @@ def record_observation(*args: object, **kwargs: object) -> bool: tip_b_observed.set() return observed - server.observe_tip_for_refresh = record_observation # type: ignore[method-assign] + tip_refresh.observe_tip = record_observation # type: ignore[method-assign] # Park the replacement build so the detected-but-unpublished window # can be asserted deterministically before tip B is published. replacement_build_started = threading.Event() release_replacement_build = threading.Event() build_calls = 0 - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle def gated_build(*args: object, **kwargs: object) -> object: nonlocal build_calls @@ -1384,7 +1399,7 @@ def gated_build(*args: object, **kwargs: object) -> object: raise AssertionError("test did not release replacement build") return original_build(*args, **kwargs) # type: ignore[arg-type] - server.build_shared_job_bundle = gated_build # type: ignore[method-assign] + service.build_shared_job_bundle = gated_build # type: ignore[method-assign] results: dict[str, list[int]] = {"old": [], "new": []} errors: dict[str, list[BaseException]] = {"old": [], "new": []} @@ -1400,49 +1415,54 @@ def poll(label: str) -> None: try: self.assertTrue(first_send_started.wait(5)) with server.lock: - active = server._active_tip_refresh + active = server._ensure_tip_refresh_service().active_refresh_snapshot() self.assertIsNotNone(active) - assert active is not None _advance_fake_tip(rpc, tip_b, 11) new_poll.start() self.assertTrue(tip_b_observed.wait(5)) - self.assertTrue(active[1].is_set()) - self.assertFalse(replacement_build_started.wait(0.1)) + self.assertFalse(replacement_build_started.is_set()) + active = server._ensure_tip_refresh_service().active_refresh_snapshot() + self.assertIsNotNone(active) + assert active is not None + self.assertTrue(active.cancelling) self.assertEqual(server.latest_detected_tip[0], tip_b) self.assertEqual(server.current_tip_first_seen[0], tip_a) - pending_tip_b = server._tip_refresh_pending_token + pending_tip_b = server._ensure_tip_refresh_service().snapshot().pending_token self.assertIsNotNone(pending_tip_b) - finally: + scheduler = server._ensure_tip_refresh_service().scheduler_snapshot() + self.assertEqual(scheduler.active.tip_hash, tip_a) # type: ignore[union-attr] + self.assertEqual(scheduler.pending.tip_hash, tip_b) # type: ignore[union-attr] + self.assertEqual(scheduler.pending_capacity, 1) release_first_send.set() old_poll.join(5) - self.assertFalse(old_poll.is_alive()) - self.assertEqual(results["old"], []) - self.assertEqual(len(errors["old"]), 1) - self.assertIsInstance(errors["old"][0], TemplateRefreshBlocked) - if server._tip_refresh_pending_token is not None: - self.assertEqual(server._tip_refresh_pending_token, pending_tip_b) + self.assertFalse(old_poll.is_alive()) + self.assertEqual(results["old"], []) + self.assertEqual(len(errors["old"]), 1) + self.assertIsInstance(errors["old"][0], TemplateRefreshBlocked) + self.assertTrue(replacement_build_started.wait(5)) + self.assertEqual(server.current_tip_first_seen[0], tip_a) + release_replacement_build.set() + new_poll.join(5) + self.assertTrue(tip_refresh.wait_for_scheduler_idle_for_test(5.0)) + finally: + release_replacement_build.set() + release_first_send.set() + old_poll.join(5) + new_poll.join(5) + tip_refresh.wait_for_scheduler_idle_for_test(5.0) + server.shutdown_tip_refresh_executor() - new_poll.join(5) self.assertFalse(new_poll.is_alive()) self.assertEqual(errors["new"], []) self.assertEqual(results["new"], [0]) - self.assertFalse(replacement_build_started.is_set()) - - release_replacement_build.set() - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 2) - finally: - server.shutdown_tip_refresh_executor() - - self.assertTrue(replacement_build_started.is_set()) self.assertEqual(sent_tips, {1: [tip_a, tip_b], 2: [tip_b]}) self.assertEqual(server.current_tip_first_seen[0], tip_b) self.assertFalse(server.tip_refresh_is_pending()) - def test_replacement_build_waits_until_obsolete_owner_is_released(self) -> None: + def test_replacement_build_waits_for_single_refresh_scheduler(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) state = client(1) @@ -1451,7 +1471,7 @@ def test_replacement_build_waits_until_obsolete_owner_is_released(self) -> None: server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() + server._ensure_tip_refresh_service().clear_retry_for_test() bundle_started = threading.Event() replacement_started = threading.Event() @@ -1460,7 +1480,8 @@ def test_replacement_build_waits_until_obsolete_owner_is_released(self) -> None: active_builders = 0 max_active_builders = 0 build_calls = 0 - original_build = server.build_shared_job_bundle + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle def blocking_build(*args: object, **kwargs: object) -> object: nonlocal active_builders, max_active_builders, build_calls @@ -1481,7 +1502,7 @@ def blocking_build(*args: object, **kwargs: object) -> object: with builder_lock: active_builders -= 1 - server.build_shared_job_bundle = blocking_build # type: ignore[method-assign] + service.build_shared_job_bundle = blocking_build # type: ignore[method-assign] sent_generations: list[int] = [] def record_send(payload: dict[str, object]) -> None: @@ -1509,14 +1530,19 @@ def poll(label: str) -> None: # qbit confirms a direct PRISM block, before final audit/payout # publication completes. server._cancel_obsolete_job_builds("direct PRISM block accepted") - with server._job_build_scheduler_lock: - active = server._job_build_active + with server._ensure_job_bundle_service()._scheduler_lock: + active = server._ensure_job_bundle_service()._active self.assertIsNotNone(active) assert active is not None self.assertTrue(active.request.cancellation.is_set()) self.assertEqual(server._advance_payout_state_generation(), 1) new_poll.start() - self.assertFalse(replacement_started.wait(0.1)) + self.assertFalse(replacement_started.is_set()) + scheduler = server._ensure_tip_refresh_service().scheduler_snapshot() + self.assertIsNotNone(scheduler.active) + self.assertIsNotNone(scheduler.pending) + self.assertEqual(scheduler.pending.payout_state_generation, 1) # type: ignore[union-attr] + self.assertEqual(scheduler.pending_capacity, 1) new_poll.join(5) self.assertFalse(new_poll.is_alive()) self.assertEqual(errors["new"], []) @@ -1526,9 +1552,9 @@ def poll(label: str) -> None: release_bundle.set() old_poll.join(5) new_poll.join(5) + server.shutdown_tip_refresh_executor() self.assertFalse(old_poll.is_alive()) - server.shutdown_tip_refresh_executor() self.assertEqual(errors["old"], []) self.assertEqual(results["old"], [1]) self.assertTrue(replacement_started.is_set()) @@ -1537,7 +1563,7 @@ def poll(label: str) -> None: self.assertTrue( all( bundle.payout_state_generation == 1 - for bundle in server._job_bundle_cache.values() + for bundle in server._ensure_job_bundle_service()._bundle_cache.values() ) ) self.assertFalse(server.tip_refresh_is_pending()) @@ -1564,15 +1590,15 @@ def test_tip_change_cancels_each_expensive_build_phase(self) -> None: server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] server._ensure_tip_refresh_state() self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() + server._ensure_tip_refresh_service().clear_retry_for_test() phase_entered = threading.Event() release_phase = threading.Event() - original_checkpoint = server._job_build_checkpoint + original_checkpoint = JobBuildCancellation.raise_if_cancelled def blocking_checkpoint( + cancellation: JobBuildCancellation, phase: str, - cancellation: object, ) -> None: if phase == blocked_phase and not phase_entered.is_set(): phase_entered.set() @@ -1580,9 +1606,8 @@ def blocking_checkpoint( raise AssertionError( f"test did not release {blocked_phase}" ) - original_checkpoint(phase, cancellation) # type: ignore[arg-type] + original_checkpoint(cancellation, phase) - server._job_build_checkpoint = blocking_checkpoint # type: ignore[method-assign] errors: list[BaseException] = [] def poll() -> None: @@ -1591,17 +1616,22 @@ def poll() -> None: except BaseException as exc: # noqa: BLE001 - asserted below errors.append(exc) - poll_thread = threading.Thread(target=poll) - poll_thread.start() - try: - self.assertTrue(phase_entered.wait(5)) - next_tip = f"{index + 120:02x}" * 32 - _advance_fake_tip(rpc, next_tip, 10 + index) - self.assertTrue(server.observe_tip_first_seen(next_tip)) - finally: - release_phase.set() - poll_thread.join(5) - server.shutdown_tip_refresh_executor() + with patch.object( + JobBuildCancellation, + "raise_if_cancelled", + blocking_checkpoint, + ): + poll_thread = threading.Thread(target=poll) + poll_thread.start() + try: + self.assertTrue(phase_entered.wait(5)) + next_tip = f"{index + 120:02x}" * 32 + _advance_fake_tip(rpc, next_tip, 10 + index) + self.assertTrue(server.observe_tip_first_seen(next_tip)) + finally: + release_phase.set() + poll_thread.join(5) + server.shutdown_tip_refresh_executor() self.assertFalse(poll_thread.is_alive()) self.assertEqual(len(errors), 1) @@ -1611,343 +1641,151 @@ def poll() -> None: self.assertTrue( all( entry.template["previousblockhash"] == next_tip - for entry in server._job_bundle_cache.values() + for entry in server._ensure_job_bundle_service()._bundle_cache.values() ) ) - def test_new_tip_during_ledger_snapshot_retries_and_publishes_latest(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - state = client(1) - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - - snapshot_complete = threading.Event() - release_snapshot = threading.Event() - original_checkpoint = server._job_build_checkpoint - - def slow_snapshot_checkpoint( - phase: str, - cancellation: object, - ) -> None: - if phase == "ledger_snapshot_complete" and not snapshot_complete.is_set(): - snapshot_complete.set() - if not release_snapshot.wait(5): - raise AssertionError("test did not release ledger snapshot") - original_checkpoint(phase, cancellation) # type: ignore[arg-type] - - server._job_build_checkpoint = slow_snapshot_checkpoint # type: ignore[method-assign] - notifications: list[str] = [] - - def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - notifications.append( - str(state.active_job.template["previousblockhash"]) - ) - - state.send = record_send # type: ignore[method-assign] - errors: list[BaseException] = [] - - def poll() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # noqa: BLE001 - asserted below - errors.append(exc) - - owner = threading.Thread(target=poll) - owner.start() - try: - self.assertTrue(snapshot_complete.wait(5)) - latest_tip = "88" * 32 - _advance_fake_tip(rpc, latest_tip, 11) - self.assertTrue(server.observe_tip_for_refresh(latest_tip)) - finally: - release_snapshot.set() - owner.join(5) - - self.assertFalse(owner.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], TemplateRefreshBlocked) - self.assertEqual(notifications, []) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - finally: - server.shutdown_tip_refresh_executor() - - self.assertEqual(notifications, ["88" * 32]) - self.assertEqual(server.current_tip_first_seen[0], "88" * 32) - self.assertFalse(server.tip_refresh_is_pending()) - - def test_refresh_exception_releases_owner_and_coalesces_one_retry(self) -> None: - server, _rpc = coordinator() - original_fetch = server.fetch_qbit_tip_template_snapshot - fetch_started = threading.Event() - release_fetch = threading.Event() - fetches = 0 - errors: list[BaseException] = [] - - def fail_once() -> object: - nonlocal fetches - fetches += 1 - if fetches == 1: - fetch_started.set() - if not release_fetch.wait(5): - raise AssertionError("test did not release failing fetch") - raise RuntimeError("transient template failure") - return original_fetch() - - server.fetch_qbit_tip_template_snapshot = fail_once # type: ignore[method-assign] - - def run_owner() -> None: - try: - server.poll_qbit_tip_template_once() - except BaseException as exc: - errors.append(exc) - - owner = threading.Thread(target=run_owner) - owner.start() - self.assertTrue(fetch_started.wait(5)) - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - finally: - release_fetch.set() - owner.join(5) - - self.assertFalse(owner.is_alive()) - self.assertEqual(len(errors), 1) - self.assertIsInstance(errors[0], RuntimeError) - self.assertEqual(str(errors[0]), "transient template failure") - self.assertTrue(server._tip_refresh_singleflight_lock.acquire(blocking=False)) - server._tip_refresh_singleflight_lock.release() - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - self.assertEqual(fetches, 2) - self.assertIsNone(server.template_refresh_failure_started_monotonic) - - def test_refresh_exception_does_not_self_schedule_immediate_retry(self) -> None: - server, _rpc = coordinator() - - def fail_fetch() -> object: - raise RuntimeError("persistent template failure") - - server.fetch_qbit_tip_template_snapshot = fail_fetch # type: ignore[method-assign] - - with self.assertRaisesRegex(RuntimeError, "persistent template failure"): - server.poll_qbit_tip_template_once() - - self.assertTrue(server._tip_refresh_singleflight_lock.acquire(blocking=False)) - server._tip_refresh_singleflight_lock.release() - self.assertFalse(server._consume_tip_refresh_retry()) - self.assertIsNotNone(server.template_refresh_failure_started_monotonic) - - def test_post_accept_rpc_failure_still_wakes_refresh_driver(self) -> None: - server, rpc = coordinator() - original_call = rpc.call - - def fail_best_tip(method: str, *args: object) -> object: - if method == "getbestblockhash": - raise RuntimeError("temporary best-tip failure") - return original_call(method, *args) - - rpc.call = fail_best_tip # type: ignore[method-assign] - - with patch("lab.prism.prism_coordinator.traceback.print_exc"): - self.assertEqual( - server.refresh_jobs_after_accepted_block( - block_height=14, - block_hash="88" * 32, - heartbeat_name="block_submitter", - ), - 0, - ) - - self.assertEqual(server.post_accept_refresh_failure_count, 1) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - - def test_post_accept_supersession_is_not_counted_as_failure(self) -> None: - server, _rpc = coordinator() - server.observe_tip_for_refresh = ( # type: ignore[method-assign] - lambda *_args, **_kwargs: False - ) - - with patch("lab.prism.prism_coordinator.traceback.print_exc") as print_exc: - self.assertEqual( - server.refresh_jobs_after_accepted_block( - block_height=15, - block_hash="99" * 32, - heartbeat_name="block_submitter", - ), - 0, - ) - - self.assertEqual(server.post_accept_refresh_failure_count, 0) - print_exc.assert_not_called() - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - - def test_tip_flicker_back_to_published_work_clears_watchdog(self) -> None: - server, rpc = coordinator() - install_fake_bundle_builder(server) - server.template_refresh_failure_exit_seconds = 10.0 - server.min_ready_miners = 10_000 - state = client(1) - state.send = lambda _payload: None # type: ignore[method-assign] - server.clients = [state] # type: ignore[assignment] - server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] - server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] - original_tip = rpc.tip - original_template = rpc.template - - self.assertEqual(server.poll_qbit_tip_template_once(), 1) - published_job = state.active_job - self.assertIsNotNone(published_job) - - _advance_fake_tip(rpc, "aa" * 32, 11) - self.assertTrue(server.observe_tip_for_refresh(rpc.tip)) - rpc.tip = original_tip - rpc.template = original_template - self.assertTrue(server.observe_tip_for_refresh(original_tip)) - divergence_started = ( - server._progress_publication_divergence_since_monotonic - ) - self.assertIsNotNone(divergence_started) - assert divergence_started is not None - self.assertTrue( - server.publication_progress_failure_expired( - divergence_started + 10.001 - ) - ) - - # The coherent A poll has no replacement delivery to make because the - # existing A job is already current. Completion itself must close the - # A -> B -> A publication-divergence epoch. - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - self.assertIs(state.active_job, published_job) - self.assertIsNone( - server._progress_publication_divergence_since_monotonic - ) - self.assertFalse( - server.publication_progress_failure_expired(10_000_000_000.0) - ) - - def test_rapid_contention_coalesces_one_latest_tip_retry(self) -> None: + def test_rapid_contention_observations_are_latest_wins(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) state = client(1) server.clients = [state] # type: ignore[assignment] + server._ensure_tip_refresh_state() server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] self.assertTrue(server.observe_tip_first_seen(rpc.tip)) + server._ensure_tip_refresh_service().clear_retry_for_test() - build_started = threading.Event() - release_build = threading.Event() - original_build = server.build_shared_job_bundle - build_lock = threading.Lock() + tip_a = rpc.tip + tip_b = "55" * 32 + tip_c = "66" * 32 + tip_d = "77" * 32 + build_started = [threading.Event() for _index in range(2)] + release_old = threading.Event() + release_latest = threading.Event() + trigger_submitted = { + tip_b: threading.Event(), + tip_c: threading.Event(), + tip_d: threading.Event(), + } + tip_refresh = server._ensure_tip_refresh_service() + original_submit = tip_refresh.submit_trigger + service = server._ensure_job_bundle_service() + original_build = service.build_shared_job_bundle + builder_lock = threading.Lock() + build_calls = 0 active_builders = 0 - maximum_builders = 0 + max_active_builders = 0 - def blocked_first_build(*args: object, **kwargs: object) -> object: - nonlocal active_builders, maximum_builders - with build_lock: + def record_submit(trigger: object) -> Future[int]: + completion = original_submit(trigger) # type: ignore[arg-type] + submitted = trigger_submitted.get(getattr(trigger, "tip_hash")) + if submitted is not None: + submitted.set() + return completion + + def blocking_build(*args: object, **kwargs: object) -> object: + nonlocal active_builders, build_calls, max_active_builders + with builder_lock: + call_index = build_calls + build_calls += 1 active_builders += 1 - maximum_builders = max(maximum_builders, active_builders) + max_active_builders = max(max_active_builders, active_builders) + build_started[call_index].set() try: - if not build_started.is_set(): - build_started.set() - if not release_build.wait(5): - raise AssertionError("test did not release obsolete build") + if call_index == 0 and not release_old.wait(5): + raise AssertionError("test did not release oldest build") + if call_index == 1 and not release_latest.wait(5): + raise AssertionError("test did not release latest build") return original_build(*args, **kwargs) # type: ignore[arg-type] finally: - with build_lock: + with builder_lock: active_builders -= 1 - server.build_shared_job_bundle = blocked_first_build # type: ignore[method-assign] + tip_refresh.submit_trigger = record_submit # type: ignore[method-assign] + service.build_shared_job_bundle = blocking_build # type: ignore[method-assign] sent_tips: list[str] = [] def record_send(payload: dict[str, object]) -> None: - if payload["method"] == "mining.notify": - assert state.active_job is not None - sent_tips.append(str(state.active_job.template["previousblockhash"])) + if payload["method"] != "mining.notify": + return + assert state.active_job is not None + sent_tips.append(str(state.active_job.template["previousblockhash"])) state.send = record_send # type: ignore[method-assign] - owner_errors: list[BaseException] = [] + results = {label: [] for label in ("old", "b", "c", "d")} + errors: dict[str, list[BaseException]] = { + label: [] for label in ("old", "b", "c", "d") + } - def own_refresh() -> None: + def poll(label: str) -> None: try: - server.poll_qbit_tip_template_once() - except BaseException as exc: # noqa: BLE001 - asserted below - owner_errors.append(exc) + results[label].append(server.poll_qbit_tip_template_once()) + except BaseException as exc: # noqa: BLE001 - surface thread errors + errors[label].append(exc) - owner = threading.Thread(target=own_refresh) - owner.start() + polls = { + label: threading.Thread(target=poll, args=(label,)) + for label in ("old", "b", "c", "d") + } + polls["old"].start() try: - self.assertTrue(build_started.wait(5)) - # blockpoll observes B while the A owner is still building. - _advance_fake_tip(rpc, "55" * 32, 11) - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - - # blockwait observes C and only wakes the driver; it never enters - # the heavy lane itself. - blockwait_calls = 0 - - def blockwait_once(known_tip: str) -> str: - nonlocal blockwait_calls - blockwait_calls += 1 - if blockwait_calls == 1: - _advance_fake_tip(rpc, "66" * 32, 12) - return rpc.tip - server.stop_event.set() - return known_tip - - server.blockwait_once = blockwait_once # type: ignore[method-assign] - blockwait = threading.Thread(target=server.blockwait_loop) - blockwait.start() - blockwait.join(5) - self.assertFalse(blockwait.is_alive()) - server.stop_event.clear() - - # Accepted-block finalization observes D and also only wakes the - # same driver. - _advance_fake_tip(rpc, "77" * 32, 13) - self.assertEqual( - server.refresh_jobs_after_accepted_block( - block_height=13, - block_hash=rpc.tip, - heartbeat_name="block_submitter", - ), - 0, - ) - - self.assertEqual(server.latest_detected_tip[0], "77" * 32) - self.assertEqual(maximum_builders, 1) - self.assertEqual(sent_tips, []) - finally: - release_build.set() - owner.join(5) + self.assertTrue(build_started[0].wait(5)) - self.assertFalse(owner.is_alive()) - self.assertEqual(len(owner_errors), 1) - self.assertIsInstance(owner_errors[0], TemplateRefreshBlocked) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) + _advance_fake_tip(rpc, tip_b, 11) + polls["b"].start() + self.assertTrue(trigger_submitted[tip_b].wait(5)) + self.assertFalse(build_started[1].is_set()) - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 1) + _advance_fake_tip(rpc, tip_c, 12) + polls["c"].start() + self.assertTrue(trigger_submitted[tip_c].wait(5)) + self.assertFalse(build_started[1].is_set()) + + _advance_fake_tip(rpc, tip_d, 13) + polls["d"].start() + self.assertTrue(trigger_submitted[tip_d].wait(5)) + self.assertFalse(build_started[1].is_set()) + + scheduler = tip_refresh.scheduler_snapshot() + self.assertIsNotNone(scheduler.active) + self.assertIsNotNone(scheduler.pending) + self.assertEqual(scheduler.active.tip_hash, tip_a) # type: ignore[union-attr] + self.assertEqual(scheduler.pending.tip_hash, tip_d) # type: ignore[union-attr] + self.assertEqual(scheduler.pending_capacity, 1) + self.assertEqual(build_calls, 1) + self.assertEqual(max_active_builders, 1) + + release_old.set() + self.assertTrue(build_started[1].wait(5)) + self.assertEqual(server.current_tip_first_seen[0], tip_a) + release_latest.set() + self.assertTrue(tip_refresh.wait_for_scheduler_idle_for_test(5.0)) finally: + release_old.set() + release_latest.set() + for poll_thread in polls.values(): + poll_thread.join(5) + tip_refresh.wait_for_scheduler_idle_for_test(5.0) server.shutdown_tip_refresh_executor() - self.assertEqual(maximum_builders, 1) - self.assertEqual(sent_tips, ["77" * 32]) - self.assertEqual(server.current_tip_first_seen[0], "77" * 32) + self.assertTrue(all(not thread.is_alive() for thread in polls.values())) + self.assertEqual(results["old"], []) + self.assertEqual(len(errors["old"]), 1) + self.assertIsInstance(errors["old"][0], TemplateRefreshBlocked) + for label in ("b", "c", "d"): + self.assertEqual(results[label], [0]) + self.assertEqual(errors[label], []) + self.assertEqual(build_calls, 2) + self.assertEqual(max_active_builders, 1) + self.assertEqual(sent_tips, [tip_d]) + scheduler_metrics = tip_refresh.metrics_snapshot() + self.assertEqual(scheduler_metrics["trigger_supersessions"], 3) + self.assertEqual(scheduler_metrics["trigger_coalesces"], 2) + with server._ensure_job_bundle_service()._scheduler_lock: + self.assertIsNone(server._ensure_job_bundle_service()._active) + self.assertIsNone(server._ensure_job_bundle_service()._retiring) + self.assertIsNone(server._ensure_job_bundle_service()._pending) self.assertFalse(server.tip_refresh_is_pending()) def _legacy_rapid_contention_observations_are_latest_wins(self) -> None: @@ -1957,11 +1795,11 @@ def _legacy_rapid_contention_observations_are_latest_wins(self) -> None: server.clients = [state] # type: ignore[assignment] server._ensure_tip_refresh_state() refresh_lock = _ControlledTipRefreshLock() - server._tip_refresh_lock = refresh_lock # type: ignore[assignment] + server._ensure_tip_refresh_service().replace_refresh_lock_for_test(refresh_lock) server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() + server._ensure_tip_refresh_service().clear_retry_for_test() tip_a = rpc.tip published_tip_a_sequence = server.current_tip_observation_sequence @@ -2012,7 +1850,8 @@ def ordered_rpc_call( return original_rpc_call(method, params) rpc.call = ordered_rpc_call # type: ignore[method-assign] - original_observe = server.observe_tip_for_refresh + tip_refresh = server._ensure_tip_refresh_service() + original_observe = tip_refresh.observe_tip def record_observation(*args: object, **kwargs: object) -> bool: observed = original_observe(*args, **kwargs) # type: ignore[arg-type] @@ -2027,7 +1866,7 @@ def record_observation(*args: object, **kwargs: object) -> bool: tip_c_observed.set() return observed - server.observe_tip_for_refresh = record_observation # type: ignore[method-assign] + tip_refresh.observe_tip = record_observation # type: ignore[method-assign] sent_tips: list[str] = [] def record_send(payload: dict[str, object]) -> None: @@ -2077,7 +1916,7 @@ def poll(label: str) -> None: tip_c_poll.start() refresh_lock.next_probe_gate().set() self.assertTrue(tip_c_observed.wait(5)) - pending_tip_c = server._tip_refresh_pending_token + pending_tip_c = server._ensure_tip_refresh_service().snapshot().pending_token detected_tip_c_sequence = server.latest_detected_tip[1] self.assertEqual(server.latest_detected_tip[0], tip_c) self.assertEqual(server.current_tip_first_seen[0], tip_a) @@ -2099,7 +1938,7 @@ def poll(label: str) -> None: server.current_tip_observation_sequence, published_tip_a_sequence, ) - self.assertEqual(server._tip_refresh_pending_token, pending_tip_c) + self.assertEqual(server._ensure_tip_refresh_service().snapshot().pending_token, pending_tip_c) finally: release_stale_probe.set() release_bundle.set() @@ -2109,7 +1948,7 @@ def poll(label: str) -> None: self.assertEqual(results["old"], []) self.assertEqual(len(errors["old"]), 1) self.assertIsInstance(errors["old"][0], TemplateRefreshBlocked) - self.assertEqual(server._tip_refresh_pending_token, pending_tip_c) + self.assertEqual(server._ensure_tip_refresh_service().snapshot().pending_token, pending_tip_c) self.assertEqual(sent_tips, []) refresh_lock.allow_waiters_to_acquire.set() @@ -2176,7 +2015,7 @@ def poll() -> None: ) self.assertEqual(state.active_job.template["previousblockhash"], rpc.tip) self.assertGreaterEqual( - server.tip_refresh_cancellation_counts["client_lock"], + server._ensure_tip_refresh_service().metrics_snapshot()["cancellation_counts"]["client_lock"], 1, ) @@ -2187,8 +2026,8 @@ def test_publication_gate_waits_for_mutation_and_rechecks_supersession(self) -> notifications: list[dict[str, object]] = [] state.send = notifications.append # type: ignore[method-assign] server.clients = [state] # type: ignore[assignment] - gate = ObservedPayoutGate(server._payout_state_delivery_gate) - server._payout_state_delivery_gate = gate # type: ignore[assignment] + gate = ObservedPayoutGate(server._payout_state_service._delivery_gate) + server._payout_state_service._delivery_gate = gate # type: ignore[assignment] poll_done = threading.Event() errors: list[BaseException] = [] @@ -2224,12 +2063,15 @@ def poll() -> None: 1, ) self.assertEqual(state.active_job.template["previousblockhash"], rpc.tip) - self.assertEqual(server.tip_refresh_cancellation_counts["payout_gate"], 0) + self.assertEqual(server._ensure_tip_refresh_service().metrics_snapshot()["cancellation_counts"]["payout_gate"], 0) def test_obsolete_backlog_never_starts_the_unsubmitted_fleet(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 2 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) clients = [client(index + 1) for index in range(8)] observed_locks = [ObservedRLock(), ObservedRLock()] for state, observed_lock in zip(clients, observed_locks): @@ -2292,7 +2134,7 @@ def poll() -> None: self.assertEqual(notifications, {state.connection_id for state in clients}) self.assertGreaterEqual( - server.tip_refresh_cancellation_counts["client_lock"], + server._ensure_tip_refresh_service().metrics_snapshot()["cancellation_counts"]["client_lock"], 2, ) @@ -2300,6 +2142,9 @@ def test_obsolete_executor_queue_entry_is_canceled_and_counted(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 2 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) server._ensure_tip_refresh_state() class ObservedExecutor: @@ -2319,7 +2164,9 @@ def shutdown(self, **kwargs: object) -> None: self.delegate.shutdown(**kwargs) # type: ignore[arg-type] observed_executor = ObservedExecutor() - server._tip_refresh_executor = observed_executor # type: ignore[assignment] + server._ensure_tip_refresh_service().replace_executor_for_test( + observed_executor + ) admitted = client(1) queued = client(2) server.clients = [admitted, queued] # type: ignore[assignment] @@ -2335,14 +2182,15 @@ def block_notify(payload: dict[str, object]) -> None: admitted.send = block_notify # type: ignore[method-assign] queued.send = queued_notifications.append # type: ignore[method-assign] - original_record_cancellation = server._record_tip_refresh_cancellation + tip_refresh = server._ensure_tip_refresh_service() + original_record_cancellation = tip_refresh.record_cancellation def record_cancellation(stage: str) -> None: original_record_cancellation(stage) if stage == "executor_queue": queued_canceled.set() - server._record_tip_refresh_cancellation = record_cancellation # type: ignore[method-assign] + tip_refresh.record_cancellation = record_cancellation # type: ignore[method-assign] errors: list[BaseException] = [] def poll() -> None: @@ -2369,7 +2217,7 @@ def poll() -> None: self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0], TemplateRefreshBlocked) self.assertEqual( - server.tip_refresh_cancellation_counts["executor_queue"], + server._ensure_tip_refresh_service().metrics_snapshot()["cancellation_counts"]["executor_queue"], 1, ) @@ -2404,12 +2252,12 @@ def poll() -> None: self.assertTrue(observed_lock.acquire_attempted.wait(5)) self.advance_tip(server, rpc, tip_b, height=11) self.advance_tip(server, rpc, tip_c, height=12) - newest_pending_token = server._tip_refresh_pending_token + newest_pending_token = server._ensure_tip_refresh_service().snapshot().pending_token self.assertEqual(server.latest_detected_tip[0], tip_c) self.assertEqual(server.current_tip_first_seen[0], tip_a) self.assertTrue(poll_done.wait(5)) self.assertTrue(server.tip_refresh_is_pending()) - self.assertEqual(server._tip_refresh_pending_token, newest_pending_token) + self.assertEqual(server._ensure_tip_refresh_service().snapshot().pending_token, newest_pending_token) self.assertEqual(server.latest_detected_tip[0], tip_c) self.assertEqual(server.current_tip_first_seen[0], tip_a) self.assertEqual(notifications, []) @@ -2436,6 +2284,9 @@ def test_admitted_old_send_finishes_before_new_tip_delivery(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 2 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) admitted = client(1) waiting = client(2) waiting_lock = ObservedRLock() @@ -2574,7 +2425,7 @@ def delivery_cancelable( yield self.Admission() state.job_update_lock = AdvancingLock() # type: ignore[assignment] - server._payout_state_delivery_gate = AdvancingGate() # type: ignore[assignment] + server._payout_state_service._delivery_gate = AdvancingGate() # type: ignore[assignment] original_stamp = server.stamp_job_for_client def advancing_stamp(*args: object, **kwargs: object) -> object: @@ -2609,10 +2460,10 @@ def advancing_stamp(*args: object, **kwargs: object) -> object: } for phase, expected in expected_phases.items(): self.assertAlmostEqual( - server.job_build_phase_seconds[phase], + server._ensure_job_bundle_service()._phase_seconds[phase], expected, ) - self.assertAlmostEqual(server.job_build_seconds_sum, 20.0) + self.assertAlmostEqual(server._ensure_job_bundle_service()._build_seconds_sum, 20.0) self.assertAlmostEqual(sum(expected_phases.values()), 20.0) metrics = server.metrics_payload() self.assertIn( @@ -2679,6 +2530,9 @@ def test_routine_same_tip_observation_keeps_active_fanout_valid(self) -> None: install_fake_bundle_builder(server) server.reorg_reconciler_enabled = True server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first, second, third = client(1), client(2), client(3) clients = [first, second, third] server.clients = clients # type: ignore[assignment] @@ -2742,12 +2596,15 @@ def poll() -> None: self.assertEqual(errors, []) self.assertEqual(results, [3]) self.assertEqual(notifications, {1, 2, 3}) - self.assertFalse(server._tip_refresh_retry.is_set()) + self.assertFalse(server._ensure_tip_refresh_service().snapshot().retry_requested) - def test_repeated_same_tip_contenders_do_not_supersede_active_fanout(self) -> None: + def test_same_tip_contention_probe_keeps_active_fanout_valid(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first, second = client(1), client(2) server.clients = [first, second] # type: ignore[assignment] server._ensure_tip_refresh_state() @@ -2757,7 +2614,7 @@ def test_repeated_same_tip_contenders_do_not_supersede_active_fanout(self) -> No lambda **_kwargs: True ) self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() + server._ensure_tip_refresh_service().clear_retry_for_test() first_send_started = threading.Event() release_first_send = threading.Event() @@ -2799,17 +2656,16 @@ def poll() -> None: try: self.assertTrue(first_send_started.wait(5)) with server.lock: - active = server._active_tip_refresh + active = server._ensure_tip_refresh_service().active_refresh_snapshot() active_sequence = server.current_tip_observation_sequence self.assertIsNotNone(active) assert active is not None - contender_results = [ - server.poll_qbit_tip_template_once() - for _attempt in range(10) - ] - self.assertEqual(contender_results, [0] * 10) - self.assertFalse(active[1].is_set()) + server._probe_tip_while_refresh_waiting() + active = server._ensure_tip_refresh_service().active_refresh_snapshot() + self.assertIsNotNone(active) + assert active is not None + self.assertFalse(active.cancelling) self.assertEqual( server.current_tip_observation_sequence, active_sequence, @@ -2818,36 +2674,33 @@ def poll() -> None: finally: release_first_send.set() active_poll.join(5) + server.shutdown_tip_refresh_executor() self.assertFalse(active_poll.is_alive()) self.assertEqual(errors, []) self.assertEqual(results, [2]) self.assertEqual(notifications, [1, 2]) self.assertFalse(server.tip_refresh_is_pending()) - self.assertTrue(server._consume_tip_refresh_retry()) - self.assertFalse(server._consume_tip_refresh_retry()) - try: - self.assertEqual(server.poll_qbit_tip_template_once(), 0) - finally: - server.shutdown_tip_refresh_executor() - self.assertEqual(notifications, [1, 2]) def _legacy_same_tip_contention_probe_keeps_active_fanout_valid(self) -> None: server, rpc = coordinator() install_fake_bundle_builder(server) server.tip_refresh_max_workers = 1 + server._ensure_tip_refresh_service().reconfigure_for_test( + max_workers=server.tip_refresh_max_workers + ) first, second = client(1), client(2) server.clients = [first, second] # type: ignore[assignment] server._ensure_tip_refresh_state() refresh_lock = _ControlledTipRefreshLock() - server._tip_refresh_lock = refresh_lock # type: ignore[assignment] + server._ensure_tip_refresh_service().replace_refresh_lock_for_test(refresh_lock) server.ensure_reorg_reconciled_for_tip = lambda _tip: True # type: ignore[method-assign] server.qbit_chain_view_untrusted = lambda: False # type: ignore[method-assign] server.ensure_reorg_reconciled_for_current_tip = ( # type: ignore[method-assign] lambda **_kwargs: True ) self.assertTrue(server.observe_tip_first_seen(rpc.tip)) - server._tip_refresh_retry.clear() + server._ensure_tip_refresh_service().clear_retry_for_test() first_send_started = threading.Event() release_first_send = threading.Event() @@ -2898,7 +2751,7 @@ def poll(label: str) -> None: try: self.assertTrue(first_send_started.wait(5)) with server.lock: - active = server._active_tip_refresh + active = server._ensure_tip_refresh_service().active_refresh_snapshot() active_sequence = server.current_tip_observation_sequence self.assertIsNotNone(active) assert active is not None @@ -2906,7 +2759,10 @@ def poll(label: str) -> None: waiting_poll.start() refresh_lock.next_probe_gate().set() self.assertTrue(contention_probe_finished.wait(5)) - self.assertFalse(active[1].is_set()) + active = server._ensure_tip_refresh_service().active_refresh_snapshot() + self.assertIsNotNone(active) + assert active is not None + self.assertFalse(active.cancelling) self.assertEqual( server.current_tip_observation_sequence, active_sequence, @@ -2960,8 +2816,8 @@ def advance_during_post_fanout_tip_check( self.assertIsNotNone(state.active_job) self.assertEqual(state.active_job.payout_state_generation, 0) - self.assertEqual(server._payout_state_generation, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertEqual(server._payout_state_service._generation, 1) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_payout_mutation_waits_for_prepared_network_delivery(self) -> None: server, _rpc = coordinator() @@ -3001,18 +2857,18 @@ def mutate() -> None: mutation_thread.start() self.assertTrue(mutation_started.wait(5)) self.assertFalse(mutation_completed.wait(0.1)) - self.assertEqual(server._payout_state_generation, 0) - with server._payout_state_delivery_gate._condition: + self.assertEqual(server._payout_state_service._generation, 0) + with server._payout_state_service._delivery_gate._condition: self.assertTrue( - server._payout_state_delivery_gate._publisher_waiting + server._payout_state_service._delivery_gate._publisher_waiting ) self.assertIsNone( - server._payout_state_delivery_gate._mutation_owner + server._payout_state_service._delivery_gate._mutation_owner ) self.assertTrue( - server._payout_state_prepare_lock.acquire(timeout=1) + server._payout_state_service._prepare_lock.acquire(timeout=1) ) - server._payout_state_prepare_lock.release() + server._payout_state_service._prepare_lock.release() finally: release_send.set() mutation_thread.join(5) @@ -3022,13 +2878,13 @@ def mutate() -> None: self.assertFalse(mutation_thread.is_alive()) self.assertFalse(poll_thread.is_alive()) self.assertTrue(mutation_completed.is_set()) - self.assertEqual(server._payout_state_generation, 1) + self.assertEqual(server._payout_state_service._generation, 1) self.assertEqual(len(poll_results) + len(poll_errors), 1) if poll_errors: self.assertIsInstance(poll_errors[0], TemplateRefreshBlocked) else: self.assertEqual(poll_results, [1]) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_prepared_disconnection_after_admission_releases_cancellation_gate(self) -> None: server, _rpc = coordinator() @@ -3053,7 +2909,7 @@ def test_prepared_disconnection_after_admission_releases_cancellation_gate(self) sequence, ) cancellation = _FanoutCancellation() - original_gate = server._payout_state_delivery_gate + original_gate = server._payout_state_service._delivery_gate class RemoveClientAtAdmission: @contextmanager @@ -3076,7 +2932,7 @@ def delivery_cancelable( def mutation(self) -> object: return original_gate.mutation() - server._payout_state_delivery_gate = RemoveClientAtAdmission() + server._payout_state_service._delivery_gate = RemoveClientAtAdmission() result = server.send_prepared_job( state, @@ -3139,13 +2995,13 @@ def test_publication_block_fences_escaped_prepared_bundle(self) -> None: self.assertEqual(result.result, "skipped") self.assertEqual(sent, []) self.assertIsNone(state.active_job) - self.assertTrue(server._payout_state_delivery_gate._delivery_blocked) + self.assertTrue(server._payout_state_service._delivery_gate._delivery_blocked) published = server._publish_payout_state_candidate( server._current_payout_state_candidate() ) self.assertEqual(published, 1) - with server._payout_state_delivery_gate.delivery_cancelable( + with server._payout_state_service._delivery_gate.delivery_cancelable( lambda: False, generation=1, priority=True, @@ -3187,7 +3043,7 @@ def advance_between_clients(*args: object, **kwargs: object) -> bool: self.assertIsNotNone(second.active_job) self.assertEqual(first.active_job.payout_state_generation, 0) self.assertEqual(second.active_job.payout_state_generation, 1) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) def test_payout_change_after_client_selection_coalesces_full_same_tip_set( self, @@ -3208,7 +3064,8 @@ def test_payout_change_after_client_selection_coalesces_full_same_tip_set( self.assertEqual(second.active_job.payout_state_generation, 0) server.clients = [first, second] # type: ignore[assignment] - original_prepare = server.prepare_tip_refresh_bundle + tip_refresh = server._ensure_tip_refresh_service() + original_prepare = tip_refresh.prepare_bundle advanced = False def advance_before_prepare(*args: object, **kwargs: object) -> object: @@ -3218,7 +3075,7 @@ def advance_before_prepare(*args: object, **kwargs: object) -> object: server._advance_payout_state_generation() return original_prepare(*args, **kwargs) # type: ignore[arg-type] - server.prepare_tip_refresh_bundle = advance_before_prepare # type: ignore[method-assign] + tip_refresh.prepare_bundle = advance_before_prepare # type: ignore[method-assign] self.assertEqual(server.poll_qbit_tip_template_once(), 2) finally: @@ -3247,13 +3104,12 @@ def test_payout_change_during_build_coalesces_latest_generation(self) -> None: def record_send(payload: dict[str, object]) -> None: if payload["method"] == "mining.notify": assert state.active_job is not None - sent_generations.append( - state.active_job.payout_state_generation - ) + sent_generations.append(state.active_job.payout_state_generation) state.send = record_send # type: ignore[method-assign] server.clients = [state] # type: ignore[assignment] - original_build = server.build_shared_job_bundle + job_bundles = server._ensure_job_bundle_service() + original_build = job_bundles.build_shared_job_bundle build_calls = 0 def advance_first_build(*args: object, **kwargs: object) -> object: @@ -3263,7 +3119,7 @@ def advance_first_build(*args: object, **kwargs: object) -> object: self.assertEqual(server._advance_payout_state_generation(), 1) return original_build(*args, **kwargs) # type: ignore[arg-type] - server.build_shared_job_bundle = advance_first_build # type: ignore[method-assign] + job_bundles.build_shared_job_bundle = advance_first_build # type: ignore[method-assign] try: self.assertEqual(server.poll_qbit_tip_template_once(), 1) @@ -3277,7 +3133,7 @@ def advance_first_build(*args: object, **kwargs: object) -> object: self.assertTrue( all( bundle.payout_state_generation == 1 - for bundle in server._job_bundle_cache.values() + for bundle in job_bundles.bundle_cache_snapshot() ) ) self.assertFalse(server.tip_refresh_is_pending()) @@ -3319,7 +3175,9 @@ def submit(self, function: object, *args: object) -> Future[object]: return future raise RuntimeError("executor rejected queued fanout task") - server.tip_refresh_executor = lambda: RejectSecondSubmission() # type: ignore[method-assign] + server._ensure_tip_refresh_service().executor = ( # type: ignore[method-assign] + lambda: RejectSecondSubmission() + ) errors: list[BaseException] = [] def poll() -> None: @@ -3340,8 +3198,8 @@ def poll() -> None: self.assertEqual(later_notifications, []) self.assertEqual(len(errors), 1) self.assertIsInstance(errors[0], RuntimeError) - self.assertIsNone(server._active_tip_refresh) - self.assertTrue(server._tip_refresh_retry.is_set()) + self.assertIsNone(server._ensure_tip_refresh_service().active_refresh_snapshot()) + self.assertTrue(server._ensure_tip_refresh_service().snapshot().retry_requested) if __name__ == "__main__": diff --git a/tests/test_prism_vardiff.py b/tests/test_prism_vardiff.py index 65e3b60..a84ce5b 100644 --- a/tests/test_prism_vardiff.py +++ b/tests/test_prism_vardiff.py @@ -76,7 +76,7 @@ def failing_build(client: ClientState, *, clean_jobs: bool) -> object: server.note_vardiff_submitted_share(state) server.note_vardiff_accepted_share(state, FakeJob(Decimal("1"))) # type: ignore[arg-type] - self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 1) self.assertIsNone(state.pending_share_difficulty) # rolled back, not left at the new value self.assertEqual(state.share_difficulty, Decimal("1")) # unchanged self.assertEqual(state.active_job_ids, {"old-job"}) # old job retained, still submittable @@ -284,7 +284,7 @@ def fail_stamp(*_args: object, **_kwargs: object) -> None: ), window_state, ) - self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 1) self.assertEqual(server.vardiff_idle_task_failures, 1) def test_idle_cached_bundle_requires_live_reorg_trust(self) -> None: server = coordinator() @@ -323,8 +323,8 @@ def test_idle_retarget_defers_detected_payout_source_during_tip_divergence( state = client() prepare_idle_client(server, state, tip=old_tip) install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - published_artifacts = server._template_artifacts + repository = server._ensure_job_bundle_service().template_repository + published_artifacts = repository.current_artifacts() assert published_artifacts is not None detected_template = gbt_template(new_tip, height=11) detected_artifacts = CachedTemplateArtifacts( @@ -349,10 +349,10 @@ def test_idle_retarget_defers_detected_payout_source_during_tip_divergence( template_generation=published_artifacts.generation, template_artifacts=published_artifacts, ) - with server._job_cache_lock: - server._template_artifacts = detected_artifacts - server._published_payout_state = dataclass_replace( - server._published_payout_state, + repository.replace_for_test(detected_artifacts) + with server._payout_state_service._lock: + server._payout_state_service._published = dataclass_replace( + server._payout_state_service._published, source_tip_hash=new_tip, ) window_state = ( @@ -391,8 +391,8 @@ def test_replacement_tip_build_survives_repeated_idle_sweeps(self) -> None: state = client() prepare_idle_client(server, state, tip=old_tip) install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - published_artifacts = server._template_artifacts + repository = server._ensure_job_bundle_service().template_repository + published_artifacts = repository.current_artifacts() assert published_artifacts is not None detected_template = gbt_template(new_tip, height=11) detected_artifacts = CachedTemplateArtifacts( @@ -417,8 +417,7 @@ def test_replacement_tip_build_survives_repeated_idle_sweeps(self) -> None: template_generation=published_artifacts.generation, template_artifacts=published_artifacts, ) - with server._job_cache_lock: - server._template_artifacts = detected_artifacts + repository.replace_for_test(detected_artifacts) server._ensure_job_cache_state() replacement_cancellation = _JobBuildCancellation(timeout_seconds=60.0) replacement_request = SimpleNamespace( @@ -426,8 +425,8 @@ def test_replacement_tip_build_survives_repeated_idle_sweeps(self) -> None: cancellation=replacement_cancellation, ) replacement_flight = SimpleNamespace(request=replacement_request) - with server._job_build_scheduler_lock: - server._job_build_active = replacement_flight + with server._ensure_job_bundle_service()._scheduler_lock: + server._ensure_job_bundle_service()._active = replacement_flight build_called = threading.Event() window_started = state.vardiff_window_started_monotonic @@ -453,15 +452,15 @@ def unexpected_idle_build(_request: object) -> CachedJobBundle: with self.assertRaises(JobBuildCancelled): racing_idle_promise.result() - with server._job_build_scheduler_lock: - self.assertIs(server._job_build_active, replacement_flight) + with server._ensure_job_bundle_service()._scheduler_lock: + self.assertIs(server._ensure_job_bundle_service()._active, replacement_flight) self.assertFalse(replacement_cancellation.is_set()) self.assertTrue(racing_idle_cancellation.is_set()) self.assertFalse(build_called.is_set()) self.assertEqual(state.vardiff_window_started_monotonic, window_started) self.assertEqual(server.vardiff_idle_skip_counts["superseded"], 4) - with server._job_build_scheduler_lock: - server._job_build_active = None + with server._ensure_job_bundle_service()._scheduler_lock: + server._ensure_job_bundle_service()._active = None server.shutdown_vardiff_idle_executor() def test_idle_shared_build_does_not_retry_scheduler_divergence_race( self, @@ -472,9 +471,10 @@ def test_idle_shared_build_does_not_retry_scheduler_divergence_race( state = client() prepare_idle_client(server, state, tip=old_tip) install_idle_job_cache(server, tip=old_tip) - with server._job_cache_lock: - old_artifacts = server._template_artifacts - server._job_bundle_cache.clear() + repository = server._ensure_job_bundle_service().template_repository + old_artifacts = repository.current_artifacts() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() assert old_artifacts is not None now = time.monotonic() server.current_tip_first_seen = (old_tip, now) @@ -495,12 +495,13 @@ def test_idle_shared_build_does_not_retry_scheduler_divergence_race( cancellation=replacement_cancellation, ) ) - with server._job_build_scheduler_lock: - server._job_build_active = replacement_flight + with server._ensure_job_bundle_service()._scheduler_lock: + server._ensure_job_bundle_service()._active = replacement_flight request_builds = 0 admission_attempts = 0 idle_cancellation: _JobBuildCancellation | None = None - original_request_job_build = server._request_job_build + service = server._ensure_job_bundle_service() + original_request_job_build = service.request_build def make_idle_request( _artifacts: CachedTemplateArtifacts, @@ -525,8 +526,8 @@ def detect_before_admission(request: object) -> Future[CachedJobBundle]: server.tip_refresh_divergence_started_monotonic = time.monotonic() return original_request_job_build(request) # type: ignore[arg-type] - server._new_job_build_request = make_idle_request # type: ignore[method-assign] - server._request_job_build = detect_before_admission # type: ignore[method-assign] + service.new_build_request = make_idle_request # type: ignore[method-assign] + service.request_build = detect_before_admission # type: ignore[method-assign] server._schedule_tip_refresh_retry = lambda: None # type: ignore[method-assign] assert state.worker is not None @@ -535,14 +536,14 @@ def detect_before_admission(request: object) -> Future[CachedJobBundle]: self.assertEqual(request_builds, 1) self.assertEqual(admission_attempts, 1) - self.assertIs(server._template_artifacts, old_artifacts) + self.assertIs(repository.current_artifacts(), old_artifacts) self.assertIsNotNone(idle_cancellation) assert idle_cancellation is not None self.assertTrue(idle_cancellation.is_set()) self.assertFalse(replacement_cancellation.is_set()) - with server._job_build_scheduler_lock: - self.assertIs(server._job_build_active, replacement_flight) - server._job_build_active = None + with server._ensure_job_bundle_service()._scheduler_lock: + self.assertIs(server._ensure_job_bundle_service()._active, replacement_flight) + server._ensure_job_bundle_service()._active = None server.shutdown_vardiff_idle_executor() def test_idle_cached_collection_bundle_refreshes_readiness(self) -> None: server = coordinator() @@ -550,13 +551,16 @@ def test_idle_cached_collection_bundle_refreshes_readiness(self) -> None: prepare_idle_client(server, state) ready_bundle = install_idle_job_cache(server) assert state.worker is not None - with server._job_cache_lock: - artifacts = server._template_artifacts - assert artifacts is not None + artifacts = ( + server._ensure_job_bundle_service() + .template_repository.current_artifacts() + ) + assert artifacts is not None + with server._ensure_job_bundle_service()._cache_lock: collection_key = server._job_bundle_key( artifacts, mode="collection", - payout_state_generation=server._payout_state_generation, + payout_state_generation=server._payout_state_service._generation, payout_artifact_generation=0, worker=state.worker, ) @@ -569,18 +573,19 @@ def test_idle_cached_collection_bundle_refreshes_readiness(self) -> None: state.worker.p2mr_program_hex, ), ) - server._job_bundle_cache.clear() - server._job_bundle_cache[collection_key] = collection_bundle - server._pool_ready_latched = False + server._ensure_job_bundle_service()._bundle_cache.clear() + server._ensure_job_bundle_service()._bundle_cache[collection_key] = collection_bundle + server._ensure_job_bundle_service().set_ready_for_test(False) server.min_ready_miners = 3 + server._ensure_job_bundle_service().set_min_ready_miners_for_test(3) server.accepted_share_stats = lambda: (3, 3) # type: ignore[method-assign] rebuilt = threading.Event() sent: list[dict[str, object]] = [] def build_ready(_request: object) -> CachedJobBundle: rebuilt.set() - with server._job_cache_lock: - server._job_bundle_cache[ready_bundle.key] = ready_bundle + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache[ready_bundle.key] = ready_bundle return ready_bundle state.send = sent.append # type: ignore[method-assign] @@ -589,7 +594,7 @@ def build_ready(_request: object) -> CachedJobBundle: self.assertEqual(server.vardiff_idle_sweep_once(), 1) server.shutdown_vardiff_idle_executor() - self.assertTrue(server._pool_ready_latched) + self.assertTrue(server._ensure_job_bundle_service().ready_latched()) self.assertTrue(rebuilt.is_set()) self.assertEqual( [payload["method"] for payload in sent], @@ -631,7 +636,7 @@ def record(payload: dict[str, object]) -> None: delivered.set() state.send = record # type: ignore[method-assign] - server._bind_cached_bundle_to_artifacts = ( # type: ignore[method-assign] + server._ensure_job_bundle_service().bind_cached_bundle = ( # type: ignore[method-assign] bind_current_observation ) @@ -721,8 +726,8 @@ def test_stuck_bundle_builder_does_not_stale_idle_sweep_heartbeat(self) -> None: state = client() prepare_idle_client(server, state) bundle = install_idle_job_cache(server) - with server._job_cache_lock: - server._job_bundle_cache.clear() + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() state.send = lambda _payload: None # type: ignore[method-assign] build_started = threading.Event() release_build = threading.Event() @@ -736,7 +741,7 @@ def blocked_build( raise AssertionError("idle retarget bundle build was not released") return bundle - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] + server._ensure_job_bundle_service().build_shared_job_bundle = blocked_build # type: ignore[method-assign] server._record_heartbeat("vardiff_idle_sweep") heartbeat_before = server._heartbeats["vardiff_idle_sweep"] started = time.monotonic() @@ -760,12 +765,13 @@ def test_idle_sweep_cache_miss_builds_only_on_bounded_worker(self) -> None: prepare_idle_client(server, state) bundle = install_idle_job_cache(server) server.job_bundle_cache_seconds = 10.0 + server._ensure_job_bundle_service().set_cache_seconds_for_test(10.0) expired_bundle = dataclass_replace( bundle, built_monotonic=time.monotonic() - 11.0, ) - with server._job_cache_lock: - server._job_bundle_cache[bundle.key] = expired_bundle + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache[bundle.key] = expired_bundle build_started = threading.Event() release_build = threading.Event() build_thread_ids: list[int] = [] @@ -782,7 +788,7 @@ def blocked_build( return bundle state.send = sent.append # type: ignore[method-assign] - server.build_shared_job_bundle = blocked_build # type: ignore[method-assign] + server._ensure_job_bundle_service().build_shared_job_bundle = blocked_build # type: ignore[method-assign] sweep_thread_id = threading.get_ident() started = time.monotonic() try: @@ -818,8 +824,9 @@ def test_idle_retarget_delivers_fresh_bundle_when_cache_is_disabled( prepare_idle_client(server, state) bundle = install_idle_job_cache(server) server.job_bundle_cache_seconds = 0.0 - with server._job_cache_lock: - server._job_bundle_cache.clear() + server._ensure_job_bundle_service().set_cache_seconds_for_test(0.0) + with server._ensure_job_bundle_service()._cache_lock: + server._ensure_job_bundle_service()._bundle_cache.clear() built = threading.Event() sent: list[dict[str, object]] = [] @@ -842,8 +849,8 @@ def build_uncached(_request: object) -> CachedJobBundle: self.assertEqual(server.idle_retarget_count, 1) self.assertEqual(state.share_difficulty, Decimal("4")) self.assertIsNone(state.pending_share_difficulty) - with server._job_cache_lock: - self.assertEqual(server._job_bundle_cache, {}) + with server._ensure_job_bundle_service()._cache_lock: + self.assertEqual(server._ensure_job_bundle_service()._bundle_cache, {}) def test_idle_preparation_oserror_keeps_client_connected(self) -> None: for failure_phase in ("bundle", "reorg"): with self.subTest(failure_phase=failure_phase): @@ -1030,7 +1037,7 @@ def boom(client: ClientState, *, clean_jobs: bool) -> None: # It must now be swallowed so the client thread survives a single bad template. server.maybe_send_job(state, clean_jobs=True) - self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 1) self.assertEqual(state.active_job_ids, set()) self.assertEqual(server.jobs, {}) self.assertEqual(sent, []) # no difficulty / mining.notify pushed for the failed build @@ -1051,7 +1058,7 @@ def boom(client: ClientState, *, clean_jobs: bool) -> None: server.maybe_send_job(state, clean_jobs=True) - self.assertEqual(server.job_build_failure_count, 1) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 1) self.assertEqual(state.active_job_ids, {"job-ok"}) self.assertEqual(sent, ["notify"]) def test_maybe_send_job_does_not_swallow_send_failures_as_build_failures(self) -> None: @@ -1091,7 +1098,7 @@ def dead_socket(client: ClientState, job: object) -> None: # The send failure is not a build failure, and handle_client (not us) owns # the disconnect/cleanup of the registered job for the dead connection. - self.assertEqual(server.job_build_failure_count, 0) + self.assertEqual(server._ensure_job_bundle_service().metrics_snapshot()["failure_count"], 0) def _pending_append(self, tag: str, accepted_at_ms: int = 2) -> PendingShareAppend: from lab.prism.share_ledger import PendingShare