diff --git a/VERSION b/VERSION index ab0fa336..c20c645d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.5 +5.0.6 diff --git a/docs/Generative-Mining.md b/docs/Generative-Mining.md index 1f7c898d..9f98cc8e 100644 --- a/docs/Generative-Mining.md +++ b/docs/Generative-Mining.md @@ -12,7 +12,7 @@ Generative miners create synthetic media (images and videos) according to prompt 3. **Sample volume** -- verified volume ramps through the first 10 samples, then logarithmically 4. **Model choice** -- more expensive generation models earn higher per-unit rewards (see [Model Pricing](#model-pricing-and-rewards) below) -Validators still challenge onboarding and under-bar miners (8 + 6 of 50 slots) so new UIDs can build a fool-rate sample. Those slots do not pay until the miner qualifies. See [Incentive Mechanism](Incentive.md) for the full reward formula. +Validators still challenge onboarding and under-bar miners (8 + 6 of 50 slots) so new UIDs can build a fool-rate sample. Those slots do not pay until the miner qualifies. A miner this validator has asked enough times with no answer is skipped for that modality instead of sitting in onboarding. See [Incentive Mechanism](Incentive.md) for the full reward formula. Generative miners operate as FastAPI servers that receive generation requests from validators and respond asynchronously via webhooks. diff --git a/docs/Incentive.md b/docs/Incentive.md index db36e56e..1d4cc458 100644 --- a/docs/Incentive.md +++ b/docs/Incentive.md @@ -97,8 +97,11 @@ Each validator still sends `--neuron.sample-size` (default 50) requests per roun | Qualified | Over the bar for that modality | 36 | | Onboarding | $n < 20$ or no fool-rate row | 8 | | Probe | $n \ge 20$ but under the bar | 6 | +| Unresponsive | This validator asked enough times and got no answer | 0 | -Onboarding and probe miners still receive prompts; they do not earn until they clear. If the onboarding set is empty, the unused 8 slots split 4+4 (40 qualified / 10 probe). Leftover slots overflow qualified → probe → qualified, then any remaining live generator, so the round never goes out under-filled when miners exist. A missing or stale generator-results cache treats everyone as onboarding so sampling does not freeze on the last qualified set. +Onboarding and probe miners still receive prompts; they do not earn until they clear. If the onboarding set is empty, the unused 8 slots split 4+4 (40 qualified / 10 probe). Leftover slots overflow qualified → probe → qualified, then any remaining live generator who still answers that modality, so the round never goes out under-filled when miners exist. A missing or stale generator-results cache treats everyone as onboarding so sampling does not freeze on the last qualified set. + +Unresponsive is local to each validator: refused challenge POSTs (`no_answer`) and accepted tasks that never deliver (`challenge_timeout`). After `--scoring.min-no-answer-attempts` (default 5) in `--scoring.no-answer-lookback-hours` (default 24) with zero answers in that modality, the miner is skipped for that modality. A video-only miner who ignores image is not image-onboarding. One real answer (media or a miner-reported failure) clears the flag. This design incentivizes generators to: 1. Produce valid, C2PA-signed content (base reward) diff --git a/gas/__init__.py b/gas/__init__.py index 21a585e3..6bbd1085 100644 --- a/gas/__init__.py +++ b/gas/__init__.py @@ -1,4 +1,4 @@ -__version__ = "5.0.5" +__version__ = "5.0.6" version_split = __version__.split(".") __spec_version__ = ( diff --git a/gas/cache/content_manager.py b/gas/cache/content_manager.py index 813ed24c..0372b10c 100644 --- a/gas/cache/content_manager.py +++ b/gas/cache/content_manager.py @@ -652,6 +652,12 @@ def update_challenge_outcome( media_id=media_id, ) + def get_challenge_response_stats( + self, lookback_hours: float = 24.0 + ) -> Dict[str, Dict[str, Dict[str, int]]]: + """Per-hotkey answer / no-answer counts for challenge slot allocation.""" + return self.challenges.get_challenge_response_stats(lookback_hours=lookback_hours) + def store_clip_embedding(self, media_id: str, embedding_blob: bytes) -> bool: """Store a CLIP embedding for a media entry (deep-feature duplicate detection).""" return self.media.update_media_embedding(media_id, embedding_blob) diff --git a/gas/cache/db/challenge_store.py b/gas/cache/db/challenge_store.py index b18ac2a0..fe1e7d1a 100644 --- a/gas/cache/db/challenge_store.py +++ b/gas/cache/db/challenge_store.py @@ -161,7 +161,9 @@ def get_outcomes_last_n_hours( SELECT o.*, m.resolution AS media_resolution, m.has_audio AS media_has_audio FROM generator_challenge_outcomes o LEFT JOIN media m ON o.media_id = m.id - WHERE o.status IN ('verified', 'failed') AND o.updated_at >= ? + WHERE o.status IN ('verified', 'failed') + AND o.updated_at >= ? + AND COALESCE(o.failure_reason, '') != 'no_answer' ORDER BY o.updated_at DESC LIMIT ? """, (cutoff, int(limit)), @@ -200,6 +202,9 @@ def get_outcome_stats_last_n_hours( outcomes = self.get_outcomes_last_n_hours(lookback_hours, limit) miner_stats: Dict[str, Dict[str, Any]] = {} for outcome in outcomes: + if outcome.status == "failed" and (outcome.failure_reason or "") == "no_answer": + # Refused POSTs are sampling signal, not a verification miss. + continue hotkey = outcome.hotkey if hotkey not in miner_stats: miner_stats[hotkey] = { @@ -280,3 +285,65 @@ def get_outcome_stats_last_n_hours( "last_timestamp": stats["last_timestamp"], } return result + + def get_challenge_response_stats( + self, lookback_hours: float = 24.0 + ) -> Dict[str, Dict[str, Dict[str, int]]]: + """Count answers vs no-answers per hotkey and modality. + + Keyed by hotkey so a replacement at a recycled UID does not inherit + the previous occupant's totals. Callers resolve against the current + metagraph the same way qualification does. + + An answer is stored/verified media, or a failed attempt that still + engaged (miner-reported failure, C2PA, CLIP, etc.). ``no_answer`` is + a refused/unreached POST or an accepted task that never delivered. + In-flight ``pending`` rows are ignored so a live challenge cannot + mark a miner unresponsive. + """ + try: + cutoff = time.time() - (lookback_hours * 3600) + with self.db.connect() as conn: + cursor = conn.execute( + """ + SELECT hotkey, modality, + SUM( + CASE + WHEN status IN ('stored', 'verified') THEN 1 + WHEN status = 'failed' + AND COALESCE(failure_reason, '') + NOT IN ('no_answer', 'challenge_timeout') + THEN 1 + ELSE 0 + END + ) AS answered, + SUM( + CASE + WHEN status = 'failed' + AND failure_reason IN ('no_answer', 'challenge_timeout') + THEN 1 + ELSE 0 + END + ) AS no_answer + FROM generator_challenge_outcomes + WHERE created_at >= ? + GROUP BY hotkey, modality + """, + (cutoff,), + ) + stats: Dict[str, Dict[str, Dict[str, int]]] = {} + for hotkey, modality, answered, no_answer in cursor.fetchall(): + if not hotkey: + continue + mod = str(modality or "").strip().lower() + if mod not in ("image", "video"): + continue + row = stats.setdefault(str(hotkey), {}) + row[mod] = { + "answered": int(answered or 0), + "no_answer": int(no_answer or 0), + } + return stats + except Exception as e: + bt.logging.error(f"Error getting challenge response stats: {e}") + return {} diff --git a/gas/config.py b/gas/config.py index 75f4f732..e3d5b8be 100644 --- a/gas/config.py +++ b/gas/config.py @@ -364,6 +364,23 @@ def add_validator_args(parser): default=20, ) + parser.add_argument( + "--scoring.min-no-answer-attempts", + type=int, + help=( + "Local unanswered challenges (this validator) before a modality is " + "treated as unresponsive instead of onboarding" + ), + default=5, + ) + + parser.add_argument( + "--scoring.no-answer-lookback-hours", + type=float, + help="Window for no-answer / answer counts used in challenge sampling", + default=24.0, + ) + parser.add_argument( "--benchmark-api-url", type=str, diff --git a/gas/evaluation/__init__.py b/gas/evaluation/__init__.py index eb9a61e8..8c755f66 100644 --- a/gas/evaluation/__init__.py +++ b/gas/evaluation/__init__.py @@ -1,6 +1,7 @@ from .challenge_allocation import ( allocate_challenge_slots, classify_modality_bucket, + resolve_challenge_response_stats, ) from .miner_type_tracker import MinerTypeTracker from .rewards import ( @@ -17,6 +18,7 @@ "GeneratorQualification", "allocate_challenge_slots", "classify_modality_bucket", + "resolve_challenge_response_stats", "combine_generator_rewards", "get_generator_base_rewards", "get_generator_qualification", diff --git a/gas/evaluation/challenge_allocation.py b/gas/evaluation/challenge_allocation.py index 21de86b0..2d298cef 100644 --- a/gas/evaluation/challenge_allocation.py +++ b/gas/evaluation/challenge_allocation.py @@ -6,19 +6,43 @@ from .rewards import GeneratorQualification -Bucket = str # "qualified" | "onboarding" | "probe" +Bucket = str # "qualified" | "onboarding" | "probe" | "unresponsive" +ModalityResponseStats = Dict[str, Dict[str, int]] + +# Local challenge outcomes that mean "we asked, they never produced media." +NO_ANSWER_REASONS = frozenset({"no_answer", "challenge_timeout"}) + + +def _is_unresponsive( + response_stats: Optional[ModalityResponseStats], + modality: str, + min_no_answer_attempts: int, +) -> bool: + """True when this validator asked often enough and got zero answers.""" + if not response_stats or min_no_answer_attempts <= 0: + return False + row = response_stats.get(modality) or {} + answered = int(row.get("answered") or 0) + no_answer = int(row.get("no_answer") or 0) + return answered == 0 and no_answer >= min_no_answer_attempts def classify_modality_bucket( qualification: Optional[GeneratorQualification], modality: str, min_fool_samples: int = 20, + response_stats: Optional[ModalityResponseStats] = None, + min_no_answer_attempts: int = 5, ) -> Bucket: """Classify one UID for one modality. + Repeated no-answers on this validator (never accepted, or accepted and + never delivered) are unresponsive: they do not occupy onboarding slots. Missing qualification or n < min_fool_samples is onboarding. Over the cutoff is qualified. n >= min and under the cutoff is probe. """ + if _is_unresponsive(response_stats, modality, min_no_answer_attempts): + return "unresponsive" if qualification is None: return "onboarding" if modality == "image": @@ -36,6 +60,24 @@ def classify_modality_bucket( return "probe" +def resolve_challenge_response_stats( + response_stats: Optional[Dict[str, ModalityResponseStats]], + metagraph, +) -> Optional[Dict[int, ModalityResponseStats]]: + """Map hotkey-keyed no-answer counts onto current UIDs. + + A replacement at a recycled UID starts at zero; the prior occupant's + totals stay on the old hotkey and do not transfer. + """ + if not response_stats: + return None + return { + uid: response_stats[hotkey] + for uid, hotkey in enumerate(list(metagraph.hotkeys)) + if hotkey in response_stats + } + + def _slot_targets( qualified_slots: int, onboarding_slots: int, @@ -65,15 +107,19 @@ def allocate_challenge_slots( onboarding_slots: int = 8, probe_slots: int = 6, min_fool_samples: int = 20, + response_stats: Optional[Dict[int, ModalityResponseStats]] = None, + min_no_answer_attempts: int = 5, rng: Optional[np.random.Generator] = None, ) -> Tuple[List[Tuple[int, str]], Dict[str, object]]: """Pick unique (uid, modality) pairs for one validator challenge round. Modality is chosen first for each slot, then a UID from the matching bucket. If qualification is None (stale/missing API), every UID is - onboarding. Unused onboarding slots when that set is empty become - 4+4 extra qualified/probe (40/10 at the defaults). Remaining unused - slots overflow qualified → probe → qualified, then any leftover miner. + onboarding unless local no-answer stats mark that modality unresponsive. + Unused onboarding slots when that set is empty become 4+4 extra + qualified/probe (40/10 at the defaults). Remaining unused slots overflow + qualified → probe → qualified, then any leftover miner who still answers + that modality. Returns (assignments, stats) where stats includes pool sizes and rolled_onboarding. @@ -82,23 +128,31 @@ def allocate_challenge_slots( uids = [int(u) for u in dict.fromkeys(miner_uids)] mods = [str(m).strip().lower() for m in available_modalities] mods = [m for m in mods if m in ("image", "video")] + empty_stats = { + "image_qualified": 0, + "video_qualified": 0, + "onboarding": 0, + "probe": 0, + "image_unresponsive": 0, + "video_unresponsive": 0, + "rolled_onboarding": False, + } if not uids or not mods or sample_size <= 0: - return [], { - "image_qualified": 0, - "video_qualified": 0, - "onboarding": 0, - "probe": 0, - "rolled_onboarding": False, - } + return [], empty_stats def bucket_for(uid: int, modality: str) -> Bucket: - if qualification is None: - return "onboarding" + q = None if qualification is None else qualification.get(uid) + stats = None if response_stats is None else response_stats.get(uid) return classify_modality_bucket( - qualification.get(uid), modality, min_fool_samples + q, + modality, + min_fool_samples, + response_stats=stats, + min_no_answer_attempts=min_no_answer_attempts, ) - image_qualified = video_qualified = onboarding_n = probe_n = 0 + image_qualified = video_qualified = 0 + image_unresponsive = video_unresponsive = 0 onboarding_uids = set() probe_uids = set() for uid in uids: @@ -111,8 +165,13 @@ def bucket_for(uid: int, modality: str) -> Bucket: video_qualified += 1 elif kind == "onboarding": onboarding_uids.add(uid) - else: + elif kind == "probe": probe_uids.add(uid) + elif kind == "unresponsive": + if mod == "image": + image_unresponsive += 1 + else: + video_unresponsive += 1 onboarding_n = len(onboarding_uids) probe_n = len(probe_uids) @@ -156,9 +215,13 @@ def fill(kind: Bucket, count: int) -> int: for i in range(n): if assigned[i] is not None: continue - pool = [uid for uid in uids if uid not in used] + pool = [ + uid + for uid in uids + if uid not in used and bucket_for(uid, slot_mods[i]) != "unresponsive" + ] if not pool: - break + continue uid = int(pool[int(rng.integers(0, len(pool)))]) assigned[i] = uid used.add(uid) @@ -173,6 +236,8 @@ def fill(kind: Bucket, count: int) -> int: "video_qualified": video_qualified, "onboarding": onboarding_n, "probe": probe_n, + "image_unresponsive": image_unresponsive, + "video_unresponsive": video_unresponsive, "rolled_onboarding": rolled, } return assignments, stats @@ -182,15 +247,20 @@ def summarize_assignment_buckets( assignments: Iterable[Tuple[int, str]], qualification: Optional[Dict[int, GeneratorQualification]], min_fool_samples: int = 20, + response_stats: Optional[Dict[int, ModalityResponseStats]] = None, + min_no_answer_attempts: int = 5, ) -> Dict[str, int]: """Count assigned UIDs by the bucket used for their chosen modality.""" - counts = {"qualified": 0, "onboarding": 0, "probe": 0} + counts = {"qualified": 0, "onboarding": 0, "probe": 0, "unresponsive": 0} for uid, modality in assignments: - if qualification is None: - kind = "onboarding" - else: - kind = classify_modality_bucket( - qualification.get(uid), modality, min_fool_samples - ) + q = None if qualification is None else qualification.get(uid) + stats = None if response_stats is None else response_stats.get(uid) + kind = classify_modality_bucket( + q, + modality, + min_fool_samples, + response_stats=stats, + min_no_answer_attempts=min_no_answer_attempts, + ) counts[kind] += 1 return counts diff --git a/gas/evaluation/generative_challenge_manager.py b/gas/evaluation/generative_challenge_manager.py index 70d77cdb..b5721f58 100644 --- a/gas/evaluation/generative_challenge_manager.py +++ b/gas/evaluation/generative_challenge_manager.py @@ -7,6 +7,7 @@ import tempfile import threading import time +import uuid from concurrent.futures import ThreadPoolExecutor import aiohttp @@ -21,7 +22,10 @@ from typing import Dict, Optional from gas.cache.content_manager import ContentManager -from gas.evaluation.challenge_allocation import allocate_challenge_slots +from gas.evaluation.challenge_allocation import ( + allocate_challenge_slots, + resolve_challenge_response_stats, +) from gas.evaluation.resolution_tiers import sample_challenge_tier from gas.evaluation.rewards import GeneratorQualification, resolve_generator_qualification from gas.protocol.epistula import get_verifier @@ -152,6 +156,14 @@ async def issue_generative_challenge(self): resolve_generator_qualification(self.qualification, self.metagraph) if self.qualification_fresh and self.qualification is not None else None ) + scoring = getattr(self.config, "scoring", None) + lookback_hours = float(getattr(scoring, "no_answer_lookback_hours", 24.0)) + response_stats = resolve_challenge_response_stats( + self.content_manager.get_challenge_response_stats( + lookback_hours=lookback_hours + ), + self.metagraph, + ) assignments, pool_stats = allocate_challenge_slots( miner_uids, available_names, @@ -160,8 +172,10 @@ async def issue_generative_challenge(self): qualified_slots=int(getattr(self.config.neuron, "qualified_slots", 36)), onboarding_slots=int(getattr(self.config.neuron, "onboarding_slots", 8)), probe_slots=int(getattr(self.config.neuron, "probe_slots", 6)), - min_fool_samples=int( - getattr(getattr(self.config, "scoring", None), "min_fool_samples", 20) + min_fool_samples=int(getattr(scoring, "min_fool_samples", 20)), + response_stats=response_stats, + min_no_answer_attempts=int( + getattr(scoring, "min_no_answer_attempts", 5) ), ) @@ -173,6 +187,8 @@ async def issue_generative_challenge(self): f"Challenge pools: image_qualified={pool_stats['image_qualified']} " f"video_qualified={pool_stats['video_qualified']} " f"onboarding={pool_stats['onboarding']} probe={pool_stats['probe']} " + f"image_unresponsive={pool_stats['image_unresponsive']} " + f"video_unresponsive={pool_stats['video_unresponsive']} " f"rolled_onboarding={pool_stats['rolled_onboarding']}" ) bt.logging.info(f"Issuing generative challenge to UIDs: {[uid for uid, _ in assignments]}") @@ -243,7 +259,21 @@ async def send_generative_request(self, uid: int, prompt_entry, modality: Modali ) else: error = response_data.get("error") if response_data else "Unknown error" - bt.logging.error(f"Failed to send challenge to UID {uid}. Error: {error}") + miner_hotkey = self.metagraph.hotkeys[uid] + self.content_manager.record_challenge_outcome( + task_id=f"no-answer-{uid}-{uuid.uuid4()}", + uid=uid, + hotkey=miner_hotkey, + prompt_id=prompt_entry.id, + modality=modality.value, + status="failed", + failure_reason="no_answer", + requested_resolution=requested_resolution, + ) + bt.logging.error( + f"Failed to send challenge to UID {uid}. Error: {error} " + f"(recorded no_answer for {modality.value})" + ) async def generative_callback(self, request: Request): """Callback endpoint for generative challenges. diff --git a/tests/test_challenge_allocation.py b/tests/test_challenge_allocation.py index 59294784..85fea354 100644 --- a/tests/test_challenge_allocation.py +++ b/tests/test_challenge_allocation.py @@ -9,6 +9,7 @@ from gas.evaluation.challenge_allocation import ( allocate_challenge_slots, classify_modality_bucket, + resolve_challenge_response_stats, summarize_assignment_buckets, ) from gas.evaluation.rewards import ( @@ -233,5 +234,79 @@ def test_fresh_cache_does_not_give_replacement_qualified_challenge_slots(): assert stats["image_qualified"] == 1 assert stats["onboarding"] == 1 assert summarize_assignment_buckets(assignments, current) == { - "qualified": 1, "onboarding": 1, "probe": 0, + "qualified": 1, "onboarding": 1, "probe": 0, "unresponsive": 0, } + + +def test_no_answers_are_unresponsive_not_onboarding(): + q = _q(video_n=40, qualified_video=False) + stats = {"image": {"answered": 0, "no_answer": 5}, "video": {"answered": 8, "no_answer": 0}} + assert classify_modality_bucket(q, "image", response_stats=stats) == "unresponsive" + assert classify_modality_bucket(q, "video", response_stats=stats) == "probe" + assert classify_modality_bucket(q, "image") == "onboarding" + few = {"image": {"answered": 0, "no_answer": 4}} + assert classify_modality_bucket(q, "image", response_stats=few) == "onboarding" + answered = {"image": {"answered": 1, "no_answer": 20}} + assert classify_modality_bucket(q, "image", response_stats=answered) == "onboarding" + + +def test_video_specialist_who_ignores_image_is_not_assigned_image(): + qualification = { + 0: _q(video_n=40, qualified_video=False), + } + for uid in range(1, 40): + qualification[uid] = _q( + image_n=40, qualified_image=True, video_n=40, qualified_video=True + ) + response_stats = { + 0: { + "image": {"answered": 0, "no_answer": 5}, + "video": {"answered": 12, "no_answer": 0}, + } + } + video_hits = 0 + for seed in range(40): + assignments, stats = allocate_challenge_slots( + list(qualification), + ["image", "video"], + qualification, + response_stats=response_stats, + rng=np.random.default_rng(seed), + ) + assert stats["image_unresponsive"] == 1 + assert 0 not in {uid for uid, _ in assignments if _ == "image"} + if (0, "video") in assignments: + video_hits += 1 + assert video_hits > 0 + + +def test_unresponsive_still_applies_when_qualification_is_missing(): + response_stats = {0: {"image": {"answered": 0, "no_answer": 5}}} + assignments, stats = allocate_challenge_slots( + [0, 1], + ["image"], + None, + sample_size=2, + qualified_slots=1, + onboarding_slots=1, + probe_slots=0, + response_stats=response_stats, + rng=np.random.default_rng(8), + ) + assert stats["image_unresponsive"] == 1 + assert stats["onboarding"] == 1 + assert dict(assignments) == {1: "image"} + + +def test_response_stats_do_not_follow_recycled_uid(): + metagraph = SimpleNamespace(hotkeys=["replacement", "unchanged"]) + by_hotkey = { + "old-owner": {"image": {"answered": 0, "no_answer": 9}}, + "unchanged": {"image": {"answered": 3, "no_answer": 0}}, + } + resolved = resolve_challenge_response_stats(by_hotkey, metagraph) + assert 0 not in resolved + assert resolved[1]["image"]["answered"] == 3 + assert classify_modality_bucket( + None, "image", response_stats=resolved.get(0) + ) == "onboarding" diff --git a/tests/test_challenge_outcomes.py b/tests/test_challenge_outcomes.py index a1948620..ebbc6e6f 100644 --- a/tests/test_challenge_outcomes.py +++ b/tests/test_challenge_outcomes.py @@ -68,6 +68,88 @@ def test_challenge_outcomes_count_pre_storage_failures(self): self.assertEqual(stats["hotkey-1"]["image_model_names"], []) self.assertEqual(stats["hotkey-1"]["video_model_names"], []) + challenges.record_outcome( + task_id="task-no-answer-hk1", + uid=1, + hotkey="hotkey-1", + prompt_id=prompt_id, + modality="image", + status="failed", + failure_reason="no_answer", + ) + stats_after = challenges.get_outcome_stats_last_n_hours(lookback_hours=1) + self.assertEqual(stats_after["hotkey-1"]["total_failed"], 1) + self.assertEqual(stats_after["hotkey-1"]["pass_rate"], 0.5) + + challenges.record_outcome( + task_id="task-no-answer", + uid=2, + hotkey="hotkey-2", + prompt_id=prompt_id, + modality="image", + status="failed", + failure_reason="no_answer", + ) + challenges.record_outcome( + task_id="task-timeout", + uid=2, + hotkey="hotkey-2", + prompt_id=prompt_id, + modality="image", + status="failed", + failure_reason="challenge_timeout", + ) + challenges.record_outcome( + task_id="task-video-ok", + uid=2, + hotkey="hotkey-2", + prompt_id=prompt_id, + modality="video", + status="verified", + media_id=media_id, + ) + response = challenges.get_challenge_response_stats(lookback_hours=1) + self.assertEqual(response["hotkey-1"]["image"]["answered"], 2) + self.assertEqual(response["hotkey-1"]["image"]["no_answer"], 1) + self.assertEqual(response["hotkey-2"]["image"]["answered"], 0) + self.assertEqual(response["hotkey-2"]["image"]["no_answer"], 2) + self.assertEqual(response["hotkey-2"]["video"]["answered"], 1) + self.assertEqual(response["hotkey-2"]["video"]["no_answer"], 0) + + def test_no_answer_rows_do_not_consume_reward_stats_limit(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "prompts.db" + conn = ConnectionManager(db_path) + with conn.connect() as c: + create_schema(c) + + prompts = PromptStore(conn) + challenges = ChallengeStore(conn) + prompt_id = prompts.add_prompt_entry("a test prompt", modality="image") + challenges.record_outcome( + task_id="task-verified", + uid=1, + hotkey="hotkey-1", + prompt_id=prompt_id, + modality="image", + status="verified", + ) + for i in range(5): + challenges.record_outcome( + task_id=f"task-no-answer-{i}", + uid=1, + hotkey="hotkey-1", + prompt_id=prompt_id, + modality="image", + status="failed", + failure_reason="no_answer", + ) + + stats = challenges.get_outcome_stats_last_n_hours(lookback_hours=1, limit=1) + self.assertEqual(stats["hotkey-1"]["total_verified"], 1) + self.assertEqual(stats["hotkey-1"]["total_failed"], 0) + self.assertEqual(stats["hotkey-1"]["pass_rate"], 1.0) + if __name__ == "__main__": unittest.main()