From 680a9fb11b476420cd81d0da726f62d3dd11ca1e Mon Sep 17 00:00:00 2001 From: dylan Date: Thu, 17 Sep 2026 13:30:36 +0000 Subject: [PATCH 1/2] Restore uniform random generator challenge sampling. Bucketed qualified/onboarding/probe slots were starving some miners; the default is a uniform draw again, with buckets still available as an opt-in. Co-authored-by: Cursor --- .env.validator.template | 2 + docs/Incentive.md | 6 +- gas/config.py | 8 +++ gas/evaluation/__init__.py | 2 + gas/evaluation/challenge_allocation.py | 32 ++++++++++ .../generative_challenge_manager.py | 60 ++++++++++++------- tests/test_challenge_allocation.py | 26 ++++++++ validator.config.js | 3 + 8 files changed, 116 insertions(+), 23 deletions(-) diff --git a/.env.validator.template b/.env.validator.template index 5e83daa2..9a242995 100644 --- a/.env.validator.template +++ b/.env.validator.template @@ -38,6 +38,8 @@ DEVICE=cuda # Other LOGLEVEL=INFO AUTO_UPDATE=true +# Challenge target pick: random (default) or buckets +#CHALLENGE_ALLOCATION=random # Service toggles (set to false to disable individual services) #START_VALIDATOR=true diff --git a/docs/Incentive.md b/docs/Incentive.md index 1d4cc458..7d17ef77 100644 --- a/docs/Incentive.md +++ b/docs/Incentive.md @@ -90,7 +90,11 @@ Scores use separate image and video exponential moving averages (50% current rew ### Challenge slots -Each validator still sends `--neuron.sample-size` (default 50) requests per round — one UID, one modality, no replacement. Slots are filled from three buckets **for the chosen modality**: +Each validator still sends `--neuron.sample-size` (default 50) requests per round — one UID, one modality, no replacement. + +The default `--neuron.challenge-allocation random` draws those UIDs uniformly from every registered generator and assigns image or video at random. Qualification, onboarding, and unresponsive status still gate **pay**; they do not change who gets asked. + +`--neuron.challenge-allocation buckets` is the older slot fill. Slots are filled from three buckets **for the chosen modality**: | Bucket | Who | Default slots | |---|---|---| diff --git a/gas/config.py b/gas/config.py index e3d5b8be..d6a3f2f7 100644 --- a/gas/config.py +++ b/gas/config.py @@ -308,6 +308,14 @@ def add_validator_args(parser): default=50, ) + parser.add_argument( + "--neuron.challenge-allocation", + type=str, + choices=["random", "buckets"], + default="random", + help="Pick challenge targets uniformly at random, or via qualified/onboarding/probe buckets", + ) + parser.add_argument( "--neuron.qualified-slots", type=int, diff --git a/gas/evaluation/__init__.py b/gas/evaluation/__init__.py index 8c755f66..c8edd5fa 100644 --- a/gas/evaluation/__init__.py +++ b/gas/evaluation/__init__.py @@ -1,5 +1,6 @@ from .challenge_allocation import ( allocate_challenge_slots, + allocate_random_slots, classify_modality_bucket, resolve_challenge_response_stats, ) @@ -17,6 +18,7 @@ "MinerTypeTracker", "GeneratorQualification", "allocate_challenge_slots", + "allocate_random_slots", "classify_modality_bucket", "resolve_challenge_response_stats", "combine_generator_rewards", diff --git a/gas/evaluation/challenge_allocation.py b/gas/evaluation/challenge_allocation.py index 2d298cef..ed628a06 100644 --- a/gas/evaluation/challenge_allocation.py +++ b/gas/evaluation/challenge_allocation.py @@ -97,6 +97,38 @@ def _slot_targets( ) +def allocate_random_slots( + miner_uids: Sequence[int], + available_modalities: Sequence[str], + *, + sample_size: int = 50, + rng: Optional[np.random.Generator] = None, +) -> Tuple[List[Tuple[int, str]], Dict[str, object]]: + """Pick unique generators uniformly, each with a random image/video modality.""" + rng = rng or np.random.default_rng() + uids = [int(u) for u in dict.fromkeys(miner_uids)] + mods = [str(m).strip().lower() for m in available_modalities] + mods = [m for m in mods if m in ("image", "video")] + stats = { + "mode": "random", + "pool": len(uids), + "image_qualified": 0, + "video_qualified": 0, + "onboarding": 0, + "probe": 0, + "image_unresponsive": 0, + "video_unresponsive": 0, + "rolled_onboarding": False, + } + if not uids or not mods or sample_size <= 0: + return [], stats + n = min(int(sample_size), len(uids)) + chosen = rng.choice(np.array(uids, dtype=int), size=n, replace=False) + slot_mods = [mods[int(i)] for i in rng.integers(0, len(mods), size=n)] + assignments = [(int(uid), slot_mods[i]) for i, uid in enumerate(chosen)] + return assignments, stats + + def allocate_challenge_slots( miner_uids: Sequence[int], available_modalities: Sequence[str], diff --git a/gas/evaluation/generative_challenge_manager.py b/gas/evaluation/generative_challenge_manager.py index b5721f58..ba46b81b 100644 --- a/gas/evaluation/generative_challenge_manager.py +++ b/gas/evaluation/generative_challenge_manager.py @@ -24,6 +24,7 @@ from gas.cache.content_manager import ContentManager from gas.evaluation.challenge_allocation import ( allocate_challenge_slots, + allocate_random_slots, resolve_challenge_response_stats, ) from gas.evaluation.resolution_tiers import sample_challenge_tier @@ -164,33 +165,48 @@ async def issue_generative_challenge(self): ), self.metagraph, ) - assignments, pool_stats = allocate_challenge_slots( - miner_uids, - available_names, - qualification, - sample_size=sample_size, - qualified_slots=int(getattr(self.config.neuron, "qualified_slots", 36)), - onboarding_slots=int(getattr(self.config.neuron, "onboarding_slots", 8)), - probe_slots=int(getattr(self.config.neuron, "probe_slots", 6)), - min_fool_samples=int(getattr(scoring, "min_fool_samples", 20)), - response_stats=response_stats, - min_no_answer_attempts=int( - getattr(scoring, "min_no_answer_attempts", 5) - ), - ) + allocation = str( + getattr(self.config.neuron, "challenge_allocation", "random") + ).strip().lower() + if allocation == "random": + assignments, pool_stats = allocate_random_slots( + miner_uids, + available_names, + sample_size=sample_size, + ) + else: + assignments, pool_stats = allocate_challenge_slots( + miner_uids, + available_names, + qualification, + sample_size=sample_size, + qualified_slots=int(getattr(self.config.neuron, "qualified_slots", 36)), + onboarding_slots=int(getattr(self.config.neuron, "onboarding_slots", 8)), + probe_slots=int(getattr(self.config.neuron, "probe_slots", 6)), + min_fool_samples=int(getattr(scoring, "min_fool_samples", 20)), + response_stats=response_stats, + min_no_answer_attempts=int( + getattr(scoring, "min_no_answer_attempts", 5) + ), + ) if not assignments: bt.logging.trace("No generative miners found to challenge.") return - bt.logging.info( - f"Challenge pools: image_qualified={pool_stats['image_qualified']} " - f"video_qualified={pool_stats['video_qualified']} " - f"onboarding={pool_stats['onboarding']} probe={pool_stats['probe']} " - f"image_unresponsive={pool_stats['image_unresponsive']} " - f"video_unresponsive={pool_stats['video_unresponsive']} " - f"rolled_onboarding={pool_stats['rolled_onboarding']}" - ) + if pool_stats.get("mode") == "random": + bt.logging.info( + f"Challenge allocation: random {len(assignments)}/{pool_stats.get('pool', 0)} generators" + ) + else: + bt.logging.info( + f"Challenge pools: image_qualified={pool_stats['image_qualified']} " + f"video_qualified={pool_stats['video_qualified']} " + f"onboarding={pool_stats['onboarding']} probe={pool_stats['probe']} " + f"image_unresponsive={pool_stats['image_unresponsive']} " + f"video_unresponsive={pool_stats['video_unresponsive']} " + f"rolled_onboarding={pool_stats['rolled_onboarding']}" + ) bt.logging.info(f"Issuing generative challenge to UIDs: {[uid for uid, _ in assignments]}") modality_for = {Modality.IMAGE.value: Modality.IMAGE, Modality.VIDEO.value: Modality.VIDEO} diff --git a/tests/test_challenge_allocation.py b/tests/test_challenge_allocation.py index 85fea354..a5029983 100644 --- a/tests/test_challenge_allocation.py +++ b/tests/test_challenge_allocation.py @@ -8,6 +8,7 @@ from gas.config import validate_config_and_neuron_path from gas.evaluation.challenge_allocation import ( allocate_challenge_slots, + allocate_random_slots, classify_modality_bucket, resolve_challenge_response_stats, summarize_assignment_buckets, @@ -23,6 +24,31 @@ def _q(**kwargs): return GeneratorQualification(**kwargs) +def test_allocate_random_slots_picks_unique_uids_uniformly(): + assignments, stats = allocate_random_slots( + list(range(80)), + ["image", "video"], + sample_size=50, + rng=np.random.default_rng(0), + ) + assert stats["mode"] == "random" + assert stats["pool"] == 80 + assert len(assignments) == 50 + assert len({uid for uid, _ in assignments}) == 50 + assert {mod for _, mod in assignments} <= {"image", "video"} + + +def test_allocate_random_slots_includes_every_generator(): + assignments, stats = allocate_random_slots( + [7], + ["image"], + sample_size=50, + rng=np.random.default_rng(1), + ) + assert stats["pool"] == 1 + assert assignments == [(7, "image")] + + def test_classify_missing_and_short_samples_are_onboarding(): assert classify_modality_bucket(None, "image") == "onboarding" assert classify_modality_bucket(_q(image_n=19, qualified_image=True), "image") == "onboarding" diff --git a/validator.config.js b/validator.config.js index f180dc93..fb11ec54 100644 --- a/validator.config.js +++ b/validator.config.js @@ -139,6 +139,9 @@ if (config.startValidator) { if (process.env.EPOCH_LENGTH) { validatorArgs.push('--epoch-length', process.env.EPOCH_LENGTH); } + if (process.env.CHALLENGE_ALLOCATION) { + validatorArgs.push('--neuron.challenge-allocation', process.env.CHALLENGE_ALLOCATION); + } // Add external callback port if provided if (config.externalCallbackPort) { From e9f8e494f1194ce6d03cb2ad73945bc8a18e372a Mon Sep 17 00:00:00 2001 From: dylan Date: Thu, 17 Sep 2026 13:38:55 +0000 Subject: [PATCH 2/2] Bump subnet version to 5.0.7 so validators autoupdate. Co-authored-by: Cursor --- VERSION | 2 +- gas/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index c20c645d..00433367 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.6 +5.0.7 diff --git a/gas/__init__.py b/gas/__init__.py index 6bbd1085..80f31e6b 100644 --- a/gas/__init__.py +++ b/gas/__init__.py @@ -1,4 +1,4 @@ -__version__ = "5.0.6" +__version__ = "5.0.7" version_split = __version__.split(".") __spec_version__ = (