Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .env.validator.template
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ DEVICE=cuda
# Other
LOGLEVEL=INFO
AUTO_UPDATE=true
# Challenge target pick: random (default) or buckets
#CHALLENGE_ALLOCATION=random

# Service toggles (set to false to disable individual services)
#START_VALIDATOR=true
Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
5.0.6
5.0.7
6 changes: 5 additions & 1 deletion docs/Incentive.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,11 @@ Scores use separate image and video exponential moving averages (50% current rew

### Challenge slots

Each validator still sends `--neuron.sample-size` (default 50) requests per round — one UID, one modality, no replacement. Slots are filled from three buckets **for the chosen modality**:
Each validator still sends `--neuron.sample-size` (default 50) requests per round — one UID, one modality, no replacement.

The default `--neuron.challenge-allocation random` draws those UIDs uniformly from every registered generator and assigns image or video at random. Qualification, onboarding, and unresponsive status still gate **pay**; they do not change who gets asked.

`--neuron.challenge-allocation buckets` is the older slot fill. Slots are filled from three buckets **for the chosen modality**:

| Bucket | Who | Default slots |
|---|---|---|
Expand Down
2 changes: 1 addition & 1 deletion gas/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
__version__ = "5.0.6"
__version__ = "5.0.7"

version_split = __version__.split(".")
__spec_version__ = (
Expand Down
8 changes: 8 additions & 0 deletions gas/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,14 @@ def add_validator_args(parser):
default=50,
)

parser.add_argument(
"--neuron.challenge-allocation",
type=str,
choices=["random", "buckets"],
default="random",
help="Pick challenge targets uniformly at random, or via qualified/onboarding/probe buckets",
)

parser.add_argument(
"--neuron.qualified-slots",
type=int,
Expand Down
2 changes: 2 additions & 0 deletions gas/evaluation/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from .challenge_allocation import (
allocate_challenge_slots,
allocate_random_slots,
classify_modality_bucket,
resolve_challenge_response_stats,
)
Expand All @@ -17,6 +18,7 @@
"MinerTypeTracker",
"GeneratorQualification",
"allocate_challenge_slots",
"allocate_random_slots",
"classify_modality_bucket",
"resolve_challenge_response_stats",
"combine_generator_rewards",
Expand Down
32 changes: 32 additions & 0 deletions gas/evaluation/challenge_allocation.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,38 @@ def _slot_targets(
)


def allocate_random_slots(
miner_uids: Sequence[int],
available_modalities: Sequence[str],
*,
sample_size: int = 50,
rng: Optional[np.random.Generator] = None,
) -> Tuple[List[Tuple[int, str]], Dict[str, object]]:
"""Pick unique generators uniformly, each with a random image/video modality."""
rng = rng or np.random.default_rng()
uids = [int(u) for u in dict.fromkeys(miner_uids)]
mods = [str(m).strip().lower() for m in available_modalities]
mods = [m for m in mods if m in ("image", "video")]
stats = {
"mode": "random",
"pool": len(uids),
"image_qualified": 0,
"video_qualified": 0,
"onboarding": 0,
"probe": 0,
"image_unresponsive": 0,
"video_unresponsive": 0,
"rolled_onboarding": False,
}
if not uids or not mods or sample_size <= 0:
return [], stats
n = min(int(sample_size), len(uids))
chosen = rng.choice(np.array(uids, dtype=int), size=n, replace=False)
slot_mods = [mods[int(i)] for i in rng.integers(0, len(mods), size=n)]
assignments = [(int(uid), slot_mods[i]) for i, uid in enumerate(chosen)]
return assignments, stats


def allocate_challenge_slots(
miner_uids: Sequence[int],
available_modalities: Sequence[str],
Expand Down
60 changes: 38 additions & 22 deletions gas/evaluation/generative_challenge_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from gas.cache.content_manager import ContentManager
from gas.evaluation.challenge_allocation import (
allocate_challenge_slots,
allocate_random_slots,
resolve_challenge_response_stats,
)
from gas.evaluation.resolution_tiers import sample_challenge_tier
Expand Down Expand Up @@ -164,33 +165,48 @@ async def issue_generative_challenge(self):
),
self.metagraph,
)
assignments, pool_stats = allocate_challenge_slots(
miner_uids,
available_names,
qualification,
sample_size=sample_size,
qualified_slots=int(getattr(self.config.neuron, "qualified_slots", 36)),
onboarding_slots=int(getattr(self.config.neuron, "onboarding_slots", 8)),
probe_slots=int(getattr(self.config.neuron, "probe_slots", 6)),
min_fool_samples=int(getattr(scoring, "min_fool_samples", 20)),
response_stats=response_stats,
min_no_answer_attempts=int(
getattr(scoring, "min_no_answer_attempts", 5)
),
)
allocation = str(
getattr(self.config.neuron, "challenge_allocation", "random")
).strip().lower()
if allocation == "random":
assignments, pool_stats = allocate_random_slots(
miner_uids,
available_names,
sample_size=sample_size,
)
else:
assignments, pool_stats = allocate_challenge_slots(
miner_uids,
available_names,
qualification,
sample_size=sample_size,
qualified_slots=int(getattr(self.config.neuron, "qualified_slots", 36)),
onboarding_slots=int(getattr(self.config.neuron, "onboarding_slots", 8)),
probe_slots=int(getattr(self.config.neuron, "probe_slots", 6)),
min_fool_samples=int(getattr(scoring, "min_fool_samples", 20)),
response_stats=response_stats,
min_no_answer_attempts=int(
getattr(scoring, "min_no_answer_attempts", 5)
),
)

if not assignments:
bt.logging.trace("No generative miners found to challenge.")
return

bt.logging.info(
f"Challenge pools: image_qualified={pool_stats['image_qualified']} "
f"video_qualified={pool_stats['video_qualified']} "
f"onboarding={pool_stats['onboarding']} probe={pool_stats['probe']} "
f"image_unresponsive={pool_stats['image_unresponsive']} "
f"video_unresponsive={pool_stats['video_unresponsive']} "
f"rolled_onboarding={pool_stats['rolled_onboarding']}"
)
if pool_stats.get("mode") == "random":
bt.logging.info(
f"Challenge allocation: random {len(assignments)}/{pool_stats.get('pool', 0)} generators"
)
else:
bt.logging.info(
f"Challenge pools: image_qualified={pool_stats['image_qualified']} "
f"video_qualified={pool_stats['video_qualified']} "
f"onboarding={pool_stats['onboarding']} probe={pool_stats['probe']} "
f"image_unresponsive={pool_stats['image_unresponsive']} "
f"video_unresponsive={pool_stats['video_unresponsive']} "
f"rolled_onboarding={pool_stats['rolled_onboarding']}"
)
bt.logging.info(f"Issuing generative challenge to UIDs: {[uid for uid, _ in assignments]}")

modality_for = {Modality.IMAGE.value: Modality.IMAGE, Modality.VIDEO.value: Modality.VIDEO}
Expand Down
8 changes: 6 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,18 @@ urls = { homepage = "http://bitmind.ai" }
dependencies = [
"bittensor==10.4.0",
"bittensor-cli==9.22.0",
"pillow==12.2.0",
"pillow==12.3.0", # 12.3.0 closes GHSA heap/DoS alerts
"numpy==2.0.1",
"pandas==2.3.0",
"torch==2.7.1",
"torchvision==0.22.1",
"flash-attn>=2.7.0",
"httpcore==1.0.9", # >=1.0.9 required for h11 0.16 (CVE: malformed chunked-encoding)
"httpx==0.28.1",
"GitPython==3.1.62", # 3.1.62 closes CRITICAL GHSA-284h-m62q-gf8w and later HIGH git option/RCE advisories
"cryptography==50.0.1", # Includes the 50.0.0 fix for HIGH PKCS#7 Bleichenbacher oracle
"fastapi==0.141.1",
"starlette==1.6.0", # Includes the 1.3.1 fixes for multipart/form DoS and StaticFiles SSRF
"pyarrow==23.0.1",
"ffmpeg-python==0.2.0",
"imageio-ffmpeg==0.6.0",
Expand All @@ -36,7 +40,7 @@ dependencies = [
"uvicorn==0.27.1",
"httptools>=0.6.0",
"peft==0.17.0",
"aiohttp>=3.10.2",
"aiohttp==3.14.3", # 3.14.3 closes HIGH C parser OOB read
"hf_xet==1.1.7",
"python-dotenv==1.2.2",
"accelerate==1.8.1",
Expand Down
26 changes: 26 additions & 0 deletions tests/test_challenge_allocation.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from gas.config import validate_config_and_neuron_path
from gas.evaluation.challenge_allocation import (
allocate_challenge_slots,
allocate_random_slots,
classify_modality_bucket,
resolve_challenge_response_stats,
summarize_assignment_buckets,
Expand All @@ -23,6 +24,31 @@ def _q(**kwargs):
return GeneratorQualification(**kwargs)


def test_allocate_random_slots_picks_unique_uids_uniformly():
assignments, stats = allocate_random_slots(
list(range(80)),
["image", "video"],
sample_size=50,
rng=np.random.default_rng(0),
)
assert stats["mode"] == "random"
assert stats["pool"] == 80
assert len(assignments) == 50
assert len({uid for uid, _ in assignments}) == 50
assert {mod for _, mod in assignments} <= {"image", "video"}


def test_allocate_random_slots_includes_every_generator():
assignments, stats = allocate_random_slots(
[7],
["image"],
sample_size=50,
rng=np.random.default_rng(1),
)
assert stats["pool"] == 1
assert assignments == [(7, "image")]


def test_classify_missing_and_short_samples_are_onboarding():
assert classify_modality_bucket(None, "image") == "onboarding"
assert classify_modality_bucket(_q(image_n=19, qualified_image=True), "image") == "onboarding"
Expand Down
Loading
Loading