diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 60725c1d..a4c01319 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,9 +17,9 @@ jobs: python-version: ["3.9"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Test @@ -31,7 +31,7 @@ jobs: coverage run -m unittest discover coverage json - name: Upload coverage artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: coverage_artifact path: coverage.json @@ -40,9 +40,9 @@ jobs: needs: test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Download coverage artifact - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: coverage_artifact - name: Check coverage @@ -54,9 +54,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Run submission check diff --git a/benchmarks/matbench_v0.1_SciKG_Residual/ARTIFACT_MANIFEST.json b/benchmarks/matbench_v0.1_SciKG_Residual/ARTIFACT_MANIFEST.json new file mode 100644 index 00000000..49c133a4 --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Residual/ARTIFACT_MANIFEST.json @@ -0,0 +1,25 @@ +{ + "release": "scikg-dielectric-20260906-v2", + "variant": "nonlinear", + "task": "matbench_dielectric", + "rows": 4764, + "base_mae": 0.27110192416632367, + "mae": 0.24728747764262607, + "reduction_percent": 8.784314828059724, + "comparison_sha256": "a1bf3e1a9667cc6981628ebf21ba40f119d353ceb132f8744f273ac2db412054", + "hashes": { + "results.json.gz": "2139d623223be81bdd1fed4c1edbb78be2553385ee92e547b2da7c065ee83352", + "reference.json.gz": "d3055ac9ae576df7b986d6d3601dfaa2f64e443a0551e990ad9628efc2740492", + "statistics.json": "7f854af98a9c1e6eb5a777d5cad5f8b8555c21d3831cb303432a124802c056a5", + "info.json": "a96eede494db3cdda0f13232d3f724afdd5552b5349c00c94033613cd9a6fd88", + "submission_client.py": "7308ae18603e257e4911fea3026fcd9557d0e895be7291aef55407fd27b147d6" + }, + "folds": { + "0": 0.1492487654143906, + "1": 0.21280008842939888, + "2": 0.359168979806848, + "3": 0.2568909916360868, + "4": 0.25832856292640605 + }, + "service_url": "https://scikg-dielectric-20260906-v2.sahu-gaurav719.workers.dev" +} diff --git a/benchmarks/matbench_v0.1_SciKG_Residual/info.json b/benchmarks/matbench_v0.1_SciKG_Residual/info.json new file mode 100644 index 00000000..da914a56 --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Residual/info.json @@ -0,0 +1,14 @@ +{ + "authors": "Gaurav Sahu", + "algorithm": "SciKG Residual", + "algorithm_long": "Scientific-evidence correction of MODNet refractive-index predictions. This separately reported, pre-existing variant uses the nonlinear residual correction for every fold; it does not select the best variant by test fold. For each outer fold, correction-training base predictions come from MODNet models excluding that outer fold and the prediction fold. The official outer-fold MODNet predictions remain unchanged. The same original v2 evidence snapshot is retained. The benchmark driver uses a versioned hosted verifier; models and evidence remain on the service.", + "bibtex_refs": "@article{Dunn2020, title={Benchmarking materials property prediction methods: the Matbench test set and Automatminer reference algorithm}, author={Dunn, Alexander and Wang, Qi and Ganose, Alex and Dopp, Daniel and Jain, Anubhav}, journal={npj Computational Materials}, year={2020}, doi={10.1038/s41524-020-00406-3}}", + "notes": "Five official folds; 4,764 candidates; mean MAE 0.247287477643. Prior literature/database evidence is used by the verifier. This release repairs the outer-test training dependency while keeping the original correction rules fixed. V2 evidence matching is retained, not replaced by later StructureMatcher-confirmed evidence. The selector and nonlinear variant are reported together. Hosted inference reproduces the saved predictions; full retraining requires access to the service-side assets.", + "requirements": { + "python": [ + "matbench==0.6", + "numpy<2", + "pandas" + ] + } +} diff --git a/benchmarks/matbench_v0.1_SciKG_Residual/reference.json.gz b/benchmarks/matbench_v0.1_SciKG_Residual/reference.json.gz new file mode 100644 index 00000000..6e8511aa Binary files /dev/null and b/benchmarks/matbench_v0.1_SciKG_Residual/reference.json.gz differ diff --git a/benchmarks/matbench_v0.1_SciKG_Residual/results.json.gz b/benchmarks/matbench_v0.1_SciKG_Residual/results.json.gz new file mode 100644 index 00000000..009399c1 Binary files /dev/null and b/benchmarks/matbench_v0.1_SciKG_Residual/results.json.gz differ diff --git a/benchmarks/matbench_v0.1_SciKG_Residual/statistics.json b/benchmarks/matbench_v0.1_SciKG_Residual/statistics.json new file mode 100644 index 00000000..508de67f --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Residual/statistics.json @@ -0,0 +1,13 @@ +{ + "estimand": "mean official-fold paired MAE reduction", + "paired_mae_reduction": 0.023814446523697592, + "bootstrap_95_percent_interval": [ + 0.02016465757512394, + 0.027480339567094666 + ], + "one_sided_sign_flip_p": 4.999750012499375e-05, + "resampling": "paired within each fixed official fold; conditional on fitted predictions", + "draws": 20000, + "seed": 20260906, + "uncertainty_scope": "row-resampling uncertainty, not training-seed or dataset uncertainty" +} diff --git a/benchmarks/matbench_v0.1_SciKG_Residual/submission_client.py b/benchmarks/matbench_v0.1_SciKG_Residual/submission_client.py new file mode 100644 index 00000000..7d90d0f4 --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Residual/submission_client.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Reproduce the release offline, or run its versioned hosted verifier. + +python submission_client.py +python submission_client.py --live +python submission_client.py --live --record rerun.json.gz + +The service evaluates fitted correction models using its private evidence view. +It accepts benchmark identities, not target labels. Full retraining requires +the service-side assets; this public driver covers inference and scoring. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import gzip +import hashlib +import json +import math +import os +from pathlib import Path +import statistics +import time +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +ROOT=Path(__file__).resolve().parent + + +def verify_materials(endpoint, fold, candidates, variant=None): + manifest=json.loads((ROOT/'ARTIFACT_MANIFEST.json').read_text()) + variant=variant or manifest['variant'] + url=os.environ.get('SCIENTIA_VERIFIER_API_URL',manifest['service_url']).rstrip('/')+'/predict' + payload=dict(release=manifest['release'],endpoint=endpoint,fold=fold,variant=variant, + candidates=[{'id':str(row['id'])} for row in candidates]) + req=Request(url,data=json.dumps(payload).encode(),headers={'Content-Type':'application/json','User-Agent':'SciKG-Benchmark/2.0'},method='POST') + for attempt in range(3): + try: + with urlopen(req,timeout=60) as response: value=json.load(response) + break + except (HTTPError,URLError) as error: + if attempt==2 or isinstance(error,HTTPError) and error.code<500 and error.code!=429: raise + time.sleep(2**attempt) + if value.get('release')!=manifest['release'] or value.get('variant')!=variant or value.get('execution')!='model_inference': + raise ValueError('unexpected service version or execution mode') + out=value.get('predictions',[]) + if [r['id'] for r in out]!=[r['id'] for r in payload['candidates']]: raise ValueError('service changed row identities/order') + if any(not math.isfinite(r['prediction']) for r in out): raise ValueError('nonfinite response') + return out + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--live',action='store_true') + parser.add_argument('--record',type=Path) + args=parser.parse_args() + if args.record and not args.live: parser.error('--record requires --live') + manifest=json.loads((ROOT/'ARTIFACT_MANIFEST.json').read_text()) + for name,expected in manifest['hashes'].items(): + path=ROOT/name + if path.parent!=ROOT or hashlib.sha256(path.read_bytes()).hexdigest()!=expected: raise ValueError('artifact hash mismatch: '+name) + with gzip.open(ROOT/'reference.json.gz','rt') as handle: reference=json.load(handle) + with gzip.open(ROOT/'results.json.gz','rt') as handle: recording=json.load(handle) + stored=recording['tasks'][manifest['task']]['results'] + if set(reference)!=set(stored) or set(stored)!={'fold_'+str(i) for i in range(5)}: raise ValueError('fold coverage differs') + benchmark=None + if args.record: + from matbench.bench import MatbenchBenchmark + benchmark=MatbenchBenchmark(benchmark='matbench_v0.1',autoload=True,subset=[manifest['task']]) + metrics={}; maximum=0.; total=0 + for fold in range(5): + key='fold_'+str(fold); rows=reference[key]; table=stored[key]['data']; ids=[r['id'] for r in rows] + if len(ids)!=len(set(ids)) or set(ids)!=set(table): raise ValueError('row coverage differs') + predictions=table + if args.live: + batches=[rows[i:i+32] for i in range(0,len(rows),32)] + with ThreadPoolExecutor(max_workers=4) as pool: + responses=pool.map(lambda batch:verify_materials(manifest['task'],fold,batch),batches) + predictions={r['id']:r['prediction'] for response in responses for r in response} + difference=max(abs(predictions[i]-table[i]) for i in ids) + maximum=max(maximum,difference) + if difference>1e-12: raise ValueError('hosted prediction differs: '+str(difference)) + score=statistics.mean(abs(predictions[r['id']]-r['target']) for r in rows) + if not math.isclose(score,manifest['folds'][str(fold)],rel_tol=0,abs_tol=1e-12): raise ValueError('fold MAE differs') + if not math.isclose(score,stored[key]['scores']['mae'],rel_tol=0,abs_tol=1e-12): raise ValueError('native score differs') + metrics[str(fold)]=score; total+=len(rows) + if benchmark: + task=getattr(benchmark,manifest['task']); _,targets=task.get_test_data(fold,as_type='tuple',include_target=True) + official=list(map(str,targets.index)) + if official!=ids or any(float(t)!=r['target'] for t,r in zip(targets.to_numpy(),rows)): raise ValueError('official test set differs') + task.record(fold,[predictions[i] for i in official],params={'release':manifest['release'],'variant':manifest['variant']}) + mean=statistics.mean(metrics.values()) + if total!=manifest['rows'] or not math.isclose(mean,manifest['mae'],rel_tol=0,abs_tol=1e-12): raise ValueError('aggregate differs') + if benchmark: + if args.record.exists(): raise FileExistsError(args.record) + benchmark.add_metadata({'release':manifest['release'],'variant':manifest['variant']}) + if benchmark.validate(): raise ValueError('MatBench validation failed') + benchmark.to_file(str(args.record)) + if args.record.suffix == '.gz' and not args.record.read_bytes().startswith(b'\x1f\x8b'): + args.record.write_bytes(gzip.compress(args.record.read_bytes(),mtime=0)) + print(json.dumps(dict(release=manifest['release'],variant=manifest['variant'],rows=total,folds=metrics, + mean_mae=mean,reduction_percent=manifest['reduction_percent'],hosted_inference_checked=args.live, + maximum_hosted_prediction_difference=maximum if args.live else None),indent=2)) + + +if __name__=='__main__': main() diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/ARTIFACT_MANIFEST.json b/benchmarks/matbench_v0.1_SciKG_Verify/ARTIFACT_MANIFEST.json new file mode 100644 index 00000000..7234098e --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Verify/ARTIFACT_MANIFEST.json @@ -0,0 +1,25 @@ +{ + "release": "scikg-dielectric-20260906-v2", + "variant": "selected", + "task": "matbench_dielectric", + "rows": 4764, + "base_mae": 0.27110192416632367, + "mae": 0.25118012222453423, + "reduction_percent": 7.348454645997725, + "comparison_sha256": "a1bf3e1a9667cc6981628ebf21ba40f119d353ceb132f8744f273ac2db412054", + "hashes": { + "results.json.gz": "8504dd15b57c45616e2d946569a0326a6e69d06d85c99d350cb7025f1d058005", + "reference.json.gz": "d3055ac9ae576df7b986d6d3601dfaa2f64e443a0551e990ad9628efc2740492", + "statistics.json": "e1bef714f3e269a36b24513d1926c7bdc5b38f6a6699cde3e6c284c8a1537471", + "info.json": "037fbe2d5ec4435804e496095983caab7dd606758a99114e48ac23bbd3bfccbf", + "submission_client.py": "7308ae18603e257e4911fea3026fcd9557d0e895be7291aef55407fd27b147d6" + }, + "folds": { + "0": 0.15277098561203584, + "1": 0.22464586677727075, + "2": 0.36255234210722104, + "3": 0.25624146079630267, + "4": 0.2596899558298407 + }, + "service_url": "https://scikg-dielectric-20260906-v2.sahu-gaurav719.workers.dev" +} diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/info.json b/benchmarks/matbench_v0.1_SciKG_Verify/info.json new file mode 100644 index 00000000..5fbb40e6 --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Verify/info.json @@ -0,0 +1,14 @@ +{ + "authors": "Gaurav Sahu", + "algorithm": "SciKG Verify", + "algorithm_long": "Scientific-evidence correction of MODNet refractive-index predictions. The original selector chooses between a direct-evidence correction and a nonlinear residual correction. For each outer fold, correction-training base predictions come from MODNet models excluding that outer fold and the prediction fold. The official outer-fold MODNet predictions remain unchanged. The same original v2 evidence snapshot is retained. The benchmark driver uses a versioned hosted verifier; models and evidence remain on the service.", + "bibtex_refs": "@article{Dunn2020, title={Benchmarking materials property prediction methods: the Matbench test set and Automatminer reference algorithm}, author={Dunn, Alexander and Wang, Qi and Ganose, Alex and Dopp, Daniel and Jain, Anubhav}, journal={npj Computational Materials}, year={2020}, doi={10.1038/s41524-020-00406-3}}", + "notes": "Five official folds; 4,764 candidates; mean MAE 0.251180122225. Prior literature/database evidence is used by the verifier. This release repairs the outer-test training dependency while keeping the original correction rules fixed. V2 evidence matching is retained, not replaced by later StructureMatcher-confirmed evidence. The selector and nonlinear variant are reported together. Hosted inference reproduces the saved predictions; full retraining requires access to the service-side assets.", + "requirements": { + "python": [ + "matbench==0.6", + "numpy<2", + "pandas" + ] + } +} diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/reference.json.gz b/benchmarks/matbench_v0.1_SciKG_Verify/reference.json.gz new file mode 100644 index 00000000..6e8511aa Binary files /dev/null and b/benchmarks/matbench_v0.1_SciKG_Verify/reference.json.gz differ diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/results.json.gz b/benchmarks/matbench_v0.1_SciKG_Verify/results.json.gz new file mode 100644 index 00000000..9d3884e1 Binary files /dev/null and b/benchmarks/matbench_v0.1_SciKG_Verify/results.json.gz differ diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/statistics.json b/benchmarks/matbench_v0.1_SciKG_Verify/statistics.json new file mode 100644 index 00000000..eb4f55e1 --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Verify/statistics.json @@ -0,0 +1,13 @@ +{ + "estimand": "mean official-fold paired MAE reduction", + "paired_mae_reduction": 0.019921801941789444, + "bootstrap_95_percent_interval": [ + 0.01604759425046051, + 0.023865664127118196 + ], + "one_sided_sign_flip_p": 4.999750012499375e-05, + "resampling": "paired within each fixed official fold; conditional on fitted predictions", + "draws": 20000, + "seed": 20260906, + "uncertainty_scope": "row-resampling uncertainty, not training-seed or dataset uncertainty" +} diff --git a/benchmarks/matbench_v0.1_SciKG_Verify/submission_client.py b/benchmarks/matbench_v0.1_SciKG_Verify/submission_client.py new file mode 100644 index 00000000..7d90d0f4 --- /dev/null +++ b/benchmarks/matbench_v0.1_SciKG_Verify/submission_client.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Reproduce the release offline, or run its versioned hosted verifier. + +python submission_client.py +python submission_client.py --live +python submission_client.py --live --record rerun.json.gz + +The service evaluates fitted correction models using its private evidence view. +It accepts benchmark identities, not target labels. Full retraining requires +the service-side assets; this public driver covers inference and scoring. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import gzip +import hashlib +import json +import math +import os +from pathlib import Path +import statistics +import time +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +ROOT=Path(__file__).resolve().parent + + +def verify_materials(endpoint, fold, candidates, variant=None): + manifest=json.loads((ROOT/'ARTIFACT_MANIFEST.json').read_text()) + variant=variant or manifest['variant'] + url=os.environ.get('SCIENTIA_VERIFIER_API_URL',manifest['service_url']).rstrip('/')+'/predict' + payload=dict(release=manifest['release'],endpoint=endpoint,fold=fold,variant=variant, + candidates=[{'id':str(row['id'])} for row in candidates]) + req=Request(url,data=json.dumps(payload).encode(),headers={'Content-Type':'application/json','User-Agent':'SciKG-Benchmark/2.0'},method='POST') + for attempt in range(3): + try: + with urlopen(req,timeout=60) as response: value=json.load(response) + break + except (HTTPError,URLError) as error: + if attempt==2 or isinstance(error,HTTPError) and error.code<500 and error.code!=429: raise + time.sleep(2**attempt) + if value.get('release')!=manifest['release'] or value.get('variant')!=variant or value.get('execution')!='model_inference': + raise ValueError('unexpected service version or execution mode') + out=value.get('predictions',[]) + if [r['id'] for r in out]!=[r['id'] for r in payload['candidates']]: raise ValueError('service changed row identities/order') + if any(not math.isfinite(r['prediction']) for r in out): raise ValueError('nonfinite response') + return out + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--live',action='store_true') + parser.add_argument('--record',type=Path) + args=parser.parse_args() + if args.record and not args.live: parser.error('--record requires --live') + manifest=json.loads((ROOT/'ARTIFACT_MANIFEST.json').read_text()) + for name,expected in manifest['hashes'].items(): + path=ROOT/name + if path.parent!=ROOT or hashlib.sha256(path.read_bytes()).hexdigest()!=expected: raise ValueError('artifact hash mismatch: '+name) + with gzip.open(ROOT/'reference.json.gz','rt') as handle: reference=json.load(handle) + with gzip.open(ROOT/'results.json.gz','rt') as handle: recording=json.load(handle) + stored=recording['tasks'][manifest['task']]['results'] + if set(reference)!=set(stored) or set(stored)!={'fold_'+str(i) for i in range(5)}: raise ValueError('fold coverage differs') + benchmark=None + if args.record: + from matbench.bench import MatbenchBenchmark + benchmark=MatbenchBenchmark(benchmark='matbench_v0.1',autoload=True,subset=[manifest['task']]) + metrics={}; maximum=0.; total=0 + for fold in range(5): + key='fold_'+str(fold); rows=reference[key]; table=stored[key]['data']; ids=[r['id'] for r in rows] + if len(ids)!=len(set(ids)) or set(ids)!=set(table): raise ValueError('row coverage differs') + predictions=table + if args.live: + batches=[rows[i:i+32] for i in range(0,len(rows),32)] + with ThreadPoolExecutor(max_workers=4) as pool: + responses=pool.map(lambda batch:verify_materials(manifest['task'],fold,batch),batches) + predictions={r['id']:r['prediction'] for response in responses for r in response} + difference=max(abs(predictions[i]-table[i]) for i in ids) + maximum=max(maximum,difference) + if difference>1e-12: raise ValueError('hosted prediction differs: '+str(difference)) + score=statistics.mean(abs(predictions[r['id']]-r['target']) for r in rows) + if not math.isclose(score,manifest['folds'][str(fold)],rel_tol=0,abs_tol=1e-12): raise ValueError('fold MAE differs') + if not math.isclose(score,stored[key]['scores']['mae'],rel_tol=0,abs_tol=1e-12): raise ValueError('native score differs') + metrics[str(fold)]=score; total+=len(rows) + if benchmark: + task=getattr(benchmark,manifest['task']); _,targets=task.get_test_data(fold,as_type='tuple',include_target=True) + official=list(map(str,targets.index)) + if official!=ids or any(float(t)!=r['target'] for t,r in zip(targets.to_numpy(),rows)): raise ValueError('official test set differs') + task.record(fold,[predictions[i] for i in official],params={'release':manifest['release'],'variant':manifest['variant']}) + mean=statistics.mean(metrics.values()) + if total!=manifest['rows'] or not math.isclose(mean,manifest['mae'],rel_tol=0,abs_tol=1e-12): raise ValueError('aggregate differs') + if benchmark: + if args.record.exists(): raise FileExistsError(args.record) + benchmark.add_metadata({'release':manifest['release'],'variant':manifest['variant']}) + if benchmark.validate(): raise ValueError('MatBench validation failed') + benchmark.to_file(str(args.record)) + if args.record.suffix == '.gz' and not args.record.read_bytes().startswith(b'\x1f\x8b'): + args.record.write_bytes(gzip.compress(args.record.read_bytes(),mtime=0)) + print(json.dumps(dict(release=manifest['release'],variant=manifest['variant'],rows=total,folds=metrics, + mean_mae=mean,reduction_percent=manifest['reduction_percent'],hosted_inference_checked=args.live, + maximum_hosted_prediction_difference=maximum if args.live else None),indent=2)) + + +if __name__=='__main__': main() diff --git a/requirements-dev.txt b/requirements-dev.txt index 4c2ec0e9..97a35f33 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,12 @@ pytest coverage==6.4.3 wheel==0.37.1 -monty==2022.4.26 +# The current test suite still references np.float_, removed in NumPy 2. +numpy<2 +# Matminer and pymatgen installed by matbench require a current Monty. The +# previous 2022 pin downgraded the compatible version selected by pip and made +# both the test and benchmark-submission jobs fail during import. +monty>=2024.7.29 isort==5.10.1 black==22.3.0 -flake8==4.0.1 \ No newline at end of file +flake8==4.0.1