New Benchmark: SciKG Verify and SciKG Residual dielectric results

.github/workflows/python-package.yml

@@ -17,9 +17,9 @@ jobs:
         python-version: ["3.9"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Test
@@ -31,7 +31,7 @@ jobs:
           coverage run -m unittest discover
           coverage json
       - name: Upload coverage artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: coverage_artifact
           path: coverage.json
@@ -40,9 +40,9 @@ jobs:
     needs: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Download coverage artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: coverage_artifact
       - name: Check coverage
@@ -54,9 +54,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up python 3.9
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: 3.9
     - name: Run submission check

benchmarks/matbench_v0.1_SciKG_Residual/ARTIFACT_MANIFEST.json

@@ -0,0 +1,25 @@
+{
+  "release": "scikg-dielectric-20260906-v2",
+  "variant": "nonlinear",
+  "task": "matbench_dielectric",
+  "rows": 4764,
+  "base_mae": 0.27110192416632367,
+  "mae": 0.24728747764262607,
+  "reduction_percent": 8.784314828059724,
+  "comparison_sha256": "a1bf3e1a9667cc6981628ebf21ba40f119d353ceb132f8744f273ac2db412054",
+  "hashes": {
+    "results.json.gz": "2139d623223be81bdd1fed4c1edbb78be2553385ee92e547b2da7c065ee83352",
+    "reference.json.gz": "d3055ac9ae576df7b986d6d3601dfaa2f64e443a0551e990ad9628efc2740492",
+    "statistics.json": "7f854af98a9c1e6eb5a777d5cad5f8b8555c21d3831cb303432a124802c056a5",
+    "info.json": "a96eede494db3cdda0f13232d3f724afdd5552b5349c00c94033613cd9a6fd88",
+    "submission_client.py": "7308ae18603e257e4911fea3026fcd9557d0e895be7291aef55407fd27b147d6"
+  },
+  "folds": {
+    "0": 0.1492487654143906,
+    "1": 0.21280008842939888,
+    "2": 0.359168979806848,
+    "3": 0.2568909916360868,
+    "4": 0.25832856292640605
+  },
+  "service_url": "https://scikg-dielectric-20260906-v2.sahu-gaurav719.workers.dev"
+}

benchmarks/matbench_v0.1_SciKG_Residual/info.json

@@ -0,0 +1,14 @@
+{
+  "authors": "Gaurav Sahu",
+  "algorithm": "SciKG Residual",
+  "algorithm_long": "Scientific-evidence correction of MODNet refractive-index predictions. This separately reported, pre-existing variant uses the nonlinear residual correction for every fold; it does not select the best variant by test fold. For each outer fold, correction-training base predictions come from MODNet models excluding that outer fold and the prediction fold. The official outer-fold MODNet predictions remain unchanged. The same original v2 evidence snapshot is retained. The benchmark driver uses a versioned hosted verifier; models and evidence remain on the service.",
+  "bibtex_refs": "@article{Dunn2020, title={Benchmarking materials property prediction methods: the Matbench test set and Automatminer reference algorithm}, author={Dunn, Alexander and Wang, Qi and Ganose, Alex and Dopp, Daniel and Jain, Anubhav}, journal={npj Computational Materials}, year={2020}, doi={10.1038/s41524-020-00406-3}}",
+  "notes": "Five official folds; 4,764 candidates; mean MAE 0.247287477643. Prior literature/database evidence is used by the verifier. This release repairs the outer-test training dependency while keeping the original correction rules fixed. V2 evidence matching is retained, not replaced by later StructureMatcher-confirmed evidence. The selector and nonlinear variant are reported together. Hosted inference reproduces the saved predictions; full retraining requires access to the service-side assets.",
+  "requirements": {
+    "python": [
+      "matbench==0.6",
+      "numpy<2",
+      "pandas"
+    ]
+  }
+}

benchmarks/matbench_v0.1_SciKG_Residual/statistics.json

@@ -0,0 +1,13 @@
+{
+  "estimand": "mean official-fold paired MAE reduction",
+  "paired_mae_reduction": 0.023814446523697592,
+  "bootstrap_95_percent_interval": [
+    0.02016465757512394,
+    0.027480339567094666
+  ],
+  "one_sided_sign_flip_p": 4.999750012499375e-05,
+  "resampling": "paired within each fixed official fold; conditional on fitted predictions",
+  "draws": 20000,
+  "seed": 20260906,
+  "uncertainty_scope": "row-resampling uncertainty, not training-seed or dataset uncertainty"
+}

benchmarks/matbench_v0.1_SciKG_Residual/submission_client.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Reproduce the release offline, or run its versioned hosted verifier.
+
+python submission_client.py
+python submission_client.py --live
+python submission_client.py --live --record rerun.json.gz
+
+The service evaluates fitted correction models using its private evidence view.
+It accepts benchmark identities, not target labels. Full retraining requires
+the service-side assets; this public driver covers inference and scoring.
+"""
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+import gzip
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import statistics
+import time
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+ROOT=Path(__file__).resolve().parent
+
+
+def verify_materials(endpoint, fold, candidates, variant=None):
+    manifest=json.loads((ROOT/'ARTIFACT_MANIFEST.json').read_text())
+    variant=variant or manifest['variant']
+    url=os.environ.get('SCIENTIA_VERIFIER_API_URL',manifest['service_url']).rstrip('/')+'/predict'
+    payload=dict(release=manifest['release'],endpoint=endpoint,fold=fold,variant=variant,
+                 candidates=[{'id':str(row['id'])} for row in candidates])
+    req=Request(url,data=json.dumps(payload).encode(),headers={'Content-Type':'application/json','User-Agent':'SciKG-Benchmark/2.0'},method='POST')
+    for attempt in range(3):
+        try:
+            with urlopen(req,timeout=60) as response: value=json.load(response)
+            break
+        except (HTTPError,URLError) as error:
+            if attempt==2 or isinstance(error,HTTPError) and error.code<500 and error.code!=429: raise
+            time.sleep(2**attempt)
+    if value.get('release')!=manifest['release'] or value.get('variant')!=variant or value.get('execution')!='model_inference':
+        raise ValueError('unexpected service version or execution mode')
+    out=value.get('predictions',[])
+    if [r['id'] for r in out]!=[r['id'] for r in payload['candidates']]: raise ValueError('service changed row identities/order')
+    if any(not math.isfinite(r['prediction']) for r in out): raise ValueError('nonfinite response')
+    return out
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--live',action='store_true')
+    parser.add_argument('--record',type=Path)
+    args=parser.parse_args()
+    if args.record and not args.live: parser.error('--record requires --live')
+    manifest=json.loads((ROOT/'ARTIFACT_MANIFEST.json').read_text())
+    for name,expected in manifest['hashes'].items():
+        path=ROOT/name
+        if path.parent!=ROOT or hashlib.sha256(path.read_bytes()).hexdigest()!=expected: raise ValueError('artifact hash mismatch: '+name)
+    with gzip.open(ROOT/'reference.json.gz','rt') as handle: reference=json.load(handle)
+    with gzip.open(ROOT/'results.json.gz','rt') as handle: recording=json.load(handle)
+    stored=recording['tasks'][manifest['task']]['results']
+    if set(reference)!=set(stored) or set(stored)!={'fold_'+str(i) for i in range(5)}: raise ValueError('fold coverage differs')
+    benchmark=None
+    if args.record:
+        from matbench.bench import MatbenchBenchmark
+        benchmark=MatbenchBenchmark(benchmark='matbench_v0.1',autoload=True,subset=[manifest['task']])
+    metrics={}; maximum=0.; total=0
+    for fold in range(5):
+        key='fold_'+str(fold); rows=reference[key]; table=stored[key]['data']; ids=[r['id'] for r in rows]
+        if len(ids)!=len(set(ids)) or set(ids)!=set(table): raise ValueError('row coverage differs')
+        predictions=table
+        if args.live:
+            batches=[rows[i:i+32] for i in range(0,len(rows),32)]
+            with ThreadPoolExecutor(max_workers=4) as pool:
+                responses=pool.map(lambda batch:verify_materials(manifest['task'],fold,batch),batches)
+                predictions={r['id']:r['prediction'] for response in responses for r in response}
+            difference=max(abs(predictions[i]-table[i]) for i in ids)
+            maximum=max(maximum,difference)
+            if difference>1e-12: raise ValueError('hosted prediction differs: '+str(difference))
+        score=statistics.mean(abs(predictions[r['id']]-r['target']) for r in rows)
+        if not math.isclose(score,manifest['folds'][str(fold)],rel_tol=0,abs_tol=1e-12): raise ValueError('fold MAE differs')
+        if not math.isclose(score,stored[key]['scores']['mae'],rel_tol=0,abs_tol=1e-12): raise ValueError('native score differs')
+        metrics[str(fold)]=score; total+=len(rows)
+        if benchmark:
+            task=getattr(benchmark,manifest['task']); _,targets=task.get_test_data(fold,as_type='tuple',include_target=True)
+            official=list(map(str,targets.index))
+            if official!=ids or any(float(t)!=r['target'] for t,r in zip(targets.to_numpy(),rows)): raise ValueError('official test set differs')
+            task.record(fold,[predictions[i] for i in official],params={'release':manifest['release'],'variant':manifest['variant']})
+    mean=statistics.mean(metrics.values())
+    if total!=manifest['rows'] or not math.isclose(mean,manifest['mae'],rel_tol=0,abs_tol=1e-12): raise ValueError('aggregate differs')
+    if benchmark:
+        if args.record.exists(): raise FileExistsError(args.record)
+        benchmark.add_metadata({'release':manifest['release'],'variant':manifest['variant']})
+        if benchmark.validate(): raise ValueError('MatBench validation failed')
+        benchmark.to_file(str(args.record))
+        if args.record.suffix == '.gz' and not args.record.read_bytes().startswith(b'\x1f\x8b'):
+            args.record.write_bytes(gzip.compress(args.record.read_bytes(),mtime=0))
+    print(json.dumps(dict(release=manifest['release'],variant=manifest['variant'],rows=total,folds=metrics,
+        mean_mae=mean,reduction_percent=manifest['reduction_percent'],hosted_inference_checked=args.live,
+        maximum_hosted_prediction_difference=maximum if args.live else None),indent=2))
+
+
+if __name__=='__main__': main()

benchmarks/matbench_v0.1_SciKG_Verify/ARTIFACT_MANIFEST.json

@@ -0,0 +1,25 @@
+{
+  "release": "scikg-dielectric-20260906-v2",
+  "variant": "selected",
+  "task": "matbench_dielectric",
+  "rows": 4764,
+  "base_mae": 0.27110192416632367,
+  "mae": 0.25118012222453423,
+  "reduction_percent": 7.348454645997725,
+  "comparison_sha256": "a1bf3e1a9667cc6981628ebf21ba40f119d353ceb132f8744f273ac2db412054",
+  "hashes": {
+    "results.json.gz": "8504dd15b57c45616e2d946569a0326a6e69d06d85c99d350cb7025f1d058005",
+    "reference.json.gz": "d3055ac9ae576df7b986d6d3601dfaa2f64e443a0551e990ad9628efc2740492",
+    "statistics.json": "e1bef714f3e269a36b24513d1926c7bdc5b38f6a6699cde3e6c284c8a1537471",
+    "info.json": "037fbe2d5ec4435804e496095983caab7dd606758a99114e48ac23bbd3bfccbf",
+    "submission_client.py": "7308ae18603e257e4911fea3026fcd9557d0e895be7291aef55407fd27b147d6"
+  },
+  "folds": {
+    "0": 0.15277098561203584,
+    "1": 0.22464586677727075,
+    "2": 0.36255234210722104,
+    "3": 0.25624146079630267,
+    "4": 0.2596899558298407
+  },
+  "service_url": "https://scikg-dielectric-20260906-v2.sahu-gaurav719.workers.dev"
+}

benchmarks/matbench_v0.1_SciKG_Verify/info.json

@@ -0,0 +1,14 @@
+{
+  "authors": "Gaurav Sahu",
+  "algorithm": "SciKG Verify",
+  "algorithm_long": "Scientific-evidence correction of MODNet refractive-index predictions. The original selector chooses between a direct-evidence correction and a nonlinear residual correction. For each outer fold, correction-training base predictions come from MODNet models excluding that outer fold and the prediction fold. The official outer-fold MODNet predictions remain unchanged. The same original v2 evidence snapshot is retained. The benchmark driver uses a versioned hosted verifier; models and evidence remain on the service.",
+  "bibtex_refs": "@article{Dunn2020, title={Benchmarking materials property prediction methods: the Matbench test set and Automatminer reference algorithm}, author={Dunn, Alexander and Wang, Qi and Ganose, Alex and Dopp, Daniel and Jain, Anubhav}, journal={npj Computational Materials}, year={2020}, doi={10.1038/s41524-020-00406-3}}",
+  "notes": "Five official folds; 4,764 candidates; mean MAE 0.251180122225. Prior literature/database evidence is used by the verifier. This release repairs the outer-test training dependency while keeping the original correction rules fixed. V2 evidence matching is retained, not replaced by later StructureMatcher-confirmed evidence. The selector and nonlinear variant are reported together. Hosted inference reproduces the saved predictions; full retraining requires access to the service-side assets.",
+  "requirements": {
+    "python": [
+      "matbench==0.6",
+      "numpy<2",
+      "pandas"
+    ]
+  }
+}

benchmarks/matbench_v0.1_SciKG_Verify/statistics.json

@@ -0,0 +1,13 @@
+{
+  "estimand": "mean official-fold paired MAE reduction",
+  "paired_mae_reduction": 0.019921801941789444,
+  "bootstrap_95_percent_interval": [
+    0.01604759425046051,
+    0.023865664127118196
+  ],
+  "one_sided_sign_flip_p": 4.999750012499375e-05,
+  "resampling": "paired within each fixed official fold; conditional on fitted predictions",
+  "draws": 20000,
+  "seed": 20260906,
+  "uncertainty_scope": "row-resampling uncertainty, not training-seed or dataset uncertainty"
+}

benchmarks/matbench_v0.1_SciKG_Verify/submission_client.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Reproduce the release offline, or run its versioned hosted verifier.
+
+python submission_client.py
+python submission_client.py --live
+python submission_client.py --live --record rerun.json.gz
+
+The service evaluates fitted correction models using its private evidence view.
+It accepts benchmark identities, not target labels. Full retraining requires
+the service-side assets; this public driver covers inference and scoring.
+"""
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+import gzip
+import hashlib
+import json
+import math
+import os
+from pathlib import Path
+import statistics
+import time
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+ROOT=Path(__file__).resolve().parent
+
+
+def verify_materials(endpoint, fold, candidates, variant=None):
+    manifest=json.loads((ROOT/'ARTIFACT_MANIFEST.json').read_text())
+    variant=variant or manifest['variant']
+    url=os.environ.get('SCIENTIA_VERIFIER_API_URL',manifest['service_url']).rstrip('/')+'/predict'
+    payload=dict(release=manifest['release'],endpoint=endpoint,fold=fold,variant=variant,
+                 candidates=[{'id':str(row['id'])} for row in candidates])
+    req=Request(url,data=json.dumps(payload).encode(),headers={'Content-Type':'application/json','User-Agent':'SciKG-Benchmark/2.0'},method='POST')
+    for attempt in range(3):
+        try:
+            with urlopen(req,timeout=60) as response: value=json.load(response)
+            break
+        except (HTTPError,URLError) as error:
+            if attempt==2 or isinstance(error,HTTPError) and error.code<500 and error.code!=429: raise
+            time.sleep(2**attempt)
+    if value.get('release')!=manifest['release'] or value.get('variant')!=variant or value.get('execution')!='model_inference':
+        raise ValueError('unexpected service version or execution mode')
+    out=value.get('predictions',[])
+    if [r['id'] for r in out]!=[r['id'] for r in payload['candidates']]: raise ValueError('service changed row identities/order')
+    if any(not math.isfinite(r['prediction']) for r in out): raise ValueError('nonfinite response')
+    return out
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--live',action='store_true')
+    parser.add_argument('--record',type=Path)
+    args=parser.parse_args()
+    if args.record and not args.live: parser.error('--record requires --live')
+    manifest=json.loads((ROOT/'ARTIFACT_MANIFEST.json').read_text())
+    for name,expected in manifest['hashes'].items():
+        path=ROOT/name
+        if path.parent!=ROOT or hashlib.sha256(path.read_bytes()).hexdigest()!=expected: raise ValueError('artifact hash mismatch: '+name)
+    with gzip.open(ROOT/'reference.json.gz','rt') as handle: reference=json.load(handle)
+    with gzip.open(ROOT/'results.json.gz','rt') as handle: recording=json.load(handle)
+    stored=recording['tasks'][manifest['task']]['results']
+    if set(reference)!=set(stored) or set(stored)!={'fold_'+str(i) for i in range(5)}: raise ValueError('fold coverage differs')
+    benchmark=None
+    if args.record:
+        from matbench.bench import MatbenchBenchmark
+        benchmark=MatbenchBenchmark(benchmark='matbench_v0.1',autoload=True,subset=[manifest['task']])
+    metrics={}; maximum=0.; total=0
+    for fold in range(5):
+        key='fold_'+str(fold); rows=reference[key]; table=stored[key]['data']; ids=[r['id'] for r in rows]
+        if len(ids)!=len(set(ids)) or set(ids)!=set(table): raise ValueError('row coverage differs')
+        predictions=table
+        if args.live:
+            batches=[rows[i:i+32] for i in range(0,len(rows),32)]
+            with ThreadPoolExecutor(max_workers=4) as pool:
+                responses=pool.map(lambda batch:verify_materials(manifest['task'],fold,batch),batches)
+                predictions={r['id']:r['prediction'] for response in responses for r in response}
+            difference=max(abs(predictions[i]-table[i]) for i in ids)
+            maximum=max(maximum,difference)
+            if difference>1e-12: raise ValueError('hosted prediction differs: '+str(difference))
+        score=statistics.mean(abs(predictions[r['id']]-r['target']) for r in rows)
+        if not math.isclose(score,manifest['folds'][str(fold)],rel_tol=0,abs_tol=1e-12): raise ValueError('fold MAE differs')
+        if not math.isclose(score,stored[key]['scores']['mae'],rel_tol=0,abs_tol=1e-12): raise ValueError('native score differs')
+        metrics[str(fold)]=score; total+=len(rows)
+        if benchmark:
+            task=getattr(benchmark,manifest['task']); _,targets=task.get_test_data(fold,as_type='tuple',include_target=True)
+            official=list(map(str,targets.index))
+            if official!=ids or any(float(t)!=r['target'] for t,r in zip(targets.to_numpy(),rows)): raise ValueError('official test set differs')
+            task.record(fold,[predictions[i] for i in official],params={'release':manifest['release'],'variant':manifest['variant']})
+    mean=statistics.mean(metrics.values())
+    if total!=manifest['rows'] or not math.isclose(mean,manifest['mae'],rel_tol=0,abs_tol=1e-12): raise ValueError('aggregate differs')
+    if benchmark:
+        if args.record.exists(): raise FileExistsError(args.record)
+        benchmark.add_metadata({'release':manifest['release'],'variant':manifest['variant']})
+        if benchmark.validate(): raise ValueError('MatBench validation failed')
+        benchmark.to_file(str(args.record))
+        if args.record.suffix == '.gz' and not args.record.read_bytes().startswith(b'\x1f\x8b'):
+            args.record.write_bytes(gzip.compress(args.record.read_bytes(),mtime=0))
+    print(json.dumps(dict(release=manifest['release'],variant=manifest['variant'],rows=total,folds=metrics,
+        mean_mae=mean,reduction_percent=manifest['reduction_percent'],hosted_inference_checked=args.live,
+        maximum_hosted_prediction_difference=maximum if args.live else None),indent=2))
+
+
+if __name__=='__main__': main()

requirements-dev.txt

@@ -1,7 +1,12 @@
 pytest
 coverage==6.4.3
 wheel==0.37.1
-monty==2022.4.26
+# The current test suite still references np.float_, removed in NumPy 2.
+numpy<2
+# Matminer and pymatgen installed by matbench require a current Monty.  The
+# previous 2022 pin downgraded the compatible version selected by pip and made
+# both the test and benchmark-submission jobs fail during import.
+monty>=2024.7.29
 isort==5.10.1
 black==22.3.0
-flake8==4.0.1
+flake8==4.0.1