From e10bf9847af15b2fe92652f0cfd03face7dc78bb Mon Sep 17 00:00:00 2001 From: Thomas Prest Date: Thu, 9 Jul 2026 23:55:15 +0200 Subject: [PATCH] Fix to_json() crashing on the test batteries' numpy scalars to_json() raised `TypeError: Object of type bool is not JSON serializable` on both UnivariateSamples and MultivariateSamples as soon as the extended battery had been run: the battery results carry numpy scalars (np.bool_ for `passes`, np.float64 for statistics like tail_exceedance's `expected`), which json.dumps refuses. This made the advertised programmatic API -- to_dict()/to_json() for batch analysis -- unusable in exactly the case it exists for. The scripts already worked around it with their own local encoders (NumpyEncoder in run_baseline.py, SafeEncoder in calibration.py); the methods never got one. Add a recursive _jsonable() helper and apply it in both to_dict() methods, so to_dict() returns plain Python and both to_dict() and to_json() are safe (a user calling json.dumps(obj.to_dict()) themselves now works too). Added test_to_json_after_battery on both classes; they fail on all seven good vectors without this fix. Co-Authored-By: Claude Opus 4.8 --- code/saga.py | 29 ++++++++++++++++++++++++----- code/tests/test_multivariate.py | 11 +++++++++++ code/tests/test_univariate.py | 12 ++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/code/saga.py b/code/saga.py index 88d4467..610f7b8 100644 --- a/code/saga.py +++ b/code/saga.py @@ -16,7 +16,7 @@ # Distributions from scipy.stats import chi2, norm # Numpy stuff -from numpy import cov, set_printoptions, diag, array, mean +from numpy import cov, set_printoptions, diag, array, mean, ndarray, generic from numpy.linalg import matrix_rank, inv, eig, eigh import matplotlib.pyplot as plt @@ -62,6 +62,25 @@ set_printoptions(precision=4) +def _jsonable(obj): + """ + Recursively convert numpy scalars and arrays to plain Python types. + + The test batteries return numpy scalars (np.bool_ for `passes`, np.float64 + for statistics), which json.dumps refuses. Applying this to the output of + to_dict() keeps both to_dict() and to_json() usable for batch analysis. + """ + if isinstance(obj, dict): + return {key: _jsonable(value) for key, value in obj.items()} + if isinstance(obj, (list, tuple)): + return [_jsonable(value) for value in obj] + if isinstance(obj, ndarray): + return _jsonable(obj.tolist()) + if isinstance(obj, generic): # any numpy scalar (bool_, int64, ...) + return obj.item() + return obj + + def gaussian(x, mu, sigma): """ Gaussian function of center mu and "standard deviation" sigma. @@ -224,7 +243,7 @@ def effect_sizes(self): def to_dict(self): es = self.effect_sizes() - return { + return _jsonable({ "test": "univariate", "params": { "mu": self.exp_mu, "sigma": self.exp_sigma, @@ -249,7 +268,7 @@ def to_dict(self): "effect_sizes": es, "extended_tests": self._extended, "is_valid_extended": getattr(self, 'is_valid_extended', None), - } + }) def to_json(self): return json.dumps(self.to_dict(), indent=2) @@ -392,7 +411,7 @@ def __repr__(self): return rep def to_dict(self): - return { + return _jsonable({ "test": "multivariate", "params": {"sigma": self.exp_si, "dim": self.dim, "n": self.nsamples}, "doornik_hansen": {"stat": float(self.DH), "pvalue": float(self.PO)}, @@ -404,7 +423,7 @@ def to_dict(self): "total": self.dim, "applicable": self.is_integer}, "extended_tests": self._mv_extended, - } + }) def to_json(self): return json.dumps(self.to_dict(), indent=2) diff --git a/code/tests/test_multivariate.py b/code/tests/test_multivariate.py index 84ed249..553bd3b 100644 --- a/code/tests/test_multivariate.py +++ b/code/tests/test_multivariate.py @@ -1,4 +1,5 @@ """Test SAGA multivariate analysis against generated test vectors.""" +import json import sys import os @@ -50,6 +51,16 @@ def test_extended_battery_passes(self, good_multivariate_vector): if isinstance(r, dict) and not r.get("passes", True)) ) + def test_to_json_after_battery(self, good_multivariate_vector): + """to_dict()/to_json() must survive the battery's numpy scalars.""" + v = good_multivariate_vector + mv = MultivariateSamples(v["params"]["sigma"], v["samples"]) + mv.run_multivariate_battery() + parsed = json.loads(mv.to_json()) + assert parsed["test"] == "multivariate" + assert parsed["extended_tests"]["all_pass"] is True + json.dumps(mv.to_dict()) # a user dumping to_dict() directly also works + def test_per_coordinate_channel(self, good_multivariate_vector): """Integer (discrete-Gaussian) good vectors must light up the per-coordinate discrete channel; continuous vectors skip it.""" diff --git a/code/tests/test_univariate.py b/code/tests/test_univariate.py index 6c77a4d..1871170 100644 --- a/code/tests/test_univariate.py +++ b/code/tests/test_univariate.py @@ -1,4 +1,5 @@ """Test SAGA univariate analysis against generated test vectors.""" +import json import sys import os @@ -49,6 +50,17 @@ def test_extended_battery_passes(self, good_univariate_vector): if isinstance(r, dict) and not r.get("passes", True)) ) + def test_to_json_after_battery(self, good_univariate_vector): + """to_dict()/to_json() must survive the battery's numpy scalars.""" + v = good_univariate_vector + uv = UnivariateSamples(v["params"]["mu"], v["params"]["sigma"], v["samples"]) + uv.run_extended_battery(samples=v["samples"], mc_B=100) + # to_dict() must be plain Python, to_json() must not raise + parsed = json.loads(uv.to_json()) + assert parsed["test"] == "univariate" + assert parsed["extended_tests"]["all_pass"] is True + json.dumps(uv.to_dict()) # a user dumping to_dict() directly also works + class TestBadVectors: """Flawed distributions must be detected by chi-square or extended battery."""