From abb1e24b34193d4eb2369a949fe03042e3356334 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 09:22:40 +0200 Subject: [PATCH 01/26] add combinations from parser branch --- src/detectmatelibrary/utils/finetune.py | 51 +++++++++++++++ tests/test_utils/test_finetune.py | 82 +++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 src/detectmatelibrary/utils/finetune.py create mode 100644 tests/test_utils/test_finetune.py diff --git a/src/detectmatelibrary/utils/finetune.py b/src/detectmatelibrary/utils/finetune.py new file mode 100644 index 00000000..c29c6d47 --- /dev/null +++ b/src/detectmatelibrary/utils/finetune.py @@ -0,0 +1,51 @@ +from detectmatelibrary.common.core import CoreConfig + +from typing import Any +import numpy as np + +import itertools +import warnings +import typing +import copy + + +class Combinations: + def __init__(self, config: CoreConfig) -> None: + self.config = config + + self.paths: list[str] = [] + self.combs: list[tuple[Any, ...]] = [] + if "finetune" not in dir(config): + warnings.warn("No finetune options found") + else: + self.paths = [path[0] for path in getattr(config, "finetune")] + self.combs = list(itertools.product( + *[path[-1] for path in getattr(config, "finetune")] + )) + self.values: list[float] = [] + + def add_value(self, value: float) -> None: + self.values.append(value) + + def get_best(self) -> CoreConfig: + if self.values == []: + return self.config + + idx = np.argmin(self.values) + for i, combo in enumerate(self.combs): + if i == idx: + config = copy.deepcopy(self.config) + for path, value in zip(self.paths, combo): + if path in dir(self.config): + setattr(config, path, value) + return config + return self.config + + def __call__(self) -> typing.Iterable[CoreConfig]: + for combo in self.combs: + config = copy.deepcopy(self.config) + for path, value in zip(self.paths, combo): + if path in dir(config): + setattr(config, path, value) + + yield config diff --git a/tests/test_utils/test_finetune.py b/tests/test_utils/test_finetune.py new file mode 100644 index 00000000..feb9c0a3 --- /dev/null +++ b/tests/test_utils/test_finetune.py @@ -0,0 +1,82 @@ + +from detectmatelibrary.utils.finetune import Combinations + +from detectmatelibrary.common.core import CoreConfig + +from typing import Any +import pytest + + +class DummyConfig(CoreConfig): + a: int = 2 + b: int = 10 + c: float = 0.2 + + finetune: list[tuple[str, list[Any]]] = [ + ["a", [1, 2, 3, 4]], + ["b", [10, 40]], + ["c", [0.2, 0.4, 0.6, 0.8]] + ] + + +class DummyConfig2(CoreConfig): + a: int = 2 + b: int = 10 + c: float = 0.4 + + +class DummyConfig3(CoreConfig): + a: int = 2 + b: int = 10 + + finetune: list[tuple[str, list[Any]]] = [ + ["a", [1, 2, 3, 4]], + ["b", [10, 40]], + ["c", [0.2, 0.4, 0.6, 0.8]] + ] + + +class TestCombinations: + def test_combination_no_overwrite(self): + comb = Combinations(config := DummyConfig()) + assert config != next(comb()) + + def test_call_format(self): + comb = Combinations(DummyConfig()) + config = next(comb()) + + assert config.a == 1 + assert config.b == 10 + assert config.c == 0.2 + + def test_get_best(self): + comb = Combinations(DummyConfig()) + for j, _ in enumerate(comb()): + comb.add_value(j) + + best_config = comb.get_best() + assert best_config.a == 1 + assert best_config.b == 10 + assert best_config.c == 0.2 + + def test_finetune_not_found(self): + with pytest.warns(UserWarning): + comb = Combinations(DummyConfig2()) + + for _ in comb(): + assert False + config = comb.get_best() + + assert config.a == 2 + assert config.b == 10 + assert config.c == 0.4 + + def test_argument_missing(self): + comb = Combinations(DummyConfig3()) + + for j, _ in enumerate(comb()): + comb.add_value(j) + config = comb.get_best() + + assert config.a == 1 + assert config.b == 10 From 9368328eab45fb93acf95c93a490e7ab838a1e34 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 09:27:18 +0200 Subject: [PATCH 02/26] add deep learning dependencies --- pyproject.toml | 2 + uv.lock | 429 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 427 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 904edf5f..c3f92210 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,8 @@ dependencies = [ "msgpack>=1.0.0", "fsspec>=2024.1.0", "pyarrow>=24.0.0", + "jax>=0.11.0", + "flax>=0.12.8", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index d2ccd2b3..c655e443 100644 --- a/uv.lock +++ b/uv.lock @@ -1,13 +1,34 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version < '3.14' and sys_platform == 'win32'", - "python_full_version < '3.14' and sys_platform == 'emscripten'", - "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'win32'", + "python_full_version < '3.13' and sys_platform == 'win32'", + "python_full_version == '3.13.*' and sys_platform == 'emscripten'", + "python_full_version < '3.13' and sys_platform == 'emscripten'", + "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] + +[[package]] +name = "absl-py" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/4f/d79676ab82f2e42fc3611618139f13a9c4c31d0cff4b486982047679a802/absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f", size = 118119, upload-time = "2026-07-03T10:57:48.157Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/0a/a10b45aab35b175aded078a462dc8d0c698f5b13946e7cb0869097b78bb6/absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba", size = 137410, upload-time = "2026-07-03T10:57:46.735Z" }, +] + +[[package]] +name = "aiofiles" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/c3/534eac40372d8ee36ef40df62ec129bee4fdb5ad9706e58a29be53b2c970/aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2", size = 46354, upload-time = "2025-10-09T20:51:04.358Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" }, ] [[package]] @@ -197,7 +218,9 @@ name = "detectmatelibrary" source = { editable = "." } dependencies = [ { name = "detectmateperformance" }, + { name = "flax" }, { name = "fsspec" }, + { name = "jax" }, { name = "msgpack" }, { name = "numpy" }, { name = "protobuf" }, @@ -248,7 +271,9 @@ requires-dist = [ { name = "detectmatelibrary", extras = ["llm"], marker = "extra == 'full'" }, { name = "detectmatelibrary", extras = ["polars-rtcompat"], marker = "extra == 'full'" }, { name = "detectmateperformance", specifier = ">=0.1.0" }, + { name = "flax", specifier = ">=0.12.8" }, { name = "fsspec", specifier = ">=2024.1.0" }, + { name = "jax", specifier = ">=0.11.0" }, { name = "msgpack", specifier = ">=1.0.0" }, { name = "numpy", specifier = ">=2.3.2" }, { name = "openai", marker = "extra == 'llm'", specifier = ">=2.26.0" }, @@ -303,6 +328,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "etils" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/ce/6e067242fde898841922ac6fc82b0bb2fe35c38e995880bdffdfbe30182a/etils-1.14.0.tar.gz", hash = "sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4", size = 108127, upload-time = "2026-03-04T17:41:36.291Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/3d/589663aeeacd59bb2f3e8596bfd3e81cf0fb18d70bb433199041f469771b/etils-1.14.0-py3-none-any.whl", hash = "sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4", size = 172934, upload-time = "2026-03-04T17:41:35.01Z" }, +] + +[package.optional-dependencies] +epath = [ + { name = "fsspec" }, + { name = "typing-extensions" }, + { name = "zipp" }, +] +epy = [ + { name = "typing-extensions" }, +] + +[[package]] +name = "flax" +version = "0.12.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jax" }, + { name = "msgpack" }, + { name = "numpy" }, + { name = "optax" }, + { name = "orbax-checkpoint" }, + { name = "pyyaml" }, + { name = "rich" }, + { name = "tensorstore" }, + { name = "treescope" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/b3/d2effc3a26f61135a90c7b282473b8c7963d124856ce860743a12c99a0da/flax-0.12.8.tar.gz", hash = "sha256:1bb438775b967a3f1f42b9c3d4ddf1d2883cd6406589f9cecdc244634036bdc4", size = 15710344, upload-time = "2026-07-20T18:29:40.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/da/2bda90cbe3bab93a5ce37263d6a461ee12e851d8d32b1a86bc3e682d651a/flax-0.12.8-py3-none-any.whl", hash = "sha256:807931d369cbc716fc447801ab39cd6c3c2614d573387a2319a028675dd34fe5", size = 531332, upload-time = "2026-07-20T18:29:38.971Z" }, +] + [[package]] name = "fsspec" version = "2026.6.0" @@ -361,6 +426,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "humanize" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/ea/13a1ef3c12d12662905801495283530251918b70d62d368f1d2e0272c70d/humanize-4.16.0.tar.gz", hash = "sha256:7dc2244a2f84a4bfb1d36c37bac80cd78e35cdc5c119206d87b018e1445f3a3f", size = 89515, upload-time = "2026-06-30T16:17:29.859Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/aa/0b7365d30fed43e7a3449aba1fe20a0a7174d9cf13e282af4e69ac825441/humanize-4.16.0-py3-none-any.whl", hash = "sha256:353eb2f34c09d098b2880eee8bef21832eae6d174f48c5762fff7e5fcb74d01d", size = 137209, upload-time = "2026-06-30T16:17:28.36Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -379,6 +453,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jax" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jaxlib" }, + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "opt-einsum" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/20/90a137c178de8f2b9170bd4dfb934e261213fdaac67e5c5183b132be85e7/jax-0.11.0.tar.gz", hash = "sha256:a2feb7cfa48bb35d36b8ecec16a4ec24044dec01935f779b28b017213697b195", size = 2813185, upload-time = "2026-07-16T19:00:14.519Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/5a/26a1305f8978bc41f889a4172198c7d8976e8e394fc4d03597d9edfd6b7e/jax-0.11.0-py3-none-any.whl", hash = "sha256:b31ed66c4321d5c9e48b861f51a72ed5f7960c93514296202d2041cbb9ac45bf", size = 3255281, upload-time = "2026-07-16T18:58:25.02Z" }, +] + +[[package]] +name = "jaxlib" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, + { name = "scipy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/13/629fd9f50e15424358b7c53b6ec729e2d7163d64942817bee9adeee04ed1/jaxlib-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5bf0b3bfc2ef4778580047389dacb51ae000e50c6b3b6f98d10788927066c97", size = 62546563, upload-time = "2026-07-16T18:59:16.788Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1d/a85feebcfc6fa702bcaadd74f3c13283e1613f7598de5ca69bdbaf4c2671/jaxlib-0.11.0-cp312-cp312-manylinux_2_27_aarch64.whl", hash = "sha256:1b767c4d7ad4dfc6358313b63ae6d83c3571d3ce966de04d8aaf29eb44633786", size = 82155667, upload-time = "2026-07-16T18:59:20.477Z" }, + { url = "https://files.pythonhosted.org/packages/86/f0/89c34a4877628edea6585699e42f7d3ed4c1e50140ade9c6efce85a0ab84/jaxlib-0.11.0-cp312-cp312-manylinux_2_27_x86_64.whl", hash = "sha256:7ea9dc06d94f536deabcf304513143bc89f51173ddfd786d44f8ac303efdf643", size = 87263507, upload-time = "2026-07-16T18:59:24.258Z" }, + { url = "https://files.pythonhosted.org/packages/59/6b/e6caeaff6bd1537becbbcde8985fd65a71374e33f9e8c1aa3371bd16c349/jaxlib-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:166ca16586a8aeb23f9c5e322845f3789037ed294f0a1539404a50f7c5fb13cf", size = 67743601, upload-time = "2026-07-16T18:59:27.981Z" }, + { url = "https://files.pythonhosted.org/packages/81/3b/cc587eaa7d66aad1cddc77f5e10bca086a7d252891d359e64795521f5a2b/jaxlib-0.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:88b349aa95e452caa30a6723320e93ec7bd1ab249e9f0bf29000da1b5efae733", size = 62547263, upload-time = "2026-07-16T18:59:31.414Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/8c71ff16e019554a07dfcc206034b0182fba00bca1d7aaada7bcdb32b595/jaxlib-0.11.0-cp313-cp313-manylinux_2_27_aarch64.whl", hash = "sha256:112a01c584707a7d0e8634fd55dd7efffe812966e60c2d3ac84e8c24e4571336", size = 82152577, upload-time = "2026-07-16T18:59:35.392Z" }, + { url = "https://files.pythonhosted.org/packages/0f/85/82e456879df00e8bed7074028b0e1ddf2ce4e58dd83b699b9e9ef8306542/jaxlib-0.11.0-cp313-cp313-manylinux_2_27_x86_64.whl", hash = "sha256:33dd9405cab05ee4f3ecc3a91289323e9bd2f08d25990e5d45ffb8524f519506", size = 87263717, upload-time = "2026-07-16T18:59:39.194Z" }, + { url = "https://files.pythonhosted.org/packages/e9/87/96670370fd97cb79ad84d8becc8878e6b4aa52fd052cf5f3c063add9e7af/jaxlib-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:aced7b1ca4e338b5248821a397d2d53008894672be6fee762f391cfdd7272e1e", size = 67744784, upload-time = "2026-07-16T18:59:43.711Z" }, + { url = "https://files.pythonhosted.org/packages/50/6e/8fa567a498ce9e1966d035efd6a4be92e4a8b788b321549ec06fbb7ff25c/jaxlib-0.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7e6c0e2374943ba7191fe196e22067b789894c58cf6ddd24c9971ae642de58ee", size = 62546481, upload-time = "2026-07-16T18:59:48.212Z" }, + { url = "https://files.pythonhosted.org/packages/75/a3/159e82715919b8945107c35051ac4fa6e2578640c484e17ea6590914b1a5/jaxlib-0.11.0-cp314-cp314-manylinux_2_27_aarch64.whl", hash = "sha256:483429cc7fa7fd6a20cb50f666c8d2d499efb7e9ba3163811c01f051ad6057c4", size = 82171230, upload-time = "2026-07-16T18:59:51.677Z" }, + { url = "https://files.pythonhosted.org/packages/4e/e6/b6e4154b24d5a6bdcae715ad32030e7954bd793c590afa91b693d70e2719/jaxlib-0.11.0-cp314-cp314-manylinux_2_27_x86_64.whl", hash = "sha256:6125da8532610641b7d3ea7926217cdab52dccb294ef6615455e2b6ecb9e2ee6", size = 87271644, upload-time = "2026-07-16T18:59:55.428Z" }, + { url = "https://files.pythonhosted.org/packages/be/79/949b0e7860787c0996af475a57b8a060fbd6b9c91b009005d91d12b4aecd/jaxlib-0.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:19888d25c2bba4240ccffe39ee4e5490a544a0bbcdcc1c171e558d3efed57d84", size = 70245063, upload-time = "2026-07-16T18:59:59.016Z" }, + { url = "https://files.pythonhosted.org/packages/09/18/04d82793d804a6766ca73e4ed30ef928aea252db7c86f73b2faf11fe9e90/jaxlib-0.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5c38a393d37b4a70569224dc8273d8ae6ccea837e45410ed53ca03e15d6fad58", size = 62642149, upload-time = "2026-07-16T19:00:02.597Z" }, + { url = "https://files.pythonhosted.org/packages/91/2e/418dae82175154f6b5f53aec8909bda800dcbc9c65d94373de146b06e3b1/jaxlib-0.11.0-cp314-cp314t-manylinux_2_27_aarch64.whl", hash = "sha256:5ed7c54a6ef02eea19c0a02e3f603e4f9ad77248098bb9fc3feffca7f4814b83", size = 82261560, upload-time = "2026-07-16T19:00:06.22Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ff/2edc74b4de40ac4222ee48b3805a6ea5e40f34abe4cd10b65bc0fa0864db/jaxlib-0.11.0-cp314-cp314t-manylinux_2_27_x86_64.whl", hash = "sha256:005ae3a663193d3a89eafadc073880c67d6b829e0ad989690275d02b5adc810a", size = 87367515, upload-time = "2026-07-16T19:00:10.661Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -543,6 +660,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -606,6 +735,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "mergedeep" version = "1.3.4" @@ -670,6 +808,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, ] +[[package]] +name = "ml-dtypes" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/b8/3c70881695e056f8a32f8b941126cf78775d9a4d7feba8abcb52cb7b04f2/ml_dtypes-0.5.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a174837a64f5b16cab6f368171a1a03a27936b31699d167684073ff1c4237dac", size = 676927, upload-time = "2025-11-17T22:31:48.182Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f0/0cfadd537c5470378b1b32bd859cf2824972174b51b873c9d95cfd7475a5/ml_dtypes-0.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:c1a953995cccb9e25a4ae19e34316671e4e2edaebe4cf538229b1fc7109087b7", size = 212222, upload-time = "2025-11-17T22:31:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/16/2e/9acc86985bfad8f2c2d30291b27cd2bb4c74cea08695bd540906ed744249/ml_dtypes-0.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:9bad06436568442575beb2d03389aa7456c690a5b05892c471215bfd8cf39460", size = 160793, upload-time = "2025-11-17T22:31:55.358Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a1/4008f14bbc616cfb1ac5b39ea485f9c63031c4634ab3f4cf72e7541f816a/ml_dtypes-0.5.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8c760d85a2f82e2bed75867079188c9d18dae2ee77c25a54d60e9cc79be1bc48", size = 676888, upload-time = "2025-11-17T22:31:56.907Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b7/dff378afc2b0d5a7d6cd9d3209b60474d9819d1189d347521e1688a60a53/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b", size = 5036993, upload-time = "2025-11-17T22:31:58.497Z" }, + { url = "https://files.pythonhosted.org/packages/eb/33/40cd74219417e78b97c47802037cf2d87b91973e18bb968a7da48a96ea44/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d", size = 5010956, upload-time = "2025-11-17T22:31:59.931Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8b/200088c6859d8221454825959df35b5244fa9bdf263fd0249ac5fb75e281/ml_dtypes-0.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328", size = 212224, upload-time = "2025-11-17T22:32:01.349Z" }, + { url = "https://files.pythonhosted.org/packages/8f/75/dfc3775cb36367816e678f69a7843f6f03bd4e2bcd79941e01ea960a068e/ml_dtypes-0.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:35f29491a3e478407f7047b8a4834e4640a77d2737e0b294d049746507af5175", size = 160798, upload-time = "2025-11-17T22:32:02.864Z" }, + { url = "https://files.pythonhosted.org/packages/4f/74/e9ddb35fd1dd43b1106c20ced3f53c2e8e7fc7598c15638e9f80677f81d4/ml_dtypes-0.5.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:304ad47faa395415b9ccbcc06a0350800bc50eda70f0e45326796e27c62f18b6", size = 702083, upload-time = "2025-11-17T22:32:04.08Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/667060b0aed1aa63166b22897fdf16dca9eb704e6b4bbf86848d5a181aa7/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d", size = 5354111, upload-time = "2025-11-17T22:32:05.546Z" }, + { url = "https://files.pythonhosted.org/packages/40/49/0f8c498a28c0efa5f5c95a9e374c83ec1385ca41d0e85e7cf40e5d519a21/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298", size = 5366453, upload-time = "2025-11-17T22:32:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/8c/27/12607423d0a9c6bbbcc780ad19f1f6baa2b68b18ce4bddcdc122c4c68dc9/ml_dtypes-0.5.4-cp313-cp313t-win_amd64.whl", hash = "sha256:cb73dccfc991691c444acc8c0012bee8f2470da826a92e3a20bb333b1a7894e6", size = 225612, upload-time = "2025-11-17T22:32:08.615Z" }, + { url = "https://files.pythonhosted.org/packages/e5/80/5a5929e92c72936d5b19872c5fb8fc09327c1da67b3b68c6a13139e77e20/ml_dtypes-0.5.4-cp313-cp313t-win_arm64.whl", hash = "sha256:3bbbe120b915090d9dd1375e4684dd17a20a2491ef25d640a908281da85e73f1", size = 164145, upload-time = "2025-11-17T22:32:09.782Z" }, + { url = "https://files.pythonhosted.org/packages/72/4e/1339dc6e2557a344f5ba5590872e80346f76f6cb2ac3dd16e4666e88818c/ml_dtypes-0.5.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:2b857d3af6ac0d39db1de7c706e69c7f9791627209c3d6dedbfca8c7e5faec22", size = 673781, upload-time = "2025-11-17T22:32:11.364Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/067b84365c7e83bda15bba2b06c6ca250ce27b20630b1128c435fb7a09aa/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465", size = 5036145, upload-time = "2025-11-17T22:32:12.783Z" }, + { url = "https://files.pythonhosted.org/packages/c6/bb/82c7dcf38070b46172a517e2334e665c5bf374a262f99a283ea454bece7c/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f", size = 5010230, upload-time = "2025-11-17T22:32:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e9/93/2bfed22d2498c468f6bcd0d9f56b033eaa19f33320389314c19ef6766413/ml_dtypes-0.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:8c6a2dcebd6f3903e05d51960a8058d6e131fe69f952a5397e5dbabc841b6d56", size = 221032, upload-time = "2025-11-17T22:32:15.763Z" }, + { url = "https://files.pythonhosted.org/packages/76/a3/9c912fe6ea747bb10fe2f8f54d027eb265db05dfb0c6335e3e063e74e6e8/ml_dtypes-0.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:5a0f68ca8fd8d16583dfa7793973feb86f2fbb56ce3966daf9c9f748f52a2049", size = 163353, upload-time = "2025-11-17T22:32:16.932Z" }, + { url = "https://files.pythonhosted.org/packages/cd/02/48aa7d84cc30ab4ee37624a2fd98c56c02326785750cd212bc0826c2f15b/ml_dtypes-0.5.4-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:bfc534409c5d4b0bf945af29e5d0ab075eae9eecbb549ff8a29280db822f34f9", size = 702085, upload-time = "2025-11-17T22:32:18.175Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e7/85cb99fe80a7a5513253ec7faa88a65306be071163485e9a626fce1b6e84/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7", size = 5355358, upload-time = "2025-11-17T22:32:19.7Z" }, + { url = "https://files.pythonhosted.org/packages/79/2b/a826ba18d2179a56e144aef69e57fb2ab7c464ef0b2111940ee8a3a223a2/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf", size = 5366332, upload-time = "2025-11-17T22:32:21.193Z" }, + { url = "https://files.pythonhosted.org/packages/84/44/f4d18446eacb20ea11e82f133ea8f86e2bf2891785b67d9da8d0ab0ef525/ml_dtypes-0.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4381fe2f2452a2d7589689693d3162e876b3ddb0a832cde7a414f8e1adf7eab1", size = 236612, upload-time = "2025-11-17T22:32:22.579Z" }, + { url = "https://files.pythonhosted.org/packages/ad/3f/3d42e9a78fe5edf792a83c074b13b9b770092a4fbf3462872f4303135f09/ml_dtypes-0.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d", size = 168825, upload-time = "2025-11-17T22:32:23.766Z" }, +] + [[package]] name = "msgpack" version = "1.2.1" @@ -731,6 +905,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/4e/afc8c31605cb8be1d3bb4438c4d979daa104dab6306cd2b87abe9c3a7299/narwhals-2.23.0-py3-none-any.whl", hash = "sha256:769e7b9ab102c93d8fa019f6b4cd1a657909b04a20bf6210e5a35aae06814ae9", size = 458938, upload-time = "2026-07-01T11:21:51.677Z" }, ] +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418, upload-time = "2024-01-21T14:25:19.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195, upload-time = "2024-01-21T14:25:17.223Z" }, +] + [[package]] name = "numpy" version = "2.5.1" @@ -801,6 +984,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/b0/2291689e3ec4723fbf5bbf3b54afcd7b160f9ddc98ca7aedfd0132af5677/openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777", size = 1629470, upload-time = "2026-07-09T18:02:42.21Z" }, ] +[[package]] +name = "opt-einsum" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" }, +] + +[[package]] +name = "optax" +version = "0.2.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "jax" }, + { name = "jaxlib" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/f9/e3d11ae6f298ee941a0690e353a323d158ba5dedc436e75621c310845c5c/optax-0.2.8.tar.gz", hash = "sha256:5b225b35066fc3eebaa4d798f1b4173b4d57d1a480610908981f8343b50af0b0", size = 301193, upload-time = "2026-03-20T23:30:05.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/69/6a93d8600c339d7687a05857c7907bd4dd8cf88691a5ea106d7a50af90a1/optax-0.2.8-py3-none-any.whl", hash = "sha256:e3ca2d36c99daab1800ae9dbc0545034382d6bc780b24d969e1b0df65fa31cb4", size = 402960, upload-time = "2026-03-20T23:30:03.886Z" }, +] + +[[package]] +name = "orbax-checkpoint" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "aiofiles" }, + { name = "etils", extra = ["epath", "epy"] }, + { name = "humanize" }, + { name = "jax" }, + { name = "msgpack" }, + { name = "nest-asyncio", marker = "sys_platform == 'win32'" }, + { name = "numpy" }, + { name = "prometheus-client" }, + { name = "protobuf" }, + { name = "psutil" }, + { name = "pyyaml" }, + { name = "simplejson" }, + { name = "tensorstore" }, + { name = "typing-extensions" }, + { name = "uvloop", marker = "sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/70/e8b4c7f382c77d198128c50076611b9b3b7b18e795f6d7d5931755af0106/orbax_checkpoint-0.12.1.tar.gz", hash = "sha256:e9b014ccad4c9b2648cdb9946cd2550339bc3eb388ac1f494a27fda4739b4e19", size = 703150, upload-time = "2026-06-24T19:13:15.993Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/87/0ae1baf8271a7a68bcc5b91103892b201cd723ac3dd67bc6ef3e5fc4747e/orbax_checkpoint-0.12.1-py3-none-any.whl", hash = "sha256:e123b4d6e5d9f9144bb9b5a1d88b1eddc81c71f8a12b07b995f61e0d3022b0f1", size = 1311120, upload-time = "2026-06-24T19:13:14.262Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -962,6 +1196,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/4a/57d04de49f591088901794cc22f36563a102681e3512ae17ee6085cd2f30/prek-0.4.9-py3-none-win_arm64.whl", hash = "sha256:7eab3900d9ea614c8ea0d0d55a8b708f0c88e43c966dc8b13a4e36c1e398dd16", size = 5540477, upload-time = "2026-07-11T11:04:02.815Z" }, ] +[[package]] +name = "prometheus-client" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/73/f1334c29c2af4cd9dba6c7817e61b611bd0215e2eb5565c6064a4de18802/prometheus_client-0.26.0.tar.gz", hash = "sha256:04a91bcf94e2cf74a44a1a874d651a2e853ed354b6e822f3b7487751465d5c2b", size = 92910, upload-time = "2026-07-24T19:36:41.893Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/a3/b69efbf4143b5b9859b977770bbbabcc2796b702fa69dc40271e45cd5a56/prometheus_client-0.26.0-py3-none-any.whl", hash = "sha256:fa93d06737aa02bacd05794768508bb97d2fbee28cb3bca04eaae92f0ca953d6", size = 64494, upload-time = "2026-07-24T19:36:40.854Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -977,6 +1220,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "pyarrow" version = "25.0.0" @@ -1396,6 +1667,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "scikit-learn" version = "1.9.0" @@ -1495,6 +1779,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] +[[package]] +name = "simplejson" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/2a/54837395a3487c725669428d513293612a48d82b95a0642c936932e5d898/simplejson-4.1.1.tar.gz", hash = "sha256:c08eb9f7a90f77ae470e19a07472e9a79ebc0d1c2315d86a72767665bd5ba79f", size = 118860, upload-time = "2026-04-24T19:24:59.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/25/e90998fe8e480eb43b966c09e835379887d427567ebd496563d3b1e16b19/simplejson-4.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:19040a17154dc03d289bab68d73ce0a6a0be01de30c584bbdd93490bead14b22", size = 112414, upload-time = "2026-04-24T19:23:06.084Z" }, + { url = "https://files.pythonhosted.org/packages/9c/a0/abd4785f36c3400f1fbb21f517be39295a750a714f04b7ee175adf6ef580/simplejson-4.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a94ebaecdbaa80d9551a3ec6bf0c9302fc8b53ab6c1b2bfd498a1df4cb28158d", size = 91120, upload-time = "2026-04-24T19:23:07.877Z" }, + { url = "https://files.pythonhosted.org/packages/b8/78/fc060d2e3b13c6ec59288574b8efac64075e316b2afba4396a56b2422f78/simplejson-4.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:67341c95c0a168ab4a6d1e807e50463f1c8da932c3286d81e201266c427061fa", size = 91055, upload-time = "2026-04-24T19:23:09.264Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b6/156a8de1e1b47694f0e7de6675866936608d45dc68388fd017d36f8693be/simplejson-4.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45ec18e337fec538b7e902d489505c450b2454653d1290f3f50385e6fd8aa607", size = 190297, upload-time = "2026-04-24T19:23:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/86/1c/e4d0eab695be3eb21d0f46bce820752031f03e7113f9c80a9b3c73ee7157/simplejson-4.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:820c69a4710400e9b248d5670647d60be58824369282d3925e516b3ff1a7cd82", size = 187002, upload-time = "2026-04-24T19:23:12.982Z" }, + { url = "https://files.pythonhosted.org/packages/76/0e/7f5a59d29426b062d5928fb88b403c3f797129d53be7102f955dbe51aa44/simplejson-4.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e708d373a10e4378ef2d59f8361850c7150fd907ed49efe49bc5492160476d1", size = 195146, upload-time = "2026-04-24T19:23:14.517Z" }, + { url = "https://files.pythonhosted.org/packages/78/18/9943db224dd4d5fa3c090c3e56a94c37b254338c83995ec5680285111c40/simplejson-4.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:980fc33353f81fd12d8c49d44f8c2760d1dc8192285e627c5180d141035b228a", size = 183931, upload-time = "2026-04-24T19:23:16.742Z" }, + { url = "https://files.pythonhosted.org/packages/c2/08/9a690da9a766161c06c627d805362cf159f1abe480969372b2897649b955/simplejson-4.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:de2ed102fff88dacf543699f53ee3a533cc11539a39baa176b7e09dd783069d6", size = 192228, upload-time = "2026-04-24T19:23:18.33Z" }, + { url = "https://files.pythonhosted.org/packages/05/88/bd8aad36b451ffb0e0a3f721d695a88befa6d1ac7d1e02ae788ca7ff4029/simplejson-4.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2785ff8edc0e28bf773a32543a6bbed46351453c997b3f6709c744e3c2f7eabb", size = 187808, upload-time = "2026-04-24T19:23:21.165Z" }, + { url = "https://files.pythonhosted.org/packages/04/ee/14f91db0d1f481533b651dafbf8cd0da088d9817f7af30c68f7f19f9c847/simplejson-4.1.1-cp312-cp312-win32.whl", hash = "sha256:2e0d5ead6d14610467ec356ec1f6b5d8a56aa216abaad8d41c8b873b16cf313f", size = 88512, upload-time = "2026-04-24T19:23:22.764Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c4/90de06b2d8737c68c05ff9274113f854dbf6a5f28b7a955212111672cb57/simplejson-4.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:63a5451f557d6be48a231bae932458655c620902b868170b2f1c8afed496f6b4", size = 90748, upload-time = "2026-04-24T19:23:24.494Z" }, + { url = "https://files.pythonhosted.org/packages/37/a9/47b445eeb559c9593453a0648e0fd6d08e8adff64dd5e5ced66726da8a09/simplejson-4.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dff52fc7af272e84fc21cc5a06c927c823ca6ae00af14f3b0d7707b42775ed98", size = 113160, upload-time = "2026-04-24T19:23:26.033Z" }, + { url = "https://files.pythonhosted.org/packages/4c/65/cb72db31523c164dea5dc55b02dad065a40c478856bc7534b279d2b51906/simplejson-4.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:971aed0647ad6e840a3943bec812fcda5f2d26a5497a4981d1fb49aa4f9a396c", size = 91521, upload-time = "2026-04-24T19:23:27.572Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e5/54cb7c50ad5fdc1e0a86b7df4b135c2cbd5c4623605aa94466659098e8da/simplejson-4.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:249e2e220aa6d9b9d936bde84eb7bf79d5b6c5a8273c6e411f8b1635a9073f2d", size = 91407, upload-time = "2026-04-24T19:23:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/38/2e/21a3ede87f0bf82d6c7bcb90480d50a6490eb974c6ab20881188e440957c/simplejson-4.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e5cdd6a5d52299f345c15ab5678cc4249e24f383f361d986afbc3c7072a6b6b", size = 192451, upload-time = "2026-04-24T19:23:30.56Z" }, + { url = "https://files.pythonhosted.org/packages/59/df/9903edd3102bf0b5984edfcb90c88612330996efa3b4fbf8a971d6e17839/simplejson-4.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:642cec364e0676e2d5a73fa4d31d0c7c55886997caa2fde24e8292ca44d32728", size = 189015, upload-time = "2026-04-24T19:23:32.647Z" }, + { url = "https://files.pythonhosted.org/packages/98/cd/33230927a780e1398b857e3944abb914556994d252b1d765ae40d112cb25/simplejson-4.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:76fe296ca1df23d290033f10aaacf534fd1b3e3007e7f9ff8aa68b21413aaa78", size = 196658, upload-time = "2026-04-24T19:23:34.563Z" }, + { url = "https://files.pythonhosted.org/packages/cd/84/2c5a7444eb53e9a86d3738299bffddd9f53aeed799ded2f45368221fdb19/simplejson-4.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f0ad25b7dc4e0fb23858355819f2e994f1a5badcdcde8737eac7921c2f1ed2a", size = 185967, upload-time = "2026-04-24T19:23:36.191Z" }, + { url = "https://files.pythonhosted.org/packages/d3/68/454378e06d059cd412a7ed5d87fb6d29fd5b60f13a4d89fc1f764ff434df/simplejson-4.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a59ebd0533f03fd06ff0c42ba0f02d93cbcdd7944922bf3b93911327a95b901f", size = 193940, upload-time = "2026-04-24T19:23:38.151Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d5/a15bf915f623a2c5a079d6e3be8256fdb8ef06f110669493a09b9d6933e0/simplejson-4.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bccbf4419676b517939852e5aeff2af6aee4dc046881c67a1581fa6f1cb01abd", size = 189795, upload-time = "2026-04-24T19:23:40.139Z" }, + { url = "https://files.pythonhosted.org/packages/d2/c9/37212ae7dc4b607f0978c408e8633f05c810884e054c33113184c6c2c8a2/simplejson-4.1.1-cp313-cp313-win32.whl", hash = "sha256:6c845363eb5fd166fb7c72243da38f4fcfde666ede7fdf2cc6fd7762894626f7", size = 88773, upload-time = "2026-04-24T19:23:41.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a5/c7a0a47883a9015b54c9d8a4b62f2aba17bd4335b1787b9b8a0fc2fa6d52/simplejson-4.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:104d8324c34f25b4b90800bc5fa363780cbc3d8496aef061cba7ce1af9162270", size = 90888, upload-time = "2026-04-24T19:23:43.11Z" }, + { url = "https://files.pythonhosted.org/packages/d3/18/4a118a6a92eb33bb08c8e2fe7ec85cb96f0673491bb2b829930831ee4fbe/simplejson-4.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ed7473602b6625de793b6acba49aa949f144a475f538792067e4cf2fda2071f5", size = 110492, upload-time = "2026-04-24T19:23:44.957Z" }, + { url = "https://files.pythonhosted.org/packages/07/f4/84d160e9fa8cada1e0a9381cae4fa81eecd573577a5b34366d8ced59bdf7/simplejson-4.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:225c9caa324c5b554d009fb9cac22aee7711e71bd96f487938c659af467e828e", size = 90152, upload-time = "2026-04-24T19:23:46.355Z" }, + { url = "https://files.pythonhosted.org/packages/68/31/9a5432c433a7671107182cdc9a20ea78a70f99c4e5334aa54b6d4d0d79ed/simplejson-4.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:95407269340c7f22f09776ea7b717a52cf56cfcf119b5e45f66faa4a26445bea", size = 90115, upload-time = "2026-04-24T19:23:47.743Z" }, + { url = "https://files.pythonhosted.org/packages/78/91/3635cdb13318cb0a328abaa69e2b91251caad39d6779aa308098f341f6cb/simplejson-4.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3851658d642c1184d2023f0e6c9ce44a21eb1629e74e7c84ef956b128841fe12", size = 184036, upload-time = "2026-04-24T19:23:49.472Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/149b6ec5393f6849d98c59cadba888b710a8ef4b805ab91e11a566960d40/simplejson-4.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95a3bb0f78e85f4937f99092239f2011ce06f0f2d803df5c299cc05abbeae008", size = 180543, upload-time = "2026-04-24T19:23:51.023Z" }, + { url = "https://files.pythonhosted.org/packages/df/7c/a5d968d0b527a748b667e62bea94309ccbcb1e2b108e8f0cf8547efaa12b/simplejson-4.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbfdaa7c0603f75b7b14b211b7f2be44696d4e26833ad2d91d5c87bf5fb9a920", size = 188725, upload-time = "2026-04-24T19:23:52.995Z" }, + { url = "https://files.pythonhosted.org/packages/db/e3/6a8d11181d587ef00e2db9112357e6832111e56dd56b01b5c11758a1965d/simplejson-4.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e3c584071dced8c21b4689f0254303521daeb9b5bc1f4289755d71fa3cb0d3", size = 177492, upload-time = "2026-04-24T19:23:54.581Z" }, + { url = "https://files.pythonhosted.org/packages/67/e3/8b0eb8b06e8198cfbd1270487da163d0093df05cc4f557350cd65e2f7e79/simplejson-4.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:036a27bd0469b9d79557cbddb392969f876cd7f278cfbd0fba81534927a06575", size = 185281, upload-time = "2026-04-24T19:23:56.13Z" }, + { url = "https://files.pythonhosted.org/packages/dc/5f/64990f07ec9e2cb1a814c674e2e21b5693207f74ac70eb72151b847ea4e6/simplejson-4.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b70bfd2f67f3351baba08aa3ae9233c83f21fd95ae5e6b3d0ecb8c647929112f", size = 181848, upload-time = "2026-04-24T19:23:57.92Z" }, + { url = "https://files.pythonhosted.org/packages/61/a5/bbc1bc0447f339f79f99ab8c37f7f037cb2f1f93af75d6a4d553096bb0c3/simplejson-4.1.1-cp314-cp314-win32.whl", hash = "sha256:37233c72ce88d06acb92747347742b3c07871eba6789f060c179c9302dde8efe", size = 88761, upload-time = "2026-04-24T19:23:59.397Z" }, + { url = "https://files.pythonhosted.org/packages/18/72/ec1b5cbdcb140c132e6c7bdf99bd73e4f675439e77126c88f472fcffa09c/simplejson-4.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:cc0442dea71cd9cbf30a0b8b9929ab5aa6c02c0443a3d977351e6ec5bada4388", size = 91018, upload-time = "2026-04-24T19:24:00.85Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/4fa437f68ff72219bac3bf3d050de9c6265691f3a170e16954bd69d7cddd/simplejson-4.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:c996a4d38290c515af347740659ce095b425449c164a5c9fa3977caa6eff5dbe", size = 113919, upload-time = "2026-04-24T19:24:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/c2/83/59de041d09eb4a9577f7015d7263c32095dfb7fde49717dff62145d89809/simplejson-4.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c65c763fb20d7ca113c1c14dce2fc04a0fc3a57aceff533d6fdac707c7bffb40", size = 91904, upload-time = "2026-04-24T19:24:03.812Z" }, + { url = "https://files.pythonhosted.org/packages/03/8e/46bb345d540f6eb31427d984a4e518cdb182d0621814fee4fee045e8815b/simplejson-4.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0da5c9f57206ee7ef280ff7f1d924937b0a64f9a271a5ef371a2ecdbebba7421", size = 91752, upload-time = "2026-04-24T19:24:05.622Z" }, + { url = "https://files.pythonhosted.org/packages/83/e2/1b2ce97f068835eb3d253c116a4df7a3f436b7bf2fb5ff1ba29287e8b0ec/simplejson-4.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ea3426e786425d10e9e82f8a6eda74a7d6eb10d99165ac3d0d3bbcb65c0ea343", size = 214021, upload-time = "2026-04-24T19:24:07.447Z" }, + { url = "https://files.pythonhosted.org/packages/48/70/d93e556df6a0786298644a7c08304fcbeddc248325f23f38acbebeb21165/simplejson-4.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d75cea7a1025edd7e439b2966b3d977c45b5b899e2adaf422811b3ac702ed9fb", size = 213530, upload-time = "2026-04-24T19:24:09.289Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/c93bf305b9f00d7259e09e713d60e75bd0f7f53da970f716ab90491770e7/simplejson-4.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63c2ada8e58f266491f19eed2eeeb7c25c6141e52f8f9e820f6bb94156cf8dbc", size = 218282, upload-time = "2026-04-24T19:24:10.991Z" }, + { url = "https://files.pythonhosted.org/packages/0c/20/a9b5d2e27ec44b069ee251bd55544fc76929a067107b1050001566ba86f3/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d1fffb56305c5b475ee746cf9e04f97423ba5aaacd292dc1255bd75b1d3b124b", size = 209249, upload-time = "2026-04-24T19:24:12.662Z" }, + { url = "https://files.pythonhosted.org/packages/97/e4/e06ee682ed5df67592181f5ecb062e35878967e27f5b6e087237d4548d95/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a6525ec733f43d0541206cffa64fd2aad5a7ae3eb76566aff49cd4db6382209a", size = 213963, upload-time = "2026-04-24T19:24:14.302Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9f/1e160e4cd8cdbf062bf6a454cdf814dc7a48eb47e566fdb8f80ccb202605/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:861e393260508efa64d8805a8e49c416c3484907e3f146ce966c69552b49b9a3", size = 210474, upload-time = "2026-04-24T19:24:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e6/cecd913df322df5bbe7ebb8ba39e0708e505a165553900da8a7761026d6f/simplejson-4.1.1-cp314-cp314t-win32.whl", hash = "sha256:d083b89d30948a751d3d97476c2ed91e4caaa24a1a1459bdbadb8876242c71fe", size = 91134, upload-time = "2026-04-24T19:24:17.635Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/f540dde99cc1d393bd062ab3b5735b777561a5d8f8a5f2e241164444d77a/simplejson-4.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4cbb299d0528ec0447fe366d8c9641860e28f997a62730690fef905f1f41046e", size = 94467, upload-time = "2026-04-24T19:24:19.109Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6a/8b74c52ffd33dbbde00fe7251fee6a0acdc8cea33f7a43805aed258fb79b/simplejson-4.1.1-py3-none-any.whl", hash = "sha256:2ce92b3748f02423e26d2bfb636fb9d7a8f67c8f5854dcae69d350d123b2eee2", size = 69195, upload-time = "2026-04-24T19:24:57.962Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -1522,6 +1859,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "tensorstore" +version = "0.1.84" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ml-dtypes" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/18/c8e8b4faffab1a434b6c013d54cf7f5b754a6849429d9dbb718297705796/tensorstore-0.1.84.tar.gz", hash = "sha256:3cb091dfde68600e6d8f03a389ccc92ffa7c0798a0c600d1013c0138d7163e6b", size = 7208048, upload-time = "2026-05-16T06:17:58.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/8a/1b5231e965257c3ee7d4615cb49a0fac53a71a1c34b293bcf524bb7c6d13/tensorstore-0.1.84-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:915371fc2c27540e8b69c573b7a06217fb8d161ec231cedfa9f3d264615a326d", size = 16571584, upload-time = "2026-05-16T06:17:13.283Z" }, + { url = "https://files.pythonhosted.org/packages/88/5d/52e52aa00a5ae3ebe1116ca52ac9f47ef98e94f6c4e411649cd3d1bb79cc/tensorstore-0.1.84-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4477eabe26e2f5131f1b1a3444cd9167fe69fabc29579eab8259d218399b9e6b", size = 14905169, upload-time = "2026-05-16T06:17:15.638Z" }, + { url = "https://files.pythonhosted.org/packages/61/36/f88b4bf267902f12cd2ca33aff10fabd6839dd1ce7d51876ebefa98aaf2c/tensorstore-0.1.84-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ace00cf2e45dc5d64fe3a10c2cbef61343915683808a10a3e081233566a7231", size = 19345134, upload-time = "2026-05-16T06:17:17.984Z" }, + { url = "https://files.pythonhosted.org/packages/18/7c/b7b24e10e5cb0213c85204d53fcd60d0568d986ea0001a00a815e14e01e1/tensorstore-0.1.84-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64c8039558d5607b73903948fce058725731df410c5c196cf58b3fc6222395b5", size = 20968745, upload-time = "2026-05-16T06:17:20.569Z" }, + { url = "https://files.pythonhosted.org/packages/dd/36/33ad454a2b667a93b35e74595a351dbf9b8693440bd68665990663b79164/tensorstore-0.1.84-cp312-cp312-win_amd64.whl", hash = "sha256:08e7ec5b35db5d4c4b6a867be8500448f9bd4e0c9d5a52d7f0b460650622baf6", size = 13398458, upload-time = "2026-05-16T06:17:22.701Z" }, + { url = "https://files.pythonhosted.org/packages/e7/27/3c637c0f987866f6fb92cf96ee4d40eee4b5ab699135803ada851f2a56ad/tensorstore-0.1.84-cp313-cp313-macosx_10_14_x86_64.whl", hash = "sha256:9337d96693b4a0e555fbe63bb228e3e2e681a80e4d371f351fb67810f197f74e", size = 16571472, upload-time = "2026-05-16T06:17:24.885Z" }, + { url = "https://files.pythonhosted.org/packages/41/83/4f3c6ef9bed01f384036c2030b3901cf075bbc8eff6e4529e502f0283ab5/tensorstore-0.1.84-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:028455cccdc05c31f194048cf459a26669b26d38f0516caf9213e7219b1ee79a", size = 14905288, upload-time = "2026-05-16T06:17:26.753Z" }, + { url = "https://files.pythonhosted.org/packages/4d/28/03e46405ba7c616e7a1ec5425a8f4a1b3f4d6ec2be359cb2f248199849e4/tensorstore-0.1.84-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50afb06c57a509091015af6a85da6f483a7f5ad0372284dd95d5513d877336e4", size = 19344890, upload-time = "2026-05-16T06:17:28.956Z" }, + { url = "https://files.pythonhosted.org/packages/5e/c6/82669e70cef67c803852285ba6f59d7e3d102983c0ab4be8269c14756677/tensorstore-0.1.84-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ea53a851ea86aad3d99c14a790c85468d6324be14c7ac211f1f0265e8fab707", size = 20968230, upload-time = "2026-05-16T06:17:31.374Z" }, + { url = "https://files.pythonhosted.org/packages/a5/12/97d8ad183e3130e168f2feb860edd68f1b72e57f29268d980f3b70e34cd0/tensorstore-0.1.84-cp313-cp313-win_amd64.whl", hash = "sha256:fe9bf1c7fef69884a91222179550f9b5ba6c1454f9534429221824d9b15c00ec", size = 13398858, upload-time = "2026-05-16T06:17:33.658Z" }, + { url = "https://files.pythonhosted.org/packages/39/bd/fda828e7915ddb61704a6f4568b5b7ce9fe607c33b7535cf51b4fd900b38/tensorstore-0.1.84-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:97e767a64415297f019dd1e4b9af6a23d72e126e2a2d05cf41083253abe81428", size = 16576451, upload-time = "2026-05-16T06:17:35.993Z" }, + { url = "https://files.pythonhosted.org/packages/8a/99/2e72fcb19404de43f9412880c542a8ef8651bd30183c85454d6ca14ebe56/tensorstore-0.1.84-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7d01775985fcaa2b0f100349766a953c5086e92e746bf395e936151d4d8f9ac", size = 14907896, upload-time = "2026-05-16T06:17:38.458Z" }, + { url = "https://files.pythonhosted.org/packages/25/d4/9a3964cdfdc5a15df6d0485694de9684c13990a5b0f5d88bfa365e0a2936/tensorstore-0.1.84-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b2ff5d5536a8a9b1596c51b9075cc9d40b4c4ea4e6cc03c0480111dbe5d956d", size = 19347021, upload-time = "2026-05-16T06:17:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/43/19/70532cb2bf2f6fc3bf252f850bfb528b26eeb9c30c3cafffb075cbb7c77a/tensorstore-0.1.84-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c9108ae6c29adc90b72ca267ba2b577386c5e410ea2f8e87eabce5ebdad327e", size = 20970345, upload-time = "2026-05-16T06:17:43.874Z" }, + { url = "https://files.pythonhosted.org/packages/4e/22/a523e7576c83a6c35bd1415e5f4530b0f1e448c099d7e22684f55792755c/tensorstore-0.1.84-cp314-cp314-win_amd64.whl", hash = "sha256:4096220f4b9a2411c3751597dd8ced2f671d7a217575613c915191e19d5ea150", size = 13787429, upload-time = "2026-05-16T06:17:46.135Z" }, + { url = "https://files.pythonhosted.org/packages/2b/23/40b0a0a91973431770ac09a8b69039bd7a051169f6707b63a1c51ad36782/tensorstore-0.1.84-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:70284a7225d94adad5989de7f9238638ec9fec2bdd8c0fdb86d567f16d59e615", size = 16652804, upload-time = "2026-05-16T06:17:49.226Z" }, + { url = "https://files.pythonhosted.org/packages/01/99/2ea3b37864adb2e9f6e724c6959ab2b7f56aa4dad01964d4b32adf211e68/tensorstore-0.1.84-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:98e10cae5b3fc0828967b8ddf36242b3b26ac8bc79880bc3e36346063259212a", size = 14990904, upload-time = "2026-05-16T06:17:51.464Z" }, + { url = "https://files.pythonhosted.org/packages/ff/26/fe72887d7b91e832bef3033d244c4c548993d1b6fb19177dff3895659c12/tensorstore-0.1.84-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98edbc57d453ba8ebcf0d19bc28e4de6de95922bee2eca0805955b0833b8ce26", size = 19365698, upload-time = "2026-05-16T06:17:53.748Z" }, + { url = "https://files.pythonhosted.org/packages/37/74/35a1d41343f86f6e2ef135e81f6b8107b9f16c777a3e8be9e3fbce541d18/tensorstore-0.1.84-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bd20de85c1b83dd3ca94db24e7bd449bdb055590a1b162b05691f6b81fff00f", size = 20986107, upload-time = "2026-05-16T06:17:56.571Z" }, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -1590,6 +1958,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2", size = 676612, upload-time = "2026-07-07T09:58:16.256Z" }, ] +[[package]] +name = "treescope" +version = "0.1.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/2a/d13d3c38862632742d2fe2f7ae307c431db06538fd05ca03020d207b5dcc/treescope-0.1.10.tar.gz", hash = "sha256:20f74656f34ab2d8716715013e8163a0da79bdc2554c16d5023172c50d27ea95", size = 138870, upload-time = "2025-08-08T05:43:48.048Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/2b/36e984399089c026a6499ac8f7401d38487cf0183839a4aa78140d373771/treescope-0.1.10-py3-none-any.whl", hash = "sha256:dde52f5314f4c29d22157a6fe4d3bd103f9cae02791c9e672eefa32c9aa1da51", size = 182255, upload-time = "2025-08-08T05:43:46.673Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" @@ -1629,6 +2009,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, + { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, + { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, + { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, + { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, + { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, + { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, +] + [[package]] name = "verspec" version = "0.1.0" @@ -1661,3 +2073,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, +] From d68081e6e701f56451a3687013700edaac3b15ae Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 09:40:56 +0200 Subject: [PATCH 03/26] add deeplog --- pyproject.toml | 1 + .../utils/deep_learning/_op.py | 46 ++++++++ .../utils/deep_learning/deeplog.py | 102 ++++++++++++++++++ uv.lock | 2 + 4 files changed, 151 insertions(+) create mode 100644 src/detectmatelibrary/utils/deep_learning/_op.py create mode 100644 src/detectmatelibrary/utils/deep_learning/deeplog.py diff --git a/pyproject.toml b/pyproject.toml index c3f92210..35339252 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "pyarrow>=24.0.0", "jax>=0.11.0", "flax>=0.12.8", + "optax>=0.2.8", ] [dependency-groups] diff --git a/src/detectmatelibrary/utils/deep_learning/_op.py b/src/detectmatelibrary/utils/deep_learning/_op.py new file mode 100644 index 00000000..dbdfec92 --- /dev/null +++ b/src/detectmatelibrary/utils/deep_learning/_op.py @@ -0,0 +1,46 @@ +import jax +import jax.numpy as jnp + +from typing import Any + + +class CheckPoint: + def __init__(self, patience: int) -> None: + self.last_loss = jnp.inf + self.epoch = -1 + self.param: None | dict[str, Any] = None + self.patience, self.i = patience, 0 + + def __call__(self, loss: float, epoch: int, param: dict[str, Any]) -> bool: + if self.last_loss > loss or self.param is None: + self.last_loss = loss + self.epoch = epoch + self.param = param + self.i = 0 + else: + self.i += 1 + + return self.i >= self.patience + + def load_checkpoint(self) -> tuple[int, dict[str, Any] | None]: + return self.epoch, self.param + + +class Mask: + def __init__(self, seq_size: int, mask_per: float) -> None: + self.i = 0 + self.seq_size = seq_size + self.mask_per = mask_per + self.s0 = int(seq_size * mask_per) + self.s1 = self.seq_size - self.s0 + + def __call__(self, batch_size: int) -> jnp.ndarray: + mask = jnp.concat([ + jnp.ones((batch_size, self.s1)), jnp.zeros((batch_size, self.s0)) + ], axis=1).astype(jnp.int32) + + seed = jax.random.key(self.i) + self.i += 1 + + mask = jax.random.permutation(seed, mask, independent=True, axis=1) + return mask diff --git a/src/detectmatelibrary/utils/deep_learning/deeplog.py b/src/detectmatelibrary/utils/deep_learning/deeplog.py new file mode 100644 index 00000000..27f6653a --- /dev/null +++ b/src/detectmatelibrary/utils/deep_learning/deeplog.py @@ -0,0 +1,102 @@ +import jax.numpy as jnp +import jax + +import flax.linen as nn +import optax + +from dataclasses import dataclass +from typing import Any +from tqdm import tqdm +from math import ceil + +from detectmatelibrary.utils.deep_learning._op import CheckPoint + + +## Model Deeplog +class DeepLogModel(nn.Module): + hidden_dim: int + n_layers: int + output_size: int = 1 + + @nn.compact + def __call__(self, x: jnp.ndarray) -> jnp.ndarray: + for _ in range(self.n_layers): + lstm_cell = nn.OptimizedLSTMCell(features=self.hidden_dim) + x = nn.RNN(lstm_cell)(x) + + last_step = x[:, -1, :] + return nn.Dense(features=self.output_size)(last_step) + + +## Train script +@dataclass +class TrainConfig: + seed: int = 0 + epochs: int = 40 + learning_rate: float = 0.05 + batch_size: int = 2 + patience: int = 3 + + +def loss_f(model: nn.Module, params: dict[str, Any], x: jnp.ndarray, y: jnp.ndarray) -> jnp.ndarray: + return optax.softmax_cross_entropy_with_integer_labels( + logits=model.apply({'params': params}, x), labels=y + ).mean() + + +def train( + model: nn.Module, + x: jnp.ndarray, + y: jnp.ndarray, + x_val: jnp.ndarray, + y_val: jnp.ndarray, + trainConfig: TrainConfig = TrainConfig() +): + @jax.jit + def train_step( + params: dict[str, Any], opt_state: optax.OptState, x: jnp.ndarray, y: jnp.ndarray + ) -> jnp.ndarray: + def loss_fn(params: dict[str, Any]) -> jnp.ndarray: + return loss_f(params=params, x=x, y=y, model=model) + + loss, grads = jax.value_and_grad(loss_fn)(params) + updates, opt_state = optimizer.update(grads, opt_state) + params = optax.apply_updates(params, updates) + return params, opt_state, loss + + key = jax.random.PRNGKey(trainConfig.seed) + n_steps = ceil(x.shape[0] / trainConfig.batch_size) + + variables = model.init(key, x[:1]) + params = variables['params'] + optimizer = optax.adam(learning_rate=trainConfig.learning_rate) + opt_state = optimizer.init(params) + + idx = jnp.arange(x.shape[0]) + idx = jax.random.permutation(jax.random.key(trainConfig.seed), idx) + + losses_epoch, losses_step, loss_val = [], [], [] + checkpoint = CheckPoint(trainConfig.patience) + for epoch in tqdm(range(trainConfig.epochs), desc="training..."): + step_loss = 0 + for step_idx in jnp.array_split(idx, n_steps): + params, opt_state, loss = train_step( + params, opt_state, x[step_idx], y[step_idx] + ) + step_loss += loss + losses_step.append(loss) + losses_epoch.append(step_loss / n_steps) + loss_val.append(loss_f(model=model, params=params, x=x_val, y=y_val)) + idx = jax.random.permutation(jax.random.key(epoch), idx) + if checkpoint(loss=loss_val[-1], epoch=epoch, param=params): + print("Early stop") + break + + best_e, params = checkpoint.load_checkpoint() + print(f"Best epoch {best_e} -> Train {losses_epoch[best_e]} Val {loss_val[best_e]}") + return params, { + "Loss Epoch": losses_epoch, + "Loss Step": losses_step, + "Loss Val": loss_val, + "Best val": loss_val[best_e] + } \ No newline at end of file diff --git a/uv.lock b/uv.lock index c655e443..857d110b 100644 --- a/uv.lock +++ b/uv.lock @@ -223,6 +223,7 @@ dependencies = [ { name = "jax" }, { name = "msgpack" }, { name = "numpy" }, + { name = "optax" }, { name = "protobuf" }, { name = "pyarrow" }, { name = "pydantic" }, @@ -277,6 +278,7 @@ requires-dist = [ { name = "msgpack", specifier = ">=1.0.0" }, { name = "numpy", specifier = ">=2.3.2" }, { name = "openai", marker = "extra == 'llm'", specifier = ">=2.26.0" }, + { name = "optax", specifier = ">=0.2.8" }, { name = "pandas", marker = "extra == 'dataframes'", specifier = ">=2.3.2" }, { name = "pandas", marker = "extra == 'llm'", specifier = ">=2.3.2" }, { name = "polars", marker = "extra == 'dataframes'", specifier = ">=1.40.1" }, From 4baaf357dfe8ff1783086df50b8186cebf6e6583 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 10:08:16 +0200 Subject: [PATCH 04/26] add deeplog main class --- .../utils/deep_learning/deeplog.py | 113 +++++++++++++++++- 1 file changed, 111 insertions(+), 2 deletions(-) diff --git a/src/detectmatelibrary/utils/deep_learning/deeplog.py b/src/detectmatelibrary/utils/deep_learning/deeplog.py index 27f6653a..bdaed56b 100644 --- a/src/detectmatelibrary/utils/deep_learning/deeplog.py +++ b/src/detectmatelibrary/utils/deep_learning/deeplog.py @@ -4,12 +4,15 @@ import flax.linen as nn import optax +from functools import lru_cache + from dataclasses import dataclass from typing import Any from tqdm import tqdm from math import ceil from detectmatelibrary.utils.deep_learning._op import CheckPoint +from detectmatelibrary.utils.finetune import Combinations ## Model Deeplog @@ -51,7 +54,7 @@ def train( x_val: jnp.ndarray, y_val: jnp.ndarray, trainConfig: TrainConfig = TrainConfig() -): +) -> tuple[dict[str, Any], dict[str, float]]: @jax.jit def train_step( params: dict[str, Any], opt_state: optax.OptState, x: jnp.ndarray, y: jnp.ndarray @@ -99,4 +102,110 @@ def loss_fn(params: dict[str, Any]) -> jnp.ndarray: "Loss Step": losses_step, "Loss Val": loss_val, "Best val": loss_val[best_e] - } \ No newline at end of file + } + + +def do_train( + model: nn.Module, train_seqs: jnp.ndarray, val_seqs: jnp.ndarray, config: TrainConfig +) -> tuple[dict[str, Any], dict[str, float]]: + return train( + model=model, + x=train_seqs[:, :-1, :], + y=train_seqs[:, -1, :].reshape((train_seqs.shape[0])), + x_val=val_seqs[:, :-1, :], + y_val=val_seqs[:, -1, :].reshape((val_seqs.shape[0])), + trainConfig=config + ) + + +## Final model +default_config = { + "Model": { + "hidden_dim": 64, + "n_layers": 2, + }, + "Train": { + "batch_size": 2048, + "learning_rate": 0.01, + "epochs": 10, + }, +} + + +class DeepLog: + def __init__(self, config: dict = default_config) -> None: + self.config = config + self.params = {} + self.config_train = TrainConfig(**config["Train"]) + self.model_trained = False + self.model: DeepLogModel | None = None + + def __str__(self) -> str: + return str(self.model) + "\n" + str(self.config_train) + + def top_pred(self, x: jnp.ndarray) -> jnp.ndarray: + return jnp.argsort( + self.model.apply({"params": self.params}, x[None, ..., None]).flatten(), + descending=True + ) + + def get_best_k(self, seq: jnp.ndarray) -> int: + if seq.shape[0] == 0: + return 0 + + x_s, y_s = seq[:, :-1, :], seq[:, -1] + x_s = jnp.argsort( + self.model.apply({"params": self.params}, x_s), descending=True + ) + + return int(jax.scipy.stats.mode( + (jnp.arange(x_s.shape[1]) * (x_s == y_s)).sum(1) + ).mode + 2) # give a little space for variation + + @lru_cache + def check_anomaly(self, seq: tuple[int], top_k: int) -> bool: + if not self.model_trained: + return False + + seq = jnp.array(seq) + x, y = seq[:-1], seq[-1] + return not jnp.isin(y, self.top_pred(x)[:top_k]) + + def _prepare_data(self, seqs: list[tuple[int]], var_per: float) -> tuple[jnp.ndarray]: + seed = jax.random.key(self.config_train.seed) + idx = jax.random.permutation(seed, len(seqs)) + seqs = jnp.array(seqs)[..., None][idx] + train_seqs = seqs[:ceil(len(seqs) * (1 - var_per))] + val_seqs = seqs[ceil(len(seqs) * (1 - var_per)):] + return train_seqs, val_seqs + + def train(self, seqs: list[tuple[int]], var_per: float) -> dict: + train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) + + self.config_train = TrainConfig(**self.config["Train"]) + self.config["Model"]["output_size"] = train_seqs.max() + 1 + print("Output shape:", self.config["Model"]["output_size"]) + + self.model = DeepLogModel(**self.config["Model"]) + self.params, stats = do_train( + self.model, train_seqs=train_seqs, val_seqs=val_seqs, config=self.config_train + ) + self.model_trained = True + stats["top_k"] = self.get_best_k(val_seqs) + + return stats + + def finetune(self, seqs: list[tuple[int]], var_per: float, epochs: int = 2) -> None: + train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) + combos = Combinations(config=self.config) + + for comb in combos(): + comb["Model"]["output_size"] = train_seqs.max() + 1 + comb["Train"]["epochs"] = epochs + model = DeepLogModel(**comb["Model"]) + config_train = TrainConfig(**comb["Train"]) + + _, stats = do_train(model, train_seqs=train_seqs, val_seqs=val_seqs, config=config_train) + combos.add_value(stats["Best val"]) + self.config = combos.get_best() + print(self.config) \ No newline at end of file From 2c0ee34ca81d3500b0124f9a3be70c09831b063e Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 13:29:04 +0200 Subject: [PATCH 05/26] deeplog ready in deep_learning utils --- src/detectmatelibrary/utils/finetune.py | 57 ++++++++++++--- tests/test_utils/test_deep_learning.py | 92 +++++++++++++++++++++++++ tests/test_utils/test_finetune.py | 47 +++++++++++++ 3 files changed, 185 insertions(+), 11 deletions(-) create mode 100644 tests/test_utils/test_deep_learning.py diff --git a/src/detectmatelibrary/utils/finetune.py b/src/detectmatelibrary/utils/finetune.py index c29c6d47..14da359f 100644 --- a/src/detectmatelibrary/utils/finetune.py +++ b/src/detectmatelibrary/utils/finetune.py @@ -9,25 +9,60 @@ import copy +class CombOp: + @staticmethod + def is_finetune_in_there(config: CoreConfig | dict[str, Any]) -> bool: + variables = dir(config) if isinstance(config, CoreConfig) else config + return "finetune" in variables + + @staticmethod + def get_value(config: CoreConfig | dict[str, Any], path: str) -> Any: + return getattr(config, path) if isinstance(config, CoreConfig) else config[path] + + @staticmethod + def set_value(config: CoreConfig | dict[str, Any], path: list[str], value: Any) -> None: + if isinstance(config, CoreConfig): + setattr(config, path[0], value) + else: + path = [path] if isinstance(path, str) else path + dict_ = config + for v in path[:-1]: + dict_ = dict_[v] + + dict_[path[-1]] = value + + @staticmethod + def value_exist(config: CoreConfig | dict[str, Any], path: list[str]) -> bool: + if isinstance(config, CoreConfig): + return path[0] in dir(config) + + path = [path] if isinstance(path, str) else path + dict_ = config + for v in path[:-1]: + dict_ = dict_[v] + + return path[-1] in dict_ + + class Combinations: - def __init__(self, config: CoreConfig) -> None: + def __init__(self, config: CoreConfig | dict[str, Any]) -> None: self.config = config - self.paths: list[str] = [] + self.paths: list[list[str]] = [] self.combs: list[tuple[Any, ...]] = [] - if "finetune" not in dir(config): + if not CombOp.is_finetune_in_there(config): warnings.warn("No finetune options found") else: - self.paths = [path[0] for path in getattr(config, "finetune")] + self.paths = [path[:-1] for path in CombOp.get_value(config, "finetune")] self.combs = list(itertools.product( - *[path[-1] for path in getattr(config, "finetune")] + *[path[-1] for path in CombOp.get_value(config, "finetune")] )) self.values: list[float] = [] def add_value(self, value: float) -> None: self.values.append(value) - def get_best(self) -> CoreConfig: + def get_best(self) -> CoreConfig | dict[str, Any]: if self.values == []: return self.config @@ -36,16 +71,16 @@ def get_best(self) -> CoreConfig: if i == idx: config = copy.deepcopy(self.config) for path, value in zip(self.paths, combo): - if path in dir(self.config): - setattr(config, path, value) + if CombOp.value_exist(config, path): + CombOp.set_value(config, path, value) return config return self.config - def __call__(self) -> typing.Iterable[CoreConfig]: + def __call__(self) -> typing.Iterable[CoreConfig | dict[str, Any]]: for combo in self.combs: config = copy.deepcopy(self.config) for path, value in zip(self.paths, combo): - if path in dir(config): - setattr(config, path, value) + if CombOp.value_exist(config, path): + CombOp.set_value(config, path, value) yield config diff --git a/tests/test_utils/test_deep_learning.py b/tests/test_utils/test_deep_learning.py new file mode 100644 index 00000000..6b71aa9a --- /dev/null +++ b/tests/test_utils/test_deep_learning.py @@ -0,0 +1,92 @@ + +import detectmatelibrary.utils.deep_learning._op as op +import detectmatelibrary.utils.deep_learning.deeplog as deeplog + +import jax.numpy as jnp +import jax + + +class TestDLOp: + def test_checkpoint(self) -> None: + checkpoint = op.CheckPoint(patience=2) + + assert not checkpoint(0.1, 1, {"hi": "is not mee"}) + assert not checkpoint(0.02, 2, {"hi": "is mee"}) + assert not checkpoint(0.2, 2, {"hi": "is not mee"}) + assert checkpoint(0.2, 2, {"hi": "is not mee"}) + + epoch, param = checkpoint.load_checkpoint() + assert epoch == 2 + assert {"hi": "is mee"} == param + + def test_mask(self) -> None: + mask = op.Mask(seq_size=4, mask_per=0.5) + + mask_tensor_1 = mask(3) + + assert mask_tensor_1.shape == (3, 4) + for i in range(3): + assert mask_tensor_1[i].sum() == 2 + + mask_tensor_2 = mask(3) + + assert not jnp.array_equal(mask_tensor_1, mask_tensor_2) + for i in range(3): + assert mask_tensor_2[i].sum() == 2 + + +class TestDeeplog: + def test_deeplog_model(self) -> None: + model = deeplog.DeepLogModel(hidden_dim=4, n_layers=1, output_size=1) + + x = jnp.ones((3, 4, 1), dtype=jnp.float32) + params = model.init(jax.random.PRNGKey(0), x[:1])['params'] + + y = model.apply({"params": params}, x) + assert y.shape == (3, 1) + + def test_deeplog_train(self) -> None: + config = { + "Model": { + "hidden_dim": 4, + "n_layers": 1, + }, + "Train": { + "batch_size": 3, + "learning_rate": 0.01, + "epochs": 2, + }, + } + deeplog_ = deeplog.DeepLog(config=config) + stats = deeplog_.train( + seqs=[ + [1, 2, 0, 1] for _ in range(4) + ], + var_per=0.25 + ) + assert not deeplog_.check_anomaly((1, 2, 0, 1), stats["top_k"]) + assert deeplog_.check_anomaly((1, 2, 2, 0), stats["top_k"]) + + def test_deeplog_finetune(self) -> None: + config = { + "Model": { + "hidden_dim": 8, + "n_layers": 1, + }, + "Train": { + "batch_size": 3, + "learning_rate": 0.01, + "epochs": 2, + }, + "finetune": [ + ("Model", "hidden_dim", [4, 5]) + ] + } + deeplog_ = deeplog.DeepLog(config=config) + deeplog_.finetune( + seqs=[ + [1, 2, 0, 1] for _ in range(4) + ], + var_per=0.25 + ) + assert deeplog_.config != config diff --git a/tests/test_utils/test_finetune.py b/tests/test_utils/test_finetune.py index feb9c0a3..1d609e8e 100644 --- a/tests/test_utils/test_finetune.py +++ b/tests/test_utils/test_finetune.py @@ -36,6 +36,31 @@ class DummyConfig3(CoreConfig): ] +hyperparameters_without = { + "Model": { + "a": 1, + }, + "Train": { + "b": 2, + }, +} + + +hyperparameters = { + "Model": { + "a": 1, + }, + "Train": { + "b": 2, + }, + "finetune": [ + ["Model", "a", [1, 2, 3, 4]], + ["Train", "b", [10, 40]], + ["c", [0.2, 0.4, 0.6, 0.8]] + ] +} + + class TestCombinations: def test_combination_no_overwrite(self): comb = Combinations(config := DummyConfig()) @@ -80,3 +105,25 @@ def test_argument_missing(self): assert config.a == 1 assert config.b == 10 + + def test_compatibility_dl_without(self): + with pytest.warns(UserWarning): + comb = Combinations(hyperparameters_without) + + for j, _ in enumerate(comb()): + comb.add_value(j) + config = comb.get_best() + + assert config["Model"]["a"] == 1 + assert config["Train"]["b"] == 2 + + def test_compatibility_with_dl(self): + comb = Combinations(hyperparameters) + + for j, _ in enumerate(comb()): + comb.add_value(j) + config = comb.get_best() + + print(comb.paths) + assert config["Model"]["a"] == 1 + assert config["Train"]["b"] == 10 From ff55d3ad7db64aece873a57a7dd7c373b12014d4 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 13:41:15 +0200 Subject: [PATCH 06/26] add logbert model --- .../utils/deep_learning/logbert.py | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 src/detectmatelibrary/utils/deep_learning/logbert.py diff --git a/src/detectmatelibrary/utils/deep_learning/logbert.py b/src/detectmatelibrary/utils/deep_learning/logbert.py new file mode 100644 index 00000000..69197ea3 --- /dev/null +++ b/src/detectmatelibrary/utils/deep_learning/logbert.py @@ -0,0 +1,179 @@ + +import jax.numpy as jnp +import jax + +import flax.linen as nn +import optax + + +from dataclasses import dataclass +from typing import Any +from tqdm import tqdm +from math import ceil + +from detectmatelibrary.utils.deep_learning._op import CheckPoint, Mask + + +class PositionEmbedding(nn.Module): + """ + Equation: + pi,2j = sin(i / 10000^(2j/d)) + pi,2j+1 = cos(i / 10000^(2j/d)) + """ + hidden: int + max_len: int = 1000 + + def setup(self) -> None: + x = jnp.arange(self.max_len, dtype=jnp.float32).reshape(-1, 1) + jd = jnp.power(10000, jnp.arange(0, self.hidden, 2, dtype=jnp.float32) / self.hidden) + + position = jnp.zeros((1, self.max_len, self.hidden)) + position = position.at[:, :, 0::2].set(jnp.sin(x / jd)) + position = position.at[:, :, 1::2].set(jnp.cos(x / jd)) + self.position = position + + @nn.compact + def __call__(self, X: jnp.ndarray) -> jnp.ndarray: + X = X + self.position[:, :X.shape[1] ,:] + return X + + +class Embedding(nn.Module): + n_embed: int + hidden: int + dropout: float + max_len: int = 1000 + + def setup(self) -> None: + self.position = PositionEmbedding(hidden=self.hidden, max_len=self.max_len) + self.embed = nn.Embed(num_embeddings=self.n_embed, features=self.hidden) + + @nn.compact + def __call__(self, x: jnp.ndarray, training: bool = False) -> jnp.ndarray: + h = self.embed(x) + h = h + self.position(h) + return nn.Dropout(self.dropout)(h, deterministic=not training) + + +class TransformerBlock(nn.Module): + hidden: int + num_heads: int + dropout: float + + @nn.compact + def __call__(self, x: jnp.ndarray, training: bool = False) -> jnp.ndarray: + h = nn.MultiHeadAttention( + num_heads=self.num_heads, + dropout_rate=self.dropout, + out_features=self.hidden, + deterministic=not training + )(inputs_q=x) + + h = nn.LayerNorm()(h) + + h = nn.gelu(nn.Dense(self.hidden * 2)(h)) + h = nn.Dropout(self.dropout)(h, deterministic=not training) + h = nn.Dense(self.hidden)(h) + + return h + x + + +class LogBertModel(nn.Module): + n_embed: int + hidden: int + num_heads: int + n_layers: int + dropout: float + max_len: int = 1000 + + def setup(self) -> None: + self.special_tokens = 2 + + @nn.compact + def __call__(self, x: jnp.ndarray, training: bool = False) -> tuple[jnp.ndarray, jnp.ndarray]: + x = jnp.concat([x, jnp.ones((x.shape[0], 1), dtype=jnp.int32)], axis=1) + h = Embedding( + n_embed=self.n_embed + self.special_tokens, hidden=self.hidden, dropout=self.dropout + )(x + self.special_tokens) + + for _ in range(self.n_layers): + h = TransformerBlock( + hidden=self.hidden, num_heads=self.num_heads, dropout=self.dropout + )(h, training=training) + h, dist = h[:, :-1, :], h[:, -1, :] + + return nn.Dense(self.n_embed)(h), dist + + +@dataclass +class TrainConfig: + seed: int = 0 + epochs: int = 40 + learning_rate: float = 0.01 + batch_size: int = 2 + mask_per: float = 0.4 + alpha: float = 0.0 + patience: int = 3 + + +def loss_( + model: nn.Module, params: dict[str, Any], x: jnp.ndarray, m: jnp.ndarray, alpha: float +) -> jnp.ndarray: + + logist, h_dist = model.apply({"params": params}, x * m, training=True) + loss_mlkp = ((1 - m) * optax.softmax_cross_entropy_with_integer_labels( + logits=logist, labels=x + )) + loss_vhm = jnp.linalg.norm(h_dist - h_dist.mean(axis=1)[..., None], axis=1) ** 2 + + return loss_mlkp.mean() + alpha * loss_vhm.mean() + + +def train( + model: nn.Module, x: jnp.ndarray, x_val: jnp.ndarray, mask: Mask, trainConfig=TrainConfig() +) -> tuple[dict[str, Any], dict[str, float]]: + @jax.jit + def train_step(params, opt_state, x, m, alpha): + def loss_f(params): + return loss_(model=model, params=params, x=x, m=m, alpha=alpha) + + loss, grads = jax.value_and_grad(loss_f)(params) + updates, opt_state = optimizer.update(grads, opt_state) + params = optax.apply_updates(params, updates) + return params, opt_state, loss + + seed = jax.random.key(trainConfig.seed) + n_steps = ceil(x.shape[0] / trainConfig.batch_size) + + params = model.init(seed, jnp.zeros((1, x.shape[-1]), dtype=jnp.int32))["params"] + optimizer = optax.adam(learning_rate=trainConfig.learning_rate) + opt_state = optimizer.init(params) + + idx = jnp.arange(x.shape[0]) + idx = jax.random.permutation(jax.random.key(trainConfig.seed), idx) + + losses_epoch, losses_step, loss_val = [], [], [] + checkpoint, alpha = CheckPoint(TrainConfig.patience), trainConfig.alpha + for epoch in tqdm(range(trainConfig.epochs), desc="training..."): + step_loss = 0 + for step_idx in jnp.array_split(idx, n_steps): + x_step = x[step_idx] + m = mask(x_step.shape[0]) + params, opt_state, loss = train_step(params, opt_state, x_step, m, alpha=alpha) + step_loss += loss + losses_step.append(loss) + + m = mask(x_val.shape[0]) + loss_val.append(loss_(model=model, params=params, x=x_val, m=m, alpha=alpha)) + losses_epoch.append(step_loss / n_steps) + idx = jax.random.permutation(jax.random.key(epoch), idx) + if checkpoint(loss=loss_val[-1], epoch=epoch, param=params): + print("Early stop") + break + + best_e, params = checkpoint.load_checkpoint() + print(f"Best epoch {best_e} -> Train {losses_epoch[best_e]} Val {loss_val[best_e]}") + return params, { + "Loss Epoch": losses_epoch, "Loss Step": losses_step, "Loss Val": loss_val, "Best val": loss_val[best_e] + } + \ No newline at end of file From d811a76fde75818dddd6d997a4c9be038262bcd0 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 13:43:14 +0200 Subject: [PATCH 07/26] add logbert --- .../utils/deep_learning/logbert.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/src/detectmatelibrary/utils/deep_learning/logbert.py b/src/detectmatelibrary/utils/deep_learning/logbert.py index 69197ea3..74638c89 100644 --- a/src/detectmatelibrary/utils/deep_learning/logbert.py +++ b/src/detectmatelibrary/utils/deep_learning/logbert.py @@ -5,6 +5,7 @@ import flax.linen as nn import optax +from functools import lru_cache from dataclasses import dataclass from typing import Any @@ -12,6 +13,7 @@ from math import ceil from detectmatelibrary.utils.deep_learning._op import CheckPoint, Mask +from detectmatelibrary.utils.finetune import Combinations class PositionEmbedding(nn.Module): @@ -176,4 +178,113 @@ def loss_f(params): return params, { "Loss Epoch": losses_epoch, "Loss Step": losses_step, "Loss Val": loss_val, "Best val": loss_val[best_e] } + + +## Final model +default_config = { + "Model": { + "hidden": 256, + "num_heads": 2, + "n_layers": 4, + "dropout": 0.0, + "max_len": 1000, + }, + "Train": { + "batch_size": 256, + "learning_rate": 0.01, + "epochs": 10, + }, +} + + +class LogBert: + def __init__(self, config: dict = default_config) -> None: + self.config = config + self.params = {} + self.model: LogBertModel | None = None + self.mask: Mask | None = None + self.config_train = TrainConfig(**self.config["Train"]) + + def __str__(self) -> str: + return str(self.model) + "\n" + str(self.config_train) + + def top_pred(self, x: jnp.ndarray) -> tuple[jnp.ndarray]: + m = self.mask(x.shape[0]) + y, _ = self.model.apply({"params": self.params}, x * m, training=False) + + idx = jnp.nonzero(m == 0) + pred = jnp.argsort(y[idx], axis=1, descending=True) + return pred, x[idx] + + def get_best_k(self, seq: jnp.ndarray) -> int: + if seq.shape[0] == 0: + return 0 + + pred, y = self.top_pred(seq) + return int(jax.scipy.stats.mode( + ((y[..., None] == pred) * jnp.arange(pred.shape[1])[None, ...]).sum(1) + ).mode + 2) + + @lru_cache + def check_anomaly(self, seq: tuple[int], top_k: int) -> int: + if self.model is None: + return False + + pred, y = self.top_pred(jnp.array([seq])) + pred = pred[:, :top_k] + score = 0 + + for i in range(pred.shape[0]): + score += not bool(jnp.isin(y[i], pred[i])) + return score + + def _prepare_data(self, seqs: list[tuple[int]], var_per: float) -> tuple[jnp.ndarray]: + seed = jax.random.key(self.config_train.seed) + idx = jax.random.permutation(seed, len(seqs)) + seqs = jnp.array(seqs)[idx] + train_seqs = seqs[:ceil(len(seqs) * (1 - var_per))] + val_seqs = seqs[ceil(len(seqs) * (1 - var_per)):] + return train_seqs, val_seqs + + def train(self, seqs: list[tuple[int]], var_per: float) -> dict: + train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) + + self.config_train = TrainConfig(**self.config["Train"]) + self.config["Model"]["n_embed"] = int(train_seqs.max() + 1) + self.model = LogBertModel(**self.config["Model"]) + self.mask = Mask( + seq_size=train_seqs.shape[-1], mask_per=self.config_train.mask_per + ) + self.params, stats = train( + model=self.model, + mask=self.mask, + x=train_seqs, + x_val=val_seqs, + trainConfig=self.config_train + ) + stats["top_k"] = self.get_best_k(val_seqs) + + return stats + + def finetune(self, seqs: list[tuple[int]], var_per: float) -> None: + train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) + combos = Combinations(config=self.config) + + for comb in combos(): + comb["Model"]["n_embed"] = int(train_seqs.max() + 1) + model = LogBertModel(**comb["Model"]) + config_train = TrainConfig(**comb["Train"]) + mask = Mask( + seq_size=train_seqs.shape[-1], mask_per=config_train.mask_per + ) + + _, stats = train( + model=model, + mask=mask, + x=train_seqs, + x_val=val_seqs, + trainConfig=config_train + ) + combos.add_value(stats["Best val"]) + self.config = combos.get_best() \ No newline at end of file From 952d177171f486a11e69f4a19af5a0b7867595ba Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 13:44:36 +0200 Subject: [PATCH 08/26] add logbert --- src/detectmatelibrary/utils/deep_learning/logbert.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/detectmatelibrary/utils/deep_learning/logbert.py b/src/detectmatelibrary/utils/deep_learning/logbert.py index 74638c89..863d067c 100644 --- a/src/detectmatelibrary/utils/deep_learning/logbert.py +++ b/src/detectmatelibrary/utils/deep_learning/logbert.py @@ -256,11 +256,7 @@ def train(self, seqs: list[tuple[int]], var_per: float) -> dict: seq_size=train_seqs.shape[-1], mask_per=self.config_train.mask_per ) self.params, stats = train( - model=self.model, - mask=self.mask, - x=train_seqs, - x_val=val_seqs, - trainConfig=self.config_train + model=self.model, mask=self.mask, x=train_seqs, x_val=val_seqs, trainConfig=self.config_train ) stats["top_k"] = self.get_best_k(val_seqs) @@ -279,11 +275,7 @@ def finetune(self, seqs: list[tuple[int]], var_per: float) -> None: ) _, stats = train( - model=model, - mask=mask, - x=train_seqs, - x_val=val_seqs, - trainConfig=config_train + model=model, mask=mask, x=train_seqs, x_val=val_seqs, trainConfig=config_train ) combos.add_value(stats["Best val"]) self.config = combos.get_best() From 991f090fddf01beda5e402746528b0307f5c77c5 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 14:33:24 +0200 Subject: [PATCH 09/26] add logbert tests --- .../utils/deep_learning/logbert.py | 3 +- tests/test_utils/test_deep_learning.py | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/detectmatelibrary/utils/deep_learning/logbert.py b/src/detectmatelibrary/utils/deep_learning/logbert.py index 863d067c..03e612d6 100644 --- a/src/detectmatelibrary/utils/deep_learning/logbert.py +++ b/src/detectmatelibrary/utils/deep_learning/logbert.py @@ -101,7 +101,7 @@ def __call__(self, x: jnp.ndarray, training: bool = False) -> tuple[jnp.ndarray, for _ in range(self.n_layers): h = TransformerBlock( hidden=self.hidden, num_heads=self.num_heads, dropout=self.dropout - )(h, training=training) + )(h, training=training) h, dist = h[:, :-1, :], h[:, -1, :] return nn.Dense(self.n_embed)(h), dist @@ -233,7 +233,6 @@ def check_anomaly(self, seq: tuple[int], top_k: int) -> int: pred, y = self.top_pred(jnp.array([seq])) pred = pred[:, :top_k] score = 0 - for i in range(pred.shape[0]): score += not bool(jnp.isin(y[i], pred[i])) return score diff --git a/tests/test_utils/test_deep_learning.py b/tests/test_utils/test_deep_learning.py index 6b71aa9a..8dc0c928 100644 --- a/tests/test_utils/test_deep_learning.py +++ b/tests/test_utils/test_deep_learning.py @@ -1,6 +1,7 @@ import detectmatelibrary.utils.deep_learning._op as op import detectmatelibrary.utils.deep_learning.deeplog as deeplog +import detectmatelibrary.utils.deep_learning.logbert as logbert import jax.numpy as jnp import jax @@ -90,3 +91,69 @@ def test_deeplog_finetune(self) -> None: var_per=0.25 ) assert deeplog_.config != config + + +class TestLogBert: + def test_logbert_model(self) -> None: + model = logbert.LogBertModel( + n_embed=3, hidden=8, num_heads=1, n_layers=1, dropout=0.0, max_len=10 + ) + + params = model.init(jax.random.PRNGKey(0), jnp.zeros((1, 5), dtype=jnp.int32))['params'] + x = jnp.ones((3, 10), dtype=jnp.int32) + + y, dist = model.apply({"params": params}, x) + assert y.shape == (3, 10, 3) + assert dist.shape == (3, 8) + + def test_logbert_train(self) -> None: + config = { + "Model": { + "hidden": 4, + "n_layers": 1, + "num_heads": 4, + "dropout": 0.0, + "max_len": 10, + }, + "Train": { + "batch_size": 3, + "learning_rate": 0.01, + "epochs": 2, + }, + } + logbert_ = logbert.LogBert(config=config) + stats = logbert_.train( + seqs=[ + [1, 2, 0, 1] for _ in range(4) + ], + var_per=0.25 + ) + assert not logbert_.check_anomaly((1, 2, 0, 1), stats["top_k"]) + assert logbert_.check_anomaly((1, 2, 3, 0), stats["top_k"]) + + def test_logbert_finetune(self) -> None: + config = { + "Model": { + "hidden": 4, + "n_layers": 1, + "num_heads": 1, + "dropout": 0.0, + "max_len": 10, + }, + "Train": { + "batch_size": 3, + "learning_rate": 0.01, + "epochs": 2, + }, + "finetune": [ + ("Model", "n_layers", [2, 3]) + ] + } + logbert_ = logbert.LogBert(config=config) + logbert_.finetune( + seqs=[ + [1, 2, 0, 1] for _ in range(4) + ], + var_per=0.25 + ) + assert logbert_.config != config From ae4c3d7512a913f2d1a9ce2cccd018c6fc209c7c Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 15:11:49 +0200 Subject: [PATCH 10/26] optimize tests, only run slow tests in github action --- .github/workflows/python-app.yml | 2 +- pytest.ini | 5 ++++ tests/conftest.py | 23 +++++++++++++++++++ .../test_bigram_frequency_detector.py | 5 ++++ tests/test_detectors/test_charset_detector.py | 4 ++++ .../test_detectors/test_new_event_detector.py | 3 +++ .../test_detectors/test_new_value_detector.py | 4 ++++ .../test_value_range_detector.py | 2 ++ .../test_configuration_engine.py | 5 +++- tests/test_utils/test_deep_learning.py | 8 +++++++ tests/test_workspace/test_create_workspace.py | 5 ++++ 11 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/conftest.py diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index e9727e9a..a0f4fbe4 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -30,7 +30,7 @@ jobs: run: uv run --dev prek run -a - name: Test with pytest - run: uv run --dev pytest -s + run: uv run pytest -s --run-ignored # integration tests for DetectMateService - name: Checkout DetectMateService diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..ed39a206 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +# You can use addopts to change the default terminal behavior globally +addopts = --strict-markers +markers = + ignored: tests that should only run on demand diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..e9ae1503 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +# conftest.py +import pytest + + +def pytest_addoption(parser): + # This creates the command line flag + parser.addoption( + "--run-ignored", + action="store_true", + default=False + ) + + +def pytest_collection_modifyitems(config, items): + """Automatically skips tests tagged as 'ignored' unless flag is passed.""" + if config.getoption("--run-ignored"): + # If terminal flag is present, don't skip anything + return + + skip_marker = pytest.mark.skip(reason="Skipped: requires --run-ignored flag") + for item in items: + if "ignored" in item.keywords: + item.add_marker(skip_marker) diff --git a/tests/test_detectors/test_bigram_frequency_detector.py b/tests/test_detectors/test_bigram_frequency_detector.py index 01f590b1..d898667f 100644 --- a/tests/test_detectors/test_bigram_frequency_detector.py +++ b/tests/test_detectors/test_bigram_frequency_detector.py @@ -19,6 +19,9 @@ from detectmatelibrary.utils.aux import time_test_mode from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL # Set time test mode for consistent timestamps +import pytest + + time_test_mode() @@ -235,6 +238,7 @@ def test_detect_known_value_alert(self): class TestBigramFrequencyDetectorEndToEnd: """Regression test: full configure/train/detect pipeline on audit.log.""" + @pytest.mark.ignored def test_audit_log_anomalies(self): parser = MatcherParser(config=_PARSER_CONFIG) detector = BigramFrequencyDetector( @@ -265,6 +269,7 @@ class TestBigramFrequencyDetectorAutoConfig: """Test that process() drives configure/set_configuration/train/detect automatically.""" + @pytest.mark.ignored def test_audit_log_anomalies_via_process(self): parser = MatcherParser(config=_PARSER_CONFIG) detector = BigramFrequencyDetector(config=_SKIP_REPETITIONS_CONFIG, name="MultipleDetector") diff --git a/tests/test_detectors/test_charset_detector.py b/tests/test_detectors/test_charset_detector.py index a53fa2dd..8fef5569 100644 --- a/tests/test_detectors/test_charset_detector.py +++ b/tests/test_detectors/test_charset_detector.py @@ -17,6 +17,8 @@ from detectmatelibrary.utils.aux import time_test_mode from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL +import pytest + # Set time test mode for consistent timestamps time_test_mode() @@ -288,6 +290,7 @@ def test_detect_unknown_chars_reported_per_variable(self): class TestCharsetDetectorEndToEnd: """Regression test: full configure/train/detect pipeline on audit.log.""" + @pytest.mark.ignored def test_audit_log_anomalies(self): parser = MatcherParser(config=_PARSER_CONFIG) detector = CharsetDetector() @@ -314,6 +317,7 @@ class TestCharsetDetectorAutoConfig: """Test that process() drives configure/set_configuration/train/detect automatically.""" + @pytest.mark.ignored def test_audit_log_anomalies_via_process(self): parser = MatcherParser(config=_PARSER_CONFIG) detector = CharsetDetector() diff --git a/tests/test_detectors/test_new_event_detector.py b/tests/test_detectors/test_new_event_detector.py index 62ff9f36..dd2390ca 100644 --- a/tests/test_detectors/test_new_event_detector.py +++ b/tests/test_detectors/test_new_event_detector.py @@ -18,6 +18,7 @@ from detectmatelibrary.constants import GLOBAL_EVENT_ID from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL +import pytest # Set time test mode for consistent timestamps time_test_mode() @@ -154,6 +155,7 @@ def test_detect_known_event_id_no_alert(self): class TestNewEventDetectorEndToEnd: """Regression test: full configure/train/detect pipeline on audit.log.""" + @pytest.mark.ignored def test_audit_log_anomalies(self): pars = MatcherParser(config=_PARSER_CONFIG) detector = NewEventDetector(config=config, name="NewEventDetector") @@ -180,6 +182,7 @@ class TestNewEventDetectorAutoConfig: """Test that process() drives configure/set_configuration/train/detect automatically.""" + @pytest.mark.ignored def test_audit_log_anomalies_via_process(self): pars = MatcherParser(config=_PARSER_CONFIG) detector = NewEventDetector() diff --git a/tests/test_detectors/test_new_value_detector.py b/tests/test_detectors/test_new_value_detector.py index b66a4c11..422e61da 100644 --- a/tests/test_detectors/test_new_value_detector.py +++ b/tests/test_detectors/test_new_value_detector.py @@ -18,6 +18,8 @@ from detectmatelibrary.utils.aux import time_test_mode from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL +import pytest + # Set time test mode for consistent timestamps time_test_mode() @@ -215,6 +217,7 @@ def test_detect_known_value_alert(self): class TestNewValueDetectorEndToEnd: """Regression test: full configure/train/detect pipeline on audit.log.""" + @pytest.mark.ignored def test_audit_log_anomalies(self): parser = MatcherParser(config=_PARSER_CONFIG) detector = NewValueDetector() @@ -241,6 +244,7 @@ class TestNewValueDetectorAutoConfig: """Test that process() drives configure/set_configuration/train/detect automatically.""" + @pytest.mark.ignored def test_audit_log_anomalies_via_process(self): parser = MatcherParser(config=_PARSER_CONFIG) detector = NewValueDetector() diff --git a/tests/test_detectors/test_value_range_detector.py b/tests/test_detectors/test_value_range_detector.py index ace3c604..ddfae7d9 100644 --- a/tests/test_detectors/test_value_range_detector.py +++ b/tests/test_detectors/test_value_range_detector.py @@ -324,6 +324,7 @@ def test_detect_known_value_ranges_alert(self): class TestValueRangeDetectorEndToEnd: """Regression test: full configure/train/detect pipeline on audit.log.""" + @pytest.mark.ignored def test_audit_log_anomalies(self): parser = MatcherParser(config=_PARSER_CONFIG) detector = ValueRangeDetector() @@ -353,6 +354,7 @@ class TestValueRangeDetectorAutoConfig: """Test that process() drives configure/set_configuration/train/detect automatically.""" + @pytest.mark.ignored def test_audit_log_anomalies_via_process(self): parser = MatcherParser(config=_PARSER_CONFIG) detector = ValueRangeDetector() diff --git a/tests/test_pipelines/test_configuration_engine.py b/tests/test_pipelines/test_configuration_engine.py index 28058570..6f0cdd90 100644 --- a/tests/test_pipelines/test_configuration_engine.py +++ b/tests/test_pipelines/test_configuration_engine.py @@ -2,6 +2,8 @@ from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, ANOMALY_LABELS, LOG_FORMAT, TRAIN_UNTIL + +import pytest import json @@ -26,6 +28,7 @@ def load_expected_anomaly_ids() -> set[str]: class TestConfigurationEngineManual: """Mirrors the manual flow in 05_configuration_engine/detect.py.""" + @pytest.mark.ignored def test_configure_train_detect(self) -> None: parser = MatcherParser(config=parser_config) detector = NewValueDetector() @@ -51,7 +54,7 @@ def test_configure_train_detect(self) -> None: class TestConfigurationEngineAutomatic: """Tests the automated configure phase via process().""" - + @pytest.mark.ignored def test_process_configure_train_detect(self) -> None: parser = MatcherParser(config=parser_config) config = NewValueDetectorConfig(data_use_configure=TRAIN_UNTIL) diff --git a/tests/test_utils/test_deep_learning.py b/tests/test_utils/test_deep_learning.py index 8dc0c928..09145886 100644 --- a/tests/test_utils/test_deep_learning.py +++ b/tests/test_utils/test_deep_learning.py @@ -3,6 +3,8 @@ import detectmatelibrary.utils.deep_learning.deeplog as deeplog import detectmatelibrary.utils.deep_learning.logbert as logbert +import pytest + import jax.numpy as jnp import jax @@ -37,6 +39,7 @@ def test_mask(self) -> None: class TestDeeplog: + @pytest.mark.ignored def test_deeplog_model(self) -> None: model = deeplog.DeepLogModel(hidden_dim=4, n_layers=1, output_size=1) @@ -46,6 +49,7 @@ def test_deeplog_model(self) -> None: y = model.apply({"params": params}, x) assert y.shape == (3, 1) + @pytest.mark.ignored def test_deeplog_train(self) -> None: config = { "Model": { @@ -68,6 +72,7 @@ def test_deeplog_train(self) -> None: assert not deeplog_.check_anomaly((1, 2, 0, 1), stats["top_k"]) assert deeplog_.check_anomaly((1, 2, 2, 0), stats["top_k"]) + @pytest.mark.ignored def test_deeplog_finetune(self) -> None: config = { "Model": { @@ -94,6 +99,7 @@ def test_deeplog_finetune(self) -> None: class TestLogBert: + @pytest.mark.ignored def test_logbert_model(self) -> None: model = logbert.LogBertModel( n_embed=3, hidden=8, num_heads=1, n_layers=1, dropout=0.0, max_len=10 @@ -106,6 +112,7 @@ def test_logbert_model(self) -> None: assert y.shape == (3, 10, 3) assert dist.shape == (3, 8) + @pytest.mark.ignored def test_logbert_train(self) -> None: config = { "Model": { @@ -131,6 +138,7 @@ def test_logbert_train(self) -> None: assert not logbert_.check_anomaly((1, 2, 0, 1), stats["top_k"]) assert logbert_.check_anomaly((1, 2, 3, 0), stats["top_k"]) + @pytest.mark.ignored def test_logbert_finetune(self) -> None: config = { "Model": { diff --git a/tests/test_workspace/test_create_workspace.py b/tests/test_workspace/test_create_workspace.py index c6571daf..d717c910 100644 --- a/tests/test_workspace/test_create_workspace.py +++ b/tests/test_workspace/test_create_workspace.py @@ -16,6 +16,7 @@ def temp_dir(tmp_path: Path) -> Path: return tmp_path +@pytest.mark.ignored def test_create_parser_workspace(temp_dir: Path): ws_name = "myParser" workspace_root = temp_dir @@ -55,6 +56,7 @@ def test_create_parser_workspace(temp_dir: Path): assert (tests_dir / f"test_{ws_name}.py").exists() +@pytest.mark.ignored def test_create_detector_workspace(temp_dir: Path): ws_name = "myDetector" workspace_root = temp_dir @@ -87,6 +89,7 @@ def test_create_detector_workspace(temp_dir: Path): assert (tests_dir / f"test_{ws_name}.py").exists() +@pytest.mark.ignored def test_create_workspace_with_dash_name(temp_dir: Path): ws_name = "custom-parser" workspace_root = temp_dir @@ -137,6 +140,7 @@ def test_fail_if_dir_exists(temp_dir: Path): assert "already exists" in result.stderr +@pytest.mark.ignored def test_generated_detector_tests_pass(temp_dir: Path): """Run pytest inside the generated workspace on the generated detector test file.""" @@ -174,6 +178,7 @@ def test_generated_detector_tests_pass(temp_dir: Path): sys.path[:] = old_sys_path +@pytest.mark.ignored def test_generated_parser_tests_pass(temp_dir: Path): ws_name = "MyCoolParser" workspace_root = temp_dir From ee278360de2c18b6c7839ba38b00c7163d9185f7 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 15:41:30 +0200 Subject: [PATCH 11/26] mark as ignored test with sleep method --- tests/test_utils/test_aux.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_utils/test_aux.py b/tests/test_utils/test_aux.py index 6ef95456..11b4abd3 100644 --- a/tests/test_utils/test_aux.py +++ b/tests/test_utils/test_aux.py @@ -4,6 +4,8 @@ from datetime import datetime from time import sleep +import pytest + class TestTimestamp: def test_test_mode(self) -> None: @@ -14,6 +16,7 @@ def test_no_test_mode(self) -> None: time_test_mode(False) assert get_timestamp() != 0 + @pytest.mark.ignored def test_get_timestamp(self) -> None: time = get_timestamp() sleep(1) From 282d8c3be74c3d4a3749527dff08e4c9394f13ce Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 16:02:29 +0200 Subject: [PATCH 12/26] add DeepModel class --- .../common/deeplearning_detector.py | 36 +++++++++++++++++++ .../utils/deep_learning/deeplog.py | 6 ++-- .../utils/deep_learning/imodel.py | 19 ++++++++++ .../utils/deep_learning/logbert.py | 5 +-- 4 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 src/detectmatelibrary/common/deeplearning_detector.py create mode 100644 src/detectmatelibrary/utils/deep_learning/imodel.py diff --git a/src/detectmatelibrary/common/deeplearning_detector.py b/src/detectmatelibrary/common/deeplearning_detector.py new file mode 100644 index 00000000..53ff8ea9 --- /dev/null +++ b/src/detectmatelibrary/common/deeplearning_detector.py @@ -0,0 +1,36 @@ + +from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig +from detectmatelibrary.utils.data_buffer import BufferMode + + +class DeepLearningDetectorConfig(CoreDetectorConfig): + window_size: int = 10 + hyperparameters: list[tuple[str | dict[str, str], ...]] = { # type: ignore + "Model": { + + }, + "Train": { + + }, + "finetune": { + + }, + } + + +class DeepLearningDetector(CoreDetector): + def __init__( + self, + name: str = "CoreDetector", + config: DeepLearningDetectorConfig = DeepLearningDetectorConfig() + ) -> None: + + if isinstance(config, dict): + config = DeepLearningDetectorConfig.from_dict(config, name) + + super().__init__( + name=name, + buffer_mode=BufferMode.WINDOW, + buffer_size=config.window_size, + config=config + ) diff --git a/src/detectmatelibrary/utils/deep_learning/deeplog.py b/src/detectmatelibrary/utils/deep_learning/deeplog.py index bdaed56b..6106be55 100644 --- a/src/detectmatelibrary/utils/deep_learning/deeplog.py +++ b/src/detectmatelibrary/utils/deep_learning/deeplog.py @@ -11,10 +11,10 @@ from tqdm import tqdm from math import ceil +from detectmatelibrary.utils.deep_learning.imodel import DeepModel from detectmatelibrary.utils.deep_learning._op import CheckPoint from detectmatelibrary.utils.finetune import Combinations - ## Model Deeplog class DeepLogModel(nn.Module): hidden_dim: int @@ -132,7 +132,7 @@ def do_train( } -class DeepLog: +class DeepLog(DeepModel): def __init__(self, config: dict = default_config) -> None: self.config = config self.params = {} @@ -179,7 +179,7 @@ def _prepare_data(self, seqs: list[tuple[int]], var_per: float) -> tuple[jnp.nda val_seqs = seqs[ceil(len(seqs) * (1 - var_per)):] return train_seqs, val_seqs - def train(self, seqs: list[tuple[int]], var_per: float) -> dict: + def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, float]: train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) self.config_train = TrainConfig(**self.config["Train"]) diff --git a/src/detectmatelibrary/utils/deep_learning/imodel.py b/src/detectmatelibrary/utils/deep_learning/imodel.py new file mode 100644 index 00000000..18e3643e --- /dev/null +++ b/src/detectmatelibrary/utils/deep_learning/imodel.py @@ -0,0 +1,19 @@ + +from abc import ABC, abstractmethod + + +class DeepModel(ABC): + def __init__(self) -> None: + pass + + @abstractmethod + def check_anomaly(self, seq: tuple[int], top_k: int) -> bool: + pass + + @abstractmethod + def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, float]: + pass + + @abstractmethod + def finetune(self, seqs: list[tuple[int]], var_per: float, epochs: int = 2) -> None: + pass diff --git a/src/detectmatelibrary/utils/deep_learning/logbert.py b/src/detectmatelibrary/utils/deep_learning/logbert.py index 03e612d6..bfa10b9b 100644 --- a/src/detectmatelibrary/utils/deep_learning/logbert.py +++ b/src/detectmatelibrary/utils/deep_learning/logbert.py @@ -13,6 +13,7 @@ from math import ceil from detectmatelibrary.utils.deep_learning._op import CheckPoint, Mask +from detectmatelibrary.utils.deep_learning.imodel import DeepModel from detectmatelibrary.utils.finetune import Combinations @@ -197,7 +198,7 @@ def loss_f(params): } -class LogBert: +class LogBert(DeepModel): def __init__(self, config: dict = default_config) -> None: self.config = config self.params = {} @@ -245,7 +246,7 @@ def _prepare_data(self, seqs: list[tuple[int]], var_per: float) -> tuple[jnp.nda val_seqs = seqs[ceil(len(seqs) * (1 - var_per)):] return train_seqs, val_seqs - def train(self, seqs: list[tuple[int]], var_per: float) -> dict: + def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, float]: train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) self.config_train = TrainConfig(**self.config["Train"]) From 0feadb976f251449e7250b8198b8f00b5efbd5e6 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 16:29:29 +0200 Subject: [PATCH 13/26] add first version deeplog and logbert detectors --- .../common/deeplearning_detector.py | 58 +++++++++++++++++-- .../detectors/deeplog_detector.py | 44 ++++++++++++++ .../detectors/logbert_detector.py | 48 +++++++++++++++ 3 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 src/detectmatelibrary/detectors/deeplog_detector.py create mode 100644 src/detectmatelibrary/detectors/logbert_detector.py diff --git a/src/detectmatelibrary/common/deeplearning_detector.py b/src/detectmatelibrary/common/deeplearning_detector.py index 53ff8ea9..0e575f68 100644 --- a/src/detectmatelibrary/common/deeplearning_detector.py +++ b/src/detectmatelibrary/common/deeplearning_detector.py @@ -1,26 +1,35 @@ from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig + +from detectmatelibrary.utils.deep_learning.imodel import DeepModel from detectmatelibrary.utils.data_buffer import BufferMode +from detectmatelibrary import schemas + class DeepLearningDetectorConfig(CoreDetectorConfig): window_size: int = 10 - hyperparameters: list[tuple[str | dict[str, str], ...]] = { # type: ignore + validation_per: float = 0.2 + + hyperparameters: list[tuple[str | dict[str, str] | list[str], ...]] = { # type: ignore "Model": { }, "Train": { }, - "finetune": { - - }, + "finetune": [], } +def build_seq(input_: list[schemas.ParserSchema]) -> tuple[int]: + return tuple([in_["EventID"] for in_ in input_]) + + class DeepLearningDetector(CoreDetector): def __init__( self, + model_cls: DeepModel, name: str = "CoreDetector", config: DeepLearningDetectorConfig = DeepLearningDetectorConfig() ) -> None: @@ -34,3 +43,44 @@ def __init__( buffer_size=config.window_size, config=config ) + self.config: DeepLearningDetectorConfig + + self.model = model_cls(config=self.config.hyperparameters) # type: ignore + + self.train_seqs: list[tuple[int]] = [] + self.config_seqs: list[tuple[int]] = [] + self.stats: dict[str, float] = {} + self.top_k: float = 0 + + def train(self, input_: list[schemas.ParserSchema]) -> None: # type: ignore + self.train_seqs.append(build_seq(input_)) + + def configure(self, input_: list[schemas.ParserSchema]) -> None: # type: ignore + self.config_seqs.append(build_seq(input_)) + + def set_configuration(self) -> None: + self.model.finetune(self.config_seqs, var_per=self.config.validation_per) + self.config_seqs = [] + + def post_train(self) -> None: + self.stats = self.model.train(self.train_seqs, var_per=self.config.validation_per) + self.train_seqs = [] + + if "top_k" in self.stats: + self.top_k = self.stats["top_k"] + print(self.model) + print("Top k assigned", self.top_k) + + def detect( + self, + input_: list[schemas.ParserSchema], # type: ignore + output_: schemas.DetectorSchema, + ) -> bool: + + alert = self.model.check_anomaly(build_seq(input_), top_k=self.top_k) + if alert: + output_["score"] = 1.0 + output_["description"] = f"{self.name} found an anomaly in the sequence" + return True + + return False diff --git a/src/detectmatelibrary/detectors/deeplog_detector.py b/src/detectmatelibrary/detectors/deeplog_detector.py new file mode 100644 index 00000000..f9797db1 --- /dev/null +++ b/src/detectmatelibrary/detectors/deeplog_detector.py @@ -0,0 +1,44 @@ +from detectmatelibrary.common.deeplearning_detector import ( + DeepLearningDetectorConfig, DeepLearningDetector +) + +from detectmatelibrary.utils.deep_learning.deeplog import DeepLog + + +from typing import Any + + +class DeeplogDetectorConfig(DeepLearningDetectorConfig): + method_type: str = "deeplog_detector" + + hyperparameters: list[tuple[str | dict[str, str] | list[str], ...]] = { # type: ignore + "Model": { + "hidden_dim": 64, + "n_layers": 2, + }, + "Train": { + "batch_size": 2048, + "learning_rate": 0.01, + "epochs": 10, + }, + "finetune": [ + ["Model", "hidden_dim", [128, 256, 512]] + ["Model", "n_layers", [1, 2, 3]] + ["Train", "learning_rate", [0.01, 0.02, 0.03]] + ], + } + + +class DeeplogDetector(DeepLearningDetector): + def __init__( + self, + name: str = "DeeplogDetector", + config: DeeplogDetectorConfig | dict[str, Any] = DeeplogDetectorConfig(), + ) -> None: + + if isinstance(config, dict): + config = DeeplogDetectorConfig.from_dict(config, name) + + super().__init__( + name=name, model_cls=DeepLog, config=config + ) \ No newline at end of file diff --git a/src/detectmatelibrary/detectors/logbert_detector.py b/src/detectmatelibrary/detectors/logbert_detector.py new file mode 100644 index 00000000..77bb465a --- /dev/null +++ b/src/detectmatelibrary/detectors/logbert_detector.py @@ -0,0 +1,48 @@ +from detectmatelibrary.common.deeplearning_detector import ( + DeepLearningDetectorConfig, DeepLearningDetector +) + +from detectmatelibrary.utils.deep_learning.logbert import LogBert + + +from typing import Any + + +class LogBertDetectorConfig(DeepLearningDetectorConfig): + method_type: str = "logbert_detector" + + hyperparameters: list[tuple[str | dict[str, str] | list[str], ...]] = { # type: ignore + "Model": { + "n_embed": 10, + "hidden": 32, + "num_heads": 2, + "n_layers": 1, + "dropout": 0.0, + "max_len": 1000, + }, + "Train": { + "batch_size": 256, + "learning_rate": 0.01, + "epochs": 10, + }, + "finetune": [ + ["Model", "hidden", [64, 128, 256]] + ["Model", "n_layers", [1, 2, 3]] + ["Train", "learning_rate", [0.002, 0.001, 0.005]] + ] + } + + +class LogBertDetector(DeepLearningDetector): + def __init__( + self, + name: str = "LogBertDetector", + config: LogBertDetectorConfig | dict[str, Any] = LogBertDetectorConfig(), + ) -> None: + + if isinstance(config, dict): + config = LogBertDetectorConfig.from_dict(config, name) + + super().__init__( + name=name, model_cls=LogBert, config=config + ) \ No newline at end of file From 431d0e393e625d7577d4daafa37bcf8275834d76 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 16:38:23 +0200 Subject: [PATCH 14/26] minor cleaning --- src/detectmatelibrary/common/deeplearning_detector.py | 11 +++++------ src/detectmatelibrary/detectors/deeplog_detector.py | 2 +- src/detectmatelibrary/detectors/logbert_detector.py | 2 +- src/detectmatelibrary/utils/deep_learning/deeplog.py | 2 +- src/detectmatelibrary/utils/deep_learning/imodel.py | 2 +- src/detectmatelibrary/utils/deep_learning/logbert.py | 2 +- src/detectmatelibrary/utils/finetune.py | 6 +++--- tests/test_utils/test_deep_learning.py | 4 ++-- tests/test_utils/test_finetune.py | 6 +++--- 9 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/detectmatelibrary/common/deeplearning_detector.py b/src/detectmatelibrary/common/deeplearning_detector.py index 0e575f68..564f7d07 100644 --- a/src/detectmatelibrary/common/deeplearning_detector.py +++ b/src/detectmatelibrary/common/deeplearning_detector.py @@ -18,7 +18,7 @@ class DeepLearningDetectorConfig(CoreDetectorConfig): "Train": { }, - "finetune": [], + "Finetune": [], } @@ -44,13 +44,12 @@ def __init__( config=config ) self.config: DeepLearningDetectorConfig - - self.model = model_cls(config=self.config.hyperparameters) # type: ignore + self.model: DeepModel = model_cls(config=self.config.hyperparameters) # type: ignore self.train_seqs: list[tuple[int]] = [] self.config_seqs: list[tuple[int]] = [] - self.stats: dict[str, float] = {} - self.top_k: float = 0 + self.stats: dict[str, float | int] = {} + self.top_k: int = 0 def train(self, input_: list[schemas.ParserSchema]) -> None: # type: ignore self.train_seqs.append(build_seq(input_)) @@ -67,7 +66,7 @@ def post_train(self) -> None: self.train_seqs = [] if "top_k" in self.stats: - self.top_k = self.stats["top_k"] + self.top_k = int(self.stats["top_k"]) print(self.model) print("Top k assigned", self.top_k) diff --git a/src/detectmatelibrary/detectors/deeplog_detector.py b/src/detectmatelibrary/detectors/deeplog_detector.py index f9797db1..398edad5 100644 --- a/src/detectmatelibrary/detectors/deeplog_detector.py +++ b/src/detectmatelibrary/detectors/deeplog_detector.py @@ -21,7 +21,7 @@ class DeeplogDetectorConfig(DeepLearningDetectorConfig): "learning_rate": 0.01, "epochs": 10, }, - "finetune": [ + "Finetune": [ ["Model", "hidden_dim", [128, 256, 512]] ["Model", "n_layers", [1, 2, 3]] ["Train", "learning_rate", [0.01, 0.02, 0.03]] diff --git a/src/detectmatelibrary/detectors/logbert_detector.py b/src/detectmatelibrary/detectors/logbert_detector.py index 77bb465a..c94da589 100644 --- a/src/detectmatelibrary/detectors/logbert_detector.py +++ b/src/detectmatelibrary/detectors/logbert_detector.py @@ -25,7 +25,7 @@ class LogBertDetectorConfig(DeepLearningDetectorConfig): "learning_rate": 0.01, "epochs": 10, }, - "finetune": [ + "Finetune": [ ["Model", "hidden", [64, 128, 256]] ["Model", "n_layers", [1, 2, 3]] ["Train", "learning_rate", [0.002, 0.001, 0.005]] diff --git a/src/detectmatelibrary/utils/deep_learning/deeplog.py b/src/detectmatelibrary/utils/deep_learning/deeplog.py index 6106be55..3331d2fd 100644 --- a/src/detectmatelibrary/utils/deep_learning/deeplog.py +++ b/src/detectmatelibrary/utils/deep_learning/deeplog.py @@ -179,7 +179,7 @@ def _prepare_data(self, seqs: list[tuple[int]], var_per: float) -> tuple[jnp.nda val_seqs = seqs[ceil(len(seqs) * (1 - var_per)):] return train_seqs, val_seqs - def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, float]: + def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, int | float]: train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) self.config_train = TrainConfig(**self.config["Train"]) diff --git a/src/detectmatelibrary/utils/deep_learning/imodel.py b/src/detectmatelibrary/utils/deep_learning/imodel.py index 18e3643e..51446d50 100644 --- a/src/detectmatelibrary/utils/deep_learning/imodel.py +++ b/src/detectmatelibrary/utils/deep_learning/imodel.py @@ -11,7 +11,7 @@ def check_anomaly(self, seq: tuple[int], top_k: int) -> bool: pass @abstractmethod - def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, float]: + def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, int | float]: pass @abstractmethod diff --git a/src/detectmatelibrary/utils/deep_learning/logbert.py b/src/detectmatelibrary/utils/deep_learning/logbert.py index bfa10b9b..e7097956 100644 --- a/src/detectmatelibrary/utils/deep_learning/logbert.py +++ b/src/detectmatelibrary/utils/deep_learning/logbert.py @@ -246,7 +246,7 @@ def _prepare_data(self, seqs: list[tuple[int]], var_per: float) -> tuple[jnp.nda val_seqs = seqs[ceil(len(seqs) * (1 - var_per)):] return train_seqs, val_seqs - def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, float]: + def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, int | float]: train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) self.config_train = TrainConfig(**self.config["Train"]) diff --git a/src/detectmatelibrary/utils/finetune.py b/src/detectmatelibrary/utils/finetune.py index 14da359f..8015531a 100644 --- a/src/detectmatelibrary/utils/finetune.py +++ b/src/detectmatelibrary/utils/finetune.py @@ -13,7 +13,7 @@ class CombOp: @staticmethod def is_finetune_in_there(config: CoreConfig | dict[str, Any]) -> bool: variables = dir(config) if isinstance(config, CoreConfig) else config - return "finetune" in variables + return "Finetune" in variables @staticmethod def get_value(config: CoreConfig | dict[str, Any], path: str) -> Any: @@ -53,9 +53,9 @@ def __init__(self, config: CoreConfig | dict[str, Any]) -> None: if not CombOp.is_finetune_in_there(config): warnings.warn("No finetune options found") else: - self.paths = [path[:-1] for path in CombOp.get_value(config, "finetune")] + self.paths = [path[:-1] for path in CombOp.get_value(config, "Finetune")] self.combs = list(itertools.product( - *[path[-1] for path in CombOp.get_value(config, "finetune")] + *[path[-1] for path in CombOp.get_value(config, "Finetune")] )) self.values: list[float] = [] diff --git a/tests/test_utils/test_deep_learning.py b/tests/test_utils/test_deep_learning.py index 09145886..a2dba4c5 100644 --- a/tests/test_utils/test_deep_learning.py +++ b/tests/test_utils/test_deep_learning.py @@ -84,7 +84,7 @@ def test_deeplog_finetune(self) -> None: "learning_rate": 0.01, "epochs": 2, }, - "finetune": [ + "Finetune": [ ("Model", "hidden_dim", [4, 5]) ] } @@ -153,7 +153,7 @@ def test_logbert_finetune(self) -> None: "learning_rate": 0.01, "epochs": 2, }, - "finetune": [ + "Finetune": [ ("Model", "n_layers", [2, 3]) ] } diff --git a/tests/test_utils/test_finetune.py b/tests/test_utils/test_finetune.py index 1d609e8e..2c558e83 100644 --- a/tests/test_utils/test_finetune.py +++ b/tests/test_utils/test_finetune.py @@ -12,7 +12,7 @@ class DummyConfig(CoreConfig): b: int = 10 c: float = 0.2 - finetune: list[tuple[str, list[Any]]] = [ + Finetune: list[tuple[str, list[Any]]] = [ ["a", [1, 2, 3, 4]], ["b", [10, 40]], ["c", [0.2, 0.4, 0.6, 0.8]] @@ -29,7 +29,7 @@ class DummyConfig3(CoreConfig): a: int = 2 b: int = 10 - finetune: list[tuple[str, list[Any]]] = [ + Finetune: list[tuple[str, list[Any]]] = [ ["a", [1, 2, 3, 4]], ["b", [10, 40]], ["c", [0.2, 0.4, 0.6, 0.8]] @@ -53,7 +53,7 @@ class DummyConfig3(CoreConfig): "Train": { "b": 2, }, - "finetune": [ + "Finetune": [ ["Model", "a", [1, 2, 3, 4]], ["Train", "b", [10, 40]], ["c", [0.2, 0.4, 0.6, 0.8]] From 9215836433441768eace7e0ae16401724dd687ea Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 16:47:37 +0200 Subject: [PATCH 15/26] final touches --- src/detectmatelibrary/detectors/deeplog_detector.py | 2 ++ src/detectmatelibrary/detectors/logbert_detector.py | 4 ++++ .../utils/deep_learning/deeplog.py | 13 +++++++++---- .../utils/deep_learning/logbert.py | 11 +++++++++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/detectmatelibrary/detectors/deeplog_detector.py b/src/detectmatelibrary/detectors/deeplog_detector.py index 398edad5..95d36c87 100644 --- a/src/detectmatelibrary/detectors/deeplog_detector.py +++ b/src/detectmatelibrary/detectors/deeplog_detector.py @@ -17,9 +17,11 @@ class DeeplogDetectorConfig(DeepLearningDetectorConfig): "n_layers": 2, }, "Train": { + "seed": 0, "batch_size": 2048, "learning_rate": 0.01, "epochs": 10, + "patience": 3, }, "Finetune": [ ["Model", "hidden_dim", [128, 256, 512]] diff --git a/src/detectmatelibrary/detectors/logbert_detector.py b/src/detectmatelibrary/detectors/logbert_detector.py index c94da589..3e185944 100644 --- a/src/detectmatelibrary/detectors/logbert_detector.py +++ b/src/detectmatelibrary/detectors/logbert_detector.py @@ -21,9 +21,13 @@ class LogBertDetectorConfig(DeepLearningDetectorConfig): "max_len": 1000, }, "Train": { + "seed": 0, "batch_size": 256, "learning_rate": 0.01, "epochs": 10, + "mask_per": 0.4, + "alpha": 0.0, + "patience": 3, }, "Finetune": [ ["Model", "hidden", [64, 128, 256]] diff --git a/src/detectmatelibrary/utils/deep_learning/deeplog.py b/src/detectmatelibrary/utils/deep_learning/deeplog.py index 3331d2fd..40bac7dc 100644 --- a/src/detectmatelibrary/utils/deep_learning/deeplog.py +++ b/src/detectmatelibrary/utils/deep_learning/deeplog.py @@ -15,6 +15,9 @@ from detectmatelibrary.utils.deep_learning._op import CheckPoint from detectmatelibrary.utils.finetune import Combinations +import logging + + ## Model Deeplog class DeepLogModel(nn.Module): hidden_dim: int @@ -92,11 +95,11 @@ def loss_fn(params: dict[str, Any]) -> jnp.ndarray: loss_val.append(loss_f(model=model, params=params, x=x_val, y=y_val)) idx = jax.random.permutation(jax.random.key(epoch), idx) if checkpoint(loss=loss_val[-1], epoch=epoch, param=params): - print("Early stop") + logging.info("Early stop") break best_e, params = checkpoint.load_checkpoint() - print(f"Best epoch {best_e} -> Train {losses_epoch[best_e]} Val {loss_val[best_e]}") + logging.info(f"Best epoch {best_e} -> Train {losses_epoch[best_e]} Val {loss_val[best_e]}") return params, { "Loss Epoch": losses_epoch, "Loss Step": losses_step, @@ -125,9 +128,11 @@ def do_train( "n_layers": 2, }, "Train": { + "seed": 0, "batch_size": 2048, "learning_rate": 0.01, "epochs": 10, + "patience": 3, }, } @@ -184,7 +189,7 @@ def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, int | float self.config_train = TrainConfig(**self.config["Train"]) self.config["Model"]["output_size"] = train_seqs.max() + 1 - print("Output shape:", self.config["Model"]["output_size"]) + logging.info(f"Output shape: {self.config["Model"]["output_size"]}") self.model = DeepLogModel(**self.config["Model"]) self.params, stats = do_train( @@ -208,4 +213,4 @@ def finetune(self, seqs: list[tuple[int]], var_per: float, epochs: int = 2) -> N _, stats = do_train(model, train_seqs=train_seqs, val_seqs=val_seqs, config=config_train) combos.add_value(stats["Best val"]) self.config = combos.get_best() - print(self.config) \ No newline at end of file + logging.info(self.config) \ No newline at end of file diff --git a/src/detectmatelibrary/utils/deep_learning/logbert.py b/src/detectmatelibrary/utils/deep_learning/logbert.py index e7097956..755a5d04 100644 --- a/src/detectmatelibrary/utils/deep_learning/logbert.py +++ b/src/detectmatelibrary/utils/deep_learning/logbert.py @@ -16,6 +16,8 @@ from detectmatelibrary.utils.deep_learning.imodel import DeepModel from detectmatelibrary.utils.finetune import Combinations +import logging + class PositionEmbedding(nn.Module): """ @@ -171,11 +173,11 @@ def loss_f(params): losses_epoch.append(step_loss / n_steps) idx = jax.random.permutation(jax.random.key(epoch), idx) if checkpoint(loss=loss_val[-1], epoch=epoch, param=params): - print("Early stop") + logging.info("Early stop") break best_e, params = checkpoint.load_checkpoint() - print(f"Best epoch {best_e} -> Train {losses_epoch[best_e]} Val {loss_val[best_e]}") + logging.info(f"Best epoch {best_e} -> Train {losses_epoch[best_e]} Val {loss_val[best_e]}") return params, { "Loss Epoch": losses_epoch, "Loss Step": losses_step, "Loss Val": loss_val, "Best val": loss_val[best_e] } @@ -191,9 +193,13 @@ def loss_f(params): "max_len": 1000, }, "Train": { + "seed": 0, "batch_size": 256, "learning_rate": 0.01, "epochs": 10, + "mask_per": 0.4, + "alpha": 0.0, + "patience": 3, }, } @@ -279,4 +285,5 @@ def finetune(self, seqs: list[tuple[int]], var_per: float) -> None: ) combos.add_value(stats["Best val"]) self.config = combos.get_best() + logging.info(self.config) \ No newline at end of file From 7794798062173caae5ccb8011e938986d35b4422 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 27 Jul 2026 17:01:09 +0200 Subject: [PATCH 16/26] unittest test deep learning --- .../test_deeplearning_detectors.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/test_common/test_deeplearning_detectors.py diff --git a/tests/test_common/test_deeplearning_detectors.py b/tests/test_common/test_deeplearning_detectors.py new file mode 100644 index 00000000..9740b208 --- /dev/null +++ b/tests/test_common/test_deeplearning_detectors.py @@ -0,0 +1,69 @@ + +from detectmatelibrary.common.deeplearning_detector import ( + DeepLearningDetector, DeepLearningDetectorConfig +) + +from detectmatelibrary.utils.deep_learning.imodel import DeepModel +from detectmatelibrary import schemas + +import pytest + + +class Flag(Exception): + pass + + +class DummyDeepModel(DeepModel): + def __init__(self, *args, **kargs): + super().__init__() + + def check_anomaly(self, seq, top_k): + raise Flag() + + def train(self, seqs, var_per): + return {} + + def finetune(self, seqs, var_per, epochs): + return None + + +class TestDeepLearning: + def test_normal_run_configure(self): + deep_learning_detector = DeepLearningDetector( + model_cls=DummyDeepModel, config=DeepLearningDetectorConfig(window_size=1) + ) + + for i in range(10): + deep_learning_detector.configure( + [schemas.ParserSchema({"EventID": i})] + ) + + for i in range(10): + assert deep_learning_detector.config_seqs[i] == (i,) + + deep_learning_detector.set_configuration() + assert len(deep_learning_detector.config_seqs) == 0 + + def test_normal_run_train(self): + deep_learning_detector = DeepLearningDetector( + model_cls=DummyDeepModel, config=DeepLearningDetectorConfig(window_size=1) + ) + + for i in range(10): + deep_learning_detector.train( + [schemas.ParserSchema({"EventID": i})] + ) + + for i in range(10): + assert deep_learning_detector.train_seqs[i] == (i,) + + deep_learning_detector.post_train() + assert len(deep_learning_detector.train_seqs) == 0 + + def test_check_that_check_anomaly_is_call(self): + deep_learning_detector = DeepLearningDetector( + model_cls=DummyDeepModel, config=DeepLearningDetectorConfig(window_size=1) + ) + + with pytest.raises(Flag): + deep_learning_detector.process(schemas.ParserSchema({"EventID": 0})) From 343400f1e8babd4c1a0004bf762d4bdee4c3d5f7 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 28 Jul 2026 08:17:00 +0200 Subject: [PATCH 17/26] add finetune epochs in DLDetector --- src/detectmatelibrary/common/deeplearning_detector.py | 5 ++++- tests/test_common/test_deeplearning_detectors.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/detectmatelibrary/common/deeplearning_detector.py b/src/detectmatelibrary/common/deeplearning_detector.py index 564f7d07..ed41d4b5 100644 --- a/src/detectmatelibrary/common/deeplearning_detector.py +++ b/src/detectmatelibrary/common/deeplearning_detector.py @@ -10,6 +10,7 @@ class DeepLearningDetectorConfig(CoreDetectorConfig): window_size: int = 10 validation_per: float = 0.2 + finetune_epochs: int = 2 hyperparameters: list[tuple[str | dict[str, str] | list[str], ...]] = { # type: ignore "Model": { @@ -58,7 +59,9 @@ def configure(self, input_: list[schemas.ParserSchema]) -> None: # type: ignore self.config_seqs.append(build_seq(input_)) def set_configuration(self) -> None: - self.model.finetune(self.config_seqs, var_per=self.config.validation_per) + self.model.finetune( + self.config_seqs, var_per=self.config.validation_per, epochs=self.config.finetune_epochs + ) self.config_seqs = [] def post_train(self) -> None: diff --git a/tests/test_common/test_deeplearning_detectors.py b/tests/test_common/test_deeplearning_detectors.py index 9740b208..c2260fe6 100644 --- a/tests/test_common/test_deeplearning_detectors.py +++ b/tests/test_common/test_deeplearning_detectors.py @@ -23,7 +23,7 @@ def check_anomaly(self, seq, top_k): def train(self, seqs, var_per): return {} - def finetune(self, seqs, var_per, epochs): + def finetune(self, seqs, var_per, epochs=2): return None From b5e71c1d170293554958ba5ac925d0a0a4964fe4 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 28 Jul 2026 08:51:41 +0200 Subject: [PATCH 18/26] adding deeplog tests --- .../common/_core_op/_fit_logic.py | 2 +- .../common/deeplearning_detector.py | 4 +- .../detectors/deeplog_detector.py | 8 +- tests/test_detectors/test_deeplog_detector.py | 82 +++++++++++++++++++ 4 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 tests/test_detectors/test_deeplog_detector.py diff --git a/src/detectmatelibrary/common/_core_op/_fit_logic.py b/src/detectmatelibrary/common/_core_op/_fit_logic.py index d0699191..b99a352a 100644 --- a/src/detectmatelibrary/common/_core_op/_fit_logic.py +++ b/src/detectmatelibrary/common/_core_op/_fit_logic.py @@ -67,7 +67,7 @@ class FitLogicState(Enum): def describe(self) -> str: descriptions = [ "Configuring", - "Training.", + "Training", "Default" ] return descriptions[self.value] diff --git a/src/detectmatelibrary/common/deeplearning_detector.py b/src/detectmatelibrary/common/deeplearning_detector.py index ed41d4b5..1189483e 100644 --- a/src/detectmatelibrary/common/deeplearning_detector.py +++ b/src/detectmatelibrary/common/deeplearning_detector.py @@ -6,13 +6,15 @@ from detectmatelibrary import schemas +from typing import Any + class DeepLearningDetectorConfig(CoreDetectorConfig): window_size: int = 10 validation_per: float = 0.2 finetune_epochs: int = 2 - hyperparameters: list[tuple[str | dict[str, str] | list[str], ...]] = { # type: ignore + hyperparameters: dict[str, Any] = { "Model": { }, diff --git a/src/detectmatelibrary/detectors/deeplog_detector.py b/src/detectmatelibrary/detectors/deeplog_detector.py index 95d36c87..f575887f 100644 --- a/src/detectmatelibrary/detectors/deeplog_detector.py +++ b/src/detectmatelibrary/detectors/deeplog_detector.py @@ -11,7 +11,7 @@ class DeeplogDetectorConfig(DeepLearningDetectorConfig): method_type: str = "deeplog_detector" - hyperparameters: list[tuple[str | dict[str, str] | list[str], ...]] = { # type: ignore + hyperparameters: dict[str, Any] = { # type: ignore "Model": { "hidden_dim": 64, "n_layers": 2, @@ -24,9 +24,9 @@ class DeeplogDetectorConfig(DeepLearningDetectorConfig): "patience": 3, }, "Finetune": [ - ["Model", "hidden_dim", [128, 256, 512]] - ["Model", "n_layers", [1, 2, 3]] - ["Train", "learning_rate", [0.01, 0.02, 0.03]] + ["Model", "hidden_dim", [128, 256, 512]], + ["Model", "n_layers", [1, 2, 3]], + ["Train", "learning_rate", [0.01, 0.02, 0.03]], ], } diff --git a/tests/test_detectors/test_deeplog_detector.py b/tests/test_detectors/test_deeplog_detector.py new file mode 100644 index 00000000..24ec186e --- /dev/null +++ b/tests/test_detectors/test_deeplog_detector.py @@ -0,0 +1,82 @@ +""" +Main tests are done at low level in the deeplearning module as DeeplogDetector is an interface. + +Tests at this level are only end to end pipeline. +""" + +from detectmatelibrary.detectors.deeplog_detector import DeeplogDetector + +import detectmatelibrary.schemas as schemas + +import pytest + + +class TestDeeplog: + @pytest.mark.ignored + def test_end2end_autoconfig(self) -> None: + config = { + "detectors": { + "DeeplogDetector": { + "method_type": "deeplog_detector", + "auto_config": True, + "data_use_configure": 10, + "data_use_training": 10, + "window_size": 3, + } + } + } + + deeplog = DeeplogDetector(config=config) + assert deeplog.get_state() == "Default" + + for j in range(2): + for i in [1, 2, 3, 4, 5, 1, 2]: + deeplog.process(schemas.ParserSchema({"EventID": i})) + + if j == 0: + assert deeplog.get_state() == "Configuring" + + assert deeplog.get_state() == "Training" + + for _ in range(2): + for i in [1, 2, 3, 4, 5, 1, 2]: + deeplog.process(schemas.ParserSchema({"EventID": i})) + assert deeplog.get_state() == "Default" + + @pytest.mark.ignored + def test_end2end_no_autoconfig(self) -> None: + config = { + "detectors": { + "DeeplogDetector": { + "method_type": "deeplog_detector", + "auto_config": False, + "data_use_training": 10, + "window_size": 3, + "hyperparameters": { + "Model": { + "hidden_dim": 64, + "n_layers": 2, + }, + "Train": { + "seed": 0, + "batch_size": 2048, + "learning_rate": 0.01, + "epochs": 10, + "patience": 3, + }, + "Finetune": [], + } + } + } + } + + deeplog = DeeplogDetector(config=config) + assert deeplog.get_state() == "Default" + + for j in range(2): + for i in [1, 2, 3, 4, 5, 1, 2]: + deeplog.process(schemas.ParserSchema({"EventID": i})) + if j == 0: + assert deeplog.get_state() == "Training" + + assert deeplog.get_state() == "Default" From b1df3a4a5ad26457eac0a4a474ad06f31e6c46d1 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 28 Jul 2026 09:06:59 +0200 Subject: [PATCH 19/26] add tests for logbert --- .../detectors/logbert_detector.py | 8 +- .../utils/deep_learning/logbert.py | 3 +- tests/test_detectors/test_logbert_detector.py | 87 +++++++++++++++++++ 3 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 tests/test_detectors/test_logbert_detector.py diff --git a/src/detectmatelibrary/detectors/logbert_detector.py b/src/detectmatelibrary/detectors/logbert_detector.py index 3e185944..d9a0e5e5 100644 --- a/src/detectmatelibrary/detectors/logbert_detector.py +++ b/src/detectmatelibrary/detectors/logbert_detector.py @@ -11,7 +11,7 @@ class LogBertDetectorConfig(DeepLearningDetectorConfig): method_type: str = "logbert_detector" - hyperparameters: list[tuple[str | dict[str, str] | list[str], ...]] = { # type: ignore + hyperparameters: dict[str, Any] = { # type: ignore "Model": { "n_embed": 10, "hidden": 32, @@ -30,9 +30,9 @@ class LogBertDetectorConfig(DeepLearningDetectorConfig): "patience": 3, }, "Finetune": [ - ["Model", "hidden", [64, 128, 256]] - ["Model", "n_layers", [1, 2, 3]] - ["Train", "learning_rate", [0.002, 0.001, 0.005]] + ["Model", "hidden", [64, 128, 256]], + ["Model", "n_layers", [1, 2, 3]], + ["Train", "learning_rate", [0.002, 0.001, 0.005]], ] } diff --git a/src/detectmatelibrary/utils/deep_learning/logbert.py b/src/detectmatelibrary/utils/deep_learning/logbert.py index 755a5d04..cb7f8793 100644 --- a/src/detectmatelibrary/utils/deep_learning/logbert.py +++ b/src/detectmatelibrary/utils/deep_learning/logbert.py @@ -268,12 +268,13 @@ def train(self, seqs: list[tuple[int]], var_per: float) -> dict[str, int | float return stats - def finetune(self, seqs: list[tuple[int]], var_per: float) -> None: + def finetune(self, seqs: list[tuple[int]], var_per: float, epochs: int = 2) -> None: train_seqs, val_seqs = self._prepare_data(seqs=seqs, var_per=var_per) combos = Combinations(config=self.config) for comb in combos(): comb["Model"]["n_embed"] = int(train_seqs.max() + 1) + comb["Train"]["epochs"] = epochs model = LogBertModel(**comb["Model"]) config_train = TrainConfig(**comb["Train"]) mask = Mask( diff --git a/tests/test_detectors/test_logbert_detector.py b/tests/test_detectors/test_logbert_detector.py new file mode 100644 index 00000000..ecd21a1c --- /dev/null +++ b/tests/test_detectors/test_logbert_detector.py @@ -0,0 +1,87 @@ +""" +Main tests are done at low level in the deeplearning module as LogBertDetector is an interface. + +Tests at this level are only end to end pipeline. +""" + +from detectmatelibrary.detectors.logbert_detector import LogBertDetector + +import detectmatelibrary.schemas as schemas + +import pytest + + +class TestDeeplog: + @pytest.mark.ignored + def test_end2end_autoconfig(self) -> None: + config = { + "detectors": { + "LogBertDetector": { + "method_type": "logbert_detector", + "auto_config": True, + "data_use_configure": 10, + "data_use_training": 10, + "window_size": 4, + } + } + } + + logbert = LogBertDetector(config=config) + assert logbert.get_state() == "Default" + + for j in range(2): + for i in [1, 2, 3, 4, 5, 1, 2]: + logbert.process(schemas.ParserSchema({"EventID": i})) + + if j == 0: + assert logbert.get_state() == "Configuring" + + assert logbert.get_state() == "Training" + + for _ in range(2): + for i in [1, 2, 3, 4, 5, 1, 2]: + logbert.process(schemas.ParserSchema({"EventID": i})) + assert logbert.get_state() == "Default" + + @pytest.mark.ignored + def test_end2end_no_autoconfig(self) -> None: + config = { + "detectors": { + "LogBertDetector": { + "method_type": "logbert_detector", + "auto_config": False, + "data_use_training": 10, + "window_size": 4, + "hyperparameters": { + "Model": { + "hidden": 256, + "num_heads": 2, + "n_layers": 4, + "dropout": 0.0, + "max_len": 1000, + }, + "Train": { + "seed": 0, + "batch_size": 256, + "learning_rate": 0.01, + "epochs": 10, + "mask_per": 0.4, + "alpha": 0.0, + "patience": 3, + }, + "Finetune": [], + } + } + } + } + + logbert = LogBertDetector(config=config) + assert logbert.get_state() == "Default" + + for j in range(2): + for i in [1, 2, 3, 4, 5, 1, 2]: + logbert.process(schemas.ParserSchema({"EventID": i})) + if j == 0: + assert logbert.get_state() == "Training" + + assert logbert.get_state() == "Default" From a626f89640aac7c59eb73c6bb0ac3f5875017db2 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 28 Jul 2026 09:24:13 +0200 Subject: [PATCH 20/26] add deeplog detector --- docs/detectors.md | 3 ++ docs/detectors/deeplog.md | 63 +++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 67 insertions(+) create mode 100644 docs/detectors/deeplog.md diff --git a/docs/detectors.md b/docs/detectors.md index e3e744a3..a9ff709b 100644 --- a/docs/detectors.md +++ b/docs/detectors.md @@ -92,6 +92,7 @@ List of detectors: * [Rule Based](detectors/rule_based.md): Detect anomalies based in a set of rules. * [Bigram Frequency](detectors/bigram_frequency.md): Detect bigram-frequency-based anomalies in the logs. * [Charset](detectors/charset.md): Detect new characters in the variables in the logs. +* [Deeplog](detectors/deeplog.md): Detect anomalies of a sequence of evend IDs with a LSTM. ## Configuration @@ -102,6 +103,8 @@ detectors: NewValueDetector: method_type: new_value_detector auto_config: False + data_use_configure: None # Data use for configuration + data_use_training: 199 # Data use for training params: {} # global parameters events: # event-specific configuration 1: # event_id diff --git a/docs/detectors/deeplog.md b/docs/detectors/deeplog.md new file mode 100644 index 00000000..a8ed225c --- /dev/null +++ b/docs/detectors/deeplog.md @@ -0,0 +1,63 @@ +# Deeplog Detector + +The Deeplog Detector is inspired from [Deeplog paper](https://dl.acm.org/doi/10.1145/3133956.3134015). + +| | Schema | Description | +|------------|----------------------------|--------------------| +| **Input** | [ParserSchema](../schemas.md) | Structured log | +| **Output** | [DetectorSchema](../schemas.md) | Combined alert / finding | + +## Description +Deep learning method that look at the event ID sequence + +## Configuration + +```yaml +detectors: + DeeplogDetector: + method_type: deeplog_detector + auto_config: False + data_use_training: 10 + window_size: 3 + hyperparameters: + Model: + hidden_dim: 64 + n_layers: 2 + Train: + seed: 0 + batch_size: 2048 + learning_rate: 0.01 + epochs: 10 + patience: 3 + Finetune: + - ["Model", "hidden_dim", [128, 256, 512]] + - ["Model", "n_layers", [1, 2, 3]] + - ["Train", "learning_rate", [0.01, 0.02, 0.03]] +``` + +## Example usage + +```python +from detectmatelibrary.detectors.deeplog_detector import DeeplogDetector + +import detectmatelibrary.schemas as schemas + +detector = DeeplogDetector(name="DeeplogDetector", config=cfg) + +test_data = schemas.ParserSchema({ + "parserType": "test", + "EventID": 12, + "template": "test template", + "variables": ["adsasd", "asdasd"], + "logID": "2", + "parsedLogID": "2", + "parserID": "test_parser", + "log": "test log message", + "logFormatVariables": {"level": "CRITICAL"} +}) +output = schemas.DetectorSchema() + +result = detector.detect(test_data, output) + +``` +Go back [Index](../index.md) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 49b0eceb..1333a63a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - BiGram Frequency: detectors/bigram_frequency.md - CharSet: detectors/charset.md - Value Range: detectors/value_range.md + - Deeplog Detector: detectors/deeplog.md - Alert Aggregation Methods: - Basic Concat: alert_aggregators/basic_concatenation.md - Auxiliar: From 81ecfd2c26839cf62e64fecbfd052f624f23a633 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Tue, 28 Jul 2026 09:32:43 +0200 Subject: [PATCH 21/26] add logbert documentation --- docs/detectors.md | 1 + docs/detectors/logbert.md | 68 +++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 70 insertions(+) create mode 100644 docs/detectors/logbert.md diff --git a/docs/detectors.md b/docs/detectors.md index a9ff709b..385c90aa 100644 --- a/docs/detectors.md +++ b/docs/detectors.md @@ -93,6 +93,7 @@ List of detectors: * [Bigram Frequency](detectors/bigram_frequency.md): Detect bigram-frequency-based anomalies in the logs. * [Charset](detectors/charset.md): Detect new characters in the variables in the logs. * [Deeplog](detectors/deeplog.md): Detect anomalies of a sequence of evend IDs with a LSTM. +* [LogBert](detectors/logbert.md): Detect anomalies of a sequence of evend IDs with a Transformer. ## Configuration diff --git a/docs/detectors/logbert.md b/docs/detectors/logbert.md new file mode 100644 index 00000000..6e42444c --- /dev/null +++ b/docs/detectors/logbert.md @@ -0,0 +1,68 @@ +# LogBert Detector + +The LogBert Detector is inspired from [LogBert paper](https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=9534113). + +| | Schema | Description | +|------------|----------------------------|--------------------| +| **Input** | [ParserSchema](../schemas.md) | Structured log | +| **Output** | [DetectorSchema](../schemas.md) | Combined alert / finding | + +## Description +Deep learning method that look at the event ID sequence + +## Configuration + +```yaml +detectors: + LogBertDetector: + method_type: logbert_detector + auto_config: False + data_use_training: 10 + window_size: 4 + hyperparameters: + Model: + hidden: 32 + num_heads: 2 + n_layers: 1 + dropout: 0.0 + max_len: 1000 + Train: + seed: 0 + batch_size: 256 + learning_rate: 0.01 + epochs: 10 + mask_per: 0.4 + alpha: 0.0 + patience: 3 + Finetune: + - ["Model", "hidden", [64, 128, 256]] + - ["Model", "n_layers", [1, 2, 3]] + - ["Train", "learning_rate", [0.002, 0.001, 0.005]] +``` + +## Example usage + +```python +from detectmatelibrary.detectors.logbert_detector import LogBertDetector + +import detectmatelibrary.schemas as schemas + +detector = LogBertDetector(name="LogBertDetector", config=cfg) + +test_data = schemas.ParserSchema({ + "parserType": "test", + "EventID": 12, + "template": "test template", + "variables": ["adsasd", "asdasd"], + "logID": "2", + "parsedLogID": "2", + "parserID": "test_parser", + "log": "test log message", + "logFormatVariables": {"level": "CRITICAL"} +}) +output = schemas.DetectorSchema() + +result = detector.detect(test_data, output) + +``` +Go back [Index](../index.md) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 1333a63a..e4e268ca 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,7 @@ nav: - CharSet: detectors/charset.md - Value Range: detectors/value_range.md - Deeplog Detector: detectors/deeplog.md + - LogBert Detector: detectors/logbert.md - Alert Aggregation Methods: - Basic Concat: alert_aggregators/basic_concatenation.md - Auxiliar: From 9473d9facea6d3c43fc5959ba0b1e08eeb4f19be Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Wed, 5 Aug 2026 10:08:04 +0200 Subject: [PATCH 22/26] add end to end in deeplog --- tests/test_detectors/test_deeplog_detector.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/test_detectors/test_deeplog_detector.py b/tests/test_detectors/test_deeplog_detector.py index 24ec186e..c4b9d8ad 100644 --- a/tests/test_detectors/test_deeplog_detector.py +++ b/tests/test_detectors/test_deeplog_detector.py @@ -5,6 +5,10 @@ """ from detectmatelibrary.detectors.deeplog_detector import DeeplogDetector +from detectmatelibrary.parsers.template_matcher import MatcherParser +from detectmatelibrary.helper.from_to import From + +from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL import detectmatelibrary.schemas as schemas @@ -80,3 +84,65 @@ def test_end2end_no_autoconfig(self) -> None: assert deeplog.get_state() == "Training" assert deeplog.get_state() == "Default" + + +PIPELINE_CONFIG = { + "parsers": { + "MatcherParser": { + "method_type": "matcher_parser", + "auto_config": False, + "log_format": "type= msg=audit(