From 2167b789ab9bb89bfc41df8a64db632b06f4cd83 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Thu, 24 Apr 2025 18:24:12 +0000 Subject: [PATCH 01/72] Start work on a libpressio codec wrapper --- Cargo.toml | 2 + codecs/pressio/Cargo.toml | 27 +++++++++++++ codecs/pressio/LICENSE | 1 + codecs/pressio/README.md | 38 ++++++++++++++++++ codecs/pressio/src/lib.rs | 82 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100644 codecs/pressio/Cargo.toml create mode 120000 codecs/pressio/LICENSE create mode 100644 codecs/pressio/README.md create mode 100644 codecs/pressio/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 322b2f7fb..bd6bbed89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ members = [ "codecs/log", "codecs/onion", "codecs/pco", + "codecs/pressio", "codecs/qpet-sperr", "codecs/random-projection", "codecs/reinterpret", @@ -73,6 +74,7 @@ numcodecs-linear-quantize = { version = "0.5", path = "codecs/linear-quantize", numcodecs-log = { version = "0.5", path = "codecs/log", default-features = false } numcodecs-onion = { version = "0.1", path = "codecs/onion", default-features = false } numcodecs-pco = { version = "0.4", path = "codecs/pco", default-features = false } +numcodecs-pressio = { version = "0.1", path = "codecs/pressio", default-features = false } numcodecs-qpet-sperr = { version = "0.2.2", path = "codecs/qpet-sperr", default-features = false } numcodecs-random-projection = { version = "0.4", path = "codecs/random-projection", default-features = false } numcodecs-reinterpret = { version = "0.4", path = "codecs/reinterpret", default-features = false } diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml new file mode 100644 index 000000000..2a8830c3f --- /dev/null +++ b/codecs/pressio/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "numcodecs-pressio" +version = "0.1.0" +edition = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +rust-version = { workspace = true } + +description = "libpressio codec wrapper for the numcodecs API" +readme = "README.md" +categories = ["compression", "encoding"] +keywords = ["libpressio", "numcodecs", "compression", "encoding"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +numcodecs = { workspace = true } +schemars = { workspace = true, features = ["derive", "preserve_order"] } +serde = { workspace = true, features = ["std", "derive"] } +thiserror = { workspace = true } + +# FIXME: move into workspace dependencies +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "ddceba6" } + +[lints] +workspace = true diff --git a/codecs/pressio/LICENSE b/codecs/pressio/LICENSE new file mode 120000 index 000000000..30cff7403 --- /dev/null +++ b/codecs/pressio/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/codecs/pressio/README.md b/codecs/pressio/README.md new file mode 100644 index 000000000..2a211f673 --- /dev/null +++ b/codecs/pressio/README.md @@ -0,0 +1,38 @@ +[![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![PyPi Release]][pypi] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] [![Read the Docs]][rtdocs] + +[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain + +[MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue +[repo]: https://github.com/juntyr/numcodecs-rs + +[Latest Version]: https://img.shields.io/crates/v/numcodecs-pressio +[crates.io]: https://crates.io/crates/numcodecs-pressio + +[PyPi Release]: https://img.shields.io/pypi/v/numcodecs-wasm-pressio.svg +[pypi]: https://pypi.python.org/pypi/numcodecs-wasm-pressio + +[Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-pressio +[docs.rs]: https://docs.rs/numcodecs-pressio/ + +[Rust Doc Main]: https://img.shields.io/badge/docs-main-blue +[docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_pressio + +[Read the Docs]: https://img.shields.io/readthedocs/numcodecs-wasm?label=readthedocs +[rtdocs]: https://numcodecs-wasm.readthedocs.io/en/stable/api/numcodecs_wasm_pressio/ + +# numcodecs-pressio + +libpressio codec wrapper for the [`numcodecs`] API. + +[`numcodecs`]: https://docs.rs/numcodecs/0.2/numcodecs/ + +## License + +Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/). + +## Funding + +The `numcodecs-pressio` crate has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe. + +Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054. diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs new file mode 100644 index 000000000..cfe500cb9 --- /dev/null +++ b/codecs/pressio/src/lib.rs @@ -0,0 +1,82 @@ +//! [![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] +//! +//! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +//! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain +//! +//! [MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue +//! [repo]: https://github.com/juntyr/numcodecs-rs +//! +//! [Latest Version]: https://img.shields.io/crates/v/numcodecs-pressio +//! [crates.io]: https://crates.io/crates/numcodecs-pressio +//! +//! [Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-pressio +//! [docs.rs]: https://docs.rs/numcodecs-pressio/ +//! +//! [Rust Doc Main]: https://img.shields.io/badge/docs-main-blue +//! [docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_pressio +//! +//! libpressio codec wrapper for the [`numcodecs`] API. + +use numcodecs::{ + AnyArray, AnyArrayAssignError, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, + StaticCodecConfig, StaticCodecVersion, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +/// Identity codec which applies the identity function, i.e. passes through the +/// input unchanged during encoding and decoding. +pub struct IdentityCodec { + /// The codec's encoding format version. Do not provide this parameter explicitly. + #[serde(default, rename = "_version")] + pub version: StaticCodecVersion<1, 0, 0>, +} + +impl Codec for IdentityCodec { + type Error = IdentityCodecError; + + fn encode(&self, data: AnyCowArray) -> Result { + Ok(data.into_owned()) + } + + fn decode(&self, encoded: AnyCowArray) -> Result { + Ok(encoded.into_owned()) + } + + fn decode_into( + &self, + encoded: AnyArrayView, + mut decoded: AnyArrayViewMut, + ) -> Result<(), Self::Error> { + Ok(decoded.assign(&encoded)?) + } +} + +impl StaticCodec for IdentityCodec { + const CODEC_ID: &'static str = "identity.rs"; + + type Config<'de> = Self; + + fn from_config(config: Self::Config<'_>) -> Self { + config + } + + fn get_config(&self) -> StaticCodecConfig { + StaticCodecConfig::from(self) + } +} + +#[derive(Debug, Error)] +/// Errors that may occur when applying the [`IdentityCodec`]. +pub enum IdentityCodecError { + /// [`IdentityCodec`] cannot decode into the provided array + #[error("Identity cannot decode into the provided array")] + MismatchedDecodeIntoArray { + /// The source of the error + #[from] + source: AnyArrayAssignError, + }, +} From ce13ba6ee363681c2d98f77c78cabcba0a86e48e Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Fri, 25 Apr 2025 07:50:21 +0000 Subject: [PATCH 02/72] Try with some libpressio fixes --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 2a8830c3f..bec78ed0a 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -21,7 +21,7 @@ serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } # FIXME: move into workspace dependencies -libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "ddceba6" } +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "82afbea" } [lints] workspace = true From b2f1667da2fc99236bc71f700b2fae31b66d3069 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Fri, 25 Apr 2025 08:10:37 +0000 Subject: [PATCH 03/72] Skip numcodecs-pressio in non-builder WASM CI --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c64cfe95a..1ea7c8fe8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,7 @@ jobs: --exclude numcodecs-ebcc \ --exclude numcodecs-jpeg2000 \ --exclude numcodecs-lc \ + --exclude numcodecs-pressio \ --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ @@ -269,6 +270,7 @@ jobs: --exclude numcodecs-ebcc \ --exclude numcodecs-jpeg2000 \ --exclude numcodecs-lc \ + --exclude numcodecs-pressio \ --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ @@ -288,6 +290,7 @@ jobs: --exclude numcodecs-ebcc \ --exclude numcodecs-jpeg2000 \ --exclude numcodecs-lc \ + --exclude numcodecs-pressio \ --exclude numcodecs-qpet-sperr \ --exclude numcodecs-sperr \ --exclude numcodecs-sz3 \ From f250c1dd35388b01438ee411b1add9479f088940 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 25 Apr 2025 12:12:29 +0300 Subject: [PATCH 04/72] Update libpressio rev --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index bec78ed0a..b3b7233ef 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -21,7 +21,7 @@ serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } # FIXME: move into workspace dependencies -libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "82afbea" } +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "6d25c06", default-features = false } [lints] workspace = true From 179acfde1cd2491a5fc1cc3c5f17f2b65d941f24 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 25 Apr 2025 12:29:15 +0300 Subject: [PATCH 05/72] Fix clippy lints --- codecs/pressio/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index cfe500cb9..4026c773d 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,6 +17,7 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. +use ::libpressio as _; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, From fe7168cba185713a7fb4f537062630dc2d1493e5 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 10:56:19 +0200 Subject: [PATCH 06/72] Some WASM hacks --- Cargo.toml | 1 + codecs/pressio/Cargo.toml | 4 +--- codecs/pressio/src/lib.rs | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bd6bbed89..d831b9099 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,6 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "cad5bcc", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index b3b7233ef..d4cc6179a 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -15,13 +15,11 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +libpressio = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } -# FIXME: move into workspace dependencies -libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "6d25c06", default-features = false } - [lints] workspace = true diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 4026c773d..814f054be 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -28,16 +28,16 @@ use thiserror::Error; #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] -/// Identity codec which applies the identity function, i.e. passes through the +/// Pressio codec which applies the identity function, i.e. passes through the /// input unchanged during encoding and decoding. -pub struct IdentityCodec { +pub struct PressioCodec { /// The codec's encoding format version. Do not provide this parameter explicitly. #[serde(default, rename = "_version")] pub version: StaticCodecVersion<1, 0, 0>, } -impl Codec for IdentityCodec { - type Error = IdentityCodecError; +impl Codec for PressioCodec { + type Error = PressioCodecError; fn encode(&self, data: AnyCowArray) -> Result { Ok(data.into_owned()) @@ -56,8 +56,8 @@ impl Codec for IdentityCodec { } } -impl StaticCodec for IdentityCodec { - const CODEC_ID: &'static str = "identity.rs"; +impl StaticCodec for PressioCodec { + const CODEC_ID: &'static str = "pressio.rs"; type Config<'de> = Self; @@ -65,16 +65,16 @@ impl StaticCodec for IdentityCodec { config } - fn get_config(&self) -> StaticCodecConfig { + fn get_config(&self) -> StaticCodecConfig<'_, Self> { StaticCodecConfig::from(self) } } #[derive(Debug, Error)] -/// Errors that may occur when applying the [`IdentityCodec`]. -pub enum IdentityCodecError { - /// [`IdentityCodec`] cannot decode into the provided array - #[error("Identity cannot decode into the provided array")] +/// Errors that may occur when applying the [`PressioCodec`]. +pub enum PressioCodecError { + /// [`PressioCodec`] cannot decode into the provided array + #[error("Pressio cannot decode into the provided array")] MismatchedDecodeIntoArray { /// The source of the error #[from] From bba64e48eee1beea9d7ef0a2a4fec30ea37124b0 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 11:03:24 +0200 Subject: [PATCH 07/72] Clean up libpressio dependencies --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d831b9099..0db29ebe7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "cad5bcc", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "3a00062", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 9c8d129eed3d9bf37eae2a482d590fb21eaabd1c Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 12:25:26 +0200 Subject: [PATCH 08/72] Some experimentation to produce more link errors --- codecs/pressio/src/lib.rs | 183 ++++++++++++++++++++++++++++++++++---- 1 file changed, 164 insertions(+), 19 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 814f054be..d35129594 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,42 +17,191 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. -use ::libpressio as _; +use std::{borrow::Cow, collections::BTreeMap, sync::LazyLock}; + use numcodecs::{ - AnyArray, AnyArrayAssignError, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, - StaticCodecConfig, StaticCodecVersion, + AnyArray, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, + StaticCodecVersion, }; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use schemars::{JsonSchema, Schema, SchemaGenerator}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; +static PRESSIO: LazyLock = LazyLock::new(Pressio::new); + #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] /// Pressio codec which applies the identity function, i.e. passes through the /// input unchanged during encoding and decoding. pub struct PressioCodec { + /// The Pressio compressor + #[serde(flatten)] + pub compressor: PressioCompressor, /// The codec's encoding format version. Do not provide this parameter explicitly. #[serde(default, rename = "_version")] pub version: StaticCodecVersion<1, 0, 0>, } +/// Pressio compressor +pub struct PressioCompressor { + format: PressioCompressorFormat, + compressor: libpressio::PressioCompressor, +} + +// FIXME: UNSOUND +#[expect(unsafe_code, clippy::non_send_fields_in_send_ty)] +unsafe impl Send for PressioCompressor {} +#[expect(unsafe_code)] +unsafe impl Sync for PressioCompressor {} + +impl Clone for PressioCompressor { + #[expect(clippy::unwrap_used)] + fn clone(&self) -> Self { + let pressio = PRESSIO.get_or_unwrap(); + let compressor = pressio.get_compressor(self.format.id.as_str()).unwrap(); + let options = self.compressor.get_options().unwrap(); + compressor.set_options(&options).unwrap(); + + Self { + format: self.format.clone(), + compressor, + } + } +} + +impl Serialize for PressioCompressor { + fn serialize(&self, serializer: S) -> Result { + self.format.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for PressioCompressor { + fn deserialize>(deserializer: D) -> Result { + let pressio = PRESSIO + .get() + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + + // TODO: better error handling + let format = PressioCompressorFormat::deserialize(deserializer)?; + + let compressor = pressio + .get_compressor(format.id.as_str()) + .map_err(|err| serde::de::Error::custom(err.message))?; + let mut options = compressor + .get_options() + .map_err(|err| serde::de::Error::custom(err.message))?; + + for (key, value) in &format.options { + options = options + .set( + key, + match value { + PressioOption::U8(x) => libpressio::PressioOption::uint8(Some(*x)), + PressioOption::I8(x) => libpressio::PressioOption::int8(Some(*x)), + PressioOption::U16(x) => libpressio::PressioOption::uint16(Some(*x)), + PressioOption::I16(x) => libpressio::PressioOption::int16(Some(*x)), + PressioOption::U32(x) => libpressio::PressioOption::uint32(Some(*x)), + PressioOption::I32(x) => libpressio::PressioOption::int32(Some(*x)), + PressioOption::U64(x) => libpressio::PressioOption::uint64(Some(*x)), + PressioOption::I64(x) => libpressio::PressioOption::int64(Some(*x)), + PressioOption::F32(x) => libpressio::PressioOption::float32(Some(*x)), + PressioOption::F64(x) => libpressio::PressioOption::float64(Some(*x)), + PressioOption::String(x) => { + libpressio::PressioOption::string(Some(x.clone())) + } + PressioOption::VecString(x) => { + libpressio::PressioOption::vec_string(Some(x.clone())) + } + }, + ) + .map_err(|err| serde::de::Error::custom(err.message))?; + } + + Ok(Self { format, compressor }) + } +} + +impl JsonSchema for PressioCompressor { + fn schema_name() -> Cow<'static, str> { + PressioCompressorFormat::schema_name() + } + + fn json_schema(generator: &mut SchemaGenerator) -> Schema { + PressioCompressorFormat::json_schema(generator) + } +} + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename = "PressioCompressor")] +struct PressioCompressorFormat { + id: String, + #[serde(flatten)] + options: BTreeMap, +} + +#[expect(missing_docs)] +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(untagged)] +/// Pressio option value +pub enum PressioOption { + U8(u8), + I8(i8), + U16(u16), + I16(i16), + U32(u32), + I32(i32), + U64(u64), + I64(i64), + F32(f32), + F64(f64), + String(String), + VecString(Vec), +} + +struct Pressio { + pressio: Result, +} + +impl Pressio { + fn new() -> Self { + Self { + pressio: libpressio::Pressio::new(), + } + } + + const fn get(&self) -> Result<&libpressio::Pressio, &libpressio::PressioError> { + self.pressio.as_ref() + } + + #[expect(clippy::unwrap_used)] + fn get_or_unwrap(&self) -> &libpressio::Pressio { + self.pressio.as_ref().unwrap() + } +} + +// FIXME: UNSOUND +#[expect(unsafe_code, clippy::non_send_fields_in_send_ty)] +unsafe impl Send for Pressio {} +#[expect(unsafe_code)] +unsafe impl Sync for Pressio {} + impl Codec for PressioCodec { type Error = PressioCodecError; - fn encode(&self, data: AnyCowArray) -> Result { - Ok(data.into_owned()) + fn encode(&self, _data: AnyCowArray) -> Result { + Err(PressioCodecError::Unimplemented) } - fn decode(&self, encoded: AnyCowArray) -> Result { - Ok(encoded.into_owned()) + fn decode(&self, _encoded: AnyCowArray) -> Result { + Err(PressioCodecError::Unimplemented) } fn decode_into( &self, - encoded: AnyArrayView, - mut decoded: AnyArrayViewMut, + _encoded: AnyArrayView, + _decoded: AnyArrayViewMut, ) -> Result<(), Self::Error> { - Ok(decoded.assign(&encoded)?) + Err(PressioCodecError::Unimplemented) } } @@ -73,11 +222,7 @@ impl StaticCodec for PressioCodec { #[derive(Debug, Error)] /// Errors that may occur when applying the [`PressioCodec`]. pub enum PressioCodecError { - /// [`PressioCodec`] cannot decode into the provided array - #[error("Pressio cannot decode into the provided array")] - MismatchedDecodeIntoArray { - /// The source of the error - #[from] - source: AnyArrayAssignError, - }, + /// [`PressioCodec`] does not yet implement this functionality + #[error("Pressio does not yet implement this functionality")] + Unimplemented, } From 364f9cfee1a4ab08d38a9ef392897c89d18c776e Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 19:03:37 +0200 Subject: [PATCH 09/72] Upgrade to upstream std_compat --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 0db29ebe7..83b18e63d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "3a00062", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5d91490", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From ab750eb026dff0cddc4036f8d379d511d423daf0 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 21:19:54 +0200 Subject: [PATCH 10/72] Only link libstdc++ in libpressio on Linux target --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 83b18e63d..cecc8b6cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5d91490", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "818f30d", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index d35129594..e69c734b5 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -226,3 +226,8 @@ pub enum PressioCodecError { #[error("Pressio does not yet implement this functionality")] Unimplemented, } + +// FIXME: don't stub +#[expect(unsafe_code)] +#[unsafe(no_mangle)] +const extern "C" fn pressio_register_all() {} From 789fdbab1f8e9f0d59152c310d203dd6c537e6e5 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Feb 2026 22:15:54 +0200 Subject: [PATCH 11/72] Fix target_os check in libpressio build.rs --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index cecc8b6cf..18dd37d59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "818f30d", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a1f99af", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 9866cee12c6bc399b152e0a6a1a1c486d498c175 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 17 Feb 2026 09:50:48 +0200 Subject: [PATCH 12/72] no really really don't ask --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 18dd37d59..b3e3949ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a1f99af", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a664de1", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index e69c734b5..d35129594 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -226,8 +226,3 @@ pub enum PressioCodecError { #[error("Pressio does not yet implement this functionality")] Unimplemented, } - -// FIXME: don't stub -#[expect(unsafe_code)] -#[unsafe(no_mangle)] -const extern "C" fn pressio_register_all() {} From 9d89e525e71459e6d2d3d9193b27d53c6ba1e0a7 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 17 Feb 2026 14:19:26 +0200 Subject: [PATCH 13/72] add some option parsing and inspection support --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 67 +++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b3e3949ec..ab11cc72f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a664de1", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "435c82a", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index d35129594..40bcbbeda 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -30,7 +30,7 @@ use thiserror::Error; static PRESSIO: LazyLock = LazyLock::new(Pressio::new); #[derive(Clone, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] /// Pressio codec which applies the identity function, i.e. passes through the /// input unchanged during encoding and decoding. pub struct PressioCodec { @@ -58,7 +58,9 @@ impl Clone for PressioCompressor { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { let pressio = PRESSIO.get_or_unwrap(); - let compressor = pressio.get_compressor(self.format.id.as_str()).unwrap(); + let compressor = pressio + .get_compressor(self.format.compressor.as_str()) + .unwrap(); let options = self.compressor.get_options().unwrap(); compressor.set_options(&options).unwrap(); @@ -80,13 +82,26 @@ impl<'de> Deserialize<'de> for PressioCompressor { let pressio = PRESSIO .get() .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; - // TODO: better error handling let format = PressioCompressorFormat::deserialize(deserializer)?; - let compressor = pressio - .get_compressor(format.id.as_str()) - .map_err(|err| serde::de::Error::custom(err.message))?; + .get_compressor(format.compressor.as_str()) + .map_err(|err| { + let supported_compressors = + pressio + .supported_compressors() + .map_or(String::from(""), |x| { + x.iter() + .map(|x| format!("`{x}`")) + .collect::>() + .join(", ") + }); + + serde::de::Error::custom(format_args!( + "{}, choose one of: {}", + err.message, supported_compressors + )) + })?; let mut options = compressor .get_options() .map_err(|err| serde::de::Error::custom(err.message))?; @@ -96,6 +111,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { .set( key, match value { + PressioOption::Bool(x) => libpressio::PressioOption::bool(Some(*x)), PressioOption::U8(x) => libpressio::PressioOption::uint8(Some(*x)), PressioOption::I8(x) => libpressio::PressioOption::int8(Some(*x)), PressioOption::U16(x) => libpressio::PressioOption::uint16(Some(*x)), @@ -117,6 +133,35 @@ impl<'de> Deserialize<'de> for PressioCompressor { .map_err(|err| serde::de::Error::custom(err.message))?; } + let mut format = format; + if let Ok(format_options) = options.get_options() { + format.options = format_options + .into_iter() + .filter_map(|(k, v)| match v { + libpressio::PressioOption::bool(Some(x)) => Some((k, PressioOption::Bool(x))), + libpressio::PressioOption::int8(Some(x)) => Some((k, PressioOption::I8(x))), + libpressio::PressioOption::int16(Some(x)) => Some((k, PressioOption::I16(x))), + libpressio::PressioOption::int32(Some(x)) => Some((k, PressioOption::I32(x))), + libpressio::PressioOption::int64(Some(x)) => Some((k, PressioOption::I64(x))), + libpressio::PressioOption::uint8(Some(x)) => Some((k, PressioOption::U8(x))), + libpressio::PressioOption::uint16(Some(x)) => Some((k, PressioOption::U16(x))), + libpressio::PressioOption::uint32(Some(x)) => Some((k, PressioOption::U32(x))), + libpressio::PressioOption::uint64(Some(x)) => Some((k, PressioOption::U64(x))), + libpressio::PressioOption::float32(Some(x)) => Some((k, PressioOption::F32(x))), + libpressio::PressioOption::float64(Some(x)) => Some((k, PressioOption::F64(x))), + libpressio::PressioOption::string(Some(x)) => { + Some((k, PressioOption::String(x))) + } + // FIXME: seems to return strings as a single joined string + libpressio::PressioOption::vec_string(Some(x)) => { + Some((k, PressioOption::VecString(x))) + } + _ => None, + }) + .collect(); + } + let format = format; + Ok(Self { format, compressor }) } } @@ -131,19 +176,21 @@ impl JsonSchema for PressioCompressor { } } -#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename = "PressioCompressor")] struct PressioCompressorFormat { - id: String, - #[serde(flatten)] + compressor: String, + // TODO: flatten + #[serde(default)] options: BTreeMap, } #[expect(missing_docs)] -#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] /// Pressio option value pub enum PressioOption { + Bool(bool), U8(u8), I8(i8), U16(u16), From d734f275898e2f4f0f982e5a4d46ba0c3a607b49 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 18 Feb 2026 08:43:40 +0200 Subject: [PATCH 14/72] some more plugins --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ab11cc72f..921bd5bda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "435c82a", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "e562964", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 8ca81a5bc3ebc6a99a8ce77ed0f2db84a5016ac1 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 18 Feb 2026 15:04:06 +0200 Subject: [PATCH 15/72] enable more plugins --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 921bd5bda..d33d437bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "e562964", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "f142487", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 1912be64a46ba033e2485f5927bad87248964b02 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 21 Feb 2026 06:36:13 +0200 Subject: [PATCH 16/72] Simple encode/decode implementation --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 1 + codecs/pressio/src/lib.rs | 167 +++++++++++++++++++++++++++++++++++--- 3 files changed, 157 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d33d437bf..d0e4071b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "f142487", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "4e7b44f", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index d4cc6179a..e664605e6 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -16,6 +16,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] [dependencies] libpressio = { workspace = true } +ndarray = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 40bcbbeda..56fd0f3ea 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -19,9 +19,10 @@ use std::{borrow::Cow, collections::BTreeMap, sync::LazyLock}; +use ndarray::{CowArray, IxDyn}; use numcodecs::{ - AnyArray, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, - StaticCodecVersion, + AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, + Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, }; use schemars::{JsonSchema, Schema, SchemaGenerator}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -235,20 +236,126 @@ unsafe impl Sync for Pressio {} impl Codec for PressioCodec { type Error = PressioCodecError; - fn encode(&self, _data: AnyCowArray) -> Result { - Err(PressioCodecError::Unimplemented) + fn encode(&self, data: AnyCowArray) -> Result { + fn encode_typed( + compressor: &libpressio::PressioCompressor, + data: CowArray, + ) -> Result { + let data = match data.try_into_owned_nocopy() { + Ok(data) => libpressio::PressioData::new(data), + Err(data) => libpressio::PressioData::new_copied(data.view()), + }; + + let compressed_data = + libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); + + let compressed_data = compressor.compress(&data, compressed_data).map_err(|err| { + PressioCodecError::PressioEncodeFailed { + source: PressioCodingError(err), + } + })?; + + let Some(compressed_data) = compressed_data.clone_into_array() else { + return Err(PressioCodecError::EncodeToUnknownDtype); + }; + + match compressed_data { + libpressio::PressioArray::Bool(_) => Err(PressioCodecError::EncodeToBoolArray), + libpressio::PressioArray::U8(a) | libpressio::PressioArray::Byte(a) => { + Ok(AnyArray::U8(a)) + } + libpressio::PressioArray::U16(a) => Ok(AnyArray::U16(a)), + libpressio::PressioArray::U32(a) => Ok(AnyArray::U32(a)), + libpressio::PressioArray::U64(a) => Ok(AnyArray::U64(a)), + libpressio::PressioArray::I8(a) => Ok(AnyArray::I8(a)), + libpressio::PressioArray::I16(a) => Ok(AnyArray::I16(a)), + libpressio::PressioArray::I32(a) => Ok(AnyArray::I32(a)), + libpressio::PressioArray::I64(a) => Ok(AnyArray::I64(a)), + libpressio::PressioArray::F32(a) => Ok(AnyArray::F32(a)), + libpressio::PressioArray::F64(a) => Ok(AnyArray::F64(a)), + } + } + + match data { + AnyCowArray::U8(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::U16(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::U32(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::U64(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::I8(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::I16(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::I32(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::I64(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::F32(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::F64(data) => encode_typed(&self.compressor.compressor, data), + data => Err(PressioCodecError::UnsupportedDtype(data.dtype())), + } } - fn decode(&self, _encoded: AnyCowArray) -> Result { - Err(PressioCodecError::Unimplemented) + fn decode(&self, encoded: AnyCowArray) -> Result { + fn decode_typed( + compressor: &libpressio::PressioCompressor, + encoded: CowArray, + ) -> Result { + let encoded = match encoded.try_into_owned_nocopy() { + Ok(encoded) => libpressio::PressioData::new(encoded), + Err(encoded) => libpressio::PressioData::new_copied(encoded.view()), + }; + + let decompressed_data = + libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); + + let decompressed_data = + compressor + .compress(&encoded, decompressed_data) + .map_err(|err| PressioCodecError::PressioDecodeFailed { + source: PressioCodingError(err), + })?; + + let Some(decompressed_data) = decompressed_data.clone_into_array() else { + return Err(PressioCodecError::DecodeToUnknownDtype); + }; + + match decompressed_data { + libpressio::PressioArray::Bool(_) => Err(PressioCodecError::DecodeToBoolArray), + libpressio::PressioArray::U8(a) | libpressio::PressioArray::Byte(a) => { + Ok(AnyArray::U8(a)) + } + libpressio::PressioArray::U16(a) => Ok(AnyArray::U16(a)), + libpressio::PressioArray::U32(a) => Ok(AnyArray::U32(a)), + libpressio::PressioArray::U64(a) => Ok(AnyArray::U64(a)), + libpressio::PressioArray::I8(a) => Ok(AnyArray::I8(a)), + libpressio::PressioArray::I16(a) => Ok(AnyArray::I16(a)), + libpressio::PressioArray::I32(a) => Ok(AnyArray::I32(a)), + libpressio::PressioArray::I64(a) => Ok(AnyArray::I64(a)), + libpressio::PressioArray::F32(a) => Ok(AnyArray::F32(a)), + libpressio::PressioArray::F64(a) => Ok(AnyArray::F64(a)), + } + } + + match encoded { + AnyCowArray::U8(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::U16(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::U32(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::U64(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::I8(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::I16(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::I32(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::I64(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::F32(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::F64(encoded) => decode_typed(&self.compressor.compressor, encoded), + encoded => Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), + } } fn decode_into( &self, - _encoded: AnyArrayView, - _decoded: AnyArrayViewMut, + encoded: AnyArrayView, + mut decoded: AnyArrayViewMut, ) -> Result<(), Self::Error> { - Err(PressioCodecError::Unimplemented) + // TODO: optimize + let decoded_in = self.decode(encoded.cow())?; + + Ok(decoded.assign(&decoded_in)?) } } @@ -269,7 +376,43 @@ impl StaticCodec for PressioCodec { #[derive(Debug, Error)] /// Errors that may occur when applying the [`PressioCodec`]. pub enum PressioCodecError { - /// [`PressioCodec`] does not yet implement this functionality - #[error("Pressio does not yet implement this functionality")] - Unimplemented, + /// [`PressioCodec`] does not support the dtype + #[error("Pressio does not support the dtype {0}")] + UnsupportedDtype(AnyArrayDType), + /// [`PressioCodec`] failed to encode the data + #[error("Pressio failed to encode the data")] + PressioEncodeFailed { + /// Opaque source error + source: PressioCodingError, + }, + /// [`PressioCodec`] encoded to an unknown unsupported dtype + #[error("Pressio encoded to an unknown unsupported dtype")] + EncodeToUnknownDtype, + /// [`PressioCodec`] encoded to a bool array, which is unsupported + #[error("Pressio encoded to a bool array, which is unsupported")] + EncodeToBoolArray, + /// [`PressioCodec`] failed to decode the data + #[error("Pressio failed to decode the data")] + PressioDecodeFailed { + /// Opaque source error + source: PressioCodingError, + }, + /// [`PressioCodec`] decoded to an unknown unsupported dtype + #[error("Pressio decoded to an unknown unsupported dtype")] + DecodeToUnknownDtype, + /// [`PressioCodec`] decoded to a bool array, which is unsupported + #[error("Pressio decoded to a bool array, which is unsupported")] + DecodeToBoolArray, + /// [`PressioCodec`] cannot decode into the provided array + #[error("Pressio cannot decode into the provided array")] + MismatchedDecodeIntoArray { + /// The source of the error + #[from] + source: AnyArrayAssignError, + }, } + +#[derive(Debug, Error)] +#[error(transparent)] +/// Opaque error for when encoding or decoding with libpressio fails +pub struct PressioCodingError(libpressio::PressioError); From 0e110633c58c7809c120f69059cb47e61ecb0ac6 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 23 Feb 2026 12:37:52 +0200 Subject: [PATCH 17/72] add host libcxx to the nix flake --- Cargo.toml | 2 +- .../numcodecs-wasm-builder/buildenv/flake.nix | 2 ++ crates/numcodecs-wasm-builder/src/main.rs | 20 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d0e4071b2..f89724c35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "4e7b44f", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "3b90d94", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.nix b/crates/numcodecs-wasm-builder/buildenv/flake.nix index b02efb029..3e4b8bab1 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.nix +++ b/crates/numcodecs-wasm-builder/buildenv/flake.nix @@ -61,6 +61,7 @@ packages = [ (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain) pkgs."llvmPackages_${llvmVersion}".libclang + pkgs."llvmPackages_${llvmVersion}".libcxx wasi-sysroot libclang_rt pkgs.cmake @@ -84,6 +85,7 @@ MY_WASM_OPT = "${pkgs.binaryen}/bin/wasm-opt"; MY_PKG_CONFIG = "${pkgs.pkg-config}/bin/pkg-config"; MY_PYTHON3 = "${pkgs.python3}/bin/python3"; + MY_HOST_LIBCXX = "${pkgs."llvmPackages_${llvmVersion}".libcxx}"; }; }; }); diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 9e0e12693..45f0f9934 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -212,6 +212,8 @@ struct NixEnv { pkg_config: PathBuf, #[expect(dead_code)] python3: PathBuf, + host_libcxx: PathBuf, + host_sysroot: PathBuf, } impl NixEnv { @@ -279,6 +281,11 @@ impl NixEnv { wasm_opt: try_read_env(&env, "MY_WASM_OPT")?, pkg_config: try_read_env(&env, "MY_PKG_CONFIG")?, python3: try_read_env(&env, "MY_PYTHON3")?, + host_libcxx: try_read_env(&env, "MY_HOST_LIBCXX")?, + // FIXME + host_sysroot: PathBuf::from( + "/nix/store/5gfsv5n8zhpnl9yhggjpxrxg0jyflwja-apple-sdk-11.3/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk", + ), }) } } @@ -304,6 +311,8 @@ fn configure_cargo_cmd( wasi_sysroot, libclang_rt, pkg_config, + host_libcxx, + host_sysroot, .. } = nix_env; @@ -388,6 +397,17 @@ fn configure_cargo_cmd( cpp_include_path = crate_dir.join("include.hpp").display(), debug = if debug { "-g" } else { "" }, )); + cmd.arg(format!( + "CXXFLAGSHOST=-isysroot {host_sysroot} -isystem {host_libcxx_include} \ + -isystem {clang_include}", + host_sysroot = host_sysroot.display(), + host_libcxx_include = host_libcxx.join("include").join("c++").join("v1").display(), + clang_include = libclang + .join("clang") + .join(llvm_version) + .join("include") + .display(), + )); cmd.arg(format!( "BINDGEN_EXTRA_CLANG_ARGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir \ {resource_dir} --sysroot={wasi_sysroot} -isystem {wasm32_wasi_cxx_include} -isystem \ From 53ca2a8cfd3d74168b8edc464bccb86a85b00a14 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 23 Feb 2026 13:04:04 +0200 Subject: [PATCH 18/72] fix cmake env lookup --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f89724c35..615be135d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "3b90d94", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5bc8ff0", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From c624e6a510445535b291e12b958b901d882e3ca3 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 23 Feb 2026 14:52:10 +0200 Subject: [PATCH 19/72] Fix clippy lint --- codecs/pressio/src/lib.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 56fd0f3ea..ad5a58597 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -88,15 +88,15 @@ impl<'de> Deserialize<'de> for PressioCompressor { let compressor = pressio .get_compressor(format.compressor.as_str()) .map_err(|err| { - let supported_compressors = - pressio - .supported_compressors() - .map_or(String::from(""), |x| { - x.iter() - .map(|x| format!("`{x}`")) - .collect::>() - .join(", ") - }); + let supported_compressors = pressio.supported_compressors().map_or_else( + |_| String::from(""), + |x| { + x.iter() + .map(|x| format!("`{x}`")) + .collect::>() + .join(", ") + }, + ); serde::de::Error::custom(format_args!( "{}, choose one of: {}", From bcf600a8cd827867bee84fb0bdd68ef2d4b47d69 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 1 Mar 2026 23:08:36 +0200 Subject: [PATCH 20/72] Pressio and PressioCompressor are Send + !Sync --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 148 ++++++++++++++++++++------------------ 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 615be135d..ed915987b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5bc8ff0", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "fea47ef", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index ad5a58597..40b008b06 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,7 +17,11 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. -use std::{borrow::Cow, collections::BTreeMap, sync::LazyLock}; +use std::{ + borrow::Cow, + collections::BTreeMap, + sync::{LazyLock, Mutex}, +}; use ndarray::{CowArray, IxDyn}; use numcodecs::{ @@ -46,28 +50,24 @@ pub struct PressioCodec { /// Pressio compressor pub struct PressioCompressor { format: PressioCompressorFormat, - compressor: libpressio::PressioCompressor, + compressor: Mutex, } -// FIXME: UNSOUND -#[expect(unsafe_code, clippy::non_send_fields_in_send_ty)] -unsafe impl Send for PressioCompressor {} -#[expect(unsafe_code)] -unsafe impl Sync for PressioCompressor {} - impl Clone for PressioCompressor { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { - let pressio = PRESSIO.get_or_unwrap(); - let compressor = pressio - .get_compressor(self.format.compressor.as_str()) - .unwrap(); - let options = self.compressor.get_options().unwrap(); + let mut compressor = { + let mut pressio = PRESSIO.get_or_unwrap().lock().unwrap(); + pressio + .get_compressor(self.format.compressor.as_str()) + .unwrap() + }; + let options = self.compressor.lock().unwrap().get_options().unwrap(); compressor.set_options(&options).unwrap(); Self { format: self.format.clone(), - compressor, + compressor: Mutex::new(compressor), } } } @@ -80,29 +80,32 @@ impl Serialize for PressioCompressor { impl<'de> Deserialize<'de> for PressioCompressor { fn deserialize>(deserializer: D) -> Result { - let pressio = PRESSIO - .get() - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; // TODO: better error handling let format = PressioCompressorFormat::deserialize(deserializer)?; - let compressor = pressio - .get_compressor(format.compressor.as_str()) - .map_err(|err| { - let supported_compressors = pressio.supported_compressors().map_or_else( - |_| String::from(""), - |x| { - x.iter() - .map(|x| format!("`{x}`")) - .collect::>() - .join(", ") - }, - ); - - serde::de::Error::custom(format_args!( - "{}, choose one of: {}", - err.message, supported_compressors - )) - })?; + let compressor = { + let pressio = PRESSIO + .get() + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + let mut pressio = pressio.lock().map_err(serde::de::Error::custom)?; + pressio + .get_compressor(format.compressor.as_str()) + .map_err(|err| { + let supported_compressors = pressio.supported_compressors().map_or_else( + |_| String::from(""), + |x| { + x.iter() + .map(|x| format!("`{x}`")) + .collect::>() + .join(", ") + }, + ); + + serde::de::Error::custom(format_args!( + "{}, choose one of: {}", + err.message, supported_compressors + )) + })? + }; let mut options = compressor .get_options() .map_err(|err| serde::de::Error::custom(err.message))?; @@ -161,9 +164,11 @@ impl<'de> Deserialize<'de> for PressioCompressor { }) .collect(); } - let format = format; - Ok(Self { format, compressor }) + Ok(Self { + format, + compressor: Mutex::new(compressor), + }) } } @@ -207,38 +212,32 @@ pub enum PressioOption { } struct Pressio { - pressio: Result, + pressio: Result, libpressio::PressioError>, } impl Pressio { fn new() -> Self { Self { - pressio: libpressio::Pressio::new(), + pressio: libpressio::Pressio::new().map(Mutex::new), } } - const fn get(&self) -> Result<&libpressio::Pressio, &libpressio::PressioError> { + const fn get(&self) -> Result<&Mutex, &libpressio::PressioError> { self.pressio.as_ref() } #[expect(clippy::unwrap_used)] - fn get_or_unwrap(&self) -> &libpressio::Pressio { + fn get_or_unwrap(&self) -> &Mutex { self.pressio.as_ref().unwrap() } } -// FIXME: UNSOUND -#[expect(unsafe_code, clippy::non_send_fields_in_send_ty)] -unsafe impl Send for Pressio {} -#[expect(unsafe_code)] -unsafe impl Sync for Pressio {} - impl Codec for PressioCodec { type Error = PressioCodecError; fn encode(&self, data: AnyCowArray) -> Result { fn encode_typed( - compressor: &libpressio::PressioCompressor, + compressor: &mut libpressio::PressioCompressor, data: CowArray, ) -> Result { let data = match data.try_into_owned_nocopy() { @@ -276,24 +275,28 @@ impl Codec for PressioCodec { } } + let Ok(mut compressor) = self.compressor.compressor.lock() else { + return Err(PressioCodecError::PressioPoisonedMutex); + }; + match data { - AnyCowArray::U8(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::U16(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::U32(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::U64(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::I8(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::I16(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::I32(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::I64(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::F32(data) => encode_typed(&self.compressor.compressor, data), - AnyCowArray::F64(data) => encode_typed(&self.compressor.compressor, data), + AnyCowArray::U8(data) => encode_typed(&mut compressor, data), + AnyCowArray::U16(data) => encode_typed(&mut compressor, data), + AnyCowArray::U32(data) => encode_typed(&mut compressor, data), + AnyCowArray::U64(data) => encode_typed(&mut compressor, data), + AnyCowArray::I8(data) => encode_typed(&mut compressor, data), + AnyCowArray::I16(data) => encode_typed(&mut compressor, data), + AnyCowArray::I32(data) => encode_typed(&mut compressor, data), + AnyCowArray::I64(data) => encode_typed(&mut compressor, data), + AnyCowArray::F32(data) => encode_typed(&mut compressor, data), + AnyCowArray::F64(data) => encode_typed(&mut compressor, data), data => Err(PressioCodecError::UnsupportedDtype(data.dtype())), } } fn decode(&self, encoded: AnyCowArray) -> Result { fn decode_typed( - compressor: &libpressio::PressioCompressor, + compressor: &mut libpressio::PressioCompressor, encoded: CowArray, ) -> Result { let encoded = match encoded.try_into_owned_nocopy() { @@ -332,17 +335,21 @@ impl Codec for PressioCodec { } } + let Ok(mut compressor) = self.compressor.compressor.lock() else { + return Err(PressioCodecError::PressioPoisonedMutex); + }; + match encoded { - AnyCowArray::U8(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::U16(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::U32(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::U64(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::I8(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::I16(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::I32(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::I64(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::F32(encoded) => decode_typed(&self.compressor.compressor, encoded), - AnyCowArray::F64(encoded) => decode_typed(&self.compressor.compressor, encoded), + AnyCowArray::U8(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::U16(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::U32(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::U64(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::I8(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::I16(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::I32(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::I64(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::F32(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::F64(encoded) => decode_typed(&mut compressor, encoded), encoded => Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), } } @@ -379,6 +386,9 @@ pub enum PressioCodecError { /// [`PressioCodec`] does not support the dtype #[error("Pressio does not support the dtype {0}")] UnsupportedDtype(AnyArrayDType), + /// [`PressioCodec`] lock was poisoned + #[error("Pressio lock was poisoned")] + PressioPoisonedMutex, /// [`PressioCodec`] failed to encode the data #[error("Pressio failed to encode the data")] PressioEncodeFailed { From 0639c4e8e431f506164b6b1e84a207fc395a532f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 10:03:15 +0200 Subject: [PATCH 21/72] derive host sysroot in builder from clang include paths --- Cargo.toml | 2 +- .../numcodecs-wasm-builder/buildenv/flake.nix | 2 +- crates/numcodecs-wasm-builder/src/main.rs | 68 ++++++++++++++++--- 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ed915987b..fc1945c7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "fea47ef", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6329913", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.nix b/crates/numcodecs-wasm-builder/buildenv/flake.nix index 3e4b8bab1..22ce9af96 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.nix +++ b/crates/numcodecs-wasm-builder/buildenv/flake.nix @@ -85,7 +85,7 @@ MY_WASM_OPT = "${pkgs.binaryen}/bin/wasm-opt"; MY_PKG_CONFIG = "${pkgs.pkg-config}/bin/pkg-config"; MY_PYTHON3 = "${pkgs.python3}/bin/python3"; - MY_HOST_LIBCXX = "${pkgs."llvmPackages_${llvmVersion}".libcxx}"; + MY_HOST_LIBCXX = "${pkgs."llvmPackages_${llvmVersion}".libcxx.dev}"; }; }; }); diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 45f0f9934..3d1037eae 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -3,9 +3,12 @@ use std::{ collections::HashMap, - env, fs, io, + env, + ffi::OsStr, + fs, io, + os::unix::ffi::OsStrExt, path::{Path, PathBuf}, - process::Command, + process::{Command, Stdio}, str::FromStr, }; @@ -77,9 +80,11 @@ fn main() -> io::Result<()> { copy_buildenv_to_crate(&crate_dir)?; let nix_env = NixEnv::new(&crate_dir)?; + let host_sysroot = find_clang_host_sysroot(&nix_env, &crate_dir)?; let wasm = build_wasm_codec( &nix_env, + &host_sysroot, &target_dir, &crate_dir, &format!("{}-wasm", args.crate_), @@ -213,7 +218,6 @@ struct NixEnv { #[expect(dead_code)] python3: PathBuf, host_libcxx: PathBuf, - host_sysroot: PathBuf, } impl NixEnv { @@ -282,17 +286,63 @@ impl NixEnv { pkg_config: try_read_env(&env, "MY_PKG_CONFIG")?, python3: try_read_env(&env, "MY_PYTHON3")?, host_libcxx: try_read_env(&env, "MY_HOST_LIBCXX")?, - // FIXME - host_sysroot: PathBuf::from( - "/nix/store/5gfsv5n8zhpnl9yhggjpxrxg0jyflwja-apple-sdk-11.3/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk", - ), }) } } +fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Result { + let NixEnv { clang, .. } = nix_env; + + let mut cmd = Command::new("nix"); + cmd.current_dir(flake_parent_dir); + cmd.arg("develop"); + // cmd.arg("--store"); + // cmd.arg(nix_store_path); + cmd.arg("--no-update-lock-file"); + cmd.arg("--ignore-environment"); + cmd.arg("path:."); + cmd.arg("--command"); + cmd.arg(clang.join("clang")); + cmd.arg("-v"); + cmd.arg("-x"); + cmd.arg("c"); + cmd.arg("-c"); + cmd.arg("-"); + cmd.stdin(Stdio::null()); + + eprintln!("executing {cmd:?}"); + + let output = cmd.output()?; + let Some(include) = output + .stderr + .split(|x| *x == b'\n') + .skip_while(|x| x.trim_ascii() != b"#include <...> search starts here:") + .nth(1) + else { + return Err(io::Error::other( + "failed to find #include <...> search path for clang", + )); + }; + let include = Path::new(OsStr::from_bytes(include.trim_ascii())); + let include = if include.ends_with("include") + && let Some(include) = include.parent() + && include.ends_with("usr") + && let Some(include) = include.parent() + { + include + } else { + return Err(io::Error::other( + "clang #include <...> search path should end in /usr/include", + )); + }; + + Ok(PathBuf::from(include)) +} + #[expect(clippy::too_many_lines)] fn configure_cargo_cmd( nix_env: &NixEnv, + host_sysroot: &Path, target_dir: &Path, crate_dir: &Path, debug: bool, @@ -312,7 +362,6 @@ fn configure_cargo_cmd( libclang_rt, pkg_config, host_libcxx, - host_sysroot, .. } = nix_env; @@ -480,13 +529,14 @@ fn configure_cargo_cmd( fn build_wasm_codec( nix_env: &NixEnv, + host_sysroot: &Path, target_dir: &Path, crate_dir: &Path, crate_name: &str, debug: bool, verbose: bool, ) -> io::Result { - let mut cmd = configure_cargo_cmd(nix_env, target_dir, crate_dir, debug); + let mut cmd = configure_cargo_cmd(nix_env, host_sysroot, target_dir, crate_dir, debug); cmd.arg("rustc") .arg("--crate-type=cdylib") .arg("-Z") From 74fe2e26a5ada1ac0a070e270c65137d59509004 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 10:08:46 +0200 Subject: [PATCH 22/72] debug clang include path on Linux --- crates/numcodecs-wasm-builder/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 3d1037eae..f21ecc86a 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -313,6 +313,7 @@ fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Res eprintln!("executing {cmd:?}"); let output = cmd.output()?; + eprintln!("output={:?}", String::from_utf8_lossy(&output.stderr)); let Some(include) = output .stderr .split(|x| *x == b'\n') @@ -323,7 +324,9 @@ fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Res "failed to find #include <...> search path for clang", )); }; + eprintln!("include={:?}", String::from_utf8_lossy(include)); let include = Path::new(OsStr::from_bytes(include.trim_ascii())); + eprintln!("include={}", include.display()); let include = if include.ends_with("include") && let Some(include) = include.parent() && include.ends_with("usr") From 78da45fab87018143f70cc2d21128f5a23ce8209 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 10:22:13 +0200 Subject: [PATCH 23/72] more flexible sysroot finding --- crates/numcodecs-wasm-builder/src/main.rs | 40 +++++++++++++++-------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index f21ecc86a..7922680b8 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -313,30 +313,42 @@ fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Res eprintln!("executing {cmd:?}"); let output = cmd.output()?; - eprintln!("output={:?}", String::from_utf8_lossy(&output.stderr)); - let Some(include) = output + let Some(full_include) = output .stderr .split(|x| *x == b'\n') .skip_while(|x| x.trim_ascii() != b"#include <...> search starts here:") .nth(1) else { - return Err(io::Error::other( - "failed to find #include <...> search path for clang", - )); + return Err(io::Error::other(format!( + "failed to find #include <...> search path for clang in {:?}", + String::from_utf8_lossy(&output.stderr) + ))); }; - eprintln!("include={:?}", String::from_utf8_lossy(include)); - let include = Path::new(OsStr::from_bytes(include.trim_ascii())); - eprintln!("include={}", include.display()); - let include = if include.ends_with("include") - && let Some(include) = include.parent() - && include.ends_with("usr") + let full_include = Path::new(OsStr::from_bytes(full_include.trim_ascii())); + let mut include = if full_include.ends_with("include") + && let Some(include) = full_include.parent() + { + include + } else { + return Err(io::Error::other(format!( + "clang #include <...> search path {} should end in /usr/.../include", + full_include.display() + ))); + }; + while !include.ends_with("usr") + && let Some(include_parent) = include.parent() + { + include = include_parent; + } + let include = if include.ends_with("usr") && let Some(include) = include.parent() { include } else { - return Err(io::Error::other( - "clang #include <...> search path should end in /usr/include", - )); + return Err(io::Error::other(format!( + "clang #include <...> search path {} should end in /usr/.../include", + full_include.display() + ))); }; Ok(PathBuf::from(include)) From 42c4a7048bbec2fd8ca95adf98671da28408c5c9 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 10:57:55 +0200 Subject: [PATCH 24/72] why not? --- crates/numcodecs-wasm-builder/src/main.rs | 79 ++--------------------- 1 file changed, 4 insertions(+), 75 deletions(-) diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 7922680b8..f26634b16 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -3,12 +3,9 @@ use std::{ collections::HashMap, - env, - ffi::OsStr, - fs, io, - os::unix::ffi::OsStrExt, + env, fs, io, path::{Path, PathBuf}, - process::{Command, Stdio}, + process::Command, str::FromStr, }; @@ -80,11 +77,9 @@ fn main() -> io::Result<()> { copy_buildenv_to_crate(&crate_dir)?; let nix_env = NixEnv::new(&crate_dir)?; - let host_sysroot = find_clang_host_sysroot(&nix_env, &crate_dir)?; let wasm = build_wasm_codec( &nix_env, - &host_sysroot, &target_dir, &crate_dir, &format!("{}-wasm", args.crate_), @@ -290,74 +285,9 @@ impl NixEnv { } } -fn find_clang_host_sysroot(nix_env: &NixEnv, flake_parent_dir: &Path) -> io::Result { - let NixEnv { clang, .. } = nix_env; - - let mut cmd = Command::new("nix"); - cmd.current_dir(flake_parent_dir); - cmd.arg("develop"); - // cmd.arg("--store"); - // cmd.arg(nix_store_path); - cmd.arg("--no-update-lock-file"); - cmd.arg("--ignore-environment"); - cmd.arg("path:."); - cmd.arg("--command"); - cmd.arg(clang.join("clang")); - cmd.arg("-v"); - cmd.arg("-x"); - cmd.arg("c"); - cmd.arg("-c"); - cmd.arg("-"); - cmd.stdin(Stdio::null()); - - eprintln!("executing {cmd:?}"); - - let output = cmd.output()?; - let Some(full_include) = output - .stderr - .split(|x| *x == b'\n') - .skip_while(|x| x.trim_ascii() != b"#include <...> search starts here:") - .nth(1) - else { - return Err(io::Error::other(format!( - "failed to find #include <...> search path for clang in {:?}", - String::from_utf8_lossy(&output.stderr) - ))); - }; - let full_include = Path::new(OsStr::from_bytes(full_include.trim_ascii())); - let mut include = if full_include.ends_with("include") - && let Some(include) = full_include.parent() - { - include - } else { - return Err(io::Error::other(format!( - "clang #include <...> search path {} should end in /usr/.../include", - full_include.display() - ))); - }; - while !include.ends_with("usr") - && let Some(include_parent) = include.parent() - { - include = include_parent; - } - let include = if include.ends_with("usr") - && let Some(include) = include.parent() - { - include - } else { - return Err(io::Error::other(format!( - "clang #include <...> search path {} should end in /usr/.../include", - full_include.display() - ))); - }; - - Ok(PathBuf::from(include)) -} - #[expect(clippy::too_many_lines)] fn configure_cargo_cmd( nix_env: &NixEnv, - host_sysroot: &Path, target_dir: &Path, crate_dir: &Path, debug: bool, @@ -464,7 +394,7 @@ fn configure_cargo_cmd( cmd.arg(format!( "CXXFLAGSHOST=-isysroot {host_sysroot} -isystem {host_libcxx_include} \ -isystem {clang_include}", - host_sysroot = host_sysroot.display(), + host_sysroot = wasi_sysroot.display(), // I mean, what could go wrong? host_libcxx_include = host_libcxx.join("include").join("c++").join("v1").display(), clang_include = libclang .join("clang") @@ -544,14 +474,13 @@ fn configure_cargo_cmd( fn build_wasm_codec( nix_env: &NixEnv, - host_sysroot: &Path, target_dir: &Path, crate_dir: &Path, crate_name: &str, debug: bool, verbose: bool, ) -> io::Result { - let mut cmd = configure_cargo_cmd(nix_env, host_sysroot, target_dir, crate_dir, debug); + let mut cmd = configure_cargo_cmd(nix_env, target_dir, crate_dir, debug); cmd.arg("rustc") .arg("--crate-type=cdylib") .arg("-Z") From 5453840c4794bc6fa7e4a36b87635613d8b08ba8 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 11:10:43 +0200 Subject: [PATCH 25/72] try again --- crates/numcodecs-wasm-builder/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index f26634b16..8dccdb9de 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -394,7 +394,7 @@ fn configure_cargo_cmd( cmd.arg(format!( "CXXFLAGSHOST=-isysroot {host_sysroot} -isystem {host_libcxx_include} \ -isystem {clang_include}", - host_sysroot = wasi_sysroot.display(), // I mean, what could go wrong? + host_sysroot = wasi_sysroot.join("include").join("wasm32-wasi").display(), // I mean, what could go wrong? host_libcxx_include = host_libcxx.join("include").join("c++").join("v1").display(), clang_include = libclang .join("clang") From f31c699eb1ebb601ca546874fcd438cc22ce5959 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 2 Mar 2026 16:30:45 +0200 Subject: [PATCH 26/72] separate error for en/de-code to array without data --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fc1945c7c..708398cac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6329913", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "7c38eb3", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 40b008b06..296d3d7d2 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -244,18 +244,43 @@ impl Codec for PressioCodec { Ok(data) => libpressio::PressioData::new(data), Err(data) => libpressio::PressioData::new_copied(data.view()), }; + eprintln!( + "data: {} {} {} {:?}", + data.has_data(), + data.len(), + data.ndim(), + data.dtype() + ); let compressed_data = libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); + eprintln!( + "compressed: {} {} {} {:?}", + compressed_data.has_data(), + compressed_data.len(), + compressed_data.ndim(), + compressed_data.dtype() + ); let compressed_data = compressor.compress(&data, compressed_data).map_err(|err| { PressioCodecError::PressioEncodeFailed { source: PressioCodingError(err), } })?; + eprintln!( + "compressed: {} {} {} {:?}", + compressed_data.has_data(), + compressed_data.len(), + compressed_data.ndim(), + compressed_data.dtype() + ); let Some(compressed_data) = compressed_data.clone_into_array() else { - return Err(PressioCodecError::EncodeToUnknownDtype); + if compressed_data.has_data() { + return Err(PressioCodecError::EncodeToUnknownDtype); + } + + return Err(PressioCodecError::EncodeToArrayWithoutData); }; match compressed_data { @@ -315,7 +340,11 @@ impl Codec for PressioCodec { })?; let Some(decompressed_data) = decompressed_data.clone_into_array() else { - return Err(PressioCodecError::DecodeToUnknownDtype); + if decompressed_data.has_data() { + return Err(PressioCodecError::DecodeToUnknownDtype); + } + + return Err(PressioCodecError::DecodeToArrayWithoutData); }; match decompressed_data { @@ -398,6 +427,9 @@ pub enum PressioCodecError { /// [`PressioCodec`] encoded to an unknown unsupported dtype #[error("Pressio encoded to an unknown unsupported dtype")] EncodeToUnknownDtype, + /// [`PressioCodec`] encoded to an array without data + #[error("Pressio encoded to an array without data")] + EncodeToArrayWithoutData, /// [`PressioCodec`] encoded to a bool array, which is unsupported #[error("Pressio encoded to a bool array, which is unsupported")] EncodeToBoolArray, @@ -410,6 +442,9 @@ pub enum PressioCodecError { /// [`PressioCodec`] decoded to an unknown unsupported dtype #[error("Pressio decoded to an unknown unsupported dtype")] DecodeToUnknownDtype, + /// [`PressioCodec`] decoded to an array without data + #[error("Pressio decoded to an array without data")] + DecodeToArrayWithoutData, /// [`PressioCodec`] decoded to a bool array, which is unsupported #[error("Pressio decoded to a bool array, which is unsupported")] DecodeToBoolArray, From 02d5be5d1abbd15c1cfda84f5218bcf42097dd70 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 9 Mar 2026 10:37:44 +0200 Subject: [PATCH 27/72] Enable more plugins --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 58 ++++++++++++++++----------------------- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 708398cac..915c040ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "7c38eb3", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5e533d8", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 296d3d7d2..df38fad0f 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -240,33 +240,17 @@ impl Codec for PressioCodec { compressor: &mut libpressio::PressioCompressor, data: CowArray, ) -> Result { - let data = match data.try_into_owned_nocopy() { - Ok(data) => libpressio::PressioData::new(data), - Err(data) => libpressio::PressioData::new_copied(data.view()), - }; - eprintln!( - "data: {} {} {} {:?}", - data.has_data(), - data.len(), - data.ndim(), - data.dtype() - ); - - let compressed_data = - libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); - eprintln!( - "compressed: {} {} {} {:?}", - compressed_data.has_data(), - compressed_data.len(), - compressed_data.ndim(), - compressed_data.dtype() - ); + let compressed_data = libpressio::PressioData::new_with_shared(data, |data| { + let compressed_data = + libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); - let compressed_data = compressor.compress(&data, compressed_data).map_err(|err| { - PressioCodecError::PressioEncodeFailed { - source: PressioCodingError(err), - } + compressor.compress(data, compressed_data).map_err(|err| { + PressioCodecError::PressioEncodeFailed { + source: PressioCodingError(err), + } + }) })?; + eprintln!( "compressed: {} {} {} {:?}", compressed_data.has_data(), @@ -324,20 +308,24 @@ impl Codec for PressioCodec { compressor: &mut libpressio::PressioCompressor, encoded: CowArray, ) -> Result { - let encoded = match encoded.try_into_owned_nocopy() { - Ok(encoded) => libpressio::PressioData::new(encoded), - Err(encoded) => libpressio::PressioData::new_copied(encoded.view()), - }; + let decompressed_data = libpressio::PressioData::new_with_shared(encoded, |encoded| { + let decompressed_data = + libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); - let decompressed_data = - libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); - - let decompressed_data = compressor - .compress(&encoded, decompressed_data) + .compress(encoded, decompressed_data) .map_err(|err| PressioCodecError::PressioDecodeFailed { source: PressioCodingError(err), - })?; + }) + })?; + + eprintln!( + "decompressed: {} {} {} {:?}", + decompressed_data.has_data(), + decompressed_data.len(), + decompressed_data.ndim(), + decompressed_data.dtype() + ); let Some(decompressed_data) = decompressed_data.clone_into_array() else { if decompressed_data.has_data() { From 49e3f815171d906627250b33127384dfd172ea25 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 10 Mar 2026 09:31:27 +0200 Subject: [PATCH 28/72] port pressio_register_generator to CMake --- Cargo.toml | 2 +- crates/numcodecs-wasm-builder/buildenv/flake.nix | 2 -- crates/numcodecs-wasm-builder/src/main.rs | 14 -------------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 915c040ac..82d6c2740 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -151,7 +151,7 @@ zstd = { version = "0.13", default-features = false } zstd-sys = { version = "2.0.16", default-features = false } # git third-party dependencies with non-upstream fixes -wasm_component_layer = { git = "https://github.com/juntyr/wasm_component_layer.git", rev = "e923536", version = "0.1", default-features = false } +wasm_component_layer = { git = "https://github.com/juntyr/wasm_component_layer.git", rev = "1b8708e", version = "0.1", default-features = false } [workspace.lints.rust] unsafe_code = "deny" diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.nix b/crates/numcodecs-wasm-builder/buildenv/flake.nix index 22ce9af96..b02efb029 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.nix +++ b/crates/numcodecs-wasm-builder/buildenv/flake.nix @@ -61,7 +61,6 @@ packages = [ (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain) pkgs."llvmPackages_${llvmVersion}".libclang - pkgs."llvmPackages_${llvmVersion}".libcxx wasi-sysroot libclang_rt pkgs.cmake @@ -85,7 +84,6 @@ MY_WASM_OPT = "${pkgs.binaryen}/bin/wasm-opt"; MY_PKG_CONFIG = "${pkgs.pkg-config}/bin/pkg-config"; MY_PYTHON3 = "${pkgs.python3}/bin/python3"; - MY_HOST_LIBCXX = "${pkgs."llvmPackages_${llvmVersion}".libcxx.dev}"; }; }; }); diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 8dccdb9de..9e0e12693 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -212,7 +212,6 @@ struct NixEnv { pkg_config: PathBuf, #[expect(dead_code)] python3: PathBuf, - host_libcxx: PathBuf, } impl NixEnv { @@ -280,7 +279,6 @@ impl NixEnv { wasm_opt: try_read_env(&env, "MY_WASM_OPT")?, pkg_config: try_read_env(&env, "MY_PKG_CONFIG")?, python3: try_read_env(&env, "MY_PYTHON3")?, - host_libcxx: try_read_env(&env, "MY_HOST_LIBCXX")?, }) } } @@ -306,7 +304,6 @@ fn configure_cargo_cmd( wasi_sysroot, libclang_rt, pkg_config, - host_libcxx, .. } = nix_env; @@ -391,17 +388,6 @@ fn configure_cargo_cmd( cpp_include_path = crate_dir.join("include.hpp").display(), debug = if debug { "-g" } else { "" }, )); - cmd.arg(format!( - "CXXFLAGSHOST=-isysroot {host_sysroot} -isystem {host_libcxx_include} \ - -isystem {clang_include}", - host_sysroot = wasi_sysroot.join("include").join("wasm32-wasi").display(), // I mean, what could go wrong? - host_libcxx_include = host_libcxx.join("include").join("c++").join("v1").display(), - clang_include = libclang - .join("clang") - .join(llvm_version) - .join("include") - .display(), - )); cmd.arg(format!( "BINDGEN_EXTRA_CLANG_ARGS=--target=wasm32-wasip1 -nodefaultlibs -resource-dir \ {resource_dir} --sysroot={wasi_sysroot} -isystem {wasm32_wasi_cxx_include} -isystem \ From f362d151e26f74fb6f62bcd869f8d3273c385f4c Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 10 Mar 2026 09:33:15 +0200 Subject: [PATCH 29/72] Fix git rev --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 82d6c2740..720340c58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5e533d8", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1b8708e", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy @@ -151,7 +151,7 @@ zstd = { version = "0.13", default-features = false } zstd-sys = { version = "2.0.16", default-features = false } # git third-party dependencies with non-upstream fixes -wasm_component_layer = { git = "https://github.com/juntyr/wasm_component_layer.git", rev = "1b8708e", version = "0.1", default-features = false } +wasm_component_layer = { git = "https://github.com/juntyr/wasm_component_layer.git", rev = "e923536", version = "0.1", default-features = false } [workspace.lints.rust] unsafe_code = "deny" From 27b6700430d228af542a62dfb5e87ebc86e29761 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 10 Mar 2026 09:50:47 +0200 Subject: [PATCH 30/72] add LIBPRESSIO_WITH_EXTERNAL option --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 720340c58..ac180526c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1b8708e", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6b65157", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 1829212727eaba03faacf49b68ecf6a444d01a06 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 11 Mar 2026 10:53:12 +0200 Subject: [PATCH 31/72] Start bringing codec config closer to existing libpressio numcodecs interface --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 412 ++++++++++++++++++++++++++++---------- 2 files changed, 304 insertions(+), 110 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ac180526c..3b8620af6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6b65157", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "ec5df41", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index df38fad0f..fde1210f6 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,23 +17,17 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. -use std::{ - borrow::Cow, - collections::BTreeMap, - sync::{LazyLock, Mutex}, -}; +use std::{borrow::Cow, collections::BTreeMap, sync::Mutex}; -use ndarray::{CowArray, IxDyn}; +use ndarray::{ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, }; -use schemars::{JsonSchema, Schema, SchemaGenerator}; +use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; -static PRESSIO: LazyLock = LazyLock::new(Pressio::new); - #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[schemars(deny_unknown_fields)] /// Pressio codec which applies the identity function, i.e. passes through the @@ -56,12 +50,10 @@ pub struct PressioCompressor { impl Clone for PressioCompressor { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { - let mut compressor = { - let mut pressio = PRESSIO.get_or_unwrap().lock().unwrap(); - pressio - .get_compressor(self.format.compressor.as_str()) - .unwrap() - }; + let mut pressio = libpressio::Pressio::new().unwrap(); + let mut compressor = pressio + .get_compressor(self.format.compressor_id.as_str()) + .unwrap(); let options = self.compressor.lock().unwrap().get_options().unwrap(); compressor.set_options(&options).unwrap(); @@ -80,89 +72,102 @@ impl Serialize for PressioCompressor { impl<'de> Deserialize<'de> for PressioCompressor { fn deserialize>(deserializer: D) -> Result { - // TODO: better error handling - let format = PressioCompressorFormat::deserialize(deserializer)?; - let compressor = { - let pressio = PRESSIO - .get() - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; - let mut pressio = pressio.lock().map_err(serde::de::Error::custom)?; - pressio - .get_compressor(format.compressor.as_str()) - .map_err(|err| { - let supported_compressors = pressio.supported_compressors().map_or_else( - |_| String::from(""), - |x| { - x.iter() - .map(|x| format!("`{x}`")) - .collect::>() - .join(", ") - }, - ); - - serde::de::Error::custom(format_args!( - "{}, choose one of: {}", - err.message, supported_compressors - )) - })? - }; - let mut options = compressor - .get_options() - .map_err(|err| serde::de::Error::custom(err.message))?; - - for (key, value) in &format.options { - options = options - .set( - key, - match value { - PressioOption::Bool(x) => libpressio::PressioOption::bool(Some(*x)), - PressioOption::U8(x) => libpressio::PressioOption::uint8(Some(*x)), - PressioOption::I8(x) => libpressio::PressioOption::int8(Some(*x)), - PressioOption::U16(x) => libpressio::PressioOption::uint16(Some(*x)), - PressioOption::I16(x) => libpressio::PressioOption::int16(Some(*x)), - PressioOption::U32(x) => libpressio::PressioOption::uint32(Some(*x)), - PressioOption::I32(x) => libpressio::PressioOption::int32(Some(*x)), - PressioOption::U64(x) => libpressio::PressioOption::uint64(Some(*x)), - PressioOption::I64(x) => libpressio::PressioOption::int64(Some(*x)), - PressioOption::F32(x) => libpressio::PressioOption::float32(Some(*x)), - PressioOption::F64(x) => libpressio::PressioOption::float64(Some(*x)), + fn convert_to_pressio_options( + config: &BTreeMap, + ) -> Result { + let mut options = libpressio::PressioOptions::new()?; + + let mut entries = vec![(vec![], config)]; + + while let Some((path, entry)) = entries.pop() { + for (key, value) in entry { + let option = match value { + PressioOption::None(None) => Option::None, + PressioOption::Bool(x) => Some(libpressio::PressioOption::bool(Some(*x))), + PressioOption::U8(x) => Some(libpressio::PressioOption::uint8(Some(*x))), + PressioOption::I8(x) => Some(libpressio::PressioOption::int8(Some(*x))), + PressioOption::U16(x) => Some(libpressio::PressioOption::uint16(Some(*x))), + PressioOption::I16(x) => Some(libpressio::PressioOption::int16(Some(*x))), + PressioOption::U32(x) => Some(libpressio::PressioOption::uint32(Some(*x))), + PressioOption::I32(x) => Some(libpressio::PressioOption::int32(Some(*x))), + PressioOption::U64(x) => Some(libpressio::PressioOption::uint64(Some(*x))), + PressioOption::I64(x) => Some(libpressio::PressioOption::int64(Some(*x))), + PressioOption::F32(x) => Some(libpressio::PressioOption::float32(Some(*x))), + PressioOption::F64(x) => Some(libpressio::PressioOption::float64(Some(*x))), PressioOption::String(x) => { - libpressio::PressioOption::string(Some(x.clone())) + Some(libpressio::PressioOption::string(Some(x.clone()))) } PressioOption::VecString(x) => { - libpressio::PressioOption::vec_string(Some(x.clone())) + Some(libpressio::PressioOption::vec_string(Some(x.clone()))) } + PressioOption::Nested(entry) => { + let mut nested_path = path.clone(); + nested_path.push(key.clone()); + entries.push((nested_path, entry)); + continue; + } + }; + + let name = if path.is_empty() { + key.clone() + } else { + format!("{path}:{key}", path = path.join("/")) + }; + + if let Some(option) = option { + options = options.set(name, option)?; + } + } + } + + Ok(options) + } + + // TODO: better error handling + let format = PressioCompressorFormat::deserialize(deserializer)?; + + let mut pressio = libpressio::Pressio::new() + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + let mut compressor = pressio + .get_compressor(format.compressor_id.as_str()) + .map_err(|err| { + let supported_compressors = pressio.supported_compressors().map_or_else( + |_| String::from(""), + |x| { + x.iter() + .map(|x| format!("`{x}`")) + .collect::>() + .join(", ") }, - ) - .map_err(|err| serde::de::Error::custom(err.message))?; + ); + + serde::de::Error::custom(format_args!( + "{}, choose one of: {}", + err.message, supported_compressors + )) + })?; + + if let Some(name) = &format.name { + compressor + .set_name(name) + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; } - let mut format = format; - if let Ok(format_options) = options.get_options() { - format.options = format_options - .into_iter() - .filter_map(|(k, v)| match v { - libpressio::PressioOption::bool(Some(x)) => Some((k, PressioOption::Bool(x))), - libpressio::PressioOption::int8(Some(x)) => Some((k, PressioOption::I8(x))), - libpressio::PressioOption::int16(Some(x)) => Some((k, PressioOption::I16(x))), - libpressio::PressioOption::int32(Some(x)) => Some((k, PressioOption::I32(x))), - libpressio::PressioOption::int64(Some(x)) => Some((k, PressioOption::I64(x))), - libpressio::PressioOption::uint8(Some(x)) => Some((k, PressioOption::U8(x))), - libpressio::PressioOption::uint16(Some(x)) => Some((k, PressioOption::U16(x))), - libpressio::PressioOption::uint32(Some(x)) => Some((k, PressioOption::U32(x))), - libpressio::PressioOption::uint64(Some(x)) => Some((k, PressioOption::U64(x))), - libpressio::PressioOption::float32(Some(x)) => Some((k, PressioOption::F32(x))), - libpressio::PressioOption::float64(Some(x)) => Some((k, PressioOption::F64(x))), - libpressio::PressioOption::string(Some(x)) => { - Some((k, PressioOption::String(x))) - } - // FIXME: seems to return strings as a single joined string - libpressio::PressioOption::vec_string(Some(x)) => { - Some((k, PressioOption::VecString(x))) - } - _ => None, - }) - .collect(); + let early_options = convert_to_pressio_options(&format.early_config) + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + compressor + .set_options(&early_options) + .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + + let _options_template = compressor + .get_options() + .map_err(|err| serde::de::Error::custom(err.message))?; + + if !format.compressor_config.is_empty() { + // TODO + return Err(serde::de::Error::custom( + "compressor_config is not yet supported", + )); } Ok(Self { @@ -185,10 +190,17 @@ impl JsonSchema for PressioCompressor { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename = "PressioCompressor")] struct PressioCompressorFormat { - compressor: String, - // TODO: flatten + /// The id of the compressor + compressor_id: String, + /// Configuration for the structure of the compressor #[serde(default)] - options: BTreeMap, + early_config: BTreeMap, + /// Configuration for the compressor + #[serde(default)] + compressor_config: BTreeMap, + /// Optional name for the compressor when used in hierarchical mode + #[serde(default)] + name: Option, } #[expect(missing_docs)] @@ -196,6 +208,7 @@ struct PressioCompressorFormat { #[serde(untagged)] /// Pressio option value pub enum PressioOption { + None(None), Bool(bool), U8(u8), I8(i8), @@ -209,26 +222,50 @@ pub enum PressioOption { F64(f64), String(String), VecString(Vec), + Nested(BTreeMap), } -struct Pressio { - pressio: Result, libpressio::PressioError>, +#[derive(Copy, Clone, Debug)] +/// Equivalent of `Option::None` +pub struct None; + +impl Serialize for None { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_none() + } } -impl Pressio { - fn new() -> Self { - Self { - pressio: libpressio::Pressio::new().map(Mutex::new), +impl<'de> Deserialize<'de> for None { + fn deserialize>(deserializer: D) -> Result { + enum Never {} + + impl<'de> Deserialize<'de> for Never { + fn deserialize>(_deserializer: D) -> Result { + Err(serde::de::Error::custom("never")) + } } + + match Option::::deserialize(deserializer) { + Ok(Option::Some(x)) => match x {}, + Ok(Option::None) => Ok(Self), + Err(err) => Err(err), + } + } +} + +impl JsonSchema for None { + fn schema_name() -> Cow<'static, str> { + Cow::Borrowed("null") } - const fn get(&self) -> Result<&Mutex, &libpressio::PressioError> { - self.pressio.as_ref() + fn inline_schema() -> bool { + true } - #[expect(clippy::unwrap_used)] - fn get_or_unwrap(&self) -> &Mutex { - self.pressio.as_ref().unwrap() + fn json_schema(_generator: &mut SchemaGenerator) -> Schema { + json_schema!({ + "type": "null" + }) } } @@ -371,15 +408,172 @@ impl Codec for PressioCodec { } } + #[expect(clippy::too_many_lines)] // FIXME fn decode_into( &self, encoded: AnyArrayView, - mut decoded: AnyArrayViewMut, + decoded: AnyArrayViewMut, ) -> Result<(), Self::Error> { - // TODO: optimize - let decoded_in = self.decode(encoded.cow())?; + fn decompress_typed( + compressor: &mut libpressio::PressioCompressor, + encoded: ArrayView, + decoded_dtype: libpressio::PressioDtype, + decoded_shape: &[usize], + ) -> Result { + libpressio::PressioData::new_with_shared(encoded, |encoded| { + let decompressed_data = + libpressio::PressioData::new_empty(decoded_dtype, decoded_shape); + + compressor + .compress(encoded, decompressed_data) + .map_err(|err| PressioCodecError::PressioDecodeFailed { + source: PressioCodingError(err), + }) + }) + } + + fn decode_into_typed( + decompressed_data: &libpressio::PressioData, + mut decoded: ArrayViewMut, + ) -> Result<(), PressioCodecError> { + eprintln!( + "decompressed into: {} {} {} {:?}", + decompressed_data.has_data(), + decompressed_data.len(), + decompressed_data.ndim(), + decompressed_data.dtype() + ); + + if !decompressed_data.has_data() { + return Err(PressioCodecError::DecodeToArrayWithoutData); + } + + let dtype = match ::DTYPE { + libpressio::PressioDtype::Bool => { + return Err(PressioCodecError::DecodeToBoolArray); + } + libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8 => AnyArrayDType::U8, + libpressio::PressioDtype::U16 => AnyArrayDType::U16, + libpressio::PressioDtype::U32 => AnyArrayDType::U32, + libpressio::PressioDtype::U64 => AnyArrayDType::U64, + libpressio::PressioDtype::I8 => AnyArrayDType::I8, + libpressio::PressioDtype::I16 => AnyArrayDType::I16, + libpressio::PressioDtype::I32 => AnyArrayDType::I32, + libpressio::PressioDtype::I64 => AnyArrayDType::I64, + libpressio::PressioDtype::F32 => AnyArrayDType::F32, + libpressio::PressioDtype::F64 => AnyArrayDType::F64, + }; + let decompressed_dtype = match decompressed_data.dtype() { + Option::None => return Err(PressioCodecError::DecodeToUnknownDtype), + Some(libpressio::PressioDtype::Bool) => { + return Err(PressioCodecError::DecodeToBoolArray); + } + Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => { + AnyArrayDType::U8 + } + Some(libpressio::PressioDtype::U16) => AnyArrayDType::U16, + Some(libpressio::PressioDtype::U32) => AnyArrayDType::U32, + Some(libpressio::PressioDtype::U64) => AnyArrayDType::U64, + Some(libpressio::PressioDtype::I8) => AnyArrayDType::I8, + Some(libpressio::PressioDtype::I16) => AnyArrayDType::I16, + Some(libpressio::PressioDtype::I32) => AnyArrayDType::I32, + Some(libpressio::PressioDtype::I64) => AnyArrayDType::I64, + Some(libpressio::PressioDtype::F32) => AnyArrayDType::F32, + Some(libpressio::PressioDtype::F64) => AnyArrayDType::F64, + }; + + if dtype != decompressed_dtype { + return Err(PressioCodecError::MismatchedDecodeIntoArray { + source: AnyArrayAssignError::DTypeMismatch { + src: decompressed_dtype, + dst: dtype, + }, + }); + } - Ok(decoded.assign(&decoded_in)?) + if decompressed_data + .with_shared::(decoded.dim(), |decompressed| { + decoded.assign(&decompressed); + }) + .is_none() + { + return Err(PressioCodecError::MismatchedDecodeIntoArray { + source: AnyArrayAssignError::ShapeMismatch { + src: decompressed_data.shape(), + dst: decoded.shape().to_vec(), + }, + }); + } + + Ok(()) + } + + let Ok(mut compressor) = self.compressor.compressor.lock() else { + return Err(PressioCodecError::PressioPoisonedMutex); + }; + + let decoded_dtype = match decoded.dtype() { + AnyArrayDType::U8 => libpressio::PressioDtype::U8, + AnyArrayDType::U16 => libpressio::PressioDtype::U16, + AnyArrayDType::U32 => libpressio::PressioDtype::U32, + AnyArrayDType::U64 => libpressio::PressioDtype::U64, + AnyArrayDType::I8 => libpressio::PressioDtype::I8, + AnyArrayDType::I16 => libpressio::PressioDtype::I16, + AnyArrayDType::I32 => libpressio::PressioDtype::I32, + AnyArrayDType::I64 => libpressio::PressioDtype::I64, + AnyArrayDType::F32 => libpressio::PressioDtype::F32, + AnyArrayDType::F64 => libpressio::PressioDtype::F64, + decoded_dtype => return Err(PressioCodecError::UnsupportedDtype(decoded_dtype)), + }; + let decoded_shape = decoded.shape(); + + let decompressed_data = match encoded { + AnyArrayView::U8(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::U16(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::U32(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::U64(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::I8(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::I16(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::I32(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::I64(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::F32(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + AnyArrayView::F64(encoded) => { + decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + } + encoded => return Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), + }?; + + match decoded { + AnyArrayViewMut::U8(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::U16(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::U32(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::U64(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::I8(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::I16(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::I32(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::I64(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::F32(decoded) => decode_into_typed(&decompressed_data, decoded), + AnyArrayViewMut::F64(decoded) => decode_into_typed(&decompressed_data, decoded), + decoded => Err(PressioCodecError::UnsupportedDtype(decoded.dtype())), + } } } From ef41a7b7293ceb5c087dfcb1a45bd6a4edf04bdc Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 11 Mar 2026 11:52:07 +0200 Subject: [PATCH 32/72] Fix clippy lint --- codecs/pressio/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index fde1210f6..7c2723349 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -222,7 +222,7 @@ pub enum PressioOption { F64(f64), String(String), VecString(Vec), - Nested(BTreeMap), + Nested(BTreeMap), } #[derive(Copy, Clone, Debug)] From 6fbd2b06997e1c626985e65ee3fada405425316a Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 12 Mar 2026 12:11:19 +0200 Subject: [PATCH 33/72] Progress with serialising the compressor config --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 197 +++++++++++++++++++++++++++++++------- 2 files changed, 165 insertions(+), 34 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3b8620af6..8e2d8b966 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "ec5df41", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "bed5dc3", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 7c2723349..a664e0acc 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,7 +17,11 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. -use std::{borrow::Cow, collections::BTreeMap, sync::Mutex}; +use std::{ + borrow::Cow, + collections::{BTreeMap, btree_map::Entry}, + sync::Mutex, +}; use ndarray::{ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ @@ -43,39 +47,137 @@ pub struct PressioCodec { /// Pressio compressor pub struct PressioCompressor { - format: PressioCompressorFormat, compressor: Mutex, + compressor_id: String, + early_config: BTreeMap, + name: Option, } impl Clone for PressioCompressor { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { let mut pressio = libpressio::Pressio::new().unwrap(); - let mut compressor = pressio - .get_compressor(self.format.compressor_id.as_str()) - .unwrap(); + let mut compressor = pressio.get_compressor(self.compressor_id.as_str()).unwrap(); + if let Some(name) = &self.name { + compressor.set_name(name).unwrap(); + } let options = self.compressor.lock().unwrap().get_options().unwrap(); compressor.set_options(&options).unwrap(); Self { - format: self.format.clone(), compressor: Mutex::new(compressor), + compressor_id: self.compressor_id.clone(), + early_config: self.early_config.clone(), + name: self.name.clone(), } } } impl Serialize for PressioCompressor { fn serialize(&self, serializer: S) -> Result { - self.format.serialize(serializer) + fn convert_from_pressio_options( + options: BTreeMap, + ) -> Result, E> { + let mut config = BTreeMap::new(); + + for (name, option) in options { + let value = match option { + libpressio::PressioOption::bool(Some(x)) => PressioOption::Bool(x), + libpressio::PressioOption::int8(Some(x)) => PressioOption::I8(x), + libpressio::PressioOption::int16(Some(x)) => PressioOption::I16(x), + libpressio::PressioOption::int32(Some(x)) => PressioOption::I32(x), + libpressio::PressioOption::int64(Some(x)) => PressioOption::I64(x), + libpressio::PressioOption::uint8(Some(x)) => PressioOption::U8(x), + libpressio::PressioOption::uint16(Some(x)) => PressioOption::U16(x), + libpressio::PressioOption::uint32(Some(x)) => PressioOption::U32(x), + libpressio::PressioOption::uint64(Some(x)) => PressioOption::U64(x), + libpressio::PressioOption::float32(Some(x)) => PressioOption::F32(x), + libpressio::PressioOption::float64(Some(x)) => PressioOption::F64(x), + libpressio::PressioOption::string(Some(x)) => PressioOption::String(x), + // FIXME: seems to return strings as a single joined string + libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), + libpressio::PressioOption::data(_) + | libpressio::PressioOption::user_ptr(_) + | libpressio::PressioOption::unset + | _ /* non-exhaustive */ => continue, + }; + + let Some(nested_name) = name.strip_prefix('/') else { + // global option + if config.insert(name.clone(), value).is_some() { + return Err(serde::ser::Error::custom(format!( + "duplicate option {name:?}" + ))); + } + continue; + }; + + // hierarchical option + let mut parts = nested_name.split(':').peekable(); + + let Some(first) = parts.next() else { + return Err(serde::ser::Error::custom(format!( + "invalid hierarchical config name {name:?}" + ))); + }; + let paths = first.split('/'); + + if parts.peek().is_none() { + return Err(serde::ser::Error::custom(format!( + "invalid hierarchical config name {name:?}" + ))); + } + let option_name = parts.map(String::from).collect::>().join(":"); + + let mut it = &mut config; + for path in paths { + if let Entry::Vacant(entry) = it.entry(String::from(path)) { + entry.insert(PressioOption::Nested(BTreeMap::new())); + } + + let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { + return Err(serde::ser::Error::custom(format!( + "duplicate option {path:?}" + ))); + }; + it = entry; + } + if it.insert(option_name.clone(), value).is_some() { + return Err(serde::ser::Error::custom(format!( + "duplicate option {option_name:?}" + ))); + } + } + + Ok(config) + } + + let options = { + let compressor = self.compressor.lock().map_err(serde::ser::Error::custom)?; + compressor + .get_options() + .map_err(serde::ser::Error::custom)? + }; + let options = options.get_options().map_err(serde::ser::Error::custom)?; + + PressioCompressorBorrowedFormat { + compressor_id: self.compressor_id.as_str(), + early_config: &self.early_config, + compressor_config: &convert_from_pressio_options(options)?, + name: self.name.as_deref(), + } + .serialize(serializer) } } impl<'de> Deserialize<'de> for PressioCompressor { fn deserialize>(deserializer: D) -> Result { - fn convert_to_pressio_options( + fn convert_to_pressio_options( config: &BTreeMap, - ) -> Result { - let mut options = libpressio::PressioOptions::new()?; + template: Option<&libpressio::PressioOptions>, + ) -> Result { + let mut options = + libpressio::PressioOptions::new().map_err(serde::de::Error::custom)?; let mut entries = vec![(vec![], config)]; @@ -114,8 +216,25 @@ impl<'de> Deserialize<'de> for PressioCompressor { format!("{path}:{key}", path = path.join("/")) }; + if let Some(template) = template { + if !template + .has_option(&name) + .map_err(serde::de::Error::custom)? + { + return Err(serde::de::Error::custom(format!( + "unknown compressor configuration option: {name:?}" + ))); + } + } + + // TODO: handle conversion and type errors + // TODO: check if the options were actually set + // (e.g. compressor names are validated and fallback to noop) + if let Some(option) = option { - options = options.set(name, option)?; + options + .set(name, option) + .map_err(serde::de::Error::custom)?; } } } @@ -124,10 +243,9 @@ impl<'de> Deserialize<'de> for PressioCompressor { } // TODO: better error handling - let format = PressioCompressorFormat::deserialize(deserializer)?; + let format = PressioCompressorOwnedFormat::deserialize(deserializer)?; - let mut pressio = libpressio::Pressio::new() - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + let mut pressio = libpressio::Pressio::new().map_err(serde::de::Error::custom)?; let mut compressor = pressio .get_compressor(format.compressor_id.as_str()) .map_err(|err| { @@ -150,46 +268,43 @@ impl<'de> Deserialize<'de> for PressioCompressor { if let Some(name) = &format.name { compressor .set_name(name) - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + .map_err(serde::de::Error::custom)?; } - let early_options = convert_to_pressio_options(&format.early_config) - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; + let early_options = convert_to_pressio_options(&format.early_config, Option::None)?; compressor .set_options(&early_options) - .map_err(|err| serde::de::Error::custom(err.message.as_str()))?; - - let _options_template = compressor - .get_options() - .map_err(|err| serde::de::Error::custom(err.message))?; + .map_err(serde::de::Error::custom)?; + let options_template = compressor.get_options().map_err(serde::de::Error::custom)?; - if !format.compressor_config.is_empty() { - // TODO - return Err(serde::de::Error::custom( - "compressor_config is not yet supported", - )); - } + let options = + convert_to_pressio_options(&format.compressor_config, Some(&options_template))?; + compressor + .set_options(&options) + .map_err(serde::de::Error::custom)?; Ok(Self { - format, compressor: Mutex::new(compressor), + compressor_id: format.compressor_id, + early_config: format.early_config, + name: format.name, }) } } impl JsonSchema for PressioCompressor { fn schema_name() -> Cow<'static, str> { - PressioCompressorFormat::schema_name() + PressioCompressorOwnedFormat::schema_name() } fn json_schema(generator: &mut SchemaGenerator) -> Schema { - PressioCompressorFormat::json_schema(generator) + PressioCompressorOwnedFormat::json_schema(generator) } } -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[derive(Debug, Deserialize, JsonSchema)] #[serde(rename = "PressioCompressor")] -struct PressioCompressorFormat { +struct PressioCompressorOwnedFormat { /// The id of the compressor compressor_id: String, /// Configuration for the structure of the compressor @@ -203,6 +318,22 @@ struct PressioCompressorFormat { name: Option, } +#[derive(Debug, Serialize)] +#[serde(rename = "PressioCompressor")] +struct PressioCompressorBorrowedFormat<'a> { + /// The id of the compressor + compressor_id: &'a str, + /// Configuration for the structure of the compressor + #[serde(default)] + early_config: &'a BTreeMap, + /// Configuration for the compressor + #[serde(default)] + compressor_config: &'a BTreeMap, + /// Optional name for the compressor when used in hierarchical mode + #[serde(default)] + name: Option<&'a str>, +} + #[expect(missing_docs)] #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] From 67d9f60531d837c67845c18170a5199df259b7a2 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 13 Mar 2026 09:51:52 +0200 Subject: [PATCH 34/72] small code updates --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8e2d8b966..9bd9714af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "bed5dc3", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "095bd73", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index a664e0acc..6e2580128 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -76,11 +76,16 @@ impl Clone for PressioCompressor { impl Serialize for PressioCompressor { fn serialize(&self, serializer: S) -> Result { fn convert_from_pressio_options( - options: BTreeMap, + options: impl Iterator, Option)>, ) -> Result, E> { let mut config = BTreeMap::new(); for (name, option) in options { + // skip invalid option names and values + let (Some(name), Some(option)) = (name, option) else { + continue; + }; + let value = match option { libpressio::PressioOption::bool(Some(x)) => PressioOption::Bool(x), libpressio::PressioOption::int8(Some(x)) => PressioOption::I8(x), @@ -158,12 +163,11 @@ impl Serialize for PressioCompressor { .get_options() .map_err(serde::ser::Error::custom)? }; - let options = options.get_options().map_err(serde::ser::Error::custom)?; PressioCompressorBorrowedFormat { compressor_id: self.compressor_id.as_str(), early_config: &self.early_config, - compressor_config: &convert_from_pressio_options(options)?, + compressor_config: &convert_from_pressio_options(options.iter())?, name: self.name.as_deref(), } .serialize(serializer) @@ -249,7 +253,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { let mut compressor = pressio .get_compressor(format.compressor_id.as_str()) .map_err(|err| { - let supported_compressors = pressio.supported_compressors().map_or_else( + let supported_compressors = libpressio::supported_compressors().map_or_else( |_| String::from(""), |x| { x.iter() From af554caafd6cc05728fc780b797a0b134deb17b1 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 13 Mar 2026 13:11:06 +0200 Subject: [PATCH 35/72] fix some config bugs --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 62 +++++++++++++++------------------------ 2 files changed, 25 insertions(+), 39 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9bd9714af..dcc87f7d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "095bd73", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "d928ca7", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 6e2580128..1bb50d640 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -101,6 +101,8 @@ impl Serialize for PressioCompressor { libpressio::PressioOption::string(Some(x)) => PressioOption::String(x), // FIXME: seems to return strings as a single joined string libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), + libpressio::PressioOption::dtype(Some(x)) => PressioOption::String(format!("{x}")), + libpressio::PressioOption::thread_safety(Some(x)) => PressioOption::String(format!("{x}")), libpressio::PressioOption::data(_) | libpressio::PressioOption::user_ptr(_) | libpressio::PressioOption::unset @@ -111,7 +113,7 @@ impl Serialize for PressioCompressor { // global option if config.insert(name.clone(), value).is_some() { return Err(serde::ser::Error::custom(format!( - "duplicate option {name:?}" + "duplicate global option: {name:?}" ))); } continue; @@ -142,14 +144,14 @@ impl Serialize for PressioCompressor { let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { return Err(serde::ser::Error::custom(format!( - "duplicate option {path:?}" + "duplicate option nesting: {path:?} in {name:?}" ))); }; it = entry; } if it.insert(option_name.clone(), value).is_some() { return Err(serde::ser::Error::custom(format!( - "duplicate option {option_name:?}" + "duplicate nested option: {option_name:?} in {name:?}" ))); } } @@ -175,6 +177,7 @@ impl Serialize for PressioCompressor { } impl<'de> Deserialize<'de> for PressioCompressor { + #[expect(clippy::too_many_lines)] // FIXME fn deserialize>(deserializer: D) -> Result { fn convert_to_pressio_options( config: &BTreeMap, @@ -217,25 +220,32 @@ impl<'de> Deserialize<'de> for PressioCompressor { let name = if path.is_empty() { key.clone() } else { - format!("{path}:{key}", path = path.join("/")) + format!("/{path}:{key}", path = path.join("/")) }; if let Some(template) = template { - if !template - .has_option(&name) - .map_err(serde::de::Error::custom)? - { + let Some(option_template) = + template.get(&name).map_err(serde::de::Error::custom)? + else { return Err(serde::de::Error::custom(format!( "unknown compressor configuration option: {name:?}" ))); - } - } + }; - // TODO: handle conversion and type errors - // TODO: check if the options were actually set - // (e.g. compressor names are validated and fallback to noop) + options + .set(&name, option_template.copy_type_only()) + .map_err(serde::de::Error::custom)?; - if let Some(option) = option { + if let Some(option) = option { + options + .set_with_cast( + name, + option, + libpressio::PressioConversionSafety::Special, + ) + .map_err(serde::de::Error::custom)?; + } + } else if let Some(option) = option { options .set(name, option) .map_err(serde::de::Error::custom)?; @@ -423,14 +433,6 @@ impl Codec for PressioCodec { }) })?; - eprintln!( - "compressed: {} {} {} {:?}", - compressed_data.has_data(), - compressed_data.len(), - compressed_data.ndim(), - compressed_data.dtype() - ); - let Some(compressed_data) = compressed_data.clone_into_array() else { if compressed_data.has_data() { return Err(PressioCodecError::EncodeToUnknownDtype); @@ -491,14 +493,6 @@ impl Codec for PressioCodec { }) })?; - eprintln!( - "decompressed: {} {} {} {:?}", - decompressed_data.has_data(), - decompressed_data.len(), - decompressed_data.ndim(), - decompressed_data.dtype() - ); - let Some(decompressed_data) = decompressed_data.clone_into_array() else { if decompressed_data.has_data() { return Err(PressioCodecError::DecodeToUnknownDtype); @@ -571,14 +565,6 @@ impl Codec for PressioCodec { decompressed_data: &libpressio::PressioData, mut decoded: ArrayViewMut, ) -> Result<(), PressioCodecError> { - eprintln!( - "decompressed into: {} {} {} {:?}", - decompressed_data.has_data(), - decompressed_data.len(), - decompressed_data.ndim(), - decompressed_data.dtype() - ); - if !decompressed_data.has_data() { return Err(PressioCodecError::DecodeToArrayWithoutData); } From a378bce8442ac9dec9add8e652d086872fbf9e23 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 14 Mar 2026 08:51:26 +0200 Subject: [PATCH 36/72] include metric results in config and add help when deserialising options fails --- codecs/pressio/src/lib.rs | 95 ++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 27 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 1bb50d640..8fcd29d35 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -113,7 +113,7 @@ impl Serialize for PressioCompressor { // global option if config.insert(name.clone(), value).is_some() { return Err(serde::ser::Error::custom(format!( - "duplicate global option: {name:?}" + "duplicate global option: `{name}`" ))); } continue; @@ -124,14 +124,14 @@ impl Serialize for PressioCompressor { let Some(first) = parts.next() else { return Err(serde::ser::Error::custom(format!( - "invalid hierarchical config name {name:?}" + "invalid hierarchical config name `{name}`" ))); }; let paths = first.split('/'); if parts.peek().is_none() { return Err(serde::ser::Error::custom(format!( - "invalid hierarchical config name {name:?}" + "invalid hierarchical config name `{name}`" ))); } let option_name = parts.map(String::from).collect::>().join(":"); @@ -144,14 +144,14 @@ impl Serialize for PressioCompressor { let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { return Err(serde::ser::Error::custom(format!( - "duplicate option nesting: {path:?} in {name:?}" + "duplicate option nesting: `{path}` in `{name}`" ))); }; it = entry; } if it.insert(option_name.clone(), value).is_some() { return Err(serde::ser::Error::custom(format!( - "duplicate nested option: {option_name:?} in {name:?}" + "duplicate nested option: `{option_name}` in `{name}`" ))); } } @@ -159,20 +159,28 @@ impl Serialize for PressioCompressor { Ok(config) } - let options = { - let compressor = self.compressor.lock().map_err(serde::ser::Error::custom)?; - compressor - .get_options() - .map_err(serde::ser::Error::custom)? - }; + let compressor = self.compressor.lock().map_err(serde::ser::Error::custom)?; + let options = compressor + .get_options() + .map_err(serde::ser::Error::custom)?; + let metric_results = compressor + .get_metric_results() + .map_err(serde::ser::Error::custom)?; + let name = compressor.get_name().map_err(serde::ser::Error::custom)?; - PressioCompressorBorrowedFormat { + let result = PressioCompressorBorrowedFormat { compressor_id: self.compressor_id.as_str(), early_config: &self.early_config, compressor_config: &convert_from_pressio_options(options.iter())?, - name: self.name.as_deref(), + metric_results: &convert_from_pressio_options(metric_results.iter())?, + name: match name { + "" => Option::None, + name => Some(name), + }, } - .serialize(serializer) + .serialize(serializer); + std::mem::drop(compressor); + result } } @@ -182,6 +190,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { fn convert_to_pressio_options( config: &BTreeMap, template: Option<&libpressio::PressioOptions>, + documentation: &libpressio::PressioOptions, ) -> Result { let mut options = libpressio::PressioOptions::new().map_err(serde::de::Error::custom)?; @@ -227,8 +236,15 @@ impl<'de> Deserialize<'de> for PressioCompressor { let Some(option_template) = template.get(&name).map_err(serde::de::Error::custom)? else { + let supported_options = template + .iter() + .filter_map(|(key, _value)| key) + .map(|x| format!("`{x}`")) + .collect::>() + .join(", "); + return Err(serde::de::Error::custom(format!( - "unknown compressor configuration option: {name:?}" + "unknown compressor configuration option: `{name}`, use one of {supported_options}" ))); }; @@ -239,11 +255,24 @@ impl<'de> Deserialize<'de> for PressioCompressor { if let Some(option) = option { options .set_with_cast( - name, + &name, option, libpressio::PressioConversionSafety::Special, ) - .map_err(serde::de::Error::custom)?; + .map_err(|err| { + let docs = match documentation.get(&name) { + Ok(Some(libpressio::PressioOption::string(Some(docs)))) => { + Some(docs) + } + _ => Option::None, + }; + + if let Some(docs) = docs { + serde::de::Error::custom(format_args!("{err} ({docs})")) + } else { + serde::de::Error::custom(err) + } + })?; } } else if let Some(option) = option { options @@ -258,6 +287,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { // TODO: better error handling let format = PressioCompressorOwnedFormat::deserialize(deserializer)?; + std::mem::drop(format.metric_results); let mut pressio = libpressio::Pressio::new().map_err(serde::de::Error::custom)?; let mut compressor = pressio @@ -274,8 +304,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { ); serde::de::Error::custom(format_args!( - "{}, choose one of: {}", - err.message, supported_compressors + "{err}, choose one of: {supported_compressors}" )) })?; @@ -285,14 +314,22 @@ impl<'de> Deserialize<'de> for PressioCompressor { .map_err(serde::de::Error::custom)?; } - let early_options = convert_to_pressio_options(&format.early_config, Option::None)?; + let documentation = compressor + .get_documentation() + .map_err(serde::de::Error::custom)?; + + let early_options = + convert_to_pressio_options(&format.early_config, Option::None, &documentation)?; compressor .set_options(&early_options) .map_err(serde::de::Error::custom)?; let options_template = compressor.get_options().map_err(serde::de::Error::custom)?; - let options = - convert_to_pressio_options(&format.compressor_config, Some(&options_template))?; + let options = convert_to_pressio_options( + &format.compressor_config, + Some(&options_template), + &documentation, + )?; compressor .set_options(&options) .map_err(serde::de::Error::custom)?; @@ -327,6 +364,9 @@ struct PressioCompressorOwnedFormat { /// Configuration for the compressor #[serde(default)] compressor_config: BTreeMap, + /// Results of the compressor metrics (output-only) + #[serde(default)] + metric_results: BTreeMap, /// Optional name for the compressor when used in hierarchical mode #[serde(default)] name: Option, @@ -338,13 +378,14 @@ struct PressioCompressorBorrowedFormat<'a> { /// The id of the compressor compressor_id: &'a str, /// Configuration for the structure of the compressor - #[serde(default)] early_config: &'a BTreeMap, /// Configuration for the compressor - #[serde(default)] compressor_config: &'a BTreeMap, + /// Results of the compressor metrics (output-only) + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + metric_results: &'a BTreeMap, /// Optional name for the compressor when used in hierarchical mode - #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] name: Option<&'a str>, } @@ -487,7 +528,7 @@ impl Codec for PressioCodec { libpressio::PressioData::new_empty(libpressio::PressioDtype::Byte, []); compressor - .compress(encoded, decompressed_data) + .decompress(encoded, decompressed_data) .map_err(|err| PressioCodecError::PressioDecodeFailed { source: PressioCodingError(err), }) @@ -554,7 +595,7 @@ impl Codec for PressioCodec { libpressio::PressioData::new_empty(decoded_dtype, decoded_shape); compressor - .compress(encoded, decompressed_data) + .decompress(encoded, decompressed_data) .map_err(|err| PressioCodecError::PressioDecodeFailed { source: PressioCodingError(err), }) From a32b82b2d92c5b143bed76079574c6ac22d42eed Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 14 Mar 2026 13:40:18 +0200 Subject: [PATCH 37/72] support inline data nd arrays in the config --- Cargo.toml | 1 + codecs/pressio/Cargo.toml | 1 + codecs/pressio/src/lib.rs | 140 +++- codecs/pressio/tests/schema.json | 1331 ++++++++++++++++++++++++++++++ codecs/pressio/tests/schema.rs | 20 + 5 files changed, 1490 insertions(+), 3 deletions(-) create mode 100644 codecs/pressio/tests/schema.json create mode 100644 codecs/pressio/tests/schema.rs diff --git a/Cargo.toml b/Cargo.toml index dcc87f7d2..4deb0343f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,7 @@ schemars = { version = "1.0.3", default-features = false } scratch = { version = "1.0", default-features = false } semver = { version = "1.0.23", default-features = false } serde = { version = "1.0.218", default-features = false } +serde-ndim = { version = "=2.1.0", default-features = false } serde-transcode = { version = "1.1", default-features = false } serde_json = { version = "1.0.140", default-features = false } serde_repr = { version = "0.1.5", default-features = false } diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index e664605e6..5eb125e6f 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -20,6 +20,7 @@ ndarray = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } +serde-ndim = { workspace = true, features = ["ndarray"] } thiserror = { workspace = true } [lints] diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 8fcd29d35..112897d53 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -23,7 +23,7 @@ use std::{ sync::Mutex, }; -use ndarray::{ArrayView, ArrayViewMut, CowArray, IxDyn}; +use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, @@ -74,6 +74,7 @@ impl Clone for PressioCompressor { } impl Serialize for PressioCompressor { + #[expect(clippy::too_many_lines)] fn serialize(&self, serializer: S) -> Result { fn convert_from_pressio_options( options: impl Iterator, Option)>, @@ -103,8 +104,21 @@ impl Serialize for PressioCompressor { libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), libpressio::PressioOption::dtype(Some(x)) => PressioOption::String(format!("{x}")), libpressio::PressioOption::thread_safety(Some(x)) => PressioOption::String(format!("{x}")), - libpressio::PressioOption::data(_) - | libpressio::PressioOption::user_ptr(_) + libpressio::PressioOption::data(Some(x)) => match x.clone_into_array() { + Option::None => continue, + Some(libpressio::PressioArray::Bool(x)) => PressioOption::DataBool(NdArray(x)), + Some(libpressio::PressioArray::Byte(x) | libpressio::PressioArray::U8(x)) => PressioOption::DataU8(NdArray(x)), + Some(libpressio::PressioArray::U16(x)) => PressioOption::DataU16(NdArray(x)), + Some(libpressio::PressioArray::U32(x)) => PressioOption::DataU32(NdArray(x)), + Some(libpressio::PressioArray::U64(x)) => PressioOption::DataU64(NdArray(x)), + Some(libpressio::PressioArray::I8(x)) => PressioOption::DataI8(NdArray(x)), + Some(libpressio::PressioArray::I16(x)) => PressioOption::DataI16(NdArray(x)), + Some(libpressio::PressioArray::I32(x)) => PressioOption::DataI32(NdArray(x)), + Some(libpressio::PressioArray::I64(x)) => PressioOption::DataI64(NdArray(x)), + Some(libpressio::PressioArray::F32(x)) => PressioOption::DataF32(NdArray(x)), + Some(libpressio::PressioArray::F64(x)) => PressioOption::DataF64(NdArray(x)), + }, + libpressio::PressioOption::user_ptr(_) | libpressio::PressioOption::unset | _ /* non-exhaustive */ => continue, }; @@ -218,6 +232,57 @@ impl<'de> Deserialize<'de> for PressioCompressor { PressioOption::VecString(x) => { Some(libpressio::PressioOption::vec_string(Some(x.clone()))) } + PressioOption::DataBool(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataU8(NdArray(x)) => Some(libpressio::PressioOption::data( + Some(libpressio::PressioData::new_copied(x)), + )), + PressioOption::DataU16(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataU32(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataU64(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataI8(NdArray(x)) => Some(libpressio::PressioOption::data( + Some(libpressio::PressioData::new_copied(x)), + )), + PressioOption::DataI16(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataI32(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataI64(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataF32(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } + PressioOption::DataF64(NdArray(x)) => { + Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))) + } PressioOption::Nested(entry) => { let mut nested_path = path.clone(); nested_path.push(key.clone()); @@ -408,9 +473,78 @@ pub enum PressioOption { F64(f64), String(String), VecString(Vec), + DataBool(NdArray), + DataU8(NdArray), + DataU16(NdArray), + DataU32(NdArray), + DataU64(NdArray), + DataI8(NdArray), + DataI16(NdArray), + DataI32(NdArray), + DataI64(NdArray), + DataF32(NdArray), + DataF64(NdArray), Nested(BTreeMap), } +#[derive(Clone)] +/// Pressio n-dimensional data array +pub struct NdArray(Array); + +impl std::fmt::Debug for NdArray { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + self.0.fmt(fmt) + } +} + +impl Serialize for NdArray { + fn serialize(&self, serializer: S) -> Result { + serde_ndim::serialize(&self.0, serializer) + } +} + +impl<'de, T: Deserialize<'de>> Deserialize<'de> for NdArray { + fn deserialize>(deserializer: D) -> Result { + serde_ndim::deserialize(deserializer).map(Self) + } +} + +impl JsonSchema for NdArray { + fn inline_schema() -> bool { + false + } + + fn schema_name() -> Cow<'static, str> { + Cow::Owned(format!("{}NdArray", std::any::type_name::())) + } + + fn schema_id() -> Cow<'static, str> { + Cow::Owned(format!( + "{}::NdArray<{}>", + module_path!(), + std::any::type_name::() + )) + } + + fn json_schema(generator: &mut SchemaGenerator) -> Schema { + let item = generator.subschema_for::(); + let nested = generator.subschema_for::(); + + json_schema!({ + "anyOf": [ + { + "type": "array", + "items": item, + }, + { + "type": "array", + "items": nested, + } + ] + }) + } +} + #[derive(Copy, Clone, Debug)] /// Equivalent of `Option::None` pub struct None; diff --git a/codecs/pressio/tests/schema.json b/codecs/pressio/tests/schema.json new file mode 100644 index 000000000..0cec2dfe5 --- /dev/null +++ b/codecs/pressio/tests/schema.json @@ -0,0 +1,1331 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "compressor_id": { + "type": "string", + "description": "The id of the compressor" + }, + "early_config": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + { + "type": "integer", + "format": "int32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + { + "type": "integer", + "format": "int64" + }, + { + "type": "number", + "format": "float" + }, + { + "type": "number", + "format": "double" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + } + } + ], + "description": "Pressio option value" + }, + "description": "Configuration for the structure of the compressor", + "default": {} + }, + "compressor_config": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + { + "type": "integer", + "format": "int32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + { + "type": "integer", + "format": "int64" + }, + { + "type": "number", + "format": "float" + }, + { + "type": "number", + "format": "double" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + } + } + ], + "description": "Pressio option value" + }, + "description": "Configuration for the compressor", + "default": {} + }, + "metric_results": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + { + "type": "integer", + "format": "int32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + { + "type": "integer", + "format": "int64" + }, + { + "type": "number", + "format": "float" + }, + { + "type": "number", + "format": "double" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + } + } + ], + "description": "Pressio option value" + }, + "description": "Results of the compressor metrics (output-only)", + "default": {} + }, + "name": { + "type": [ + "string", + "null" + ], + "description": "Optional name for the compressor when used in hierarchical mode", + "default": null + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "compressor_id" + ], + "description": "Pressio codec which applies the identity function, i.e. passes through the\ninput unchanged during encoding and decoding.", + "title": "PressioCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "boolNdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + "u8NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + "u16NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + "u32NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + "u64NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + "i8NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + "i16NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + "i32NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + "i64NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + "f32NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + "f64NdArray": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + "PressioOption": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + { + "type": "integer", + "format": "int32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + { + "type": "integer", + "format": "int64" + }, + { + "type": "number", + "format": "float" + }, + { + "type": "number", + "format": "double" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/boolNdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/u64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i8NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767 + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i16NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/i64NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "float" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f32NdArray" + } + } + ] + }, + { + "anyOf": [ + { + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "type": "array", + "items": { + "$ref": "#/$defs/f64NdArray" + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + } + } + ], + "description": "Pressio option value" + } + } +} \ No newline at end of file diff --git a/codecs/pressio/tests/schema.rs b/codecs/pressio/tests/schema.rs new file mode 100644 index 000000000..8945df79b --- /dev/null +++ b/codecs/pressio/tests/schema.rs @@ -0,0 +1,20 @@ +#![expect(missing_docs)] + +use ::{libpressio as _, ndarray as _, schemars as _, serde as _, serde_ndim as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_pressio::PressioCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + if schema != include_str!("schema.json") { + panic!("Pressio schema has changed\n===\n{schema}\n==="); + } +} From 7b6ee10f6694ba5092e5a0a54fb6b7d46b948cfb Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 15 Mar 2026 07:07:06 +0200 Subject: [PATCH 38/72] add linear quantizer test --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 4 +++ codecs/pressio/src/lib.rs | 49 +++++++++++++++++++++++++++++++++- codecs/pressio/tests/schema.rs | 5 +++- 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4deb0343f..97f4be896 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "d928ca7", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1289c47", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 5eb125e6f..074682eee 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -23,5 +23,9 @@ serde = { workspace = true, features = ["std", "derive"] } serde-ndim = { workspace = true, features = ["ndarray"] } thiserror = { workspace = true } +[dev-dependencies] +ndarray = { workspace = true, features = ["std"] } +serde_json = { workspace = true, features = ["std"] } + [lints] workspace = true diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 112897d53..13078ca77 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -100,7 +100,6 @@ impl Serialize for PressioCompressor { libpressio::PressioOption::float32(Some(x)) => PressioOption::F32(x), libpressio::PressioOption::float64(Some(x)) => PressioOption::F64(x), libpressio::PressioOption::string(Some(x)) => PressioOption::String(x), - // FIXME: seems to return strings as a single joined string libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), libpressio::PressioOption::dtype(Some(x)) => PressioOption::String(format!("{x}")), libpressio::PressioOption::thread_safety(Some(x)) => PressioOption::String(format!("{x}")), @@ -939,3 +938,51 @@ pub enum PressioCodecError { #[error(transparent)] /// Opaque error for when encoding or decoding with libpressio fails pub struct PressioCodingError(libpressio::PressioError); + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + use ndarray::Array1; + use serde_json::json; + + #[test] + fn linear_quantizer() { + let pressio = PressioCodec::deserialize(json!({ + "compressor_id": "linear_quantizer", + "early_config": { + "pressio:metric": "composite", + }, + "compressor_config": { + "pressio:abs": 10.0, + "pressio:metric": "composite", + "composite:plugins": ["printer", "size"], + } + })) + .unwrap(); + + let data = ndarray::linspace(0.0, 100.0, 50) + .collect::>() + .into_dyn(); + + let encoded = pressio + .encode(AnyCowArray::F64(CowArray::from(&data))) + .unwrap(); + + let decoded = pressio.decode(encoded.cow()); + assert!(matches!( + decoded, + Err(PressioCodecError::DecodeToArrayWithoutData) + )); + + let mut decoded = ndarray::Array::zeros(data.dim()); + pressio + .decode_into(encoded.view(), AnyArrayViewMut::F64(decoded.view_mut())) + .unwrap(); + + for (i, o) in data.iter().zip(decoded.iter()) { + assert!(((*i) - (*o)).abs() <= 10.0); + } + } +} diff --git a/codecs/pressio/tests/schema.rs b/codecs/pressio/tests/schema.rs index 8945df79b..ae8dde7ff 100644 --- a/codecs/pressio/tests/schema.rs +++ b/codecs/pressio/tests/schema.rs @@ -1,6 +1,9 @@ #![expect(missing_docs)] -use ::{libpressio as _, ndarray as _, schemars as _, serde as _, serde_ndim as _, thiserror as _}; +use ::{ + libpressio as _, ndarray as _, schemars as _, serde as _, serde_json as _, serde_ndim as _, + thiserror as _, +}; use numcodecs::{DynCodecType, StaticCodecType}; use numcodecs_pressio::PressioCodec; From a6b6b18fedf12c94d042681339b1ce75e1c05e90 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 16 Mar 2026 01:05:53 +0200 Subject: [PATCH 39/72] fix pressio metrics results --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 153 ++++++++++++++++++++++++-------------- 2 files changed, 98 insertions(+), 57 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 97f4be896..0c661c367 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1289c47", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5be5b1b", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 13078ca77..64400c9dc 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -20,7 +20,7 @@ use std::{ borrow::Cow, collections::{BTreeMap, btree_map::Entry}, - sync::Mutex, + sync::{Arc, Mutex, RwLock}, }; use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, IxDyn}; @@ -47,28 +47,51 @@ pub struct PressioCodec { /// Pressio compressor pub struct PressioCompressor { + // get_config clones the compressor, but we want the config to include the + // compressor metrics + // so we make cheap shallow clones whenever possible and then later make + // the compressor unique with the clone-on-write `Arc::make_mut` + // we pinky-promise to only lock the inner `Mutex` for immutable access + // when we have read-only access, otherwise we can go through + // `Mutex::get_mut` + inner: RwLock>, +} + +impl Clone for PressioCompressor { + #[expect(clippy::unwrap_used)] + fn clone(&self) -> Self { + Self { + inner: RwLock::new(self.inner.read().unwrap().clone()), + } + } +} + +struct PressioCompressorInner { compressor: Mutex, compressor_id: String, early_config: BTreeMap, - name: Option, } -impl Clone for PressioCompressor { +impl Clone for PressioCompressorInner { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { let mut pressio = libpressio::Pressio::new().unwrap(); - let mut compressor = pressio.get_compressor(self.compressor_id.as_str()).unwrap(); - if let Some(name) = &self.name { - compressor.set_name(name).unwrap(); - } - let options = self.compressor.lock().unwrap().get_options().unwrap(); - compressor.set_options(&options).unwrap(); + let compressor = self.compressor.lock().unwrap(); + + let mut compressor_clone = pressio.get_compressor(self.compressor_id.as_str()).unwrap(); + compressor_clone + .set_name(compressor.get_name().unwrap()) + .unwrap(); + compressor_clone + .set_options(&compressor.get_options().unwrap()) + .unwrap(); + + std::mem::drop(compressor); Self { - compressor: Mutex::new(compressor), + compressor: Mutex::new(compressor_clone), compressor_id: self.compressor_id.clone(), early_config: self.early_config.clone(), - name: self.name.clone(), } } } @@ -172,7 +195,8 @@ impl Serialize for PressioCompressor { Ok(config) } - let compressor = self.compressor.lock().map_err(serde::ser::Error::custom)?; + let inner = self.inner.read().map_err(serde::ser::Error::custom)?; + let compressor = inner.compressor.lock().map_err(serde::ser::Error::custom)?; let options = compressor .get_options() .map_err(serde::ser::Error::custom)?; @@ -182,8 +206,8 @@ impl Serialize for PressioCompressor { let name = compressor.get_name().map_err(serde::ser::Error::custom)?; let result = PressioCompressorBorrowedFormat { - compressor_id: self.compressor_id.as_str(), - early_config: &self.early_config, + compressor_id: inner.compressor_id.as_str(), + early_config: &inner.early_config, compressor_config: &convert_from_pressio_options(options.iter())?, metric_results: &convert_from_pressio_options(metric_results.iter())?, name: match name { @@ -193,6 +217,7 @@ impl Serialize for PressioCompressor { } .serialize(serializer); std::mem::drop(compressor); + std::mem::drop(inner); result } } @@ -399,10 +424,11 @@ impl<'de> Deserialize<'de> for PressioCompressor { .map_err(serde::de::Error::custom)?; Ok(Self { - compressor: Mutex::new(compressor), - compressor_id: format.compressor_id, - early_config: format.early_config, - name: format.name, + inner: RwLock::new(Arc::new(PressioCompressorInner { + compressor: Mutex::new(compressor), + compressor_id: format.compressor_id, + early_config: format.early_config, + })), }) } } @@ -632,21 +658,25 @@ impl Codec for PressioCodec { } } - let Ok(mut compressor) = self.compressor.compressor.lock() else { - return Err(PressioCodecError::PressioPoisonedMutex); + let Ok(mut inner) = self.compressor.inner.write() else { + return Err(PressioCodecError::PressioPoisonedLock); + }; + + let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { + return Err(PressioCodecError::PressioPoisonedLock); }; match data { - AnyCowArray::U8(data) => encode_typed(&mut compressor, data), - AnyCowArray::U16(data) => encode_typed(&mut compressor, data), - AnyCowArray::U32(data) => encode_typed(&mut compressor, data), - AnyCowArray::U64(data) => encode_typed(&mut compressor, data), - AnyCowArray::I8(data) => encode_typed(&mut compressor, data), - AnyCowArray::I16(data) => encode_typed(&mut compressor, data), - AnyCowArray::I32(data) => encode_typed(&mut compressor, data), - AnyCowArray::I64(data) => encode_typed(&mut compressor, data), - AnyCowArray::F32(data) => encode_typed(&mut compressor, data), - AnyCowArray::F64(data) => encode_typed(&mut compressor, data), + AnyCowArray::U8(data) => encode_typed(compressor, data), + AnyCowArray::U16(data) => encode_typed(compressor, data), + AnyCowArray::U32(data) => encode_typed(compressor, data), + AnyCowArray::U64(data) => encode_typed(compressor, data), + AnyCowArray::I8(data) => encode_typed(compressor, data), + AnyCowArray::I16(data) => encode_typed(compressor, data), + AnyCowArray::I32(data) => encode_typed(compressor, data), + AnyCowArray::I64(data) => encode_typed(compressor, data), + AnyCowArray::F32(data) => encode_typed(compressor, data), + AnyCowArray::F64(data) => encode_typed(compressor, data), data => Err(PressioCodecError::UnsupportedDtype(data.dtype())), } } @@ -692,21 +722,25 @@ impl Codec for PressioCodec { } } - let Ok(mut compressor) = self.compressor.compressor.lock() else { - return Err(PressioCodecError::PressioPoisonedMutex); + let Ok(mut inner) = self.compressor.inner.write() else { + return Err(PressioCodecError::PressioPoisonedLock); + }; + + let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { + return Err(PressioCodecError::PressioPoisonedLock); }; match encoded { - AnyCowArray::U8(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::U16(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::U32(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::U64(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::I8(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::I16(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::I32(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::I64(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::F32(encoded) => decode_typed(&mut compressor, encoded), - AnyCowArray::F64(encoded) => decode_typed(&mut compressor, encoded), + AnyCowArray::U8(encoded) => decode_typed(compressor, encoded), + AnyCowArray::U16(encoded) => decode_typed(compressor, encoded), + AnyCowArray::U32(encoded) => decode_typed(compressor, encoded), + AnyCowArray::U64(encoded) => decode_typed(compressor, encoded), + AnyCowArray::I8(encoded) => decode_typed(compressor, encoded), + AnyCowArray::I16(encoded) => decode_typed(compressor, encoded), + AnyCowArray::I32(encoded) => decode_typed(compressor, encoded), + AnyCowArray::I64(encoded) => decode_typed(compressor, encoded), + AnyCowArray::F32(encoded) => decode_typed(compressor, encoded), + AnyCowArray::F64(encoded) => decode_typed(compressor, encoded), encoded => Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), } } @@ -803,8 +837,12 @@ impl Codec for PressioCodec { Ok(()) } - let Ok(mut compressor) = self.compressor.compressor.lock() else { - return Err(PressioCodecError::PressioPoisonedMutex); + let Ok(mut inner) = self.compressor.inner.write() else { + return Err(PressioCodecError::PressioPoisonedLock); + }; + + let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { + return Err(PressioCodecError::PressioPoisonedLock); }; let decoded_dtype = match decoded.dtype() { @@ -824,34 +862,34 @@ impl Codec for PressioCodec { let decompressed_data = match encoded { AnyArrayView::U8(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::U16(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::U32(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::U64(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::I8(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::I16(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::I32(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::I64(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::F32(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } AnyArrayView::F64(encoded) => { - decompress_typed(&mut compressor, encoded, decoded_dtype, decoded_shape) + decompress_typed(compressor, encoded, decoded_dtype, decoded_shape) } encoded => return Err(PressioCodecError::UnsupportedDtype(encoded.dtype())), }?; @@ -894,7 +932,7 @@ pub enum PressioCodecError { UnsupportedDtype(AnyArrayDType), /// [`PressioCodec`] lock was poisoned #[error("Pressio lock was poisoned")] - PressioPoisonedMutex, + PressioPoisonedLock, /// [`PressioCodec`] failed to encode the data #[error("Pressio failed to encode the data")] PressioEncodeFailed { @@ -957,7 +995,7 @@ mod tests { "compressor_config": { "pressio:abs": 10.0, "pressio:metric": "composite", - "composite:plugins": ["printer", "size"], + "composite:plugins": ["printer", "size", "time"], } })) .unwrap(); @@ -984,5 +1022,8 @@ mod tests { for (i, o) in data.iter().zip(decoded.iter()) { assert!(((*i) - (*o)).abs() <= 10.0); } + + let config = serde_json::to_string(&pressio.get_config()).unwrap(); + assert!(config.contains("\"size:compressed_size\":400")); } } From f112a1e6b1fe0b5f83691583b4a23631d284b48f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 16 Mar 2026 10:59:48 +0200 Subject: [PATCH 40/72] add support for the libpressio bzip2 compressor --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 41 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0c661c367..15ec7100d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "5be5b1b", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "4508eb7", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 074682eee..1a4966743 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -15,7 +15,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libpressio = { workspace = true } +libpressio = { workspace = true, features = ["bzip2"] } ndarray = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 64400c9dc..eb6ccc0fa 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -986,7 +986,7 @@ mod tests { use serde_json::json; #[test] - fn linear_quantizer() { + fn linear_quantizer_noop() { let pressio = PressioCodec::deserialize(json!({ "compressor_id": "linear_quantizer", "early_config": { @@ -1026,4 +1026,43 @@ mod tests { let config = serde_json::to_string(&pressio.get_config()).unwrap(); assert!(config.contains("\"size:compressed_size\":400")); } + + #[test] + fn linear_quantizer_bzip2() { + let pressio = PressioCodec::deserialize(json!({ + "compressor_id": "linear_quantizer", + "early_config": { + "linear_quantizer:compressor": "bzip2", + }, + "compressor_config": { + "pressio:abs": 10.0, + "pressio:lossless": 9, + "pressio:metric": "size", + } + })) + .unwrap(); + + let data = ndarray::linspace(0.0, 100.0, 50) + .collect::>() + .into_dyn(); + + let encoded = pressio + .encode(AnyCowArray::F64(CowArray::from(&data))) + .unwrap(); + + let decoded = pressio.decode(encoded.cow()); + assert!(decoded.is_err()); + + let mut decoded = ndarray::Array::zeros(data.dim()); + pressio + .decode_into(encoded.view(), AnyArrayViewMut::F64(decoded.view_mut())) + .unwrap(); + + for (i, o) in data.iter().zip(decoded.iter()) { + assert!(((*i) - (*o)).abs() <= 10.0); + } + + let config = serde_json::to_string(&pressio.get_config()).unwrap(); + assert!(config.contains("\"size:compressed_size\":63")); + } } From eb4826a4c5c7998251897584577b9239ad42de3f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Tue, 17 Mar 2026 09:17:44 +0200 Subject: [PATCH 41/72] Try out Lua support for libpressio --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 52 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 15ec7100d..712f0ce96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "4508eb7", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "22f085e", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 1a4966743..ffe83260b 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -15,7 +15,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libpressio = { workspace = true, features = ["bzip2"] } +libpressio = { workspace = true, features = ["bzip2", "lua"] } ndarray = { workspace = true } numcodecs = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index eb6ccc0fa..8ccf0d0b3 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -1065,4 +1065,56 @@ mod tests { let config = serde_json::to_string(&pressio.get_config()).unwrap(); assert!(config.contains("\"size:compressed_size\":63")); } + + #[test] + fn lua_metrics() { + let pressio = PressioCodec::deserialize(json!({ + "compressor_id": "noop", + "early_config": { + "pressio:metric": "composite", + }, + "compressor_config": { + "pressio:metric": "composite", + "composite:plugins": ["size"], + "composite:scripts": [ + "return \"objective\", 1.2", + "return \"objective2\", metrics[\"size:compression_ratio\"] * 4.2", + ] + } + })) + .unwrap(); + + let config = serde_json::to_string(&pressio.get_config()).unwrap(); + assert!(!config.contains("\"size:compression_ratio\"")); + assert!(config.contains("\"composite:objective\":1.2")); + assert!(!config.contains("\"composite:objective2\"")); + + let data = ndarray::linspace(0.0, 100.0, 50) + .collect::>() + .into_dyn(); + + let encoded = pressio + .encode(AnyCowArray::F64(CowArray::from(&data))) + .unwrap(); + + let decoded = pressio.decode(encoded.cow()); + assert!(matches!( + decoded, + Err(PressioCodecError::DecodeToArrayWithoutData) + )); + + let mut decoded = ndarray::Array::zeros(data.dim()); + pressio + .decode_into(encoded.view(), AnyArrayViewMut::F64(decoded.view_mut())) + .unwrap(); + + for (i, o) in data.iter().zip(decoded.iter()) { + assert!(i.to_bits() == o.to_bits()); + } + + let config = serde_json::to_string(&pressio.get_config()).unwrap(); + assert!(config.contains("\"size:compression_ratio\":1.0")); + assert!(config.contains("\"composite:objective\":1.2")); + assert!(config.contains("\"composite:objective2\":4.2")); + } } From 654ab13fd55588f3115364c694ee683bba4ab4a6 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 22 Mar 2026 08:24:29 +0200 Subject: [PATCH 42/72] update libpressio-rs --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 712f0ce96..1d39d0fea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ format_serde_error = { version = "0.3", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "22f085e", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "255ed51", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 3e8756ac719f12c189415624d4305c48d3999390 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 26 Mar 2026 14:35:33 +0200 Subject: [PATCH 43/72] allow using non-sendable pressio compressors using fragile --- Cargo.toml | 3 ++- codecs/pressio/Cargo.toml | 1 + codecs/pressio/src/lib.rs | 56 +++++++++++++++++++++++++++++++++++---- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1d39d0fea..1b68f76ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,10 +98,11 @@ convert_case = { version = "0.8", default-features = false } ebcc = { version = "0.3.1-alpha", default-features = false } erased-serde = { version = "0.4", default-features = false } format_serde_error = { version = "0.3", default-features = false } +fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "255ed51", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "30545dd", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index ffe83260b..87a49a490 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -15,6 +15,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +fragile = { workspace = true } libpressio = { workspace = true, features = ["bzip2", "lua"] } ndarray = { workspace = true } numcodecs = { workspace = true } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 8ccf0d0b3..2da4148a3 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -23,6 +23,7 @@ use std::{ sync::{Arc, Mutex, RwLock}, }; +use fragile::Fragile; use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, @@ -67,7 +68,7 @@ impl Clone for PressioCompressor { } struct PressioCompressorInner { - compressor: Mutex, + compressor: Mutex, compressor_id: String, early_config: BTreeMap, } @@ -76,7 +77,8 @@ impl Clone for PressioCompressorInner { #[expect(clippy::unwrap_used)] fn clone(&self) -> Self { let mut pressio = libpressio::Pressio::new().unwrap(); - let compressor = self.compressor.lock().unwrap(); + let compressor_guard = self.compressor.lock().unwrap(); + let compressor = compressor_guard.try_get().unwrap(); let mut compressor_clone = pressio.get_compressor(self.compressor_id.as_str()).unwrap(); compressor_clone @@ -86,7 +88,12 @@ impl Clone for PressioCompressorInner { .set_options(&compressor.get_options().unwrap()) .unwrap(); - std::mem::drop(compressor); + std::mem::drop(compressor_guard); + + let compressor_clone = match compressor_clone.try_into_sendable() { + Ok(compressor) => PressioCompressorSendable::Sendable(compressor), + Err((compressor, _err)) => PressioCompressorSendable::Fragile(Fragile::new(compressor)), + }; Self { compressor: Mutex::new(compressor_clone), @@ -96,6 +103,31 @@ impl Clone for PressioCompressorInner { } } +enum PressioCompressorSendable { + Sendable(libpressio::PressioSendableCompressor), + Fragile(Fragile), +} + +impl PressioCompressorSendable { + fn try_get(&self) -> Result<&libpressio::PressioCompressor, PressioCodecError> { + match self { + Self::Sendable(compressor) => Ok(compressor), + Self::Fragile(compressor) => compressor + .try_get() + .map_err(|_| PressioCodecError::PressioNonThreadsafeSend), + } + } + + fn try_get_mut(&mut self) -> Result<&mut libpressio::PressioCompressor, PressioCodecError> { + match self { + Self::Sendable(compressor) => Ok(compressor), + Self::Fragile(compressor) => compressor + .try_get_mut() + .map_err(|_| PressioCodecError::PressioNonThreadsafeSend), + } + } +} + impl Serialize for PressioCompressor { #[expect(clippy::too_many_lines)] fn serialize(&self, serializer: S) -> Result { @@ -196,7 +228,10 @@ impl Serialize for PressioCompressor { } let inner = self.inner.read().map_err(serde::ser::Error::custom)?; - let compressor = inner.compressor.lock().map_err(serde::ser::Error::custom)?; + let compressor_guard = inner.compressor.lock().map_err(serde::ser::Error::custom)?; + let compressor = compressor_guard + .try_get() + .map_err(serde::ser::Error::custom)?; let options = compressor .get_options() .map_err(serde::ser::Error::custom)?; @@ -216,7 +251,7 @@ impl Serialize for PressioCompressor { }, } .serialize(serializer); - std::mem::drop(compressor); + std::mem::drop(compressor_guard); std::mem::drop(inner); result } @@ -423,6 +458,11 @@ impl<'de> Deserialize<'de> for PressioCompressor { .set_options(&options) .map_err(serde::de::Error::custom)?; + let compressor = match compressor.try_into_sendable() { + Ok(compressor) => PressioCompressorSendable::Sendable(compressor), + Err((compressor, _err)) => PressioCompressorSendable::Fragile(Fragile::new(compressor)), + }; + Ok(Self { inner: RwLock::new(Arc::new(PressioCompressorInner { compressor: Mutex::new(compressor), @@ -665,6 +705,7 @@ impl Codec for PressioCodec { let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { return Err(PressioCodecError::PressioPoisonedLock); }; + let compressor = compressor.try_get_mut()?; match data { AnyCowArray::U8(data) => encode_typed(compressor, data), @@ -729,6 +770,7 @@ impl Codec for PressioCodec { let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { return Err(PressioCodecError::PressioPoisonedLock); }; + let compressor = compressor.try_get_mut()?; match encoded { AnyCowArray::U8(encoded) => decode_typed(compressor, encoded), @@ -844,6 +886,7 @@ impl Codec for PressioCodec { let Ok(compressor) = Arc::make_mut(&mut inner).compressor.get_mut() else { return Err(PressioCodecError::PressioPoisonedLock); }; + let compressor = compressor.try_get_mut()?; let decoded_dtype = match decoded.dtype() { AnyArrayDType::U8 => libpressio::PressioDtype::U8, @@ -933,6 +976,9 @@ pub enum PressioCodecError { /// [`PressioCodec`] lock was poisoned #[error("Pressio lock was poisoned")] PressioPoisonedLock, + /// [`PressioCodec`] was used on a different thread with a non-threadsafe compressor + #[error("Pressio was used on a different thread with a non-threadsafe compressor")] + PressioNonThreadsafeSend, /// [`PressioCodec`] failed to encode the data #[error("Pressio failed to encode the data")] PressioEncodeFailed { From adc5f231a4fda4436d9a533bf3ae38bd64f5ed21 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 06:39:53 +0300 Subject: [PATCH 44/72] Update libpressio-rs --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1b68f76ab..0e97a8ded 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "30545dd", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "39d2fcb", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From b23d52f5713ee5d9dcb89afcad9d4024efc227f1 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 07:02:51 +0300 Subject: [PATCH 45/72] Bump MSRV to 1.88 --- .github/workflows/ci.yml | 4 ++-- Cargo.toml | 4 ++-- README.md | 2 +- codecs/asinh/README.md | 2 +- codecs/asinh/src/lib.rs | 2 +- codecs/bit-round/README.md | 2 +- codecs/bit-round/src/lib.rs | 2 +- codecs/ebcc/README.md | 2 +- codecs/ebcc/src/lib.rs | 2 +- codecs/fixed-offset-scale/README.md | 2 +- codecs/fixed-offset-scale/src/lib.rs | 2 +- codecs/fourier-network/Cargo.toml | 2 +- codecs/fourier-network/README.md | 2 +- codecs/fourier-network/src/lib.rs | 4 ++-- codecs/identity/README.md | 2 +- codecs/identity/src/lib.rs | 2 +- codecs/jpeg2000/README.md | 2 +- codecs/jpeg2000/src/lib.rs | 2 +- codecs/lc/README.md | 2 +- codecs/lc/src/lib.rs | 2 +- codecs/linear-quantize/README.md | 2 +- codecs/linear-quantize/src/lib.rs | 2 +- codecs/log/README.md | 2 +- codecs/log/src/lib.rs | 2 +- codecs/pco/README.md | 2 +- codecs/pco/src/lib.rs | 2 +- codecs/qpet-sperr/README.md | 2 +- codecs/qpet-sperr/src/lib.rs | 2 +- codecs/random-projection/README.md | 2 +- codecs/random-projection/src/lib.rs | 2 +- codecs/reinterpret/README.md | 2 +- codecs/reinterpret/src/lib.rs | 2 +- codecs/round/README.md | 2 +- codecs/round/src/lib.rs | 2 +- codecs/sperr/README.md | 2 +- codecs/sperr/src/lib.rs | 2 +- codecs/stochastic-rounding/README.md | 2 +- codecs/stochastic-rounding/src/lib.rs | 2 +- codecs/swizzle-reshape/README.md | 2 +- codecs/swizzle-reshape/src/lib.rs | 2 +- codecs/sz3/README.md | 2 +- codecs/sz3/src/lib.rs | 2 +- codecs/tthresh/README.md | 2 +- codecs/tthresh/src/lib.rs | 2 +- codecs/uniform-noise/README.md | 2 +- codecs/uniform-noise/src/lib.rs | 2 +- codecs/zfp-classic/README.md | 2 +- codecs/zfp-classic/src/lib.rs | 2 +- codecs/zfp/README.md | 2 +- codecs/zfp/src/lib.rs | 2 +- codecs/zlib/README.md | 2 +- codecs/zlib/src/lib.rs | 2 +- codecs/zstd/README.md | 2 +- codecs/zstd/src/lib.rs | 2 +- crates/numcodecs-python/README.md | 2 +- crates/numcodecs-python/src/lib.rs | 2 +- crates/numcodecs-wasm-builder/README.md | 2 +- .../buildenv/flake.lock | 20 +++++++++---------- .../buildenv/rust-toolchain | 2 +- crates/numcodecs-wasm-guest/README.md | 2 +- crates/numcodecs-wasm-guest/src/lib.rs | 2 +- .../README.md | 2 +- .../src/lib.rs | 2 +- crates/numcodecs-wasm-host/README.md | 2 +- crates/numcodecs-wasm-host/src/lib.rs | 2 +- crates/numcodecs-wasm-logging/README.md | 2 +- crates/numcodecs-wasm-logging/src/lib.rs | 2 +- crates/numcodecs/README.md | 2 +- crates/numcodecs/src/lib.rs | 2 +- 69 files changed, 81 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ea7c8fe8..7b95c84c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - rust: ["1.87", stable, nightly] + rust: ["1.88", stable, nightly] lock: ["Cargo.lock", "Cargo.lock.min"] runs-on: ${{ matrix.os }} needs: lock @@ -218,7 +218,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - rust: ["1.87", stable] + rust: ["1.88", stable] lock: ["Cargo.lock", "Cargo.lock.min"] runs-on: ${{ matrix.os }} needs: lock diff --git a/Cargo.toml b/Cargo.toml index 0e97a8ded..f737ccaef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ edition = "2024" authors = ["Juniper Tyree "] repository = "https://github.com/juntyr/numcodecs-rs" license = "MPL-2.0" -rust-version = "1.87" +rust-version = "1.88" [workspace.dependencies] # workspace-internal numcodecs crates @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "39d2fcb", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "edf7511", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/README.md b/README.md index e5d4934b4..a103393eb 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs diff --git a/codecs/asinh/README.md b/codecs/asinh/README.md index 6db74d008..14cd41347 100644 --- a/codecs/asinh/README.md +++ b/codecs/asinh/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-asinh diff --git a/codecs/asinh/src/lib.rs b/codecs/asinh/src/lib.rs index 9c1baf762..d685145a5 100644 --- a/codecs/asinh/src/lib.rs +++ b/codecs/asinh/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-asinh diff --git a/codecs/bit-round/README.md b/codecs/bit-round/README.md index b07b2be3f..fca5fcd4a 100644 --- a/codecs/bit-round/README.md +++ b/codecs/bit-round/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-bit-round diff --git a/codecs/bit-round/src/lib.rs b/codecs/bit-round/src/lib.rs index 8bbb482ca..552d1956b 100644 --- a/codecs/bit-round/src/lib.rs +++ b/codecs/bit-round/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-bit-round diff --git a/codecs/ebcc/README.md b/codecs/ebcc/README.md index 1a72e510c..5678b9f94 100644 --- a/codecs/ebcc/README.md +++ b/codecs/ebcc/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-ebcc diff --git a/codecs/ebcc/src/lib.rs b/codecs/ebcc/src/lib.rs index ed2df3016..4efa4f9da 100644 --- a/codecs/ebcc/src/lib.rs +++ b/codecs/ebcc/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-ebcc diff --git a/codecs/fixed-offset-scale/README.md b/codecs/fixed-offset-scale/README.md index 8ab537fcc..73c0f3387 100644 --- a/codecs/fixed-offset-scale/README.md +++ b/codecs/fixed-offset-scale/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-fixed-offset-scale diff --git a/codecs/fixed-offset-scale/src/lib.rs b/codecs/fixed-offset-scale/src/lib.rs index 920c0f4e8..555aea69b 100644 --- a/codecs/fixed-offset-scale/src/lib.rs +++ b/codecs/fixed-offset-scale/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-fixed-offset-scale diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index c83f840fd..305f341a6 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -18,7 +18,7 @@ keywords = ["fourier", "network", "numcodecs", "compression", "encoding"] burn = { workspace = true, features = ["std", "autodiff", "ndarray"] } itertools = { workspace = true, features = ["use_alloc"] } log = { workspace = true } -# FIXME: bytemuck 1.24 fails to compile on 1.87 +# FIXME: bytemuck 1.24 fails to compile on 1.88 bytemuck = { version = "=1.23.2", default-features = false } ndarray = { workspace = true, features = ["std"] } numcodecs = { workspace = true } diff --git a/codecs/fourier-network/README.md b/codecs/fourier-network/README.md index ff45edd5b..76cfdb5e6 100644 --- a/codecs/fourier-network/README.md +++ b/codecs/fourier-network/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-fourier-network diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index 10dddef9f..d2d1121f4 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-fourier-network @@ -46,7 +46,7 @@ use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; -// FIXME: bytemuck 1.24 fails to compile on 1.87 +// FIXME: bytemuck 1.24 fails to compile on 1.88 use ::bytemuck as _; // FIXME: burn-common -> cubecl-common brings in wasm-bindgen diff --git a/codecs/identity/README.md b/codecs/identity/README.md index e7c0e3a88..a009d45c4 100644 --- a/codecs/identity/README.md +++ b/codecs/identity/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-identity diff --git a/codecs/identity/src/lib.rs b/codecs/identity/src/lib.rs index 92aa2d649..4535da6dc 100644 --- a/codecs/identity/src/lib.rs +++ b/codecs/identity/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-identity diff --git a/codecs/jpeg2000/README.md b/codecs/jpeg2000/README.md index 8bb11ff84..17c21fbc7 100644 --- a/codecs/jpeg2000/README.md +++ b/codecs/jpeg2000/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-jpeg2000 diff --git a/codecs/jpeg2000/src/lib.rs b/codecs/jpeg2000/src/lib.rs index 8f3211db7..6f00b7b02 100644 --- a/codecs/jpeg2000/src/lib.rs +++ b/codecs/jpeg2000/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-jpeg2000 diff --git a/codecs/lc/README.md b/codecs/lc/README.md index 3511eeaf2..c3c883a8c 100644 --- a/codecs/lc/README.md +++ b/codecs/lc/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-lc diff --git a/codecs/lc/src/lib.rs b/codecs/lc/src/lib.rs index 6c4ba3116..10e96fed4 100644 --- a/codecs/lc/src/lib.rs +++ b/codecs/lc/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-lc diff --git a/codecs/linear-quantize/README.md b/codecs/linear-quantize/README.md index c93b302c3..341c0ef2f 100644 --- a/codecs/linear-quantize/README.md +++ b/codecs/linear-quantize/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-linear-quantize diff --git a/codecs/linear-quantize/src/lib.rs b/codecs/linear-quantize/src/lib.rs index 1bb5a24c8..7803aecac 100644 --- a/codecs/linear-quantize/src/lib.rs +++ b/codecs/linear-quantize/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-linear-quantize diff --git a/codecs/log/README.md b/codecs/log/README.md index 8725ba54d..90a1d758c 100644 --- a/codecs/log/README.md +++ b/codecs/log/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-log diff --git a/codecs/log/src/lib.rs b/codecs/log/src/lib.rs index 46460daed..7061b4d5a 100644 --- a/codecs/log/src/lib.rs +++ b/codecs/log/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-log diff --git a/codecs/pco/README.md b/codecs/pco/README.md index faf730df4..e8ffc68c0 100644 --- a/codecs/pco/README.md +++ b/codecs/pco/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-pco diff --git a/codecs/pco/src/lib.rs b/codecs/pco/src/lib.rs index 5f577a1d9..0b680eb5f 100644 --- a/codecs/pco/src/lib.rs +++ b/codecs/pco/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-pco diff --git a/codecs/qpet-sperr/README.md b/codecs/qpet-sperr/README.md index 8a317c0d3..058ba764d 100644 --- a/codecs/qpet-sperr/README.md +++ b/codecs/qpet-sperr/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-qpet-sperr diff --git a/codecs/qpet-sperr/src/lib.rs b/codecs/qpet-sperr/src/lib.rs index 929b8a645..445bbe24a 100644 --- a/codecs/qpet-sperr/src/lib.rs +++ b/codecs/qpet-sperr/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-qpet-sperr diff --git a/codecs/random-projection/README.md b/codecs/random-projection/README.md index 1bad2c39d..43958665d 100644 --- a/codecs/random-projection/README.md +++ b/codecs/random-projection/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-random-projection diff --git a/codecs/random-projection/src/lib.rs b/codecs/random-projection/src/lib.rs index f30c81b59..4510758bc 100644 --- a/codecs/random-projection/src/lib.rs +++ b/codecs/random-projection/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-random-projection diff --git a/codecs/reinterpret/README.md b/codecs/reinterpret/README.md index a5329096a..8e05d6bf4 100644 --- a/codecs/reinterpret/README.md +++ b/codecs/reinterpret/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-reinterpret diff --git a/codecs/reinterpret/src/lib.rs b/codecs/reinterpret/src/lib.rs index 638415d53..a39d74735 100644 --- a/codecs/reinterpret/src/lib.rs +++ b/codecs/reinterpret/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-reinterpret diff --git a/codecs/round/README.md b/codecs/round/README.md index 662671c3c..0d2fc4cd8 100644 --- a/codecs/round/README.md +++ b/codecs/round/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-round diff --git a/codecs/round/src/lib.rs b/codecs/round/src/lib.rs index 549611d39..aed61938f 100644 --- a/codecs/round/src/lib.rs +++ b/codecs/round/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-round diff --git a/codecs/sperr/README.md b/codecs/sperr/README.md index 4fbeb4542..7384e6c95 100644 --- a/codecs/sperr/README.md +++ b/codecs/sperr/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-sperr diff --git a/codecs/sperr/src/lib.rs b/codecs/sperr/src/lib.rs index 7b6435c8f..8195807c5 100644 --- a/codecs/sperr/src/lib.rs +++ b/codecs/sperr/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-sperr diff --git a/codecs/stochastic-rounding/README.md b/codecs/stochastic-rounding/README.md index 747fb4b06..9c351028f 100644 --- a/codecs/stochastic-rounding/README.md +++ b/codecs/stochastic-rounding/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-stochastic-rounding diff --git a/codecs/stochastic-rounding/src/lib.rs b/codecs/stochastic-rounding/src/lib.rs index 497f3234b..475c7e5ca 100644 --- a/codecs/stochastic-rounding/src/lib.rs +++ b/codecs/stochastic-rounding/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-stochastic-rounding diff --git a/codecs/swizzle-reshape/README.md b/codecs/swizzle-reshape/README.md index b276e8dfa..711672cff 100644 --- a/codecs/swizzle-reshape/README.md +++ b/codecs/swizzle-reshape/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-swizzle-reshape diff --git a/codecs/swizzle-reshape/src/lib.rs b/codecs/swizzle-reshape/src/lib.rs index f3bc3fa21..850d4c4d9 100644 --- a/codecs/swizzle-reshape/src/lib.rs +++ b/codecs/swizzle-reshape/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-swizzle-reshape diff --git a/codecs/sz3/README.md b/codecs/sz3/README.md index 7773e5f90..81d5c224f 100644 --- a/codecs/sz3/README.md +++ b/codecs/sz3/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-sz3 diff --git a/codecs/sz3/src/lib.rs b/codecs/sz3/src/lib.rs index 444f45877..b783e5dc7 100644 --- a/codecs/sz3/src/lib.rs +++ b/codecs/sz3/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-sz3 diff --git a/codecs/tthresh/README.md b/codecs/tthresh/README.md index 0c6e8d05e..ef0e787c3 100644 --- a/codecs/tthresh/README.md +++ b/codecs/tthresh/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-tthresh diff --git a/codecs/tthresh/src/lib.rs b/codecs/tthresh/src/lib.rs index 72e8d782b..0eceb5de3 100644 --- a/codecs/tthresh/src/lib.rs +++ b/codecs/tthresh/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-tthresh diff --git a/codecs/uniform-noise/README.md b/codecs/uniform-noise/README.md index 24392d5de..e022b51aa 100644 --- a/codecs/uniform-noise/README.md +++ b/codecs/uniform-noise/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-uniform-noise diff --git a/codecs/uniform-noise/src/lib.rs b/codecs/uniform-noise/src/lib.rs index ac1e8436d..cccedf3fa 100644 --- a/codecs/uniform-noise/src/lib.rs +++ b/codecs/uniform-noise/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-uniform-noise diff --git a/codecs/zfp-classic/README.md b/codecs/zfp-classic/README.md index 9f032069e..b553bbae0 100644 --- a/codecs/zfp-classic/README.md +++ b/codecs/zfp-classic/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-zfp-classic diff --git a/codecs/zfp-classic/src/lib.rs b/codecs/zfp-classic/src/lib.rs index f008a6cc9..47522be98 100644 --- a/codecs/zfp-classic/src/lib.rs +++ b/codecs/zfp-classic/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-zfp-classic diff --git a/codecs/zfp/README.md b/codecs/zfp/README.md index 3537c137a..0c358801a 100644 --- a/codecs/zfp/README.md +++ b/codecs/zfp/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-zfp diff --git a/codecs/zfp/src/lib.rs b/codecs/zfp/src/lib.rs index 3c16e4f87..cda53b053 100644 --- a/codecs/zfp/src/lib.rs +++ b/codecs/zfp/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-zfp diff --git a/codecs/zlib/README.md b/codecs/zlib/README.md index 37ce9b5aa..63991b516 100644 --- a/codecs/zlib/README.md +++ b/codecs/zlib/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-zlib diff --git a/codecs/zlib/src/lib.rs b/codecs/zlib/src/lib.rs index ea93ab42b..69bc3d438 100644 --- a/codecs/zlib/src/lib.rs +++ b/codecs/zlib/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-zlib diff --git a/codecs/zstd/README.md b/codecs/zstd/README.md index d3da36c92..3f437cdc5 100644 --- a/codecs/zstd/README.md +++ b/codecs/zstd/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-zstd diff --git a/codecs/zstd/src/lib.rs b/codecs/zstd/src/lib.rs index fcd3cde55..1744666a3 100644 --- a/codecs/zstd/src/lib.rs +++ b/codecs/zstd/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-zstd diff --git a/crates/numcodecs-python/README.md b/crates/numcodecs-python/README.md index 6dd4b22f0..6f762507f 100644 --- a/crates/numcodecs-python/README.md +++ b/crates/numcodecs-python/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-python diff --git a/crates/numcodecs-python/src/lib.rs b/crates/numcodecs-python/src/lib.rs index e904f8a35..c46e0ac73 100644 --- a/crates/numcodecs-python/src/lib.rs +++ b/crates/numcodecs-python/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-python diff --git a/crates/numcodecs-wasm-builder/README.md b/crates/numcodecs-wasm-builder/README.md index 7b625b2f9..86d608d63 100644 --- a/crates/numcodecs-wasm-builder/README.md +++ b/crates/numcodecs-wasm-builder/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-builder diff --git a/crates/numcodecs-wasm-builder/buildenv/flake.lock b/crates/numcodecs-wasm-builder/buildenv/flake.lock index e32be7dbf..d11c7d73e 100644 --- a/crates/numcodecs-wasm-builder/buildenv/flake.lock +++ b/crates/numcodecs-wasm-builder/buildenv/flake.lock @@ -2,12 +2,12 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1758346548, - "narHash": "sha256-afXE7AJ7MY6wY1pg/Y6UPHNYPy5GtUKeBkrZZ/gC71E=", - "rev": "b2a3852bd078e68dd2b3dfa8c00c67af1f0a7d20", - "revCount": 810175, + "lastModified": 1776560675, + "narHash": "sha256-p68udKWWh7+V4ZPpcMDq0gTHWNZJnr4JPI+kHPPE40o=", + "rev": "e07580dae39738e46609eaab8b154de2488133ce", + "revCount": 911303, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2505.810175%2Brev-b2a3852bd078e68dd2b3dfa8c00c67af1f0a7d20/01996a9c-be96-7a98-a978-e5a9ecbd877f/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2511.911303%2Brev-e07580dae39738e46609eaab8b154de2488133ce/019db12b-b7e1-7762-aadd-382b14c5b218/source.tar.gz" }, "original": { "type": "tarball", @@ -41,12 +41,12 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1758508617, - "narHash": "sha256-kx2uELmVnAbiekj/YFfWR26OXqXedImkhe2ocnbumTA=", - "rev": "d2bac276ac7e669a1f09c48614538a37e3eb6d0f", - "revCount": 1924, + "lastModified": 1776827647, + "narHash": "sha256-sYixYhp5V8jCajO8TRorE4fzs7IkL4MZdfLTKgkPQBk=", + "rev": "40e6ccc06e1245a4837cbbd6bdda64e21cc67379", + "revCount": 2151, "type": "tarball", - "url": "https://api.flakehub.com/f/pinned/oxalica/rust-overlay/0.1.1924%2Brev-d2bac276ac7e669a1f09c48614538a37e3eb6d0f/01996f4a-36eb-7480-9a05-c8caee46ccc0/source.tar.gz" + "url": "https://api.flakehub.com/f/pinned/oxalica/rust-overlay/0.1.2151%2Brev-40e6ccc06e1245a4837cbbd6bdda64e21cc67379/019db32f-dc56-7292-a296-bbe69236be74/source.tar.gz" }, "original": { "type": "tarball", diff --git a/crates/numcodecs-wasm-builder/buildenv/rust-toolchain b/crates/numcodecs-wasm-builder/buildenv/rust-toolchain index 9c12d9413..5a7ad886f 100644 --- a/crates/numcodecs-wasm-builder/buildenv/rust-toolchain +++ b/crates/numcodecs-wasm-builder/buildenv/rust-toolchain @@ -1,5 +1,5 @@ [toolchain] -channel = "1.87" # MSRV +channel = "1.88" # MSRV components = [ "cargo", "rustfmt", "clippy", "rust-src" ] targets = [ "wasm32-wasip1" ] profile = "minimal" diff --git a/crates/numcodecs-wasm-guest/README.md b/crates/numcodecs-wasm-guest/README.md index 0bea44f3a..34337f695 100644 --- a/crates/numcodecs-wasm-guest/README.md +++ b/crates/numcodecs-wasm-guest/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-guest diff --git a/crates/numcodecs-wasm-guest/src/lib.rs b/crates/numcodecs-wasm-guest/src/lib.rs index 4db5416f2..82e2f2d59 100644 --- a/crates/numcodecs-wasm-guest/src/lib.rs +++ b/crates/numcodecs-wasm-guest/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-guest diff --git a/crates/numcodecs-wasm-host-reproducible/README.md b/crates/numcodecs-wasm-host-reproducible/README.md index e32257514..b8db18826 100644 --- a/crates/numcodecs-wasm-host-reproducible/README.md +++ b/crates/numcodecs-wasm-host-reproducible/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-host-reproducible diff --git a/crates/numcodecs-wasm-host-reproducible/src/lib.rs b/crates/numcodecs-wasm-host-reproducible/src/lib.rs index b3bec752e..3f721cead 100644 --- a/crates/numcodecs-wasm-host-reproducible/src/lib.rs +++ b/crates/numcodecs-wasm-host-reproducible/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-host-reproducible diff --git a/crates/numcodecs-wasm-host/README.md b/crates/numcodecs-wasm-host/README.md index e3d4c554c..df8e3333a 100644 --- a/crates/numcodecs-wasm-host/README.md +++ b/crates/numcodecs-wasm-host/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-host diff --git a/crates/numcodecs-wasm-host/src/lib.rs b/crates/numcodecs-wasm-host/src/lib.rs index bb0620450..6e60bfc9b 100644 --- a/crates/numcodecs-wasm-host/src/lib.rs +++ b/crates/numcodecs-wasm-host/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-host diff --git a/crates/numcodecs-wasm-logging/README.md b/crates/numcodecs-wasm-logging/README.md index 1f82a67f1..41bd659ec 100644 --- a/crates/numcodecs-wasm-logging/README.md +++ b/crates/numcodecs-wasm-logging/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-logging diff --git a/crates/numcodecs-wasm-logging/src/lib.rs b/crates/numcodecs-wasm-logging/src/lib.rs index 9f073e03e..7972decd3 100644 --- a/crates/numcodecs-wasm-logging/src/lib.rs +++ b/crates/numcodecs-wasm-logging/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-wasm-logging diff --git a/crates/numcodecs/README.md b/crates/numcodecs/README.md index 1b8ebaa18..7e29e802c 100644 --- a/crates/numcodecs/README.md +++ b/crates/numcodecs/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs diff --git a/crates/numcodecs/src/lib.rs b/crates/numcodecs/src/lib.rs index 7a8f84981..bb6d1d31d 100644 --- a/crates/numcodecs/src/lib.rs +++ b/crates/numcodecs/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs From f79eaab6b1228a68b35c58b4fde8c79acc73e75f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 07:10:18 +0300 Subject: [PATCH 46/72] Try removing bytemuck version pin --- codecs/fourier-network/Cargo.toml | 2 -- codecs/fourier-network/src/lib.rs | 3 --- 2 files changed, 5 deletions(-) diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index 305f341a6..2af3c83c1 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -18,8 +18,6 @@ keywords = ["fourier", "network", "numcodecs", "compression", "encoding"] burn = { workspace = true, features = ["std", "autodiff", "ndarray"] } itertools = { workspace = true, features = ["use_alloc"] } log = { workspace = true } -# FIXME: bytemuck 1.24 fails to compile on 1.88 -bytemuck = { version = "=1.23.2", default-features = false } ndarray = { workspace = true, features = ["std"] } numcodecs = { workspace = true } num-traits = { workspace = true, features = ["std"] } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index d2d1121f4..0ad9ff479 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -46,9 +46,6 @@ use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; -// FIXME: bytemuck 1.24 fails to compile on 1.88 -use ::bytemuck as _; - // FIXME: burn-common -> cubecl-common brings in wasm-bindgen // wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi use ::wasm_bindgen as _; From 9a2aed0044fe49525d1a86ca1d55ca77a9c58339 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 07:14:06 +0300 Subject: [PATCH 47/72] Try to remove wasm-bindgen pin --- codecs/fourier-network/Cargo.toml | 3 --- codecs/fourier-network/src/lib.rs | 4 ---- 2 files changed, 7 deletions(-) diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index 2af3c83c1..a52b4439a 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -24,9 +24,6 @@ num-traits = { workspace = true, features = ["std"] } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } -# FIXME: burn-common -> cubecl-common brings in wasm-bindgen -# wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi -wasm-bindgen = { version = "=0.2.114", default-features = false } [dev-dependencies] serde_json = { workspace = true, features = ["std"] } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index 0ad9ff479..91d101399 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -46,10 +46,6 @@ use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; -// FIXME: burn-common -> cubecl-common brings in wasm-bindgen -// wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi -use ::wasm_bindgen as _; - #[cfg(test)] use ::serde_json as _; From 81a72045aebcde5bb10f2322206b6e03a04fedc8 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 09:29:47 +0300 Subject: [PATCH 48/72] Fix clippy --- Cargo.toml | 2 +- codecs/ebcc/tests/schema.rs | 1 + codecs/fourier-network/Cargo.toml | 2 ++ codecs/fourier-network/src/lib.rs | 3 +++ codecs/fourier-network/tests/config.rs | 4 ++-- codecs/fourier-network/tests/schema.rs | 2 +- codecs/jpeg2000/tests/config.rs | 2 +- codecs/jpeg2000/tests/schema.rs | 1 + codecs/pco/tests/config.rs | 2 +- codecs/pco/tests/schema.rs | 1 + codecs/pressio/tests/schema.rs | 5 +++-- codecs/qpet-sperr/src/lib.rs | 6 +++++- codecs/qpet-sperr/tests/config.rs | 2 +- codecs/qpet-sperr/tests/schema.rs | 1 + codecs/random-projection/tests/config.rs | 2 +- codecs/random-projection/tests/schema.rs | 1 + codecs/round/src/lib.rs | 8 +++++--- codecs/sperr/tests/config.rs | 6 +++++- codecs/sperr/tests/schema.rs | 1 + codecs/stochastic-rounding/src/lib.rs | 1 + codecs/swizzle-reshape/src/lib.rs | 5 ++++- codecs/sz3/src/lib.rs | 1 + codecs/sz3/tests/config.rs | 2 +- codecs/sz3/tests/schema.rs | 1 + codecs/tthresh/tests/config.rs | 2 +- codecs/tthresh/tests/schema.rs | 1 + codecs/zfp-classic/tests/config.rs | 2 +- codecs/zfp-classic/tests/schema.rs | 1 + codecs/zfp/tests/config.rs | 2 +- codecs/zfp/tests/schema.rs | 1 + crates/numcodecs-python/src/schema.rs | 14 +++++++------- crates/numcodecs-wasm-guest/src/lib.rs | 1 - .../src/tests.rs | 5 +++-- crates/numcodecs/src/codec.rs | 18 ++++++++---------- 34 files changed, 69 insertions(+), 40 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f737ccaef..80114872c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "edf7511", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "018c80b", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/ebcc/tests/schema.rs b/codecs/ebcc/tests/schema.rs index a1b75551d..8d1d776ec 100644 --- a/codecs/ebcc/tests/schema.rs +++ b/codecs/ebcc/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("EBCC schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index a52b4439a..ae7661aa7 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -16,6 +16,8 @@ keywords = ["fourier", "network", "numcodecs", "compression", "encoding"] [dependencies] burn = { workspace = true, features = ["std", "autodiff", "ndarray"] } +# FIXME: bytemuck 1.24 fails to compile on 1.88 +bytemuck = { version = "=1.23.2", default-features = false } itertools = { workspace = true, features = ["use_alloc"] } log = { workspace = true } ndarray = { workspace = true, features = ["std"] } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index 91d101399..fb87d32ae 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -46,6 +46,9 @@ use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; +// FIXME: bytemuck 1.24 fails to compile on 1.88 +use ::bytemuck as _; + #[cfg(test)] use ::serde_json as _; diff --git a/codecs/fourier-network/tests/config.rs b/codecs/fourier-network/tests/config.rs index 30d0c2d09..7c12d78f9 100644 --- a/codecs/fourier-network/tests/config.rs +++ b/codecs/fourier-network/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use numcodecs::StaticCodec; use numcodecs_fourier_network::FourierNetworkCodec; @@ -7,7 +7,7 @@ use serde_json::json; use ::{ burn as _, bytemuck as _, itertools as _, log as _, ndarray as _, num_traits as _, - schemars as _, simple_logger as _, thiserror as _, wasm_bindgen as _, + schemars as _, simple_logger as _, thiserror as _, }; #[test] diff --git a/codecs/fourier-network/tests/schema.rs b/codecs/fourier-network/tests/schema.rs index aec0f709a..4d9eeb338 100644 --- a/codecs/fourier-network/tests/schema.rs +++ b/codecs/fourier-network/tests/schema.rs @@ -6,7 +6,6 @@ use numcodecs_fourier_network::FourierNetworkCodec; use ::{ burn as _, bytemuck as _, itertools as _, log as _, ndarray as _, num_traits as _, schemars as _, serde as _, serde_json as _, simple_logger as _, thiserror as _, - wasm_bindgen as _, }; #[test] @@ -18,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("FourierNetwork schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/jpeg2000/tests/config.rs b/codecs/jpeg2000/tests/config.rs index 647901222..deb0d9e5b 100644 --- a/codecs/jpeg2000/tests/config.rs +++ b/codecs/jpeg2000/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ log as _, ndarray as _, num_traits as _, numcodecs_jpeg2000::Jpeg2000CompressionMode, diff --git a/codecs/jpeg2000/tests/schema.rs b/codecs/jpeg2000/tests/schema.rs index 393791d5b..474b627f7 100644 --- a/codecs/jpeg2000/tests/schema.rs +++ b/codecs/jpeg2000/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Jpeg2000 schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/pco/tests/config.rs b/codecs/pco/tests/config.rs index 37fb7eece..629dd771c 100644 --- a/codecs/pco/tests/config.rs +++ b/codecs/pco/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use std::num::{NonZero, NonZeroUsize}; diff --git a/codecs/pco/tests/schema.rs b/codecs/pco/tests/schema.rs index e69a83500..2e1af9925 100644 --- a/codecs/pco/tests/schema.rs +++ b/codecs/pco/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Pcodec schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/pressio/tests/schema.rs b/codecs/pressio/tests/schema.rs index ae8dde7ff..ec6b74557 100644 --- a/codecs/pressio/tests/schema.rs +++ b/codecs/pressio/tests/schema.rs @@ -1,8 +1,8 @@ #![expect(missing_docs)] use ::{ - libpressio as _, ndarray as _, schemars as _, serde as _, serde_json as _, serde_ndim as _, - thiserror as _, + fragile as _, libpressio as _, ndarray as _, schemars as _, serde as _, serde_json as _, + serde_ndim as _, thiserror as _, }; use numcodecs::{DynCodecType, StaticCodecType}; @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Pressio schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/qpet-sperr/src/lib.rs b/codecs/qpet-sperr/src/lib.rs index 445bbe24a..f90764dfc 100644 --- a/codecs/qpet-sperr/src/lib.rs +++ b/codecs/qpet-sperr/src/lib.rs @@ -602,6 +602,7 @@ mod tests { #[test] fn all_modes() { + #[expect(clippy::single_element_loop)] for mode in [QpetSperrCompressionMode::SymbolicQuantityOfInterest { qoi: String::from("x^2"), qoi_block_size: default_qoi_block_size(), @@ -685,7 +686,10 @@ mod tests { let decoded = decompress(&encoded).unwrap(); assert_eq!(decoded.dtype(), AnyArrayDType::F64); - assert_eq!(decoded.len(), 64 * 64 * 1); + #[expect(clippy::identity_op)] + { + assert_eq!(decoded.len(), 64 * 64 * 1); + } assert_eq!(decoded.shape(), &[64, 64, 1]); } } diff --git a/codecs/qpet-sperr/tests/config.rs b/codecs/qpet-sperr/tests/config.rs index ac5e7a876..662a04179 100644 --- a/codecs/qpet-sperr/tests/config.rs +++ b/codecs/qpet-sperr/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ ndarray as _, num_traits as _, postcard as _, qpet_sperr as _, schemars as _, thiserror as _, diff --git a/codecs/qpet-sperr/tests/schema.rs b/codecs/qpet-sperr/tests/schema.rs index 6b95744e1..ff9746b8c 100644 --- a/codecs/qpet-sperr/tests/schema.rs +++ b/codecs/qpet-sperr/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("QPET-SPERR schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/random-projection/tests/config.rs b/codecs/random-projection/tests/config.rs index 728223ba8..102527d63 100644 --- a/codecs/random-projection/tests/config.rs +++ b/codecs/random-projection/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use numcodecs::StaticCodec; use numcodecs_random_projection::RandomProjectionCodec; diff --git a/codecs/random-projection/tests/schema.rs b/codecs/random-projection/tests/schema.rs index 8b42d1613..cf0d03370 100644 --- a/codecs/random-projection/tests/schema.rs +++ b/codecs/random-projection/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("RandomProjection schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/round/src/lib.rs b/codecs/round/src/lib.rs index aed61938f..91eb806f6 100644 --- a/codecs/round/src/lib.rs +++ b/codecs/round/src/lib.rs @@ -198,6 +198,7 @@ mod tests { } #[test] + #[expect(clippy::float_cmp)] fn round_minimal_precision() { let data = array![0.1, 1.0, 11.0, 21.0]; @@ -219,11 +220,11 @@ mod tests { 0.0, 0.1, 0.2, - 0.30000000000000004, + 0.300_000_000_000_000_04, 0.4, 0.5, - 0.6000000000000001, - 0.7000000000000001, + 0.600_000_000_000_000_1, + 0.700_000_000_000_000_1, 0.8, 0.9, 1.0 @@ -250,6 +251,7 @@ mod tests { let rounded = round(data.view(), NonNegative(1.0)); + #[expect(clippy::float_cmp)] for (d, r) in data.into_iter().zip(rounded) { assert!(d == r || d.to_bits() == r.to_bits()); } diff --git a/codecs/sperr/tests/config.rs b/codecs/sperr/tests/config.rs index aa471e553..e20ca5139 100644 --- a/codecs/sperr/tests/config.rs +++ b/codecs/sperr/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ndarray as _, num_traits as _, postcard as _, schemars as _, sperr as _, thiserror as _}; @@ -14,6 +14,7 @@ fn empty_config() { } #[test] +#[expect(clippy::float_cmp)] fn bpp_config() { let codec = SperrCodec::from_config( Deserialize::deserialize(json!({ @@ -30,6 +31,7 @@ fn bpp_config() { } #[test] +#[expect(clippy::float_cmp)] fn psnr_config() { let codec = SperrCodec::from_config( Deserialize::deserialize(json!({ @@ -46,6 +48,7 @@ fn psnr_config() { } #[test] +#[expect(clippy::float_cmp)] fn pwe_config() { let codec = SperrCodec::from_config( Deserialize::deserialize(json!({ @@ -62,6 +65,7 @@ fn pwe_config() { } #[test] +#[expect(clippy::float_cmp)] fn q_config() { let codec = SperrCodec::from_config( Deserialize::deserialize(json!({ diff --git a/codecs/sperr/tests/schema.rs b/codecs/sperr/tests/schema.rs index b25694ea2..1e2076f3b 100644 --- a/codecs/sperr/tests/schema.rs +++ b/codecs/sperr/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Sperr schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/stochastic-rounding/src/lib.rs b/codecs/stochastic-rounding/src/lib.rs index 475c7e5ca..29844d212 100644 --- a/codecs/stochastic-rounding/src/lib.rs +++ b/codecs/stochastic-rounding/src/lib.rs @@ -333,6 +333,7 @@ mod tests { } #[test] + #[expect(clippy::float_cmp)] fn round_minimal_precision() { let data = array![0.1, 1.0, 11.0, 21.0]; diff --git a/codecs/swizzle-reshape/src/lib.rs b/codecs/swizzle-reshape/src/lib.rs index 850d4c4d9..ed8d3c3a4 100644 --- a/codecs/swizzle-reshape/src/lib.rs +++ b/codecs/swizzle-reshape/src/lib.rs @@ -779,7 +779,10 @@ mod tests { Axis::Index(2), Axis::Index(1), ])], - &[1 * 3 * 1 * 721 * 1440], + #[expect(clippy::identity_op)] + { + &[1 * 3 * 1 * 721 * 1440] + }, ); } diff --git a/codecs/sz3/src/lib.rs b/codecs/sz3/src/lib.rs index b783e5dc7..e7c76ce0a 100644 --- a/codecs/sz3/src/lib.rs +++ b/codecs/sz3/src/lib.rs @@ -859,6 +859,7 @@ mod tests { } #[test] + #[expect(clippy::redundant_closure)] // FIXME fn all_dtypes() -> Result<(), Sz3CodecError> { fn compress_decompress( iter: impl Clone + IntoIterator, diff --git a/codecs/sz3/tests/config.rs b/codecs/sz3/tests/config.rs index aa5bda0f2..e34d85ff4 100644 --- a/codecs/sz3/tests/config.rs +++ b/codecs/sz3/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ ndarray as _, num_traits as _, postcard as _, schemars as _, sz3 as _, thiserror as _, diff --git a/codecs/sz3/tests/schema.rs b/codecs/sz3/tests/schema.rs index 9e40080e0..0aab3214a 100644 --- a/codecs/sz3/tests/schema.rs +++ b/codecs/sz3/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Sz3 schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/tthresh/tests/config.rs b/codecs/tthresh/tests/config.rs index 382fc43ab..3dbeb7fbf 100644 --- a/codecs/tthresh/tests/config.rs +++ b/codecs/tthresh/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ndarray as _, num_traits as _, schemars as _, thiserror as _, tthresh as _}; diff --git a/codecs/tthresh/tests/schema.rs b/codecs/tthresh/tests/schema.rs index 20327bf24..5a5d9167a 100644 --- a/codecs/tthresh/tests/schema.rs +++ b/codecs/tthresh/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Tthresh schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/zfp-classic/tests/config.rs b/codecs/zfp-classic/tests/config.rs index e21ddc33e..944744952 100644 --- a/codecs/zfp-classic/tests/config.rs +++ b/codecs/zfp-classic/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ndarray as _, postcard as _, schemars as _, thiserror as _, zfp_sys as _}; diff --git a/codecs/zfp-classic/tests/schema.rs b/codecs/zfp-classic/tests/schema.rs index 053227517..d6240fd48 100644 --- a/codecs/zfp-classic/tests/schema.rs +++ b/codecs/zfp-classic/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("ZfpClassic schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/zfp/tests/config.rs b/codecs/zfp/tests/config.rs index dee661f09..ffe89d915 100644 --- a/codecs/zfp/tests/config.rs +++ b/codecs/zfp/tests/config.rs @@ -1,4 +1,4 @@ -#![expect(missing_docs)] +#![expect(missing_docs, clippy::unwrap_used)] use ::{ndarray as _, postcard as _, schemars as _, thiserror as _, zfp_sys as _}; diff --git a/codecs/zfp/tests/schema.rs b/codecs/zfp/tests/schema.rs index 9d8bf4d7b..be941cab9 100644 --- a/codecs/zfp/tests/schema.rs +++ b/codecs/zfp/tests/schema.rs @@ -17,6 +17,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Zfp schema has changed\n===\n{schema}\n==="); } diff --git a/crates/numcodecs-python/src/schema.rs b/crates/numcodecs-python/src/schema.rs index e43ad0261..ea728e8d3 100644 --- a/crates/numcodecs-python/src/schema.rs +++ b/crates/numcodecs-python/src/schema.rs @@ -108,13 +108,13 @@ pub fn schema_from_codec_class( schema.insert(String::from("additionalProperties"), Value::Bool(true)); } - if let Ok(doc) = class.getattr(intern!(py, "__doc__")) { - if !doc.is_none() { - let doc: String = doc - .extract() - .map_err(|err| SchemaError::InvalidClassDocs { source: err })?; - schema.insert(String::from("description"), Value::String(doc)); - } + if let Ok(doc) = class.getattr(intern!(py, "__doc__")) + && !doc.is_none() + { + let doc: String = doc + .extract() + .map_err(|err| SchemaError::InvalidClassDocs { source: err })?; + schema.insert(String::from("description"), Value::String(doc)); } let name = class diff --git a/crates/numcodecs-wasm-guest/src/lib.rs b/crates/numcodecs-wasm-guest/src/lib.rs index 82e2f2d59..47380b14a 100644 --- a/crates/numcodecs-wasm-guest/src/lib.rs +++ b/crates/numcodecs-wasm-guest/src/lib.rs @@ -44,7 +44,6 @@ use crate::convert::{ }; #[doc(hidden)] -#[expect(clippy::same_length_and_capacity)] pub mod bindings { #[cfg(not(feature = "registry"))] wit_bindgen::generate!({ diff --git a/crates/numcodecs-wasm-host-reproducible/src/tests.rs b/crates/numcodecs-wasm-host-reproducible/src/tests.rs index d22978864..260eab5d5 100644 --- a/crates/numcodecs-wasm-host-reproducible/src/tests.rs +++ b/crates/numcodecs-wasm-host-reproducible/src/tests.rs @@ -13,6 +13,7 @@ const MEMORY_GUARD_SIZE: u32 = WASM_PAGE_SIZE * 16 * 64 /* 64MiB */; const MEMORY_RESERVATION_FOR_GROWTH: u32 = WASM_PAGE_SIZE * 16 * 64 /* 64MiB */; #[test] +#[expect(clippy::unwrap_used, clippy::panic)] fn codec_roundtrip() { // keep in sync with numcodecs-wasm let mut config = wasmtime::Config::new(); @@ -71,7 +72,7 @@ fn codec_roundtrip() { let data = Array::random((256, 256), Normal::new(0.0, 1.0).unwrap()); - let encoded = match codec.encode(numcodecs::AnyArray::F64(data.clone().into_dyn()).into_cow()) { + let encoded = match codec.encode(numcodecs::AnyArray::F64(data.into_dyn()).into_cow()) { Ok(encoded) => encoded, Err(err) => panic!( "ReproducibleWasmCodec::encode:\n===\n{err}\n===\n{err:?}\n===\n{err:#}\n===\n{err:#?}\n===\n" @@ -85,7 +86,7 @@ fn codec_roundtrip() { Err(err) => panic!( "ReproducibleWasmCodec::decode_into:\n===\n{err}\n===\n{err:?}\n===\n{err:#}\n===\n{err:#?}\n===\n" ), - }; + } let decoded = match codec.decode(encoded.into_cow()) { Ok(decoded) => decoded, diff --git a/crates/numcodecs/src/codec.rs b/crates/numcodecs/src/codec.rs index ee87e241d..0e144615c 100644 --- a/crates/numcodecs/src/codec.rs +++ b/crates/numcodecs/src/codec.rs @@ -283,16 +283,14 @@ pub fn codec_from_config_with_id<'de, T: DynCodecType, D: Deserializer<'de>>( ) -> Result { let mut config = Value::deserialize(config)?; - if let Some(config) = config.as_object_mut() { - if let Some(id) = config.remove("id") { - let codec_id = ty.codec_id(); - - if !matches!(id, Value::String(ref id) if id == codec_id) { - return Err(serde::de::Error::custom(format!( - "expected codec id {codec_id:?} but found {id}" - ))); - } - } + if let Some(config) = config.as_object_mut() + && let Some(id) = config.remove("id") + && let codec_id = ty.codec_id() + && !matches!(id, Value::String(ref id) if id == codec_id) + { + return Err(serde::de::Error::custom(format!( + "expected codec id {codec_id:?} but found {id}" + ))); } ty.codec_from_config(config) From 4887d2845fb5e0a293a4d851852bc5891f1b3ebf Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 09:41:14 +0300 Subject: [PATCH 49/72] add serde(deny_unknown_fields) for pressio config --- codecs/fourier-network/Cargo.toml | 3 +++ codecs/fourier-network/src/lib.rs | 4 ++++ codecs/fourier-network/tests/config.rs | 2 +- codecs/fourier-network/tests/schema.rs | 1 + codecs/pressio/src/lib.rs | 2 ++ codecs/pressio/tests/schema.json | 6 +++--- 6 files changed, 14 insertions(+), 4 deletions(-) diff --git a/codecs/fourier-network/Cargo.toml b/codecs/fourier-network/Cargo.toml index ae7661aa7..ccf8f94d6 100644 --- a/codecs/fourier-network/Cargo.toml +++ b/codecs/fourier-network/Cargo.toml @@ -26,6 +26,9 @@ num-traits = { workspace = true, features = ["std"] } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } +# FIXME: burn-common -> cubecl-common brings in wasm-bindgen +# wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi +wasm-bindgen = { version = "=0.2.114", default-features = false } [dev-dependencies] serde_json = { workspace = true, features = ["std"] } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index fb87d32ae..d2d1121f4 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -49,6 +49,10 @@ use thiserror::Error; // FIXME: bytemuck 1.24 fails to compile on 1.88 use ::bytemuck as _; +// FIXME: burn-common -> cubecl-common brings in wasm-bindgen +// wasm-bindgen v0.2.115 has an unresolved import in wasm32-wasi +use ::wasm_bindgen as _; + #[cfg(test)] use ::serde_json as _; diff --git a/codecs/fourier-network/tests/config.rs b/codecs/fourier-network/tests/config.rs index 7c12d78f9..ab78ad450 100644 --- a/codecs/fourier-network/tests/config.rs +++ b/codecs/fourier-network/tests/config.rs @@ -7,7 +7,7 @@ use serde_json::json; use ::{ burn as _, bytemuck as _, itertools as _, log as _, ndarray as _, num_traits as _, - schemars as _, simple_logger as _, thiserror as _, + schemars as _, simple_logger as _, thiserror as _, wasm_bindgen as _, }; #[test] diff --git a/codecs/fourier-network/tests/schema.rs b/codecs/fourier-network/tests/schema.rs index 4d9eeb338..7ce541e06 100644 --- a/codecs/fourier-network/tests/schema.rs +++ b/codecs/fourier-network/tests/schema.rs @@ -6,6 +6,7 @@ use numcodecs_fourier_network::FourierNetworkCodec; use ::{ burn as _, bytemuck as _, itertools as _, log as _, ndarray as _, num_traits as _, schemars as _, serde as _, serde_json as _, simple_logger as _, thiserror as _, + wasm_bindgen as _, }; #[test] diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 2da4148a3..ea1e281c2 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -485,6 +485,7 @@ impl JsonSchema for PressioCompressor { #[derive(Debug, Deserialize, JsonSchema)] #[serde(rename = "PressioCompressor")] +#[serde(deny_unknown_fields)] struct PressioCompressorOwnedFormat { /// The id of the compressor compressor_id: String, @@ -504,6 +505,7 @@ struct PressioCompressorOwnedFormat { #[derive(Debug, Serialize)] #[serde(rename = "PressioCompressor")] +#[serde(deny_unknown_fields)] struct PressioCompressorBorrowedFormat<'a> { /// The id of the compressor compressor_id: &'a str, diff --git a/codecs/pressio/tests/schema.json b/codecs/pressio/tests/schema.json index 0cec2dfe5..7cb424a94 100644 --- a/codecs/pressio/tests/schema.json +++ b/codecs/pressio/tests/schema.json @@ -1,6 +1,9 @@ { "type": "object", "additionalProperties": false, + "required": [ + "compressor_id" + ], "properties": { "compressor_id": { "type": "string", @@ -852,9 +855,6 @@ "default": "1.0.0" } }, - "required": [ - "compressor_id" - ], "description": "Pressio codec which applies the identity function, i.e. passes through the\ninput unchanged during encoding and decoding.", "title": "PressioCodec", "$schema": "https://json-schema.org/draft/2020-12/schema", From 323ca5ae6149c73c6317d05f332bb979a8edf2eb Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 22 Apr 2026 10:03:40 +0300 Subject: [PATCH 50/72] Fix clippy --- crates/numcodecs-wasm-guest/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/numcodecs-wasm-guest/src/lib.rs b/crates/numcodecs-wasm-guest/src/lib.rs index 47380b14a..82e2f2d59 100644 --- a/crates/numcodecs-wasm-guest/src/lib.rs +++ b/crates/numcodecs-wasm-guest/src/lib.rs @@ -44,6 +44,7 @@ use crate::convert::{ }; #[doc(hidden)] +#[expect(clippy::same_length_and_capacity)] pub mod bindings { #[cfg(not(feature = "registry"))] wit_bindgen::generate!({ From 40762d7401298bf5b548c517fb8193365ddad80c Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 23 Apr 2026 10:11:25 +0300 Subject: [PATCH 51/72] Improve codec schema inlining --- Cargo.toml | 4 +- codecs/ebcc/src/lib.rs | 4 + codecs/fourier-network/src/lib.rs | 4 + codecs/lc/src/lib.rs | 6 + codecs/lc/tests/schema.json | 192 +-- codecs/pco/src/lib.rs | 3 + codecs/pressio/tests/schema.json | 1220 +++----------------- codecs/qpet-sperr/src/lib.rs | 4 + codecs/random-projection/src/lib.rs | 10 +- codecs/random-projection/tests/schema.json | 19 +- codecs/sperr/src/lib.rs | 4 + codecs/swizzle-reshape/Cargo.toml | 3 + codecs/swizzle-reshape/src/lib.rs | 9 + codecs/swizzle-reshape/tests/schema.json | 58 + codecs/swizzle-reshape/tests/schema.rs | 21 + codecs/sz3/src/lib.rs | 1 + codecs/tthresh/src/lib.rs | 4 + codecs/zfp-classic/src/lib.rs | 1 + codecs/zfp/src/lib.rs | 1 + crates/numcodecs-wasm-guest/Cargo.toml | 2 +- crates/numcodecs-wasm-guest/src/lib.rs | 8 +- crates/numcodecs/Cargo.toml | 2 +- crates/numcodecs/src/codec.rs | 16 +- 23 files changed, 355 insertions(+), 1241 deletions(-) create mode 100644 codecs/swizzle-reshape/tests/schema.json create mode 100644 codecs/swizzle-reshape/tests/schema.rs diff --git a/Cargo.toml b/Cargo.toml index 80114872c..b128c2d2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,11 +50,11 @@ rust-version = "1.88" [workspace.dependencies] # workspace-internal numcodecs crates -numcodecs = { version = "0.3.2", path = "crates/numcodecs", default-features = false } +numcodecs = { version = "0.3.3", path = "crates/numcodecs", default-features = false } numcodecs-python = { version = "0.7.1", path = "crates/numcodecs-python", default-features = false } numcodecs-registry = { version = "0.1", path = "crates/numcodecs-registry", default-features = false } numcodecs-wasm-builder = { version = "0.2", path = "crates/numcodecs-wasm-builder", default-features = false } -numcodecs-wasm-guest = { version = "0.3.1", path = "crates/numcodecs-wasm-guest", default-features = false } +numcodecs-wasm-guest = { version = "0.3.2", path = "crates/numcodecs-wasm-guest", default-features = false } numcodecs-wasm-host = { version = "0.2.1", path = "crates/numcodecs-wasm-host", default-features = false } numcodecs-wasm-host-reproducible = { version = "0.2.3", path = "crates/numcodecs-wasm-host-reproducible", default-features = false } numcodecs-wasm-logging = { version = "0.2", path = "crates/numcodecs-wasm-logging", default-features = false } diff --git a/codecs/ebcc/src/lib.rs b/codecs/ebcc/src/lib.rs index 4efa4f9da..736dabaa1 100644 --- a/codecs/ebcc/src/lib.rs +++ b/codecs/ebcc/src/lib.rs @@ -290,6 +290,10 @@ impl<'de> Deserialize<'de> for Positive { } impl JsonSchema for Positive { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("PositiveF32") } diff --git a/codecs/fourier-network/src/lib.rs b/codecs/fourier-network/src/lib.rs index d2d1121f4..b6489a18b 100644 --- a/codecs/fourier-network/src/lib.rs +++ b/codecs/fourier-network/src/lib.rs @@ -222,6 +222,10 @@ impl<'de> Deserialize<'de> for Positive { } impl JsonSchema for Positive { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("PositiveF64") } diff --git a/codecs/lc/src/lib.rs b/codecs/lc/src/lib.rs index 10e96fed4..71b1f6b01 100644 --- a/codecs/lc/src/lib.rs +++ b/codecs/lc/src/lib.rs @@ -75,6 +75,7 @@ fn deserialize_components<'de, D: Deserializer<'de>>( #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] #[serde(tag = "id")] +#[schemars(inline)] /// LC preprocessor pub enum LcPreprocessor { #[serde(rename = "NUL")] @@ -118,6 +119,7 @@ impl LcPreprocessor { #[derive( Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, )] +#[schemars(inline)] /// LC error bound kind pub enum LcErrorKind { /// pointwise absolute error bound @@ -145,6 +147,7 @@ impl LcErrorKind { #[derive( Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, )] +#[schemars(inline)] /// LC quantisation decorrelation mode pub enum LcDecorrelation { #[serde(rename = "0")] @@ -166,6 +169,7 @@ impl LcDecorrelation { #[derive( Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, )] +#[schemars(inline)] /// LC Lorenzo preprocessor dtype pub enum LcLorenzoDtype { #[serde(rename = "i32")] @@ -184,6 +188,7 @@ impl LcLorenzoDtype { #[derive( Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, JsonSchema, )] +#[schemars(inline)] /// LC quantization dtype pub enum LcQuantizeDType { #[serde(rename = "f32")] @@ -207,6 +212,7 @@ impl LcQuantizeDType { )] #[serde(deny_unknown_fields)] #[serde(tag = "id")] +#[schemars(inline)] /// LC component pub enum LcComponent { #[serde(rename = "NUL")] diff --git a/codecs/lc/tests/schema.json b/codecs/lc/tests/schema.json index 3de092cbf..2c29e9cf0 100644 --- a/codecs/lc/tests/schema.json +++ b/codecs/lc/tests/schema.json @@ -132,14 +132,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -156,14 +149,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -180,12 +166,7 @@ "type": "object", "properties": { "size": { - "description": "LC component float element size, in bytes", - "type": "integer", - "enum": [ - 4, - 8 - ] + "$ref": "#/$defs/LcFloatSize" }, "id": { "type": "string", @@ -202,12 +183,7 @@ "type": "object", "properties": { "size": { - "description": "LC component float element size, in bytes", - "type": "integer", - "enum": [ - 4, - 8 - ] + "$ref": "#/$defs/LcFloatSize" }, "id": { "type": "string", @@ -224,14 +200,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -248,24 +217,7 @@ "type": "object", "properties": { "size": { - "description": "LC tuple component element size, in tuple length _ bytes", - "type": "string", - "enum": [ - "2_1", - "3_1", - "4_1", - "6_1", - "8_1", - "12_1", - "2_2", - "3_2", - "4_2", - "6_2", - "2_4", - "6_4", - "3_8", - "6_8" - ] + "$ref": "#/$defs/LcTupleSize" }, "id": { "type": "string", @@ -282,14 +234,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -306,14 +251,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -330,14 +268,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -354,14 +285,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -378,14 +302,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -402,14 +319,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -426,14 +336,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -450,14 +353,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -474,14 +370,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -498,14 +387,7 @@ "type": "object", "properties": { "size": { - "description": "LC component element size, in bytes", - "type": "integer", - "enum": [ - 1, - 2, - 4, - 8 - ] + "$ref": "#/$defs/LcElemSize" }, "id": { "type": "string", @@ -536,5 +418,45 @@ ], "description": "Codec providing compression using LC", "title": "LcCodec", - "$schema": "https://json-schema.org/draft/2020-12/schema" + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "LcElemSize": { + "type": "integer", + "enum": [ + 1, + 2, + 4, + 8 + ], + "description": "LC component element size, in bytes" + }, + "LcFloatSize": { + "type": "integer", + "enum": [ + 4, + 8 + ], + "description": "LC component float element size, in bytes" + }, + "LcTupleSize": { + "type": "string", + "enum": [ + "2_1", + "3_1", + "4_1", + "6_1", + "8_1", + "12_1", + "2_2", + "3_2", + "4_2", + "6_2", + "2_4", + "6_4", + "3_8", + "6_8" + ], + "description": "LC tuple component element size, in tuple length _ bytes" + } + } } \ No newline at end of file diff --git a/codecs/pco/src/lib.rs b/codecs/pco/src/lib.rs index 0b680eb5f..b347b77dc 100644 --- a/codecs/pco/src/lib.rs +++ b/codecs/pco/src/lib.rs @@ -61,6 +61,7 @@ pub struct Pcodec { Copy, Clone, Debug, Default, PartialEq, Eq, Serialize_repr, Deserialize_repr, JsonSchema_repr, )] #[repr(u8)] +#[schemars(inline)] /// Pco compression level. /// /// The level ranges from 0 to 12 inclusive (default: 8): @@ -167,6 +168,7 @@ pub enum PcoDeltaSpec { #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr, JsonSchema_repr)] #[repr(u8)] +#[schemars(inline)] /// Pco delta encoding order for consecutive deltas. /// /// The order ranges from 0 to 7 inclusive. @@ -184,6 +186,7 @@ pub enum PcoDeltaEncodingConsecutiveOrder { #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize_repr, Deserialize_repr, JsonSchema_repr)] #[repr(u8)] +#[schemars(inline)] /// Pco delta encoding order for convolution deltas. /// /// The order ranges from 0 to 32 inclusive. diff --git a/codecs/pressio/tests/schema.json b/codecs/pressio/tests/schema.json index 7cb424a94..fc99ab36d 100644 --- a/codecs/pressio/tests/schema.json +++ b/codecs/pressio/tests/schema.json @@ -12,276 +12,7 @@ "early_config": { "type": "object", "additionalProperties": { - "anyOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - }, - { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - }, - { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - }, - { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - }, - { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - { - "type": "integer", - "format": "int32" - }, - { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - { - "type": "integer", - "format": "int64" - }, - { - "type": "number", - "format": "float" - }, - { - "type": "number", - "format": "double" - }, - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/boolNdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f64NdArray" - } - } - ] - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/PressioOption" - } - } - ], - "description": "Pressio option value" + "$ref": "#/$defs/PressioOption" }, "description": "Configuration for the structure of the compressor", "default": {} @@ -289,276 +20,7 @@ "compressor_config": { "type": "object", "additionalProperties": { - "anyOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - }, - { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - }, - { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - }, - { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - }, - { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - { - "type": "integer", - "format": "int32" - }, - { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - { - "type": "integer", - "format": "int64" - }, - { - "type": "number", - "format": "float" - }, - { - "type": "number", - "format": "double" - }, - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/boolNdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f64NdArray" - } - } - ] - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/PressioOption" - } - } - ], - "description": "Pressio option value" + "$ref": "#/$defs/PressioOption" }, "description": "Configuration for the compressor", "default": {} @@ -566,276 +28,7 @@ "metric_results": { "type": "object", "additionalProperties": { - "anyOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - }, - { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - }, - { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - }, - { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - }, - { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - { - "type": "integer", - "format": "int32" - }, - { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - { - "type": "integer", - "format": "int64" - }, - { - "type": "number", - "format": "float" - }, - { - "type": "number", - "format": "double" - }, - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/boolNdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f64NdArray" - } - } - ] - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/PressioOption" - } - } - ], - "description": "Pressio option value" + "$ref": "#/$defs/PressioOption" }, "description": "Results of the compressor metrics (output-only)", "default": {} @@ -859,6 +52,141 @@ "title": "PressioCodec", "$schema": "https://json-schema.org/draft/2020-12/schema", "$defs": { + "PressioOption": { + "anyOf": [ + { + "type": "null", + "title": "None" + }, + { + "type": "boolean", + "title": "Bool" + }, + { + "type": "integer", + "format": "uint8", + "minimum": 0, + "maximum": 255, + "title": "U8" + }, + { + "type": "integer", + "format": "int8", + "minimum": -128, + "maximum": 127, + "title": "I8" + }, + { + "type": "integer", + "format": "uint16", + "minimum": 0, + "maximum": 65535, + "title": "U16" + }, + { + "type": "integer", + "format": "int16", + "minimum": -32768, + "maximum": 32767, + "title": "I16" + }, + { + "type": "integer", + "format": "uint32", + "minimum": 0, + "title": "U32" + }, + { + "type": "integer", + "format": "int32", + "title": "I32" + }, + { + "type": "integer", + "format": "uint64", + "minimum": 0, + "title": "U64" + }, + { + "type": "integer", + "format": "int64", + "title": "I64" + }, + { + "type": "number", + "format": "float", + "title": "F32" + }, + { + "type": "number", + "format": "double", + "title": "F64" + }, + { + "type": "string", + "title": "String" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "title": "VecString" + }, + { + "$ref": "#/$defs/boolNdArray", + "title": "DataBool" + }, + { + "$ref": "#/$defs/u8NdArray", + "title": "DataU8" + }, + { + "$ref": "#/$defs/u16NdArray", + "title": "DataU16" + }, + { + "$ref": "#/$defs/u32NdArray", + "title": "DataU32" + }, + { + "$ref": "#/$defs/u64NdArray", + "title": "DataU64" + }, + { + "$ref": "#/$defs/i8NdArray", + "title": "DataI8" + }, + { + "$ref": "#/$defs/i16NdArray", + "title": "DataI16" + }, + { + "$ref": "#/$defs/i32NdArray", + "title": "DataI32" + }, + { + "$ref": "#/$defs/i64NdArray", + "title": "DataI64" + }, + { + "$ref": "#/$defs/f32NdArray", + "title": "DataF32" + }, + { + "$ref": "#/$defs/f64NdArray", + "title": "DataF64" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/PressioOption" + }, + "title": "Nested" + } + ], + "description": "Pressio option value" + }, "boolNdArray": { "anyOf": [ { @@ -1054,278 +382,6 @@ } } ] - }, - "PressioOption": { - "anyOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - }, - { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - }, - { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - }, - { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - }, - { - "type": "integer", - "format": "uint32", - "minimum": 0 - }, - { - "type": "integer", - "format": "int32" - }, - { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - { - "type": "integer", - "format": "int64" - }, - { - "type": "number", - "format": "float" - }, - { - "type": "number", - "format": "double" - }, - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/boolNdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint8", - "minimum": 0, - "maximum": 255 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint16", - "minimum": 0, - "maximum": 65535 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/u64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int8", - "minimum": -128, - "maximum": 127 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i8NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int16", - "minimum": -32768, - "maximum": 32767 - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i16NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/i64NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f32NdArray" - } - } - ] - }, - { - "anyOf": [ - { - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/f64NdArray" - } - } - ] - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/PressioOption" - } - } - ], - "description": "Pressio option value" } } } \ No newline at end of file diff --git a/codecs/qpet-sperr/src/lib.rs b/codecs/qpet-sperr/src/lib.rs index f90764dfc..7384bbf0c 100644 --- a/codecs/qpet-sperr/src/lib.rs +++ b/codecs/qpet-sperr/src/lib.rs @@ -482,6 +482,10 @@ impl<'de> Deserialize<'de> for Positive { } impl JsonSchema for Positive { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("PositiveF64") } diff --git a/codecs/random-projection/src/lib.rs b/codecs/random-projection/src/lib.rs index 4510758bc..9761386d1 100644 --- a/codecs/random-projection/src/lib.rs +++ b/codecs/random-projection/src/lib.rs @@ -48,7 +48,7 @@ use ::serde_json as _; /// /// This codec only supports finite floating point data. #[derive(Clone, Serialize, Deserialize, JsonSchema)] -// FIXME: #[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] // serde cannot deny unknown fields because of the flatten pub struct RandomProjectionCodec { /// Seed for generating the random projection matrix pub seed: u64, @@ -65,7 +65,7 @@ pub struct RandomProjectionCodec { /// Method with which the reduced dimensionality `$K$` is selected #[derive(Clone, Serialize, Deserialize, JsonSchema)] -// FIXME: #[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] // serde cannot deny unknown fields because of the flatten #[serde(tag = "reduction", rename_all = "kebab-case")] pub enum RandomProjectionReduction { /// The reduced dimensionality `$K$` is derived from `epsilon`, as defined @@ -84,7 +84,7 @@ pub enum RandomProjectionReduction { /// Projection kind that is used to generate the random projection matrix #[derive(Clone, Serialize, Deserialize, JsonSchema)] -// FIXME: #[serde(deny_unknown_fields)] +#[schemars(deny_unknown_fields)] // serde cannot deny unknown fields because of the flatten #[serde(tag = "projection", rename_all = "kebab-case")] pub enum RandomProjectionKind { /// The random projection matrix is dense and its components are sampled @@ -799,6 +799,10 @@ impl<'de> Deserialize<'de> for OpenClosedUnit { } impl JsonSchema for OpenClosedUnit { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("OpenClosedUnitF64") } diff --git a/codecs/random-projection/tests/schema.json b/codecs/random-projection/tests/schema.json index 9f48b20f6..5f6aa5031 100644 --- a/codecs/random-projection/tests/schema.json +++ b/codecs/random-projection/tests/schema.json @@ -1,5 +1,8 @@ { "type": "object", + "required": [ + "seed" + ], "properties": { "seed": { "type": "integer", @@ -14,15 +17,14 @@ "default": "0.1.0" } }, - "required": [ - "seed" - ], + "unevaluatedProperties": false, "description": "Codec that uses random projections to reduce the dimensionality of high-\ndimensional data to compress it.\n\nA two-dimensional array of shape `$N \\times D$` is encoded as n array of\nshape `$N \\times K$`, where `$K$` is either set explicitly or chosen using\nthe the Johnson-Lindenstrauss lemma. For `$K$` to be smaller than `$D$`,\n`$D$` must be quite large. Therefore, this codec should only applied on\nlarge datasets as it otherwise significantly inflates the data size instead\nof reducing it.\n\nChoosing a lower distortion rate `epsilon` will improve the quality of the\nlossy compression, i.e. reduce the compression error, at the cost of\nincreasing `$K$`.\n\nThis codec only supports finite floating point data.", "allOf": [ { "oneOf": [ { "type": "object", + "description": "The reduced dimensionality `$K$` is derived from `epsilon`, as defined\nby the Johnson-Lindenstrauss lemma.", "properties": { "epsilon": { "type": "number", @@ -38,11 +40,11 @@ "required": [ "reduction", "epsilon" - ], - "description": "The reduced dimensionality `$K$` is derived from `epsilon`, as defined\nby the Johnson-Lindenstrauss lemma." + ] }, { "type": "object", + "description": "The reduced dimensionality `$K$`, to which the data is projected, is\ngiven explicitly.", "properties": { "k": { "type": "integer", @@ -58,8 +60,7 @@ "required": [ "reduction", "k" - ], - "description": "The reduced dimensionality `$K$`, to which the data is projected, is\ngiven explicitly." + ] } ] }, @@ -80,6 +81,7 @@ }, { "type": "object", + "description": "The random projection matrix is sparse where only `density`% of entries\nare non-zero.\n\nThe matrix's components are sampled from\n\n- `$-\\sqrt{\\frac{1}{k \\cdot density}}$` with probability\n `$0.5 \\cdot density$`\n- `$0$` with probability `$1 - density$`\n- `$+\\sqrt{\\frac{1}{k \\cdot density}}$` with probability\n `$0.5 \\cdot density$`", "properties": { "density": { "type": [ @@ -97,8 +99,7 @@ }, "required": [ "projection" - ], - "description": "The random projection matrix is sparse where only `density`% of entries\nare non-zero.\n\nThe matrix's components are sampled from\n\n- `$-\\sqrt{\\frac{1}{k \\cdot density}}$` with probability\n `$0.5 \\cdot density$`\n- `$0$` with probability `$1 - density$`\n- `$+\\sqrt{\\frac{1}{k \\cdot density}}$` with probability\n `$0.5 \\cdot density$`" + ] } ] } diff --git a/codecs/sperr/src/lib.rs b/codecs/sperr/src/lib.rs index 8195807c5..3ced82e19 100644 --- a/codecs/sperr/src/lib.rs +++ b/codecs/sperr/src/lib.rs @@ -451,6 +451,10 @@ impl<'de> Deserialize<'de> for Positive { } impl JsonSchema for Positive { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("PositiveF64") } diff --git a/codecs/swizzle-reshape/Cargo.toml b/codecs/swizzle-reshape/Cargo.toml index 2f33b7b81..5059d2414 100644 --- a/codecs/swizzle-reshape/Cargo.toml +++ b/codecs/swizzle-reshape/Cargo.toml @@ -19,6 +19,9 @@ schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } +[dev-dependencies] +serde_json = { workspace = true, features = ["std"] } + [lints] workspace = true diff --git a/codecs/swizzle-reshape/src/lib.rs b/codecs/swizzle-reshape/src/lib.rs index ed8d3c3a4..5d8534644 100644 --- a/codecs/swizzle-reshape/src/lib.rs +++ b/codecs/swizzle-reshape/src/lib.rs @@ -36,6 +36,9 @@ use serde::{ }; use thiserror::Error; +#[cfg(test)] +use ::serde_json as _; + #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] /// Codec to swizzle/swap the axes of an array and reshape it. @@ -71,6 +74,7 @@ pub struct SwizzleReshapeCodec { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] #[serde(deny_unknown_fields)] +#[schemars(inline)] /// An axis group, potentially from a merged combination of multiple input axes pub enum AxisGroup { /// A merged combination of zero, one, or multiple input axes @@ -82,6 +86,7 @@ pub enum AxisGroup { #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] #[serde(deny_unknown_fields)] +#[schemars(inline)] /// An axis or all remaining axes pub enum Axis { /// A single axis, as determined by its index @@ -534,6 +539,10 @@ impl<'de> Deserialize<'de> for Rest { } impl JsonSchema for Rest { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("Rest") } diff --git a/codecs/swizzle-reshape/tests/schema.json b/codecs/swizzle-reshape/tests/schema.json new file mode 100644 index 000000000..3bdb0eaa7 --- /dev/null +++ b/codecs/swizzle-reshape/tests/schema.json @@ -0,0 +1,58 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "axes": { + "type": "array", + "items": { + "description": "An axis group, potentially from a merged combination of multiple input axes", + "anyOf": [ + { + "title": "Group", + "description": "A merged combination of zero, one, or multiple input axes", + "type": "array", + "items": { + "description": "An axis or all remaining axes", + "anyOf": [ + { + "title": "Index", + "description": "A single axis, as determined by its index", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + { + "title": "MergedRest", + "description": "All remaining axes, combined into one", + "type": "object", + "properties": {}, + "additionalProperties": false + } + ] + } + }, + { + "title": "AllRest", + "description": "All remaining axes, each in a separate single-axis group", + "type": "object", + "properties": {}, + "additionalProperties": false + } + ] + }, + "description": "The permutation of the axes that is applied on encoding.\n\nThe permutation is given as a list of axis groups, where each group\ncorresponds to one encoded output axis that may consist of several\ndecoded input axes. For instance, `[[0], [1, 2]]` flattens a three-\ndimensional array into a two-dimensional one by combining the second and\nthird axes.\n\nThe permutation also allows specifying a special catch-all remaining\naxes marker:\n- `[[0], {}]` moves the second axis to be the first and appends all\n other axes afterwards, i.e. the encoded array has the same number\n of axes as the input array\n- `[[0], [{}]]` in contrast collapses all other axes into one, i.e.\n the encoded array is two-dimensional" + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "axes" + ], + "description": "Codec to swizzle/swap the axes of an array and reshape it.\n\nThis codec does not store metadata about the original shape of the array.\nSince axes that have been combined during encoding cannot be split without\nfurther information, decoding may fail if an output array is not provided.\n\nSwizzling axes is always supported since no additional information about the\narray's shape is required to reconstruct it.", + "title": "SwizzleReshapeCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/swizzle-reshape/tests/schema.rs b/codecs/swizzle-reshape/tests/schema.rs new file mode 100644 index 000000000..418dbf1c2 --- /dev/null +++ b/codecs/swizzle-reshape/tests/schema.rs @@ -0,0 +1,21 @@ +#![expect(missing_docs)] + +use ::{ndarray as _, schemars as _, serde as _, serde_json as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_swizzle_reshape::SwizzleReshapeCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("SwizzleReshape schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/sz3/src/lib.rs b/codecs/sz3/src/lib.rs index e7c76ce0a..5c1016835 100644 --- a/codecs/sz3/src/lib.rs +++ b/codecs/sz3/src/lib.rs @@ -115,6 +115,7 @@ pub enum Sz3ErrorBound { /// SZ3 predictor #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] +#[schemars(inline)] pub enum Sz3Predictor { /// Interpolation #[serde(rename = "interpolation")] diff --git a/codecs/tthresh/src/lib.rs b/codecs/tthresh/src/lib.rs index 0eceb5de3..b7407f484 100644 --- a/codecs/tthresh/src/lib.rs +++ b/codecs/tthresh/src/lib.rs @@ -302,6 +302,10 @@ impl<'de> Deserialize<'de> for NonNegative { } impl JsonSchema for NonNegative { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("NonNegativeF64") } diff --git a/codecs/zfp-classic/src/lib.rs b/codecs/zfp-classic/src/lib.rs index 47522be98..a49e4c9ca 100644 --- a/codecs/zfp-classic/src/lib.rs +++ b/codecs/zfp-classic/src/lib.rs @@ -115,6 +115,7 @@ pub enum ZfpCompressionMode { } #[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +#[schemars(inline)] /// ZFP non-finite values mode pub enum ZfpNonFiniteValuesMode { /// Deny compressing non-finite values, i.e. return an error. diff --git a/codecs/zfp/src/lib.rs b/codecs/zfp/src/lib.rs index cda53b053..3d315fd60 100644 --- a/codecs/zfp/src/lib.rs +++ b/codecs/zfp/src/lib.rs @@ -120,6 +120,7 @@ pub enum ZfpCompressionMode { } #[derive(Copy, Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] +#[schemars(inline)] /// ZFP non-finite values mode pub enum ZfpNonFiniteValuesMode { /// Deny compressing non-finite values, i.e. return an error. diff --git a/crates/numcodecs-wasm-guest/Cargo.toml b/crates/numcodecs-wasm-guest/Cargo.toml index bc260d4f9..a7652a1e9 100644 --- a/crates/numcodecs-wasm-guest/Cargo.toml +++ b/crates/numcodecs-wasm-guest/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs-wasm-guest" -version = "0.3.1" +version = "0.3.2" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/crates/numcodecs-wasm-guest/src/lib.rs b/crates/numcodecs-wasm-guest/src/lib.rs index 82e2f2d59..53b453a15 100644 --- a/crates/numcodecs-wasm-guest/src/lib.rs +++ b/crates/numcodecs-wasm-guest/src/lib.rs @@ -27,8 +27,7 @@ use numcodecs::StaticCodec; #[cfg(target_arch = "wasm32")] use ::{ - numcodecs::{Codec, StaticCodec}, - schemars::schema_for, + numcodecs::{Codec, DynCodecType, StaticCodec, StaticCodecType}, serde::Deserialize, }; @@ -138,8 +137,9 @@ impl wit::codec::Guest for T { } fn codec_config_schema() -> wit::types::JsonSchema { - schema_for!(::Config<'static>) - .as_value() + StaticCodecType::::of() + .codec_config_schema() + .to_value() .to_string() } } diff --git a/crates/numcodecs/Cargo.toml b/crates/numcodecs/Cargo.toml index 324356d1c..655e4e025 100644 --- a/crates/numcodecs/Cargo.toml +++ b/crates/numcodecs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs" -version = "0.3.2" +version = "0.3.3" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true } diff --git a/crates/numcodecs/src/codec.rs b/crates/numcodecs/src/codec.rs index 0e144615c..6be95ee63 100644 --- a/crates/numcodecs/src/codec.rs +++ b/crates/numcodecs/src/codec.rs @@ -1,6 +1,10 @@ use std::{borrow::Cow, error::Error, fmt, marker::PhantomData}; -use schemars::{JsonSchema, Schema, SchemaGenerator, generate::SchemaSettings, json_schema}; +use schemars::{ + JsonSchema, Schema, SchemaGenerator, + generate::{Contract, SchemaSettings}, + json_schema, +}; use semver::{Version, VersionReq}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; @@ -152,9 +156,9 @@ impl DynCodecType for StaticCodecType { fn codec_config_schema(&self) -> Schema { let mut settings = SchemaSettings::draft2020_12(); - // TODO: perhaps this could be done as a more generally applicable - // transformation instead - settings.inline_subschemas = true; + settings.inline_subschemas = false; + settings.contract = Contract::Deserialize; + settings.untagged_enum_variant_titles = true; settings .into_generator() .into_root_schema_for::>() @@ -392,6 +396,10 @@ impl<'de, const MAJOR: u64, const MINOR: u64, const PATCH: u64> Deserialize<'de> impl JsonSchema for StaticCodecVersion { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("StaticCodecVersion") } From ed4ad447f0e62e6f92aa56cd94086dbc0dc7cb11 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 23 Apr 2026 10:38:59 +0300 Subject: [PATCH 52/72] Remove extraneous schemars dependency in numcodecs-wasm-guest --- crates/numcodecs-wasm-guest/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/numcodecs-wasm-guest/Cargo.toml b/crates/numcodecs-wasm-guest/Cargo.toml index a7652a1e9..3b7ddc76e 100644 --- a/crates/numcodecs-wasm-guest/Cargo.toml +++ b/crates/numcodecs-wasm-guest/Cargo.toml @@ -23,7 +23,7 @@ registry = ["dep:numcodecs-registry", "dep:serde-transcode"] format_serde_error = { workspace = true, features = ["serde_json"] } ndarray = { workspace = true, features = ["std"] } numcodecs-registry = { workspace = true, optional = true } -schemars = { workspace = true } +schemars = { workspace = true, optional = true } serde = { workspace = true } serde-transcode = { workspace = true, optional = true } serde_json = { workspace = true, features = ["std"] } From 573880308fc49d08eb52f9d126d1fea6c442eb5b Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 30 Apr 2026 15:54:37 +0300 Subject: [PATCH 53/72] add failing config test --- codecs/pressio/tests/config.rs | 69 ++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 codecs/pressio/tests/config.rs diff --git a/codecs/pressio/tests/config.rs b/codecs/pressio/tests/config.rs new file mode 100644 index 000000000..0eacd8eb6 --- /dev/null +++ b/codecs/pressio/tests/config.rs @@ -0,0 +1,69 @@ +#![expect(missing_docs, clippy::unwrap_used)] + +use ::{ + fragile as _, libpressio as _, ndarray as _, schemars as _, serde as _, serde_json as _, + serde_ndim as _, thiserror as _, +}; + +use numcodecs::StaticCodec; +use numcodecs_pressio::PressioCodec; +use serde::Deserialize; +use serde_json::json; + +#[test] +#[should_panic(expected = "missing field `compressor_id`")] +fn empty_config() { + let _ = PressioCodec::from_config(Deserialize::deserialize(json!({})).unwrap()); +} + +#[test] +#[should_panic(expected = "invalid compressor id ???, choose one of")] +fn invalid_compressor_id() { + let _ = PressioCodec::from_config( + Deserialize::deserialize(json!({ + "compressor_id": "???", + })) + .unwrap(), + ); +} + +#[test] +#[should_panic(expected = "unknown compressor configuration option: `abc`, use one of")] +fn unknown_compressor_config() { + let _ = PressioCodec::from_config( + Deserialize::deserialize(json!({ + "compressor_id": "linear_quantizer", + "compressor_config": { + "abc": 42, + } + })) + .unwrap(), + ); +} + +#[test] +#[should_panic(expected = "failed to cast option `pressio:abs`")] +fn option_cast_failure() { + let _ = PressioCodec::from_config( + Deserialize::deserialize(json!({ + "compressor_id": "linear_quantizer", + "compressor_config": { + "pressio:abs": "abc", + }, + })) + .unwrap(), + ); +} + +#[test] +fn array_data_option() { + let _ = PressioCodec::from_config( + Deserialize::deserialize(json!({ + "compressor_id": "mask_interpolation", + "compressor_config": { + "mask_interpolation:mask": [true, false], + }, + })) + .unwrap(), + ); +} From 9b1fbbe0c833958a9f0aa11847912aa0f0a2d305 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 30 Apr 2026 21:01:45 +0300 Subject: [PATCH 54/72] Fix pressio bool data array --- Cargo.toml | 2 +- codecs/pressio/tests/config.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b128c2d2d..4f5418c33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,7 +124,7 @@ schemars = { version = "1.0.3", default-features = false } scratch = { version = "1.0", default-features = false } semver = { version = "1.0.23", default-features = false } serde = { version = "1.0.218", default-features = false } -serde-ndim = { version = "=2.1.0", default-features = false } +serde-ndim = { version = "=2.1.1", git = "https://github.com/juntyr/serde-ndim.git", rev = "060c889", default-features = false } serde-transcode = { version = "1.1", default-features = false } serde_json = { version = "1.0.140", default-features = false } serde_repr = { version = "0.1.5", default-features = false } diff --git a/codecs/pressio/tests/config.rs b/codecs/pressio/tests/config.rs index 0eacd8eb6..e87377ca4 100644 --- a/codecs/pressio/tests/config.rs +++ b/codecs/pressio/tests/config.rs @@ -56,12 +56,12 @@ fn option_cast_failure() { } #[test] -fn array_data_option() { +fn bool_array_data_option() { let _ = PressioCodec::from_config( Deserialize::deserialize(json!({ "compressor_id": "mask_interpolation", "compressor_config": { - "mask_interpolation:mask": [true, false], + "mask_interpolation:mask": [[true, false], [false, true]], }, })) .unwrap(), From b0a699ca68432772c124d4179260e0c40d0b69f2 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Thu, 30 Apr 2026 21:14:13 +0300 Subject: [PATCH 55/72] Prepare manual v0.1.0-beta.1 release --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 87a49a490..cf76dbcae 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs-pressio" -version = "0.1.0" +version = "0.1.0-beta.1" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true } From 402edd40a039d1279d7dbcfc626b33276ed12eb2 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 3 Jul 2026 14:40:53 +0300 Subject: [PATCH 56/72] update MSRV in READMEs --- codecs/onion/README.md | 2 +- codecs/onion/src/lib.rs | 2 +- codecs/pressio/README.md | 2 +- codecs/pressio/src/lib.rs | 2 +- crates/numcodecs-registry/README.md | 2 +- crates/numcodecs-registry/src/lib.rs | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/codecs/onion/README.md b/codecs/onion/README.md index 23a7a2a7e..05112035e 100644 --- a/codecs/onion/README.md +++ b/codecs/onion/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-onion diff --git a/codecs/onion/src/lib.rs b/codecs/onion/src/lib.rs index 75ecdfff7..5b7b540df 100644 --- a/codecs/onion/src/lib.rs +++ b/codecs/onion/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-onion diff --git a/codecs/pressio/README.md b/codecs/pressio/README.md index 2a211f673..feb522c04 100644 --- a/codecs/pressio/README.md +++ b/codecs/pressio/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-pressio diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index ea1e281c2..d8ba80af0 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-pressio diff --git a/crates/numcodecs-registry/README.md b/crates/numcodecs-registry/README.md index c0ada0e18..ff962b00b 100644 --- a/crates/numcodecs-registry/README.md +++ b/crates/numcodecs-registry/README.md @@ -3,7 +3,7 @@ [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain -[MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +[MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue [repo]: https://github.com/juntyr/numcodecs-rs [Latest Version]: https://img.shields.io/crates/v/numcodecs-registry diff --git a/crates/numcodecs-registry/src/lib.rs b/crates/numcodecs-registry/src/lib.rs index e9129fad9..b0fb86f6b 100644 --- a/crates/numcodecs-registry/src/lib.rs +++ b/crates/numcodecs-registry/src/lib.rs @@ -3,7 +3,7 @@ //! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main //! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain //! -//! [MSRV]: https://img.shields.io/badge/MSRV-1.87.0-blue +//! [MSRV]: https://img.shields.io/badge/MSRV-1.88.0-blue //! [repo]: https://github.com/juntyr/numcodecs-rs //! //! [Latest Version]: https://img.shields.io/crates/v/numcodecs-registry From c000e6faf9ade18d95dc05079f700f9789085ec8 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 3 Jul 2026 14:43:24 +0300 Subject: [PATCH 57/72] Add numcodecs-wasm-pressio metadata --- codecs/pressio/Cargo.toml | 4 ++++ crates/numcodecs-wasm-guest/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index cf76dbcae..b80b44228 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -30,3 +30,7 @@ serde_json = { workspace = true, features = ["std"] } [lints] workspace = true + +[package.metadata.numcodecs-wasm] +version = "0.2.5" # wasi 0.2.6 +wasm-features = ["registry"] diff --git a/crates/numcodecs-wasm-guest/Cargo.toml b/crates/numcodecs-wasm-guest/Cargo.toml index 3b7ddc76e..c7573f0b0 100644 --- a/crates/numcodecs-wasm-guest/Cargo.toml +++ b/crates/numcodecs-wasm-guest/Cargo.toml @@ -17,7 +17,7 @@ numcodecs = { workspace = true } wit-bindgen = { workspace = true, features = ["macros", "realloc"] } [features] -registry = ["dep:numcodecs-registry", "dep:serde-transcode"] +registry = ["dep:numcodecs-registry", "dep:schemars", "dep:serde-transcode"] [target.'cfg(target_arch = "wasm32")'.dependencies] format_serde_error = { workspace = true, features = ["serde_json"] } From 708227b027d8c472fbe979d6760bd3f638b498f3 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 3 Jul 2026 16:06:31 +0300 Subject: [PATCH 58/72] start drafting libpressio codec trampoline --- Cargo.toml | 2 +- codecs/asinh/tests/schema.json | 23 ++++ codecs/asinh/tests/schema.rs | 21 +++ codecs/bit-round/src/lib.rs | 4 + codecs/bit-round/tests/schema.rs | 1 + codecs/ebcc/src/lib.rs | 1 + codecs/ebcc/tests/schema.json | 1 + codecs/fixed-offset-scale/tests/schema.json | 29 ++++ codecs/fixed-offset-scale/tests/schema.rs | 21 +++ codecs/identity/tests/schema.json | 15 ++ codecs/identity/tests/schema.rs | 21 +++ codecs/linear-quantize/src/lib.rs | 2 + codecs/linear-quantize/tests/schema.json | 99 ++++++++++++++ codecs/linear-quantize/tests/schema.rs | 24 ++++ codecs/log/tests/schema.json | 15 ++ codecs/log/tests/schema.rs | 21 +++ codecs/onion/tests/schema.rs | 1 + codecs/pressio/Cargo.toml | 3 +- codecs/pressio/src/lib.rs | 137 +++++++++++++++++-- codecs/pressio/tests/config.rs | 4 + codecs/pressio/tests/schema.rs | 4 + codecs/random-projection/src/lib.rs | 2 +- codecs/reinterpret/tests/schema.json | 71 ++++++++++ codecs/reinterpret/tests/schema.rs | 21 +++ codecs/round/src/lib.rs | 4 + codecs/round/tests/schema.json | 23 ++++ codecs/round/tests/schema.rs | 21 +++ codecs/stochastic-rounding/src/lib.rs | 4 + codecs/stochastic-rounding/tests/schema.json | 30 ++++ codecs/stochastic-rounding/tests/schema.rs | 24 ++++ codecs/uniform-noise/tests/schema.json | 30 ++++ codecs/uniform-noise/tests/schema.rs | 24 ++++ codecs/zlib/src/lib.rs | 1 + codecs/zlib/tests/schema.json | 34 +++++ codecs/zlib/tests/schema.rs | 24 ++++ codecs/zstd/src/lib.rs | 1 + codecs/zstd/tests/schema.json | 23 ++++ codecs/zstd/tests/schema.rs | 24 ++++ crates/numcodecs-python/tests/crc32.rs | 4 +- crates/numcodecs-python/tests/export.rs | 4 +- crates/numcodecs-python/tests/schema.rs | 5 +- crates/numcodecs/src/array.rs | 1 + crates/numcodecs/src/erased.rs | 1 + 43 files changed, 807 insertions(+), 18 deletions(-) create mode 100644 codecs/asinh/tests/schema.json create mode 100644 codecs/asinh/tests/schema.rs create mode 100644 codecs/fixed-offset-scale/tests/schema.json create mode 100644 codecs/fixed-offset-scale/tests/schema.rs create mode 100644 codecs/identity/tests/schema.json create mode 100644 codecs/identity/tests/schema.rs create mode 100644 codecs/linear-quantize/tests/schema.json create mode 100644 codecs/linear-quantize/tests/schema.rs create mode 100644 codecs/log/tests/schema.json create mode 100644 codecs/log/tests/schema.rs create mode 100644 codecs/reinterpret/tests/schema.json create mode 100644 codecs/reinterpret/tests/schema.rs create mode 100644 codecs/round/tests/schema.json create mode 100644 codecs/round/tests/schema.rs create mode 100644 codecs/stochastic-rounding/tests/schema.json create mode 100644 codecs/stochastic-rounding/tests/schema.rs create mode 100644 codecs/uniform-noise/tests/schema.json create mode 100644 codecs/uniform-noise/tests/schema.rs create mode 100644 codecs/zlib/tests/schema.json create mode 100644 codecs/zlib/tests/schema.rs create mode 100644 codecs/zstd/tests/schema.json create mode 100644 codecs/zstd/tests/schema.rs diff --git a/Cargo.toml b/Cargo.toml index 4f5418c33..d32ce5e94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "018c80b", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a315cb1", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/asinh/tests/schema.json b/codecs/asinh/tests/schema.json new file mode 100644 index 000000000..29018dca5 --- /dev/null +++ b/codecs/asinh/tests/schema.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "linear_width": { + "type": "number", + "format": "double", + "description": "The width of the close-to-zero input value range where the transform is\nnearly linear" + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "linear_width" + ], + "description": "Asinh codec, which applies a quasi-logarithmic transformation on encoding.\n\nFor values close to zero that are within the codec's `linear_width`, the\ntransform is close to linear. For values of larger magnitudes, the\ntransform is asymptotically logarithmic. Unlike a logarithmic transform,\nthis codec supports all finite values, including negative values and zero.\n\nIn detail, the codec calculates\n`$c = w \\cdot \\text{asinh}\\left( \\frac{x}{w} \\right)$`\non encoding and\n`$d = w \\cdot \\text{sinh}\\left( \\frac{c}{w} \\right)$`\non decoding, where `$w$` is the codec's `linear_width`.\n\nThe codec only supports finite floating point numbers.", + "title": "AsinhCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/asinh/tests/schema.rs b/codecs/asinh/tests/schema.rs new file mode 100644 index 000000000..dc5a26863 --- /dev/null +++ b/codecs/asinh/tests/schema.rs @@ -0,0 +1,21 @@ +#![expect(missing_docs)] + +use ::{ndarray as _, num_traits as _, schemars as _, serde as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_asinh::AsinhCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("Asinh schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/bit-round/src/lib.rs b/codecs/bit-round/src/lib.rs index 552d1956b..862936ff5 100644 --- a/codecs/bit-round/src/lib.rs +++ b/codecs/bit-round/src/lib.rs @@ -180,6 +180,10 @@ impl<'de> Deserialize<'de> for NonNegative { } impl JsonSchema for NonNegative { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("NonNegativeF64") } diff --git a/codecs/bit-round/tests/schema.rs b/codecs/bit-round/tests/schema.rs index da345a568..6f92b925c 100644 --- a/codecs/bit-round/tests/schema.rs +++ b/codecs/bit-round/tests/schema.rs @@ -14,6 +14,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("BitRound schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/ebcc/src/lib.rs b/codecs/ebcc/src/lib.rs index 736dabaa1..c0401b67c 100644 --- a/codecs/ebcc/src/lib.rs +++ b/codecs/ebcc/src/lib.rs @@ -64,6 +64,7 @@ pub struct EbccCodec { #[derive(Debug, Copy, Clone, Default, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] +#[schemars(inline)] /// Chunk shape that EBCC uses to handle large data. pub enum EbccChunkShape { /// EBCC chooses an appropriate chunk shape automatically. diff --git a/codecs/ebcc/tests/schema.json b/codecs/ebcc/tests/schema.json index f3fd850fd..ccf238a5a 100644 --- a/codecs/ebcc/tests/schema.json +++ b/codecs/ebcc/tests/schema.json @@ -78,6 +78,7 @@ }, "minItems": 3, "maxItems": 3, + "title": "Explicit", "description": "EBCC uses the provided explicit chunk shape." } ], diff --git a/codecs/fixed-offset-scale/tests/schema.json b/codecs/fixed-offset-scale/tests/schema.json new file mode 100644 index 000000000..edd858bcc --- /dev/null +++ b/codecs/fixed-offset-scale/tests/schema.json @@ -0,0 +1,29 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "offset": { + "type": "number", + "format": "double", + "description": "The offset of the data." + }, + "scale": { + "type": "number", + "format": "double", + "description": "The scale of the data." + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "offset", + "scale" + ], + "description": "Fixed offset-scale codec which calculates `$c = \\frac{x - o}{s}$` on\nencoding and `$d = (c \\cdot s) + o$` on decoding.\n\n- Setting `$o = \\text{mean}(x)$` and `$s = \\text{std}(x)$` normalizes that\n data.\n- Setting `$o = \\text{min}(x)$` and `$s = \\text{max}(x) - \\text{min}(x)$`\n standardizes the data.\n\nThe codec only supports floating point numbers.", + "title": "FixedOffsetScaleCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/fixed-offset-scale/tests/schema.rs b/codecs/fixed-offset-scale/tests/schema.rs new file mode 100644 index 000000000..ae1748632 --- /dev/null +++ b/codecs/fixed-offset-scale/tests/schema.rs @@ -0,0 +1,21 @@ +#![expect(missing_docs)] + +use ::{ndarray as _, num_traits as _, schemars as _, serde as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_fixed_offset_scale::FixedOffsetScaleCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("FixedOffsetScale schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/identity/tests/schema.json b/codecs/identity/tests/schema.json new file mode 100644 index 000000000..c8e648163 --- /dev/null +++ b/codecs/identity/tests/schema.json @@ -0,0 +1,15 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "description": "Identity codec which applies the identity function, i.e. passes through the\ninput unchanged during encoding and decoding.", + "title": "IdentityCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/identity/tests/schema.rs b/codecs/identity/tests/schema.rs new file mode 100644 index 000000000..039a05d01 --- /dev/null +++ b/codecs/identity/tests/schema.rs @@ -0,0 +1,21 @@ +#![expect(missing_docs)] + +use ::{schemars as _, serde as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_identity::IdentityCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("Identity schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/linear-quantize/src/lib.rs b/codecs/linear-quantize/src/lib.rs index 7803aecac..7dbdf8fa8 100644 --- a/codecs/linear-quantize/src/lib.rs +++ b/codecs/linear-quantize/src/lib.rs @@ -54,6 +54,7 @@ pub struct LinearQuantizeCodec { /// Data types which the [`LinearQuantizeCodec`] can quantize #[derive(Copy, Clone, Debug, Serialize, Deserialize, JsonSchema)] #[schemars(extend("enum" = ["f32", "float32", "f64", "float64"]))] +#[schemars(inline)] #[expect(missing_docs)] pub enum LinearQuantizeDType { #[serde(rename = "f32", alias = "float32")] @@ -78,6 +79,7 @@ impl fmt::Display for LinearQuantizeDType { /// bits used. #[derive(Copy, Clone, Serialize_repr, Deserialize_repr, JsonSchema_repr)] #[repr(u8)] +#[schemars(inline)] #[rustfmt::skip] #[expect(missing_docs)] pub enum LinearQuantizeBins { diff --git a/codecs/linear-quantize/tests/schema.json b/codecs/linear-quantize/tests/schema.json new file mode 100644 index 000000000..bef2b64a9 --- /dev/null +++ b/codecs/linear-quantize/tests/schema.json @@ -0,0 +1,99 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "dtype": { + "type": "string", + "enum": [ + "f32", + "float32", + "f64", + "float64" + ], + "description": "Dtype of the decoded data" + }, + "bits": { + "type": "integer", + "enum": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64 + ], + "description": "Binary precision of the encoded data where `$bits = \\log_{2}(bins)$`" + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "0.1.0" + } + }, + "required": [ + "dtype", + "bits" + ], + "description": "Lossy codec to reduce the precision of floating point data.\n\nThe data is quantized to unsigned integers of the best-fitting type.\nThe range and shape of the input data is stored in-band.", + "title": "LinearQuantizeCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/linear-quantize/tests/schema.rs b/codecs/linear-quantize/tests/schema.rs new file mode 100644 index 000000000..9d773d0fc --- /dev/null +++ b/codecs/linear-quantize/tests/schema.rs @@ -0,0 +1,24 @@ +#![expect(missing_docs)] + +use ::{ + ndarray as _, num_traits as _, postcard as _, schemars as _, serde as _, serde_repr as _, + thiserror as _, twofloat as _, +}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_linear_quantize::LinearQuantizeCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("LinearQuantize schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/log/tests/schema.json b/codecs/log/tests/schema.json new file mode 100644 index 000000000..3a04c5694 --- /dev/null +++ b/codecs/log/tests/schema.json @@ -0,0 +1,15 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "description": "Log codec which calculates `$c = \\ln(x)$` on encoding and `$d = {e}^{c}$`\non decoding.\n\nThe codec only supports finite positive floating point numbers.", + "title": "LogCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/log/tests/schema.rs b/codecs/log/tests/schema.rs new file mode 100644 index 000000000..44667f38c --- /dev/null +++ b/codecs/log/tests/schema.rs @@ -0,0 +1,21 @@ +#![expect(missing_docs)] + +use ::{ndarray as _, num_traits as _, schemars as _, serde as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_log::LogCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("Log schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/onion/tests/schema.rs b/codecs/onion/tests/schema.rs index 9e35990d8..0f07fcf5f 100644 --- a/codecs/onion/tests/schema.rs +++ b/codecs/onion/tests/schema.rs @@ -14,6 +14,7 @@ fn schema() { .to_value() ); + #[expect(clippy::manual_assert, clippy::panic)] if schema != include_str!("schema.json") { panic!("Onion schema has changed\n===\n{schema}\n==="); } diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index b80b44228..3e661f76a 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -16,9 +16,10 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] [dependencies] fragile = { workspace = true } -libpressio = { workspace = true, features = ["bzip2", "lua"] } +libpressio = { workspace = true, features = ["bzip2", "lua", "mpi-stubs", "opt"] } ndarray = { workspace = true } numcodecs = { workspace = true } +numcodecs-registry = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } serde-ndim = { workspace = true, features = ["ndarray"] } diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index d8ba80af0..8e2d93e66 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -27,7 +27,8 @@ use fragile::Fragile; use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, - Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, + Codec, DynCodec, DynCodecType, ErasedDynCodec, StaticCodec, StaticCodecConfig, + StaticCodecVersion, }; use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -414,6 +415,19 @@ impl<'de> Deserialize<'de> for PressioCompressor { std::mem::drop(format.metric_results); let mut pressio = libpressio::Pressio::new().map_err(serde::de::Error::custom)?; + pressio + .register_compressor( + "numcodecs.rs", + NumcodecsPressioCompressor { + codec: Option::None, + }, + "0.1.0.0", + 0, + 1, + 0, + 0, + ) + .map_err(serde::de::Error::custom)?; let mut compressor = pressio .get_compressor(format.compressor_id.as_str()) .map_err(|err| { @@ -641,14 +655,14 @@ impl<'de> Deserialize<'de> for None { } impl JsonSchema for None { - fn schema_name() -> Cow<'static, str> { - Cow::Borrowed("null") - } - fn inline_schema() -> bool { true } + fn schema_name() -> Cow<'static, str> { + Cow::Borrowed("null") + } + fn json_schema(_generator: &mut SchemaGenerator) -> Schema { json_schema!({ "type": "null" @@ -1025,6 +1039,107 @@ pub enum PressioCodecError { /// Opaque error for when encoding or decoding with libpressio fails pub struct PressioCodingError(libpressio::PressioError); +#[derive(Clone)] +struct NumcodecsPressioCompressor { + codec: Option, +} + +#[expect(clippy::expect_used)] +impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { + fn get_configuration(&self) -> libpressio::PressioOptions { + (|| -> Result { + let mut options = libpressio::PressioOptions::new()?; + options.set( + "pressio:thread_safe", + libpressio::PressioOption::thread_safety(Some( + libpressio::PressioThreadSafety::Multiple, + )), + )?; + options.set( + "pressio:stability", + libpressio::PressioOption::string(Some(String::from("experimental"))), + )?; + Ok(options) + })() + .expect("get_configuration should not fail") + } + + fn get_documentation(&self) -> libpressio::PressioOptions { + (|| -> Result { + let mut options = libpressio::PressioOptions::new()?; + options.set( + "pressio:description", + libpressio::PressioOption::string(Some(String::from( + "A numcodecs codec exposed through numcodecs-rs", + ))), + )?; + options.set( + "numcodecs.rs:id", + libpressio::PressioOption::string(Some(String::from("numcodecs codec id"))), + )?; + Ok(options) + })() + .expect("get_documentation should not fail") + } + + fn get_options(&self) -> libpressio::PressioOptions { + (|| -> Result { + let mut options = libpressio::PressioOptions::new()?; + if let Some(codec) = &self.codec { + options.set( + "numcodecs.rs:id", + libpressio::PressioOption::string(Some(String::from(codec.ty().codec_id()))), + )?; + // TODO: serialize all remaining options + } else { + options.set( + "numcodecs.rs:id", + libpressio::PressioOption::string(Option::None), + )?; + } + Ok(options) + })() + .expect("get_options should not fail") + } + + fn set_options( + &self, + options: &libpressio::PressioOptions, + ) -> Result<(), libpressio::PressioError> { + let _id = options.get("numcodecs.rs:id")?; + + todo!("deserialize options") + } + + fn compress( + &self, + _input_data: &libpressio::PressioData, + _compressed_data: libpressio::PressioData, + ) -> Result { + let Some(_codec) = &self.codec else { + todo!("error or no-op"); + }; + + todo!("compress"); + } + + fn decompress( + &self, + _compressed_data: &libpressio::PressioData, + _decompressed_data: libpressio::PressioData, + ) -> Result { + let Some(_codec) = &self.codec else { + todo!("error or no-op"); + }; + + todo!("decompress"); + } + + fn get_metrics_results(&self) -> libpressio::PressioOptions { + libpressio::PressioOptions::new().expect("get_metrics_results should not fail") + } +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { @@ -1071,7 +1186,7 @@ mod tests { assert!(((*i) - (*o)).abs() <= 10.0); } - let config = serde_json::to_string(&pressio.get_config()).unwrap(); + let config = serde_json::to_string(&StaticCodec::get_config(&pressio)).unwrap(); assert!(config.contains("\"size:compressed_size\":400")); } @@ -1110,7 +1225,7 @@ mod tests { assert!(((*i) - (*o)).abs() <= 10.0); } - let config = serde_json::to_string(&pressio.get_config()).unwrap(); + let config = serde_json::to_string(&StaticCodec::get_config(&pressio)).unwrap(); assert!(config.contains("\"size:compressed_size\":63")); } @@ -1132,7 +1247,7 @@ mod tests { })) .unwrap(); - let config = serde_json::to_string(&pressio.get_config()).unwrap(); + let config = serde_json::to_string(&StaticCodec::get_config(&pressio)).unwrap(); assert!(!config.contains("\"size:compression_ratio\"")); assert!(config.contains("\"composite:objective\":1.2")); assert!(!config.contains("\"composite:objective2\"")); @@ -1160,9 +1275,13 @@ mod tests { assert!(i.to_bits() == o.to_bits()); } - let config = serde_json::to_string(&pressio.get_config()).unwrap(); + let config = serde_json::to_string(&StaticCodec::get_config(&pressio)).unwrap(); assert!(config.contains("\"size:compression_ratio\":1.0")); assert!(config.contains("\"composite:objective\":1.2")); assert!(config.contains("\"composite:objective2\":4.2")); } + + numcodecs_registry::export_global! { + static REGISTRY: numcodecs_registry::EmptyRegistry = numcodecs_registry::EmptyRegistry; + } } diff --git a/codecs/pressio/tests/config.rs b/codecs/pressio/tests/config.rs index e87377ca4..434c8bf19 100644 --- a/codecs/pressio/tests/config.rs +++ b/codecs/pressio/tests/config.rs @@ -67,3 +67,7 @@ fn bool_array_data_option() { .unwrap(), ); } + +numcodecs_registry::export_global! { + static REGISTRY: numcodecs_registry::EmptyRegistry = numcodecs_registry::EmptyRegistry; +} diff --git a/codecs/pressio/tests/schema.rs b/codecs/pressio/tests/schema.rs index ec6b74557..a99723684 100644 --- a/codecs/pressio/tests/schema.rs +++ b/codecs/pressio/tests/schema.rs @@ -22,3 +22,7 @@ fn schema() { panic!("Pressio schema has changed\n===\n{schema}\n==="); } } + +numcodecs_registry::export_global! { + static REGISTRY: numcodecs_registry::EmptyRegistry = numcodecs_registry::EmptyRegistry; +} diff --git a/codecs/random-projection/src/lib.rs b/codecs/random-projection/src/lib.rs index 9761386d1..59a4c931a 100644 --- a/codecs/random-projection/src/lib.rs +++ b/codecs/random-projection/src/lib.rs @@ -1026,6 +1026,7 @@ mod tests { } } + #[expect(clippy::let_and_return)] fn roundtrip( data: &Array, seed: u64, @@ -1041,7 +1042,6 @@ mod tests { .expect("projecting must not fail"); let reconstructed = reconstruct_with_projection(projected, seed, projection) .expect("reconstruction must not fail"); - #[expect(clippy::let_and_return)] reconstructed } diff --git a/codecs/reinterpret/tests/schema.json b/codecs/reinterpret/tests/schema.json new file mode 100644 index 000000000..f585c5535 --- /dev/null +++ b/codecs/reinterpret/tests/schema.json @@ -0,0 +1,71 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "encode_dtype": { + "type": "string", + "enum": [ + "u8", + "uint8", + "u16", + "uint16", + "u32", + "uint32", + "u64", + "uint64", + "i8", + "int8", + "i16", + "int16", + "i32", + "int32", + "i64", + "int64", + "f32", + "float32", + "f64", + "float64" + ], + "description": "Dtype of the encoded data." + }, + "decode_dtype": { + "type": "string", + "enum": [ + "u8", + "uint8", + "u16", + "uint16", + "u32", + "uint32", + "u64", + "uint64", + "i8", + "int8", + "i16", + "int16", + "i32", + "int32", + "i64", + "int64", + "f32", + "float32", + "f64", + "float64" + ], + "description": "Dtype of the decoded data" + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "encode_dtype", + "decode_dtype" + ], + "description": "Codec to reinterpret data between different compatible types.\n\nNote that no conversion happens, only the meaning of the bits changes.\n\nReinterpreting to bytes, or to a same-sized unsigned integer type, or\nwithout the changing the dtype are supported.", + "title": "ReinterpretCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/reinterpret/tests/schema.rs b/codecs/reinterpret/tests/schema.rs new file mode 100644 index 000000000..602591aeb --- /dev/null +++ b/codecs/reinterpret/tests/schema.rs @@ -0,0 +1,21 @@ +#![expect(missing_docs)] + +use ::{ndarray as _, schemars as _, serde as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_reinterpret::ReinterpretCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("Reinterpret schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/round/src/lib.rs b/codecs/round/src/lib.rs index 91eb806f6..ac799a581 100644 --- a/codecs/round/src/lib.rs +++ b/codecs/round/src/lib.rs @@ -120,6 +120,10 @@ impl<'de> Deserialize<'de> for NonNegative { } impl JsonSchema for NonNegative { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("NonNegativeF64") } diff --git a/codecs/round/tests/schema.json b/codecs/round/tests/schema.json new file mode 100644 index 000000000..f7c9ccb9e --- /dev/null +++ b/codecs/round/tests/schema.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "precision": { + "type": "number", + "minimum": 0.0, + "description": "Precision of the rounding operation" + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "precision" + ], + "description": "Codec that rounds the data on encoding and passes through the input\nunchanged during decoding.\n\nThe codec only supports floating point data.", + "title": "RoundCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/round/tests/schema.rs b/codecs/round/tests/schema.rs new file mode 100644 index 000000000..9960425aa --- /dev/null +++ b/codecs/round/tests/schema.rs @@ -0,0 +1,21 @@ +#![expect(missing_docs)] + +use ::{ndarray as _, num_traits as _, schemars as _, serde as _, thiserror as _}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_round::RoundCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("Round schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/stochastic-rounding/src/lib.rs b/codecs/stochastic-rounding/src/lib.rs index 29844d212..5875ad0c3 100644 --- a/codecs/stochastic-rounding/src/lib.rs +++ b/codecs/stochastic-rounding/src/lib.rs @@ -147,6 +147,10 @@ impl<'de> Deserialize<'de> for NonNegative { } impl JsonSchema for NonNegative { + fn inline_schema() -> bool { + true + } + fn schema_name() -> Cow<'static, str> { Cow::Borrowed("NonNegativeF64") } diff --git a/codecs/stochastic-rounding/tests/schema.json b/codecs/stochastic-rounding/tests/schema.json new file mode 100644 index 000000000..8d17597e4 --- /dev/null +++ b/codecs/stochastic-rounding/tests/schema.json @@ -0,0 +1,30 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "precision": { + "type": "number", + "minimum": 0.0, + "description": "The precision of the rounding operation" + }, + "seed": { + "type": "integer", + "format": "uint64", + "minimum": 0, + "description": "Seed for the random generator" + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "precision", + "seed" + ], + "description": "Codec that stochastically rounds the data to the nearest multiple of\n`precision` on encoding and passes through the input unchanged during\ndecoding.\n\nThe nearest representable multiple is chosen such that the absolute\ndifference between the original value and the rounded value do not exceed\nthe precision. Therefore, the rounded value may have a non-zero remainder.\n\nThis codec first hashes the input array data and shape to then `seed` a\npseudo-random number generator that is used to sample the stochasticity for\nrounding. Therefore, passing in the same input with the same `seed` will\nproduce the same stochasticity and thus the same encoded output.", + "title": "StochasticRoundingCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/stochastic-rounding/tests/schema.rs b/codecs/stochastic-rounding/tests/schema.rs new file mode 100644 index 000000000..276a26a95 --- /dev/null +++ b/codecs/stochastic-rounding/tests/schema.rs @@ -0,0 +1,24 @@ +#![expect(missing_docs)] + +use ::{ + ndarray as _, num_traits as _, rand as _, schemars as _, serde as _, thiserror as _, + wyhash as _, +}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_stochastic_rounding::StochasticRoundingCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("StochasticRounding schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/uniform-noise/tests/schema.json b/codecs/uniform-noise/tests/schema.json new file mode 100644 index 000000000..84d273473 --- /dev/null +++ b/codecs/uniform-noise/tests/schema.json @@ -0,0 +1,30 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "scale": { + "type": "number", + "format": "double", + "description": "Scale of the uniform noise, which is sampled from\n`$\\text{U}(-0.5 \\cdot scale, 0.5 \\cdot scale)$`" + }, + "seed": { + "type": "integer", + "format": "uint64", + "minimum": 0, + "description": "Seed for the random noise generator" + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "1.0.0" + } + }, + "required": [ + "scale", + "seed" + ], + "description": "Codec that adds `seed`ed `$\\text{U}(-0.5 \\cdot scale, 0.5 \\cdot scale)$`\nuniform noise of the given `scale` during encoding and passes through the\ninput unchanged during decoding.\n\nThis codec first hashes the input array data and shape to then seed a\npseudo-random number generator that generates the uniform noise. Therefore,\npassing in the same input with the same `seed` will produce the same noise\nand thus the same encoded output.", + "title": "UniformNoiseCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/uniform-noise/tests/schema.rs b/codecs/uniform-noise/tests/schema.rs new file mode 100644 index 000000000..28483a223 --- /dev/null +++ b/codecs/uniform-noise/tests/schema.rs @@ -0,0 +1,24 @@ +#![expect(missing_docs)] + +use ::{ + ndarray as _, num_traits as _, rand as _, schemars as _, serde as _, thiserror as _, + wyhash as _, +}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_uniform_noise::UniformNoiseCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("UniformNoise schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/zlib/src/lib.rs b/codecs/zlib/src/lib.rs index 69bc3d438..18cfbe29e 100644 --- a/codecs/zlib/src/lib.rs +++ b/codecs/zlib/src/lib.rs @@ -48,6 +48,7 @@ pub struct ZlibCodec { #[derive(Copy, Clone, Serialize_repr, Deserialize_repr, JsonSchema_repr)] #[repr(u8)] +#[schemars(inline)] /// Zlib compression level. /// /// The level ranges from 0, no compression, to 9, best compression. diff --git a/codecs/zlib/tests/schema.json b/codecs/zlib/tests/schema.json new file mode 100644 index 000000000..75897bc25 --- /dev/null +++ b/codecs/zlib/tests/schema.json @@ -0,0 +1,34 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "level": { + "type": "integer", + "enum": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9 + ], + "description": "Zlib compression level.\n\nThe level ranges from 0, no compression, to 9, best compression." + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "0.1.0" + } + }, + "required": [ + "level" + ], + "description": "Codec providing compression using Zlib", + "title": "ZlibCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/zlib/tests/schema.rs b/codecs/zlib/tests/schema.rs new file mode 100644 index 000000000..19a1663c2 --- /dev/null +++ b/codecs/zlib/tests/schema.rs @@ -0,0 +1,24 @@ +#![expect(missing_docs)] + +use ::{ + miniz_oxide as _, ndarray as _, postcard as _, schemars as _, serde as _, serde_repr as _, + thiserror as _, +}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_zlib::ZlibCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("Zlib schema has changed\n===\n{schema}\n==="); + } +} diff --git a/codecs/zstd/src/lib.rs b/codecs/zstd/src/lib.rs index 1744666a3..67bd7bf05 100644 --- a/codecs/zstd/src/lib.rs +++ b/codecs/zstd/src/lib.rs @@ -108,6 +108,7 @@ impl StaticCodec for ZstdCodec { #[derive(Clone, Copy, JsonSchema)] #[schemars(transparent)] +#[schemars(inline)] /// Zstandard compression level. /// /// The level ranges from small (fastest) to large (best compression). diff --git a/codecs/zstd/tests/schema.json b/codecs/zstd/tests/schema.json new file mode 100644 index 000000000..73e5ce7cc --- /dev/null +++ b/codecs/zstd/tests/schema.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "level": { + "type": "integer", + "format": "int32", + "description": "Zstandard compression level.\n\nThe level ranges from small (fastest) to large (best compression)." + }, + "_version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$", + "description": "The codec's encoding format version. Do not provide this parameter explicitly.", + "default": "0.1.0" + } + }, + "required": [ + "level" + ], + "description": "Codec providing compression using Zstandard", + "title": "ZstdCodec", + "$schema": "https://json-schema.org/draft/2020-12/schema" +} \ No newline at end of file diff --git a/codecs/zstd/tests/schema.rs b/codecs/zstd/tests/schema.rs new file mode 100644 index 000000000..4772fd0e3 --- /dev/null +++ b/codecs/zstd/tests/schema.rs @@ -0,0 +1,24 @@ +#![expect(missing_docs)] + +use ::{ + ndarray as _, postcard as _, schemars as _, serde as _, thiserror as _, zstd as _, + zstd_sys as _, +}; + +use numcodecs::{DynCodecType, StaticCodecType}; +use numcodecs_zstd::ZstdCodec; + +#[test] +fn schema() { + let schema = format!( + "{:#}", + StaticCodecType::::of() + .codec_config_schema() + .to_value() + ); + + #[expect(clippy::manual_assert, clippy::panic)] + if schema != include_str!("schema.json") { + panic!("Zstd schema has changed\n===\n{schema}\n==="); + } +} diff --git a/crates/numcodecs-python/tests/crc32.rs b/crates/numcodecs-python/tests/crc32.rs index e6078c376..b14366c01 100644 --- a/crates/numcodecs-python/tests/crc32.rs +++ b/crates/numcodecs-python/tests/crc32.rs @@ -1,8 +1,8 @@ #![expect(missing_docs)] use ::{ - convert_case as _, pythonize as _, schemars as _, serde as _, serde_transcode as _, - thiserror as _, + convert_case as _, numcodecs_registry as _, pythonize as _, schemars as _, serde as _, + serde_transcode as _, thiserror as _, }; use ndarray::{Array1, ArrayView1}; use numcodecs::{AnyArray, AnyArrayView, AnyCowArray, Codec, DynCodec, DynCodecType}; diff --git a/crates/numcodecs-python/tests/export.rs b/crates/numcodecs-python/tests/export.rs index 68f0b228e..079e03cf4 100644 --- a/crates/numcodecs-python/tests/export.rs +++ b/crates/numcodecs-python/tests/export.rs @@ -1,8 +1,8 @@ #![expect(missing_docs)] use ::{ - convert_case as _, ndarray as _, pyo3_error as _, pythonize as _, serde as _, serde_json as _, - serde_transcode as _, thiserror as _, + convert_case as _, ndarray as _, numcodecs_registry as _, pyo3_error as _, pythonize as _, + serde as _, serde_json as _, serde_transcode as _, thiserror as _, }; use numcodecs::{ AnyArray, AnyArrayBase, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, DynCodecType, diff --git a/crates/numcodecs-python/tests/schema.rs b/crates/numcodecs-python/tests/schema.rs index 4e7abd391..9a8b48127 100644 --- a/crates/numcodecs-python/tests/schema.rs +++ b/crates/numcodecs-python/tests/schema.rs @@ -1,8 +1,9 @@ #![expect(missing_docs)] use ::{ - convert_case as _, ndarray as _, numpy as _, pyo3_error as _, pythonize as _, schemars as _, - serde as _, serde_json as _, serde_transcode as _, thiserror as _, + convert_case as _, ndarray as _, numcodecs_registry as _, numpy as _, pyo3_error as _, + pythonize as _, schemars as _, serde as _, serde_json as _, serde_transcode as _, + thiserror as _, }; use numcodecs::DynCodecType; use numcodecs_python::{PyCodecClass, PyCodecClassAdapter}; diff --git a/crates/numcodecs/src/array.rs b/crates/numcodecs/src/array.rs index 095b1a334..259867b94 100644 --- a/crates/numcodecs/src/array.rs +++ b/crates/numcodecs/src/array.rs @@ -677,6 +677,7 @@ impl< "f32", "float32", "f64", "float64" ]))] +#[schemars(inline)] #[non_exhaustive] #[expect(missing_docs)] pub enum AnyArrayDType { diff --git a/crates/numcodecs/src/erased.rs b/crates/numcodecs/src/erased.rs index 8a739ad8e..df8b44a7d 100644 --- a/crates/numcodecs/src/erased.rs +++ b/crates/numcodecs/src/erased.rs @@ -96,6 +96,7 @@ impl ErasedDynCodec { pub fn codec_config_schema(generator: &mut SchemaGenerator) -> Schema { #[derive(JsonSchema)] #[schemars(extend("additionalProperties" = {"type": "object"}))] + #[schemars(inline)] /// The configuration for a codec. struct Codec { /// The `codec_id` of the codec, which is looked up in the global From 318cce8f54a7bbe37a5bb3bdafbb751b31fb4ece Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 3 Jul 2026 20:59:58 +0300 Subject: [PATCH 59/72] disable OptZConfig support for now, fix dlib for wasm later --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 3e661f76a..630c15a3a 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -16,7 +16,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] [dependencies] fragile = { workspace = true } -libpressio = { workspace = true, features = ["bzip2", "lua", "mpi-stubs", "opt"] } +libpressio = { workspace = true, features = ["bzip2", "lua"] } ndarray = { workspace = true } numcodecs = { workspace = true } numcodecs-registry = { workspace = true } From e7f14a12fa91ac5b5518833a257b5101693481f0 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 4 Jul 2026 16:24:42 +0300 Subject: [PATCH 60/72] Implement trampoline serialise, deserialise, and set_options, get_options missing --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 1 + codecs/pressio/src/lib.rs | 502 +++++++++++++++++++++++++++++--------- 3 files changed, 386 insertions(+), 119 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d32ce5e94..48820084a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "a315cb1", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "d00d999", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 630c15a3a..fb301ff53 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -23,6 +23,7 @@ numcodecs-registry = { workspace = true } schemars = { workspace = true, features = ["derive", "preserve_order"] } serde = { workspace = true, features = ["std", "derive"] } serde-ndim = { workspace = true, features = ["ndarray"] } +serde_json = { workspace = true, features = ["std"] } thiserror = { workspace = true } [dev-dependencies] diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 8e2d93e66..219e858fa 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -20,16 +20,18 @@ use std::{ borrow::Cow, collections::{BTreeMap, btree_map::Entry}, + pin::Pin, sync::{Arc, Mutex, RwLock}, }; use fragile::Fragile; -use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, IxDyn}; +use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, Dim, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, DynCodec, DynCodecType, ErasedDynCodec, StaticCodec, StaticCodecConfig, StaticCodecVersion, }; +use numcodecs_registry::Registry; use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; @@ -130,104 +132,7 @@ impl PressioCompressorSendable { } impl Serialize for PressioCompressor { - #[expect(clippy::too_many_lines)] fn serialize(&self, serializer: S) -> Result { - fn convert_from_pressio_options( - options: impl Iterator, Option)>, - ) -> Result, E> { - let mut config = BTreeMap::new(); - - for (name, option) in options { - // skip invalid option names and values - let (Some(name), Some(option)) = (name, option) else { - continue; - }; - - let value = match option { - libpressio::PressioOption::bool(Some(x)) => PressioOption::Bool(x), - libpressio::PressioOption::int8(Some(x)) => PressioOption::I8(x), - libpressio::PressioOption::int16(Some(x)) => PressioOption::I16(x), - libpressio::PressioOption::int32(Some(x)) => PressioOption::I32(x), - libpressio::PressioOption::int64(Some(x)) => PressioOption::I64(x), - libpressio::PressioOption::uint8(Some(x)) => PressioOption::U8(x), - libpressio::PressioOption::uint16(Some(x)) => PressioOption::U16(x), - libpressio::PressioOption::uint32(Some(x)) => PressioOption::U32(x), - libpressio::PressioOption::uint64(Some(x)) => PressioOption::U64(x), - libpressio::PressioOption::float32(Some(x)) => PressioOption::F32(x), - libpressio::PressioOption::float64(Some(x)) => PressioOption::F64(x), - libpressio::PressioOption::string(Some(x)) => PressioOption::String(x), - libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), - libpressio::PressioOption::dtype(Some(x)) => PressioOption::String(format!("{x}")), - libpressio::PressioOption::thread_safety(Some(x)) => PressioOption::String(format!("{x}")), - libpressio::PressioOption::data(Some(x)) => match x.clone_into_array() { - Option::None => continue, - Some(libpressio::PressioArray::Bool(x)) => PressioOption::DataBool(NdArray(x)), - Some(libpressio::PressioArray::Byte(x) | libpressio::PressioArray::U8(x)) => PressioOption::DataU8(NdArray(x)), - Some(libpressio::PressioArray::U16(x)) => PressioOption::DataU16(NdArray(x)), - Some(libpressio::PressioArray::U32(x)) => PressioOption::DataU32(NdArray(x)), - Some(libpressio::PressioArray::U64(x)) => PressioOption::DataU64(NdArray(x)), - Some(libpressio::PressioArray::I8(x)) => PressioOption::DataI8(NdArray(x)), - Some(libpressio::PressioArray::I16(x)) => PressioOption::DataI16(NdArray(x)), - Some(libpressio::PressioArray::I32(x)) => PressioOption::DataI32(NdArray(x)), - Some(libpressio::PressioArray::I64(x)) => PressioOption::DataI64(NdArray(x)), - Some(libpressio::PressioArray::F32(x)) => PressioOption::DataF32(NdArray(x)), - Some(libpressio::PressioArray::F64(x)) => PressioOption::DataF64(NdArray(x)), - }, - libpressio::PressioOption::user_ptr(_) - | libpressio::PressioOption::unset - | _ /* non-exhaustive */ => continue, - }; - - let Some(nested_name) = name.strip_prefix('/') else { - // global option - if config.insert(name.clone(), value).is_some() { - return Err(serde::ser::Error::custom(format!( - "duplicate global option: `{name}`" - ))); - } - continue; - }; - - // hierarchical option - let mut parts = nested_name.split(':').peekable(); - - let Some(first) = parts.next() else { - return Err(serde::ser::Error::custom(format!( - "invalid hierarchical config name `{name}`" - ))); - }; - let paths = first.split('/'); - - if parts.peek().is_none() { - return Err(serde::ser::Error::custom(format!( - "invalid hierarchical config name `{name}`" - ))); - } - let option_name = parts.map(String::from).collect::>().join(":"); - - let mut it = &mut config; - for path in paths { - if let Entry::Vacant(entry) = it.entry(String::from(path)) { - entry.insert(PressioOption::Nested(BTreeMap::new())); - } - - let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { - return Err(serde::ser::Error::custom(format!( - "duplicate option nesting: `{path}` in `{name}`" - ))); - }; - it = entry; - } - if it.insert(option_name.clone(), value).is_some() { - return Err(serde::ser::Error::custom(format!( - "duplicate nested option: `{option_name}` in `{name}`" - ))); - } - } - - Ok(config) - } - let inner = self.inner.read().map_err(serde::ser::Error::custom)?; let compressor_guard = inner.compressor.lock().map_err(serde::ser::Error::custom)?; let compressor = compressor_guard @@ -244,8 +149,10 @@ impl Serialize for PressioCompressor { let result = PressioCompressorBorrowedFormat { compressor_id: inner.compressor_id.as_str(), early_config: &inner.early_config, - compressor_config: &convert_from_pressio_options(options.iter())?, - metric_results: &convert_from_pressio_options(metric_results.iter())?, + compressor_config: &convert_from_pressio_options(options.iter()) + .map_err(serde::ser::Error::custom)?, + metric_results: &convert_from_pressio_options(metric_results.iter()) + .map_err(serde::ser::Error::custom)?, name: match name { "" => Option::None, name => Some(name), @@ -258,6 +165,94 @@ impl Serialize for PressioCompressor { } } +fn convert_from_pressio_options( + options: impl Iterator, Option)>, +) -> Result, String> { + let mut config = BTreeMap::new(); + + for (name, option) in options { + // skip invalid option names and values + let (Some(name), Some(option)) = (name, option) else { + continue; + }; + + let value = match option { + libpressio::PressioOption::bool(Some(x)) => PressioOption::Bool(x), + libpressio::PressioOption::int8(Some(x)) => PressioOption::I8(x), + libpressio::PressioOption::int16(Some(x)) => PressioOption::I16(x), + libpressio::PressioOption::int32(Some(x)) => PressioOption::I32(x), + libpressio::PressioOption::int64(Some(x)) => PressioOption::I64(x), + libpressio::PressioOption::uint8(Some(x)) => PressioOption::U8(x), + libpressio::PressioOption::uint16(Some(x)) => PressioOption::U16(x), + libpressio::PressioOption::uint32(Some(x)) => PressioOption::U32(x), + libpressio::PressioOption::uint64(Some(x)) => PressioOption::U64(x), + libpressio::PressioOption::float32(Some(x)) => PressioOption::F32(x), + libpressio::PressioOption::float64(Some(x)) => PressioOption::F64(x), + libpressio::PressioOption::string(Some(x)) => PressioOption::String(x), + libpressio::PressioOption::vec_string(Some(x)) => PressioOption::VecString(x), + libpressio::PressioOption::dtype(Some(x)) => PressioOption::String(format!("{x}")), + libpressio::PressioOption::thread_safety(Some(x)) => PressioOption::String(format!("{x}")), + libpressio::PressioOption::data(Some(x)) => match x.clone_into_array() { + Option::None => continue, + Some(libpressio::PressioArray::Bool(x)) => PressioOption::DataBool(NdArray(x)), + Some(libpressio::PressioArray::Byte(x) | libpressio::PressioArray::U8(x)) => PressioOption::DataU8(NdArray(x)), + Some(libpressio::PressioArray::U16(x)) => PressioOption::DataU16(NdArray(x)), + Some(libpressio::PressioArray::U32(x)) => PressioOption::DataU32(NdArray(x)), + Some(libpressio::PressioArray::U64(x)) => PressioOption::DataU64(NdArray(x)), + Some(libpressio::PressioArray::I8(x)) => PressioOption::DataI8(NdArray(x)), + Some(libpressio::PressioArray::I16(x)) => PressioOption::DataI16(NdArray(x)), + Some(libpressio::PressioArray::I32(x)) => PressioOption::DataI32(NdArray(x)), + Some(libpressio::PressioArray::I64(x)) => PressioOption::DataI64(NdArray(x)), + Some(libpressio::PressioArray::F32(x)) => PressioOption::DataF32(NdArray(x)), + Some(libpressio::PressioArray::F64(x)) => PressioOption::DataF64(NdArray(x)), + }, + libpressio::PressioOption::user_ptr(_) + | libpressio::PressioOption::unset + | _ /* non-exhaustive */ => continue, + }; + + let Some(nested_name) = name.strip_prefix('/') else { + // global option + if config.insert(name.clone(), value).is_some() { + return Err(format!("duplicate global option: `{name}`")); + } + continue; + }; + + // hierarchical option + let mut parts = nested_name.split(':').peekable(); + + let Some(first) = parts.next() else { + return Err(format!("invalid hierarchical config name `{name}`")); + }; + let paths = first.split('/'); + + if parts.peek().is_none() { + return Err(format!("invalid hierarchical config name `{name}`")); + } + let option_name = parts.map(String::from).collect::>().join(":"); + + let mut it = &mut config; + for path in paths { + if let Entry::Vacant(entry) = it.entry(String::from(path)) { + entry.insert(PressioOption::Nested(BTreeMap::new())); + } + + let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { + return Err(format!("duplicate option nesting: `{path}` in `{name}`")); + }; + it = entry; + } + if it.insert(option_name.clone(), value).is_some() { + return Err(format!( + "duplicate nested option: `{option_name}` in `{name}`" + )); + } + } + + Ok(config) +} + impl<'de> Deserialize<'de> for PressioCompressor { #[expect(clippy::too_many_lines)] // FIXME fn deserialize>(deserializer: D) -> Result { @@ -419,7 +414,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { .register_compressor( "numcodecs.rs", NumcodecsPressioCompressor { - codec: Option::None, + codec: RwLock::new(Option::None), }, "0.1.0.0", 0, @@ -1039,9 +1034,17 @@ pub enum PressioCodecError { /// Opaque error for when encoding or decoding with libpressio fails pub struct PressioCodingError(libpressio::PressioError); -#[derive(Clone)] struct NumcodecsPressioCompressor { - codec: Option, + codec: RwLock>, +} + +impl Clone for NumcodecsPressioCompressor { + #[expect(clippy::unwrap_used)] + fn clone(&self) -> Self { + Self { + codec: RwLock::new(self.codec.read().unwrap().clone()), + } + } } #[expect(clippy::expect_used)] @@ -1085,7 +1088,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { fn get_options(&self) -> libpressio::PressioOptions { (|| -> Result { let mut options = libpressio::PressioOptions::new()?; - if let Some(codec) = &self.codec { + let codec = self.codec.read().expect("codec poisoned"); + if let Some(codec) = &*codec { options.set( "numcodecs.rs:id", libpressio::PressioOption::string(Some(String::from(codec.ty().codec_id()))), @@ -1097,6 +1101,7 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { libpressio::PressioOption::string(Option::None), )?; } + std::mem::drop(codec); Ok(options) })() .expect("get_options should not fail") @@ -1106,33 +1111,294 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { &self, options: &libpressio::PressioOptions, ) -> Result<(), libpressio::PressioError> { - let _id = options.get("numcodecs.rs:id")?; + let options = convert_from_pressio_options(options.iter()).map_err(|err| { + libpressio::PressioError { + error_code: 1, + message: err, + } + })?; + + #[expect(clippy::option_if_let_else)] + let options = options + .into_iter() + .map(|(key, value)| match key.strip_prefix("numcodecs.rs:") { + Some(key) => Ok((String::from(key), value)), + Option::None => Err(libpressio::PressioError { + error_code: 1, + message: format!( + "numcodecs.rs option `{key}` does not start with `numcodecs.rs:`" + ), + }), + }) + .collect::, libpressio::PressioError>>()?; + + let Some(id) = options.get("numcodecs.rs:id") else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("missing numcodecs.rs:id"), + }); + }; + + let PressioOption::String(_id) = id else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("numcodecs.rs:id must be a string"), + }); + }; - todo!("deserialize options") + let config = match serde_json::to_string(&options) { + Ok(config) => config, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + + let codec = match numcodecs_registry::GlobalRegistry + .get_codec(&mut serde_json::Deserializer::from_str(&config)) + { + Ok(codec) => codec, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + + *self.codec.write().expect("codec poinsoned") = Some(codec); + + Ok(()) } fn compress( &self, - _input_data: &libpressio::PressioData, - _compressed_data: libpressio::PressioData, - ) -> Result { - let Some(_codec) = &self.codec else { - todo!("error or no-op"); + input_data: &libpressio::PressioData, + compressed_data: Pin<&mut libpressio::PinnedPressioData>, + ) -> Result<(), libpressio::PressioError> { + let codec_read = self.codec.read().expect("codec poisoned"); + let Some(codec) = &*codec_read else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("uninitialized numcodecs codec"), + }); + }; + + let input_shape = input_data.shape(); + + let encoded = match input_data.dtype() { + Option::None => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unsupported input data type"), + }); + } + Some(libpressio::PressioDtype::Bool) => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unsupported input bool array"), + }); + } + Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::U8(input_data)) + }), + Some(libpressio::PressioDtype::U16) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::U16(input_data)) + }), + Some(libpressio::PressioDtype::U32) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::U32(input_data)) + }), + Some(libpressio::PressioDtype::U64) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::U64(input_data)) + }), + Some(libpressio::PressioDtype::I8) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::I8(input_data)) + }), + Some(libpressio::PressioDtype::I16) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::I16(input_data)) + }), + Some(libpressio::PressioDtype::I32) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::I32(input_data)) + }), + Some(libpressio::PressioDtype::I64) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::I64(input_data)) + }), + Some(libpressio::PressioDtype::F32) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::F32(input_data)) + }), + Some(libpressio::PressioDtype::F64) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::F64(input_data)) + }), + }; + + std::mem::drop(codec_read); + + let Some(encoded) = encoded else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unexpected encoded data type or shape mismatch"), + }); + }; + + let encoded = match encoded { + Ok(encoded) => encoded, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + + let compressed = match encoded { + AnyArray::U8(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::U16(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::U32(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::U64(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::I8(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::I16(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::I32(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::I64(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::F32(encoded) => libpressio::PressioData::new_copied(encoded), + AnyArray::F64(encoded) => libpressio::PressioData::new_copied(encoded), + encoded => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("unsupported encoded data type {}", encoded.dtype()), + }); + } }; - todo!("compress"); + compressed_data.overwrite(compressed); + + Ok(()) } fn decompress( &self, - _compressed_data: &libpressio::PressioData, - _decompressed_data: libpressio::PressioData, - ) -> Result { - let Some(_codec) = &self.codec else { - todo!("error or no-op"); + compressed_data: &libpressio::PressioData, + decompressed_data: Pin<&mut libpressio::PinnedPressioData>, + ) -> Result<(), libpressio::PressioError> { + let codec_read = self.codec.read().expect("codec poisoned"); + let Some(codec) = &*codec_read else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("uninitialized numcodecs codec"), + }); + }; + + let compressed_shape = compressed_data.shape(); + + // TODO: take decompressed_data shape and dtype into account + + let decoded = match compressed_data.dtype() { + Option::None => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unsupported compressed data type"), + }); + } + Some(libpressio::PressioDtype::Bool) => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unsupported compressed bool array"), + }); + } + Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::U8(compressed_data)) + }), + Some(libpressio::PressioDtype::U16) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::U16(compressed_data)) + }), + Some(libpressio::PressioDtype::U32) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::U32(compressed_data)) + }), + Some(libpressio::PressioDtype::U64) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::U64(compressed_data)) + }), + Some(libpressio::PressioDtype::I8) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::I8(compressed_data)) + }), + Some(libpressio::PressioDtype::I16) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::I16(compressed_data)) + }), + Some(libpressio::PressioDtype::I32) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::I32(compressed_data)) + }), + Some(libpressio::PressioDtype::I64) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::I64(compressed_data)) + }), + Some(libpressio::PressioDtype::F32) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::F32(compressed_data)) + }), + Some(libpressio::PressioDtype::F64) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::F64(compressed_data)) + }), }; - todo!("decompress"); + std::mem::drop(codec_read); + + let Some(decoded) = decoded else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unexpected decoded data type or shape mismatch"), + }); + }; + + let decoded = match decoded { + Ok(decoded) => decoded, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + + let decompressed = match decoded { + AnyArray::U8(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::U16(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::U32(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::U64(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::I8(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::I16(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::I32(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::I64(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::F32(decoded) => libpressio::PressioData::new_copied(decoded), + AnyArray::F64(decoded) => libpressio::PressioData::new_copied(decoded), + decoded => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("unsupported decoded data type {}", decoded.dtype()), + }); + } + }; + + decompressed_data.overwrite(decompressed); + + Ok(()) } fn get_metrics_results(&self) -> libpressio::PressioOptions { From c7077753a103ecbf238cd07c5f8af7c95577518f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 4 Jul 2026 18:11:10 +0300 Subject: [PATCH 61/72] draft implement get_options --- codecs/pressio/src/lib.rs | 332 +++++++++++++++++++------------------- 1 file changed, 170 insertions(+), 162 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 219e858fa..713d4347b 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -28,8 +28,7 @@ use fragile::Fragile; use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, Dim, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, - Codec, DynCodec, DynCodecType, ErasedDynCodec, StaticCodec, StaticCodecConfig, - StaticCodecVersion, + Codec, DynCodec, ErasedDynCodec, StaticCodec, StaticCodecConfig, StaticCodecVersion, }; use numcodecs_registry::Registry; use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; @@ -254,157 +253,7 @@ fn convert_from_pressio_options( } impl<'de> Deserialize<'de> for PressioCompressor { - #[expect(clippy::too_many_lines)] // FIXME fn deserialize>(deserializer: D) -> Result { - fn convert_to_pressio_options( - config: &BTreeMap, - template: Option<&libpressio::PressioOptions>, - documentation: &libpressio::PressioOptions, - ) -> Result { - let mut options = - libpressio::PressioOptions::new().map_err(serde::de::Error::custom)?; - - let mut entries = vec![(vec![], config)]; - - while let Some((path, entry)) = entries.pop() { - for (key, value) in entry { - let option = match value { - PressioOption::None(None) => Option::None, - PressioOption::Bool(x) => Some(libpressio::PressioOption::bool(Some(*x))), - PressioOption::U8(x) => Some(libpressio::PressioOption::uint8(Some(*x))), - PressioOption::I8(x) => Some(libpressio::PressioOption::int8(Some(*x))), - PressioOption::U16(x) => Some(libpressio::PressioOption::uint16(Some(*x))), - PressioOption::I16(x) => Some(libpressio::PressioOption::int16(Some(*x))), - PressioOption::U32(x) => Some(libpressio::PressioOption::uint32(Some(*x))), - PressioOption::I32(x) => Some(libpressio::PressioOption::int32(Some(*x))), - PressioOption::U64(x) => Some(libpressio::PressioOption::uint64(Some(*x))), - PressioOption::I64(x) => Some(libpressio::PressioOption::int64(Some(*x))), - PressioOption::F32(x) => Some(libpressio::PressioOption::float32(Some(*x))), - PressioOption::F64(x) => Some(libpressio::PressioOption::float64(Some(*x))), - PressioOption::String(x) => { - Some(libpressio::PressioOption::string(Some(x.clone()))) - } - PressioOption::VecString(x) => { - Some(libpressio::PressioOption::vec_string(Some(x.clone()))) - } - PressioOption::DataBool(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::DataU8(NdArray(x)) => Some(libpressio::PressioOption::data( - Some(libpressio::PressioData::new_copied(x)), - )), - PressioOption::DataU16(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::DataU32(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::DataU64(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::DataI8(NdArray(x)) => Some(libpressio::PressioOption::data( - Some(libpressio::PressioData::new_copied(x)), - )), - PressioOption::DataI16(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::DataI32(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::DataI64(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::DataF32(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::DataF64(NdArray(x)) => { - Some(libpressio::PressioOption::data(Some( - libpressio::PressioData::new_copied(x), - ))) - } - PressioOption::Nested(entry) => { - let mut nested_path = path.clone(); - nested_path.push(key.clone()); - entries.push((nested_path, entry)); - continue; - } - }; - - let name = if path.is_empty() { - key.clone() - } else { - format!("/{path}:{key}", path = path.join("/")) - }; - - if let Some(template) = template { - let Some(option_template) = - template.get(&name).map_err(serde::de::Error::custom)? - else { - let supported_options = template - .iter() - .filter_map(|(key, _value)| key) - .map(|x| format!("`{x}`")) - .collect::>() - .join(", "); - - return Err(serde::de::Error::custom(format!( - "unknown compressor configuration option: `{name}`, use one of {supported_options}" - ))); - }; - - options - .set(&name, option_template.copy_type_only()) - .map_err(serde::de::Error::custom)?; - - if let Some(option) = option { - options - .set_with_cast( - &name, - option, - libpressio::PressioConversionSafety::Special, - ) - .map_err(|err| { - let docs = match documentation.get(&name) { - Ok(Some(libpressio::PressioOption::string(Some(docs)))) => { - Some(docs) - } - _ => Option::None, - }; - - if let Some(docs) = docs { - serde::de::Error::custom(format_args!("{err} ({docs})")) - } else { - serde::de::Error::custom(err) - } - })?; - } - } else if let Some(option) = option { - options - .set(name, option) - .map_err(serde::de::Error::custom)?; - } - } - } - - Ok(options) - } - // TODO: better error handling let format = PressioCompressorOwnedFormat::deserialize(deserializer)?; std::mem::drop(format.metric_results); @@ -452,7 +301,8 @@ impl<'de> Deserialize<'de> for PressioCompressor { .map_err(serde::de::Error::custom)?; let early_options = - convert_to_pressio_options(&format.early_config, Option::None, &documentation)?; + convert_to_pressio_options(&format.early_config, Option::None, &documentation) + .map_err(serde::de::Error::custom)?; compressor .set_options(&early_options) .map_err(serde::de::Error::custom)?; @@ -462,7 +312,8 @@ impl<'de> Deserialize<'de> for PressioCompressor { &format.compressor_config, Some(&options_template), &documentation, - )?; + ) + .map_err(serde::de::Error::custom)?; compressor .set_options(&options) .map_err(serde::de::Error::custom)?; @@ -482,6 +333,133 @@ impl<'de> Deserialize<'de> for PressioCompressor { } } +#[expect(clippy::too_many_lines)] +fn convert_to_pressio_options( + config: &BTreeMap, + template: Option<&libpressio::PressioOptions>, + documentation: &libpressio::PressioOptions, +) -> Result { + let mut options = libpressio::PressioOptions::new()?; + + let mut entries = vec![(vec![], config)]; + + while let Some((path, entry)) = entries.pop() { + for (key, value) in entry { + let option = match value { + PressioOption::None(None) => Option::None, + PressioOption::Bool(x) => Some(libpressio::PressioOption::bool(Some(*x))), + PressioOption::U8(x) => Some(libpressio::PressioOption::uint8(Some(*x))), + PressioOption::I8(x) => Some(libpressio::PressioOption::int8(Some(*x))), + PressioOption::U16(x) => Some(libpressio::PressioOption::uint16(Some(*x))), + PressioOption::I16(x) => Some(libpressio::PressioOption::int16(Some(*x))), + PressioOption::U32(x) => Some(libpressio::PressioOption::uint32(Some(*x))), + PressioOption::I32(x) => Some(libpressio::PressioOption::int32(Some(*x))), + PressioOption::U64(x) => Some(libpressio::PressioOption::uint64(Some(*x))), + PressioOption::I64(x) => Some(libpressio::PressioOption::int64(Some(*x))), + PressioOption::F32(x) => Some(libpressio::PressioOption::float32(Some(*x))), + PressioOption::F64(x) => Some(libpressio::PressioOption::float64(Some(*x))), + PressioOption::String(x) => { + Some(libpressio::PressioOption::string(Some(x.clone()))) + } + PressioOption::VecString(x) => { + Some(libpressio::PressioOption::vec_string(Some(x.clone()))) + } + PressioOption::DataBool(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataU8(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataU16(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataU32(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataU64(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataI8(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataI16(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataI32(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataI64(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataF32(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::DataF64(NdArray(x)) => Some(libpressio::PressioOption::data(Some( + libpressio::PressioData::new_copied(x), + ))), + PressioOption::Nested(entry) => { + let mut nested_path = path.clone(); + nested_path.push(key.clone()); + entries.push((nested_path, entry)); + continue; + } + }; + + let name = if path.is_empty() { + key.clone() + } else { + format!("/{path}:{key}", path = path.join("/")) + }; + + if let Some(template) = template { + let Some(option_template) = template.get(&name)? else { + let supported_options = template + .iter() + .filter_map(|(key, _value)| key) + .map(|x| format!("`{x}`")) + .collect::>() + .join(", "); + + return Err(libpressio::PressioError { + error_code: 1, + message: format!( + "unknown compressor configuration option: `{name}`, use one of {supported_options}" + ), + }); + }; + + options.set(&name, option_template.copy_type_only())?; + + if let Some(option) = option { + options + .set_with_cast(&name, option, libpressio::PressioConversionSafety::Special) + .map_err(|err| { + let docs = match documentation.get(&name) { + Ok(Some(libpressio::PressioOption::string(Some(docs)))) => { + Some(docs) + } + _ => Option::None, + }; + + if let Some(docs) = docs { + libpressio::PressioError { + error_code: err.error_code, + message: format!("{err} ({docs})"), + } + } else { + err + } + })?; + } + } else if let Some(option) = option { + options.set(name, option)?; + } + } + } + + Ok(options) +} + impl JsonSchema for PressioCompressor { fn schema_name() -> Cow<'static, str> { PressioCompressorOwnedFormat::schema_name() @@ -1087,20 +1065,50 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { fn get_options(&self) -> libpressio::PressioOptions { (|| -> Result { - let mut options = libpressio::PressioOptions::new()?; let codec = self.codec.read().expect("codec poisoned"); - if let Some(codec) = &*codec { - options.set( - "numcodecs.rs:id", - libpressio::PressioOption::string(Some(String::from(codec.ty().codec_id()))), - )?; - // TODO: serialize all remaining options + let options = if let Some(codec) = &*codec { + let mut config_bytes = Vec::new(); + match codec.get_config(&mut serde_json::Serializer::new(&mut config_bytes)) { + Ok(()) => (), + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + } + let Ok(config) = String::from_utf8(config_bytes) else { + return Err(libpressio::PressioError { + error_code: 2, + message: String::from("invalid UTF-8 in JSON config"), + }); + }; + + let options: BTreeMap = + match BTreeMap::deserialize(&mut serde_json::Deserializer::from_str(&config)) { + Ok(options) => options, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + let options = options + .into_iter() + .map(|(key, value)| (format!("numcodecs.rs:{key}"), value)) + .collect(); + + let documentation = libpressio::PressioOptions::new()?; + convert_to_pressio_options(&options, Option::None, &documentation)? } else { + let mut options = libpressio::PressioOptions::new()?; options.set( "numcodecs.rs:id", libpressio::PressioOption::string(Option::None), )?; - } + options + }; std::mem::drop(codec); Ok(options) })() From 5529bad462fd6fe5855c4e05382cdc009b63a9bb Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 4 Jul 2026 18:44:25 +0300 Subject: [PATCH 62/72] fix numcodecs.rs codec validation, should support propagating error messages --- codecs/pressio/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 713d4347b..4e4bdf8ae 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -1140,7 +1140,7 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { }) .collect::, libpressio::PressioError>>()?; - let Some(id) = options.get("numcodecs.rs:id") else { + let Some(id) = options.get("id") else { return Err(libpressio::PressioError { error_code: 1, message: String::from("missing numcodecs.rs:id"), From 804baca99cd26f26bc34ad6a01cb36e3c8ed1eb1 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sat, 4 Jul 2026 22:05:43 +0300 Subject: [PATCH 63/72] trampoline improvements --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 40 ++++++++++++--------------------------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 48820084a..ac13808f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "d00d999", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "30dad37", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 4e4bdf8ae..46b9638ab 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -263,7 +263,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { .register_compressor( "numcodecs.rs", NumcodecsPressioCompressor { - codec: RwLock::new(Option::None), + codec: Option::None, }, "0.1.0.0", 0, @@ -1012,17 +1012,9 @@ pub enum PressioCodecError { /// Opaque error for when encoding or decoding with libpressio fails pub struct PressioCodingError(libpressio::PressioError); +#[derive(Clone)] struct NumcodecsPressioCompressor { - codec: RwLock>, -} - -impl Clone for NumcodecsPressioCompressor { - #[expect(clippy::unwrap_used)] - fn clone(&self) -> Self { - Self { - codec: RwLock::new(self.codec.read().unwrap().clone()), - } - } + codec: Option, } #[expect(clippy::expect_used)] @@ -1065,8 +1057,7 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { fn get_options(&self) -> libpressio::PressioOptions { (|| -> Result { - let codec = self.codec.read().expect("codec poisoned"); - let options = if let Some(codec) = &*codec { + let options = if let Some(codec) = &self.codec { let mut config_bytes = Vec::new(); match codec.get_config(&mut serde_json::Serializer::new(&mut config_bytes)) { Ok(()) => (), @@ -1109,14 +1100,13 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { )?; options }; - std::mem::drop(codec); Ok(options) })() .expect("get_options should not fail") } fn set_options( - &self, + &mut self, options: &libpressio::PressioOptions, ) -> Result<(), libpressio::PressioError> { let options = convert_from_pressio_options(options.iter()).map_err(|err| { @@ -1176,18 +1166,17 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { } }; - *self.codec.write().expect("codec poinsoned") = Some(codec); + self.codec = Some(codec); Ok(()) } fn compress( - &self, + &mut self, input_data: &libpressio::PressioData, - compressed_data: Pin<&mut libpressio::PinnedPressioData>, + compressed_data: Pin<&mut libpressio::PressioPinnedData>, ) -> Result<(), libpressio::PressioError> { - let codec_read = self.codec.read().expect("codec poisoned"); - let Some(codec) = &*codec_read else { + let Some(codec) = &self.codec else { return Err(libpressio::PressioError { error_code: 1, message: String::from("uninitialized numcodecs codec"), @@ -1251,8 +1240,6 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { }), }; - std::mem::drop(codec_read); - let Some(encoded) = encoded else { return Err(libpressio::PressioError { error_code: 1, @@ -1295,12 +1282,11 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { } fn decompress( - &self, + &mut self, compressed_data: &libpressio::PressioData, - decompressed_data: Pin<&mut libpressio::PinnedPressioData>, + decompressed_data: Pin<&mut libpressio::PressioPinnedData>, ) -> Result<(), libpressio::PressioError> { - let codec_read = self.codec.read().expect("codec poisoned"); - let Some(codec) = &*codec_read else { + let Some(codec) = &self.codec else { return Err(libpressio::PressioError { error_code: 1, message: String::from("uninitialized numcodecs codec"), @@ -1366,8 +1352,6 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { }), }; - std::mem::drop(codec_read); - let Some(decoded) = decoded else { return Err(libpressio::PressioError { error_code: 1, From d741b94f397c343d77b5fb6d0cf73ab4fca7e09f Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 5 Jul 2026 05:35:20 +0300 Subject: [PATCH 64/72] re-enable optzconfig for wasm --- Cargo.toml | 2 +- codecs/pressio/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ac13808f5..020b96720 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "30dad37", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "d0948bb", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index fb301ff53..e5c6457b9 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -16,7 +16,7 @@ keywords = ["libpressio", "numcodecs", "compression", "encoding"] [dependencies] fragile = { workspace = true } -libpressio = { workspace = true, features = ["bzip2", "lua"] } +libpressio = { workspace = true, features = ["bzip2", "lua", "opt", "mpi-stubs"] } ndarray = { workspace = true } numcodecs = { workspace = true } numcodecs-registry = { workspace = true } From f9cca13828de5caaf5341f0e44f1687c7eee046d Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 5 Jul 2026 07:06:10 +0300 Subject: [PATCH 65/72] fix libpressio-opt dlib linking --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 020b96720..90180d90e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "d0948bb", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6527504", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 173c7a05be3292ba401d8a80bfa94f6c90ab0b3e Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 5 Jul 2026 07:49:32 +0300 Subject: [PATCH 66/72] fix dlib infinite recursion --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 90180d90e..08e837938 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "6527504", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "b857bbe", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 64d792dea9ccf09c30b715607f73cb78a9f6a774 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 5 Jul 2026 07:56:42 +0300 Subject: [PATCH 67/72] add optzconfig tests --- codecs/pressio/src/lib.rs | 94 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 46b9638ab..bc4cfd83d 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -1539,6 +1539,100 @@ mod tests { assert!(config.contains("\"composite:objective2\":4.2")); } + #[test] + fn optzconfig_fraz() { + let pressio = PressioCodec::deserialize(json!({ + "compressor_id": "pressio", + "compressor_config": { + "pressio:metric": "error_stat", + "opt:compressor": "linear_quantizer", + "linear_quantizer:compressor": "bzip2", + "opt:search": "fraz", + "opt:output": ["error_stat:psnr"], + "opt:inputs": ["linear_quantizer:step"], + "opt:lower_bound": 1e-8, + "opt:upper_bound": 1e-3, + "opt:max_iterations": 30, + "opt:objective_mode_name": "max", + }, + "early_config": { + "pressio:compressor": "opt", + "opt:compressor": "linear_quantizer", + "opt:search": "fraz", + }, + })) + .unwrap(); + + let config = serde_json::to_string(&StaticCodec::get_config(&pressio)).unwrap(); + assert!(config.contains("\"opt:search\":\"fraz\"")); + + let data = ndarray::linspace(0.0, 100.0, 50) + .collect::>() + .into_dyn(); + + let encoded = pressio + .encode(AnyCowArray::F64(CowArray::from(&data))) + .unwrap(); + + let decoded = pressio.decode(encoded.cow()); + assert!(matches!( + decoded, + Err(PressioCodecError::DecodeToArrayWithoutData) + )); + + let mut decoded = ndarray::Array::zeros(data.dim()); + pressio + .decode_into(encoded.view(), AnyArrayViewMut::F64(decoded.view_mut())) + .unwrap(); + } + + #[test] + fn optzconfig_random() { + let pressio = PressioCodec::deserialize(json!({ + "compressor_id": "pressio", + "compressor_config": { + "pressio:metric": "error_stat", + "opt:compressor": "linear_quantizer", + "linear_quantizer:compressor": "bzip2", + "opt:search": "random_search", + "opt:output": ["error_stat:psnr"], + "opt:inputs": ["linear_quantizer:step"], + "opt:lower_bound": 1e-8, + "opt:upper_bound": 1e-3, + "opt:max_iterations": 30, + "opt:objective_mode_name": "min", + }, + "early_config": { + "pressio:compressor": "opt", + "opt:compressor": "linear_quantizer", + "opt:search": "random_search", + }, + })) + .unwrap(); + + let config = serde_json::to_string(&StaticCodec::get_config(&pressio)).unwrap(); + assert!(config.contains("\"opt:search\":\"random_search\"")); + + let data = ndarray::linspace(0.0, 100.0, 50) + .collect::>() + .into_dyn(); + + let encoded = pressio + .encode(AnyCowArray::F64(CowArray::from(&data))) + .unwrap(); + + let decoded = pressio.decode(encoded.cow()); + assert!(matches!( + decoded, + Err(PressioCodecError::DecodeToArrayWithoutData) + )); + + let mut decoded = ndarray::Array::zeros(data.dim()); + pressio + .decode_into(encoded.view(), AnyArrayViewMut::F64(decoded.view_mut())) + .unwrap(); + } + numcodecs_registry::export_global! { static REGISTRY: numcodecs_registry::EmptyRegistry = numcodecs_registry::EmptyRegistry; } From 9ae21101e47ee5f45407ced603aa71430ae11ab6 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Sun, 5 Jul 2026 21:31:40 +0300 Subject: [PATCH 68/72] include in dlib --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 08e837938..d8cd53c10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "b857bbe", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "043238c", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy From 259eab73fbdc250389b5a97c9ac65e7dd027a935 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 6 Jul 2026 11:35:13 +0300 Subject: [PATCH 69/72] implement numcodecs codec as metric trampoline --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 518 +++++++++++++++++++++++++++++++++++++- 2 files changed, 518 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d8cd53c10..2656a1e55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "043238c", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1411224", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index bc4cfd83d..448a5b716 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -25,7 +25,7 @@ use std::{ }; use fragile::Fragile; -use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, Dim, IxDyn}; +use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, Dim, Ix0, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, DynCodec, ErasedDynCodec, StaticCodec, StaticCodecConfig, StaticCodecVersion, @@ -272,6 +272,16 @@ impl<'de> Deserialize<'de> for PressioCompressor { 0, ) .map_err(serde::de::Error::custom)?; + pressio + .register_metric( + "numcodecs.rs-metric", + NumcodecsPressioMetric { + codec: Option::None, + compression_result: Option::None, + decompression_result: Option::None, + }, + ) + .map_err(serde::de::Error::custom)?; let mut compressor = pressio .get_compressor(format.compressor_id.as_str()) .map_err(|err| { @@ -1105,6 +1115,13 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { .expect("get_options should not fail") } + fn check_options( + &self, + _options: &libpressio::PressioOptions, + ) -> Result<(), libpressio::PressioError> { + Ok(()) + } + fn set_options( &mut self, options: &libpressio::PressioOptions, @@ -1393,11 +1410,510 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(()) } + fn compress_many( + &mut self, + input_data: &[libpressio::PressioData], + compressed_data: Pin<&mut [libpressio::PressioPinnedData]>, + ) -> Result<(), libpressio::PressioError> { + if input_data.len() != compressed_data.len() { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("mismatched number of compress_many inputs and outputs"), + }); + } + + // FIXME: https://github.com/rust-lang/libs-team/issues/607 + #[expect(unsafe_code)] + let compressed_data_unsafe = unsafe { compressed_data.get_unchecked_mut() }; + + for (i, input_data) in input_data.iter().enumerate() { + // FIXME: https://github.com/rust-lang/libs-team/issues/607 + #[expect(unsafe_code)] + let compressed_data = + unsafe { Pin::new_unchecked(compressed_data_unsafe.get_unchecked_mut(i)) }; + self.compress(input_data, compressed_data)?; + } + + Ok(()) + } + + fn decompress_many( + &mut self, + compressed_data: &[libpressio::PressioData], + decompressed_data: Pin<&mut [libpressio::PressioPinnedData]>, + ) -> Result<(), libpressio::PressioError> { + if compressed_data.len() != decompressed_data.len() { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("mismatched number of decompress_many inputs and outputs"), + }); + } + + // FIXME: https://github.com/rust-lang/libs-team/issues/607 + #[expect(unsafe_code)] + let decompressed_data_unsafe = unsafe { decompressed_data.get_unchecked_mut() }; + + for (i, compressed_data) in compressed_data.iter().enumerate() { + // FIXME: https://github.com/rust-lang/libs-team/issues/607 + #[expect(unsafe_code)] + let decompressed_data = + unsafe { Pin::new_unchecked(decompressed_data_unsafe.get_unchecked_mut(i)) }; + self.decompress(compressed_data, decompressed_data)?; + } + + Ok(()) + } + fn get_metrics_results(&self) -> libpressio::PressioOptions { libpressio::PressioOptions::new().expect("get_metrics_results should not fail") } } +#[derive(Clone)] +struct NumcodecsPressioMetric { + codec: Option, + compression_result: Option, + decompression_result: Option, +} + +#[expect(clippy::expect_used)] +impl libpressio::PressioRsMetric for NumcodecsPressioMetric { + fn get_configuration(&self) -> libpressio::PressioOptions { + (|| -> Result { + let mut options = libpressio::PressioOptions::new()?; + options.set( + "pressio:thread_safe", + libpressio::PressioOption::thread_safety(Some( + libpressio::PressioThreadSafety::Multiple, + )), + )?; + options.set( + "pressio:stability", + libpressio::PressioOption::string(Some(String::from("experimental"))), + )?; + Ok(options) + })() + .expect("get_configuration should not fail") + } + + fn get_documentation(&self) -> libpressio::PressioOptions { + (|| -> Result { + let mut options = libpressio::PressioOptions::new()?; + options.set( + "pressio:description", + libpressio::PressioOption::string(Some(String::from( + "A metric computed from a numcodecs codec exposed through numcodecs-rs", + ))), + )?; + options.set( + "numcodecs.rs-metric:id", + libpressio::PressioOption::string(Some(String::from("numcodecs metric codec id"))), + )?; + options.set( + "numcodecs.rs-metric:compression", + libpressio::PressioOption::string(Some(String::from("compression metric result"))), + )?; + options.set( + "numcodecs.rs-metric:decompression", + libpressio::PressioOption::string(Some(String::from( + "decompression metric result", + ))), + )?; + Ok(options) + })() + .expect("get_documentation should not fail") + } + + fn get_options(&self) -> libpressio::PressioOptions { + (|| -> Result { + let options = if let Some(codec) = &self.codec { + let mut config_bytes = Vec::new(); + match codec.get_config(&mut serde_json::Serializer::new(&mut config_bytes)) { + Ok(()) => (), + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + } + let Ok(config) = String::from_utf8(config_bytes) else { + return Err(libpressio::PressioError { + error_code: 2, + message: String::from("invalid UTF-8 in JSON config"), + }); + }; + + let options: BTreeMap = + match BTreeMap::deserialize(&mut serde_json::Deserializer::from_str(&config)) { + Ok(options) => options, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + let options = options + .into_iter() + .map(|(key, value)| (format!("numcodecs.rs-metric:{key}"), value)) + .collect(); + + let documentation = libpressio::PressioOptions::new()?; + convert_to_pressio_options(&options, Option::None, &documentation)? + } else { + let mut options = libpressio::PressioOptions::new()?; + options.set( + "numcodecs.rs-metric:id", + libpressio::PressioOption::string(Option::None), + )?; + options + }; + Ok(options) + })() + .expect("get_options should not fail") + } + + fn set_options( + &mut self, + options: &libpressio::PressioOptions, + ) -> Result<(), libpressio::PressioError> { + self.compression_result = Option::None; + self.decompression_result = Option::None; + + let options = convert_from_pressio_options(options.iter()).map_err(|err| { + libpressio::PressioError { + error_code: 1, + message: err, + } + })?; + + #[expect(clippy::option_if_let_else)] + let options = options + .into_iter() + .map(|(key, value)| match key.strip_prefix("numcodecs.rs-metric:") { + Some(key) => Ok((String::from(key), value)), + Option::None => Err(libpressio::PressioError { + error_code: 1, + message: format!( + "numcodecs.rs-metric option `{key}` does not start with `numcodecs.rs-metric:`" + ), + }), + }) + .collect::, libpressio::PressioError>>()?; + + let Some(id) = options.get("id") else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("missing numcodecs.rs-metric:id"), + }); + }; + + let PressioOption::String(_id) = id else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("numcodecs.rs-metric:id must be a string"), + }); + }; + + let config = match serde_json::to_string(&options) { + Ok(config) => config, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + + let codec = match numcodecs_registry::GlobalRegistry + .get_codec(&mut serde_json::Deserializer::from_str(&config)) + { + Ok(codec) => codec, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + + self.codec = Some(codec); + + Ok(()) + } + + fn end_compress( + &mut self, + input_data: &libpressio::PressioData, + _compressed_data: &libpressio::PressioData, + result: Result<(), std::ffi::c_int>, + ) -> Result<(), libpressio::PressioError> { + let Some(codec) = &self.codec else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("uninitialized numcodecs codec"), + }); + }; + + self.compression_result = Option::None; + + if result.is_err() { + return Ok(()); + } + + let input_shape = input_data.shape(); + + let encoded = match input_data.dtype() { + Option::None => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unsupported input data type"), + }); + } + Some(libpressio::PressioDtype::Bool) => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unsupported input bool array"), + }); + } + Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::U8(input_data)) + }), + Some(libpressio::PressioDtype::U16) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::U16(input_data)) + }), + Some(libpressio::PressioDtype::U32) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::U32(input_data)) + }), + Some(libpressio::PressioDtype::U64) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::U64(input_data)) + }), + Some(libpressio::PressioDtype::I8) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::I8(input_data)) + }), + Some(libpressio::PressioDtype::I16) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::I16(input_data)) + }), + Some(libpressio::PressioDtype::I32) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::I32(input_data)) + }), + Some(libpressio::PressioDtype::I64) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::I64(input_data)) + }), + Some(libpressio::PressioDtype::F32) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::F32(input_data)) + }), + Some(libpressio::PressioDtype::F64) => input_data + .with_shared(Dim(input_shape), |input_data| { + codec.encode(AnyCowArray::F64(input_data)) + }), + }; + + let Some(encoded) = encoded else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unexpected encoded data type or shape mismatch"), + }); + }; + + let encoded = match encoded { + Ok(encoded) => encoded, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + + if let AnyArray::F64(encoded) = encoded + && let Ok(encoded) = encoded.into_dimensionality::() + { + self.compression_result = Some(encoded.into_scalar()); + return Ok(()); + } + + Err(libpressio::PressioError { + error_code: 1, + message: String::from("numcodecs.rs metric codec did not compress to a float64 scalar"), + }) + } + + fn end_decompress( + &mut self, + compressed_data: &libpressio::PressioData, + _decompressed_data: &libpressio::PressioData, + result: Result<(), std::ffi::c_int>, + ) -> Result<(), libpressio::PressioError> { + let Some(codec) = &self.codec else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("uninitialized numcodecs codec"), + }); + }; + + self.decompression_result = Option::None; + + if result.is_err() { + return Ok(()); + } + + let compressed_shape = compressed_data.shape(); + + let decoded = match compressed_data.dtype() { + Option::None => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unsupported compressed data type"), + }); + } + Some(libpressio::PressioDtype::Bool) => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unsupported compressed bool array"), + }); + } + Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::U8(compressed_data)) + }), + Some(libpressio::PressioDtype::U16) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::U16(compressed_data)) + }), + Some(libpressio::PressioDtype::U32) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::U32(compressed_data)) + }), + Some(libpressio::PressioDtype::U64) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::U64(compressed_data)) + }), + Some(libpressio::PressioDtype::I8) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::I8(compressed_data)) + }), + Some(libpressio::PressioDtype::I16) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::I16(compressed_data)) + }), + Some(libpressio::PressioDtype::I32) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::I32(compressed_data)) + }), + Some(libpressio::PressioDtype::I64) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::I64(compressed_data)) + }), + Some(libpressio::PressioDtype::F32) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::F32(compressed_data)) + }), + Some(libpressio::PressioDtype::F64) => compressed_data + .with_shared(Dim(compressed_shape), |compressed_data| { + codec.decode(AnyCowArray::F64(compressed_data)) + }), + }; + + let Some(decoded) = decoded else { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("unexpected decoded data type or shape mismatch"), + }); + }; + + let decoded = match decoded { + Ok(decoded) => decoded, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + + if let AnyArray::F64(decoded) = decoded + && let Ok(decoded) = decoded.into_dimensionality::() + { + self.decompression_result = Some(decoded.into_scalar()); + return Ok(()); + } + + Err(libpressio::PressioError { + error_code: 1, + message: String::from( + "numcodecs.rs metric codec did not decompress to a float64 scalar", + ), + }) + } + + fn end_compress_many( + &mut self, + input_data: &[libpressio::PressioData], + compressed_data: &[libpressio::PressioData], + result: Result<(), std::ffi::c_int>, + ) -> Result<(), libpressio::PressioError> { + if let ([input_data], [compressed_data]) = (input_data, compressed_data) { + return self.end_compress(input_data, compressed_data, result); + } + + Err(libpressio::PressioError { + error_code: 1, + message: format!( + "end_compress_many with {} inputs and {} outputs is unsupported", + input_data.len(), + compressed_data.len() + ), + }) + } + + fn end_decompress_many( + &mut self, + compressed_data: &[libpressio::PressioData], + decompressed_data: &[libpressio::PressioData], + result: Result<(), std::ffi::c_int>, + ) -> Result<(), libpressio::PressioError> { + if let ([compressed_data], [decompressed_data]) = (compressed_data, decompressed_data) { + return self.end_decompress(compressed_data, decompressed_data, result); + } + + Err(libpressio::PressioError { + error_code: 1, + message: format!( + "end_decompress_many with {} inputs and {} outputs is unsupported", + compressed_data.len(), + decompressed_data.len() + ), + }) + } + + fn get_metrics_results(&self) -> libpressio::PressioOptions { + (|| -> Result { + let mut options = libpressio::PressioOptions::new()?; + options.set( + "numcodecs.rs-metric:compression", + libpressio::PressioOption::float64(self.compression_result), + )?; + options.set( + "numcodecs.rs-metric:decompression", + libpressio::PressioOption::float64(self.decompression_result), + )?; + Ok(options) + })() + .expect("get_metrics_results should not fail") + } +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { From 2a997252380492951a0f71c907e7e46518f1e118 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Mon, 6 Jul 2026 14:46:26 +0300 Subject: [PATCH 70/72] fixes from experimentation --- codecs/pressio/src/lib.rs | 268 +++++++++++++++++++++++++++----------- 1 file changed, 190 insertions(+), 78 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 448a5b716..3ebacbde9 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -28,7 +28,8 @@ use fragile::Fragile; use ndarray::{Array, ArrayView, ArrayViewMut, CowArray, Dim, Ix0, IxDyn}; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray, - Codec, DynCodec, ErasedDynCodec, StaticCodec, StaticCodecConfig, StaticCodecVersion, + Codec, DynCodec, DynCodecType, ErasedDynCodec, StaticCodec, StaticCodecConfig, + StaticCodecVersion, }; use numcodecs_registry::Registry; use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema}; @@ -1133,32 +1134,82 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { } })?; - #[expect(clippy::option_if_let_else)] - let options = options + let mut options = options .into_iter() - .map(|(key, value)| match key.strip_prefix("numcodecs.rs:") { - Some(key) => Ok((String::from(key), value)), - Option::None => Err(libpressio::PressioError { - error_code: 1, - message: format!( - "numcodecs.rs option `{key}` does not start with `numcodecs.rs:`" - ), - }), + .filter_map(|(key, value)| { + // TODO: should the keys also be prefixed by the codec id? + key.strip_prefix("numcodecs.rs:") + .map(|key| (String::from(key), value)) }) - .collect::, libpressio::PressioError>>()?; + .collect::>(); + + let new_codec_id = match options.get("id") { + Some(PressioOption::String(codec_id)) => Some(codec_id.as_str()), + Some(_) => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("numcodecs.rs:id must be a string"), + }); + } + Option::None => Option::None, + }; + + let old_config = if let Some(codec) = &self.codec { + let codec_ty = codec.ty(); + let codec_id = codec_ty.codec_id(); + + if match new_codec_id { + Some(new_codec_id) if new_codec_id == codec_id => true, + Some(_) => false, + Option::None => true, + } { + let mut config_bytes = Vec::new(); + match codec.get_config(&mut serde_json::Serializer::new(&mut config_bytes)) { + Ok(()) => (), + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + } + let Ok(config) = String::from_utf8(config_bytes) else { + return Err(libpressio::PressioError { + error_code: 2, + message: String::from("invalid UTF-8 in JSON config"), + }); + }; - let Some(id) = options.get("id") else { + let options: BTreeMap = + match BTreeMap::deserialize(&mut serde_json::Deserializer::from_str(&config)) { + Ok(options) => options, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + Some(options) + } else { + Option::None + } + } else if new_codec_id.is_some() { + Option::None + } else { return Err(libpressio::PressioError { error_code: 1, message: String::from("missing numcodecs.rs:id"), }); }; - let PressioOption::String(_id) = id else { - return Err(libpressio::PressioError { - error_code: 1, - message: String::from("numcodecs.rs:id must be a string"), - }); + // simple top-level merge of options + let options = match old_config { + Some(mut old_config) => { + old_config.append(&mut options); + old_config + } + Option::None => options, }; let config = match serde_json::to_string(&options) { @@ -1491,6 +1542,21 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { "pressio:stability", libpressio::PressioOption::string(Some(String::from("experimental"))), )?; + options.set( + "predictors:requires_decompress", + libpressio::PressioOption::vec_string(Some(vec![String::from( + "numcodecs.rs-metric:decompression", + )])), + )?; + options.set( + "predictors:invalidate", + libpressio::PressioOption::vec_string(Some(vec![ + String::from("predictors:runtime"), + String::from("predictors:error_dependent"), + String::from("predictors:error_agnostic"), + String::from("predictors:nondeterministc"), + ])), + )?; Ok(options) })() .expect("get_configuration should not fail") @@ -1588,32 +1654,81 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { } })?; - #[expect(clippy::option_if_let_else)] - let options = options + let mut options = options .into_iter() - .map(|(key, value)| match key.strip_prefix("numcodecs.rs-metric:") { - Some(key) => Ok((String::from(key), value)), - Option::None => Err(libpressio::PressioError { - error_code: 1, - message: format!( - "numcodecs.rs-metric option `{key}` does not start with `numcodecs.rs-metric:`" - ), - }), + .filter_map(|(key, value)| { + key.strip_prefix("numcodecs.rs-metric:") + .map(|key| (String::from(key), value)) }) - .collect::, libpressio::PressioError>>()?; + .collect::>(); - let Some(id) = options.get("id") else { + let new_codec_id = match options.get("id") { + Some(PressioOption::String(codec_id)) => Some(codec_id.as_str()), + Some(_) => { + return Err(libpressio::PressioError { + error_code: 1, + message: String::from("numcodecs.rs-metric:id must be a string"), + }); + } + Option::None => Option::None, + }; + + let old_config = if let Some(codec) = &self.codec { + let codec_ty = codec.ty(); + let codec_id = codec_ty.codec_id(); + + if match new_codec_id { + Some(new_codec_id) if new_codec_id == codec_id => true, + Some(_) => false, + Option::None => true, + } { + let mut config_bytes = Vec::new(); + match codec.get_config(&mut serde_json::Serializer::new(&mut config_bytes)) { + Ok(()) => (), + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + } + let Ok(config) = String::from_utf8(config_bytes) else { + return Err(libpressio::PressioError { + error_code: 2, + message: String::from("invalid UTF-8 in JSON config"), + }); + }; + + let options: BTreeMap = + match BTreeMap::deserialize(&mut serde_json::Deserializer::from_str(&config)) { + Ok(options) => options, + Err(err) => { + return Err(libpressio::PressioError { + error_code: 1, + message: format!("{err}"), + }); + } + }; + Some(options) + } else { + Option::None + } + } else if new_codec_id.is_some() { + Option::None + } else { return Err(libpressio::PressioError { error_code: 1, message: String::from("missing numcodecs.rs-metric:id"), }); }; - let PressioOption::String(_id) = id else { - return Err(libpressio::PressioError { - error_code: 1, - message: String::from("numcodecs.rs-metric:id must be a string"), - }); + // simple top-level merge of options + let options = match old_config { + Some(mut old_config) => { + old_config.append(&mut options); + old_config + } + Option::None => options, }; let config = match serde_json::to_string(&options) { @@ -1652,7 +1767,7 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { let Some(codec) = &self.codec else { return Err(libpressio::PressioError { error_code: 1, - message: String::from("uninitialized numcodecs codec"), + message: String::from("uninitialized numcodecs metric codec"), }); }; @@ -1751,14 +1866,14 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { fn end_decompress( &mut self, - compressed_data: &libpressio::PressioData, - _decompressed_data: &libpressio::PressioData, + _compressed_data: &libpressio::PressioData, + decompressed_data: &libpressio::PressioData, result: Result<(), std::ffi::c_int>, ) -> Result<(), libpressio::PressioError> { let Some(codec) = &self.codec else { return Err(libpressio::PressioError { error_code: 1, - message: String::from("uninitialized numcodecs codec"), + message: String::from("uninitialized numcodecs metric codec"), }); }; @@ -1768,60 +1883,61 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { return Ok(()); } - let compressed_shape = compressed_data.shape(); + let decompressed_shape = decompressed_data.shape(); - let decoded = match compressed_data.dtype() { + let decoded = match decompressed_data.dtype() { Option::None => { return Err(libpressio::PressioError { error_code: 1, - message: String::from("unsupported compressed data type"), + message: String::from("unsupported decompressed data type"), }); } Some(libpressio::PressioDtype::Bool) => { return Err(libpressio::PressioError { error_code: 1, - message: String::from("unsupported compressed bool array"), + message: String::from("unsupported decompressed bool array"), }); } - Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::U8(compressed_data)) - }), - Some(libpressio::PressioDtype::U16) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::U16(compressed_data)) + Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => { + decompressed_data.with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::U8(decompressed_data)) + }) + } + Some(libpressio::PressioDtype::U16) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::U16(decompressed_data)) }), - Some(libpressio::PressioDtype::U32) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::U32(compressed_data)) + Some(libpressio::PressioDtype::U32) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::U32(decompressed_data)) }), - Some(libpressio::PressioDtype::U64) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::U64(compressed_data)) + Some(libpressio::PressioDtype::U64) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::U64(decompressed_data)) }), - Some(libpressio::PressioDtype::I8) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::I8(compressed_data)) + Some(libpressio::PressioDtype::I8) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::I8(decompressed_data)) }), - Some(libpressio::PressioDtype::I16) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::I16(compressed_data)) + Some(libpressio::PressioDtype::I16) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::I16(decompressed_data)) }), - Some(libpressio::PressioDtype::I32) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::I32(compressed_data)) + Some(libpressio::PressioDtype::I32) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::I32(decompressed_data)) }), - Some(libpressio::PressioDtype::I64) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::I64(compressed_data)) + Some(libpressio::PressioDtype::I64) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::I64(decompressed_data)) }), - Some(libpressio::PressioDtype::F32) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::F32(compressed_data)) + Some(libpressio::PressioDtype::F32) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::F32(decompressed_data)) }), - Some(libpressio::PressioDtype::F64) => compressed_data - .with_shared(Dim(compressed_shape), |compressed_data| { - codec.decode(AnyCowArray::F64(compressed_data)) + Some(libpressio::PressioDtype::F64) => decompressed_data + .with_shared(Dim(decompressed_shape), |decompressed_data| { + codec.decode(AnyCowArray::F64(decompressed_data)) }), }; @@ -2061,9 +2177,7 @@ mod tests { "compressor_id": "pressio", "compressor_config": { "pressio:metric": "error_stat", - "opt:compressor": "linear_quantizer", "linear_quantizer:compressor": "bzip2", - "opt:search": "fraz", "opt:output": ["error_stat:psnr"], "opt:inputs": ["linear_quantizer:step"], "opt:lower_bound": 1e-8, @@ -2108,9 +2222,7 @@ mod tests { "compressor_id": "pressio", "compressor_config": { "pressio:metric": "error_stat", - "opt:compressor": "linear_quantizer", "linear_quantizer:compressor": "bzip2", - "opt:search": "random_search", "opt:output": ["error_stat:psnr"], "opt:inputs": ["linear_quantizer:step"], "opt:lower_bound": 1e-8, From 9cd4ebbb407bb5dec82364310b3a5f1e507a15a9 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 8 Jul 2026 09:56:56 +0300 Subject: [PATCH 71/72] Clean up --- Cargo.toml | 2 +- codecs/pressio/src/lib.rs | 281 +++++++++++++++++++++----------------- 2 files changed, 153 insertions(+), 130 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2656a1e55..f8b1625e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ fragile = { version = "2.0", default-features = false } indexmap = { version = "2.10", default-features = false } itertools = { version = "0.14", default-features = false } lc-framework = { version = "0.1", default-features = false } -libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "1411224", default-features = false } +libpressio = { version = "0.1", git = "https://github.com/juntyr/libpressio-rs.git", rev = "44521c4", default-features = false } log = { version = "0.4.27", default-features = false } miniz_oxide = { version = "0.8.5", default-features = false } ndarray = { version = "0.16.1", default-features = false } # keep in sync with numpy diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index 3ebacbde9..a83b17681 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -167,7 +167,7 @@ impl Serialize for PressioCompressor { fn convert_from_pressio_options( options: impl Iterator, Option)>, -) -> Result, String> { +) -> Result, Cow<'static, str>> { let mut config = BTreeMap::new(); for (name, option) in options { @@ -214,7 +214,7 @@ fn convert_from_pressio_options( let Some(nested_name) = name.strip_prefix('/') else { // global option if config.insert(name.clone(), value).is_some() { - return Err(format!("duplicate global option: `{name}`")); + return Err(Cow::Owned(format!("duplicate global option: `{name}`"))); } continue; }; @@ -223,12 +223,16 @@ fn convert_from_pressio_options( let mut parts = nested_name.split(':').peekable(); let Some(first) = parts.next() else { - return Err(format!("invalid hierarchical config name `{name}`")); + return Err(Cow::Owned(format!( + "invalid hierarchical config name `{name}`" + ))); }; let paths = first.split('/'); if parts.peek().is_none() { - return Err(format!("invalid hierarchical config name `{name}`")); + return Err(Cow::Owned(format!( + "invalid hierarchical config name `{name}`" + ))); } let option_name = parts.map(String::from).collect::>().join(":"); @@ -239,14 +243,16 @@ fn convert_from_pressio_options( } let Some(PressioOption::Nested(entry)) = it.get_mut(path) else { - return Err(format!("duplicate option nesting: `{path}` in `{name}`")); + return Err(Cow::Owned(format!( + "duplicate option nesting: `{path}` in `{name}`" + ))); }; it = entry; } if it.insert(option_name.clone(), value).is_some() { - return Err(format!( + return Err(Cow::Owned(format!( "duplicate nested option: `{option_name}` in `{name}`" - )); + ))); } } @@ -260,7 +266,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { std::mem::drop(format.metric_results); let mut pressio = libpressio::Pressio::new().map_err(serde::de::Error::custom)?; - pressio + let _: Result<(), NumcodecsPressioCompressor> = pressio .register_compressor( "numcodecs.rs", NumcodecsPressioCompressor { @@ -273,7 +279,7 @@ impl<'de> Deserialize<'de> for PressioCompressor { 0, ) .map_err(serde::de::Error::custom)?; - pressio + let _: Result<(), NumcodecsPressioMetric> = pressio .register_metric( "numcodecs.rs-metric", NumcodecsPressioMetric { @@ -432,10 +438,10 @@ fn convert_to_pressio_options( .join(", "); return Err(libpressio::PressioError { - error_code: 1, - message: format!( + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!( "unknown compressor configuration option: `{name}`, use one of {supported_options}" - ), + )), }); }; @@ -455,7 +461,7 @@ fn convert_to_pressio_options( if let Some(docs) = docs { libpressio::PressioError { error_code: err.error_code, - message: format!("{err} ({docs})"), + message: Cow::Owned(format!("{err} ({docs})")), } } else { err @@ -1074,15 +1080,15 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(()) => (), Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } } let Ok(config) = String::from_utf8(config_bytes) else { return Err(libpressio::PressioError { - error_code: 2, - message: String::from("invalid UTF-8 in JSON config"), + error_code: libpressio::PressioErrorCode::TWO, + message: Cow::Borrowed("invalid UTF-8 in JSON config"), }); }; @@ -1091,8 +1097,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(options) => options, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1129,7 +1135,7 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { ) -> Result<(), libpressio::PressioError> { let options = convert_from_pressio_options(options.iter()).map_err(|err| { libpressio::PressioError { - error_code: 1, + error_code: libpressio::PressioErrorCode::ONE, message: err, } })?; @@ -1137,7 +1143,6 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { let mut options = options .into_iter() .filter_map(|(key, value)| { - // TODO: should the keys also be prefixed by the codec id? key.strip_prefix("numcodecs.rs:") .map(|key| (String::from(key), value)) }) @@ -1147,8 +1152,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Some(PressioOption::String(codec_id)) => Some(codec_id.as_str()), Some(_) => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("numcodecs.rs:id must be a string"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("numcodecs.rs:id must be a string"), }); } Option::None => Option::None, @@ -1168,15 +1173,15 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(()) => (), Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } } let Ok(config) = String::from_utf8(config_bytes) else { return Err(libpressio::PressioError { - error_code: 2, - message: String::from("invalid UTF-8 in JSON config"), + error_code: libpressio::PressioErrorCode::TWO, + message: Cow::Borrowed("invalid UTF-8 in JSON config"), }); }; @@ -1185,8 +1190,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(options) => options, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1198,8 +1203,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Option::None } else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("missing numcodecs.rs:id"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("missing numcodecs.rs:id"), }); }; @@ -1216,8 +1221,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(config) => config, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1228,8 +1233,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(codec) => codec, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1246,8 +1251,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { ) -> Result<(), libpressio::PressioError> { let Some(codec) = &self.codec else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("uninitialized numcodecs codec"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("uninitialized numcodecs codec"), }); }; @@ -1256,14 +1261,14 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { let encoded = match input_data.dtype() { Option::None => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unsupported input data type"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unsupported input data type"), }); } Some(libpressio::PressioDtype::Bool) => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unsupported input bool array"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unsupported input bool array"), }); } Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => input_data @@ -1310,8 +1315,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { let Some(encoded) = encoded else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unexpected encoded data type or shape mismatch"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unexpected encoded data type or shape mismatch"), }); }; @@ -1319,8 +1324,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(encoded) => encoded, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1338,8 +1343,11 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { AnyArray::F64(encoded) => libpressio::PressioData::new_copied(encoded), encoded => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("unsupported encoded data type {}", encoded.dtype()), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!( + "unsupported encoded data type {}", + encoded.dtype() + )), }); } }; @@ -1356,8 +1364,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { ) -> Result<(), libpressio::PressioError> { let Some(codec) = &self.codec else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("uninitialized numcodecs codec"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("uninitialized numcodecs codec"), }); }; @@ -1368,14 +1376,14 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { let decoded = match compressed_data.dtype() { Option::None => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unsupported compressed data type"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unsupported compressed data type"), }); } Some(libpressio::PressioDtype::Bool) => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unsupported compressed bool array"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unsupported compressed bool array"), }); } Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => compressed_data @@ -1422,8 +1430,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { let Some(decoded) = decoded else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unexpected decoded data type or shape mismatch"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unexpected decoded data type or shape mismatch"), }); }; @@ -1431,8 +1439,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { Ok(decoded) => decoded, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1450,8 +1458,11 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { AnyArray::F64(decoded) => libpressio::PressioData::new_copied(decoded), decoded => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("unsupported decoded data type {}", decoded.dtype()), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!( + "unsupported decoded data type {}", + decoded.dtype() + )), }); } }; @@ -1468,8 +1479,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { ) -> Result<(), libpressio::PressioError> { if input_data.len() != compressed_data.len() { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("mismatched number of compress_many inputs and outputs"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("mismatched number of compress_many inputs and outputs"), }); } @@ -1495,8 +1506,8 @@ impl libpressio::PressioRsCompressor for NumcodecsPressioCompressor { ) -> Result<(), libpressio::PressioError> { if compressed_data.len() != decompressed_data.len() { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("mismatched number of decompress_many inputs and outputs"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("mismatched number of decompress_many inputs and outputs"), }); } @@ -1598,15 +1609,15 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Ok(()) => (), Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } } let Ok(config) = String::from_utf8(config_bytes) else { return Err(libpressio::PressioError { - error_code: 2, - message: String::from("invalid UTF-8 in JSON config"), + error_code: libpressio::PressioErrorCode::TWO, + message: Cow::Borrowed("invalid UTF-8 in JSON config"), }); }; @@ -1615,8 +1626,8 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Ok(options) => options, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1649,7 +1660,7 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { let options = convert_from_pressio_options(options.iter()).map_err(|err| { libpressio::PressioError { - error_code: 1, + error_code: libpressio::PressioErrorCode::ONE, message: err, } })?; @@ -1666,8 +1677,8 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Some(PressioOption::String(codec_id)) => Some(codec_id.as_str()), Some(_) => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("numcodecs.rs-metric:id must be a string"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("numcodecs.rs-metric:id must be a string"), }); } Option::None => Option::None, @@ -1687,15 +1698,15 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Ok(()) => (), Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } } let Ok(config) = String::from_utf8(config_bytes) else { return Err(libpressio::PressioError { - error_code: 2, - message: String::from("invalid UTF-8 in JSON config"), + error_code: libpressio::PressioErrorCode::TWO, + message: Cow::Borrowed("invalid UTF-8 in JSON config"), }); }; @@ -1704,8 +1715,8 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Ok(options) => options, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1717,8 +1728,8 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Option::None } else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("missing numcodecs.rs-metric:id"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("missing numcodecs.rs-metric:id"), }); }; @@ -1735,8 +1746,8 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Ok(config) => config, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1747,8 +1758,8 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Ok(codec) => codec, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; @@ -1762,12 +1773,12 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { &mut self, input_data: &libpressio::PressioData, _compressed_data: &libpressio::PressioData, - result: Result<(), std::ffi::c_int>, + result: Result<(), libpressio::PressioErrorCode>, ) -> Result<(), libpressio::PressioError> { let Some(codec) = &self.codec else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("uninitialized numcodecs metric codec"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("uninitialized numcodecs metric codec"), }); }; @@ -1782,14 +1793,14 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { let encoded = match input_data.dtype() { Option::None => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unsupported input data type"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unsupported input data type"), }); } Some(libpressio::PressioDtype::Bool) => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unsupported input bool array"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unsupported input bool array"), }); } Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => input_data @@ -1836,8 +1847,8 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { let Some(encoded) = encoded else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unexpected encoded data type or shape mismatch"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unexpected encoded data type or shape mismatch"), }); }; @@ -1845,22 +1856,29 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Ok(encoded) => encoded, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; - if let AnyArray::F64(encoded) = encoded - && let Ok(encoded) = encoded.into_dimensionality::() - { - self.compression_result = Some(encoded.into_scalar()); - return Ok(()); + if let AnyArray::F64(encoded) = encoded { + if encoded.is_empty() { + self.compression_result = Option::None; + return Ok(()); + } + + if let Ok(encoded) = encoded.into_dimensionality::() { + self.compression_result = Some(encoded.into_scalar()); + return Ok(()); + } } Err(libpressio::PressioError { - error_code: 1, - message: String::from("numcodecs.rs metric codec did not compress to a float64 scalar"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed( + "numcodecs.rs metric codec did not compress to a float64 scalar result or empty non-result", + ), }) } @@ -1868,12 +1886,12 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { &mut self, _compressed_data: &libpressio::PressioData, decompressed_data: &libpressio::PressioData, - result: Result<(), std::ffi::c_int>, + result: Result<(), libpressio::PressioErrorCode>, ) -> Result<(), libpressio::PressioError> { let Some(codec) = &self.codec else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("uninitialized numcodecs metric codec"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("uninitialized numcodecs metric codec"), }); }; @@ -1888,14 +1906,14 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { let decoded = match decompressed_data.dtype() { Option::None => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unsupported decompressed data type"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unsupported decompressed data type"), }); } Some(libpressio::PressioDtype::Bool) => { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unsupported decompressed bool array"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unsupported decompressed bool array"), }); } Some(libpressio::PressioDtype::Byte | libpressio::PressioDtype::U8) => { @@ -1943,8 +1961,8 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { let Some(decoded) = decoded else { return Err(libpressio::PressioError { - error_code: 1, - message: String::from("unexpected decoded data type or shape mismatch"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed("unexpected decoded data type or shape mismatch"), }); }; @@ -1952,23 +1970,28 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { Ok(decoded) => decoded, Err(err) => { return Err(libpressio::PressioError { - error_code: 1, - message: format!("{err}"), + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!("{err}")), }); } }; - if let AnyArray::F64(decoded) = decoded - && let Ok(decoded) = decoded.into_dimensionality::() - { - self.decompression_result = Some(decoded.into_scalar()); - return Ok(()); + if let AnyArray::F64(decoded) = decoded { + if decoded.is_empty() { + self.decompression_result = Option::None; + return Ok(()); + } + + if let Ok(decoded) = decoded.into_dimensionality::() { + self.decompression_result = Some(decoded.into_scalar()); + return Ok(()); + } } Err(libpressio::PressioError { - error_code: 1, - message: String::from( - "numcodecs.rs metric codec did not decompress to a float64 scalar", + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Borrowed( + "numcodecs.rs metric codec did not decompress to a float64 scalar result or empty non-result", ), }) } @@ -1977,19 +2000,19 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { &mut self, input_data: &[libpressio::PressioData], compressed_data: &[libpressio::PressioData], - result: Result<(), std::ffi::c_int>, + result: Result<(), libpressio::PressioErrorCode>, ) -> Result<(), libpressio::PressioError> { if let ([input_data], [compressed_data]) = (input_data, compressed_data) { return self.end_compress(input_data, compressed_data, result); } Err(libpressio::PressioError { - error_code: 1, - message: format!( + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!( "end_compress_many with {} inputs and {} outputs is unsupported", input_data.len(), compressed_data.len() - ), + )), }) } @@ -1997,19 +2020,19 @@ impl libpressio::PressioRsMetric for NumcodecsPressioMetric { &mut self, compressed_data: &[libpressio::PressioData], decompressed_data: &[libpressio::PressioData], - result: Result<(), std::ffi::c_int>, + result: Result<(), libpressio::PressioErrorCode>, ) -> Result<(), libpressio::PressioError> { if let ([compressed_data], [decompressed_data]) = (compressed_data, decompressed_data) { return self.end_decompress(compressed_data, decompressed_data, result); } Err(libpressio::PressioError { - error_code: 1, - message: format!( + error_code: libpressio::PressioErrorCode::ONE, + message: Cow::Owned(format!( "end_decompress_many with {} inputs and {} outputs is unsupported", compressed_data.len(), decompressed_data.len() - ), + )), }) } From eef646ac9226b0a061b5b11b11462ec95e3ffdc5 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Wed, 8 Jul 2026 09:59:17 +0300 Subject: [PATCH 72/72] Bump numcodecs-pressio to v0.1.0-beta.2 --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index e5c6457b9..cbd5106c1 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "numcodecs-pressio" -version = "0.1.0-beta.1" +version = "0.1.0-beta.2" edition = { workspace = true } authors = { workspace = true } repository = { workspace = true }