add limited support for pickled arrays

CHANGELOG.md

@@ -4,9 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [Unreleased]  (major bump needed: 0.9.0)
 
-No unreleased changes yet!
+<!-- No unreleased changes yet! -->
+
+### Changed
+- The `T: Serialize` bound has been removed from the `WriterBuilder` trait.  This may introduce places in downstream code where a `T: npyz::Serialize` bound must now be explicitly added.
+
+### Added
+- Limited support for pickled arrays.  See `examples/pickle.rs`. Thanks, @paroga!
+  - `TypeChar::Object` (numpy's `"|O"`)
+  - `{TypeStr, DType NpyHeader}::uses_pickled_array()` boolean function.
+  - `NpyFile` can open files with pickled arrays, but it cannot extract items.
+  - `WriteOptions::new_header_only`, `WriterBuilder::write_header_only` makes it possible to write headers wwith object dtypes.
 
 ## [0.8.4] - 2025-05-14
 

Cargo.toml

@@ -39,25 +39,27 @@ optional = true
 default-features = false
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-zip = { version = "0.6",optional = true}
+zip = { version = "0.6", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
-zip = { version = "0.6", default-features = false,features = ["deflate"],optional = true}
+zip = { version = "0.6", default-features = false, features = ["deflate"], optional = true }
+
 
 [dev-dependencies]
 # For examples ONLY.  We don't want to provide a public interface because ndarray undergoes
 # breaking semver bumps very frequently.
 #
 # Also, sprs has an ndarray dependency that might not be the most recent.
 ndarray = { version = "0.15" }
+serde-pickle = { version = "1.2.0" }
 sprs = { version = "0.11", default-features = false }
 bencher = { version = "0.1" }
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 zip = { version = "0.6", default-features = true }  # NOTICE: also in dependencies
 
 [target.'cfg(target_arch = "wasm32")'.dev-dependencies]
-zip = { version = "0.6", default-features = false,features = ["deflate"]}
+zip = { version = "0.6", default-features = false, features = ["deflate"] }
 wasm-bindgen = "0.2"
 wasm-bindgen-test = "0.3"
 

examples/pickle.rs

@@ -0,0 +1,72 @@
+use std::io;
+use std::fs::File;
+
+use serde_pickle;
+
+// test-data/pickle.npy is generated by this Python code:
+//
+// import numpy as np
+// a = np.array([
+//     [1, 3.5, "number"],
+//     [{}, {}, None],
+// ], dtype=object)
+// np.save('test-data/pickle.npy', a)
+
+fn main() -> io::Result<()> {
+    read_example()?;
+    write_example()?;
+    Ok(())
+}
+
+// Example of reading a pickled ndarray's header.
+fn read_example() -> Result<(), io::Error> {
+    let file = io::BufReader::new(File::open("test-data/pickle.npy")?);
+    let npy = npyz::NpyFile::new(file)?;
+
+    assert_eq!(npy.shape(), &[2, 3]);
+    println!("DType: {}", npy.dtype().descr());
+    println!("Shape: {:?}", npy.shape());
+    println!("Strides: {:?}", npy.strides());
+
+    // When an array involves objects, the **entire** array is written in a single pickle.
+    // npyz can't help us any further here.
+    let file = npy.into_inner();
+    let de_options = Default::default();
+    let value = serde_pickle::value_from_reader(file, de_options).map_err(pickle_err_to_io_err)?;
+
+    // The pickled value begins with a bunch of array metadata that's tough to make sense of,
+    // but at the end you will see the data, a flat list of the objects written by the python
+    // snippet at the top of this file.
+    println!("Pickled value: {:?}", value);
+
+    Ok(())
+}
+
+// Example of writing a pickled ndarray's header.
+fn write_example() -> Result<(), io::Error> {
+    use npyz::WriterBuilder;
+
+    let type_str = "|O".parse().unwrap();
+    let dtype = npyz::DType::new_scalar(type_str);
+
+    let mut file = io::BufWriter::new(File::create("examples/output/pickle-corrupt.npy")?);
+
+    // With npyz it is possible to write the *header*...
+    npyz::WriteOptions::new_header_only()
+        .dtype(dtype)
+        .shape(&[2, 3])
+        .writer(&mut file)
+        .write_header_only()?;
+
+    // And beyond that I have positively no idea what you should do to write the payload.
+    // Have fun!
+
+    Ok(())
+}
+
+fn pickle_err_to_io_err(error: serde_pickle::Error) -> io::Error {
+    match error {
+        serde_pickle::Error::Io(e) => e,
+        error => io::Error::new(io::ErrorKind::Other, error.to_string()),
+    }
+}

src/header.rs

@@ -112,7 +112,8 @@ impl DType {
 
     /// Get the number of bytes that each item of this type occupies.
     ///
-    /// If this value overflows the plaform's `usize` datatype, returns `None`.
+    /// If this size is not fixed (e.g. `|O`) or would overflow the platform's `usize` type, returns `None`.
+    /// You can differentiate between these two error cases by calling [`Self::has_variable_size`].
     pub fn num_bytes(&self) -> Option<usize> {
         match self {
             DType::Plain(ty) => ty.num_bytes(),
@@ -123,6 +124,20 @@ impl DType {
             },
         }
     }
+
+    /// `true` if an array using this `DType` is stored using `pickle`.
+    ///
+    /// This is true if and only if the type contains an [`'O'`](TypeChar::Object) dtype.
+    ///
+    /// Pickled arrays present unique challenges, and most of the `npyz` crate does not support them
+    /// beyond parsing or writing the file header.
+    pub fn uses_pickled_array(&self) -> bool {
+        match self {
+            DType::Plain(ty) => ty.uses_pickled_array(),
+            DType::Array(_, inner) => inner.uses_pickled_array(),
+            DType::Record(fields) => fields.iter().any(|field| field.dtype.uses_pickled_array()),
+        }
+    }
 }
 
 fn convert_list_to_record_fields(values: &[Value]) -> io::Result<Vec<Field>> {
@@ -374,7 +389,7 @@ mod tests {
     }
 
     #[test]
-    fn converts_simple_description_to_record_dtype() -> TestResult {
+    fn converts_simple_description_to_plain_dtype() -> TestResult {
         let dtype = ">f8";
         assert_eq!(
             DType::from_descr(&Value::String(dtype.to_string())).unwrap(),
@@ -384,7 +399,7 @@ mod tests {
     }
 
     #[test]
-    fn converts_non_endian_description_to_record_dtype() -> TestResult {
+    fn converts_non_endian_description_to_plain_dtype() -> TestResult {
         let dtype = "|u1";
         assert_eq!(
             DType::from_descr(&Value::String(dtype.to_string())).unwrap(),
@@ -393,6 +408,17 @@ mod tests {
         Ok(())
     }
 
+
+    #[test]
+    fn converts_object_description_to_plain_dtype() -> TestResult {
+        let dtype = "|O";
+        assert_eq!(
+            DType::from_descr(&Value::String(dtype.to_string())).unwrap(),
+            DType::Plain(dtype.parse()?),
+        );
+        Ok(())
+    }
+
     #[test]
     fn converts_record_description_to_record_dtype() -> TestResult {
         let descr = parse("[('a', '<u2'), ('b', '<f4')]");
@@ -410,6 +436,23 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn handles_variable_size_record() -> TestResult {
+        let descr = parse("[('a', '<u2'), ('b', '|O')]");
+        let expected_dtype = DType::Record(vec![
+            Field {
+                name: "a".to_string(),
+                dtype: DType::Plain("<u2".parse()?),
+            },
+            Field {
+                name: "b".to_string(),
+                dtype: DType::Plain("|O".parse()?),
+            }
+        ]);
+        assert_eq!(DType::from_descr(&descr).unwrap(), expected_dtype);
+        Ok(())
+    }
+
     #[test]
     fn funny_member_name_roundtrips() -> TestResult {
         let original_dtype = DType::Record(vec![
@@ -472,6 +515,23 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_size_of_packed_data() -> TestResult {
+        let descr = parse("[('a', '<u2'), ('b', '>i4')]");
+        let dtype = DType::from_descr(&descr).unwrap();
+        assert_eq!(dtype.num_bytes(), Some(6));
+        Ok(())
+    }
+
+    #[test]
+    fn test_variable_size() -> TestResult {
+        let descr = parse("[('a', '<u2'), ('b', '|O')]");
+        let dtype = DType::from_descr(&descr).unwrap();
+        assert_eq!(dtype.num_bytes(), None);
+        assert!(dtype.uses_pickled_array());
+        Ok(())
+    }
+
     #[test]
     fn errors_on_value_variants_that_cannot_be_converted() {
         let no_dtype = Value::Boolean(false);

src/lib.rs

@@ -26,7 +26,7 @@ No features are enabled by default.  Here is the list of existing features:
 * **`"derive"`** enables derives of traits for working with structured arrays.
 * **`"npz"`** enables adapters for working with NPZ files
   (including scipy sparse matrices),
-  adding a public dependency on the `zip` crate.
+  adding a public dependency on the [`zip`] crate.
   This requires opt-in because `zip` has a fair number of transitive dependencies.
   (note that some npz-related helper functions are available even without the feature)