diff --git a/minarrow-py/Cargo.lock b/minarrow-py/Cargo.lock index 1676d20..7739aa8 100644 --- a/minarrow-py/Cargo.lock +++ b/minarrow-py/Cargo.lock @@ -37,7 +37,7 @@ dependencies = [ [[package]] name = "minarrow-py" -version = "0.18.2" +version = "0.18.3" dependencies = [ "minarrow", "minarrow-pyo3", diff --git a/minarrow-py/Cargo.toml b/minarrow-py/Cargo.toml index 8140883..ebf55b2 100644 --- a/minarrow-py/Cargo.toml +++ b/minarrow-py/Cargo.toml @@ -22,8 +22,6 @@ name = "minarrow_py" crate-type = ["cdylib", "rlib"] [dependencies] -# `scalar_type` is required by element access. -# Note this crate's `large_string` feature controls that rather than Minarrow's default. minarrow = { version = "0.18", path = "..", default-features = false, features = [ "views", "chunked", @@ -32,17 +30,16 @@ minarrow = { version = "0.18", path = "..", default-features = false, features = "log", "scalar_type", ] } -minarrow-pyo3 = { version = "0.18", path = "../pyo3", optional = true, default-features = false } +minarrow-pyo3 = { version = "0.18", path = "../pyo3", default-features = false, features = ["scalar_type"] } pyo3 = { version = "0.29", features = ["abi3-py39"] } thiserror = "2" [features] default = ["datetime", "large_string", "matrix", "scalar_type", "value_type", "cube", "arrow_interop", "ndarray", "decimal"] extension-module = ["pyo3/extension-module"] -arrow_interop = ["dep:minarrow-pyo3"] +arrow_interop = [] simd = ["minarrow/simd"] -# N-dimensional tensor with DLPack capsule interchange. The capsule glue -# lives in minarrow-pyo3's ffi::dlpack, alongside the Arrow capsule glue. +# N-dimensional tensor with DLPack capsule interchange. ndarray = [ "scalar_type", "minarrow/ndarray", @@ -51,7 +48,6 @@ ndarray = [ "minarrow/select", "minarrow/chunked", "minarrow/xarray", - "dep:minarrow-pyo3", "minarrow-pyo3/ndarray", ] @@ -59,16 +55,16 @@ ndarray = [ # Links libpython - use the default non `extension-module` link mode and # it should not be combined with `extension-module`. embed = ["arrow_interop", "scalar_type", "value_type", "simd"] -decimal = ["minarrow/decimal", "minarrow-pyo3?/decimal"] -datetime = ["minarrow/datetime", "minarrow-pyo3?/datetime"] -extended_numeric_types = ["minarrow/extended_numeric_types", "minarrow-pyo3?/extended_numeric_types"] +decimal = ["minarrow/decimal", "minarrow-pyo3/decimal"] +datetime = ["minarrow/datetime", "minarrow-pyo3/datetime"] +extended_numeric_types = ["minarrow/extended_numeric_types", "minarrow-pyo3/extended_numeric_types"] large_string = ["minarrow/large_string"] scalar_type = ["minarrow/scalar_type"] value_type = ["minarrow/value_type"] # Dense column-major f64 buffer for the routines that hand memory to LAPACK. matrix = ["minarrow/matrix"] cube = ["minarrow/cube"] -default_categorical_8 = ["minarrow/default_categorical_8", "minarrow-pyo3?/default_categorical_8"] +default_categorical_8 = ["minarrow/default_categorical_8", "minarrow-pyo3/default_categorical_8"] # Mirrors the core crate, where `extended_categorical` implies `default_categorical_8`. # `minarrow-pyo3` needs `extended_numeric_types` too, as its dictionary-index conversion # gates the 8 and 16-bit key arms on both features. @@ -76,7 +72,7 @@ extended_categorical = [ "default_categorical_8", "extended_numeric_types", "minarrow/extended_categorical", - "minarrow-pyo3?/extended_categorical", + "minarrow-pyo3/extended_categorical", ] [[example]] diff --git a/minarrow-py/src/convert.rs b/minarrow-py/src/convert.rs index 6040f7f..1bb38eb 100644 --- a/minarrow-py/src/convert.rs +++ b/minarrow-py/src/convert.rs @@ -38,8 +38,9 @@ use crate::arrow_type::PyArrowType; use pyo3::exceptions::PyNotImplementedError; use pyo3::exceptions::{PyIndexError, PyTypeError, PyValueError}; use pyo3::ffi; +use minarrow_pyo3::PyScalar; use pyo3::prelude::*; -use pyo3::types::{PyBool, PyString}; +use pyo3::types::PyString; use pyo3::IntoPyObjectExt; /// Reads a Python sequence into a single 64-byte aligned `Vec64` buffer. @@ -527,24 +528,7 @@ pub fn categorical_from_codes( /// `int` subclass. The scalar is later converted to the target array's element /// type when it is pushed or set. pub fn py_to_scalar(value: &Bound<'_, PyAny>) -> PyResult { - if value.is_none() { - return Ok(Scalar::Null); - } - if value.is_instance_of::() { - return Ok(Scalar::Boolean(value.extract()?)); - } - if let Ok(number) = value.extract::() { - return Ok(Scalar::Int64(number)); - } - if let Ok(number) = value.extract::() { - return Ok(Scalar::Float64(number)); - } - if let Ok(text) = value.extract::() { - return Ok(Scalar::String32(text)); - } - Err(PyTypeError::new_err( - "value must be None, bool, int, float, or str", - )) + value.extract::().map(Scalar::from) } pub fn resolve_index(i: isize, len: usize) -> PyResult { diff --git a/pyo3/Cargo.toml b/pyo3/Cargo.toml index b7531a6..03b271f 100644 --- a/pyo3/Cargo.toml +++ b/pyo3/Cargo.toml @@ -16,6 +16,8 @@ keywords = [ ] categories = ["external-ffi-bindings", "api-bindings"] description = "PyO3 bindings for MinArrow - zero-copy Arrow interop with Python via PyArrow" +# Gives dependent build scripts the source root as `DEP_MINARROW_PYO3_ROOT` +links = "minarrow_pyo3" [workspace] @@ -41,6 +43,8 @@ default_categorical_8 = ["minarrow/default_categorical_8"] extended_categorical = ["minarrow/extended_categorical"] decimal = ["minarrow/decimal"] table_metadata = ["minarrow/table_metadata"] +# Python scalar extraction through `PyScalar`. +scalar_type = ["minarrow/scalar_type"] # N-dimensional tensor bridging with DLPack capsule interchange. ndarray = ["minarrow/ndarray", "minarrow/dlpack", "minarrow/views", "minarrow/select"] diff --git a/pyo3/build.rs b/pyo3/build.rs index 7d1fba7..9507614 100644 --- a/pyo3/build.rs +++ b/pyo3/build.rs @@ -14,4 +14,9 @@ fn main() { pyo3_build_config::add_extension_module_link_args(); + // The source root, published to dependent build scripts as `DEP_MINARROW_PYO3_ROOT`. + println!( + "cargo:root={}", + std::env::var("CARGO_MANIFEST_DIR").expect("cargo sets CARGO_MANIFEST_DIR") + ); } diff --git a/pyo3/src/lib.rs b/pyo3/src/lib.rs index 2180f21..01172fd 100644 --- a/pyo3/src/lib.rs +++ b/pyo3/src/lib.rs @@ -120,8 +120,10 @@ use crate::ffi::dlpack::PyNdArrayInner; pub use error::{PyMinarrowError, PyMinarrowResult}; pub use types::{ PyArray, PyArrayView, PyChunkedArray, PyChunkedArrayView, PyField, PyRecordBatch, - PyRecordBatchView, PyTable, PyTableView, + PyRecordBatchView, PyTable, PyTableView, PyTextArrayView, }; +#[cfg(feature = "scalar_type")] +pub use types::PyScalar; // Re-export minarrow types that users might need pub use minarrow::{Array, Field, FieldArray, MaskedArray, NumericArray, SuperArray, SuperTable, Table, TextArray}; diff --git a/pyo3/src/types.rs b/pyo3/src/types.rs index 5f67bb7..ad30b34 100644 --- a/pyo3/src/types.rs +++ b/pyo3/src/types.rs @@ -19,9 +19,14 @@ use minarrow::{ Array, ArrayV, Field, FieldArray, SuperArray, SuperArrayV, SuperTable, SuperTableV, Table, - TableV, + TableV, TextArrayV, }; +#[cfg(feature = "scalar_type")] +use minarrow::Scalar; +use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; +#[cfg(feature = "scalar_type")] +use pyo3::types::PyBool; use pyo3::Borrowed; use std::sync::Arc; @@ -109,6 +114,12 @@ impl From for Arc { } } +impl From for Array { + fn from(value: PyArray) -> Self { + value.0.array + } +} + impl AsRef for PyArray { fn as_ref(&self) -> &Array { &self.0.array @@ -135,6 +146,118 @@ impl<'py> IntoPyObject<'py> for PyArray { } } +// PyTextArrayView - Wrapper around MinArrow's TextArrayV + +/// Transparent wrapper around MinArrow's [`TextArrayV`]. +/// +/// Extracts from a Python list of `str`, or from any object exporting a string +/// or categorical array through the Arrow PyCapsule interface, such as a +/// PyArrow string array. +/// +/// # Example (Rust) +/// ```ignore +/// use minarrow_pyo3::PyTextArrayView; +/// use minarrow::TextArrayV; +/// +/// #[pyfunction] +/// fn add_labels(labels: PyTextArrayView) -> PyResult<()> { +/// let labels: TextArrayV = labels.into(); +/// // Process... +/// Ok(()) +/// } +/// ``` +#[repr(transparent)] +#[derive(Debug, Clone)] +pub struct PyTextArrayView(pub TextArrayV); + +impl From for TextArrayV { + fn from(value: PyTextArrayView) -> Self { + value.0 + } +} + +impl<'py> FromPyObject<'_, 'py> for PyTextArrayView { + type Error = PyErr; + + fn extract(ob: Borrowed<'_, 'py, PyAny>) -> PyResult { + if let Ok(strings) = ob.extract::>() { + return Ok(PyTextArrayView(TextArrayV::from(strings))); + } + match to_rust::array_to_rust(&ob)?.array { + Array::TextArray(text) => Ok(PyTextArrayView(TextArrayV::from(text))), + _ => Err(PyTypeError::new_err( + "expected a list of str or an Arrow string or categorical array", + )), + } + } +} + +// PyScalar - Wrapper around MinArrow's Scalar + +/// Transparent wrapper around MinArrow's [`Scalar`]. +/// +/// Extracts from a single Python value. `None` becomes `Scalar::Null`, and +/// `bool`, `int`, `float` and `str` become `Boolean`, `Int64`, `Float64` and +/// `String32`. `bool` is checked before `int` because a Python `bool` is an +/// `int` subclass. Any other value raises `TypeError`. +/// +/// # Example (Rust) +/// ```ignore +/// use minarrow_pyo3::PyScalar; +/// use minarrow::Scalar; +/// +/// #[pyfunction] +/// fn fill_value(value: PyScalar) -> PyResult<()> { +/// let value: Scalar = value.into(); +/// // Process... +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "scalar_type")] +#[repr(transparent)] +#[derive(Debug, Clone)] +pub struct PyScalar(pub Scalar); + +#[cfg(feature = "scalar_type")] +impl From for PyScalar { + fn from(scalar: Scalar) -> Self { + Self(scalar) + } +} + +#[cfg(feature = "scalar_type")] +impl From for Scalar { + fn from(value: PyScalar) -> Self { + value.0 + } +} + +#[cfg(feature = "scalar_type")] +impl<'py> FromPyObject<'_, 'py> for PyScalar { + type Error = PyErr; + + fn extract(ob: Borrowed<'_, 'py, PyAny>) -> PyResult { + if ob.is_none() { + return Ok(PyScalar(Scalar::Null)); + } + if ob.is_instance_of::() { + return Ok(PyScalar(Scalar::Boolean(ob.extract()?))); + } + if let Ok(number) = ob.extract::() { + return Ok(PyScalar(Scalar::Int64(number))); + } + if let Ok(number) = ob.extract::() { + return Ok(PyScalar(Scalar::Float64(number))); + } + if let Ok(text) = ob.extract::() { + return Ok(PyScalar(Scalar::String32(text))); + } + Err(PyTypeError::new_err( + "value must be None, bool, int, float, or str", + )) + } +} + // PyRecordBatch - Wrapper around MinArrow's Table /// Transparent wrapper around MinArrow's Table.