Add PyTextArrayView and PyScalar argument wrappers to minarrow-pyo3

minarrow-py/Cargo.toml

@@ -22,8 +22,6 @@ name = "minarrow_py"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-# `scalar_type` is required by element access.
-# Note this crate's `large_string` feature controls that rather than Minarrow's default.
 minarrow = { version = "0.18", path = "..", default-features = false, features = [
     "views",
     "chunked",
@@ -32,17 +30,16 @@ minarrow = { version = "0.18", path = "..", default-features = false, features =
     "log",
     "scalar_type",
 ] }
-minarrow-pyo3 = { version = "0.18", path = "../pyo3", optional = true, default-features = false }
+minarrow-pyo3 = { version = "0.18", path = "../pyo3", default-features = false, features = ["scalar_type"] }
 pyo3 = { version = "0.29", features = ["abi3-py39"] }
 thiserror = "2"
 
 [features]
 default = ["datetime", "large_string", "matrix", "scalar_type", "value_type", "cube", "arrow_interop", "ndarray", "decimal"]
 extension-module = ["pyo3/extension-module"]
-arrow_interop = ["dep:minarrow-pyo3"]
+arrow_interop = []
 simd = ["minarrow/simd"]
-# N-dimensional tensor with DLPack capsule interchange. The capsule glue
-# lives in minarrow-pyo3's ffi::dlpack, alongside the Arrow capsule glue.
+# N-dimensional tensor with DLPack capsule interchange.
 ndarray = [
     "scalar_type",
     "minarrow/ndarray",
@@ -51,32 +48,31 @@ ndarray = [
     "minarrow/select",
     "minarrow/chunked",
     "minarrow/xarray",
-    "dep:minarrow-pyo3",
     "minarrow-pyo3/ndarray",
 ]
 
 # Embeds a CPython interpreter so Rust can drive Python through `PyLiquid`.
 # Links libpython - use the default non `extension-module` link mode and
 # it should not be combined with `extension-module`.
 embed = ["arrow_interop", "scalar_type", "value_type", "simd"]
-decimal = ["minarrow/decimal", "minarrow-pyo3?/decimal"]
-datetime = ["minarrow/datetime", "minarrow-pyo3?/datetime"]
-extended_numeric_types = ["minarrow/extended_numeric_types", "minarrow-pyo3?/extended_numeric_types"]
+decimal = ["minarrow/decimal", "minarrow-pyo3/decimal"]
+datetime = ["minarrow/datetime", "minarrow-pyo3/datetime"]
+extended_numeric_types = ["minarrow/extended_numeric_types", "minarrow-pyo3/extended_numeric_types"]
 large_string = ["minarrow/large_string"]
 scalar_type = ["minarrow/scalar_type"]
 value_type = ["minarrow/value_type"]
 # Dense column-major f64 buffer for the routines that hand memory to LAPACK.
 matrix = ["minarrow/matrix"]
 cube = ["minarrow/cube"]
-default_categorical_8 = ["minarrow/default_categorical_8", "minarrow-pyo3?/default_categorical_8"]
+default_categorical_8 = ["minarrow/default_categorical_8", "minarrow-pyo3/default_categorical_8"]
 # Mirrors the core crate, where `extended_categorical` implies `default_categorical_8`.
 # `minarrow-pyo3` needs `extended_numeric_types` too, as its dictionary-index conversion
 # gates the 8 and 16-bit key arms on both features.
 extended_categorical = [
     "default_categorical_8",
     "extended_numeric_types",
     "minarrow/extended_categorical",
-    "minarrow-pyo3?/extended_categorical",
+    "minarrow-pyo3/extended_categorical",
 ]
 
 [[example]]

minarrow-py/src/convert.rs

@@ -38,8 +38,9 @@ use crate::arrow_type::PyArrowType;
 use pyo3::exceptions::PyNotImplementedError;
 use pyo3::exceptions::{PyIndexError, PyTypeError, PyValueError};
 use pyo3::ffi;
+use minarrow_pyo3::PyScalar;
 use pyo3::prelude::*;
-use pyo3::types::{PyBool, PyString};
+use pyo3::types::PyString;
 use pyo3::IntoPyObjectExt;
 
 /// Reads a Python sequence into a single 64-byte aligned `Vec64` buffer.
@@ -527,24 +528,7 @@ pub fn categorical_from_codes(
 /// `int` subclass. The scalar is later converted to the target array's element
 /// type when it is pushed or set.
 pub fn py_to_scalar(value: &Bound<'_, PyAny>) -> PyResult<Scalar> {
-    if value.is_none() {
-        return Ok(Scalar::Null);
-    }
-    if value.is_instance_of::<PyBool>() {
-        return Ok(Scalar::Boolean(value.extract()?));
-    }
-    if let Ok(number) = value.extract::<i64>() {
-        return Ok(Scalar::Int64(number));
-    }
-    if let Ok(number) = value.extract::<f64>() {
-        return Ok(Scalar::Float64(number));
-    }
-    if let Ok(text) = value.extract::<String>() {
-        return Ok(Scalar::String32(text));
-    }
-    Err(PyTypeError::new_err(
-        "value must be None, bool, int, float, or str",
-    ))
+    value.extract::<PyScalar>().map(Scalar::from)
 }
 
 pub fn resolve_index(i: isize, len: usize) -> PyResult<usize> {

pyo3/Cargo.toml

@@ -16,6 +16,8 @@ keywords = [
 ]
 categories = ["external-ffi-bindings", "api-bindings"]
 description = "PyO3 bindings for MinArrow - zero-copy Arrow interop with Python via PyArrow"
+# Gives dependent build scripts the source root as `DEP_MINARROW_PYO3_ROOT`
+links = "minarrow_pyo3"
 
 [workspace]
 
@@ -41,6 +43,8 @@ default_categorical_8 = ["minarrow/default_categorical_8"]
 extended_categorical = ["minarrow/extended_categorical"]
 decimal = ["minarrow/decimal"]
 table_metadata = ["minarrow/table_metadata"]
+# Python scalar extraction through `PyScalar`.
+scalar_type = ["minarrow/scalar_type"]
 # N-dimensional tensor bridging with DLPack capsule interchange.
 ndarray = ["minarrow/ndarray", "minarrow/dlpack", "minarrow/views", "minarrow/select"]
 

pyo3/build.rs

@@ -14,4 +14,9 @@
 
 fn main() {
     pyo3_build_config::add_extension_module_link_args();
+    // The source root, published to dependent build scripts as `DEP_MINARROW_PYO3_ROOT`.
+    println!(
+        "cargo:root={}",
+        std::env::var("CARGO_MANIFEST_DIR").expect("cargo sets CARGO_MANIFEST_DIR")
+    );
 }

pyo3/src/lib.rs

@@ -120,8 +120,10 @@ use crate::ffi::dlpack::PyNdArrayInner;
 pub use error::{PyMinarrowError, PyMinarrowResult};
 pub use types::{
     PyArray, PyArrayView, PyChunkedArray, PyChunkedArrayView, PyField, PyRecordBatch,
-    PyRecordBatchView, PyTable, PyTableView,
+    PyRecordBatchView, PyTable, PyTableView, PyTextArrayView,
 };
+#[cfg(feature = "scalar_type")]
+pub use types::PyScalar;
 
 // Re-export minarrow types that users might need
 pub use minarrow::{Array, Field, FieldArray, MaskedArray, NumericArray, SuperArray, SuperTable, Table, TextArray};

pyo3/src/types.rs

@@ -19,9 +19,14 @@
 
 use minarrow::{
     Array, ArrayV, Field, FieldArray, SuperArray, SuperArrayV, SuperTable, SuperTableV, Table,
-    TableV,
+    TableV, TextArrayV,
 };
+#[cfg(feature = "scalar_type")]
+use minarrow::Scalar;
+use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
+#[cfg(feature = "scalar_type")]
+use pyo3::types::PyBool;
 use pyo3::Borrowed;
 use std::sync::Arc;
 
@@ -109,6 +114,12 @@ impl From<PyArray> for Arc<Array> {
     }
 }
 
+impl From<PyArray> for Array {
+    fn from(value: PyArray) -> Self {
+        value.0.array
+    }
+}
+
 impl AsRef<Array> for PyArray {
     fn as_ref(&self) -> &Array {
         &self.0.array
@@ -135,6 +146,118 @@ impl<'py> IntoPyObject<'py> for PyArray {
     }
 }
 
+// PyTextArrayView - Wrapper around MinArrow's TextArrayV
+
+/// Transparent wrapper around MinArrow's [`TextArrayV`].
+///
+/// Extracts from a Python list of `str`, or from any object exporting a string
+/// or categorical array through the Arrow PyCapsule interface, such as a
+/// PyArrow string array.
+///
+/// # Example (Rust)
+/// ```ignore
+/// use minarrow_pyo3::PyTextArrayView;
+/// use minarrow::TextArrayV;
+///
+/// #[pyfunction]
+/// fn add_labels(labels: PyTextArrayView) -> PyResult<()> {
+///     let labels: TextArrayV = labels.into();
+///     // Process...
+///     Ok(())
+/// }
+/// ```
+#[repr(transparent)]
+#[derive(Debug, Clone)]
+pub struct PyTextArrayView(pub TextArrayV);
+
+impl From<PyTextArrayView> for TextArrayV {
+    fn from(value: PyTextArrayView) -> Self {
+        value.0
+    }
+}
+
+impl<'py> FromPyObject<'_, 'py> for PyTextArrayView {
+    type Error = PyErr;
+
+    fn extract(ob: Borrowed<'_, 'py, PyAny>) -> PyResult<Self> {
+        if let Ok(strings) = ob.extract::<Vec<String>>() {
+            return Ok(PyTextArrayView(TextArrayV::from(strings)));
+        }
+        match to_rust::array_to_rust(&ob)?.array {
+            Array::TextArray(text) => Ok(PyTextArrayView(TextArrayV::from(text))),
+            _ => Err(PyTypeError::new_err(
+                "expected a list of str or an Arrow string or categorical array",
+            )),
+        }
+    }
+}
+
+// PyScalar - Wrapper around MinArrow's Scalar
+
+/// Transparent wrapper around MinArrow's [`Scalar`].
+///
+/// Extracts from a single Python value. `None` becomes `Scalar::Null`, and
+/// `bool`, `int`, `float` and `str` become `Boolean`, `Int64`, `Float64` and
+/// `String32`. `bool` is checked before `int` because a Python `bool` is an
+/// `int` subclass. Any other value raises `TypeError`.
+///
+/// # Example (Rust)
+/// ```ignore
+/// use minarrow_pyo3::PyScalar;
+/// use minarrow::Scalar;
+///
+/// #[pyfunction]
+/// fn fill_value(value: PyScalar) -> PyResult<()> {
+///     let value: Scalar = value.into();
+///     // Process...
+///     Ok(())
+/// }
+/// ```
+#[cfg(feature = "scalar_type")]
+#[repr(transparent)]
+#[derive(Debug, Clone)]
+pub struct PyScalar(pub Scalar);
+
+#[cfg(feature = "scalar_type")]
+impl From<Scalar> for PyScalar {
+    fn from(scalar: Scalar) -> Self {
+        Self(scalar)
+    }
+}
+
+#[cfg(feature = "scalar_type")]
+impl From<PyScalar> for Scalar {
+    fn from(value: PyScalar) -> Self {
+        value.0
+    }
+}
+
+#[cfg(feature = "scalar_type")]
+impl<'py> FromPyObject<'_, 'py> for PyScalar {
+    type Error = PyErr;
+
+    fn extract(ob: Borrowed<'_, 'py, PyAny>) -> PyResult<Self> {
+        if ob.is_none() {
+            return Ok(PyScalar(Scalar::Null));
+        }
+        if ob.is_instance_of::<PyBool>() {
+            return Ok(PyScalar(Scalar::Boolean(ob.extract()?)));
+        }
+        if let Ok(number) = ob.extract::<i64>() {
+            return Ok(PyScalar(Scalar::Int64(number)));
+        }
+        if let Ok(number) = ob.extract::<f64>() {
+            return Ok(PyScalar(Scalar::Float64(number)));
+        }
+        if let Ok(text) = ob.extract::<String>() {
+            return Ok(PyScalar(Scalar::String32(text)));
+        }
+        Err(PyTypeError::new_err(
+            "value must be None, bool, int, float, or str",
+        ))
+    }
+}
+
 // PyRecordBatch - Wrapper around MinArrow's Table
 
 /// Transparent wrapper around MinArrow's Table.