Lower pyarrow requirement from 18 to 16 (#328)

* Lower pyarrow requirement from 18 to 17

This required a shim function, which should be removed
in the future, when we finally drop old pyarrow versions

Fixes #326

* Lower pyarrow req to 16

* do what Melissa said

Author: hombit

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
     "numpy>=2",
     # We use internal pd._libs.missing and experimental ArrowExtensionArray
     "pandas>=2.2.3,<2.4",
-    "pyarrow>=18",
+    "pyarrow>=16",  # remove struct_field_names when upgraded to 18+
     "universal_pathlib>=0.2",
 ]
 

src/nested_pandas/series/ext_array.py

@@ -59,6 +59,7 @@
     chunk_lengths,
     is_pa_type_a_list,
     rechunk,
+    struct_field_names,
     transpose_struct_list_type,
 )
 
@@ -186,7 +187,7 @@ def convert_df_to_pa_scalar(df: pd.DataFrame, *, pa_type: pa.StructType | None)
     types = {}
     columns = df.columns
     if pa_type is not None:
-        names = pa_type.names
+        names = struct_field_names(pa_type)
         columns = names + list(set(columns) - set(names))
     for column in columns:
         series = df[column]
@@ -913,7 +914,7 @@ def list_offsets(self) -> pa.Array:
     @property
     def field_names(self) -> list[str]:
         """Names of the nested columns"""
-        return [field for field in self._pyarrow_dtype.names]
+        return struct_field_names(self._pyarrow_dtype)
 
     @property
     def list_lengths(self) -> np.ndarray:

src/nested_pandas/series/utils.py

@@ -11,6 +11,15 @@
     from nested_pandas.series.dtype import NestedDtype
 
 
+def struct_field_names(struct_type: pa.StructType) -> list[str]:
+    """Return field names for a pyarrow.StructType in a pyarrow<18-compatible way.
+
+    Note: Once we bump our pyarrow requirement to ">=18", this helper can be
+    replaced with direct usage of ``struct_type.names`` throughout the codebase.
+    """
+    return [f.name for f in struct_type]
+
+
 def is_pa_type_a_list(pa_type: pa.DataType) -> bool:
     """Check if the given pyarrow type is a list type.
 
@@ -149,7 +158,7 @@ def transpose_struct_list_array(array: pa.StructArray, validate: bool = True) ->
     struct_flat_array = pa.StructArray.from_arrays(
         # Select values within the offsets
         [field.values[field.offsets[0].as_py() : field.offsets[-1].as_py()] for field in array.flatten()],
-        names=array.type.names,
+        names=struct_field_names(array.type),
     )
     return pa.ListArray.from_arrays(
         offsets=offsets,
@@ -196,7 +205,7 @@ def transpose_list_struct_scalar(scalar: pa.ListScalar) -> pa.StructScalar:
     """
     struct_type = transpose_list_struct_type(scalar.type)
     struct_scalar = pa.scalar(
-        {field: scalar.values.field(field) for field in struct_type.names},
+        {field.name: scalar.values.field(field.name) for field in struct_type},
         type=struct_type,
     )
     return cast(pa.StructScalar, struct_scalar)
@@ -265,7 +274,7 @@ def transpose_list_struct_array(array: pa.ListArray) -> pa.StructArray:
 
     return pa.StructArray.from_arrays(
         arrays=fields,
-        names=array.type.value_type.names,
+        names=struct_field_names(array.type.value_type),
         mask=mask,
     )
 

tests/nested_pandas/series/test_series_utils.py

@@ -4,6 +4,7 @@
 from nested_pandas import NestedDtype
 from nested_pandas.series.utils import (
     nested_types_mapper,
+    struct_field_names,
     transpose_list_struct_array,
     transpose_list_struct_scalar,
     transpose_list_struct_type,
@@ -125,6 +126,21 @@ def test_transpose_list_struct_scalar():
     assert actual == desired
 
 
+def test_struct_field_names():
+    """Test struct_field_names and guard against requirement bumps."""
+
+    # Otherwise, validate the shim works as expected (for pyarrow<=17 requirement)
+    t = pa.struct(
+        [
+            pa.field("a", pa.list_(pa.int64())),
+            pa.field("b", pa.list_(pa.float64())),
+            pa.field("c", pa.list_(pa.string())),
+        ]
+    )
+    # Ensure we get names in the correct order
+    assert struct_field_names(t) == ["a", "b", "c"]
+
+
 @pytest.mark.parametrize(
     "pa_type,is_nested",
     [