From b141fb31030fdf2f31df3c4d65d148f9d9e96556 Mon Sep 17 00:00:00 2001 From: gbranaa4-hue Date: Sun, 6 Sep 2026 04:31:49 -0700 Subject: [PATCH 1/2] GH-39914: [Python] Fall back to schema field type for unparseable complex dtype metadata pd.read_parquet() (and Table.to_pandas() generally) raises TypeError when a column's pandas metadata records a complex ArrowDtype (list, struct, dictionary, ...) as its 'numpy_type', e.g. "list[pyarrow]". _get_extension_dtypes() passes this string straight to pandas.api.types.pandas_dtype(), which cannot parse nested Arrow type syntax and raises TypeError -- so any DataFrame that has such a column can never round-trip through parquet with the default (numpy) backend, and this has stayed broken since 2023. GH-39914 previously fixed the case where an explicit types_mapper / dtype_backend="pyarrow" is passed, by resolving those columns earlier so this code path is skipped entirely. The default backend, and any metadata-only path with no types_mapper, still hit the crash. Fix: when pandas_dtype(dtype) raises TypeError, fall back to building the ArrowDtype directly from the schema's actual field type instead of giving up. The schema always has the precise type available regardless of whether the metadata string can be parsed back. Verified against pandas 3.0.4 + pyarrow 24.0.0: list, struct, and dictionary ArrowDtype columns now round-trip through to_pandas() and parquet with no types_mapper/dtype_backend needed, a double write/read/write/read round-trip of a struct column still works, and plain numpy dtypes plus simple extension dtypes (Int64) are unaffected. Fixes pandas-dev/pandas#53011 Co-Authored-By: Claude Sonnet 5 --- python/pyarrow/pandas_compat.py | 16 ++++++++++++++- python/pyarrow/tests/test_pandas.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 36d336d51351..cd772a7b5e7b 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -902,7 +902,21 @@ def _get_extension_dtypes(table, columns_metadata, types_mapper, options, catego if name not in ext_columns and dtype not in _pandas_supported_numpy_types: # pandas_dtype is expensive, so avoid doing this for types # that are certainly numpy dtypes - pandas_dtype = _pandas_api.pandas_dtype(dtype) + try: + pandas_dtype = _pandas_api.pandas_dtype(dtype) + except TypeError: + # Complex/nested Arrow types (list, struct, dictionary, ...) + # serialize to a 'numpy_type' string (e.g. + # "list[pyarrow]") that pandas_dtype() cannot + # parse back. Fall back to building the ArrowDtype directly + # from the schema's actual field type instead of giving up + # on round-tripping the dtype entirely. + # See GH-39914 / pandas-dev/pandas#53011. + try: + field = table.schema.field(name) + except KeyError: + continue + pandas_dtype = _pandas_api.pd.ArrowDtype(field.type) if isinstance(pandas_dtype, _pandas_api.extension_dtype): if isinstance(pandas_dtype, _pandas_api.pd.StringDtype): # when the metadata indicate to use the string dtype, diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index dd20a0aa9777..8f9d198f4a81 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -4566,6 +4566,37 @@ def test_to_pandas_extension_dtypes_mapping_complex_type(): pd.testing.assert_frame_equal(df0, df1) +def test_to_pandas_extension_dtypes_mapping_complex_type_no_types_mapper(): + # GH-39914: without an explicit types_mapper, the pandas metadata + # embedded by from_pandas() stores a 'numpy_type' string like + # "list[pyarrow]" for complex/nested ArrowDtype columns. + # pandas_dtype() cannot parse that string back, so to_pandas() (and, by + # extension, pd.read_parquet() without dtype_backend="pyarrow") used to + # raise a TypeError instead of falling back to the schema's actual field + # type. See pandas-dev/pandas#53011. + list_type = pd.ArrowDtype(pa.list_(pa.string())) + df0 = pd.DataFrame({ + "a": pd.Series([["x"], ["x", "y"]], dtype=list_type), + }) + + table = pa.Table.from_pandas(df0) + df1 = table.to_pandas() + pd.testing.assert_frame_equal(df0, df1) + + struct_type = pd.ArrowDtype( + pa.struct([pa.field("bar", pa.bool_()), pa.field("baz", pa.float32())]) + ) + df2 = pd.DataFrame({ + "a": pd.Series( + [{"bar": True, "baz": 1.0}, {"bar": False, "baz": None}], + dtype=struct_type, + ), + }) + table2 = pa.Table.from_pandas(df2) + df3 = table2.to_pandas() + pd.testing.assert_frame_equal(df2, df3) + + def test_array_to_pandas(): for arr in [pd.period_range("2012-01-01", periods=3, freq="D").array, pd.interval_range(1, 4).array]: From 0eb1e18554970a9d36f0f029eb7bd6eba77e5d44 Mon Sep 17 00:00:00 2001 From: tritsystem Date: Tue, 20 Oct 2026 20:49:53 -0700 Subject: [PATCH 2/2] GH-39914: [Python] Skip unparseable complex dtype metadata instead of guessing ArrowDtype Address review from @jorisvandenbossche: a parse failure on the 'numpy_type' metadata string shouldn't be read as "the user wants an ArrowDtype" -- it can happen for reasons other than a complex/nested Arrow type. Replace the schema-field-type fallback with `continue`, leaving the column out of the metadata-driven extension-dtype handling entirely, same as the existing KeyError case right above it. Also broadened the except clause from `TypeError` to `(TypeError, ValueError)`. Traced why: np.dtype() is tried before the ArrowDtype parser, and for a 'numpy_type' string containing a comma (struct, dictionary, map -- anything but a single-field list) it misreads the comma as its own legacy multi-field format syntax and raises ValueError, never reaching the TypeError the list case gets. Verified directly against pandas_dtype() and np.dtype() on pandas 2.3.3: the previous TypeError-only fix left struct/dictionary/map still crashing. Updated the regression test to match: it now asserts the column lands as object (list/struct) or category (dictionary) dtype with no crash, rather than an exact ArrowDtype round-trip -- which this fix no longer attempts, per the review above. Added a dictionary-dtype case alongside the existing list/struct ones since it exercises the same ValueError path. Co-Authored-By: Claude Sonnet 5 --- python/pyarrow/pandas_compat.py | 19 +++++++++-------- python/pyarrow/tests/test_pandas.py | 33 +++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index cd772a7b5e7b..4c13b752e32c 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -904,19 +904,20 @@ def _get_extension_dtypes(table, columns_metadata, types_mapper, options, catego # that are certainly numpy dtypes try: pandas_dtype = _pandas_api.pandas_dtype(dtype) - except TypeError: + except (TypeError, ValueError): # Complex/nested Arrow types (list, struct, dictionary, ...) # serialize to a 'numpy_type' string (e.g. # "list[pyarrow]") that pandas_dtype() cannot - # parse back. Fall back to building the ArrowDtype directly - # from the schema's actual field type instead of giving up - # on round-tripping the dtype entirely. + # parse back; np.dtype() is tried first and, for strings + # with a comma (struct, dictionary, map), misreads it as + # its legacy multi-field format syntax and raises + # ValueError before the ArrowDtype parser is ever reached, + # rather than the TypeError a comma-free string like list + # gets. Either way, don't guess that the user wants an + # ArrowDtype for a dtype string we can't actually parse; + # leave this column out so it isn't treated specially. # See GH-39914 / pandas-dev/pandas#53011. - try: - field = table.schema.field(name) - except KeyError: - continue - pandas_dtype = _pandas_api.pd.ArrowDtype(field.type) + continue if isinstance(pandas_dtype, _pandas_api.extension_dtype): if isinstance(pandas_dtype, _pandas_api.pd.StringDtype): # when the metadata indicate to use the string dtype, diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 8f9d198f4a81..8352a1bb5615 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -4570,18 +4570,27 @@ def test_to_pandas_extension_dtypes_mapping_complex_type_no_types_mapper(): # GH-39914: without an explicit types_mapper, the pandas metadata # embedded by from_pandas() stores a 'numpy_type' string like # "list[pyarrow]" for complex/nested ArrowDtype columns. - # pandas_dtype() cannot parse that string back, so to_pandas() (and, by - # extension, pd.read_parquet() without dtype_backend="pyarrow") used to - # raise a TypeError instead of falling back to the schema's actual field - # type. See pandas-dev/pandas#53011. + # pandas_dtype() cannot parse that string back -- with TypeError for a + # comma-free string like the list case, or ValueError for a string + # with a comma (struct, dictionary, map), which np.dtype() misreads as + # its legacy multi-field format syntax before ever reaching the + # ArrowDtype parser. Either way, to_pandas() (and, by extension, + # pd.read_parquet() without dtype_backend="pyarrow") used to crash + # outright instead of leaving the column out of the metadata-driven + # extension-dtype handling. See pandas-dev/pandas#53011. + # + # The ArrowDtype is not expected to round-trip here: an unparseable + # 'numpy_type' string isn't necessarily an ArrowDtype the caller still + # wants, so the column just falls back to its normal (non-metadata) + # conversion. list_type = pd.ArrowDtype(pa.list_(pa.string())) df0 = pd.DataFrame({ "a": pd.Series([["x"], ["x", "y"]], dtype=list_type), }) - table = pa.Table.from_pandas(df0) df1 = table.to_pandas() - pd.testing.assert_frame_equal(df0, df1) + assert df1["a"].dtype == np.dtype("object") + assert [list(v) for v in df1["a"]] == [["x"], ["x", "y"]] struct_type = pd.ArrowDtype( pa.struct([pa.field("bar", pa.bool_()), pa.field("baz", pa.float32())]) @@ -4594,7 +4603,17 @@ def test_to_pandas_extension_dtypes_mapping_complex_type_no_types_mapper(): }) table2 = pa.Table.from_pandas(df2) df3 = table2.to_pandas() - pd.testing.assert_frame_equal(df2, df3) + assert df3["a"].dtype == np.dtype("object") + assert df3["a"].tolist() == [ + {"bar": True, "baz": 1.0}, {"bar": False, "baz": None} + ] + + dict_type = pd.ArrowDtype(pa.dictionary(pa.int32(), pa.string())) + df4 = pd.DataFrame({"a": pd.Series(["x", "y", "x"], dtype=dict_type)}) + table4 = pa.Table.from_pandas(df4) + df5 = table4.to_pandas() + assert isinstance(df5["a"].dtype, pd.CategoricalDtype) + assert df5["a"].tolist() == ["x", "y", "x"] def test_array_to_pandas():