From aef05e4a36ec49af94cf685c1e51b638a2b6a597 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sun, 4 Oct 2026 01:30:07 +0900 Subject: [PATCH] Read multi-line CSV values that cross a block in ArrowCursor The CSV branch of AthenaArrowResultSet._read_csv() did not set newlines_in_values, so pyarrow split blocks at any newline and failed with "CSV parser got out of sync with chunker" when a quoted value containing a newline crossed a block boundary. This happened with CSV results larger than the 128 MiB default block size or with a smaller block_size passed to execute(). Closes #1040 Co-Authored-By: Claude Opus 5.5 --- pyathena/arrow/result_set.py | 3 +++ tests/pyathena/arrow/test_cursor.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/pyathena/arrow/result_set.py b/pyathena/arrow/result_set.py index 1acd654b..8b05ef54 100644 --- a/pyathena/arrow/result_set.py +++ b/pyathena/arrow/result_set.py @@ -324,6 +324,9 @@ def _read_csv(self) -> Table: ignore_empty_lines=False, double_quote=True, escape_char=False, + # A quoted value can contain a newline, so the reader must not split + # blocks inside quotes. + newlines_in_values=True, ) else: return pa.Table.from_pydict({}) diff --git a/tests/pyathena/arrow/test_cursor.py b/tests/pyathena/arrow/test_cursor.py index fd3f04b4..40d95327 100644 --- a/tests/pyathena/arrow/test_cursor.py +++ b/tests/pyathena/arrow/test_cursor.py @@ -45,6 +45,18 @@ def test_binary_null_vs_empty(self, arrow_cursor): ] assert [row[3] for row in rows] == ["", "", "NULL"] + def test_multiline_values_across_blocks(self, arrow_cursor): + # The 50 two-line values of 301 bytes span several 1024-byte blocks. + arrow_cursor.execute( + """ + SELECT array_join(repeat('x', 150), '') || chr(10) || array_join(repeat('y', 150), '') + AS v + FROM UNNEST(sequence(1, 50)) AS t(i) + """, + block_size=1024, + ) + assert arrow_cursor.fetchall() == [("x" * 150 + "\n" + "y" * 150,)] * 50 + def test_binary_single_null(self, arrow_cursor): arrow_cursor.execute("SELECT CAST(NULL AS VARBINARY) AS value") assert arrow_cursor.fetchall() == [(None,)]