Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions pyathena/aio/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ def __init__(
)
self._result_set: AthenaAioResultSet | None = None
self._result_set_class = AthenaAioResultSet
self._result_set_kwargs: dict[str, Any] = {}

@property # type: ignore[explicit-override] # python/mypy#15900
@override
Expand Down Expand Up @@ -174,6 +175,7 @@ async def execute(
self.arraysize,
self._retry_config,
result_set_type_hints=options.result_set_type_hints,
**self._result_set_kwargs,
)
else:
raise OperationalError(query_execution.state_change_reason)
Expand Down Expand Up @@ -246,15 +248,15 @@ class AioDictCursor(AioCursor):
... print(row["name"])
"""

def __init__(self, **kwargs) -> None:
def __init__(self, dict_type: type[Any] | None = None, **kwargs) -> None:
"""Initialize an AioDictCursor.

Args:
**kwargs: Arguments forwarded to ``AioCursor.__init__``. If they include
``dict_type``, it is also assigned to the class attribute
``AthenaAioDictResultSet.dict_type``, the type used to build each row.
dict_type: The type used to build each row of this cursor's result
sets. If None, the result set class's ``dict_type`` is used.
**kwargs: Arguments forwarded to ``AioCursor.__init__``.
"""
super().__init__(**kwargs)
self._result_set_class = AthenaAioDictResultSet
if "dict_type" in kwargs:
AthenaAioDictResultSet.dict_type = kwargs["dict_type"]
if dict_type is not None:
self._result_set_kwargs = {"dict_type": dict_type}
4 changes: 4 additions & 0 deletions pyathena/aio/result_set.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ async def create(
arraysize: int,
retry_config: RetryConfig,
result_set_type_hints: dict[str | int, str] | None = None,
**kwargs: Any,
) -> AthenaAioResultSet:
"""Async factory method.

Expand All @@ -88,6 +89,8 @@ async def create(
retry_config: Retry configuration for API calls.
result_set_type_hints: Optional dictionary mapping column names to
Athena DDL type signatures for precise type conversion.
**kwargs: Additional arguments passed to the constructor of ``cls``,
such as ``dict_type`` for ``AthenaAioDictResultSet``.

Returns:
A fully initialized ``AthenaAioResultSet``.
Expand All @@ -99,6 +102,7 @@ async def create(
arraysize,
retry_config,
result_set_type_hints=result_set_type_hints,
**kwargs,
)
if result_set.state == AthenaQueryExecution.STATE_SUCCEEDED:
await result_set._async_pre_fetch()
Expand Down
14 changes: 8 additions & 6 deletions pyathena/async_cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ def __init__(
self._max_workers = max_workers
self._executor = ThreadPoolExecutor(max_workers=max_workers)
self._result_set_class = AthenaResultSet
self._result_set_kwargs: dict[str, Any] = {}

@property
def arraysize(self) -> int:
Expand Down Expand Up @@ -198,6 +199,7 @@ def _collect_result_set(
arraysize=self._arraysize,
retry_config=self._retry_config,
result_set_type_hints=result_set_type_hints,
**self._result_set_kwargs,

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Independent review (relayed), rewrite — CLEAN

Reviewer: Codex CLI 0.160.0, model gpt-6-astra, reasoning effort max, sandbox read-only, --ephemeral, session 01a0fff0-6833-72c2-a3c5-6e9e897e0f6e. Static review only (no builds, tests, mypy or network). Scope: full range merge-base 6be315ec4c87758f993778b0b2167696e848390a .. head ee872f7a91da39ff2e60969718c57366a723ee6e in a detached snapshot without .env; the prompt carried the diff and intended behavior, not the PR number, description, commits or earlier findings. Snapshot and PR worktree verified unchanged after the run.

Reviewer output (verbatim)

Surfaces covered:

  • Sync execute, thread-pool _collect_result_set (including description), asyncio create, repeated execution, and pagination.
  • _result_set_class and _result_set_kwargs, direct constructor arguments, connection cursor_kwargs precedence, and dict_type=None.
  • Cooperative initialization through AthenaAioDictResultSet’s MRO; row type assignment occurs before synchronous or asynchronous pre-fetch.
  • Cursor isolation, preservation of shared classes, and custom result-set subclasses.
  • Arrow, Pandas, Polars, S3FS, and Spark inheritance/construction paths; existing result-set callers.
  • Constructor compatibility, mypy annotations/configuration, and changed docstrings, inspected statically.
  • All six new tests and their fixtures. The three isolation tests would fail on the old code. The three custom-result-set tests would also pass on the old code, but assert observable subclass selection and fetched row types.

CLEAN

No actionable regression identified in the supplied range. Builds, tests, and mypy were not run; no network access was used. The checkout remains clean at the requested head.

Author note: the reviewer's observation that test_dict_type_custom_result_set would also pass on the original code is correct (the shared class attribute reached the subclass there); it guards the subclass path of the new argument passing, while test_dict_type is the regression test for #923.

)

@override
Expand Down Expand Up @@ -335,15 +337,15 @@ class AsyncDictCursor(AsyncCursor):
>>> print(f"User: {row['name']} ({row['email']})")
"""

def __init__(self, **kwargs) -> None:
def __init__(self, dict_type: type[Any] | None = None, **kwargs) -> None:
"""Initialize an AsyncDictCursor.

Args:
**kwargs: Arguments forwarded to ``AsyncCursor.__init__``. If they include
``dict_type``, it is also assigned to the class attribute
``AthenaDictResultSet.dict_type``, the type used to build each row.
dict_type: The type used to build each row of this cursor's result
sets. If None, the result set class's ``dict_type`` is used.
**kwargs: Arguments forwarded to ``AsyncCursor.__init__``.
"""
super().__init__(**kwargs)
self._result_set_class = AthenaDictResultSet
if "dict_type" in kwargs:
AthenaDictResultSet.dict_type = kwargs["dict_type"]
if dict_type is not None:
self._result_set_kwargs = {"dict_type": dict_type}
14 changes: 8 additions & 6 deletions pyathena/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ def __init__(
**kwargs,
)
self._result_set_class = AthenaResultSet
self._result_set_kwargs: dict[str, Any] = {}

@property # type: ignore[explicit-override] # python/mypy#15900
@override
Expand Down Expand Up @@ -192,6 +193,7 @@ def execute(
self.arraysize,
self._retry_config,
result_set_type_hints=options.result_set_type_hints,
**self._result_set_kwargs,
)
else:
raise OperationalError(query_execution.state_change_reason)
Expand All @@ -216,15 +218,15 @@ class DictCursor(Cursor):
... print(f"Product {row['id']}: {row['name']} - ${row['price']}")
"""

def __init__(self, **kwargs) -> None:
def __init__(self, dict_type: type[Any] | None = None, **kwargs) -> None:
"""Initialize a DictCursor.

Args:
**kwargs: Arguments forwarded to ``Cursor.__init__``. If they include
``dict_type``, it is also assigned to the class attribute
``AthenaDictResultSet.dict_type``, the type used to build each row.
dict_type: The type used to build each row of this cursor's result
sets. If None, the result set class's ``dict_type`` is used.
**kwargs: Arguments forwarded to ``Cursor.__init__``.
"""
super().__init__(**kwargs)
self._result_set_class = AthenaDictResultSet
if "dict_type" in kwargs:
AthenaDictResultSet.dict_type = kwargs["dict_type"]
if dict_type is not None:

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Self-review round one (implementation behavior), rewrite — FINDINGS, repaired

The maintainer replaced the per-cursor subclass / _result_set_class property design with plain argument passing (e27e6e20), so this round is a full pass, not a narrow follow-up.
Scope: base 6be315ec4c87758f993778b0b2167696e848390a .. head e27e6e206520264a6e5ada11cfed553c50e37134, full diff (5 source, 3 test modules).

Covered:

  • Construction paths: Cursor.execute passes 5 positional arguments plus keywords, AsyncCursor._collect_result_set keywords only, AioCursor.execute via create(), which now forwards **kwargs. AthenaDictResultSet.__init__ sets the instance dict_type before super().__init__(), so the sync pre-fetch in AthenaResultSet.__init__ already uses it; aio pre-fetches in create() after construction.
  • MRO: for AthenaAioDictResultSet the next __init__ is AthenaAioResultSet.__init__ (5 positional + result_set_type_hints), which receives no dict_type because it is consumed first.
  • Base cursors: _result_set_kwargs defaults to {}; non-dict cursors construct result sets exactly as before.
  • Shared state: no class attribute is written; the no-AWS reproduction keeps AthenaDictResultSet.dict_type / AthenaAioDictResultSet.dict_type at dict.
  • Tests: test_dict_type (failed on the original code) and test_dict_type_custom_result_set; 16 passed in the three dict cursor classes, 4 in tests/pyathena/aio/test_result_set.py.

Finding (introduced, repaired in ee872f7a): the dict cursors always passed dict_type (also None), so a user AthenaDictResultSet subclass with a fixed __init__ signature would raise TypeError even for cursors created without dict_type. They now add it to _result_set_kwargs only when it is not None; lint and the 16 tests re-run on ee872f7a.

self._result_set_kwargs = {"dict_type": dict_type}
13 changes: 13 additions & 0 deletions pyathena/result_set.py
Original file line number Diff line number Diff line change
Expand Up @@ -818,6 +818,19 @@ class AthenaDictResultSet(AthenaResultSet):
# You can override this to use OrderedDict or other dict-like types.
dict_type: type[Any] = dict

def __init__(self, *args: Any, dict_type: type[Any] | None = None, **kwargs: Any) -> None:

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Self-review round two (claims, callers, operations), rewrite — FINDINGS, PR description corrected

Scope: base 6be315ec4c87758f993778b0b2167696e848390a .. head ee872f7a91da39ff2e60969718c57366a723ee6e, full pass over the rewritten PR body, the changed docstrings (AthenaDictResultSet.__init__, AthenaAioResultSet.create, three dict cursor __init__s), commit messages and docs.

Claims checked:

  • "sets dict_type on that instance before the first page is read": true for sync (set before super().__init__(), which pre-fetches) and aio (create() pre-fetches after the constructor). The AthenaDictResultSet.__init__ docstring originally said it fetches the first page, which is false for aio; reworded in e27e6e20 before commit.
  • cursor_kwargs and direct arguments: Connection.cursor passes everything as keywords, so the new explicit dict_type parameter receives both; nothing passes cursor arguments positionally.
  • Global default: a value assigned to AthenaDictResultSet.dict_type still applies to cursors without dict_type, including aio through inheritance.
  • Docs: docs/cursor.md and docs/aio.md examples pass dict_type per cursor and stay correct.
  • AWS: no API change; the tests add one or two one_row queries per test.

Findings (PR description, corrected):

  1. "A cursor subclass that assigns its own AthenaDictResultSet subclass ... gets dict_type applied" holds only if that class's __init__ accepts dict_type; now qualified.
  2. Missing compatibility note: a result set subclass whose __init__ does not accept dict_type now raises TypeError when its cursor is given dict_type (before, the class attribute reached it). Added to the release-note list; cursors without dict_type are unaffected after ee872f7a.

Independent review of this revision is next.

"""Initialize the result set with an optional row type for this instance.

Args:
*args: Positional arguments passed to the next ``__init__`` in the MRO.
dict_type: The type used to build each row of this result set. If
None, the class attribute ``dict_type`` is used.
**kwargs: Keyword arguments passed to the next ``__init__`` in the MRO.
"""
if dict_type is not None:
self.dict_type = dict_type
super().__init__(*args, **kwargs)

@override
def _get_rows(
self,
Expand Down
28 changes: 27 additions & 1 deletion tests/pyathena/aio/test_cursor.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
import asyncio
import re
import threading
from collections import OrderedDict
from datetime import UTC, datetime, timedelta
from unittest.mock import AsyncMock, MagicMock, call, patch

import pytest
from botocore.exceptions import ClientError

from pyathena import BINARY, Binary, ExecuteOptions
from pyathena.aio.cursor import AioCursor
from pyathena.aio.cursor import AioCursor, AioDictCursor
from pyathena.aio.result_set import AthenaAioDictResultSet
from pyathena.error import DatabaseError, OperationalError, ProgrammingError
from pyathena.glue import GlueMetadataClient
from pyathena.model import AthenaQueryExecution
Expand Down Expand Up @@ -896,3 +898,27 @@ async def test_fetchall(self, aio_dict_cursor):
assert await aio_dict_cursor.fetchall() == [{"number_of_rows": 1}]
await aio_dict_cursor.execute("SELECT a FROM many_rows ORDER BY a")
assert await aio_dict_cursor.fetchall() == [{"a": i} for i in range(10000)]

async def test_dict_type(self, aio_dict_cursor):
async with aio_dict_cursor.connection.cursor(dict_type=OrderedDict) as ordered_cursor:
await ordered_cursor.execute("SELECT * FROM one_row")
assert type(await ordered_cursor.fetchone()) is OrderedDict
# dict_type of another cursor does not change the row type of this one.
await aio_dict_cursor.execute("SELECT * FROM one_row")
assert type(await aio_dict_cursor.fetchone()) is dict

async def test_dict_type_custom_result_set(self, aio_dict_cursor):
class CustomResultSet(AthenaAioDictResultSet):
pass

class CustomDictCursor(AioDictCursor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._result_set_class = CustomResultSet

async with aio_dict_cursor.connection.cursor(
CustomDictCursor, dict_type=OrderedDict
) as cursor:
await cursor.execute("SELECT * FROM one_row")
assert isinstance(cursor.result_set, CustomResultSet)
assert type(await cursor.fetchone()) is OrderedDict
28 changes: 26 additions & 2 deletions tests/pyathena/test_async_cursor.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
import contextlib
import time
from collections import OrderedDict
from datetime import datetime
from random import randint

import pytest

from pyathena.async_cursor import AsyncCursor
from pyathena.async_cursor import AsyncCursor, AsyncDictCursor
from pyathena.error import NotSupportedError, ProgrammingError
from pyathena.model import AthenaQueryExecution
from pyathena.result_set import AthenaResultSet
from pyathena.result_set import AthenaDictResultSet, AthenaResultSet
from tests import ENV
from tests.pyathena.conftest import connect

Expand Down Expand Up @@ -246,3 +247,26 @@ def test_fetchall(self, async_dict_cursor):
query_id, future = async_dict_cursor.execute("SELECT a FROM many_rows ORDER BY a")
result_set = future.result()
assert result_set.fetchall() == [{"a": i} for i in range(10000)]

def test_dict_type(self, async_dict_cursor):
with async_dict_cursor.connection.cursor(dict_type=OrderedDict) as ordered_cursor:
_, future = ordered_cursor.execute("SELECT * FROM one_row")
assert type(future.result().fetchone()) is OrderedDict
# dict_type of another cursor does not change the row type of this one.
_, future = async_dict_cursor.execute("SELECT * FROM one_row")
assert type(future.result().fetchone()) is dict

def test_dict_type_custom_result_set(self, async_dict_cursor):
class CustomResultSet(AthenaDictResultSet):
pass

class CustomDictCursor(AsyncDictCursor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._result_set_class = CustomResultSet

with async_dict_cursor.connection.cursor(CustomDictCursor, dict_type=OrderedDict) as cursor:
_, future = cursor.execute("SELECT * FROM one_row")
result_set = future.result()
assert isinstance(result_set, CustomResultSet)
assert type(result_set.fetchone()) is OrderedDict
26 changes: 25 additions & 1 deletion tests/pyathena/test_cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import threading
import time
import uuid
from collections import OrderedDict
from concurrent import futures
from concurrent.futures.thread import ThreadPoolExecutor
from datetime import UTC, date, datetime, timedelta
Expand All @@ -31,9 +32,10 @@
)
from pyathena.async_cursor import AsyncCursor
from pyathena.converter import _to_array, _to_map, _to_struct
from pyathena.cursor import Cursor
from pyathena.cursor import Cursor, DictCursor
from pyathena.error import DatabaseError, NotSupportedError, OperationalError, ProgrammingError
from pyathena.model import AthenaQueryExecution
from pyathena.result_set import AthenaDictResultSet
from pyathena.util import RetryConfig
from tests import ENV
from tests.pyathena.conftest import connect
Expand Down Expand Up @@ -1876,6 +1878,28 @@ def test_fetchall(self, dict_cursor):
dict_cursor.execute("SELECT a FROM many_rows ORDER BY a")
assert dict_cursor.fetchall() == [{"a": i} for i in range(10000)]

def test_dict_type(self, dict_cursor):
with dict_cursor.connection.cursor(dict_type=OrderedDict) as ordered_cursor:
ordered_cursor.execute("SELECT * FROM one_row")
assert type(ordered_cursor.fetchone()) is OrderedDict
# dict_type of another cursor does not change the row type of this one.
dict_cursor.execute("SELECT * FROM one_row")
assert type(dict_cursor.fetchone()) is dict

def test_dict_type_custom_result_set(self, dict_cursor):
class CustomResultSet(AthenaDictResultSet):
pass

class CustomDictCursor(DictCursor):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._result_set_class = CustomResultSet

with dict_cursor.connection.cursor(CustomDictCursor, dict_type=OrderedDict) as cursor:
cursor.execute("SELECT * FROM one_row")
assert isinstance(cursor.result_set, CustomResultSet)
assert type(cursor.fetchone()) is OrderedDict

def test_null_vs_empty_string(self, dict_cursor):
"""
DictCursor should properly distinguish NULL from empty string.
Expand Down
Loading