From 4a668bc158df41504116c3204c0df7cb72b9fe31 Mon Sep 17 00:00:00 2001 From: suryaacharan Date: Mon, 28 Sep 2026 14:27:38 -0700 Subject: [PATCH 1/4] feat(pyamber): clear errors for return-instead-of-yield and output mismatches - `return` instead of `yield` in process_tuple, process_batch, process_table or produce now raises " must `yield` results, not `return` them" instead of MatchError / 'NoneType' not iterable. Returning a list still works. - UDFTableOperator / UDFBatchOperator create their buffers lazily, so an __init__ that skips super().__init__() no longer crashes with '_TableOperator__table_data' (same approach as the lazy template decoder in Operator). - Yielding a dict of DataFrames says to yield the DataFrame itself. - Output column mismatches point to 'Retain input columns' and 'Extra output column(s)'. Refs #8736 Co-Authored-By: Claude Opus 5.5 --- amber/src/main/python/core/models/operator.py | 56 +++++++++--- .../python/core/runnables/data_processor.py | 30 ++++++- .../test/python/core/models/test_operator.py | 89 +++++++++++++++++++ .../core/runnables/test_data_processor.py | 59 ++++++++++++ 4 files changed, 217 insertions(+), 17 deletions(-) diff --git a/amber/src/main/python/core/models/operator.py b/amber/src/main/python/core/models/operator.py index 4e65fbb2ab2..ee4faf31f34 100644 --- a/amber/src/main/python/core/models/operator.py +++ b/amber/src/main/python/core/models/operator.py @@ -29,6 +29,20 @@ import base64 +def require_yielded(result, method_name: str): + """ + Check that a user method such as process_table produced an iterator. + Using `return` instead of `yield` hands back a single value (or None), + which would otherwise fail later with a confusing error. + """ + if result is None or isinstance(result, (Tuple, dict, pandas.DataFrame)): + raise TypeError( + f"{method_name} must `yield` results, not `return` them. " + "Replace `return x` with `yield x`." + ) + return result + + class Operator(ABC): """ Abstract base class for all operators. @@ -179,7 +193,7 @@ def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]: @overrides.final def on_finish(self, port: int) -> Iterator[Optional[TupleLike]]: # TODO: change on_finish to output Iterator[Union[TupleLike, TableLike, None]] - for i in self.produce(): + for i in require_yielded(self.produce(), "produce"): yield from all_output_to_tuple(i) @overrides.final @@ -210,26 +224,35 @@ def _validate_batch_size(value): if value <= 0: raise ValueError("BATCH_SIZE should be positive.") + def _get_batch_data(self) -> MutableMapping[int, List[Tuple]]: + # Created lazily so a subclass whose __init__ skips super().__init__() + # still works, like Operator._get_template_decoder. + if not hasattr(self, "_BatchOperator__batch_data"): + self._validate_batch_size(self.BATCH_SIZE) + self.__batch_data = defaultdict(list) + return self.__batch_data + @overrides.final def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: - self.__batch_data[port].append(tuple_) - if ( - self.BATCH_SIZE is not None - and len(self.__batch_data[port]) >= self.BATCH_SIZE - ): + batch_data = self._get_batch_data() + batch_data[port].append(tuple_) + if self.BATCH_SIZE is not None and len(batch_data[port]) >= self.BATCH_SIZE: yield from self._process_batch(port) @overrides.final def _process_batch(self, port: int) -> Iterator[Optional[BatchLike]]: + batch_data = self._get_batch_data() batch = Batch( pandas.DataFrame( [ - self.__batch_data[port].pop(0).as_series() - for _ in range(min(len(self.__batch_data[port]), self.BATCH_SIZE)) + batch_data[port].pop(0).as_series() + for _ in range(min(len(batch_data[port]), self.BATCH_SIZE)) ] ) ) - for output_batch in self.process_batch(batch, port): + for output_batch in require_yielded( + self.process_batch(batch, port), "process_batch" + ): if output_batch is not None: if isinstance(output_batch, pandas.DataFrame): # TODO: integrate into Batch as a helper function. @@ -242,7 +265,7 @@ def _process_batch(self, port: int) -> Iterator[Optional[BatchLike]]: @overrides.final def on_finish(self, port: int) -> Iterator[Optional[BatchLike]]: - while len(self.__batch_data[port]) != 0: + while len(self._get_batch_data()[port]) != 0: yield from self._process_batch(port) @abstractmethod @@ -270,14 +293,21 @@ def __init__(self): self._Operator__internal_is_source: bool = False self.__table_data: Mapping[int, List[Tuple]] = defaultdict(list) + def _get_table_data(self) -> Mapping[int, List[Tuple]]: + # Created lazily so a subclass whose __init__ skips super().__init__() + # still works, like Operator._get_template_decoder. + if not hasattr(self, "_TableOperator__table_data"): + self.__table_data = defaultdict(list) + return self.__table_data + @overrides.final def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]: - self.__table_data[port].append(tuple_) + self._get_table_data()[port].append(tuple_) yield def on_finish(self, port: int) -> Iterator[Optional[TableLike]]: - table = Table(self.__table_data[port]) - yield from self.process_table(table, port) + table = Table(self._get_table_data()[port]) + yield from require_yielded(self.process_table(table, port), "process_table") @abstractmethod def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: diff --git a/amber/src/main/python/core/runnables/data_processor.py b/amber/src/main/python/core/runnables/data_processor.py index eaee8660cd5..1fa49fd7750 100644 --- a/amber/src/main/python/core/runnables/data_processor.py +++ b/amber/src/main/python/core/runnables/data_processor.py @@ -16,12 +16,15 @@ # under the License. from contextlib import contextmanager + +import pandas from threading import Event from typing import Iterator, Optional from core.architecture.managers import Context from core.models import State, TupleLike from core.models.internal_marker import EndChannel, PortMarker, StartChannel +from core.models.operator import require_yielded from core.models.table import all_output_to_tuple from core.util import Stoppable from core.util.console_message.replace_print import replace_print @@ -94,7 +97,11 @@ def process_tuple(self) -> None: while not finished_current.is_set(): with self._executor_session() as (executor, port_id): tuple_ = self._context.tuple_processing_manager.get_input_tuple() - self._set_output_tuple(executor.process_tuple(tuple_, port_id)) + self._set_output_tuple( + require_yielded( + executor.process_tuple(tuple_, port_id), "process_tuple" + ) + ) @contextmanager def _executor_session(self, marker_port_id: int | None = None): @@ -130,11 +137,26 @@ def _set_output_tuple(self, output_iterator: Iterator[Optional[TupleLike]]) -> N """ for output in output_iterator: # output could be a None, a TupleLike, or a TableLike. + if isinstance(output, dict) and any( + isinstance(value, pandas.DataFrame) for value in output.values() + ): + raise TypeError( + "A dict of DataFrames was yielded: yield the DataFrame itself, " + "not a dict of DataFrames." + ) for output_tuple in all_output_to_tuple(output): if output_tuple is not None: - output_tuple.finalize( - self._context.output_manager.get_port().get_schema() - ) + try: + output_tuple.finalize( + self._context.output_manager.get_port().get_schema() + ) + except KeyError as error: + raise KeyError( + f"{error.args[0]} The yielded columns must match the " + "operator's output columns. For a Python UDF, check " + "'Retain input columns' and 'Extra output column(s)' " + "in the property panel." + ) from error self._switch_context() self._context.tuple_processing_manager.current_output_tuple = ( output_tuple diff --git a/amber/src/test/python/core/models/test_operator.py b/amber/src/test/python/core/models/test_operator.py index bb18853350e..fe9e85f0aba 100644 --- a/amber/src/test/python/core/models/test_operator.py +++ b/amber/src/test/python/core/models/test_operator.py @@ -514,3 +514,92 @@ def test_batch_buffers_are_keyed_by_port(self): assert op.seen == [[[1], [2]]] # the port-1 tuple is not in here assert op.ports == [0] assert out == [Tuple({"batched": 2})] + + +class _TableWithoutSuperInit(TableOperator): + def __init__(self): + self.threshold = 1 # forgets super().__init__() + + def process_table(self, table, port): + yield table + + +class _BatchWithoutSuperInit(BatchOperator): + BATCH_SIZE = 2 + + def __init__(self): + self.seen = 0 # forgets super().__init__() + + def process_batch(self, batch, port): + yield batch + + +class _TableReturningDataFrame(TableOperator): + def process_table(self, table, port): + return table + + +class _TableReturningNone(TableOperator): + def process_table(self, table, port): + return None + + +class _TableReturningList(TableOperator): + def process_table(self, table, port): + return [{"x": 1}] + + +class _BatchReturningBatch(BatchOperator): + BATCH_SIZE = 1 + + def process_batch(self, batch, port): + return batch + + +class _SourceReturningDict(SourceOperator): + def produce(self): + return {"x": 1} + + +class TestCommonUDFMistakes: + """Mistakes UDF authors make should either work or say how to fix them.""" + + def test_table_operator_works_without_super_init(self): + op = _TableWithoutSuperInit() + list(op.process_tuple(Tuple({"x": 1}), 0)) + (table,) = list(op.on_finish(0)) + assert list(table["x"]) == [1] + + def test_batch_operator_works_without_super_init(self): + op = _BatchWithoutSuperInit() + list(op.process_tuple(Tuple({"x": 1}), 0)) + outputs = list(op.process_tuple(Tuple({"x": 2}), 0)) + assert [t["x"] for t in outputs] == [1, 2] + + @pytest.mark.parametrize( + "operator", [_TableReturningDataFrame, _TableReturningNone] + ) + def test_process_table_return_says_to_yield(self, operator): + op = operator() + list(op.process_tuple(Tuple({"x": 1}), 0)) + with pytest.raises(TypeError) as exc_info: + list(op.on_finish(0)) + assert "process_table must `yield` results, not `return` them" in str( + exc_info.value + ) + + def test_process_table_returning_a_list_still_works(self): + op = _TableReturningList() + assert list(op.on_finish(0)) == [{"x": 1}] + + def test_process_batch_return_says_to_yield(self): + op = _BatchReturningBatch() + with pytest.raises(TypeError) as exc_info: + list(op.process_tuple(Tuple({"x": 1}), 0)) + assert "process_batch must `yield` results" in str(exc_info.value) + + def test_produce_return_says_to_yield(self): + op = _SourceReturningDict() + with pytest.raises(TypeError) as exc_info: + list(op.on_finish(0)) + assert "produce must `yield` results" in str(exc_info.value) diff --git a/amber/src/test/python/core/runnables/test_data_processor.py b/amber/src/test/python/core/runnables/test_data_processor.py index e185872f2e2..274d1301fdb 100644 --- a/amber/src/test/python/core/runnables/test_data_processor.py +++ b/amber/src/test/python/core/runnables/test_data_processor.py @@ -290,3 +290,62 @@ def test_two_queued_inputs_raises_invariant_error(self, context, monkeypatch): assert "expected exactly one queued input" in str(excinfo.value) assert "marker=True" in str(excinfo.value) assert "tuple=True" in str(excinfo.value) + + +class _ReturningExecutor: + """process_tuple uses `return` instead of `yield`.""" + + def process_tuple(self, tuple_, port_id): + return tuple_ + + +class TestUDFOutputMistakes: + @pytest.fixture + def output_schema(self, context, monkeypatch): + schema = Schema(raw_schema={"x": "INTEGER"}) + monkeypatch.setattr( + context.output_manager, + "get_port", + lambda *args, **kwargs: SimpleNamespace(get_schema=lambda: schema), + ) + return schema + + @pytest.mark.timeout(2) + def test_process_tuple_return_says_to_yield(self, context, data_processor): + context.executor_manager.executor = _ReturningExecutor() + context.tuple_processing_manager.current_input_tuple = Tuple({"x": 1}) + + def switch_and_stop(): + data_processor.switch_calls += 1 + context.tuple_processing_manager.finished_current.set() + + data_processor._switch_context = switch_and_stop + data_processor.process_tuple() + + exc_info = context.exception_manager.get_exc_info() + assert exc_info[0] is TypeError + assert "process_tuple must `yield` results, not `return` them" in str( + exc_info[1] + ) + + def test_dict_of_dataframes_says_to_yield_the_dataframe( + self, data_processor, output_schema + ): + import pandas + + with pytest.raises(TypeError) as exc_info: + data_processor._set_output_tuple( + iter([{"a": pandas.DataFrame({"x": [1]})}]) + ) + assert "yield the DataFrame itself, not a dict of DataFrames" in str( + exc_info.value + ) + + def test_schema_mismatch_points_to_output_columns( + self, data_processor, output_schema + ): + with pytest.raises(KeyError) as exc_info: + data_processor._set_output_tuple(iter([{"y": 1}])) + message = str(exc_info.value) + assert "expected but missing" in message + assert "Extra output column(s)" in message From 300360e9e09c26cfc20cb09c8ffb0d0e57b7b7d1 Mon Sep 17 00:00:00 2001 From: suryaacharan Date: Tue, 6 Oct 2026 16:18:09 -0700 Subject: [PATCH 2/4] fix(pyamber): address review on UDF output error hints - Only explain a schema failure after finalize fails, catching KeyError and TypeError. A dict holding a DataFrame into a BINARY column is pickled on main and works again; valid rows are no longer inspected. - A wrong column type now gets the same output-columns hint. - on_finish goes through require_yielded like process_tuple. - require_yielded's docstring says which returned values it catches. - Table and batch buffers are created only by their getters. Co-Authored-By: Claude Opus 5.5 --- amber/src/main/python/core/models/operator.py | 18 +++---- .../python/core/runnables/data_processor.py | 41 +++++++++------ amber/src/main/python/proto | 1 + .../test/python/core/models/test_operator.py | 22 ++++++++ .../core/runnables/test_data_processor.py | 50 +++++++++++++++++++ 5 files changed, 108 insertions(+), 24 deletions(-) create mode 120000 amber/src/main/python/proto diff --git a/amber/src/main/python/core/models/operator.py b/amber/src/main/python/core/models/operator.py index ee4faf31f34..bbba116da56 100644 --- a/amber/src/main/python/core/models/operator.py +++ b/amber/src/main/python/core/models/operator.py @@ -31,9 +31,11 @@ def require_yielded(result, method_name: str): """ - Check that a user method such as process_table produced an iterator. - Using `return` instead of `yield` hands back a single value (or None), - which would otherwise fail later with a confusing error. + Catch the common case of a user method such as process_table using + `return` instead of `yield`: it rejects a returned None, Tuple, dict or + DataFrame, which would otherwise fail later with a confusing error. + Other values (e.g. a list, or a pandas Series) are passed through + unchecked. """ if result is None or isinstance(result, (Tuple, dict, pandas.DataFrame)): raise TypeError( @@ -211,7 +213,6 @@ class BatchOperator(TupleOperatorV2): def __init__(self): super().__init__() - self.__batch_data: MutableMapping[int, List[Tuple]] = defaultdict(list) self._validate_batch_size(self.BATCH_SIZE) @staticmethod @@ -225,8 +226,8 @@ def _validate_batch_size(value): raise ValueError("BATCH_SIZE should be positive.") def _get_batch_data(self) -> MutableMapping[int, List[Tuple]]: - # Created lazily so a subclass whose __init__ skips super().__init__() - # still works, like Operator._get_template_decoder. + # Created on first use (not in __init__) so a subclass whose __init__ + # skips super().__init__() still works, like _get_template_decoder. if not hasattr(self, "_BatchOperator__batch_data"): self._validate_batch_size(self.BATCH_SIZE) self.__batch_data = defaultdict(list) @@ -291,11 +292,10 @@ class TableOperator(TupleOperatorV2): def __init__(self): super().__init__() self._Operator__internal_is_source: bool = False - self.__table_data: Mapping[int, List[Tuple]] = defaultdict(list) def _get_table_data(self) -> Mapping[int, List[Tuple]]: - # Created lazily so a subclass whose __init__ skips super().__init__() - # still works, like Operator._get_template_decoder. + # Created on first use (not in __init__) so a subclass whose __init__ + # skips super().__init__() still works, like _get_template_decoder. if not hasattr(self, "_TableOperator__table_data"): self.__table_data = defaultdict(list) return self.__table_data diff --git a/amber/src/main/python/core/runnables/data_processor.py b/amber/src/main/python/core/runnables/data_processor.py index 1fa49fd7750..171a7b1b06e 100644 --- a/amber/src/main/python/core/runnables/data_processor.py +++ b/amber/src/main/python/core/runnables/data_processor.py @@ -79,7 +79,9 @@ def process_internal_marker(self, internal_marker: PortMarker) -> None: # Flush the state to MainLoop before producing tuples so the # state and the tuple stream don't share a single switch. self._switch_context() - self._set_output_tuple(executor.on_finish(port_id)) + self._set_output_tuple( + require_yielded(executor.on_finish(port_id), "on_finish") + ) def process_state(self, state: State) -> None: """ @@ -137,26 +139,15 @@ def _set_output_tuple(self, output_iterator: Iterator[Optional[TupleLike]]) -> N """ for output in output_iterator: # output could be a None, a TupleLike, or a TableLike. - if isinstance(output, dict) and any( - isinstance(value, pandas.DataFrame) for value in output.values() - ): - raise TypeError( - "A dict of DataFrames was yielded: yield the DataFrame itself, " - "not a dict of DataFrames." - ) for output_tuple in all_output_to_tuple(output): if output_tuple is not None: try: output_tuple.finalize( self._context.output_manager.get_port().get_schema() ) - except KeyError as error: - raise KeyError( - f"{error.args[0]} The yielded columns must match the " - "operator's output columns. For a Python UDF, check " - "'Retain input columns' and 'Extra output column(s)' " - "in the property panel." - ) from error + except (KeyError, TypeError) as error: + # Only explain on failure, so valid rows are not inspected. + raise self._explain_schema_mismatch(output, error) from error self._switch_context() self._context.tuple_processing_manager.current_output_tuple = ( output_tuple @@ -164,6 +155,26 @@ def _set_output_tuple(self, output_iterator: Iterator[Optional[TupleLike]]) -> N self._switch_context() self._context.tuple_processing_manager.finished_current.set() + @staticmethod + def _explain_schema_mismatch(output, error: Exception) -> Exception: + """ + Rebuild a schema check failure with a hint on how to fix the output. + A dict holding DataFrames is valid for BINARY columns (they are + pickled), so it is only called out once the schema check has failed. + """ + if isinstance(output, dict) and any( + isinstance(value, pandas.DataFrame) for value in output.values() + ): + return TypeError( + "A dict of DataFrames was yielded: yield the DataFrame itself, " + "not a dict of DataFrames." + ) + return type(error)( + f"{error.args[0]} The yielded columns and their types must match the " + "operator's output columns. For a Python UDF, check 'Retain input " + "columns' and 'Extra output column(s)' in the property panel." + ) + def _set_output_state(self, output_state: State) -> None: """ Set the output state after processing by the executor. diff --git a/amber/src/main/python/proto b/amber/src/main/python/proto new file mode 120000 index 00000000000..619a2a1e166 --- /dev/null +++ b/amber/src/main/python/proto @@ -0,0 +1 @@ +/Users/suryaacharan/orca/workspaces/texera/tex-86-make-texera-self-explanatory-for-users-ud/amber/src/main/python/proto \ No newline at end of file diff --git a/amber/src/test/python/core/models/test_operator.py b/amber/src/test/python/core/models/test_operator.py index fe9e85f0aba..6ca9ed5658c 100644 --- a/amber/src/test/python/core/models/test_operator.py +++ b/amber/src/test/python/core/models/test_operator.py @@ -603,3 +603,25 @@ def test_produce_return_says_to_yield(self): with pytest.raises(TypeError) as exc_info: list(op.on_finish(0)) assert "produce must `yield` results" in str(exc_info.value) + + +class TestBufferCreatedOnlyByGetter: + def test_table_operator_has_no_buffer_until_first_tuple(self): + op = _TableWithoutSuperInit() + assert "_TableOperator__table_data" not in op.__dict__ + + class _WithSuper(TableOperator): + def process_table(self, table, port): + yield table + + assert "_TableOperator__table_data" not in _WithSuper().__dict__ + + def test_batch_operator_still_rejects_a_bad_batch_size_at_construction(self): + class _BadSize(BatchOperator): + BATCH_SIZE = 0 + + def process_batch(self, batch, port): + yield batch + + with pytest.raises(ValueError): + _BadSize() diff --git a/amber/src/test/python/core/runnables/test_data_processor.py b/amber/src/test/python/core/runnables/test_data_processor.py index 274d1301fdb..bcf9c25a44f 100644 --- a/amber/src/test/python/core/runnables/test_data_processor.py +++ b/amber/src/test/python/core/runnables/test_data_processor.py @@ -349,3 +349,53 @@ def test_schema_mismatch_points_to_output_columns( message = str(exc_info.value) assert "expected but missing" in message assert "Extra output column(s)" in message + + +class _FinishReturningExecutor: + """on_finish uses `return` instead of `yield`.""" + + def produce_state_on_finish(self, port_id): + return None + + def on_finish(self, port_id): + return {"x": 1} + + +class TestUDFOutputMistakesFollowUp: + def _use_schema(self, context, monkeypatch, raw_schema): + schema = Schema(raw_schema=raw_schema) + monkeypatch.setattr( + context.output_manager, + "get_port", + lambda *args, **kwargs: SimpleNamespace(get_schema=lambda: schema), + ) + + def test_dict_holding_a_dataframe_into_a_binary_column_still_works( + self, context, data_processor, monkeypatch + ): + import pandas + + # BINARY columns pickle any value, so this is valid output on main. + self._use_schema(context, monkeypatch, {"result": "BINARY"}) + data_processor._set_output_tuple( + iter([{"result": pandas.DataFrame({"x": [1]})}]) + ) + assert context.tuple_processing_manager.finished_current.is_set() + + def test_wrong_column_type_points_to_output_columns( + self, context, data_processor, monkeypatch + ): + self._use_schema(context, monkeypatch, {"x": "INTEGER"}) + with pytest.raises(TypeError) as exc_info: + data_processor._set_output_tuple(iter([{"x": "not a number"}])) + message = str(exc_info.value) + assert "Unmatched type for field 'x'" in message + assert "Extra output column(s)" in message + + @pytest.mark.timeout(2) + def test_on_finish_return_says_to_yield(self, context, data_processor): + context.executor_manager.executor = _FinishReturningExecutor() + data_processor.process_internal_marker(EndChannel(0)) + exc_info = context.exception_manager.get_exc_info() + assert exc_info[0] is TypeError + assert "on_finish must `yield` results, not `return` them" in str(exc_info[1]) From 858cc9c87812c6cbb97e79bf2648759256a08e8d Mon Sep 17 00:00:00 2001 From: suryaacharan Date: Tue, 6 Oct 2026 17:52:24 -0700 Subject: [PATCH 3/4] chore(pyamber): remove a proto symlink committed by mistake A local symlink at amber/src/main/python/proto slipped into the previous commit. It blocked CI's proto generation (mkdir: File exists). Co-Authored-By: Claude Opus 5.5 --- amber/src/main/python/proto | 1 - 1 file changed, 1 deletion(-) delete mode 120000 amber/src/main/python/proto diff --git a/amber/src/main/python/proto b/amber/src/main/python/proto deleted file mode 120000 index 619a2a1e166..00000000000 --- a/amber/src/main/python/proto +++ /dev/null @@ -1 +0,0 @@ -/Users/suryaacharan/orca/workspaces/texera/tex-86-make-texera-self-explanatory-for-users-ud/amber/src/main/python/proto \ No newline at end of file From cc6412a803acc56b1f7aa1c373aae3d2e35279da Mon Sep 17 00:00:00 2001 From: suryaacharan Date: Thu, 8 Oct 2026 11:19:33 -0700 Subject: [PATCH 4/4] fix(pyamber): only blame a DataFrame when it caused the type error - The DataFrame hint now fires only for a TypeError (a wrong type in a column), never for a missing or extra column, and only when a DataFrame sits in an output column that isn't BINARY. A misspelled or missing column next to a DataFrame gets the output-columns hint. - The hint keeps the original message and names the column. - The output-columns hint also mentions 'Columns' for 1-out Python UDFs. Co-Authored-By: Claude Opus 5.5 --- .../python/core/runnables/data_processor.py | 52 +++++++------ .../core/runnables/test_data_processor.py | 74 ++++++++++++++++++- 2 files changed, 101 insertions(+), 25 deletions(-) diff --git a/amber/src/main/python/core/runnables/data_processor.py b/amber/src/main/python/core/runnables/data_processor.py index 171a7b1b06e..4466cc6adb4 100644 --- a/amber/src/main/python/core/runnables/data_processor.py +++ b/amber/src/main/python/core/runnables/data_processor.py @@ -22,7 +22,7 @@ from typing import Iterator, Optional from core.architecture.managers import Context -from core.models import State, TupleLike +from core.models import AttributeType, Schema, State, TupleLike from core.models.internal_marker import EndChannel, PortMarker, StartChannel from core.models.operator import require_yielded from core.models.table import all_output_to_tuple @@ -141,13 +141,14 @@ def _set_output_tuple(self, output_iterator: Iterator[Optional[TupleLike]]) -> N # output could be a None, a TupleLike, or a TableLike. for output_tuple in all_output_to_tuple(output): if output_tuple is not None: + schema = self._context.output_manager.get_port().get_schema() try: - output_tuple.finalize( - self._context.output_manager.get_port().get_schema() - ) + output_tuple.finalize(schema) except (KeyError, TypeError) as error: # Only explain on failure, so valid rows are not inspected. - raise self._explain_schema_mismatch(output, error) from error + raise self._explain_schema_mismatch( + output, schema, error + ) from error self._switch_context() self._context.tuple_processing_manager.current_output_tuple = ( output_tuple @@ -156,23 +157,32 @@ def _set_output_tuple(self, output_iterator: Iterator[Optional[TupleLike]]) -> N self._context.tuple_processing_manager.finished_current.set() @staticmethod - def _explain_schema_mismatch(output, error: Exception) -> Exception: - """ - Rebuild a schema check failure with a hint on how to fix the output. - A dict holding DataFrames is valid for BINARY columns (they are - pickled), so it is only called out once the schema check has failed. - """ - if isinstance(output, dict) and any( - isinstance(value, pandas.DataFrame) for value in output.values() - ): - return TypeError( - "A dict of DataFrames was yielded: yield the DataFrame itself, " - "not a dict of DataFrames." - ) + def _explain_schema_mismatch(output, schema: Schema, error: Exception) -> Exception: + """ + Rebuild a schema check failure with a hint on how to fix the output, + keeping the original message. The schema check reports missing or + extra columns (KeyError) before wrong types (TypeError), so only a + TypeError can be caused by a DataFrame sitting in a column; a + DataFrame is valid in a BINARY column, where it is pickled. + """ + message = error.args[0] if error.args else str(error) + if isinstance(error, TypeError) and isinstance(output, dict): + columns = set(schema.get_attr_names()) + for key, value in output.items(): + if ( + isinstance(value, pandas.DataFrame) + and key in columns + and schema.get_attr_type(key) != AttributeType.BINARY + ): + return TypeError( + f"{message} A DataFrame was put in column '{key}'. To " + "output its rows, yield the DataFrame itself instead of " + "a dict; to keep it as one value, make the column binary." + ) return type(error)( - f"{error.args[0]} The yielded columns and their types must match the " - "operator's output columns. For a Python UDF, check 'Retain input " - "columns' and 'Extra output column(s)' in the property panel." + f"{message} The yielded columns and their types must match the " + "operator's output columns: 'Retain input columns' and 'Extra output " + "column(s)' for a Python UDF, or 'Columns' for a 1-out Python UDF." ) def _set_output_state(self, output_state: State) -> None: diff --git a/amber/src/test/python/core/runnables/test_data_processor.py b/amber/src/test/python/core/runnables/test_data_processor.py index bcf9c25a44f..6b321dd299a 100644 --- a/amber/src/test/python/core/runnables/test_data_processor.py +++ b/amber/src/test/python/core/runnables/test_data_processor.py @@ -335,11 +335,9 @@ def test_dict_of_dataframes_says_to_yield_the_dataframe( with pytest.raises(TypeError) as exc_info: data_processor._set_output_tuple( - iter([{"a": pandas.DataFrame({"x": [1]})}]) + iter([{"x": pandas.DataFrame({"x": [1]})}]) ) - assert "yield the DataFrame itself, not a dict of DataFrames" in str( - exc_info.value - ) + assert "yield the DataFrame itself instead of a dict" in str(exc_info.value) def test_schema_mismatch_points_to_output_columns( self, data_processor, output_schema @@ -399,3 +397,71 @@ def test_on_finish_return_says_to_yield(self, context, data_processor): exc_info = context.exception_manager.get_exc_info() assert exc_info[0] is TypeError assert "on_finish must `yield` results, not `return` them" in str(exc_info[1]) + + def test_misspelled_column_next_to_a_dataframe_gets_the_column_hint( + self, context, data_processor, monkeypatch + ): + import pandas + + # The DataFrame is fine (BINARY column); the column name is wrong. + self._use_schema(context, monkeypatch, {"result": "BINARY"}) + with pytest.raises(KeyError) as exc_info: + data_processor._set_output_tuple( + iter([{"res": pandas.DataFrame({"x": [1]})}]) + ) + message = str(exc_info.value) + assert "Extra output column(s)" in message + assert "dict of DataFrames" not in message + + def test_dataframe_in_a_non_binary_column_says_to_yield_the_dataframe( + self, context, data_processor, monkeypatch + ): + import pandas + + self._use_schema(context, monkeypatch, {"result": "STRING"}) + with pytest.raises(TypeError) as exc_info: + data_processor._set_output_tuple( + iter([{"result": pandas.DataFrame({"x": [1]})}]) + ) + assert "yield the DataFrame itself" in str(exc_info.value) + + def test_missing_column_is_reported_even_with_a_dataframe_in_another_column( + self, context, data_processor, monkeypatch + ): + import pandas + + # The real problem is the missing "count" column, not the DataFrame. + self._use_schema( + context, + monkeypatch, + {"name": "STRING", "count": "INTEGER", "stats": "STRING"}, + ) + with pytest.raises(KeyError) as exc_info: + data_processor._set_output_tuple( + iter([{"name": "a", "stats": pandas.DataFrame({"v": [1]})}]) + ) + message = str(exc_info.value) + assert "'count'" in message and "expected but missing" in message + assert "DataFrame" not in message.split("]")[-1] + + def test_dataframe_hint_names_the_column_and_keeps_the_original_error( + self, context, data_processor, monkeypatch + ): + import pandas + + self._use_schema(context, monkeypatch, {"r": "LARGE_BINARY"}) + with pytest.raises(TypeError) as exc_info: + data_processor._set_output_tuple( + iter([{"r": pandas.DataFrame({"x": [1]})}]) + ) + message = str(exc_info.value) + assert "Unmatched type for field 'r'" in message + assert "column 'r'" in message + + def test_column_hint_mentions_the_source_udf_columns_setting( + self, context, data_processor, monkeypatch + ): + self._use_schema(context, monkeypatch, {"x": "INTEGER"}) + with pytest.raises(KeyError) as exc_info: + data_processor._set_output_tuple(iter([{"y": 1}])) + assert "'Columns'" in str(exc_info.value)