Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 45 additions & 15 deletions amber/src/main/python/core/models/operator.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,22 @@
import base64


def require_yielded(result, method_name: str):
"""
Catch the common case of a user method such as process_table using
`return` instead of `yield`: it rejects a returned None, Tuple, dict or
DataFrame, which would otherwise fail later with a confusing error.
Other values (e.g. a list, or a pandas Series) are passed through
unchecked.
"""
if result is None or isinstance(result, (Tuple, dict, pandas.DataFrame)):
raise TypeError(
f"{method_name} must `yield` results, not `return` them. "
"Replace `return x` with `yield x`."
)
return result


class Operator(ABC):
"""
Abstract base class for all operators.
Expand Down Expand Up @@ -179,7 +195,7 @@ def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]:
@overrides.final
def on_finish(self, port: int) -> Iterator[Optional[TupleLike]]:
# TODO: change on_finish to output Iterator[Union[TupleLike, TableLike, None]]
for i in self.produce():
for i in require_yielded(self.produce(), "produce"):
yield from all_output_to_tuple(i)

@overrides.final
Expand All @@ -197,7 +213,6 @@ class BatchOperator(TupleOperatorV2):

def __init__(self):
super().__init__()
self.__batch_data: MutableMapping[int, List[Tuple]] = defaultdict(list)
self._validate_batch_size(self.BATCH_SIZE)

@staticmethod
Expand All @@ -210,26 +225,35 @@ def _validate_batch_size(value):
if value <= 0:
raise ValueError("BATCH_SIZE should be positive.")

def _get_batch_data(self) -> MutableMapping[int, List[Tuple]]:
# Created on first use (not in __init__) so a subclass whose __init__
# skips super().__init__() still works, like _get_template_decoder.
if not hasattr(self, "_BatchOperator__batch_data"):
self._validate_batch_size(self.BATCH_SIZE)
self.__batch_data = defaultdict(list)
return self.__batch_data

@overrides.final
def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:
self.__batch_data[port].append(tuple_)
if (
self.BATCH_SIZE is not None
and len(self.__batch_data[port]) >= self.BATCH_SIZE
):
batch_data = self._get_batch_data()
batch_data[port].append(tuple_)
if self.BATCH_SIZE is not None and len(batch_data[port]) >= self.BATCH_SIZE:
yield from self._process_batch(port)

@overrides.final
def _process_batch(self, port: int) -> Iterator[Optional[BatchLike]]:
batch_data = self._get_batch_data()
batch = Batch(
pandas.DataFrame(
[
self.__batch_data[port].pop(0).as_series()
for _ in range(min(len(self.__batch_data[port]), self.BATCH_SIZE))
batch_data[port].pop(0).as_series()
for _ in range(min(len(batch_data[port]), self.BATCH_SIZE))
]
)
)
for output_batch in self.process_batch(batch, port):
for output_batch in require_yielded(
self.process_batch(batch, port), "process_batch"
):
if output_batch is not None:
if isinstance(output_batch, pandas.DataFrame):
# TODO: integrate into Batch as a helper function.
Expand All @@ -242,7 +266,7 @@ def _process_batch(self, port: int) -> Iterator[Optional[BatchLike]]:

@overrides.final
def on_finish(self, port: int) -> Iterator[Optional[BatchLike]]:
while len(self.__batch_data[port]) != 0:
while len(self._get_batch_data()[port]) != 0:
yield from self._process_batch(port)

@abstractmethod
Expand All @@ -268,16 +292,22 @@ class TableOperator(TupleOperatorV2):
def __init__(self):
super().__init__()
self._Operator__internal_is_source: bool = False
self.__table_data: Mapping[int, List[Tuple]] = defaultdict(list)

def _get_table_data(self) -> Mapping[int, List[Tuple]]:
Comment thread
Xiao-zhen-Liu marked this conversation as resolved.
Resolved
# Created on first use (not in __init__) so a subclass whose __init__
# skips super().__init__() still works, like _get_template_decoder.
if not hasattr(self, "_TableOperator__table_data"):
self.__table_data = defaultdict(list)
return self.__table_data

@overrides.final
def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:
self.__table_data[port].append(tuple_)
self._get_table_data()[port].append(tuple_)
yield

def on_finish(self, port: int) -> Iterator[Optional[TableLike]]:
table = Table(self.__table_data[port])
yield from self.process_table(table, port)
table = Table(self._get_table_data()[port])
yield from require_yielded(self.process_table(table, port), "process_table")

@abstractmethod
def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
Expand Down
55 changes: 49 additions & 6 deletions amber/src/main/python/core/runnables/data_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,15 @@
# under the License.

from contextlib import contextmanager

import pandas
from threading import Event
from typing import Iterator, Optional

from core.architecture.managers import Context
from core.models import State, TupleLike
from core.models import AttributeType, Schema, State, TupleLike
from core.models.internal_marker import EndChannel, PortMarker, StartChannel
from core.models.operator import require_yielded
from core.models.table import all_output_to_tuple
from core.util import Stoppable
from core.util.console_message.replace_print import replace_print
Expand Down Expand Up @@ -76,7 +79,9 @@ def process_internal_marker(self, internal_marker: PortMarker) -> None:
# Flush the state to MainLoop before producing tuples so the
# state and the tuple stream don't share a single switch.
self._switch_context()
self._set_output_tuple(executor.on_finish(port_id))
self._set_output_tuple(
require_yielded(executor.on_finish(port_id), "on_finish")
)

def process_state(self, state: State) -> None:
"""
Expand All @@ -94,7 +99,11 @@ def process_tuple(self) -> None:
while not finished_current.is_set():
with self._executor_session() as (executor, port_id):
tuple_ = self._context.tuple_processing_manager.get_input_tuple()
self._set_output_tuple(executor.process_tuple(tuple_, port_id))
self._set_output_tuple(
require_yielded(
Comment thread
Xiao-zhen-Liu marked this conversation as resolved.
Resolved
executor.process_tuple(tuple_, port_id), "process_tuple"
)
)

@contextmanager
def _executor_session(self, marker_port_id: int | None = None):
Expand Down Expand Up @@ -132,16 +141,50 @@ def _set_output_tuple(self, output_iterator: Iterator[Optional[TupleLike]]) -> N
# output could be a None, a TupleLike, or a TableLike.
for output_tuple in all_output_to_tuple(output):
if output_tuple is not None:
output_tuple.finalize(
self._context.output_manager.get_port().get_schema()
)
schema = self._context.output_manager.get_port().get_schema()
try:
output_tuple.finalize(schema)
except (KeyError, TypeError) as error:
# Only explain on failure, so valid rows are not inspected.
raise self._explain_schema_mismatch(
output, schema, error
) from error
self._switch_context()
self._context.tuple_processing_manager.current_output_tuple = (
output_tuple
)
self._switch_context()
self._context.tuple_processing_manager.finished_current.set()

@staticmethod
def _explain_schema_mismatch(output, schema: Schema, error: Exception) -> Exception:
"""
Rebuild a schema check failure with a hint on how to fix the output,
keeping the original message. The schema check reports missing or
extra columns (KeyError) before wrong types (TypeError), so only a
TypeError can be caused by a DataFrame sitting in a column; a
DataFrame is valid in a BINARY column, where it is pickled.
"""
message = error.args[0] if error.args else str(error)
if isinstance(error, TypeError) and isinstance(output, dict):
columns = set(schema.get_attr_names())
for key, value in output.items():
if (
isinstance(value, pandas.DataFrame)
and key in columns
and schema.get_attr_type(key) != AttributeType.BINARY
):
return TypeError(
f"{message} A DataFrame was put in column '{key}'. To "
"output its rows, yield the DataFrame itself instead of "
"a dict; to keep it as one value, make the column binary."
)
return type(error)(
f"{message} The yielded columns and their types must match the "
"operator's output columns: 'Retain input columns' and 'Extra output "
"column(s)' for a Python UDF, or 'Columns' for a 1-out Python UDF."
)

def _set_output_state(self, output_state: State) -> None:
"""
Set the output state after processing by the executor.
Expand Down
111 changes: 111 additions & 0 deletions amber/src/test/python/core/models/test_operator.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,3 +514,114 @@ def test_batch_buffers_are_keyed_by_port(self):
assert op.seen == [[[1], [2]]] # the port-1 tuple is not in here
assert op.ports == [0]
assert out == [Tuple({"batched": 2})]


class _TableWithoutSuperInit(TableOperator):
def __init__(self):
self.threshold = 1 # forgets super().__init__()

def process_table(self, table, port):
yield table


class _BatchWithoutSuperInit(BatchOperator):
BATCH_SIZE = 2

def __init__(self):
self.seen = 0 # forgets super().__init__()

def process_batch(self, batch, port):
yield batch


class _TableReturningDataFrame(TableOperator):
def process_table(self, table, port):
return table


class _TableReturningNone(TableOperator):
def process_table(self, table, port):
return None


class _TableReturningList(TableOperator):
def process_table(self, table, port):
return [{"x": 1}]


class _BatchReturningBatch(BatchOperator):
BATCH_SIZE = 1

def process_batch(self, batch, port):
return batch


class _SourceReturningDict(SourceOperator):
def produce(self):
return {"x": 1}


class TestCommonUDFMistakes:
"""Mistakes UDF authors make should either work or say how to fix them."""

def test_table_operator_works_without_super_init(self):
op = _TableWithoutSuperInit()
list(op.process_tuple(Tuple({"x": 1}), 0))
(table,) = list(op.on_finish(0))
assert list(table["x"]) == [1]

def test_batch_operator_works_without_super_init(self):
op = _BatchWithoutSuperInit()
list(op.process_tuple(Tuple({"x": 1}), 0))
outputs = list(op.process_tuple(Tuple({"x": 2}), 0))
assert [t["x"] for t in outputs] == [1, 2]

@pytest.mark.parametrize(
"operator", [_TableReturningDataFrame, _TableReturningNone]
)
def test_process_table_return_says_to_yield(self, operator):
op = operator()
list(op.process_tuple(Tuple({"x": 1}), 0))
with pytest.raises(TypeError) as exc_info:
list(op.on_finish(0))
assert "process_table must `yield` results, not `return` them" in str(
exc_info.value
)

def test_process_table_returning_a_list_still_works(self):
op = _TableReturningList()
assert list(op.on_finish(0)) == [{"x": 1}]

def test_process_batch_return_says_to_yield(self):
op = _BatchReturningBatch()
with pytest.raises(TypeError) as exc_info:
list(op.process_tuple(Tuple({"x": 1}), 0))
assert "process_batch must `yield` results" in str(exc_info.value)

def test_produce_return_says_to_yield(self):
op = _SourceReturningDict()
with pytest.raises(TypeError) as exc_info:
list(op.on_finish(0))
assert "produce must `yield` results" in str(exc_info.value)


class TestBufferCreatedOnlyByGetter:
def test_table_operator_has_no_buffer_until_first_tuple(self):
op = _TableWithoutSuperInit()
assert "_TableOperator__table_data" not in op.__dict__

class _WithSuper(TableOperator):
def process_table(self, table, port):
yield table

assert "_TableOperator__table_data" not in _WithSuper().__dict__

def test_batch_operator_still_rejects_a_bad_batch_size_at_construction(self):
class _BadSize(BatchOperator):
BATCH_SIZE = 0

def process_batch(self, batch, port):
yield batch

with pytest.raises(ValueError):
_BadSize()
Loading
Loading