Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 112 additions & 2 deletions amber/src/main/python/core/architecture/managers/executor_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import itertools
import sys
import tempfile
import traceback
from cached_property import cached_property
from loguru import logger
from pathlib import Path
Expand All @@ -28,6 +29,13 @@
from core.models import Operator, SourceOperator


def pytexera_names() -> set:
"""Names that `from pytexera import *` brings into a UDF."""
import pytexera

return set(pytexera.__all__)


class ExecutorManager:
# Process-wide monotonically increasing counter used to generate the
# tmp module names ExecutorManager hands to importlib. Making this a
Expand Down Expand Up @@ -119,15 +127,117 @@ def load_executor_definition(self, code: str) -> type(Operator):
# gen_module_file_name guarantees module_name is unique across
# the process, so import_module will always cleanly load source
# from the tmp directory we just wrote — no re-import / reload dance.
executor_module = importlib.import_module(module_name)
try:
executor_module = importlib.import_module(module_name)
except SyntaxError:
# SyntaxError already carries the user's line number.
raise
except Exception as error:
raise self._with_user_code_hint(error, str(file_path)) from error
self.operator_module_name = module_name

executors = list(
filter(self.is_concrete_operator, executor_module.__dict__.values())
)
assert len(executors) == 1, "There should be one and only one Operator defined"
assert len(executors) == 1, (
"There should be one and only one Operator defined. "
+ self._explain_operator_count(executors, executor_module)
)
return executors[0]

# Each pytexera base class and the one method a subclass must implement.
_BASE_CLASS_METHODS = {
"UDFOperatorV2": "process_tuple",
"UDFBatchOperator": "process_batch",
"UDFTableOperator": "process_table",
"UDFSourceOperator": "produce",
}

@classmethod
def _base_class_pairing(cls) -> str:
return ", ".join(
f"{base} -> {method}" for base, method in cls._BASE_CLASS_METHODS.items()
)

@classmethod
def _explain_operator_count(cls, executors: list, module) -> str:
"""
Explain why a UDF module does not define exactly one concrete Operator,
naming the classes involved and how to fix them.
"""
if len(executors) > 1:
names = ", ".join(executor.__name__ for executor in executors)
return (
f"Found {len(executors)}: {names}. Keep one and delete or comment "
"out the others. Helper classes must not subclass an Operator."
)

abstract_classes = [
value
for value in module.__dict__.values()
if inspect.isclass(value)
and issubclass(value, Operator)
and value.__module__ == module.__name__
]
if not abstract_classes:
return (
"No Operator class was found. Uncomment or write exactly one class, "
"e.g. `class ProcessTupleOperator(UDFOperatorV2):`, and keep "
"`from pytexera import *` as the first line."
)

explanations = []
for abstract_class in abstract_classes:
base_names = [
base.__name__
for base in abstract_class.__mro__
if base.__name__ in cls._BASE_CLASS_METHODS
]
base_name = base_names[0] if base_names else "Operator"
missing = ", ".join(sorted(abstract_class.__abstractmethods__))
explanation = (
f"{abstract_class.__name__} subclasses {base_name} but does not "
f"implement {missing}."
)
for other_base, method in cls._BASE_CLASS_METHODS.items():
if other_base != base_name and method in abstract_class.__dict__:
explanation += (
f" If you meant {method}, subclass {other_base} instead."
)
explanations.append(explanation)
return (
" ".join(explanations)
+ f" Each base class needs its own method: {cls._base_class_pairing()}."
)

@classmethod
def _with_user_code_hint(cls, error: Exception, file_path: str) -> Exception:
"""
Rebuild an error raised while importing user code so its message says
which line of the UDF failed and, for common mistakes, how to fix it.
Returns the original error if it cannot be rebuilt with a new message.
"""
message = str(error)
user_lines = [
frame.lineno
for frame in traceback.extract_tb(error.__traceback__)
if frame.filename == file_path
]
if user_lines:
message += f" (line {user_lines[-1]} of the UDF code)"
if isinstance(error, NameError) and error.name in pytexera_names():
message += ". Add `from pytexera import *` at the top of the UDF."
elif "No super class method found" in message:
message += (
f". The method must match the base class: {cls._base_class_pairing()}."
)
if message == str(error):
return error
try:
return type(error)(message)
except Exception:
return error

def close(self) -> None:
"""
Remove the tmp directory and release all resources created within it.
Expand Down
56 changes: 43 additions & 13 deletions amber/src/main/python/core/models/operator.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,20 @@
import base64


def require_yielded(result, method_name: str):
"""
Check that a user method such as process_table produced an iterator.
Using `return` instead of `yield` hands back a single value (or None),
which would otherwise fail later with a confusing error.
"""
if result is None or isinstance(result, (Tuple, dict, pandas.DataFrame)):
raise TypeError(
f"{method_name} must `yield` results, not `return` them. "
"Replace `return x` with `yield x`."
)
return result


class Operator(ABC):
"""
Abstract base class for all operators.
Expand Down Expand Up @@ -179,7 +193,7 @@ def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]:
@overrides.final
def on_finish(self, port: int) -> Iterator[Optional[TupleLike]]:
# TODO: change on_finish to output Iterator[Union[TupleLike, TableLike, None]]
for i in self.produce():
for i in require_yielded(self.produce(), "produce"):
yield from all_output_to_tuple(i)

@overrides.final
Expand Down Expand Up @@ -210,26 +224,35 @@ def _validate_batch_size(value):
if value <= 0:
raise ValueError("BATCH_SIZE should be positive.")

def _get_batch_data(self) -> MutableMapping[int, List[Tuple]]:
# Created lazily so a subclass whose __init__ skips super().__init__()
# still works, like Operator._get_template_decoder.
if not hasattr(self, "_BatchOperator__batch_data"):
self._validate_batch_size(self.BATCH_SIZE)
self.__batch_data = defaultdict(list)
return self.__batch_data

@overrides.final
def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:
self.__batch_data[port].append(tuple_)
if (
self.BATCH_SIZE is not None
and len(self.__batch_data[port]) >= self.BATCH_SIZE
):
batch_data = self._get_batch_data()
batch_data[port].append(tuple_)
if self.BATCH_SIZE is not None and len(batch_data[port]) >= self.BATCH_SIZE:
yield from self._process_batch(port)

@overrides.final
def _process_batch(self, port: int) -> Iterator[Optional[BatchLike]]:
batch_data = self._get_batch_data()
batch = Batch(
pandas.DataFrame(
[
self.__batch_data[port].pop(0).as_series()
for _ in range(min(len(self.__batch_data[port]), self.BATCH_SIZE))
batch_data[port].pop(0).as_series()
for _ in range(min(len(batch_data[port]), self.BATCH_SIZE))
]
)
)
for output_batch in self.process_batch(batch, port):
for output_batch in require_yielded(
self.process_batch(batch, port), "process_batch"
):
if output_batch is not None:
if isinstance(output_batch, pandas.DataFrame):
# TODO: integrate into Batch as a helper function.
Expand All @@ -242,7 +265,7 @@ def _process_batch(self, port: int) -> Iterator[Optional[BatchLike]]:

@overrides.final
def on_finish(self, port: int) -> Iterator[Optional[BatchLike]]:
while len(self.__batch_data[port]) != 0:
while len(self._get_batch_data()[port]) != 0:
yield from self._process_batch(port)

@abstractmethod
Expand Down Expand Up @@ -270,14 +293,21 @@ def __init__(self):
self._Operator__internal_is_source: bool = False
self.__table_data: Mapping[int, List[Tuple]] = defaultdict(list)

def _get_table_data(self) -> Mapping[int, List[Tuple]]:
# Created lazily so a subclass whose __init__ skips super().__init__()
# still works, like Operator._get_template_decoder.
if not hasattr(self, "_TableOperator__table_data"):
self.__table_data = defaultdict(list)
return self.__table_data

@overrides.final
def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:
self.__table_data[port].append(tuple_)
self._get_table_data()[port].append(tuple_)
yield

def on_finish(self, port: int) -> Iterator[Optional[TableLike]]:
table = Table(self.__table_data[port])
yield from self.process_table(table, port)
table = Table(self._get_table_data()[port])
yield from require_yielded(self.process_table(table, port), "process_table")

@abstractmethod
def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
Expand Down
30 changes: 26 additions & 4 deletions amber/src/main/python/core/runnables/data_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,15 @@
# under the License.

from contextlib import contextmanager

import pandas
from threading import Event
from typing import Iterator, Optional

from core.architecture.managers import Context
from core.models import State, TupleLike
from core.models.internal_marker import EndChannel, PortMarker, StartChannel
from core.models.operator import require_yielded
from core.models.table import all_output_to_tuple
from core.util import Stoppable
from core.util.console_message.replace_print import replace_print
Expand Down Expand Up @@ -94,7 +97,11 @@ def process_tuple(self) -> None:
while not finished_current.is_set():
with self._executor_session() as (executor, port_id):
tuple_ = self._context.tuple_processing_manager.get_input_tuple()
self._set_output_tuple(executor.process_tuple(tuple_, port_id))
self._set_output_tuple(
require_yielded(
executor.process_tuple(tuple_, port_id), "process_tuple"
)
)

@contextmanager
def _executor_session(self, marker_port_id: int | None = None):
Expand Down Expand Up @@ -130,11 +137,26 @@ def _set_output_tuple(self, output_iterator: Iterator[Optional[TupleLike]]) -> N
"""
for output in output_iterator:
# output could be a None, a TupleLike, or a TableLike.
if isinstance(output, dict) and any(
isinstance(value, pandas.DataFrame) for value in output.values()
):
raise TypeError(
"A dict of DataFrames was yielded: yield the DataFrame itself, "
"not a dict of DataFrames."
)
for output_tuple in all_output_to_tuple(output):
if output_tuple is not None:
output_tuple.finalize(
self._context.output_manager.get_port().get_schema()
)
try:
output_tuple.finalize(
self._context.output_manager.get_port().get_schema()
)
except KeyError as error:
raise KeyError(
f"{error.args[0]} The yielded columns must match the "
"operator's output columns. For a Python UDF, check "
"'Retain input columns' and 'Extra output column(s)' "
"in the property panel."
) from error
self._switch_context()
self._context.tuple_processing_manager.current_output_tuple = (
output_tuple
Expand Down
Loading
Loading