Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,70 @@ def test_close_is_noop_when_tmp_dir_never_materialized(self):
assert "tmp_dir" not in manager.__dict__


# A reduced copy of the default Python UDF template (PythonUDFOpDescV2.
# defaultValue): the import is active and every class is commented out. The
# Scala specs check the real template text; this checks how it loads.
DEFAULT_TEMPLATE_CODE = """# Keep exactly ONE class below: uncomment it and delete the others.

from pytexera import *

# class ProcessTupleOperator(UDFOperatorV2):
#
# @overrides
# def process_tuple(self, tuple_: Tuple, port: int) -> Iterator[Optional[TupleLike]]:
# yield tuple_
#
# class ProcessBatchOperator(UDFBatchOperator):
# BATCH_SIZE = 10 # must be a positive integer
#
# @overrides
# def process_batch(self, batch: Batch, port: int) -> Iterator[Optional[BatchLike]]:
# yield batch
#
# class ProcessTableOperator(UDFTableOperator):
#
# @overrides
# def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
# yield table
"""


def uncomment_class(code: str, class_name: str) -> str:
"""Uncomment one `# class <name>` block of a template, as a user would."""
lines, in_block = [], False
for line in code.splitlines():
if line.startswith("# class "):
in_block = line.startswith(f"# class {class_name}")
if in_block and line.startswith("#"):
line = line[1:].removeprefix(" ")
lines.append(line)
return "\n".join(lines) + "\n"


class TestDefaultTemplate:
@pytest.fixture
def executor_manager(self):
manager = ExecutorManager()
yield manager
manager.close()

def test_untouched_template_reports_no_operator(self, executor_manager):
with pytest.raises(AssertionError) as exc_info:
executor_manager.load_executor_definition(DEFAULT_TEMPLATE_CODE)
assert "one and only one Operator" in str(exc_info.value)

@pytest.mark.parametrize(
"class_name",
["ProcessTupleOperator", "ProcessBatchOperator", "ProcessTableOperator"],
)
def test_uncommenting_one_class_loads_without_adding_an_import(
self, executor_manager, class_name
):
code = uncomment_class(DEFAULT_TEMPLATE_CODE, class_name)
executor = executor_manager.load_executor_definition(code)
assert executor.__name__ == class_name


REPLACEMENT_OPERATOR_CODE = """
from pytexera import *

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,23 @@ class DualInputPortsPythonUDFOpDescV2 extends LogicalOp with PythonUdfUiParamete
@JsonProperty(
required = true,
defaultValue =
"# Choose from the following templates:\n" +
"# Keep exactly ONE class below: uncomment it and delete the others.\n" +
"# Its method must match its base class:\n" +
"# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch,\n" +
"# UDFTableOperator -> process_table\n" +
"# Always `yield` results (never `return` a value). Put setup code in open();\n" +
"# if you override __init__, call super().__init__() first.\n" +
"# \n" +
"# Port 0 (model) is read completely before port 1 (tuples) starts.\n" +
"# Use the `port` argument to tell them apart, e.g. `if port == 0:`.\n" +
"# \n" +
"# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.\n" +
"# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" +
"# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.\n" +
"# See the Python UDF operator documentation for supported types and behavior.\n" +
"# \n" +
"# from pytexera import *\n" +
"# \n" +
"\n" +
"from pytexera import *\n" +
"\n" +
"# class ProcessTupleOperator(UDFOperatorV2):\n" +
"# \n" +
"# @overrides\n" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,20 @@ class PythonUDFOpDescV2 extends LogicalOp with PythonUdfUiParameterSupport {
@JsonProperty(
required = true,
defaultValue =
"# Choose from the following templates:\n" +
"# Keep exactly ONE class below: uncomment it and delete the others.\n" +
"# Its method must match its base class:\n" +
"# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch,\n" +
"# UDFTableOperator -> process_table\n" +
"# Always `yield` results (never `return` a value). Put setup code in open();\n" +
"# if you override __init__, call super().__init__() first.\n" +
"# \n" +
"# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.\n" +
"# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" +
"# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.\n" +
"# See the Python UDF operator documentation for supported types and behavior.\n" +
"# \n" +
"# from pytexera import *\n" +
"# \n" +
"\n" +
"from pytexera import *\n" +
"\n" +
"# class ProcessTupleOperator(UDFOperatorV2):\n" +
"# \n" +
"# @overrides\n" +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,22 @@ class PythonUDFSourceOpDescV2 extends SourceOperatorDescriptor with PythonUdfUiP

@JsonProperty(
required = true,
defaultValue = "# Define UiParameter inside GenerateOperator.open().\n" +
defaultValue = "# Uncomment the class below. Its method must match its base class:\n" +
"# UDFSourceOperator -> produce\n" +
"# Always `yield` results (never `return` a value). Put setup code in open();\n" +
"# if you override __init__, call super().__init__() first.\n" +
"# Keep exactly ONE class in this script.\n" +
"# \n" +
"# Define UiParameter inside GenerateOperator.open().\n" +
"# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" +
"# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.\n" +
"# See the Python UDF operator documentation for supported types and behavior.\n" +
"# \n" +
"# from pytexera import *\n" +
"\n" +
"from pytexera import *\n" +
"\n" +
"# class GenerateOperator(UDFSourceOperator):\n" +
"# \n" +
"# @overrides\n" +
"# \n" +
"# def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]:\n" +
"# yield\n"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -209,4 +209,32 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
r.uiParameters.map(_.attribute) shouldBe List(new Attribute("count", AttributeType.INTEGER))
r.uiParameters.map(_.value) shouldBe List("7")
}

private val defaultCode: String =
PythonUdfTemplateTestUtils.defaultCode(classOf[DualInputPortsPythonUDFOpDescV2])

"DualInputPortsPythonUDFOpDescV2 default code" should "keep the pytexera import active and explain the rules" in {
PythonUdfTemplateTestUtils.activeLines(defaultCode) shouldBe List("from pytexera import *")
defaultCode should include("Keep exactly ONE class")
defaultCode should include("never `return` a value")
defaultCode should include("super().__init__()")
defaultCode should include("UDFOperatorV2 -> process_tuple")
defaultCode should include("UDFBatchOperator -> process_batch")
defaultCode should include("UDFTableOperator -> process_table")
defaultCode should include("Port 0 (model) is read completely before port 1 (tuples)")
defaultCode should include("UiParameter")
}

it should "leave exactly one active class after uncommenting any one template class" in {
for (
className <- List("ProcessTupleOperator", "ProcessBatchOperator", "ProcessTableOperator")
) {
val active = PythonUdfTemplateTestUtils.activeLines(
PythonUdfTemplateTestUtils.uncommentClass(defaultCode, className)
)
active.head shouldBe "from pytexera import *"
active.count(_.startsWith("class ")) shouldBe 1
active.exists(_.startsWith(s"class $className(")) shouldBe true
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -304,4 +304,31 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
case other => fail(s"expected OpExecWithCode, got $other")
}
}

private val defaultCode: String =
PythonUdfTemplateTestUtils.defaultCode(classOf[PythonUDFOpDescV2])

"PythonUDFOpDescV2 default code" should "keep the pytexera import active and explain the rules" in {
PythonUdfTemplateTestUtils.activeLines(defaultCode) shouldBe List("from pytexera import *")
defaultCode should include("Keep exactly ONE class")
defaultCode should include("never `return` a value")
defaultCode should include("super().__init__()")
defaultCode should include("UDFOperatorV2 -> process_tuple")
defaultCode should include("UDFBatchOperator -> process_batch")
defaultCode should include("UDFTableOperator -> process_table")
defaultCode should include("UiParameter")
}

it should "leave exactly one active class after uncommenting any one template class" in {
for (
className <- List("ProcessTupleOperator", "ProcessBatchOperator", "ProcessTableOperator")
) {
val active = PythonUdfTemplateTestUtils.activeLines(
PythonUdfTemplateTestUtils.uncommentClass(defaultCode, className)
)
active.head shouldBe "from pytexera import *"
active.count(_.startsWith("class ")) shouldBe 1
active.exists(_.startsWith(s"class $className(")) shouldBe true
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.texera.amber.operator.udf.python

import com.fasterxml.jackson.annotation.JsonProperty

/**
* Shared helpers for the Python UDF template specs: read a descriptor's default
* code template and simulate a user uncommenting one of its classes.
*/
object PythonUdfTemplateTestUtils {

/** The `defaultValue` of the descriptor's `code` property, i.e. the template. */
def defaultCode(descriptorClass: Class[_]): String =
descriptorClass
.getDeclaredField("code")
.getAnnotation(classOf[JsonProperty])
.defaultValue()

/** Lines that are neither blank nor comments. */
def activeLines(code: String): List[String] =
code.linesIterator.map(_.trim).filterNot(l => l.isEmpty || l.startsWith("#")).toList

/** Uncomment the template lines from `# class <name>` up to the next commented class. */
def uncommentClass(code: String, className: String): String = {
var inBlock = false
code.linesIterator
.map { line =>
if (line.startsWith("# class ")) inBlock = line.startsWith(s"# class $className")
if (inBlock && line.startsWith("#")) line.stripPrefix("#").stripPrefix(" ") else line
}
.mkString("\n")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import org.apache.texera.amber.operator.LogicalOp
import org.apache.texera.amber.operator.metadata.OperatorGroupConstants
import org.apache.texera.amber.operator.udf.python.UiUDFParameter
import org.apache.texera.amber.util.JSONUtils.objectMapper
import org.apache.texera.amber.operator.udf.python.PythonUdfTemplateTestUtils
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers

Expand Down Expand Up @@ -159,4 +160,27 @@ class PythonUDFSourceOpDescV2Spec extends AnyFlatSpec with Matchers {
p.uiParameters.map(_.attribute) shouldBe List(new Attribute("count", AttributeType.INTEGER))
p.uiParameters.map(_.value) shouldBe List("7")
}

private val defaultCode: String =
PythonUdfTemplateTestUtils.defaultCode(classOf[PythonUDFSourceOpDescV2])

"PythonUDFSourceOpDescV2 default code" should "keep the pytexera import active and explain the rules" in {
PythonUdfTemplateTestUtils.activeLines(defaultCode) shouldBe List("from pytexera import *")
defaultCode should include("Keep exactly ONE class")
defaultCode should include("never `return` a value")
defaultCode should include("super().__init__()")
defaultCode should include("UDFSourceOperator -> produce")
defaultCode should include("UiParameter")
}

it should "leave exactly one active class after uncommenting any one template class" in {
for (className <- List("GenerateOperator")) {
val active = PythonUdfTemplateTestUtils.activeLines(
PythonUdfTemplateTestUtils.uncommentClass(defaultCode, className)
)
active.head shouldBe "from pytexera import *"
active.count(_.startsWith("class ")) shouldBe 1
active.exists(_.startsWith(s"class $className(")) shouldBe true
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,22 @@ tags: [user-defined-functions, python]
**Python script**

```python
# Uncomment the class below. Its method must match its base class:
# UDFSourceOperator -> produce
# Always `yield` results (never `return` a value). Put setup code in open();
# if you override __init__, call super().__init__() first.
# Keep exactly ONE class in this script.
#
# Define UiParameter inside GenerateOperator.open().
# Example: self.count = self.UiParameter("count", AttributeType.INT).value
# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.
# See the Python UDF operator documentation for supported types and behavior.
#
# from pytexera import *

from pytexera import *

# class GenerateOperator(UDFSourceOperator):
#
# @overrides
#
# def produce(self) -> Iterator[Union[TupleLike, TableLike, None]]:
# yield

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,23 @@ tags: [user-defined-functions, python]
**Python script**

```python
# Choose from the following templates:
#
# Keep exactly ONE class below: uncomment it and delete the others.
# Its method must match its base class:
# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch,
# UDFTableOperator -> process_table
# Always `yield` results (never `return` a value). Put setup code in open();
# if you override __init__, call super().__init__() first.
#
# Port 0 (model) is read completely before port 1 (tuples) starts.
# Use the `port` argument to tell them apart, e.g. `if port == 0:`.
#
# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.
# Example: self.count = self.UiParameter("count", AttributeType.INT).value
# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.
# See the Python UDF operator documentation for supported types and behavior.
#
# from pytexera import *
#

from pytexera import *

# class ProcessTupleOperator(UDFOperatorV2):
#
# @overrides
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,20 @@ tags: [user-defined-functions, python]
**Python script**

```python
# Choose from the following templates:
#
# Keep exactly ONE class below: uncomment it and delete the others.
# Its method must match its base class:
# UDFOperatorV2 -> process_tuple, UDFBatchOperator -> process_batch,
# UDFTableOperator -> process_table
# Always `yield` results (never `return` a value). Put setup code in open();
# if you override __init__, call super().__init__() first.
#
# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.
# Example: self.count = self.UiParameter("count", AttributeType.INT).value
# Add value=Resource.MODEL or Resource.DATASET to pick a version; the value is its mount directory.
# See the Python UDF operator documentation for supported types and behavior.
#
# from pytexera import *
#

from pytexera import *

# class ProcessTupleOperator(UDFOperatorV2):
#
# @overrides
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ Batch API consumes a batch of tuples at a time. Similar to Table, a Batch is als
All three APIs can return an empty iterator by yield None.

The template code for a Python UDF follows: MAKE SURE TO USE THE CLASS NAMES AND FUNCTIONS DEFINED, THIS IS A MUST FOR THE PROGRAM TO WORK. SELECT 1 OUT OF THE 3 PROCESSING OPERATOR FUNCTIONS TO BUILD DEPENDING ON THE CONTEXT OF CODE TRANSLATION.
# Choose from the following templates:
#
# from pytexera import *
# Keep exactly ONE class below: uncomment it and delete the others.
#
from pytexera import *
#
# class ProcessTupleOperator(UDFOperatorV2):
#
Expand Down
Loading