diff --git a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyServer.scala b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyServer.scala index 82b5cea85e2..019303a505b 100644 --- a/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyServer.scala +++ b/amber/src/main/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyServer.scala @@ -145,9 +145,10 @@ private class AmberProducer( ) ) case _ => // normal data batches + val schema = ArrowUtils.toTexeraSchema(root.getSchema) val queue = mutable.Queue[Tuple]() for (i <- 0 until root.getRowCount) - queue.enqueue(ArrowUtils.getTexeraTuple(i, root)) + queue.enqueue(ArrowUtils.getTexeraTuple(i, root, schema)) outputPort.sendTo(to, DataFrame(queue.toArray)) } } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyServerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyServerSpec.scala index 1e4c531a0b5..a8350fc3277 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyServerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/pythonworker/PythonProxyServerSpec.scala @@ -247,6 +247,22 @@ class PythonProxyServerSpec extends AnyFlatSpec { } } + it should "deliver an empty Data put as an empty DataFrame" in { + withServer { fixture => + val to = + ChannelIdentity(pythonWorkerId, ActorVirtualIdentity("downstream"), isControl = false) + val schema = Schema().add(new Attribute("v", AttributeType.INTEGER)) + + putBatch(fixture, PythonDataHeader(to, "Data"), ArrowUtils.fromTexeraSchema(schema), _ => ()) + + awaitTrue()(fixture.messages.nonEmpty) + val out = fixture.messages + assert(out.size == 1) + assert(out.head.channelId == to) + assert(out.head.payload == DataFrame(Array.empty[Tuple])) + } + } + it should "reassemble a single-row State put into a StateFrame carrying the loop envelope" in { withServer { fixture => val to = diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Schema.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Schema.scala index 1dd38fbfe47..2f8df0cd85d 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Schema.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Schema.scala @@ -90,7 +90,7 @@ case class Schema @JsonCreator() ( } } - override def toString: String = s"Schema[${attributes.map(_.toString).mkString(", ")}]" + override def toString: String = s"Schema[${attributes.mkString(", ")}]" /** * Creates a new Schema containing only the specified attributes. @@ -113,7 +113,7 @@ case class Schema @JsonCreator() ( * Throws an exception if any attribute name already exists in the schema. */ def add(attributesToAdd: Iterable[Attribute]): Schema = { - val existingNames = this.getAttributeNames.map(_.toLowerCase).toSet + val existingNames = attributeIndex.keySet val duplicateNames = attributesToAdd.map(_.getName.toLowerCase).toSet.intersect(existingNames) if (duplicateNames.nonEmpty) { @@ -171,7 +171,7 @@ case class Schema @JsonCreator() ( val attributesToRemove = attributeNames.map(_.toLowerCase).toSet // Check for non-existent attributes - val nonExistentAttributes = attributesToRemove.diff(attributes.map(_.getName.toLowerCase).toSet) + val nonExistentAttributes = attributesToRemove.diff(attributeIndex.keySet) if (nonExistentAttributes.nonEmpty) { throw new IllegalArgumentException( s"Cannot remove non-existent attributes: ${nonExistentAttributes.mkString(", ")}" diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Tuple.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Tuple.scala index f7a962eb4ad..f419d6aec8a 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Tuple.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Tuple.scala @@ -83,25 +83,35 @@ case class Tuple @JsonCreator() ( this } - override def hashCode: Int = util.Arrays.deepHashCode(getFields.map(_.asInstanceOf[AnyRef])) + override def hashCode: Int = util.Arrays.deepHashCode(getFields.asInstanceOf[Array[AnyRef]]) override def equals(obj: Any): Boolean = obj match { - case that: Tuple => - this.schema == that.schema && - this.getFields.zip(that.getFields).forall { - case (field1: Array[Byte], field2: Array[Byte]) => - field1.sameElements(field2) // for Binary, use sameElements instead of == to compare - case (field1, field2) => - field1 == field2 - } + case that: Tuple if this.schema == that.schema => + val left = this.getFields + val right = that.getFields + val fieldCount = math.min(left.length, right.length) + var index = 0 + while (index < fieldCount && fieldEquals(left(index), right(index))) index += 1 + index == fieldCount case _ => false } + private def fieldEquals(left: Any, right: Any): Boolean = + if (left.isInstanceOf[Array[Byte]] && right.isInstanceOf[Array[Byte]]) + util.Arrays.equals(left.asInstanceOf[Array[Byte]], right.asInstanceOf[Array[Byte]]) + else left == right + def getPartialTuple(attributeNames: List[String]): Tuple = { val partialSchema = schema.getPartialSchema(attributeNames) val builder = Tuple.Builder(partialSchema) - val partialArray = attributeNames.map(getField[Any]).toArray + val partialArray = new Array[Any](attributeNames.length) + val names = attributeNames.iterator + var index = 0 + while (names.hasNext) { + partialArray(index) = getField[Any](names.next()) + index += 1 + } builder.addSequentially(partialArray) builder.build() } @@ -132,10 +142,8 @@ object Tuple { ) } - (attributeList zip fieldList).foreach { - case (attribute, field) => - checkAttributeMatchesField(attribute, field) - } + val fieldIterator = fieldList.iterator + attributeList.foreach(attribute => checkAttributeMatchesField(attribute, fieldIterator.next())) } /** @@ -177,14 +185,12 @@ object Tuple { def add(tuple: Tuple, isStrictSchemaMatch: Boolean = true): Builder = { require(tuple != null, "Tuple cannot be null") - tuple.getFields.zipWithIndex.foreach { - case (field, i) => - val attribute = tuple.schema.getAttributes(i) - if (!isStrictSchemaMatch && !schema.containsAttribute(attribute.getName)) { - // Skip if not matching in non-strict mode - } else { - add(attribute, tuple.getFields(i)) - } + val attributes = tuple.schema.getAttributes.iterator + tuple.getFields.foreach { field => + val attribute = attributes.next() + if (isStrictSchemaMatch || schema.containsAttribute(attribute.getName)) { + this.add(attribute, field) + } } this } @@ -215,10 +221,8 @@ object Tuple { def addSequentially(fields: Array[Any]): Builder = { require(fields != null, "Fields cannot be null") checkSchemaMatchesFields(schema.getAttributes, fields) - schema.getAttributes.zip(fields).foreach { - case (attribute, field) => - this.add(attribute, field) - } + val attributes = schema.getAttributes.iterator + fields.foreach(field => this.add(attributes.next(), field)) this } @@ -231,8 +235,14 @@ object Tuple { ) } - val fields = - schema.getAttributes.map(attr => fieldNameMap(attr.getName.toLowerCase)).toArray + val attributes = schema.getAttributes + val fields = new Array[Any](attributes.length) + val attributeIterator = attributes.iterator + var index = 0 + while (attributeIterator.hasNext) { + fields(index) = fieldNameMap(attributeIterator.next().getName.toLowerCase) + index += 1 + } new Tuple(schema, fields) } } diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/TupleLike.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/TupleLike.scala index 040b94977a1..6dffb604e2c 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/TupleLike.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/TupleLike.scala @@ -57,9 +57,13 @@ trait SeqTupleLike extends TupleLike with SchemaEnforceable { override def enforceSchema(schema: Schema): Tuple = { val attributes = schema.getAttributes val builder = Tuple.builder(schema) - getFields.zipWithIndex.foreach { - case (value, i) => - builder.add(attributes(i), value) + val fields = getFields + val attributeIterator = attributes.iterator + var index = 0 + while (index < fields.length) { + val attribute = if (attributeIterator.hasNext) attributeIterator.next() else attributes(index) + builder.add(attribute, fields(index)) + index += 1 } builder.build() } diff --git a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala index 5678948bc0d..d8bcca13d84 100644 --- a/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala +++ b/common/workflow-core/src/main/scala/org/apache/texera/amber/util/ArrowUtils.scala @@ -70,41 +70,55 @@ object ArrowUtils extends LazyLogging { def getTexeraTuple( rowIndex: Int, vectorSchemaRoot: VectorSchemaRoot + ): Tuple = getTexeraTuple(rowIndex, vectorSchemaRoot, toTexeraSchema(vectorSchemaRoot.getSchema)) + + /** + * Reads a row like `getTexeraTuple(rowIndex, vectorSchemaRoot)`, given the root's schema + * already converted by `toTexeraSchema`, so a caller reading many rows of one root converts + * its schema once instead of once per row. + * + * @param schema `toTexeraSchema(vectorSchemaRoot.getSchema)`. + */ + def getTexeraTuple( + rowIndex: Int, + vectorSchemaRoot: VectorSchemaRoot, + schema: Schema ): Tuple = { - val arrowSchema = vectorSchemaRoot.getSchema - val schema = toTexeraSchema(arrowSchema) - - Tuple - .builder(schema) - .addSequentially( - vectorSchemaRoot.getFieldVectors.asScala.zipWithIndex.map { - case (fieldVector: FieldVector, index: Int) => - val value: AnyRef = fieldVector.getObject(rowIndex) - try { - // Use the attribute type from the schema (which includes metadata) - // instead of deriving it from the Arrow type - val attributeType = schema.getAttributes(index).getType - // A timestamp and an unsigned integer are the types whose field - // says more than the schema does, so they are the only ones that - // read the field: an unsigned column arrives as INTEGER or LONG. - attributeType match { - case AttributeType.TIMESTAMP => wallClockOf(value, fieldVector.getField.getType) - case AttributeType.INTEGER | AttributeType.LONG => - AttributeTypeUtils.parseField( - unsignedValueOf(value, fieldVector.getField.getType), - attributeType - ) - case _ => AttributeTypeUtils.parseField(value, attributeType) - } - } catch { - case e: Exception => - logger.warn("Caught error during parsing Arrow value back to Texera value", e) - null - } - - }.toArray - ) - .build() + val fieldVectors = vectorSchemaRoot.getFieldVectors + val fieldValues = new Array[Any](fieldVectors.size()) + val vectors = fieldVectors.iterator() + val attributes = schema.getAttributes.iterator + var index = 0 + while (vectors.hasNext) { + val fieldVector = vectors.next() + val value: AnyRef = fieldVector.getObject(rowIndex) + val fieldValue = + try { + // Use the attribute type from the schema (which includes metadata) + // instead of deriving it from the Arrow type + val attributeType = attributes.next().getType + // A timestamp and an unsigned integer are the types whose field + // says more than the schema does, so they are the only ones that + // read the field: an unsigned column arrives as INTEGER or LONG. + attributeType match { + case AttributeType.TIMESTAMP => wallClockOf(value, fieldVector.getField.getType) + case AttributeType.INTEGER | AttributeType.LONG => + AttributeTypeUtils.parseField( + unsignedValueOf(value, fieldVector.getField.getType), + attributeType + ) + case _ => AttributeTypeUtils.parseField(value, attributeType) + } + } catch { + case e: Exception => + logger.warn("Caught error during parsing Arrow value back to Texera value", e) + null + } + fieldValues(index) = fieldValue + index += 1 + } + + Tuple.builder(schema).addSequentially(fieldValues).build() } /** The number an unsigned column counts to, out of the storage it counts in. @@ -192,7 +206,7 @@ object ArrowUtils extends LazyLogging { */ def toTexeraSchema(arrowSchema: org.apache.arrow.vector.types.pojo.Schema): Schema = Schema( - arrowSchema.getFields.asScala.map { field => + arrowSchema.getFields.asScala.iterator.map { field => val taggedType = Option(field.getMetadata) .flatMap(m => Option(m.get(TexeraTypeMetadataKey))) .collect { @@ -259,9 +273,7 @@ object ArrowUtils extends LazyLogging { } def appendTexeraTuple(tuple: Tuple, vectorSchemaRoot: VectorSchemaRoot): Unit = { - val currentRowCount = vectorSchemaRoot.getRowCount - val nextRowIndex = currentRowCount - setTexeraTuple(tuple, nextRowIndex, vectorSchemaRoot) + setTexeraTuple(tuple, vectorSchemaRoot.getRowCount, vectorSchemaRoot) } /** @@ -274,14 +286,13 @@ object ArrowUtils extends LazyLogging { * multiple Vectors. */ def setTexeraTuple(tuple: Tuple, index: Int, vectorSchemaRoot: VectorSchemaRoot): Unit = { - val arrowSchema = vectorSchemaRoot.getSchema - val arrowFields = arrowSchema.getFields.asScala.toList + val arrowFields = vectorSchemaRoot.getSchema.getFields - for (i <- arrowFields.indices) { + for (i <- 0 until arrowFields.size()) { val vector: FieldVector = vectorSchemaRoot.getVector(i) val value = tuple.getField[AnyRef](i) val isNull = value == null - arrowFields.apply(i).getFieldType.getType match { + arrowFields.get(i).getFieldType.getType match { case _: ArrowType.Int => vector.getField.getFieldType.getType.asInstanceOf[ArrowType.Int].getBitWidth match { case 16 | 32 => @@ -363,7 +374,9 @@ object ArrowUtils extends LazyLogging { * @return An Arrow Schema. */ def fromTexeraSchema(schema: Schema): org.apache.arrow.vector.types.pojo.Schema = { - val arrowFields = schema.getAttributes.map { attribute => + val attributes = schema.getAttributes + val arrowFields = new util.ArrayList[Field](attributes.size) + attributes.foreach { attribute => val metadataTag = attribute.getType match { case AttributeType.LARGE_BINARY => "LARGE_BINARY" case AttributeType.ANY => "ANY" @@ -375,14 +388,16 @@ object ArrowUtils extends LazyLogging { map } else null - new Field( - attribute.getName, - new FieldType(true, fromAttributeType(attribute.getType), null, metadata), - null + arrowFields.add( + new Field( + attribute.getName, + new FieldType(true, fromAttributeType(attribute.getType), null, metadata), + null + ) ) } - new org.apache.arrow.vector.types.pojo.Schema(util.Arrays.asList(arrowFields: _*)) + new org.apache.arrow.vector.types.pojo.Schema(arrowFields) } /** diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/SchemaSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/SchemaSpec.scala index cf62f7ba156..4d3b4917d12 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/SchemaSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/SchemaSpec.scala @@ -322,4 +322,30 @@ class SchemaSpec extends AnyFlatSpec { val restored = objectMapper.readValue(objectMapper.writeValueAsString(schema), classOf[Schema]) assert(restored == schema) } + + it should "reject duplicate adds and remove attributes case-insensitively while failing on absent removals" in { + val schema = Schema( + List( + new Attribute("Id", AttributeType.INTEGER), + new Attribute("Name", AttributeType.STRING) + ) + ) + val dupEx = intercept[RuntimeException] { + schema.add( + List( + new Attribute("ID", AttributeType.STRING), + new Attribute("Extra", AttributeType.STRING) + ) + ) + } + assert(dupEx.getMessage == "Cannot add attributes with duplicate names: id") + + val removed = schema.remove(List("ID", "name")) + assert(removed.getAttributes.isEmpty) + + val missingEx = intercept[IllegalArgumentException] { + schema.remove(List("ID", "MissingCol")) + } + assert(missingEx.getMessage == "Cannot remove non-existent attributes: missingcol") + } } diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/TupleLikeSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/TupleLikeSpec.scala index 1d259fc7bf1..528be95f07a 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/TupleLikeSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/TupleLikeSpec.scala @@ -67,6 +67,13 @@ class TupleLikeSpec extends AnyFlatSpec { assertThrows[IndexOutOfBoundsException](TupleLike(42, "hello", true).enforceSchema(schema)) } + it should "enforce null positional fields and reject field type mismatches" in { + val tuple = TupleLike(null, "hello").enforceSchema(schema) + assert(tuple.getField[Any]("col-int") == null) + assert(tuple.getField[String]("col-string") == "hello") + assertThrows[RuntimeException](TupleLike("wrong-type", "hello").enforceSchema(schema)) + } + // ----- MapTupleLike ----- "MapTupleLike" should "expose its mapping values as fields and not implement inMemSize" in { diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/TupleSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/TupleSpec.scala index 3471d740048..e958cd48481 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/TupleSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/core/tuple/TupleSpec.scala @@ -391,4 +391,39 @@ class TupleSpec extends AnyFlatSpec { tuple.getPartialTuple(List("col-missing")) } } + + it should "preserve valid fields and compare null vs binary fields safely across equality and non-strict builds" in { + val binSchema = Schema().add(stringAttribute).add(binaryAttribute) + val withNull = + Tuple.builder(binSchema).add(stringAttribute, "a").add(binaryAttribute, null).build() + val otherNull = + Tuple.builder(binSchema).add(stringAttribute, "a").add(binaryAttribute, null).build() + val withBytes = + Tuple + .builder(binSchema) + .add(stringAttribute, "a") + .add(binaryAttribute, Array[Byte](1, 2)) + .build() + val diffString = + Tuple + .builder(binSchema) + .add(stringAttribute, "b") + .add(binaryAttribute, Array[Byte](1, 2)) + .build() + + assert(withNull == otherNull) + assert(withNull != withBytes) + assert(withBytes != withNull) + assert(withBytes != diffString) + + val narrowedSchema = Schema().add(stringAttribute) + val nonStrict = + Tuple.builder(narrowedSchema).add(withBytes, isStrictSchemaMatch = false).build() + assert(nonStrict.getSchema == narrowedSchema) + assert(nonStrict.getField[String]("col-string") == "a") + + val reordered = withBytes.getPartialTuple(List("col-binary", "col-string")) + assert(reordered.getField[Array[Byte]](0).sameElements(Array[Byte](1, 2))) + assert(reordered.getField[String](1) == "a") + } } diff --git a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala index e157f9892eb..6ce294e2a0b 100644 --- a/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala +++ b/common/workflow-core/src/test/scala/org/apache/texera/amber/util/ArrowUtilsSpec.scala @@ -420,6 +420,58 @@ class ArrowUtilsSpec extends AnyFlatSpec with Matchers { } } + it should "read every row with a schema converted once as it does converting per row" in { + val tuples = Seq( + Array[Any]( + Int.box(1), + Long.box(10L), + Double.box(1.5), + Boolean.box(true), + new Timestamp(1000L), + "a", + Array[Byte](1) + ), + Array[Any](null, null, null, null, null, null, null), + Array[Any]( + Int.box(-3), + Long.box(Long.MaxValue), + Double.box(-0.25), + Boolean.box(false), + new Timestamp(-5000L), + "b", + Array[Byte](2, 3) + ) + ).map(fields => Tuple.builder(tupleSchema).addSequentially(fields).build()) + withRoot(tupleSchema) { root => + tuples.foreach(ArrowUtils.appendTexeraTuple(_, root)) + val schema = ArrowUtils.toTexeraSchema(root.getSchema) + tuples.indices.foreach { i => + val read = ArrowUtils.getTexeraTuple(i, root, schema) + read shouldBe ArrowUtils.getTexeraTuple(i, root) + read shouldBe tuples(i) + } + } + } + + it should "reject a converted schema narrower or wider than the root" in { + val rootSchema = Schema( + List(new Attribute("i", AttributeType.INTEGER), new Attribute("s", AttributeType.STRING)) + ) + val narrower = Schema(List(new Attribute("i", AttributeType.INTEGER))) + val wider = rootSchema.add(new Attribute("extra", AttributeType.STRING)) + withRoot(rootSchema) { root => + ArrowUtils.appendTexeraTuple( + Tuple.builder(rootSchema).addSequentially(Array[Any](Int.box(1), "a")).build(), + root + ) + Seq(narrower, wider).foreach { schema => + intercept[RuntimeException] { + ArrowUtils.getTexeraTuple(0, root, schema) + }.getMessage should include("are different") + } + } + } + // ----- Tuple <-> Arrow data round-trip (the State wire-hop contract) ----- "tuple round-trip through Arrow vectors" should "preserve every column of a multi-column State tuple" in { diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpDesc.scala index 0bd9f5c79d5..2222a05756e 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpDesc.scala @@ -108,7 +108,7 @@ class DictionaryMatcherOpDesc extends MapOpDesc with StandaloneCodeGenerator { // JVM splits the dictionary on "," and lowercases each entry — no trim, // so leading/trailing whitespace around a comma becomes part of the entry. val rawDict = Option(dictionary).getOrElse("") - val entries = rawDict.split(",").toList.map(_.toLowerCase) + val entries = rawDict.split(",").iterator.map(_.toLowerCase).toList val resultCol = Option(resultAttribute) .filter(_.trim.nonEmpty) .getOrElse("matched") @@ -119,10 +119,9 @@ class DictionaryMatcherOpDesc extends MapOpDesc with StandaloneCodeGenerator { val resultPy = pyStringLiteral(resultCol) val mt = Option(matchingType).getOrElse(MatchingType.SCANBASED) - val entriesLiteral = entries.map(pyStringLiteral).mkString("[", ", ", "]") - mt match { case MatchingType.SCANBASED => + val entriesLiteral = DictionaryMatcherOpDesc.renderEntries(entries) // Exact case-insensitive equality between the (lowercased) cell value // and any dictionary entry. Null or empty text never matches, matching // JVM behavior. @@ -135,6 +134,7 @@ class DictionaryMatcherOpDesc extends MapOpDesc with StandaloneCodeGenerator { |)""".stripMargin case MatchingType.SUBSTRING => + val entriesLiteral = DictionaryMatcherOpDesc.renderEntries(entries) // JVM checks dictionaryEntries.exists(entry => entry.contains(text)) — // the cell value (lowercased) is a substring of some dictionary entry, // NOT the other way around. @@ -153,13 +153,14 @@ class DictionaryMatcherOpDesc extends MapOpDesc with StandaloneCodeGenerator { // effort reproduction: regex \w+ tokens, lowercase, drop the same stop // word lists, but NO stemming — morphological variants ("book" vs "books") // will diverge from JVM behavior. - val tokenSetsLiteral = entries - .map(tokenizeForConjunction) - .map(toks => toks.toList.sorted.map(pyStringLiteral).mkString("frozenset({", ", ", "})")) + val tokenSetsLiteral = entries.iterator + .map { entry => + tokenizeForConjunction(entry).toList.sorted.iterator + .map(pyStringLiteral) + .mkString("frozenset({", ", ", "})") + } .mkString("[", ", ", "]") - val stopWordsLiteral = DictionaryMatcherOpDesc.STOP_WORDS.toList.sorted - .map(pyStringLiteral) - .mkString("frozenset({", ", ", "})") + val stopWordsLiteral = DictionaryMatcherOpDesc.StopWordsLiteral s"""import re as _texera_dm_re |_TEXERA_DM_STOPWORDS = $stopWordsLiteral |def _texera_dm_tokenize(text): @@ -175,15 +176,22 @@ class DictionaryMatcherOpDesc extends MapOpDesc with StandaloneCodeGenerator { } private def tokenizeForConjunction(text: String): Set[String] = { - val wordRe = "\\w+".r - wordRe + DictionaryMatcherOpDesc.WordRegex .findAllIn(text.toLowerCase) - .toSet .filterNot(DictionaryMatcherOpDesc.STOP_WORDS.contains) + .toSet } } private object DictionaryMatcherOpDesc { + private val WordRegex = "\\w+".r + + private def renderEntries(entries: List[String]): String = + entries.iterator.map(pyStringLiteral).mkString("[", ", ", "]") + + private lazy val StopWordsLiteral: String = + STOP_WORDS.toList.sorted.iterator.map(pyStringLiteral).mkString("frozenset({", ", ", "})") + // Mirrors Lucene's EnglishAnalyzer.ENGLISH_STOP_WORDS_SET plus the URL stop // words filtered by DictionaryMatcherOpExec. val STOP_WORDS: Set[String] = Set( diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpExec.scala index 13e62ec7807..3290f75f14b 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpExec.scala @@ -39,27 +39,13 @@ class DictionaryMatcherOpExec(descString: String) extends MapOpExec { var dictionaryEntries: List[String] = _ var luceneAnalyzer: Analyzer = _ - /** An unmodifiable set containing some common URL words that are not usually useful - * for searching. - */ - private final val URL_STOP_WORDS_SET = List[String]( - "http", - "https", - "org", - "net", - "com", - "store", - "www", - "html" - ) - /** * first prepare the dictionary by splitting the values using a comma delimiter then tokenize the split values * at the first tuple: inside a loop block the loop state may write the dictionary after open() */ private def prepareDictionary(): Unit = { // create the dictionary by splitting the values first - dictionaryEntries = desc.dictionary.split(",").toList.map(_.toLowerCase) + dictionaryEntries = desc.dictionary.split(",").iterator.map(_.toLowerCase).toList if (desc.matchingType == MatchingType.CONJUNCTION_INDEXBASED) { // then tokenize each entry this.luceneAnalyzer = new EnglishAnalyzer @@ -131,8 +117,8 @@ class DictionaryMatcherOpExec(descString: String) extends MapOpExec { while (tokenStream.incrementToken()) { val term = charTermAttribute.toString.toLowerCase if ( - !EnglishAnalyzer.ENGLISH_STOP_WORDS_SET.contains(term) && !URL_STOP_WORDS_SET - .contains(term) + !EnglishAnalyzer.ENGLISH_STOP_WORDS_SET.contains(term) && + !DictionaryMatcherOpExec.UrlStopWords.contains(term) ) { tokens += term } @@ -158,3 +144,16 @@ class DictionaryMatcherOpExec(descString: String) extends MapOpExec { setMapFunc(labelTupleIfMatched) } + +private object DictionaryMatcherOpExec { + private val UrlStopWords: Set[String] = Set( + "http", + "https", + "org", + "net", + "com", + "store", + "www", + "html" + ) +} diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala index 1edf0877382..12d2fda4836 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/CaseSensitiveAnalyzer.scala @@ -18,10 +18,8 @@ package org.apache.texera.amber.operator.keywordSearch -import org.apache.lucene.analysis.{Analyzer, TokenStream} +import org.apache.lucene.analysis.Analyzer import org.apache.lucene.analysis.standard.StandardTokenizer -import org.apache.lucene.analysis.CharArraySet -import org.apache.lucene.analysis.StopFilter import org.apache.lucene.analysis.Analyzer.TokenStreamComponents // Mirrors StandardAnalyzer but omits the LowerCaseFilter, so tokens keep their @@ -29,8 +27,6 @@ import org.apache.lucene.analysis.Analyzer.TokenStreamComponents // would instead glue punctuation to tokens (e.g. "perfect." would not match "perfect"). class CaseSensitiveAnalyzer extends Analyzer { override protected def createComponents(fieldName: String): TokenStreamComponents = { - val tokenizer = new StandardTokenizer() - val stream: TokenStream = new StopFilter(tokenizer, CharArraySet.EMPTY_SET) - new TokenStreamComponents(tokenizer, stream) + new TokenStreamComponents(new StandardTokenizer()) } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpDesc.scala index f54989b2c6f..0cce7ba0240 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpDesc.scala @@ -95,11 +95,12 @@ class KeywordSearchOpDesc extends FilterOpDesc with StandaloneCodeGenerator { // themselves, so a query that uses the syntax — a phrase, a boolean, a // wildcard, a fuzzy match — reads here as the words it is written with. val raw = Option(keyword).getOrElse("") - val terms = raw.trim.split("\\s+").filter(_.nonEmpty).toList - if (terms.isEmpty) return "out1df = in1df" + val terms = raw.trim.split("\\s+").iterator.filter(_.nonEmpty) + if (!terms.hasNext) return "out1df = in1df" - val regexSpecials = Set('.', '^', '$', '*', '+', '?', '(', ')', '[', ']', '{', '}', '|', '\\') - val escaped = terms.map(_.flatMap(c => if (regexSpecials.contains(c)) s"\\$c" else c.toString)) + val regexSpecials = ".^$*+?()[]{}|\\" + val escaped = + terms.map(_.flatMap(c => if (regexSpecials.indexOf(c) >= 0) s"\\$c" else c.toString)) val pattern = escaped.mkString("\\b(?:", "|", ")\\b") val pyLiteral = pyStringLiteral(pattern) // The engine reads the field through Schema, which ignores case. See [[columnAsStored]]. diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExec.scala index 99ce77ba748..4d79ce6614f 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpExec.scala @@ -45,7 +45,8 @@ class KeywordSearchOpExec(descString: String) extends FilterOpExec { this.setFilterFunc(findKeyword) private def findKeyword(tuple: Tuple): Boolean = { - Option[Any](tuple.getField(desc.attribute)).map(_.toString).exists { fieldValue => + Option[Any](tuple.getField(desc.attribute)).exists { value => + val fieldValue = value.toString memoryIndex.addField(desc.attribute, fieldValue, analyzer) val isMatch = memoryIndex.search(query) > 0.0f memoryIndex.reset() diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala index 1a8b5d79c12..8c51f1e5a73 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDesc.scala @@ -38,6 +38,11 @@ import org.apache.texera.amber.operator.metadata.annotations.{ import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo} import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.pyStringLiteral +object MachineLearningScorerOpDesc { + private val NumericTypes: Set[AttributeType] = + Set(AttributeType.INTEGER, AttributeType.LONG, AttributeType.DOUBLE) +} + // Whichever metric list the task shows must hold at least one metric: an empty // one renders as `metric_list = ['']` and the metric lookup fails on the empty // name. The rule is conditional because only one of the two lists is visible at @@ -135,33 +140,25 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with Standalo * cannot tell the two apart. */ private def validateScoredColumns(inputSchemas: Map[PortIdentity, Schema]): Unit = { - val numeric = Set(AttributeType.INTEGER, AttributeType.LONG, AttributeType.DOUBLE) + val numeric = MachineLearningScorerOpDesc.NumericTypes // Blank names are what the form reports before the user has picked; the // schema's own `required` already says so, and saying it twice would put a // second message on a half-filled operator. if (actualValueColumn.isEmpty || predictValueColumn.isEmpty) return inputSchemas.get(operatorInfo.inputPorts.head.id).foreach { schema => - Seq(("Actual Value", actualValueColumn), ("Predicted Value", predictValueColumn)) - .foreach { - case (label, column) => - if (!schema.containsAttribute(column)) - throw new RuntimeException(s"$label column '$column' is not in the input table") - } + requireScoredColumn(schema, "Actual Value", actualValueColumn) + requireScoredColumn(schema, "Predicted Value", predictValueColumn) val actualType = schema.getAttribute(actualValueColumn).getType val predictType = schema.getAttribute(predictValueColumn).getType if (isRegression) { - Seq( - ("Actual Value", actualValueColumn, actualType), - ("Predicted Value", predictValueColumn, predictType) - ) - .foreach { - case (label, column, attributeType) => - if (!numeric.contains(attributeType)) - throw new RuntimeException( - s"A regression metric needs a numeric $label column, but '$column' is " + - s"${attributeType.getName}" - ) + def requireNumeric(label: String, column: String, attrType: AttributeType): Unit = + if (!numeric.contains(attrType)) { + throw new RuntimeException( + s"A regression metric needs a numeric $label column, but '$column' is ${attrType.getName}" + ) } + requireNumeric("Actual Value", actualValueColumn, actualType) + requireNumeric("Predicted Value", predictValueColumn, predictType) } else { val comparable = (numeric.contains(actualType) && numeric.contains(predictType)) || actualType == predictType @@ -174,12 +171,12 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with Standalo // A matching pair can still be one sklearn refuses as labels. Bytes fail // every metric. Timestamps only fail the per-class ones, since Accuracy // just counts equal rows. - val perClass = classificationMetrics.filterNot(_ == classificationMetricsFnc.accuracy) val refusedBy = - if (Set(AttributeType.BINARY, AttributeType.LARGE_BINARY).contains(actualType)) + if (actualType == AttributeType.BINARY || actualType == AttributeType.LARGE_BINARY) classificationMetrics - else if (actualType == AttributeType.TIMESTAMP) perClass - else List() + else if (actualType == AttributeType.TIMESTAMP) + classificationMetrics.filterNot(_ == classificationMetricsFnc.accuracy) + else Nil if (refusedBy.nonEmpty) throw new RuntimeException( s"Actual Value '$actualValueColumn' and Predicted Value '$predictValueColumn' are " + @@ -190,23 +187,27 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with Standalo } } + private def requireScoredColumn( + schema: Schema, + label: String, + column: String + ): Unit = { + if (!schema.containsAttribute(column)) { + throw new RuntimeException(s"$label column '$column' is not in the input table") + } + } + override def getOutputSchemas( inputSchemas: Map[PortIdentity, Schema] ): Map[PortIdentity, Schema] = { validateScoredColumns(inputSchemas) - val metrics = if (isRegression) { - regressionMetrics.map(_.getName()) - } else { - classificationMetrics.map(_.getName()) - } - val baseSchema = if (!isRegression) { - Schema(List(new Attribute("Class", AttributeType.STRING))) - } else { - Schema(List()) - } - - val outputSchema = metrics.foldLeft(baseSchema) { (currentSchema, metricName) => - currentSchema.add(new Attribute(metricName, AttributeType.DOUBLE)) + val metricNames = + if (isRegression) regressionMetrics.map(_.getName()) + else classificationMetrics.map(_.getName()) + val baseSchema = + if (isRegression) Schema() else Schema(List(new Attribute("Class", AttributeType.STRING))) + val outputSchema = metricNames.foldLeft(baseSchema) { (schema, metricName) => + schema.add(new Attribute(metricName, AttributeType.DOUBLE)) } Map(operatorInfo.outputPorts.head.id -> outputSchema) @@ -224,7 +225,7 @@ class MachineLearningScorerOpDesc extends PythonOperatorDescriptor with Standalo // re-encoded as one quoted value, collapsing the list into a single element. private def getSelectedMetrics(): String = { val metric = if (isRegression) regressionMetrics else classificationMetrics - metric.map(metric => getMetricName(metric)).mkString("'", "','", "'") + metric.iterator.map(getMetricName).mkString("'", "','", "'") } override def generatePythonCode(): String = { diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/regex/RegexOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/regex/RegexOpDesc.scala index fd2009a5b65..eacb1a264ca 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/regex/RegexOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/regex/RegexOpDesc.scala @@ -125,11 +125,14 @@ object RegexOpDesc { def unicodeCase(pattern: String): String = { if (TurnsUnicodeOn.findFirstIn(pattern).isEmpty) return pattern val lead = LeadingFlags.findPrefixOf(pattern).getOrElse("") - val leadGroups = FlagGroup.findAllMatchIn(lead).toList - val leadOn = leadGroups.exists(_.group(1).contains('u')) val out = new StringBuilder - leadGroups.foreach { g => - val on = g.group(1).filterNot(_ == 'u') + var leadOn = false + var leadingVerbose = false + FlagGroup.findAllMatchIn(lead).foreach { g => + val flags = g.group(1) + leadOn = leadOn || flags.contains('u') + leadingVerbose = leadingVerbose || flags.contains('x') + val on = flags.filterNot(_ == 'u') val off = Option(g.group(2)).getOrElse("").filterNot(_ == 'u') if (on.nonEmpty || off.nonEmpty) out.append(s"(?$on${if (off.isEmpty) "" else s"-$off"})") } @@ -178,7 +181,7 @@ object RegexOpDesc { } } // Under the x flag a trailing comment would swallow the closing parenthesis. - if (leadOn) out.append(if (leadGroups.exists(_.group(1).contains('x'))) "\n)" else ")") + if (leadOn) out.append(if (leadingVerbose) "\n)" else ")") out.toString } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExec.scala index f5d2460025e..03e5a212385 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExec.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExec.scala @@ -61,6 +61,8 @@ class ArrowSourceOpExec( val rowIterator = new Iterator[TupleLike] { private var currentIndex = 0 private var currentBatchIndex = 0 + // Every batch loads into the same root, so its schema is converted once. + private lazy val schema = ArrowUtils.toTexeraSchema(root.get.getSchema) override def hasNext: Boolean = { if (root.exists(_.getRowCount > currentIndex)) { @@ -81,7 +83,7 @@ class ArrowSourceOpExec( override def next(): TupleLike = { root.map { vectorSchemaRoot => - val tuple = ArrowUtils.getTexeraTuple(currentIndex, vectorSchemaRoot) + val tuple = ArrowUtils.getTexeraTuple(currentIndex, vectorSchemaRoot, schema) currentIndex += 1 tuple }.get diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/UdfUiParameterSupport.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/UdfUiParameterSupport.scala index 8c43763b7e6..f72e36aaa52 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/UdfUiParameterSupport.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/UdfUiParameterSupport.scala @@ -39,20 +39,23 @@ trait UdfUiParameterSupport { protected def mounts: RepositoryMountManager = RepositoryMountManager protected final def injectUiParameters(code: String): String = { - resourceParameters(uiParameters).foreach(selectedVersionPath) + Option(uiParameters).getOrElse(Nil).foreach { parameter => + if (isResource(parameter)) selectedVersionPath(parameter) + } injectParameterValues(code, uiParameters) } protected final def executionBinding(code: String): Option[ExecutionTimeBinding] = - Option.when(resourceParameters(uiParameters).nonEmpty) { + Option.when(Option(uiParameters).exists(_.exists(isResource))) { // Snapshot the compiled descriptor; resolving versions stays lazy until execution. val parameters = uiParameters val language = uiParameterLanguage val mountManager = mounts new ExecutionTimeBinding { private lazy val resolved = { + val locators = Set.newBuilder[String] val bound = parameters.map { parameter => - if (!isResource(parameter)) (parameter, None) + if (!isResource(parameter)) parameter else { val (repositoryName, commitHash) = FileResolver.resolveRepositoryVersion(selectedVersionPath(parameter)) @@ -61,10 +64,11 @@ trait UdfUiParameterSupport { mounted.attribute = parameter.attribute mounted.inputType = parameter.inputType mounted.value = mountManager.mountPointOf(locator).toString - (mounted, Some(locator)) + locators += locator + mounted } } - (bound.map(_._1), bound.flatMap(_._2).toSet) + (bound, locators.result()) } override lazy val opExecInitInfo: OpExecWithCode = @@ -75,12 +79,8 @@ trait UdfUiParameterSupport { } private def isResource(parameter: UiUDFParameter): Boolean = - parameter != null && UiUdfParameterContract.resourceInputTypes.values.exists( - _ == Option(parameter.inputType).getOrElse("") - ) - - private def resourceParameters(parameters: List[UiUDFParameter]): List[UiUDFParameter] = - Option(parameters).getOrElse(Nil).filter(isResource) + parameter != null && + UiUdfParameterContract.isResourceInputType(Option(parameter.inputType).getOrElse("")) private def selectedVersionPath(parameter: UiUDFParameter): String = Option(parameter.value).map(_.trim).filter(_.nonEmpty).getOrElse { diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/UiUdfParameterContract.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/UiUdfParameterContract.scala index f9c7ea7c11a..8bcc6d68369 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/UiUdfParameterContract.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/UiUdfParameterContract.scala @@ -43,6 +43,11 @@ object UiUdfParameterContract { val javaTypes: Map[AttributeType, List[String]] = manifest.javaTypes val resourceInputTypes: Map[String, String] = manifest.resources + private lazy val resourceInputTypeNames = resourceInputTypes.values.toSet + + private[udf] def isResourceInputType(inputType: String): Boolean = + resourceInputTypeNames.contains(inputType) + def validate(uiParameters: List[UiUDFParameter]): Unit = { val names = Option(uiParameters).getOrElse(Nil).map { parameter => val attribute = Option(parameter) @@ -64,7 +69,7 @@ object UiUdfParameterContract { if (parameter.value == null) throw new RuntimeException(s"UiParameter '${attribute.getName}' value must be a string.") val resource = Option(parameter.inputType).getOrElse("") - if (resource.nonEmpty && !resourceInputTypes.values.toSet.contains(resource)) + if (resource.nonEmpty && !isResourceInputType(resource)) throw new RuntimeException(s"Unknown UiParameter resource '$resource'.") if (resource.nonEmpty && attribute.getType != AttributeType.STRING) throw new RuntimeException("Resource UiParameters must have type STRING.") diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/java/JavaUdfUiParameterInjector.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/java/JavaUdfUiParameterInjector.scala index 5588e4d76bd..bc4c872d3a7 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/java/JavaUdfUiParameterInjector.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/java/JavaUdfUiParameterInjector.scala @@ -29,7 +29,8 @@ import java.nio.charset.StandardCharsets import java.time.{LocalDateTime, ZoneOffset} import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverStyle} import java.time.temporal.ChronoField -import java.util.{Base64, Locale} +import java.util.regex.Pattern +import java.util.{Base64, Collections, Locale} import javax.lang.model.element.Modifier import javax.tools._ import scala.jdk.CollectionConverters._ @@ -41,6 +42,9 @@ object JavaUdfUiParameterInjector { private val nativeTypes = UiUdfParameterContract.javaTypes.flatMap { case (kind, names) => names.map(_ -> kind) } + private lazy val decimalPattern = + Pattern.compile("[+-]?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE][+-]?[0-9]+)?") + private lazy val compactOffsetPattern = Pattern.compile("([+-][0-9]{2})([0-9]{2})$") private val timestampFormat = new DateTimeFormatterBuilder() .append(DateTimeFormatter.ISO_LOCAL_DATE) .optionalStart() @@ -75,7 +79,7 @@ object JavaUdfUiParameterInjector { case "nan" | "+nan" | "-nan" => Double.NaN case "inf" | "+inf" | "infinity" | "+infinity" => Double.PositiveInfinity case "-inf" | "-infinity" => Double.NegativeInfinity - case value if value.matches("[+-]?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE][+-]?[0-9]+)?") => + case value if decimalPattern.matcher(value).matches() => value.toDouble case _ => refuse("requires a decimal numeric value.") } @@ -95,7 +99,7 @@ object JavaUdfUiParameterInjector { case _ => number(raw) != 0d }).toString case AttributeType.TIMESTAMP => - val normalized = raw.replace(' ', 'T').replaceFirst("([+-][0-9]{2})([0-9]{2})$", "$1:$2") + val normalized = compactOffsetPattern.matcher(raw.replace(' ', 'T')).replaceFirst("$1:$2") val parsed = timestampFormat.parse(normalized) val offset = if (parsed.isSupported(ChronoField.OFFSET_SECONDS)) ZoneOffset.from(parsed) @@ -128,19 +132,19 @@ object JavaUdfUiParameterInjector { // Keep javac's tree classes unloaded on JRE hosts, even for a bare template's comment. private object Javac { + private val sourceUri = URI.create("string:///JavaUDFOpExec.java") + private val options = Collections.singletonList("-proc:none") + def inject(source: String, parameters: List[UiUDFParameter], compiler: JavaCompiler): String = { val diagnostics = new DiagnosticCollector[JavaFileObject]() Using.resource( compiler.getStandardFileManager(diagnostics, Locale.ROOT, StandardCharsets.UTF_8) ) { manager => - val file = new SimpleJavaFileObject( - URI.create("string:///JavaUDFOpExec.java"), - JavaFileObject.Kind.SOURCE - ) { + val file = new SimpleJavaFileObject(sourceUri, JavaFileObject.Kind.SOURCE) { override def getCharContent(ignoreEncodingErrors: Boolean): CharSequence = source } val task = compiler - .getTask(null, manager, diagnostics, List("-proc:none").asJava, null, List(file).asJava) + .getTask(null, manager, diagnostics, options, null, Collections.singletonList(file)) .asInstanceOf[JavacTask] val unit = task.parse().iterator().next() if (diagnostics.getDiagnostics.asScala.exists(_.getKind == Diagnostic.Kind.ERROR)) @@ -151,44 +155,50 @@ object JavaUdfUiParameterInjector { if (classes.size != 1) refuse("declarations require one JavaUDFOpExec class.") val positions = Trees.instance(task).getSourcePositions val fields = classes.head.getMembers.asScala.collect { case field: VariableTree => field } - val supplied = parameters.map(p => p.attribute.getName -> p).toMap + val supplied = parameters.iterator.map(p => p.attribute.getName -> p).toMap val used = scala.collection.mutable.Set.empty[String] val edits = fields.flatMap { field => - val annotations = field.getModifiers.getAnnotations.asScala.filter(a => - Set("UiParameter", annotationName)(a.getAnnotationType.toString) - ) - if (annotations.isEmpty) None - else { - if (annotations.size != 1) refuse("must be declared once per field.") - val args = annotations.head.getArguments.asScala.map { - case argument: AssignmentTree => - argument.getVariable.toString -> argument.getExpression - case argument => "value" -> argument + val annotations = + field.getModifiers.getAnnotations.iterator().asScala.filter { annotation => + val name = annotation.getAnnotationType.toString + name == "UiParameter" || name == annotationName } - val values = args.map { - case (key, tree) => - tree match { - case literal: LiteralTree if literal.getValue.isInstanceOf[String] => - key -> literal.getValue.toString - case _ => refuse("annotation arguments must be string literals.") - } - }.toMap + if (!annotations.hasNext) None + else { + val annotation = annotations.next() + if (annotations.hasNext) refuse("must be declared once per field.") + val args = annotation.getArguments + val values = args + .iterator() + .asScala + .map { + case argument: AssignmentTree => + argument.getVariable.toString -> argument.getExpression + case argument => "value" -> argument + } + .map { + case (key, tree) => + tree match { + case literal: LiteralTree if literal.getValue.isInstanceOf[String] => + key -> literal.getValue.toString + case _ => refuse("annotation arguments must be string literals.") + } + } + .toMap if ( - values.size != args.size || values.keys + values.size != args.size() || values.keysIterator .exists(key => key != "value" && key != "resource") ) refuse("annotation arguments are invalid.") val name = values.getOrElse("value", "").trim if (used(name)) refuse(s"name '$name' is declared more than once.") used += name - val start = positions.getStartPosition(unit, field) + // javac includes the comma in the end position of a non-final declarator. val end = positions.getEndPosition(unit, field).toInt if ( field.getModifiers.getFlags.contains( Modifier.STATIC - ) || field.getInitializer != null || fields.count(f => - positions.getStartPosition(unit, f) == start - ) != 1 || end <= 0 || source.charAt(end - 1) != ';' + ) || field.getInitializer != null || end <= 0 || source.charAt(end - 1) != ';' ) refuse("requires a single uninitialized instance field per declaration.") val kind = @@ -203,10 +213,17 @@ object JavaUdfUiParameterInjector { Some((end - 1, initializer(parameter))) } } - if (used.toSet != supplied.keySet) refuse("saved values do not match the declared fields.") - edits.toList.sortBy(-_._1).foldLeft(source) { - case (text, (offset, value)) => - text.patch(offset, s" = $value", 0) + if (used != supplied.keySet) refuse("saved values do not match the declared fields.") + if (edits.isEmpty) source + else { + val result = new java.lang.StringBuilder(source.length) + var start = 0 + edits.sortBy(_._1).foreach { + case (offset, value) => + result.append(source, start, offset).append(" = ").append(value) + start = offset + } + result.append(source, start, source.length).toString } } } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjector.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjector.scala index 5863a95d122..d9838f72ba2 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjector.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjector.scala @@ -42,6 +42,8 @@ object PythonUdfUiParameterInjector { // Keep supported user-facing UDF class names in sync with the frontend parser. private val SupportedPythonUdfClassHeaderRegex: Regex = """(?m)^([ \t]*)class\s+(ProcessTupleOperator|ProcessBatchOperator|ProcessTableOperator|GenerateOperator)\s*\([^)]*\)\s*:\s*(?:#.*)?$""".r + private val ReservedHookRegex: Regex = + ("""(?m)^[ \t]+def\s+""" + Regex.quote(InjectedUiParametersHookMethodName) + """\s*\(""").r private def buildInjectedParameterEntry(parameter: UiUDFParameter): PythonTemplateBuilder = { val encodedValue: EncodableString = parameter.value @@ -51,7 +53,7 @@ object PythonUdfUiParameterInjector { private def buildInjectedParametersMap( uiParameters: List[UiUDFParameter] ): PythonTemplateBuilder = { - val entries = uiParameters.map(buildInjectedParameterEntry) + val entries = uiParameters.iterator.map(buildInjectedParameterEntry) entries.reduceOption((acc, entry) => acc + pyb", " + entry).getOrElse(pyb"") } @@ -107,11 +109,8 @@ object PythonUdfUiParameterInjector { code.length } - private def containsReservedHook(classBlock: String): Boolean = { - val hookRegex = - ("""(?m)^[ \t]+def\s+""" + Regex.quote(InjectedUiParametersHookMethodName) + """\s*\(""").r - hookRegex.findFirstIn(classBlock).isDefined - } + private def containsReservedHook(classBlock: String): Boolean = + ReservedHookRegex.findFirstIn(classBlock).isDefined private def injectHookIntoUserClass(userCode: String, hookMethod: String): String = { val classHeaderMatch = @@ -146,13 +145,17 @@ object PythonUdfUiParameterInjector { } private def inferClassBodyIndent(classBlock: String, classIndent: String): Option[String] = { - val lines = classBlock.split("\n", -1).toList.drop(1) - - lines.collectFirst { - case line if line.trim.nonEmpty => + var lineStart = lineEndIndex(classBlock, 0) + 1 + while (lineStart < classBlock.length) { + val lineEnd = lineEndIndex(classBlock, lineStart) + val line = classBlock.substring(lineStart, lineEnd) + if (line.trim.nonEmpty) { val leading = line.takeWhile(ch => ch == ' ' || ch == '\t') - if (leading.length > classIndent.length) leading else classIndent + " " + return Some(if (leading.length > classIndent.length) leading else classIndent + " ") + } + lineStart = lineEnd + 1 } + None } /** diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/r/RUdfUiParameterInjector.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/r/RUdfUiParameterInjector.scala index 1096108a4d5..937e4d6cad1 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/r/RUdfUiParameterInjector.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/r/RUdfUiParameterInjector.scala @@ -22,34 +22,46 @@ package org.apache.texera.amber.operator.udf.r import org.apache.texera.amber.operator.udf.{UiUDFParameter, UiUdfParameterContract} import java.nio.charset.StandardCharsets +import java.util.HexFormat /** Emits calls to the R plugin SDK; metadata policy is shared and runtime behavior stays in texera-rudf. */ object RUdfUiParameterInjector { + private val byteLiterals = HexFormat.ofDelimiter(",").withPrefix("0x") + // Encode names and values as UTF-8 bytes so user-entered text never becomes executable R syntax. private def rString(value: String): String = { val bytes = Option(value).getOrElse("").getBytes(StandardCharsets.UTF_8) - s".texera_decode(c(${bytes.map(byte => f"0x${byte & 0xff}%02x").mkString(",")}))" + s".texera_decode(c(${byteLiterals.formatHex(bytes)}))" + } + + private lazy val manifestArguments = { + def formatEntries[T](entries: Map[String, T])(render: T => String): String = + entries.toList + .sortBy(_._1) + .iterator + .map { + case (token, value) => s"$token=${rString(render(value))}" + } + .mkString(",") + + val typeEntries = formatEntries(UiUdfParameterContract.typesByToken)(_.getName) + val resourceEntries = formatEntries(UiUdfParameterContract.resourceInputTypes)(identity) + s"attribute_types=list($typeEntries), resources=list($resourceEntries)" } def inject(code: String, uiParameters: List[UiUDFParameter]): String = { val parameters = Option(uiParameters).getOrElse(Nil) UiUdfParameterContract.validate(parameters) - val names = parameters.map(_.attribute.getName) val userCode = Option(code).getOrElse("") if (parameters.isEmpty) return userCode // Unlike Rscript's file reader, rpy2 parses the string directly and rejects CRLF input. val executableCode = userCode.replace("\r\n", "\n").replace('\r', '\n') - val entries = parameters.map { parameter => - s"list(type=${rString(parameter.attribute.getType.getName)}, value=${rString(parameter.value)}, inputType=${rString(parameter.inputType)})" - } - val typeEntries = UiUdfParameterContract.typesByToken.toList.sortBy(_._1).map { - case (token, attributeType) => s"$token=${rString(attributeType.getName)}" - } - val resourceEntries = UiUdfParameterContract.resourceInputTypes.toList.sortBy(_._1).map { - case (token, value) => s"$token=${rString(value)}" - } - s"# Texera UI parameters: SDK v1\ntexera_ui_parameters({\n$executableCode\n}, parameters=setNames(list(${entries - .mkString(",")}), c(${names.map(rString).mkString(",")})), attribute_types=list(${typeEntries - .mkString(",")}), resources=list(${resourceEntries.mkString(",")}))" + val names = parameters.iterator.map(p => rString(p.attribute.getName)).mkString(",") + val entries = parameters.iterator + .map { parameter => + s"list(type=${rString(parameter.attribute.getType.getName)}, value=${rString(parameter.value)}, inputType=${rString(parameter.inputType)})" + } + .mkString(",") + s"# Texera UI parameters: SDK v1\ntexera_ui_parameters({\n$executableCode\n}, parameters=setNames(list($entries), c($names)), $manifestArguments)" } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpDescSpec.scala index edfd348c566..93ed5eef5da 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/dictionary/DictionaryMatcherOpDescSpec.scala @@ -117,4 +117,26 @@ class DictionaryMatcherOpDescSpec extends AnyFlatSpec with Matchers { code should not include "Text" } } + + it should "render stopwords and tokenized frozensets for conjunction matching and preserve case-folded entries for scan/substring" in { + val inputSchemas = + Map(PortIdentity() -> Schema().add(new Attribute("text", AttributeType.STRING))) + + val conjunction = + newDesc("A Quick,Fox", "text", " ", MatchingType.CONJUNCTION_INDEXBASED) + .generateStandaloneCode(inputSchemas) + conjunction should include("""out1df["matched"]""") + conjunction should include("_TEXERA_DM_STOPWORDS") + conjunction should include("""_entries=[frozenset({"quick"}), frozenset({"fox"})]""") + + val scan = + newDesc(" Alpha , BETA ", "text", "hit", MatchingType.SCANBASED) + .generateStandaloneCode(inputSchemas) + scan should include("""_entries=[" alpha ", " beta "]""") + scan should include("""out1df["hit"]""") + + val sub = + newDesc(null, "text", "hit", MatchingType.SUBSTRING).generateStandaloneCode(inputSchemas) + sub should include("""_entries=[""]""") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpDescSpec.scala index 79c1d4ccae3..c3dbd46ac57 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/keywordSearch/KeywordSearchOpDescSpec.scala @@ -161,4 +161,18 @@ class KeywordSearchOpDescSpec extends AnyFlatSpec with Matchers { code should include("""in1df["title"]""") code should not include "Title" } + + it should "emit a passthrough frame on blank keywords and escape regex metacharacters otherwise" in { + val inputSchemas = + Map(PortIdentity() -> Schema().add(new Attribute("title", AttributeType.STRING))) + newDesc("title", "", caseSensitive = false).generateStandaloneCode(inputSchemas) shouldBe + "out1df = in1df" + newDesc("title", " \t ", caseSensitive = true).generateStandaloneCode(inputSchemas) shouldBe + "out1df = in1df" + + val escaped = + newDesc("title", "a.b+c (foo)", caseSensitive = true).generateStandaloneCode(inputSchemas) + escaped should include("""\\b(?:a\\.b\\+c|\\(foo\\))\\b""") + escaped should include("case=True") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala index 98ccb2f400c..b316b00a227 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/machineLearning/Scorer/MachineLearningScorerOpDescSpec.scala @@ -406,4 +406,38 @@ class MachineLearningScorerOpDescSpec extends AnyFlatSpec with GeneratedCodeAsse } } } + + "MachineLearningScorerOpDesc.getOutputSchemas" should + "propagate classification errors eagerly on null metrics and refuse duplicate metric names" in { + val inputSchemas = Map( + PortIdentity() -> Schema( + List( + new Attribute("y", AttributeType.INTEGER), + new Attribute("yhat", AttributeType.INTEGER) + ) + ) + ) + + val nullMetricsDesc = new MachineLearningScorerOpDesc + nullMetricsDesc.actualValueColumn = "y" + nullMetricsDesc.predictValueColumn = "yhat" + nullMetricsDesc.classificationMetrics = null + intercept[NullPointerException](nullMetricsDesc.getOutputSchemas(inputSchemas)) + + val dupMetricsDesc = new MachineLearningScorerOpDesc + dupMetricsDesc.actualValueColumn = "y" + dupMetricsDesc.predictValueColumn = "yhat" + dupMetricsDesc.classificationMetrics = + List(classificationMetricsFnc.accuracy, classificationMetricsFnc.accuracy) + intercept[RuntimeException]( + dupMetricsDesc.getOutputSchemas(inputSchemas) + ).getMessage should include("already exists in the schema") + + val regDesc = new MachineLearningScorerOpDesc + regDesc.isRegression = true + regDesc.actualValueColumn = "y" + regDesc.predictValueColumn = "yhat" + regDesc.regressionMetrics = List(regressionMetricsFnc.mse, regressionMetricsFnc.r2) + regDesc.getOutputSchemas(inputSchemas).values.head.getAttributeNames shouldBe List("MSE", "R2") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/regex/RegexOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/regex/RegexOpDescSpec.scala index 0016a6eedb1..de2fff00606 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/regex/RegexOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/regex/RegexOpDescSpec.scala @@ -102,6 +102,9 @@ class RegexOpDescSpec extends AnyFlatSpec with Matchers { """(?u:(?a:\w)(?a:[\s])(?:[é]|(?a:[\d]))(?a:[^\W])(?:(?![é])(?a:[^\S]))(?:[\^]|(?a:[\w])))""" RegexOpDesc.unicodeCase("""x(?iu:é)\d\\w""") shouldBe """x(?iu:é)(?a:\d)\\w""" RegexOpDesc.unicodeCase("(?iux)é # note") shouldBe "(?ix)(?u:é # note\n)" + RegexOpDesc.unicodeCase("(?i-u)é") shouldBe "(?i-u)é" + RegexOpDesc.unicodeCase("(?u-i)é") shouldBe "(?-i)(?u:é)" + RegexOpDesc.unicodeCase("(?x)(?u)é # cmt") shouldBe "(?x)(?u:é # cmt\n)" } // Pattern without UNICODE_CASE ignores case for ASCII letters only, and its diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExecSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExecSpec.scala index 7e7d98dca28..440c83dcfec 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExecSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/arrow/ArrowSourceOpExecSpec.scala @@ -36,7 +36,8 @@ class ArrowSourceOpExecSpec extends AnyFlatSpec with Matchers { private val schema = Schema(List(new Attribute("s", AttributeType.STRING))) - private def writeArrowFile(rows: Seq[String]): File = { + /** Writes one record batch per argument, so a file can hold several or none. */ + private def writeArrowFile(batches: Seq[String]*): File = { val file = File.createTempFile("arrow-src-", ".arrow") file.deleteOnExit() val allocator = new RootAllocator() @@ -45,17 +46,19 @@ class ArrowSourceOpExecSpec extends AnyFlatSpec with Matchers { val writer = new ArrowFileWriter(root, null, Channels.newChannel(out)) try { writer.start() - root.allocateNew() - rows.zipWithIndex.foreach { - case (value, i) => - ArrowUtils.setTexeraTuple( - Tuple.builder(schema).addSequentially(Array[Any](value)).build(), - i, - root - ) + batches.foreach { rows => + root.allocateNew() + rows.zipWithIndex.foreach { + case (value, i) => + ArrowUtils.setTexeraTuple( + Tuple.builder(schema).addSequentially(Array[Any](value)).build(), + i, + root + ) + } + root.setRowCount(rows.size) + writer.writeBatch() } - root.setRowCount(rows.size) - writer.writeBatch() writer.end() } finally { writer.close() @@ -98,6 +101,33 @@ class ArrowSourceOpExecSpec extends AnyFlatSpec with Matchers { finally exec.close() } + it should "read every record batch in order, each row with the file's schema" in { + val exec = + new ArrowSourceOpExec(descString(writeArrowFile(Seq("a", "b"), Seq("c"), Seq("d", "e")))) + exec.open() + try { + val tuples = exec.produceTuple().toList.map(_.asInstanceOf[Tuple]) + tuples.map(_.getField[String]("s")) shouldBe List("a", "b", "c", "d", "e") + tuples.map(_.getSchema).distinct shouldBe List(schema) + } finally exec.close() + } + + it should "apply offset and limit across record batches" in { + val exec = new ArrowSourceOpExec( + descString(writeArrowFile(Seq("a", "b"), Seq("c"), Seq("d", "e")), Some(1), Some(3)) + ) + exec.open() + try assert(values(exec) == List("b", "c", "d")) + finally exec.close() + } + + it should "read no rows from a file without record batches" in { + val exec = new ArrowSourceOpExec(descString(writeArrowFile())) + exec.open() + try assert(values(exec).isEmpty) + finally exec.close() + } + it should "wrap failures to open an invalid Arrow file" in { val bogus = File.createTempFile("not-arrow-", ".arrow") bogus.deleteOnExit() diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/java/JavaUdfUiParameterInjectorSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/java/JavaUdfUiParameterInjectorSpec.scala index 9b017b2c586..223ad6a394e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/java/JavaUdfUiParameterInjectorSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/java/JavaUdfUiParameterInjectorSpec.scala @@ -226,6 +226,18 @@ class JavaUdfUiParameterInjectorSpec extends AnyFlatSpec with Matchers { executor.close() } } + + it should "preserve unicode escapes and mixed line endings" in { + val injected = JavaUdfUiParameterInjector.inject( + "public class JavaUDFOpExec {\r\n @UiParameter(\"x\") public final String value;\n}", + List(parameter("x", AttributeType.STRING, "a\nb\r\nc\\u0020")) + ) + injected should include("\r\n") + evaluate( + source("@UiParameter(\"x\") public String value;"), + List(parameter("x", AttributeType.STRING, "a\nb\r\nc\\u0020")) + ) shouldBe "a\nb\r\nc\\u0020" + } } object JavaUiParameterJreProbe { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjectorSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjectorSpec.scala index 043efeb8648..81cbe9b320f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjectorSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjectorSpec.scala @@ -280,4 +280,20 @@ class PythonUdfUiParameterInjectorSpec extends AnyFlatSpec with Matchers { exception.getMessage should include("no supported Python UDF class was found") } + + it should "preserve tab indentation and fall back to four spaces on empty single-line classes" in { + val tabIndented = + """class ProcessTupleOperator(UDFOperatorV2): + | def open(self): + | pass + |""".stripMargin + inject(tabIndented, uiParameter("k", AttributeType.STRING, "v")) should include( + "\n\tdef _texera_injected_ui_parameters(self) -> Dict[str, Any]:" + ) + + val emptyClass = " class ProcessTupleOperator(UDFOperatorV2):\n" + inject(emptyClass, uiParameter("k", AttributeType.STRING, "v")) should include( + "\n def _texera_injected_ui_parameters(self) -> Dict[str, Any]:" + ) + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/r/RUdfUiParameterInjectorSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/r/RUdfUiParameterInjectorSpec.scala index a1ab124c7f7..0bdcc2cc4f0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/r/RUdfUiParameterInjectorSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/r/RUdfUiParameterInjectorSpec.scala @@ -73,4 +73,21 @@ class RUdfUiParameterInjectorSpec extends AnyFlatSpec with Matchers { code should not include "UiParameter <- function" code should not include "stop('injected')" } + + it should "normalize CRLF and bare CR line endings and sort manifest entries deterministically" in { + val code = RUdfUiParameterInjector.inject( + "function() {\r\n x <- 1\r x\r\n}", + List(parameter("empty", AttributeType.STRING, "")) + ) + code should not include "\r" + code should include("value=.texera_decode(c())") + code should include("0x73,0x74,0x72,0x69,0x6e,0x67") + val typesSection = + code.substring(code.indexOf("attribute_types=list("), code.indexOf("resources=list(")) + val resourcesSection = code.substring(code.indexOf("resources=list(")) + typesSection should include("BOOL=") + typesSection.indexOf("BOOL=") should be < typesSection.indexOf("TIMESTAMP=") + resourcesSection should include("DATASET=") + resourcesSection.indexOf("DATASET=") should be < resourcesSection.indexOf("MODEL=") + } }