Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -145,9 +145,10 @@ private class AmberProducer(
)
)
case _ => // normal data batches
val schema = ArrowUtils.toTexeraSchema(root.getSchema)
val queue = mutable.Queue[Tuple]()
for (i <- 0 until root.getRowCount)
queue.enqueue(ArrowUtils.getTexeraTuple(i, root))
queue.enqueue(ArrowUtils.getTexeraTuple(i, root, schema))
outputPort.sendTo(to, DataFrame(queue.toArray))
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,22 @@ class PythonProxyServerSpec extends AnyFlatSpec {
}
}

it should "deliver an empty Data put as an empty DataFrame" in {
withServer { fixture =>
val to =
ChannelIdentity(pythonWorkerId, ActorVirtualIdentity("downstream"), isControl = false)
val schema = Schema().add(new Attribute("v", AttributeType.INTEGER))

putBatch(fixture, PythonDataHeader(to, "Data"), ArrowUtils.fromTexeraSchema(schema), _ => ())

awaitTrue()(fixture.messages.nonEmpty)
val out = fixture.messages
assert(out.size == 1)
assert(out.head.channelId == to)
assert(out.head.payload == DataFrame(Array.empty[Tuple]))
}
}

it should "reassemble a single-row State put into a StateFrame carrying the loop envelope" in {
withServer { fixture =>
val to =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ case class Schema @JsonCreator() (
}
}

override def toString: String = s"Schema[${attributes.map(_.toString).mkString(", ")}]"
override def toString: String = s"Schema[${attributes.mkString(", ")}]"

/**
* Creates a new Schema containing only the specified attributes.
Expand All @@ -113,7 +113,7 @@ case class Schema @JsonCreator() (
* Throws an exception if any attribute name already exists in the schema.
*/
def add(attributesToAdd: Iterable[Attribute]): Schema = {
val existingNames = this.getAttributeNames.map(_.toLowerCase).toSet
val existingNames = attributeIndex.keySet
val duplicateNames = attributesToAdd.map(_.getName.toLowerCase).toSet.intersect(existingNames)

if (duplicateNames.nonEmpty) {
Expand Down Expand Up @@ -171,7 +171,7 @@ case class Schema @JsonCreator() (
val attributesToRemove = attributeNames.map(_.toLowerCase).toSet

// Check for non-existent attributes
val nonExistentAttributes = attributesToRemove.diff(attributes.map(_.getName.toLowerCase).toSet)
val nonExistentAttributes = attributesToRemove.diff(attributeIndex.keySet)
if (nonExistentAttributes.nonEmpty) {
throw new IllegalArgumentException(
s"Cannot remove non-existent attributes: ${nonExistentAttributes.mkString(", ")}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,25 +83,35 @@ case class Tuple @JsonCreator() (
this
}

override def hashCode: Int = util.Arrays.deepHashCode(getFields.map(_.asInstanceOf[AnyRef]))
override def hashCode: Int = util.Arrays.deepHashCode(getFields.asInstanceOf[Array[AnyRef]])

override def equals(obj: Any): Boolean =
obj match {
case that: Tuple =>
this.schema == that.schema &&
this.getFields.zip(that.getFields).forall {
case (field1: Array[Byte], field2: Array[Byte]) =>
field1.sameElements(field2) // for Binary, use sameElements instead of == to compare
case (field1, field2) =>
field1 == field2
}
case that: Tuple if this.schema == that.schema =>
val left = this.getFields
val right = that.getFields
val fieldCount = math.min(left.length, right.length)
var index = 0
while (index < fieldCount && fieldEquals(left(index), right(index))) index += 1
index == fieldCount
case _ => false
}

private def fieldEquals(left: Any, right: Any): Boolean =
if (left.isInstanceOf[Array[Byte]] && right.isInstanceOf[Array[Byte]])
util.Arrays.equals(left.asInstanceOf[Array[Byte]], right.asInstanceOf[Array[Byte]])
else left == right

def getPartialTuple(attributeNames: List[String]): Tuple = {
val partialSchema = schema.getPartialSchema(attributeNames)
val builder = Tuple.Builder(partialSchema)
val partialArray = attributeNames.map(getField[Any]).toArray
val partialArray = new Array[Any](attributeNames.length)
val names = attributeNames.iterator
var index = 0
while (names.hasNext) {
partialArray(index) = getField[Any](names.next())
index += 1
}
builder.addSequentially(partialArray)
builder.build()
}
Expand Down Expand Up @@ -132,10 +142,8 @@ object Tuple {
)
}

(attributeList zip fieldList).foreach {
case (attribute, field) =>
checkAttributeMatchesField(attribute, field)
}
val fieldIterator = fieldList.iterator
attributeList.foreach(attribute => checkAttributeMatchesField(attribute, fieldIterator.next()))
}

/**
Expand Down Expand Up @@ -177,14 +185,12 @@ object Tuple {
def add(tuple: Tuple, isStrictSchemaMatch: Boolean = true): Builder = {
require(tuple != null, "Tuple cannot be null")

tuple.getFields.zipWithIndex.foreach {
case (field, i) =>
val attribute = tuple.schema.getAttributes(i)
if (!isStrictSchemaMatch && !schema.containsAttribute(attribute.getName)) {
// Skip if not matching in non-strict mode
} else {
add(attribute, tuple.getFields(i))
}
val attributes = tuple.schema.getAttributes.iterator
tuple.getFields.foreach { field =>
val attribute = attributes.next()
if (isStrictSchemaMatch || schema.containsAttribute(attribute.getName)) {
this.add(attribute, field)
}
}
this
}
Expand Down Expand Up @@ -215,10 +221,8 @@ object Tuple {
def addSequentially(fields: Array[Any]): Builder = {
require(fields != null, "Fields cannot be null")
checkSchemaMatchesFields(schema.getAttributes, fields)
schema.getAttributes.zip(fields).foreach {
case (attribute, field) =>
this.add(attribute, field)
}
val attributes = schema.getAttributes.iterator
fields.foreach(field => this.add(attributes.next(), field))
this
}

Expand All @@ -231,8 +235,14 @@ object Tuple {
)
}

val fields =
schema.getAttributes.map(attr => fieldNameMap(attr.getName.toLowerCase)).toArray
val attributes = schema.getAttributes
val fields = new Array[Any](attributes.length)
val attributeIterator = attributes.iterator
var index = 0
while (attributeIterator.hasNext) {
fields(index) = fieldNameMap(attributeIterator.next().getName.toLowerCase)
index += 1
}
new Tuple(schema, fields)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,13 @@ trait SeqTupleLike extends TupleLike with SchemaEnforceable {
override def enforceSchema(schema: Schema): Tuple = {
val attributes = schema.getAttributes
val builder = Tuple.builder(schema)
getFields.zipWithIndex.foreach {
case (value, i) =>
builder.add(attributes(i), value)
val fields = getFields
val attributeIterator = attributes.iterator
var index = 0
while (index < fields.length) {
val attribute = if (attributeIterator.hasNext) attributeIterator.next() else attributes(index)
builder.add(attribute, fields(index))
index += 1
}
builder.build()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,41 +70,55 @@ object ArrowUtils extends LazyLogging {
def getTexeraTuple(
rowIndex: Int,
vectorSchemaRoot: VectorSchemaRoot
): Tuple = getTexeraTuple(rowIndex, vectorSchemaRoot, toTexeraSchema(vectorSchemaRoot.getSchema))

/**
* Reads a row like `getTexeraTuple(rowIndex, vectorSchemaRoot)`, given the root's schema
* already converted by `toTexeraSchema`, so a caller reading many rows of one root converts
* its schema once instead of once per row.
*
* @param schema `toTexeraSchema(vectorSchemaRoot.getSchema)`.
*/
def getTexeraTuple(
rowIndex: Int,
vectorSchemaRoot: VectorSchemaRoot,
schema: Schema
): Tuple = {
val arrowSchema = vectorSchemaRoot.getSchema
val schema = toTexeraSchema(arrowSchema)

Tuple
.builder(schema)
.addSequentially(
vectorSchemaRoot.getFieldVectors.asScala.zipWithIndex.map {
case (fieldVector: FieldVector, index: Int) =>
val value: AnyRef = fieldVector.getObject(rowIndex)
try {
// Use the attribute type from the schema (which includes metadata)
// instead of deriving it from the Arrow type
val attributeType = schema.getAttributes(index).getType
// A timestamp and an unsigned integer are the types whose field
// says more than the schema does, so they are the only ones that
// read the field: an unsigned column arrives as INTEGER or LONG.
attributeType match {
case AttributeType.TIMESTAMP => wallClockOf(value, fieldVector.getField.getType)
case AttributeType.INTEGER | AttributeType.LONG =>
AttributeTypeUtils.parseField(
unsignedValueOf(value, fieldVector.getField.getType),
attributeType
)
case _ => AttributeTypeUtils.parseField(value, attributeType)
}
} catch {
case e: Exception =>
logger.warn("Caught error during parsing Arrow value back to Texera value", e)
null
}

}.toArray
)
.build()
val fieldVectors = vectorSchemaRoot.getFieldVectors
val fieldValues = new Array[Any](fieldVectors.size())
val vectors = fieldVectors.iterator()
val attributes = schema.getAttributes.iterator
var index = 0
while (vectors.hasNext) {
val fieldVector = vectors.next()
val value: AnyRef = fieldVector.getObject(rowIndex)
val fieldValue =
try {
// Use the attribute type from the schema (which includes metadata)
// instead of deriving it from the Arrow type
val attributeType = attributes.next().getType
// A timestamp and an unsigned integer are the types whose field
// says more than the schema does, so they are the only ones that
// read the field: an unsigned column arrives as INTEGER or LONG.
attributeType match {
case AttributeType.TIMESTAMP => wallClockOf(value, fieldVector.getField.getType)
case AttributeType.INTEGER | AttributeType.LONG =>
AttributeTypeUtils.parseField(
unsignedValueOf(value, fieldVector.getField.getType),
attributeType
)
case _ => AttributeTypeUtils.parseField(value, attributeType)
}
} catch {
case e: Exception =>
logger.warn("Caught error during parsing Arrow value back to Texera value", e)
null
}
fieldValues(index) = fieldValue
index += 1
}

Tuple.builder(schema).addSequentially(fieldValues).build()
}

/** The number an unsigned column counts to, out of the storage it counts in.
Expand Down Expand Up @@ -192,7 +206,7 @@ object ArrowUtils extends LazyLogging {
*/
def toTexeraSchema(arrowSchema: org.apache.arrow.vector.types.pojo.Schema): Schema =
Schema(
arrowSchema.getFields.asScala.map { field =>
arrowSchema.getFields.asScala.iterator.map { field =>
val taggedType = Option(field.getMetadata)
.flatMap(m => Option(m.get(TexeraTypeMetadataKey)))
.collect {
Expand Down Expand Up @@ -259,9 +273,7 @@ object ArrowUtils extends LazyLogging {
}

def appendTexeraTuple(tuple: Tuple, vectorSchemaRoot: VectorSchemaRoot): Unit = {
val currentRowCount = vectorSchemaRoot.getRowCount
val nextRowIndex = currentRowCount
setTexeraTuple(tuple, nextRowIndex, vectorSchemaRoot)
setTexeraTuple(tuple, vectorSchemaRoot.getRowCount, vectorSchemaRoot)
}

/**
Expand All @@ -274,14 +286,13 @@ object ArrowUtils extends LazyLogging {
* multiple Vectors.
*/
def setTexeraTuple(tuple: Tuple, index: Int, vectorSchemaRoot: VectorSchemaRoot): Unit = {
val arrowSchema = vectorSchemaRoot.getSchema
val arrowFields = arrowSchema.getFields.asScala.toList
val arrowFields = vectorSchemaRoot.getSchema.getFields

for (i <- arrowFields.indices) {
for (i <- 0 until arrowFields.size()) {
val vector: FieldVector = vectorSchemaRoot.getVector(i)
val value = tuple.getField[AnyRef](i)
val isNull = value == null
arrowFields.apply(i).getFieldType.getType match {
arrowFields.get(i).getFieldType.getType match {
case _: ArrowType.Int =>
vector.getField.getFieldType.getType.asInstanceOf[ArrowType.Int].getBitWidth match {
case 16 | 32 =>
Expand Down Expand Up @@ -363,7 +374,9 @@ object ArrowUtils extends LazyLogging {
* @return An Arrow Schema.
*/
def fromTexeraSchema(schema: Schema): org.apache.arrow.vector.types.pojo.Schema = {
val arrowFields = schema.getAttributes.map { attribute =>
val attributes = schema.getAttributes
val arrowFields = new util.ArrayList[Field](attributes.size)
attributes.foreach { attribute =>
val metadataTag = attribute.getType match {
case AttributeType.LARGE_BINARY => "LARGE_BINARY"
case AttributeType.ANY => "ANY"
Expand All @@ -375,14 +388,16 @@ object ArrowUtils extends LazyLogging {
map
} else null

new Field(
attribute.getName,
new FieldType(true, fromAttributeType(attribute.getType), null, metadata),
null
arrowFields.add(
new Field(
attribute.getName,
new FieldType(true, fromAttributeType(attribute.getType), null, metadata),
null
)
)
}

new org.apache.arrow.vector.types.pojo.Schema(util.Arrays.asList(arrowFields: _*))
new org.apache.arrow.vector.types.pojo.Schema(arrowFields)
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -322,4 +322,30 @@ class SchemaSpec extends AnyFlatSpec {
val restored = objectMapper.readValue(objectMapper.writeValueAsString(schema), classOf[Schema])
assert(restored == schema)
}

it should "reject duplicate adds and remove attributes case-insensitively while failing on absent removals" in {
val schema = Schema(
List(
new Attribute("Id", AttributeType.INTEGER),
new Attribute("Name", AttributeType.STRING)
)
)
val dupEx = intercept[RuntimeException] {
schema.add(
List(
new Attribute("ID", AttributeType.STRING),
new Attribute("Extra", AttributeType.STRING)
)
)
}
assert(dupEx.getMessage == "Cannot add attributes with duplicate names: id")

val removed = schema.remove(List("ID", "name"))
assert(removed.getAttributes.isEmpty)

val missingEx = intercept[IllegalArgumentException] {
schema.remove(List("ID", "MissingCol"))
}
assert(missingEx.getMessage == "Cannot remove non-existent attributes: missingcol")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,13 @@ class TupleLikeSpec extends AnyFlatSpec {
assertThrows[IndexOutOfBoundsException](TupleLike(42, "hello", true).enforceSchema(schema))
}

it should "enforce null positional fields and reject field type mismatches" in {
val tuple = TupleLike(null, "hello").enforceSchema(schema)
assert(tuple.getField[Any]("col-int") == null)
assert(tuple.getField[String]("col-string") == "hello")
assertThrows[RuntimeException](TupleLike("wrong-type", "hello").enforceSchema(schema))
}

// ----- MapTupleLike -----

"MapTupleLike" should "expose its mapping values as fields and not implement inMemSize" in {
Expand Down
Loading
Loading