Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,14 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
dataPayload match {
case DataFrame(frame) =>
writeArrowStream(mutable.Queue(ArraySeq.unsafeWrapArray(frame): _*), from, "Data")
case ColumnarFrame(arrowIpcBytes, _, _) =>
// A columnar-wire batch headed to a Python worker: decode it back to tuples
// and hand it over as a Data batch, same as a DataFrame.
writeArrowStream(
mutable.Queue(ArraySeq.unsafeWrapArray(ArrowUtils.deserializeTuples(arrowIpcBytes)): _*),
from,
"Data"
)
case StateFrame(state, loopCounter, loopStartId) =>
// The Arrow wire format for states IS the State row (content +
// loop_counter + loop_start_id), so the envelope rides its own
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ import org.apache.texera.amber.core.state.State
import org.apache.texera.amber.core.tuple.Tuple
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
import org.apache.texera.amber.engine.architecture.messaginglayer.NetworkOutputGateway
import org.apache.texera.amber.engine.common.ambermessage.{DataFrame, StateFrame}
import org.apache.texera.amber.engine.common.ambermessage.{ColumnarFrame, DataFrame, StateFrame}
import org.apache.texera.amber.util.ArrowUtils

import scala.collection.mutable.ArrayBuffer

Expand All @@ -37,7 +38,10 @@ trait Partitioner extends Serializable {
class NetworkOutputBuffer(
val to: ActorVirtualIdentity,
val dataOutputPort: NetworkOutputGateway,
val batchSize: Int = ApplicationConfig.defaultDataTransferBatchSize
val batchSize: Int = ApplicationConfig.defaultDataTransferBatchSize,
// Columnar wire (opt-in): flush as ColumnarFrame (Arrow IPC) instead of
// DataFrame. Off by default, so the default wire is unchanged.
val columnar: Boolean = ApplicationConfig.enableColumnarWire
) {

var buffer = new ArrayBuffer[Tuple]()
Expand All @@ -57,7 +61,17 @@ class NetworkOutputBuffer(

def flush(): Unit = {
if (buffer.nonEmpty) {
dataOutputPort.sendTo(to, DataFrame(buffer.toArray))
val batch = buffer.toArray
if (columnar) {
// Serialize the batch to Arrow once; all tuples on a link share the schema.
val schema = batch(0).getSchema
dataOutputPort.sendTo(
to,
ColumnarFrame(ArrowUtils.serializeTuples(schema, batch), batch.length, schema)
)
} else {
dataOutputPort.sendTo(to, DataFrame(batch))
}
buffer = new ArrayBuffer[Tuple]()
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@ package org.apache.texera.amber.engine.architecture.worker

import com.softwaremill.macwire.wire
import io.grpc.MethodDescriptor
import org.apache.texera.amber.core.executor.OperatorExecutor
import org.apache.texera.amber.core.executor.{
ColumnarOperatorExecutor,
ColumnarResult,
OperatorExecutor
}
import org.apache.texera.amber.core.state.State
import org.apache.texera.amber.core.tuple._
import org.apache.texera.amber.core.virtualidentity.{
Expand Down Expand Up @@ -59,6 +63,7 @@ import org.apache.texera.amber.engine.common.ambermessage._
import org.apache.texera.amber.engine.common.statetransition.WorkerStateManager
import org.apache.texera.amber.engine.common.virtualidentity.util.COORDINATOR
import org.apache.texera.amber.error.ErrorUtils.{mkConsoleMessage, mkPrintConsoleMessage, safely}
import org.apache.texera.amber.util.ArrowUtils

import java.util.concurrent.LinkedBlockingQueue

Expand Down Expand Up @@ -114,6 +119,90 @@ class DataProcessor(
* process currentInputTuple through executor logic.
* this function is only called by the DP thread.
*/
// Start processing one data batch: announce the worker is running, load the
// batch, and process its first tuple (the DP loop drains the rest). Shared by the
// DataFrame and ColumnarFrame input paths.
private[this] def startInputBatch(
channelId: ChannelIdentity,
tuples: Array[Tuple]
): Unit = {
transitToRunning()
inputManager.initBatch(channelId, tuples)
processInputTuple(inputManager.getNextTuple)
}

// Announce the worker is running (READY -> RUNNING, once). Shared by the row and
// columnar input paths.
private[this] def transitToRunning(): Unit = {
stateManager.conditionalTransitTo(
READY,
RUNNING,
() => {
val (state, stateVersion) = stateManager.getStateWithVersion
asyncRPCClient.coordinatorInterface.workerStateUpdated(
WorkerStateUpdatedRequest(state, stateVersion),
asyncRPCClient.mkContext(COORDINATOR)
)
}
)
}

// Columnar wire: an Arrow batch arrived. If the executor can consume Arrow
// columns directly (a ColumnarOperatorExecutor), hand it the batch and route the
// result; otherwise, or if the executor declines the batch (Unsupported), decode
// to tuples and process it exactly as a DataFrame.
private[this] def processColumnarFrame(
channelId: ChannelIdentity,
portId: PortIdentity,
arrowIpcBytes: Array[Byte],
rowCount: Int
): Unit = {
val columnarExecutor = executor match {
case c: ColumnarOperatorExecutor => c
case _ => null
}
if (columnarExecutor == null) {
startInputBatch(channelId, ArrowUtils.deserializeTuples(arrowIpcBytes))
return
}
transitToRunning()
inputManager.currentChannelId = channelId
// The executor sees data: bind its loop variables first (parity with the row path).
bindStateReferences()
try {
columnarExecutor.processColumnarBatch(arrowIpcBytes, portId.id) match {
case ColumnarResult.Unsupported =>
// The executor declined this batch: fall back to the row path.
startInputBatch(channelId, ArrowUtils.deserializeTuples(arrowIpcBytes))
case consumed =>
// Consumed natively: no per-tuple input remains for the DP loop to drain.
// Report the whole batch's input statistics in one call.
inputManager.initBatch(channelId, Array.empty)
statisticsManager.increaseInputStatistics(
portId,
rowCount.toLong,
arrowIpcBytes.length.toLong
)
consumed match {
case ColumnarResult.EmitRows(rows) =>
outputManager.outputIterator.setTupleOutput(rows)
case ColumnarResult.Emit(root) =>
// Materialize the emitted Arrow batch to output tuples (the existing
// output path is tuple-based), then free the root.
val tuples =
try ArrowUtils.tuplesOfRoot(root)
finally root.close()
outputManager.outputIterator.setTupleOutput(
tuples.iterator.map(t => (t: TupleLike, None: Option[PortIdentity]))
)
case _ => // Consumed: a blocking operator accumulated the batch, nothing to emit now
}
}
} catch safely {
case e => handleExecutorException(e)
}
}

private[this] def processInputTuple(tuple: Tuple): Unit = {
try {
val portIdentity: PortIdentity =
Expand Down Expand Up @@ -243,19 +332,13 @@ class DataProcessor(
val portId = this.inputGateway.getChannel(channelId).getPortId
dataPayload match {
case DataFrame(tuples) =>
stateManager.conditionalTransitTo(
READY,
RUNNING,
() => {
val (state, stateVersion) = stateManager.getStateWithVersion
asyncRPCClient.coordinatorInterface.workerStateUpdated(
WorkerStateUpdatedRequest(state, stateVersion),
asyncRPCClient.mkContext(COORDINATOR)
)
}
)
inputManager.initBatch(channelId, tuples)
processInputTuple(inputManager.getNextTuple)
startInputBatch(channelId, tuples)
case ColumnarFrame(arrowIpcBytes, rowCount, _) =>
// Columnar wire: the batch arrived as Arrow IPC bytes. A
// ColumnarOperatorExecutor consumes the Arrow columns directly (no per-row
// decode); any other operator, or one that declines the batch, decodes it
// back to tuples and processes it exactly as a DataFrame.
processColumnarFrame(channelId, portId, arrowIpcBytes, rowCount)
case StateFrame(state, loopCounter, loopStartId) =>
processInputState(state, portId.id, loopCounter, loopStartId)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,21 @@ class StatisticsManager {
* @param portId the port identity
* @param size the size of the tuple
*/
def increaseInputStatistics(portId: PortIdentity, size: Long): Unit = {
def increaseInputStatistics(portId: PortIdentity, size: Long): Unit =
increaseInputStatistics(portId, 1L, size)

/**
* Increases the input statistics for a given port by a whole batch at once:
* `count` tuples totalling `size` bytes. The columnar-wire path consumes an
* Arrow batch without a per-tuple loop, so it reports the batch's row count and
* serialized size in one call rather than once per tuple.
*/
def increaseInputStatistics(portId: PortIdentity, count: Long, size: Long): Unit = {
require(count >= 0, "Tuple count must be non-negative")
require(size >= 0, "Tuple size must be non-negative")
inputStatistics.updateWith(portId) {
case Some((count, totalSize)) => Some((count + 1, totalSize + size))
case None => Some((1L, size))
case Some((c, totalSize)) => Some((c + count, totalSize + size))
case None => Some((count, size))
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
package org.apache.texera.amber.engine.common.ambermessage

import org.apache.texera.amber.core.state.State
import org.apache.texera.amber.core.tuple.Tuple
import org.apache.texera.amber.core.tuple.{Schema, Tuple}

sealed trait DataPayload extends WorkflowFIFOMessagePayload {}

Expand All @@ -37,6 +37,19 @@ sealed trait DataPayload extends WorkflowFIFOMessagePayload {}
final case class StateFrame(frame: State, loopCounter: Long = 0L, loopStartId: String = "")
extends DataPayload

/**
* A batch of rows carried over the wire as Arrow IPC stream bytes (the columnar
* wire format), instead of as decoded `Tuple`s in a `DataFrame`. `rowCount` and
* the Texera `schema` describe the batch without decoding it. The receiver either
* consumes the bytes directly (a `ColumnarOperatorExecutor`) or decodes them back
* to tuples (`ArrowUtils.deserializeTuples`). Produced only when the columnar wire
* is enabled; off by default, so the default wire stays `DataFrame`.
*/
final case class ColumnarFrame(arrowIpcBytes: Array[Byte], rowCount: Int, schema: Schema)
extends DataPayload {
val inMemSize: Long = arrowIpcBytes.length.toLong
}

final case class DataFrame(frame: Array[Tuple]) extends DataPayload {
val inMemSize: Long = {
frame.map(_.inMemSize).sum
Expand Down
Loading
Loading