Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,8 @@ object ArrowFlightActorBench {
// batch_size = cfg.batchSize keeps the Python-side partitioning
// buffer aligned with our send size — one Java DataFrame in maps
// to exactly one Python DataFrame out.
OneToOnePartitioning(batchSize = cfg.batchSize, channels = Seq(dataChannelOut))
OneToOnePartitioning(batchSize = cfg.batchSize, channels = Seq(dataChannelOut)),
columnar = false
),
ctx,
4L
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,10 @@ message AddInputChannelRequest {
message AddPartitioningRequest {
core.PhysicalLink tag = 1 [(scalapb.field).no_box = true];
sendsemantics.Partitioning partitioning = 2 [(scalapb.field).no_box = true];
// Per-edge columnar-wire decision from the planner: ship this link's batches as
// Arrow (ColumnarFrame) rather than rows. True only when the downstream operator
// consumes columns natively and the columnar wire is enabled.
bool columnar = 3;
}

message AssignPortRequest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,19 @@ class PekkoActorRefMappingService(actorService: PekkoActorService) extends Amber
actorRefMapping.contains(id)
}

/**
* Whether worker `id` resolves to an actor in this same ActorSystem, i.e. a local,
* by-reference hop that never serializes, as opposed to a remote node. None when the
* ref is not resolved yet (the caller treats unknown as remote). The columnar output
* path uses this to keep a local edge on rows, since serializing it to Arrow would be
* pure overhead for a hop that otherwise copies nothing.
*/
def isLocalWorker(id: ActorVirtualIdentity): Option[Boolean] =
actorRefMapping.get(id).map { ref =>
val address = ref.path.address
address.hasLocalScope || address == self.path.address
}

def forwardToActor(msg: NetworkMessage): Unit = {
val id = msg.internalMessage.channelId.toWorkerId
if (actorRefMapping.contains(id)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ trait LinkWorkersHandler {
linkExecution.initChannelExecution(channelId)
Seq(
workerInterface.addPartitioning(
AddPartitioningRequest(msg.link, linkConfig.partitioning),
AddPartitioningRequest(msg.link, linkConfig.partitioning, linkConfig.columnar),
mkContext(channelId.fromWorkerId)
),
workerInterface.addInputChannel(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ import org.apache.texera.amber.engine.architecture.worker.managers.{
PortStorageWriterTerminateSignal
}
import org.apache.texera.amber.engine.common.AmberLogging
import org.apache.texera.amber.util.VirtualIdentityUtils
import org.apache.texera.amber.engine.common.ambermessage.ColumnarFrame
import org.apache.texera.amber.util.{ArrowUtils, VirtualIdentityUtils}
import org.apache.arrow.memory.{BufferAllocator, RootAllocator}
import org.apache.arrow.vector.VectorSchemaRoot

import java.net.URI
import scala.collection.mutable
Expand Down Expand Up @@ -112,6 +115,12 @@ class OutputManager(
) extends AmberLogging {

val outputIterator: DPOutputIterator = new DPOutputIterator()

// Set by the worker: whether a receiver worker is co-located (same ActorSystem). A local
// hop never serializes, so the columnar wire is pure overhead there and that edge stays
// on rows. None = not yet known (treated as remote). Wired into each buffer's locality
// check when the partitioner is added.
var isLocalReceiver: ActorVirtualIdentity => Option[Boolean] = _ => None
private val partitioners: mutable.Map[PhysicalLink, Partitioner] =
mutable.HashMap[PhysicalLink, Partitioner]()

Expand All @@ -134,12 +143,18 @@ class OutputManager(
*/
def addPartitionerWithPartitioning(
link: PhysicalLink,
partitioning: Partitioning
partitioning: Partitioning,
// Per-edge columnar-wire decision from the planner: ship this edge's batches as
// Arrow (ColumnarFrame) rather than rows. Defaults off so the row wire is unchanged.
columnar: Boolean = false
): Unit = {
val partitioner = toPartitioner(partitioning, actorId)
partitioners.update(link, partitioner)
partitioner.allReceivers.foreach(receiver => {
val buffer = new NetworkOutputBuffer(receiver, outputGateway, getBatchSize(partitioning))
val buffer =
new NetworkOutputBuffer(receiver, outputGateway, getBatchSize(partitioning), columnar)
// A local (same-JVM) receiver keeps the edge on rows regardless of the planner flag.
buffer.localityCheck = () => isLocalReceiver(receiver)
networkOutputBuffers.update((link, receiver), buffer)
outputGateway.addOutputChannel(ChannelIdentity(actorId, receiver, isControl = false))
})
Expand Down Expand Up @@ -168,6 +183,155 @@ class OutputManager(
}
}

/**
* Provisional (phase 1): turn the columnar output path on or off for every current
* edge. Phase 2 replaces this blanket switch with a per-edge decision made by the
* planner (local vs cross-node, and whether the receiver consumes columns).
*/
def setColumnarOutput(enabled: Boolean): Unit =
networkOutputBuffers.values.foreach(_.columnar = enabled)

// Backs the per-bucket roots the columnar shuffle materializes; each is freed right
// after it is serialized, so this allocator's outstanding bytes return to zero.
private lazy val columnarAllocator: BufferAllocator = new RootAllocator()

/**
* Columnar output path (no round-trip): ship an operator's Arrow result batch straight
* downstream without decoding it to tuples and re-encoding it. Eligible only when the
* output port needs no storage and every edge on it is in columnar mode. A
* non-splitting edge (one-to-one, broadcast) sends the whole batch, serialized once, to
* each receiver. A splitting edge (hash/range/round-robin) assigns each row a bucket
* through the same getBucketIndex the row path uses (reading only the key columns into a
* partial tuple), gathers each bucket's rows in Arrow, and sends one ColumnarFrame per
* non-empty bucket. Bucket assignment is therefore identical to the row path. Does NOT
* close `root` (caller owns it).
*
* @return Some((portId, totalSerializedByteCount)) if it shipped the batch (caller
* records output statistics), or None if the port is not eligible (fall back).
*/
def tryEmitColumnarBatch(
root: VectorSchemaRoot,
outputPortId: Option[PortIdentity]
): Option[(PortIdentity, Long)] = {
if (outputPortId.isEmpty && ports.size != 1) return None
val portId = outputPortId.getOrElse(ports.head._1)
val links = partitioners.filter(_._1.fromPortId == portId).toSeq
// Take the no-decode fast path when at least one receiver is remote-columnar, so those
// receivers get Arrow with no tuple decode; any local receivers on the same edge get
// rows. If no receiver is columnar (all local, or the wire is off), fall back so the
// caller decodes the batch once and routes it through the row path.
val anyColumnar = links.exists {
case (link, partitioner) =>
partitioner.allReceivers.exists(r =>
networkOutputBuffers.get((link, r)).exists(_.effectiveColumnar)
)
}
if (links.isEmpty || outputPortResultWriterThreads.contains(portId) || !anyColumnar) return None
val schema = getPort(portId).schema
// The executor's Arrow schema must match the declared output port schema; if not,
// fall back so the row path's enforceSchema can adjust it.
if (ArrowUtils.toTexeraSchema(root.getSchema) != schema) return None
Some((portId, links.map { case (link, p) => emitColumnarToLink(root, schema, link, p) }.sum))
}

// Ship `root` to one link's receivers, splitting per bucket if the partitioner splits.
// Each receiver gets Arrow if its buffer is columnar (remote) or rows if it is local, so
// a mixed-locality edge keeps the remote buckets columnar while sending rows locally.
// Returns the total bytes accounted for this link (serialized Arrow + local row sizes).
private def emitColumnarToLink(
root: VectorSchemaRoot,
schema: Schema,
link: PhysicalLink,
partitioner: Partitioner
): Long = {
val receivers = partitioner.allReceivers
if (!partitioner.columnarSplits) {
// One-to-one / broadcast: the whole batch goes to each receiver. Serialize to Arrow
// once for the remote receivers, decode to rows once for the local ones.
val (remote, local) =
receivers.partition(r => networkOutputBuffers((link, r)).effectiveColumnar)
var bytes = 0L
if (remote.nonEmpty) {
val arrow = ArrowUtils.serializeRoot(root)
val frame = ColumnarFrame(arrow, root.getRowCount, schema)
remote.foreach(r => networkOutputBuffers((link, r)).sendColumnarFrame(frame))
bytes += arrow.length.toLong
}
if (local.nonEmpty) {
val tuples = ArrowUtils.tuplesOfRoot(root)
local.foreach(r => networkOutputBuffers((link, r)).sendRows(tuples))
bytes += tuples.foldLeft(0L)(_ + _.inMemSize)
}
bytes
} else {
// Splitting shuffle: assign each row a bucket, then send each bucket to its receiver
// as Arrow (remote) or rows (local).
val n = root.getRowCount
val rowsPerBucket = Array.fill(receivers.length)(new mutable.ArrayBuffer[Int]())
// Build only the columns getBucketIndex needs into a partial tuple per row.
val keyCols =
if (partitioner.requiresFullTuple) schema.getAttributes.map(_.getName)
else partitioner.bucketKeyColumns
val keySchema = Schema(keyCols.map(schema.getAttribute).toList)
val keyIdx =
if (keyCols.isEmpty) Array.empty[Int] else ArrowUtils.projectionIndices(root, keyCols)
val keyTypes = keyCols.map(schema.getAttribute(_).getType).toArray
val vectors = root.getFieldVectors
var i = 0
while (i < n) {
val keyValues = new Array[Any](keyIdx.length)
var c = 0
while (c < keyIdx.length) {
keyValues(c) = ArrowUtils.getCell(vectors.get(keyIdx(c)), i, keyTypes(c)); c += 1
}
partitioner.getBucketIndex(Tuple(keySchema, keyValues)).foreach(b => rowsPerBucket(b) += i)
i += 1
}
var bytes = 0L
var b = 0
while (b < receivers.length) {
val rows = rowsPerBucket(b)
if (rows.nonEmpty) {
bytes += sendBucket(
root,
schema,
networkOutputBuffers((link, receivers(b))),
rows.toArray
)
}
b += 1
}
bytes
}
}

// Send one bucket's rows to a receiver: Arrow (gather + ColumnarFrame) if the buffer is
// columnar (remote), otherwise decode just those rows and send them as a DataFrame
// (local). Returns the bytes accounted for the send.
private def sendBucket(
root: VectorSchemaRoot,
schema: Schema,
buffer: NetworkOutputBuffer,
rows: Array[Int]
): Long = {
if (buffer.effectiveColumnar) {
val bucketRoot = ArrowUtils.gather(root, rows, columnarAllocator)
try {
val frameBytes = ArrowUtils.serializeRoot(bucketRoot)
buffer.sendColumnarFrame(ColumnarFrame(frameBytes, rows.length, schema))
frameBytes.length.toLong
} finally bucketRoot.close()
} else {
val tuples = new Array[Tuple](rows.length)
var k = 0
while (k < rows.length) {
tuples(k) = ArrowUtils.getTexeraTuple(rows(k), root, schema); k += 1
}
buffer.sendRows(tuples)
tuples.foldLeft(0L)(_ + _.inMemSize)
}
}

/**
* Flushes the network output buffers based on the specified set of physical links.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,14 @@ class PythonProxyClient(portNumberPromise: Promise[Int], val actorId: ActorVirtu
dataPayload match {
case DataFrame(frame) =>
writeArrowStream(mutable.Queue(ArraySeq.unsafeWrapArray(frame): _*), from, "Data")
case ColumnarFrame(arrowIpcBytes, _, _) =>
// A columnar-wire batch headed to a Python worker: decode it back to tuples
// and hand it over as a Data batch, same as a DataFrame.
writeArrowStream(
mutable.Queue(ArraySeq.unsafeWrapArray(ArrowUtils.deserializeTuples(arrowIpcBytes)): _*),
from,
"Data"
)
case StateFrame(state, loopCounter, loopStartId) =>
// The Arrow wire format for states IS the State row (content +
// loop_counter + loop_start_id), so the envelope rides its own
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,5 +94,8 @@ case object LinkConfig {

case class LinkConfig(
channelConfigs: List[ChannelConfig],
partitioning: Partitioning
partitioning: Partitioning,
// Per-edge columnar-wire decision: ship this link's batches as Arrow when the
// downstream operator consumes columns natively and the columnar wire is enabled.
columnar: Boolean = false
)
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.texera.amber.engine.architecture.scheduling.resourcePolicies

import org.apache.texera.common.config.ApplicationConfig
import org.apache.texera.amber.core.virtualidentity.PhysicalOpIdentity
import org.apache.texera.amber.core.workflow._
import org.apache.texera.amber.engine.architecture.scheduling.Region
Expand Down Expand Up @@ -89,7 +90,14 @@ class DefaultResourceAllocator(
operatorConfigs(physicalLink.toOpId).workerConfigs.map(_.workerId),
linkPartitionInfos(physicalLink),
workflowSettings.dataTransferBatchSize
)
),
// Per-edge columnar-wire decision: enabled only when the wire is on and the
// downstream operator consumes columns natively (otherwise shipping Arrow
// would force a costly per-tuple decode on the receiver). Locality (same-JVM
// links gain nothing) is a later refinement.
columnar =
ApplicationConfig.enableColumnarWire &&
physicalPlan.getOperator(physicalLink.toOpId).isColumnarCapable
)
}.toMap

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,7 @@ case class BroadcastPartitioner(partitioning: BroadcastPartitioning) extends Par
}

override def allReceivers: Seq[ActorVirtualIdentity] = receivers

// The whole batch goes to every receiver; no per-row split.
override def columnarSplits: Boolean = false
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,8 @@ case class HashBasedShufflePartitioner(partitioning: HashBasedShufflePartitionin
}

override def allReceivers: Seq[ActorVirtualIdentity] = receivers

// Hash reads the key columns (or the whole tuple when no keys are named).
override def bucketKeyColumns: Seq[String] = partitioning.hashAttributeNames
override def requiresFullTuple: Boolean = partitioning.hashAttributeNames.isEmpty
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,7 @@ case class OneToOnePartitioner(partitioning: OneToOnePartitioning, actorId: Acto

override def allReceivers: Seq[ActorVirtualIdentity] =
Seq(partitioning.channels.filter(_.fromWorkerId == actorId).head.toWorkerId)

// The whole batch goes to the single receiver; no per-row split.
override def columnarSplits: Boolean = false
}
Loading
Loading