Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
syntax = "proto3";
package org.apache.texera.amber.engine.architecture.rpc;

import "org/apache/texera/amber/core/virtualidentity.proto";
import "org/apache/texera/amber/engine/architecture/worker/statistics.proto";
import "scalapb/scalapb.proto";

Expand Down Expand Up @@ -65,6 +66,9 @@ message ControlError {
string errorDetails = 2;
string stackTrace = 3;
ErrorLanguage language = 4;
// The worker the error came from, when known (a WorkflowRuntimeException's
// relatedWorkerId), so the receiver can attribute it to that operator.
core.ActorVirtualIdentity relatedWorkerId = 5;
}

message ReturnInvocation {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.texera.amber.error

import com.google.protobuf.timestamp.Timestamp
import org.apache.texera.amber.core.WorkflowRuntimeException
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ConsoleMessage
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ConsoleMessageType.{
Expand Down Expand Up @@ -76,18 +77,26 @@ object ErrorUtils {
def mkControlError(err: Throwable): ControlError = {
// Format each stack trace element with "at " prefix
val stacktrace = err.getStackTrace.map(element => s"at ${element}").mkString("\n")
if (err.getCause != null) {
ControlError(err.toString, err.getCause.toString, stacktrace, ErrorLanguage.SCALA)
} else {
ControlError(err.toString, "", stacktrace, ErrorLanguage.SCALA)
val details = if (err.getCause != null) err.getCause.toString else ""
// Keep the failing worker so the receiver can attribute the error to its operator.
val relatedWorkerId = err match {
case wre: WorkflowRuntimeException => wre.relatedWorkerId
case _ => None
}
ControlError(err.toString, details, stacktrace, ErrorLanguage.SCALA, relatedWorkerId)
}

def reconstructThrowable(controlError: ControlError): Throwable = {
if (controlError.language == ErrorLanguage.PYTHON) {
// pyamber never sets relatedWorkerId; the coordinator tags a failed Python worker itself
// (RegionExecutionManager.attributeFailureTo).
return new Throwable(controlError.errorMessage)
} else {
val reconstructedThrowable = new Throwable(controlError.errorMessage)
val reconstructedThrowable = controlError.relatedWorkerId match {
// Rebuild the WorkflowRuntimeException so the failing worker is not lost.
case Some(worker) => new WorkflowRuntimeException(controlError.errorMessage, Some(worker))
case None => new Throwable(controlError.errorMessage)
}
if (controlError.errorDetails.nonEmpty) {
val causeThrowable = new Throwable(controlError.errorDetails)
reconstructedThrowable.initCause(causeThrowable)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,11 @@ class AdvanceRegionExecutionsHandlerSpec
/**
* Stands in for the real `WorkflowExecutionManager`: driving its advance to a *failed* future
* from the outside means staging a whole region launch or termination, and the causes that
* reaches the handler with are `IllegalStateException`s (unresolved output schema, termination
* retries exhausted) or the bare `Throwable` that `ErrorUtils.reconstructThrowable` builds for a
* worker RPC error — never a `WorkflowRuntimeException` carrying a worker id. This stub records
* the advance calls and hands back a future the test controls, so the handler can be observed
* both while the advance is still pending and after it has failed with either kind of cause.
* reaches the handler with are a `WorkflowRuntimeException` naming the worker whose RPC failed,
* or an `IllegalStateException` (unresolved output schema, termination retries exhausted). This
* stub records the advance calls and hands back a future the test controls, so the handler can
* be observed both while the advance is still pending and after it has failed with either kind
* of cause.
*/
private class PendingAdvanceExecutionManager(
workflowExecution: WorkflowExecution,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -683,6 +683,7 @@ class RegionExecutionManagerSpec
errorMessage = "transient EndWorker failure",
errorDetails = "",
stackTrace = "",
language = ErrorLanguage.SCALA
language = ErrorLanguage.SCALA,
None
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,8 @@ class WorkflowExecutionManagerSpec
errorMessage = udfInitError,
errorDetails = "",
stackTrace = "",
language = ErrorLanguage.PYTHON
language = ErrorLanguage.PYTHON,
None
)
else EmptyReturn()
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,33 @@ class DataProcessingSpec extends RetryingDatabaseE2ESpec("DataProcessingSpec", s
assert(error.relatedWorkerId.exists(_.name.contains("JavaUDFOpDesc")))
}

"Engine" should "name the failing worker when a udf in the first region fails to initialize" in {
// A UDF in the first region fails while the workflow starts, so the error comes back to the
// client as the reply to startWorkflow (a ControlError), not as a FatalError event. The
// failing worker must survive that hop so the UI can attribute the error to the UDF.
val headerlessCsvOpDesc = TestOperators.headerlessSmallCsvScanOpDesc()
val javaUdfOpDesc = new JavaUDFOpDesc()
javaUdfOpDesc.retainInputColumns = true
javaUdfOpDesc.code = "public class JavaUDFOpExec { not valid java }"
val workflow = buildWorkflow(
List(headerlessCsvOpDesc, javaUdfOpDesc),
List(
LogicalLink(
headerlessCsvOpDesc.operatorIdentifier,
PortIdentity(),
javaUdfOpDesc.operatorIdentifier,
PortIdentity()
)
),
workflowContext
)
val error = intercept[WorkflowRuntimeException] {
executeWorkflow(workflow)
}
assert(error.getMessage.contains("Error at line"))
assert(error.relatedWorkerId.exists(_.name.contains("JavaUDFOpDesc")))
}

"Engine" should "execute headerlessCsv->keyword workflow with MATERIALIZED mode" in {
val headerlessCsvOpDesc = TestOperators.headerlessSmallCsvScanOpDesc()
val keywordOpDesc = TestOperators.keywordSearchOpDesc("column-1", "Asia")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.texera.amber.error

import org.apache.texera.amber.core.WorkflowRuntimeException
import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity
import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ConsoleMessageType.ERROR
import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{ControlError, ErrorLanguage}
Expand Down Expand Up @@ -113,7 +114,8 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers {
"py.boom",
"ignored-details",
"at com.x.Foo.bar(Foo.scala:42)",
ErrorLanguage.PYTHON
ErrorLanguage.PYTHON,
None
)
val reconstructed = ErrorUtils.reconstructThrowable(ce)
reconstructed.getMessage shouldBe "py.boom"
Expand All @@ -123,14 +125,14 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers {
}

it should "leave the cause null when errorDetails is empty for SCALA errors" in {
val ce = ControlError("scala.boom", "", "", ErrorLanguage.SCALA)
val ce = ControlError("scala.boom", "", "", ErrorLanguage.SCALA, None)
val reconstructed = ErrorUtils.reconstructThrowable(ce)
reconstructed.getMessage shouldBe "scala.boom"
reconstructed.getCause shouldBe null
}

it should "attach a cause Throwable when errorDetails is non-empty" in {
val ce = ControlError("scala.boom", "root-cause", "", ErrorLanguage.SCALA)
val ce = ControlError("scala.boom", "root-cause", "", ErrorLanguage.SCALA, None)
val reconstructed = ErrorUtils.reconstructThrowable(ce)
reconstructed.getCause should not be null
reconstructed.getCause.getMessage shouldBe "root-cause"
Expand All @@ -141,7 +143,8 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers {
"scala.boom",
"",
"at com.x.Foo.bar(Foo.scala:42)\nat com.x.Baz.qux(Baz.scala:7)",
ErrorLanguage.SCALA
ErrorLanguage.SCALA,
None
)
val reconstructed = ErrorUtils.reconstructThrowable(ce)
val frames = reconstructed.getStackTrace
Expand All @@ -156,7 +159,8 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers {
"scala.boom",
"",
"garbage line\nat com.x.Foo.bar(Foo.scala:42)\nmore garbage",
ErrorLanguage.SCALA
ErrorLanguage.SCALA,
None
)
val reconstructed = ErrorUtils.reconstructThrowable(ce)
reconstructed.getStackTrace.length shouldBe 1
Expand Down Expand Up @@ -204,4 +208,50 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers {
operatorId shouldBe "__DummyOperator"
workerId shouldBe "COORDINATOR"
}

// ----- carrying the failing worker through ControlError -----

private val failingWorker =
ActorVirtualIdentity("Worker:WF23-PythonUDFV2-operator-udfdemo-main-0")

"mkControlError with a WorkflowRuntimeException" should "carry the failing worker" in {
val ce = ErrorUtils.mkControlError(new WorkflowRuntimeException("boom", Some(failingWorker)))
ce.relatedWorkerId shouldBe Some(failingWorker)
}

it should "leave the worker empty for errors that are not tied to one" in {
ErrorUtils.mkControlError(new RuntimeException("boom")).relatedWorkerId shouldBe None
ErrorUtils
.mkControlError(new WorkflowRuntimeException("boom", None))
.relatedWorkerId shouldBe None
}

"reconstructThrowable with a related worker" should "rebuild a WorkflowRuntimeException" in {
val original =
new WorkflowRuntimeException("boom", new IllegalStateException("root"), Some(failingWorker))
val rebuilt = ErrorUtils.reconstructThrowable(ErrorUtils.mkControlError(original))
rebuilt shouldBe a[WorkflowRuntimeException]
val wre = rebuilt.asInstanceOf[WorkflowRuntimeException]
wre.relatedWorkerId shouldBe Some(failingWorker)
wre.getMessage shouldBe "boom"
wre.getCause.getMessage shouldBe new IllegalStateException("root").toString
}

it should "let the error be attributed to the failing operator" in {
// WorkflowService reads relatedWorkerId from the rebuilt error; before this,
// the worker was dropped and the UI showed "unknown operator".
val rebuilt = ErrorUtils
.reconstructThrowable(
ErrorUtils.mkControlError(new WorkflowRuntimeException("boom", Some(failingWorker)))
)
.asInstanceOf[WorkflowRuntimeException]
ErrorUtils.getOperatorFromActorIdOpt(rebuilt.relatedWorkerId) shouldBe
("PythonUDFV2-operator-udfdemo", failingWorker.name)
}

it should "still rebuild a plain Throwable when no worker is set" in {
val rebuilt =
ErrorUtils.reconstructThrowable(ControlError("boom", "", "", ErrorLanguage.SCALA, None))
rebuilt should not be a[WorkflowRuntimeException]
}
}
Loading