From ff606f889c25400be9557a8a030725c2b8c63521 Mon Sep 17 00:00:00 2001 From: suryaacharan Date: Fri, 16 Oct 2026 15:24:22 -0700 Subject: [PATCH 1/2] fix(amber): keep the failing worker when an error crosses ControlError #8668 tags a failed worker RPC with the worker (WorkflowRuntimeException with relatedWorkerId), but ControlError had no field for it. When the coordinator returned the error to the web service, reconstructThrowable rebuilt a plain Throwable, so WorkflowService recorded "unknown operator" and the UI could not show the error under the failed operator (e.g. a Python UDF that fails while starting). - Add relatedWorkerId to ControlError. - mkControlError fills it from a WorkflowRuntimeException. - reconstructThrowable rebuilds a WorkflowRuntimeException when it is set, so the worker reaches WorkflowService's error handler. Co-Authored-By: Claude Opus 5.5 --- .../architecture/rpc/controlreturns.proto | 4 ++ .../texera/amber/error/ErrorUtils.scala | 17 ++++-- .../RegionExecutionManagerSpec.scala | 3 +- .../WorkflowExecutionManagerSpec.scala | 3 +- .../texera/amber/error/ErrorUtilsSpec.scala | 60 +++++++++++++++++-- 5 files changed, 75 insertions(+), 12 deletions(-) diff --git a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlreturns.proto b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlreturns.proto index 5ce90ebdb60..2f4191cd7cc 100644 --- a/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlreturns.proto +++ b/amber/src/main/protobuf/org/apache/texera/amber/engine/architecture/rpc/controlreturns.proto @@ -18,6 +18,7 @@ syntax = "proto3"; package org.apache.texera.amber.engine.architecture.rpc; +import "org/apache/texera/amber/core/virtualidentity.proto"; import "org/apache/texera/amber/engine/architecture/worker/statistics.proto"; import "scalapb/scalapb.proto"; @@ -65,6 +66,9 @@ message ControlError { string errorDetails = 2; string stackTrace = 3; ErrorLanguage language = 4; + // The worker the error came from, when known (a WorkflowRuntimeException's + // relatedWorkerId), so the receiver can attribute it to that operator. + core.ActorVirtualIdentity relatedWorkerId = 5; } message ReturnInvocation { diff --git a/amber/src/main/scala/org/apache/texera/amber/error/ErrorUtils.scala b/amber/src/main/scala/org/apache/texera/amber/error/ErrorUtils.scala index c1ac5090dcc..3c889ba38d9 100644 --- a/amber/src/main/scala/org/apache/texera/amber/error/ErrorUtils.scala +++ b/amber/src/main/scala/org/apache/texera/amber/error/ErrorUtils.scala @@ -20,6 +20,7 @@ package org.apache.texera.amber.error import com.google.protobuf.timestamp.Timestamp +import org.apache.texera.amber.core.WorkflowRuntimeException import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ConsoleMessage import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ConsoleMessageType.{ @@ -76,18 +77,24 @@ object ErrorUtils { def mkControlError(err: Throwable): ControlError = { // Format each stack trace element with "at " prefix val stacktrace = err.getStackTrace.map(element => s"at ${element}").mkString("\n") - if (err.getCause != null) { - ControlError(err.toString, err.getCause.toString, stacktrace, ErrorLanguage.SCALA) - } else { - ControlError(err.toString, "", stacktrace, ErrorLanguage.SCALA) + val details = if (err.getCause != null) err.getCause.toString else "" + // Keep the failing worker so the receiver can attribute the error to its operator. + val relatedWorkerId = err match { + case wre: WorkflowRuntimeException => wre.relatedWorkerId + case _ => None } + ControlError(err.toString, details, stacktrace, ErrorLanguage.SCALA, relatedWorkerId) } def reconstructThrowable(controlError: ControlError): Throwable = { if (controlError.language == ErrorLanguage.PYTHON) { return new Throwable(controlError.errorMessage) } else { - val reconstructedThrowable = new Throwable(controlError.errorMessage) + val reconstructedThrowable = controlError.relatedWorkerId match { + // Rebuild the WorkflowRuntimeException so the failing worker is not lost. + case Some(worker) => new WorkflowRuntimeException(controlError.errorMessage, Some(worker)) + case None => new Throwable(controlError.errorMessage) + } if (controlError.errorDetails.nonEmpty) { val causeThrowable = new Throwable(controlError.errorDetails) reconstructedThrowable.initCause(causeThrowable) diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerSpec.scala index afd3836a68f..1134fd3d070 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/RegionExecutionManagerSpec.scala @@ -553,6 +553,7 @@ class RegionExecutionManagerSpec errorMessage = "transient EndWorker failure", errorDetails = "", stackTrace = "", - language = ErrorLanguage.SCALA + language = ErrorLanguage.SCALA, + None ) } diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManagerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManagerSpec.scala index 141de24460d..1113b5a219a 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManagerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/scheduling/WorkflowExecutionManagerSpec.scala @@ -213,7 +213,8 @@ class WorkflowExecutionManagerSpec errorMessage = udfInitError, errorDetails = "", stackTrace = "", - language = ErrorLanguage.PYTHON + language = ErrorLanguage.PYTHON, + None ) else EmptyReturn() ) diff --git a/amber/src/test/scala/org/apache/texera/amber/error/ErrorUtilsSpec.scala b/amber/src/test/scala/org/apache/texera/amber/error/ErrorUtilsSpec.scala index 90952d06774..d01a4acc987 100644 --- a/amber/src/test/scala/org/apache/texera/amber/error/ErrorUtilsSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/error/ErrorUtilsSpec.scala @@ -19,6 +19,7 @@ package org.apache.texera.amber.error +import org.apache.texera.amber.core.WorkflowRuntimeException import org.apache.texera.amber.core.virtualidentity.ActorVirtualIdentity import org.apache.texera.amber.engine.architecture.rpc.controlcommands.ConsoleMessageType.ERROR import org.apache.texera.amber.engine.architecture.rpc.controlreturns.{ControlError, ErrorLanguage} @@ -113,7 +114,8 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers { "py.boom", "ignored-details", "at com.x.Foo.bar(Foo.scala:42)", - ErrorLanguage.PYTHON + ErrorLanguage.PYTHON, + None ) val reconstructed = ErrorUtils.reconstructThrowable(ce) reconstructed.getMessage shouldBe "py.boom" @@ -123,14 +125,14 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers { } it should "leave the cause null when errorDetails is empty for SCALA errors" in { - val ce = ControlError("scala.boom", "", "", ErrorLanguage.SCALA) + val ce = ControlError("scala.boom", "", "", ErrorLanguage.SCALA, None) val reconstructed = ErrorUtils.reconstructThrowable(ce) reconstructed.getMessage shouldBe "scala.boom" reconstructed.getCause shouldBe null } it should "attach a cause Throwable when errorDetails is non-empty" in { - val ce = ControlError("scala.boom", "root-cause", "", ErrorLanguage.SCALA) + val ce = ControlError("scala.boom", "root-cause", "", ErrorLanguage.SCALA, None) val reconstructed = ErrorUtils.reconstructThrowable(ce) reconstructed.getCause should not be null reconstructed.getCause.getMessage shouldBe "root-cause" @@ -141,7 +143,8 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers { "scala.boom", "", "at com.x.Foo.bar(Foo.scala:42)\nat com.x.Baz.qux(Baz.scala:7)", - ErrorLanguage.SCALA + ErrorLanguage.SCALA, + None ) val reconstructed = ErrorUtils.reconstructThrowable(ce) val frames = reconstructed.getStackTrace @@ -156,7 +159,8 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers { "scala.boom", "", "garbage line\nat com.x.Foo.bar(Foo.scala:42)\nmore garbage", - ErrorLanguage.SCALA + ErrorLanguage.SCALA, + None ) val reconstructed = ErrorUtils.reconstructThrowable(ce) reconstructed.getStackTrace.length shouldBe 1 @@ -204,4 +208,50 @@ class ErrorUtilsSpec extends AnyFlatSpec with Matchers { operatorId shouldBe "__DummyOperator" workerId shouldBe "COORDINATOR" } + + // ----- carrying the failing worker through ControlError ----- + + private val failingWorker = + ActorVirtualIdentity("Worker:WF23-PythonUDFV2-operator-udfdemo-main-0") + + "mkControlError with a WorkflowRuntimeException" should "carry the failing worker" in { + val ce = ErrorUtils.mkControlError(new WorkflowRuntimeException("boom", Some(failingWorker))) + ce.relatedWorkerId shouldBe Some(failingWorker) + } + + it should "leave the worker empty for errors that are not tied to one" in { + ErrorUtils.mkControlError(new RuntimeException("boom")).relatedWorkerId shouldBe None + ErrorUtils + .mkControlError(new WorkflowRuntimeException("boom", None)) + .relatedWorkerId shouldBe None + } + + "reconstructThrowable with a related worker" should "rebuild a WorkflowRuntimeException" in { + val original = + new WorkflowRuntimeException("boom", new IllegalStateException("root"), Some(failingWorker)) + val rebuilt = ErrorUtils.reconstructThrowable(ErrorUtils.mkControlError(original)) + rebuilt shouldBe a[WorkflowRuntimeException] + val wre = rebuilt.asInstanceOf[WorkflowRuntimeException] + wre.relatedWorkerId shouldBe Some(failingWorker) + wre.getMessage shouldBe "boom" + wre.getCause.getMessage shouldBe new IllegalStateException("root").toString + } + + it should "let the error be attributed to the failing operator" in { + // WorkflowService reads relatedWorkerId from the rebuilt error; before this, + // the worker was dropped and the UI showed "unknown operator". + val rebuilt = ErrorUtils + .reconstructThrowable( + ErrorUtils.mkControlError(new WorkflowRuntimeException("boom", Some(failingWorker))) + ) + .asInstanceOf[WorkflowRuntimeException] + ErrorUtils.getOperatorFromActorIdOpt(rebuilt.relatedWorkerId) shouldBe + ("PythonUDFV2-operator-udfdemo", failingWorker.name) + } + + it should "still rebuild a plain Throwable when no worker is set" in { + val rebuilt = + ErrorUtils.reconstructThrowable(ControlError("boom", "", "", ErrorLanguage.SCALA, None)) + rebuilt should not be a[WorkflowRuntimeException] + } } From c2014bcbbf5675e86b76e579943925f2c7a83f30 Mon Sep 17 00:00:00 2001 From: Suryaa Charan <122703005+sshiv012@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:03:14 -0800 Subject: [PATCH 2/2] test(amber): cover a first-region UDF failure crossing ControlError Co-Authored-By: Claude Opus 5.5 --- .../texera/amber/error/ErrorUtils.scala | 2 ++ .../AdvanceRegionExecutionsHandlerSpec.scala | 10 +++---- .../amber/engine/e2e/DataProcessingSpec.scala | 27 +++++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/amber/src/main/scala/org/apache/texera/amber/error/ErrorUtils.scala b/amber/src/main/scala/org/apache/texera/amber/error/ErrorUtils.scala index 3c889ba38d9..27affc219cd 100644 --- a/amber/src/main/scala/org/apache/texera/amber/error/ErrorUtils.scala +++ b/amber/src/main/scala/org/apache/texera/amber/error/ErrorUtils.scala @@ -88,6 +88,8 @@ object ErrorUtils { def reconstructThrowable(controlError: ControlError): Throwable = { if (controlError.language == ErrorLanguage.PYTHON) { + // pyamber never sets relatedWorkerId; the coordinator tags a failed Python worker itself + // (RegionExecutionManager.attributeFailureTo). return new Throwable(controlError.errorMessage) } else { val reconstructedThrowable = controlError.relatedWorkerId match { diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/AdvanceRegionExecutionsHandlerSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/AdvanceRegionExecutionsHandlerSpec.scala index a942c615aa4..0144bc553b5 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/AdvanceRegionExecutionsHandlerSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/architecture/coordinator/promisehandlers/AdvanceRegionExecutionsHandlerSpec.scala @@ -92,11 +92,11 @@ class AdvanceRegionExecutionsHandlerSpec /** * Stands in for the real `WorkflowExecutionManager`: driving its advance to a *failed* future * from the outside means staging a whole region launch or termination, and the causes that - * reaches the handler with are `IllegalStateException`s (unresolved output schema, termination - * retries exhausted) or the bare `Throwable` that `ErrorUtils.reconstructThrowable` builds for a - * worker RPC error — never a `WorkflowRuntimeException` carrying a worker id. This stub records - * the advance calls and hands back a future the test controls, so the handler can be observed - * both while the advance is still pending and after it has failed with either kind of cause. + * reaches the handler with are a `WorkflowRuntimeException` naming the worker whose RPC failed, + * or an `IllegalStateException` (unresolved output schema, termination retries exhausted). This + * stub records the advance calls and hands back a future the test controls, so the handler can + * be observed both while the advance is still pending and after it has failed with either kind + * of cause. */ private class PendingAdvanceExecutionManager( workflowExecution: WorkflowExecution, diff --git a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala index 12b809314b6..0237e329c95 100644 --- a/amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala +++ b/amber/src/test/scala/org/apache/texera/amber/engine/e2e/DataProcessingSpec.scala @@ -305,6 +305,33 @@ class DataProcessingSpec extends RetryingDatabaseE2ESpec("DataProcessingSpec", s assert(error.relatedWorkerId.exists(_.name.contains("JavaUDFOpDesc"))) } + "Engine" should "name the failing worker when a udf in the first region fails to initialize" in { + // A UDF in the first region fails while the workflow starts, so the error comes back to the + // client as the reply to startWorkflow (a ControlError), not as a FatalError event. The + // failing worker must survive that hop so the UI can attribute the error to the UDF. + val headerlessCsvOpDesc = TestOperators.headerlessSmallCsvScanOpDesc() + val javaUdfOpDesc = new JavaUDFOpDesc() + javaUdfOpDesc.retainInputColumns = true + javaUdfOpDesc.code = "public class JavaUDFOpExec { not valid java }" + val workflow = buildWorkflow( + List(headerlessCsvOpDesc, javaUdfOpDesc), + List( + LogicalLink( + headerlessCsvOpDesc.operatorIdentifier, + PortIdentity(), + javaUdfOpDesc.operatorIdentifier, + PortIdentity() + ) + ), + workflowContext + ) + val error = intercept[WorkflowRuntimeException] { + executeWorkflow(workflow) + } + assert(error.getMessage.contains("Error at line")) + assert(error.relatedWorkerId.exists(_.name.contains("JavaUDFOpDesc"))) + } + "Engine" should "execute headerlessCsv->keyword workflow with MATERIALIZED mode" in { val headerlessCsvOpDesc = TestOperators.headerlessSmallCsvScanOpDesc() val keywordOpDesc = TestOperators.keywordSearchOpDesc("column-1", "Asia")