From 9af5ef3460c2abcc7adc1616047c8fee7948590b Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Tue, 17 Nov 2026 20:26:19 +0100 Subject: [PATCH 1/7] fix(scheduler): fail an AQE job that loses a finished stage's output instead of hanging When an executor holding the shuffle output of a finished stage is lost, or a reduce task fails to fetch that output, the adaptive graph turned the stage back into a running one to rerun it. Its planner had already dropped the stage when it finished, though, so the first rerun task to report failed with "Can't find active stage to update stage outputs". The task manager drops the whole status update on that error, and the rolled-back reader stage is never rebuilt either, so the job hung with no running tasks until cancelled (#2539). Rerunning a finished stage under AQE is not supported yet (#1986), so fail the job instead, with a message that points to retrying the query or to the static planner, which recovers lost output: - reset_stages_on_lost_executor returns an error when the lost executor held output a finished stage's readers still need; the task manager already fails the job on that error. - A fetch failure that would resubmit a finished stage fails that stage, and with it the job. Co-Authored-By: Claude Opus 5.5 --- ballista/scheduler/src/state/aqe/mod.rs | 45 ++++++++--- .../src/state/aqe/test/job_failure.rs | 76 ++++++++++++++++++- .../src/state/aqe/test/reuse_exchange.rs | 20 ++--- docs/source/user-guide/tuning-guide.md | 4 +- 4 files changed, 122 insertions(+), 23 deletions(-) diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs index a65496dc4d..1da85bdc79 100644 --- a/ballista/scheduler/src/state/aqe/mod.rs +++ b/ballista/scheduler/src/state/aqe/mod.rs @@ -55,6 +55,7 @@ use datafusion::prelude::SessionConfig; use datafusion::scalar::ScalarValue; use log::{debug, error, info, warn}; use std::collections::{HashMap, HashSet}; +use std::mem; use std::sync::Arc; use std::vec; @@ -297,12 +298,20 @@ impl AdaptiveExecutionGraph { /// Processing stage status update after task status changing fn processing_stages_update( &mut self, - updated_stages: UpdatedStages, + mut updated_stages: UpdatedStages, ) -> ballista_core::error::Result> { let job_id = self.job_id().to_owned(); let mut has_resolved = false; let mut job_err_msg = "".to_owned(); + // A fetch failure asks to rerun the finished stage whose output was + // lost, which AQE cannot do yet; fail the job instead of hanging it. + for stage_id in mem::take(&mut updated_stages.resubmit_successful_stages) { + updated_stages + .failed_stages + .insert(stage_id, cannot_rerun_finished_stage(stage_id)); + } + for stage_id in updated_stages.resolved_stages { self.resolve_stage(stage_id)?; has_resolved = true; @@ -327,10 +336,6 @@ impl AdaptiveExecutionGraph { running_tasks_to_cancel.extend(tasks); } - for stage_id in updated_stages.resubmit_successful_stages { - self.rerun_successful_stage(stage_id); - } - if !running_tasks_to_cancel.is_empty() { events.push(QueryStageSchedulerEvent::CancelTasks( running_tasks_to_cancel, @@ -774,6 +779,14 @@ impl AdaptiveExecutionGraph { } }); } + // The lost executor held output of a finished stage that later stages + // still need, and AQE cannot rerun a finished stage yet. The caller + // fails the job on this error rather than leaving it to hang. + if let Some(stage_id) = resubmit_successful_stages.iter().min() { + return Err(BallistaError::General(cannot_rerun_finished_stage( + *stage_id, + ))); + } for stage_id in rollback_resolved_stages.iter() { self.rollback_resolved_stage(*stage_id)?; @@ -788,15 +801,10 @@ impl AdaptiveExecutionGraph { all_running_tasks.extend(tasks); } - for stage_id in resubmit_successful_stages.iter() { - self.rerun_successful_stage(*stage_id); - } - let mut reset_stage = HashSet::new(); reset_stage.extend(reset_running_stage); reset_stage.extend(rollback_resolved_stages); reset_stage.extend(rollback_running_stages); - reset_stage.extend(resubmit_successful_stages); Ok((reset_stage, all_running_tasks)) } @@ -806,6 +814,19 @@ impl AdaptiveExecutionGraph { } } +/// Why a job fails when it needs the lost output of finished stage +/// `stage_id`: AQE cannot rerun a finished stage (#1986, #2539). Its planner +/// no longer tracks the stage, so the rerun's tasks could not report, and +/// the job would hang. +fn cannot_rerun_finished_stage(stage_id: usize) -> String { + format!( + "the output of finished stage {stage_id} was lost, and adaptive query \ + execution cannot rerun a finished stage yet; retry the query, or set \ + ballista.planner.adaptive.enabled to false to use the static planner, \ + which recovers lost output" + ) +} + impl ExecutionGraph for AdaptiveExecutionGraph { fn cloned(&self) -> ExecutionGraphBox { Box::new(self.clone()) @@ -1555,7 +1576,9 @@ impl ExecutionGraph for AdaptiveExecutionGraph { } } - /// Convert successful stage to be running + /// Convert successful stage to be running. The adaptive graph does not + /// call this: its planner no longer tracks a finished stage, so it fails + /// the job instead (see [cannot_rerun_finished_stage]). fn rerun_successful_stage(&mut self, stage_id: usize) -> bool { if let Some(ExecutionStage::Successful(stage)) = self.stages.remove(&stage_id) { self.stages diff --git a/ballista/scheduler/src/state/aqe/test/job_failure.rs b/ballista/scheduler/src/state/aqe/test/job_failure.rs index 2534a03976..d374800524 100644 --- a/ballista/scheduler/src/state/aqe/test/job_failure.rs +++ b/ballista/scheduler/src/state/aqe/test/job_failure.rs @@ -18,9 +18,13 @@ //! Job-failure lifecycle tests for the adaptive execution graph. use crate::state::aqe::AdaptiveExecutionGraph; +use crate::state::aqe::test::{mock_batch, mock_context}; use crate::state::execution_graph::ExecutionGraph; use crate::state::execution_stage::ExecutionStage; -use crate::test_utils::{join_aggregate_sort_logical_plan, mock_executor}; +use crate::test_utils::{ + fetch_partition_failure, join_aggregate_sort_logical_plan, mock_executor, + mock_failed_task, revive_graph_and_complete_next_stage_with_executor, +}; use ballista_core::error::Result; use ballista_core::serde::protobuf::{ FailedTask, JobStatus, ShuffleWritePartition, SuccessfulTask, failed_task, @@ -52,6 +56,76 @@ async fn test_join_plan(partition: usize) -> AdaptiveExecutionGraph { .unwrap() } +/// Builds an adaptive graph for an aggregation: a map stage, then a reduce +/// stage reading its shuffle output. +async fn test_aggregation_plan() -> AdaptiveExecutionGraph { + let ctx = SessionContext::new_with_config(mock_context().copied_config()); + ctx.register_batch("t", mock_batch().unwrap()).unwrap(); + let sql = "select c, sum(a) from t group by c"; + let plan = ctx.sql(sql).await.unwrap().logical_plan().clone(); + AdaptiveExecutionGraph::try_new( + "localhost:50050", + &"job".into(), + "", + &ctx, + &plan, + 0, + None, + ) + .await + .unwrap() +} + +/// Runs the map stage of an aggregation on executor "a", leaving the reduce +/// stage, which reads its output, running. +async fn aggregation_with_map_output_on_a() -> AdaptiveExecutionGraph { + let mut graph = test_aggregation_plan().await; + revive_graph_and_complete_next_stage_with_executor( + &mut graph, + &mock_executor("a".to_string()), + ) + .unwrap(); + graph.revive(); + assert!(matches!(graph.stages[&0], ExecutionStage::Successful(_))); + assert!(matches!(graph.stages[&1], ExecutionStage::Running(_))); + graph +} + +const CANNOT_RERUN: &str = "cannot rerun a finished stage"; + +// AQE cannot rerun a finished stage whose output was lost (#1986): its +// planner no longer tracks the stage, so the rerun's tasks could not report +// and the job hung (#2539). Losing the executor with that output fails the +// rollback instead, and the task manager fails the job on that error. +#[tokio::test] +async fn losing_the_output_of_a_finished_stage_fails_the_rollback() { + let mut graph = aggregation_with_map_output_on_a().await; + + let err = graph.reset_stages_on_lost_executor("a").unwrap_err(); + + assert!(err.to_string().contains(CANNOT_RERUN), "{err}"); +} + +// The same loss, found by a reduce task failing to fetch the map output: the +// job fails instead of hanging. +#[tokio::test] +async fn a_fetch_failure_on_the_output_of_a_finished_stage_fails_the_job() -> Result<()> { + let mut graph = aggregation_with_map_output_on_a().await; + let b = mock_executor("b".to_string()); + let task = graph.pop_next_task(&b.id)?.unwrap(); + let status = mock_failed_task(task, fetch_partition_failure("a", 0, 0)); + + graph.update_task_status(&b, vec![status], 4, 4)?; + + match &graph.status().status { + Some(job_status::Status::Failed(failed)) => { + assert!(failed.error.contains(CANNOT_RERUN), "{}", failed.error) + } + other => panic!("expected the job to fail, got {other:?}"), + } + Ok(()) +} + // Same contract as the static graph's test: aborting transitions every running // stage to Failed and returns its in-flight tasks for cancellation. #[tokio::test] diff --git a/ballista/scheduler/src/state/aqe/test/reuse_exchange.rs b/ballista/scheduler/src/state/aqe/test/reuse_exchange.rs index cc1e5b3ed8..884308b9d3 100644 --- a/ballista/scheduler/src/state/aqe/test/reuse_exchange.rs +++ b/ballista/scheduler/src/state/aqe/test/reuse_exchange.rs @@ -25,7 +25,6 @@ use crate::state::aqe::test::{ mock_batch, mock_context, mock_partitions_with_statistics, }; use crate::state::execution_graph::ExecutionGraph; -use crate::state::execution_stage::ExecutionStage; use crate::test_utils::{ mock_executor, revive_graph_and_complete_next_stage, revive_graph_and_complete_next_stage_with_executor, @@ -141,11 +140,10 @@ async fn a_job_reading_an_aggregate_twice_runs_one_stage_fewer( } // AQE does not yet rerun a finished stage whose output was lost (#1986, -// #2539), with or without reuse. What reuse adds is a stage with more than -// one reader, and losing its output must roll back the one stage and every -// stage reading it. +// #2539), with or without reuse, and fails the job instead. What reuse adds +// is a stage with more than one reader; losing its output fails the same way. #[tokio::test] -async fn losing_a_shared_stage_output_rolls_back_the_stage_once() { +async fn losing_a_shared_stage_output_fails_the_rollback() { let mut graph = graph(&context(true), UNION).await; revive_graph_and_complete_next_stage_with_executor( &mut graph, @@ -154,12 +152,14 @@ async fn losing_a_shared_stage_output_rolls_back_the_stage_once() { .unwrap(); graph.revive(); - let (reset, _) = graph.reset_stages_on_lost_executor("a").unwrap(); + let err = graph.reset_stages_on_lost_executor("a").unwrap_err(); - assert_eq!(graph.stages.len(), 2); - assert!(reset.contains(&0) && reset.contains(&1)); - assert!(matches!(graph.stages[&0], ExecutionStage::Running(_))); - assert!(matches!(graph.stages[&1], ExecutionStage::UnResolved(_))); + assert!( + err.to_string().contains( + "finished stage 0 was lost, and adaptive query execution cannot rerun" + ), + "{err}" + ); } fn exchange(plan_id: usize) -> ExchangeExec { diff --git a/docs/source/user-guide/tuning-guide.md b/docs/source/user-guide/tuning-guide.md index 2fba2ae074..6b8fd70880 100644 --- a/docs/source/user-guide/tuning-guide.md +++ b/docs/source/user-guide/tuning-guide.md @@ -430,7 +430,9 @@ currently implemented: The implementation covers the happy path only. The following are known to be missing or incomplete: -- Executor failure handling on the AQE path ([#1986](https://github.com/apache/datafusion-ballista/issues/1986)) +- Executor failure handling on the AQE path ([#1986](https://github.com/apache/datafusion-ballista/issues/1986)): + a job that loses the output of a stage that has already finished fails + instead of rerunning the stage - Switching from hash join to sort-merge join based on runtime statistics ([#1988](https://github.com/apache/datafusion-ballista/issues/1988)) - Switching from streaming aggregation to hash aggregation based on runtime statistics ([#1989](https://github.com/apache/datafusion-ballista/issues/1989)) From 555a3f787770a38bd83e75ebfdd8146f5c31c60d Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 18 Nov 2026 12:14:14 +0100 Subject: [PATCH 2/7] Fail the reading stage on a fetch failure, and check the abort event Review comments: - On a fetch failure, the job error named the finished map stage as the one that failed ("stage 0 failed: the output of finished stage 0 was lost"), though its reader was the stage whose task failed. Fail the reading stage at the fetch failure instead, as the cache-data-lost case above it already does. - The fetch-failure test also checks that update_task_status returns the JobRunningFailed event, which is what aborts the job and cancels its tasks, and that the message names the reading stage. Co-Authored-By: Claude Opus 5.5 --- ballista/scheduler/src/state/aqe/mod.rs | 52 +++++-------------- .../src/state/aqe/test/job_failure.rs | 30 ++++++++--- 2 files changed, 36 insertions(+), 46 deletions(-) diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs index 1da85bdc79..0d0cfc62e1 100644 --- a/ballista/scheduler/src/state/aqe/mod.rs +++ b/ballista/scheduler/src/state/aqe/mod.rs @@ -55,7 +55,6 @@ use datafusion::prelude::SessionConfig; use datafusion::scalar::ScalarValue; use log::{debug, error, info, warn}; use std::collections::{HashMap, HashSet}; -use std::mem; use std::sync::Arc; use std::vec; @@ -298,20 +297,12 @@ impl AdaptiveExecutionGraph { /// Processing stage status update after task status changing fn processing_stages_update( &mut self, - mut updated_stages: UpdatedStages, + updated_stages: UpdatedStages, ) -> ballista_core::error::Result> { let job_id = self.job_id().to_owned(); let mut has_resolved = false; let mut job_err_msg = "".to_owned(); - // A fetch failure asks to rerun the finished stage whose output was - // lost, which AQE cannot do yet; fail the job instead of hanging it. - for stage_id in mem::take(&mut updated_stages.resubmit_successful_stages) { - updated_stages - .failed_stages - .insert(stage_id, cannot_rerun_finished_stage(stage_id)); - } - for stage_id in updated_stages.resolved_stages { self.resolve_stage(stage_id)?; has_resolved = true; @@ -959,9 +950,8 @@ impl ExecutionGraph for AdaptiveExecutionGraph { let mut successful_stages = HashSet::new(); let mut failed_stages = HashMap::new(); - let mut rollback_running_stages = HashMap::new(); - let mut resubmit_successful_stages: HashMap> = - HashMap::new(); + let rollback_running_stages: HashMap> = HashMap::new(); + let resubmit_successful_stages: HashMap> = HashMap::new(); let reset_running_stages: HashMap> = HashMap::new(); // @@ -1050,32 +1040,16 @@ impl ExecutionGraph for AdaptiveExecutionGraph { error!("{error_msg}"); failed_stages.insert(stage_id, error_msg); } else { - // There are different removal strategies here. - // We can choose just remove the map_partition_id in the FetchPartitionError, when resubmit the input stage, there are less tasks - // need to rerun, but this might miss many more bad input partitions, lead to more stage level retries in following. - // Here we choose remove all the bad input partitions which match the same executor id in this single input stage. - // There are other more aggressive approaches, like considering the executor is lost and check all the running stages in this graph. - // Or count the fetch failure number on executor and mark the executor lost globally. - let removed_map_partitions = running_stage - .remove_input_partitions( - map_stage_id, - map_partition_id, - &executor_id, - )?; - - let failure_reasons = rollback_running_stages - .entry(stage_id) - .or_insert_with(HashSet::new); - failure_reasons.insert(executor_id); - - let missing_inputs = - resubmit_successful_stages - .entry(map_stage_id) - .or_default(); - missing_inputs.extend(removed_map_partitions); - warn!( - "Need to resubmit the current running Stage {stage_id} and its map Stage {map_stage_id} due to FetchPartitionError from task {task_identity}" - ) + // Recovering would rerun the finished map + // stage, which AQE cannot do yet: fail this + // reading stage, and with it the job, + // instead of hanging it. + let error_msg = + cannot_rerun_finished_stage(map_stage_id); + error!( + "Stage {stage_id} failed to fetch partition {map_partition_id} of stage {map_stage_id} from executor {executor_id}: {error_msg}" + ); + failed_stages.insert(stage_id, error_msg); } } else { let mut error_msg = format!( diff --git a/ballista/scheduler/src/state/aqe/test/job_failure.rs b/ballista/scheduler/src/state/aqe/test/job_failure.rs index d374800524..cf43b1c268 100644 --- a/ballista/scheduler/src/state/aqe/test/job_failure.rs +++ b/ballista/scheduler/src/state/aqe/test/job_failure.rs @@ -17,6 +17,7 @@ //! Job-failure lifecycle tests for the adaptive execution graph. +use crate::scheduler_server::event::QueryStageSchedulerEvent; use crate::state::aqe::AdaptiveExecutionGraph; use crate::state::aqe::test::{mock_batch, mock_context}; use crate::state::execution_graph::ExecutionGraph; @@ -115,14 +116,29 @@ async fn a_fetch_failure_on_the_output_of_a_finished_stage_fails_the_job() -> Re let task = graph.pop_next_task(&b.id)?.unwrap(); let status = mock_failed_task(task, fetch_partition_failure("a", 0, 0)); - graph.update_task_status(&b, vec![status], 4, 4)?; + let events = graph.update_task_status(&b, vec![status], 4, 4)?; - match &graph.status().status { - Some(job_status::Status::Failed(failed)) => { - assert!(failed.error.contains(CANNOT_RERUN), "{}", failed.error) - } - other => panic!("expected the job to fail, got {other:?}"), - } + // The event aborts the job and cancels its tasks. The failed stage is the + // reading stage 1, not the finished stage 0 whose output was lost. + let fail_message = events + .iter() + .find_map(|event| match event { + QueryStageSchedulerEvent::JobRunningFailed { fail_message, .. } => { + Some(fail_message) + } + _ => None, + }) + .expect("a JobRunningFailed event"); + assert!(fail_message.contains("stage 1 failed"), "{fail_message}"); + assert!( + fail_message.contains("finished stage 0 was lost"), + "{fail_message}" + ); + assert!(fail_message.contains(CANNOT_RERUN), "{fail_message}"); + assert!(matches!( + graph.status().status, + Some(job_status::Status::Failed(_)) + )); Ok(()) } From cf54be0188457fda46b653202b80fcee818ccfe0 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 18 Nov 2026 12:14:39 +0100 Subject: [PATCH 3/7] Remove the fetch-failure rerun bookkeeping the adaptive graph no longer uses A fetch failure now fails its reading stage, so nothing fills rollback_running_stages or resubmit_successful_stages in AdaptiveExecutionGraph::update_task_status any more. Remove the two maps and the loop that marked a resubmitted finished stage's tasks failed, along with the imports only that loop used. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- ballista/scheduler/src/state/aqe/mod.rs | 50 +++---------------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs index 0d0cfc62e1..fe0e490154 100644 --- a/ballista/scheduler/src/state/aqe/mod.rs +++ b/ballista/scheduler/src/state/aqe/mod.rs @@ -39,8 +39,7 @@ use ballista_core::execution_plans::{ use ballista_core::serde::protobuf::failed_task::FailedReason; use ballista_core::serde::protobuf::job_status::Status; use ballista_core::serde::protobuf::{ - FailedJob, FailedTask, JobStatus, ResultLost, RunningJob, SuccessfulJob, TaskStatus, - job_status, task_status, + FailedJob, JobStatus, RunningJob, SuccessfulJob, TaskStatus, job_status, task_status, }; use ballista_core::serde::scheduler::{ ExecutorEndpoint, ExecutorMetadata, PartitionLocation, @@ -950,8 +949,6 @@ impl ExecutionGraph for AdaptiveExecutionGraph { let mut successful_stages = HashSet::new(); let mut failed_stages = HashMap::new(); - let rollback_running_stages: HashMap> = HashMap::new(); - let resubmit_successful_stages: HashMap> = HashMap::new(); let reset_running_stages: HashMap> = HashMap::new(); // @@ -1239,42 +1236,6 @@ impl ExecutionGraph for AdaptiveExecutionGraph { .insert(*stage_id, HashSet::from_iter(attempts.iter().copied())); } - // Values coming out of `remove_input_partitions` are task_ids - // (append slots) under the multi-partition-task model (see the mirror - // site in `execution_graph.rs` for the TODO on partition-id semantics). - for (stage_id, missing_task_ids) in &resubmit_successful_stages { - if let Some(stage) = self.stages.get_mut(stage_id) { - if let ExecutionStage::Successful(success_stage) = stage { - for task_id in missing_task_ids { - if *task_id >= success_stage.task_infos.len() { - return Err(BallistaError::Internal(format!( - "Invalid task_id {} in map stage {} (task_infos has {} entries)", - *task_id, - stage_id, - success_stage.task_infos.len() - ))); - } - let task_info = &mut success_stage.task_infos[*task_id]; - // Update the task info to failed - task_info.task_status = task_status::Status::Failed(FailedTask { - error: "FetchPartitionError in parent stage".to_owned(), - retryable: true, - count_to_failures: false, - failed_reason: Some(FailedReason::ResultLost(ResultLost {})), - }); - } - } else { - warn!( - "Stage {job_id}/{stage_id} is not in Successful state when try to resubmit this stage. " - ); - } - } else { - return Err(BallistaError::Internal(format!( - "Invalid stage ID {stage_id} for job {job_id}" - ))); - } - } - for (stage_id, missing_task_ids) in &reset_running_stages { if let Some(stage) = self.stages.get_mut(stage_id) { if let ExecutionStage::Running(running_stage) = stage { @@ -1305,11 +1266,10 @@ impl ExecutionGraph for AdaptiveExecutionGraph { resolved_stages: Default::default(), successful_stages, failed_stages, - rollback_running_stages, - resubmit_successful_stages: resubmit_successful_stages - .keys() - .cloned() - .collect(), + // A fetch failure fails its stage instead of rolling it back and + // rerunning its input; see above. + rollback_running_stages: Default::default(), + resubmit_successful_stages: Default::default(), }) } From 99e3fc3702c0159fc61f9ac7e6f211062270ee1e Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 18 Nov 2026 15:55:03 +0100 Subject: [PATCH 4/7] minor comment fix --- ballista/scheduler/src/state/aqe/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs index fe0e490154..011e0ec267 100644 --- a/ballista/scheduler/src/state/aqe/mod.rs +++ b/ballista/scheduler/src/state/aqe/mod.rs @@ -91,7 +91,9 @@ mod test; /// /// with many limitations, such as: /// -/// - it does not cover executor failure +/// - executor failure only partly: tasks of a running stage that ran on a +/// lost executor are rerun, but a finished stage whose output was lost is +/// not, and the job fails instead /// - dynamically coalescing shuffle partitions, not supported yet /// - does not switch from hash join to sort merge join /// - does not switch from streaming aggregation to hash aggregation From 73730ae69f747d1c347c4eea7028d21fbd7bb3d1 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 18 Nov 2026 16:19:06 +0100 Subject: [PATCH 5/7] Fail an AQE job on any rollback a lost executor would need reset_stages_internal rolled back every stage whose input was on the lost executor, and returned an error only when a finished stage of this graph had to be rerun. A rolled-back stage is unresolved, and nothing in the adaptive graph resolves a stage again: stages only come resolved from the planner. A rollback without that error, for an input that is not a finished stage of this graph, would therefore have left the job to hang. Fail on any such rollback instead. The rollback code after the checks can no longer run, so it goes. Co-Authored-By: Claude Opus 5.5 --- ballista/scheduler/src/state/aqe/mod.rs | 34 +++++++++---------- .../src/state/aqe/test/job_failure.rs | 18 ++++++++++ 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs index 011e0ec267..dcf70fdf6b 100644 --- a/ballista/scheduler/src/state/aqe/mod.rs +++ b/ballista/scheduler/src/state/aqe/mod.rs @@ -779,25 +779,25 @@ impl AdaptiveExecutionGraph { *stage_id, ))); } - - for stage_id in rollback_resolved_stages.iter() { - self.rollback_resolved_stage(*stage_id)?; - } - - let mut all_running_tasks = vec![]; - for stage_id in rollback_running_stages.iter() { - let tasks = self.rollback_running_stage( - *stage_id, - HashSet::from([executor_id.to_owned()]), - )?; - all_running_tasks.extend(tasks); + // Any other stage whose input was on the lost executor would be rolled + // back to unresolved, and nothing in the adaptive graph resolves a + // stage again: stages only come resolved from the planner. Fail rather + // than leave it to hang. + if let Some(stage_id) = rollback_resolved_stages + .union(&rollback_running_stages) + .min() + { + return Err(BallistaError::General(format!( + "stage {stage_id} lost input on executor {executor_id}, and \ + adaptive query execution cannot roll back a stage yet; retry \ + the query, or set ballista.planner.adaptive.enabled to false \ + to use the static planner, which recovers lost output" + ))); } - let mut reset_stage = HashSet::new(); - reset_stage.extend(reset_running_stage); - reset_stage.extend(rollback_resolved_stages); - reset_stage.extend(rollback_running_stages); - Ok((reset_stage, all_running_tasks)) + // Only tasks of running stages were reset; they are rescheduled, and + // none are left running to cancel. + Ok((reset_running_stage, vec![])) } /// Clear the stage failure count for this stage if the stage is finally success diff --git a/ballista/scheduler/src/state/aqe/test/job_failure.rs b/ballista/scheduler/src/state/aqe/test/job_failure.rs index cf43b1c268..6402e4d7a9 100644 --- a/ballista/scheduler/src/state/aqe/test/job_failure.rs +++ b/ballista/scheduler/src/state/aqe/test/job_failure.rs @@ -107,6 +107,24 @@ async fn losing_the_output_of_a_finished_stage_fails_the_rollback() { assert!(err.to_string().contains(CANNOT_RERUN), "{err}"); } +// Nothing in the adaptive graph resolves a stage again once it is rolled back, +// so losing a stage's input fails the job even when the input is not a +// finished stage of this graph, as when it was dropped from it. +#[tokio::test] +async fn losing_the_input_of_a_stage_fails_the_rollback() { + let mut graph = aggregation_with_map_output_on_a().await; + graph.stages.remove(&0); + + let err = graph.reset_stages_on_lost_executor("a").unwrap_err(); + + assert!( + err.to_string() + .contains("stage 1 lost input on executor a, and adaptive query execution cannot roll back a stage"), + "{err}" + ); + assert!(matches!(graph.stages[&1], ExecutionStage::Running(_))); +} + // The same loss, found by a reduce task failing to fetch the map output: the // job fails instead of hanging. #[tokio::test] From 25bc472f7f3c056d500e0dafea4e169af8ac6a28 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 18 Nov 2026 16:21:02 +0100 Subject: [PATCH 6/7] Remove the adaptive graph's never-filled reset_running_stages update_task_status declared reset_running_stages as an empty, immutable map, so the check that a successful stage was not in it always passed and the loop resetting its tasks never ran. The static graph, which this code was copied from, fills it on late fetch failures; the adaptive graph has no such path. Noted on #1986. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- ballista/scheduler/src/state/aqe/mod.rs | 30 +------------------------ 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs index dcf70fdf6b..0cf0da59fa 100644 --- a/ballista/scheduler/src/state/aqe/mod.rs +++ b/ballista/scheduler/src/state/aqe/mod.rs @@ -951,7 +951,6 @@ impl ExecutionGraph for AdaptiveExecutionGraph { let mut successful_stages = HashSet::new(); let mut failed_stages = HashMap::new(); - let reset_running_stages: HashMap> = HashMap::new(); // // @@ -1166,8 +1165,7 @@ impl ExecutionGraph for AdaptiveExecutionGraph { // all task updates have been handled // - let is_final_successful = running_stage.is_successful() - && !reset_running_stages.contains_key(&stage_id); + let is_final_successful = running_stage.is_successful(); if is_final_successful { successful_stages.insert(stage_id); @@ -1238,32 +1236,6 @@ impl ExecutionGraph for AdaptiveExecutionGraph { .insert(*stage_id, HashSet::from_iter(attempts.iter().copied())); } - for (stage_id, missing_task_ids) in &reset_running_stages { - if let Some(stage) = self.stages.get_mut(stage_id) { - if let ExecutionStage::Running(running_stage) = stage { - for task_id in missing_task_ids { - if *task_id >= running_stage.task_infos.len() { - return Err(BallistaError::Internal(format!( - "Invalid task_id {} in map stage {} (task_infos has {} entries)", - *task_id, - stage_id, - running_stage.task_infos.len() - ))); - } - running_stage.reset_task_info(*task_id); - } - } else { - warn!( - "Stage {job_id}/{stage_id} is not in Running state when try to reset the running task. " - ); - } - } else { - return Err(BallistaError::Internal(format!( - "Invalid stage ID {stage_id} for job {job_id}" - ))); - } - } - self.processing_stages_update(UpdatedStages { resolved_stages: Default::default(), successful_stages, From 755c4199b4600d0a7ce8e6455a7da197ef240e6a Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Wed, 18 Nov 2026 16:22:12 +0100 Subject: [PATCH 7/7] Put the static graph's resolve_stage doc back on resolve_stage #2730 added recovers_external_inputs and resolve_external_input between resolve_stage and its doc comment, which left "Convert unresolved stage to be resolved" on recovers_external_inputs. Move it back, and say what recovers_external_inputs does for the static graph. Co-Authored-By: Claude Opus 5.5 --- ballista/scheduler/src/state/execution_graph.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs index 14a7b3b3a2..6364388ecf 100644 --- a/ballista/scheduler/src/state/execution_graph.rs +++ b/ballista/scheduler/src/state/execution_graph.rs @@ -1492,7 +1492,8 @@ impl ExecutionGraph for StaticExecutionGraph { } } - /// Convert unresolved stage to be resolved + /// The stages reading a lost external input are rolled back, and wait + /// unresolved until `resolve_external_input` provides it again. fn recovers_external_inputs(&self) -> bool { true } @@ -1538,6 +1539,7 @@ impl ExecutionGraph for StaticExecutionGraph { }) } + /// Convert unresolved stage to be resolved fn resolve_stage(&mut self, stage_id: usize) -> Result { if let Some(ExecutionStage::UnResolved(stage)) = self.stages.get(&stage_id) { let resolved_stage = stage.to_resolved(self.session_config.options())?;