diff --git a/ballista/scheduler/src/state/aqe/mod.rs b/ballista/scheduler/src/state/aqe/mod.rs index a65496dc4d..0cf0da59fa 100644 --- a/ballista/scheduler/src/state/aqe/mod.rs +++ b/ballista/scheduler/src/state/aqe/mod.rs @@ -39,8 +39,7 @@ use ballista_core::execution_plans::{ use ballista_core::serde::protobuf::failed_task::FailedReason; use ballista_core::serde::protobuf::job_status::Status; use ballista_core::serde::protobuf::{ - FailedJob, FailedTask, JobStatus, ResultLost, RunningJob, SuccessfulJob, TaskStatus, - job_status, task_status, + FailedJob, JobStatus, RunningJob, SuccessfulJob, TaskStatus, job_status, task_status, }; use ballista_core::serde::scheduler::{ ExecutorEndpoint, ExecutorMetadata, PartitionLocation, @@ -92,7 +91,9 @@ mod test; /// /// with many limitations, such as: /// -/// - it does not cover executor failure +/// - executor failure only partly: tasks of a running stage that ran on a +/// lost executor are rerun, but a finished stage whose output was lost is +/// not, and the job fails instead /// - dynamically coalescing shuffle partitions, not supported yet /// - does not switch from hash join to sort merge join /// - does not switch from streaming aggregation to hash aggregation @@ -327,10 +328,6 @@ impl AdaptiveExecutionGraph { running_tasks_to_cancel.extend(tasks); } - for stage_id in updated_stages.resubmit_successful_stages { - self.rerun_successful_stage(stage_id); - } - if !running_tasks_to_cancel.is_empty() { events.push(QueryStageSchedulerEvent::CancelTasks( running_tasks_to_cancel, @@ -774,30 +771,33 @@ impl AdaptiveExecutionGraph { } }); } - - for stage_id in rollback_resolved_stages.iter() { - self.rollback_resolved_stage(*stage_id)?; - } - - let mut all_running_tasks = vec![]; - for stage_id in rollback_running_stages.iter() { - let tasks = self.rollback_running_stage( + // The lost executor held output of a finished stage that later stages + // still need, and AQE cannot rerun a finished stage yet. The caller + // fails the job on this error rather than leaving it to hang. + if let Some(stage_id) = resubmit_successful_stages.iter().min() { + return Err(BallistaError::General(cannot_rerun_finished_stage( *stage_id, - HashSet::from([executor_id.to_owned()]), - )?; - all_running_tasks.extend(tasks); + ))); } - - for stage_id in resubmit_successful_stages.iter() { - self.rerun_successful_stage(*stage_id); + // Any other stage whose input was on the lost executor would be rolled + // back to unresolved, and nothing in the adaptive graph resolves a + // stage again: stages only come resolved from the planner. Fail rather + // than leave it to hang. + if let Some(stage_id) = rollback_resolved_stages + .union(&rollback_running_stages) + .min() + { + return Err(BallistaError::General(format!( + "stage {stage_id} lost input on executor {executor_id}, and \ + adaptive query execution cannot roll back a stage yet; retry \ + the query, or set ballista.planner.adaptive.enabled to false \ + to use the static planner, which recovers lost output" + ))); } - let mut reset_stage = HashSet::new(); - reset_stage.extend(reset_running_stage); - reset_stage.extend(rollback_resolved_stages); - reset_stage.extend(rollback_running_stages); - reset_stage.extend(resubmit_successful_stages); - Ok((reset_stage, all_running_tasks)) + // Only tasks of running stages were reset; they are rescheduled, and + // none are left running to cancel. + Ok((reset_running_stage, vec![])) } /// Clear the stage failure count for this stage if the stage is finally success @@ -806,6 +806,19 @@ impl AdaptiveExecutionGraph { } } +/// Why a job fails when it needs the lost output of finished stage +/// `stage_id`: AQE cannot rerun a finished stage (#1986, #2539). Its planner +/// no longer tracks the stage, so the rerun's tasks could not report, and +/// the job would hang. +fn cannot_rerun_finished_stage(stage_id: usize) -> String { + format!( + "the output of finished stage {stage_id} was lost, and adaptive query \ + execution cannot rerun a finished stage yet; retry the query, or set \ + ballista.planner.adaptive.enabled to false to use the static planner, \ + which recovers lost output" + ) +} + impl ExecutionGraph for AdaptiveExecutionGraph { fn cloned(&self) -> ExecutionGraphBox { Box::new(self.clone()) @@ -938,10 +951,6 @@ impl ExecutionGraph for AdaptiveExecutionGraph { let mut successful_stages = HashSet::new(); let mut failed_stages = HashMap::new(); - let mut rollback_running_stages = HashMap::new(); - let mut resubmit_successful_stages: HashMap> = - HashMap::new(); - let reset_running_stages: HashMap> = HashMap::new(); // // @@ -1029,32 +1038,16 @@ impl ExecutionGraph for AdaptiveExecutionGraph { error!("{error_msg}"); failed_stages.insert(stage_id, error_msg); } else { - // There are different removal strategies here. - // We can choose just remove the map_partition_id in the FetchPartitionError, when resubmit the input stage, there are less tasks - // need to rerun, but this might miss many more bad input partitions, lead to more stage level retries in following. - // Here we choose remove all the bad input partitions which match the same executor id in this single input stage. - // There are other more aggressive approaches, like considering the executor is lost and check all the running stages in this graph. - // Or count the fetch failure number on executor and mark the executor lost globally. - let removed_map_partitions = running_stage - .remove_input_partitions( - map_stage_id, - map_partition_id, - &executor_id, - )?; - - let failure_reasons = rollback_running_stages - .entry(stage_id) - .or_insert_with(HashSet::new); - failure_reasons.insert(executor_id); - - let missing_inputs = - resubmit_successful_stages - .entry(map_stage_id) - .or_default(); - missing_inputs.extend(removed_map_partitions); - warn!( - "Need to resubmit the current running Stage {stage_id} and its map Stage {map_stage_id} due to FetchPartitionError from task {task_identity}" - ) + // Recovering would rerun the finished map + // stage, which AQE cannot do yet: fail this + // reading stage, and with it the job, + // instead of hanging it. + let error_msg = + cannot_rerun_finished_stage(map_stage_id); + error!( + "Stage {stage_id} failed to fetch partition {map_partition_id} of stage {map_stage_id} from executor {executor_id}: {error_msg}" + ); + failed_stages.insert(stage_id, error_msg); } } else { let mut error_msg = format!( @@ -1172,8 +1165,7 @@ impl ExecutionGraph for AdaptiveExecutionGraph { // all task updates have been handled // - let is_final_successful = running_stage.is_successful() - && !reset_running_stages.contains_key(&stage_id); + let is_final_successful = running_stage.is_successful(); if is_final_successful { successful_stages.insert(stage_id); @@ -1244,77 +1236,14 @@ impl ExecutionGraph for AdaptiveExecutionGraph { .insert(*stage_id, HashSet::from_iter(attempts.iter().copied())); } - // Values coming out of `remove_input_partitions` are task_ids - // (append slots) under the multi-partition-task model (see the mirror - // site in `execution_graph.rs` for the TODO on partition-id semantics). - for (stage_id, missing_task_ids) in &resubmit_successful_stages { - if let Some(stage) = self.stages.get_mut(stage_id) { - if let ExecutionStage::Successful(success_stage) = stage { - for task_id in missing_task_ids { - if *task_id >= success_stage.task_infos.len() { - return Err(BallistaError::Internal(format!( - "Invalid task_id {} in map stage {} (task_infos has {} entries)", - *task_id, - stage_id, - success_stage.task_infos.len() - ))); - } - let task_info = &mut success_stage.task_infos[*task_id]; - // Update the task info to failed - task_info.task_status = task_status::Status::Failed(FailedTask { - error: "FetchPartitionError in parent stage".to_owned(), - retryable: true, - count_to_failures: false, - failed_reason: Some(FailedReason::ResultLost(ResultLost {})), - }); - } - } else { - warn!( - "Stage {job_id}/{stage_id} is not in Successful state when try to resubmit this stage. " - ); - } - } else { - return Err(BallistaError::Internal(format!( - "Invalid stage ID {stage_id} for job {job_id}" - ))); - } - } - - for (stage_id, missing_task_ids) in &reset_running_stages { - if let Some(stage) = self.stages.get_mut(stage_id) { - if let ExecutionStage::Running(running_stage) = stage { - for task_id in missing_task_ids { - if *task_id >= running_stage.task_infos.len() { - return Err(BallistaError::Internal(format!( - "Invalid task_id {} in map stage {} (task_infos has {} entries)", - *task_id, - stage_id, - running_stage.task_infos.len() - ))); - } - running_stage.reset_task_info(*task_id); - } - } else { - warn!( - "Stage {job_id}/{stage_id} is not in Running state when try to reset the running task. " - ); - } - } else { - return Err(BallistaError::Internal(format!( - "Invalid stage ID {stage_id} for job {job_id}" - ))); - } - } - self.processing_stages_update(UpdatedStages { resolved_stages: Default::default(), successful_stages, failed_stages, - rollback_running_stages, - resubmit_successful_stages: resubmit_successful_stages - .keys() - .cloned() - .collect(), + // A fetch failure fails its stage instead of rolling it back and + // rerunning its input; see above. + rollback_running_stages: Default::default(), + resubmit_successful_stages: Default::default(), }) } @@ -1555,7 +1484,9 @@ impl ExecutionGraph for AdaptiveExecutionGraph { } } - /// Convert successful stage to be running + /// Convert successful stage to be running. The adaptive graph does not + /// call this: its planner no longer tracks a finished stage, so it fails + /// the job instead (see [cannot_rerun_finished_stage]). fn rerun_successful_stage(&mut self, stage_id: usize) -> bool { if let Some(ExecutionStage::Successful(stage)) = self.stages.remove(&stage_id) { self.stages diff --git a/ballista/scheduler/src/state/aqe/test/job_failure.rs b/ballista/scheduler/src/state/aqe/test/job_failure.rs index 2534a03976..6402e4d7a9 100644 --- a/ballista/scheduler/src/state/aqe/test/job_failure.rs +++ b/ballista/scheduler/src/state/aqe/test/job_failure.rs @@ -17,10 +17,15 @@ //! Job-failure lifecycle tests for the adaptive execution graph. +use crate::scheduler_server::event::QueryStageSchedulerEvent; use crate::state::aqe::AdaptiveExecutionGraph; +use crate::state::aqe::test::{mock_batch, mock_context}; use crate::state::execution_graph::ExecutionGraph; use crate::state::execution_stage::ExecutionStage; -use crate::test_utils::{join_aggregate_sort_logical_plan, mock_executor}; +use crate::test_utils::{ + fetch_partition_failure, join_aggregate_sort_logical_plan, mock_executor, + mock_failed_task, revive_graph_and_complete_next_stage_with_executor, +}; use ballista_core::error::Result; use ballista_core::serde::protobuf::{ FailedTask, JobStatus, ShuffleWritePartition, SuccessfulTask, failed_task, @@ -52,6 +57,109 @@ async fn test_join_plan(partition: usize) -> AdaptiveExecutionGraph { .unwrap() } +/// Builds an adaptive graph for an aggregation: a map stage, then a reduce +/// stage reading its shuffle output. +async fn test_aggregation_plan() -> AdaptiveExecutionGraph { + let ctx = SessionContext::new_with_config(mock_context().copied_config()); + ctx.register_batch("t", mock_batch().unwrap()).unwrap(); + let sql = "select c, sum(a) from t group by c"; + let plan = ctx.sql(sql).await.unwrap().logical_plan().clone(); + AdaptiveExecutionGraph::try_new( + "localhost:50050", + &"job".into(), + "", + &ctx, + &plan, + 0, + None, + ) + .await + .unwrap() +} + +/// Runs the map stage of an aggregation on executor "a", leaving the reduce +/// stage, which reads its output, running. +async fn aggregation_with_map_output_on_a() -> AdaptiveExecutionGraph { + let mut graph = test_aggregation_plan().await; + revive_graph_and_complete_next_stage_with_executor( + &mut graph, + &mock_executor("a".to_string()), + ) + .unwrap(); + graph.revive(); + assert!(matches!(graph.stages[&0], ExecutionStage::Successful(_))); + assert!(matches!(graph.stages[&1], ExecutionStage::Running(_))); + graph +} + +const CANNOT_RERUN: &str = "cannot rerun a finished stage"; + +// AQE cannot rerun a finished stage whose output was lost (#1986): its +// planner no longer tracks the stage, so the rerun's tasks could not report +// and the job hung (#2539). Losing the executor with that output fails the +// rollback instead, and the task manager fails the job on that error. +#[tokio::test] +async fn losing_the_output_of_a_finished_stage_fails_the_rollback() { + let mut graph = aggregation_with_map_output_on_a().await; + + let err = graph.reset_stages_on_lost_executor("a").unwrap_err(); + + assert!(err.to_string().contains(CANNOT_RERUN), "{err}"); +} + +// Nothing in the adaptive graph resolves a stage again once it is rolled back, +// so losing a stage's input fails the job even when the input is not a +// finished stage of this graph, as when it was dropped from it. +#[tokio::test] +async fn losing_the_input_of_a_stage_fails_the_rollback() { + let mut graph = aggregation_with_map_output_on_a().await; + graph.stages.remove(&0); + + let err = graph.reset_stages_on_lost_executor("a").unwrap_err(); + + assert!( + err.to_string() + .contains("stage 1 lost input on executor a, and adaptive query execution cannot roll back a stage"), + "{err}" + ); + assert!(matches!(graph.stages[&1], ExecutionStage::Running(_))); +} + +// The same loss, found by a reduce task failing to fetch the map output: the +// job fails instead of hanging. +#[tokio::test] +async fn a_fetch_failure_on_the_output_of_a_finished_stage_fails_the_job() -> Result<()> { + let mut graph = aggregation_with_map_output_on_a().await; + let b = mock_executor("b".to_string()); + let task = graph.pop_next_task(&b.id)?.unwrap(); + let status = mock_failed_task(task, fetch_partition_failure("a", 0, 0)); + + let events = graph.update_task_status(&b, vec![status], 4, 4)?; + + // The event aborts the job and cancels its tasks. The failed stage is the + // reading stage 1, not the finished stage 0 whose output was lost. + let fail_message = events + .iter() + .find_map(|event| match event { + QueryStageSchedulerEvent::JobRunningFailed { fail_message, .. } => { + Some(fail_message) + } + _ => None, + }) + .expect("a JobRunningFailed event"); + assert!(fail_message.contains("stage 1 failed"), "{fail_message}"); + assert!( + fail_message.contains("finished stage 0 was lost"), + "{fail_message}" + ); + assert!(fail_message.contains(CANNOT_RERUN), "{fail_message}"); + assert!(matches!( + graph.status().status, + Some(job_status::Status::Failed(_)) + )); + Ok(()) +} + // Same contract as the static graph's test: aborting transitions every running // stage to Failed and returns its in-flight tasks for cancellation. #[tokio::test] diff --git a/ballista/scheduler/src/state/aqe/test/reuse_exchange.rs b/ballista/scheduler/src/state/aqe/test/reuse_exchange.rs index cc1e5b3ed8..884308b9d3 100644 --- a/ballista/scheduler/src/state/aqe/test/reuse_exchange.rs +++ b/ballista/scheduler/src/state/aqe/test/reuse_exchange.rs @@ -25,7 +25,6 @@ use crate::state::aqe::test::{ mock_batch, mock_context, mock_partitions_with_statistics, }; use crate::state::execution_graph::ExecutionGraph; -use crate::state::execution_stage::ExecutionStage; use crate::test_utils::{ mock_executor, revive_graph_and_complete_next_stage, revive_graph_and_complete_next_stage_with_executor, @@ -141,11 +140,10 @@ async fn a_job_reading_an_aggregate_twice_runs_one_stage_fewer( } // AQE does not yet rerun a finished stage whose output was lost (#1986, -// #2539), with or without reuse. What reuse adds is a stage with more than -// one reader, and losing its output must roll back the one stage and every -// stage reading it. +// #2539), with or without reuse, and fails the job instead. What reuse adds +// is a stage with more than one reader; losing its output fails the same way. #[tokio::test] -async fn losing_a_shared_stage_output_rolls_back_the_stage_once() { +async fn losing_a_shared_stage_output_fails_the_rollback() { let mut graph = graph(&context(true), UNION).await; revive_graph_and_complete_next_stage_with_executor( &mut graph, @@ -154,12 +152,14 @@ async fn losing_a_shared_stage_output_rolls_back_the_stage_once() { .unwrap(); graph.revive(); - let (reset, _) = graph.reset_stages_on_lost_executor("a").unwrap(); + let err = graph.reset_stages_on_lost_executor("a").unwrap_err(); - assert_eq!(graph.stages.len(), 2); - assert!(reset.contains(&0) && reset.contains(&1)); - assert!(matches!(graph.stages[&0], ExecutionStage::Running(_))); - assert!(matches!(graph.stages[&1], ExecutionStage::UnResolved(_))); + assert!( + err.to_string().contains( + "finished stage 0 was lost, and adaptive query execution cannot rerun" + ), + "{err}" + ); } fn exchange(plan_id: usize) -> ExchangeExec { diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs index 14a7b3b3a2..6364388ecf 100644 --- a/ballista/scheduler/src/state/execution_graph.rs +++ b/ballista/scheduler/src/state/execution_graph.rs @@ -1492,7 +1492,8 @@ impl ExecutionGraph for StaticExecutionGraph { } } - /// Convert unresolved stage to be resolved + /// The stages reading a lost external input are rolled back, and wait + /// unresolved until `resolve_external_input` provides it again. fn recovers_external_inputs(&self) -> bool { true } @@ -1538,6 +1539,7 @@ impl ExecutionGraph for StaticExecutionGraph { }) } + /// Convert unresolved stage to be resolved fn resolve_stage(&mut self, stage_id: usize) -> Result { if let Some(ExecutionStage::UnResolved(stage)) = self.stages.get(&stage_id) { let resolved_stage = stage.to_resolved(self.session_config.options())?; diff --git a/docs/source/user-guide/tuning-guide.md b/docs/source/user-guide/tuning-guide.md index 2fba2ae074..6b8fd70880 100644 --- a/docs/source/user-guide/tuning-guide.md +++ b/docs/source/user-guide/tuning-guide.md @@ -430,7 +430,9 @@ currently implemented: The implementation covers the happy path only. The following are known to be missing or incomplete: -- Executor failure handling on the AQE path ([#1986](https://github.com/apache/datafusion-ballista/issues/1986)) +- Executor failure handling on the AQE path ([#1986](https://github.com/apache/datafusion-ballista/issues/1986)): + a job that loses the output of a stage that has already finished fails + instead of rerunning the stage - Switching from hash join to sort-merge join based on runtime statistics ([#1988](https://github.com/apache/datafusion-ballista/issues/1988)) - Switching from streaming aggregation to hash aggregation based on runtime statistics ([#1989](https://github.com/apache/datafusion-ballista/issues/1989))