Skip to content
185 changes: 58 additions & 127 deletions ballista/scheduler/src/state/aqe/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ use ballista_core::execution_plans::{
use ballista_core::serde::protobuf::failed_task::FailedReason;
use ballista_core::serde::protobuf::job_status::Status;
use ballista_core::serde::protobuf::{
FailedJob, FailedTask, JobStatus, ResultLost, RunningJob, SuccessfulJob, TaskStatus,
job_status, task_status,
FailedJob, JobStatus, RunningJob, SuccessfulJob, TaskStatus, job_status, task_status,
};
use ballista_core::serde::scheduler::{
ExecutorEndpoint, ExecutorMetadata, PartitionLocation,
Expand Down Expand Up @@ -92,7 +91,9 @@ mod test;
///
/// with many limitations, such as:
///
/// - it does not cover executor failure
/// - executor failure only partly: tasks of a running stage that ran on a
/// lost executor are rerun, but a finished stage whose output was lost is
/// not, and the job fails instead
/// - dynamically coalescing shuffle partitions, not supported yet
/// - does not switch from hash join to sort merge join
/// - does not switch from streaming aggregation to hash aggregation
Expand Down Expand Up @@ -327,10 +328,6 @@ impl AdaptiveExecutionGraph {
running_tasks_to_cancel.extend(tasks);
}

for stage_id in updated_stages.resubmit_successful_stages {
self.rerun_successful_stage(stage_id);
}

if !running_tasks_to_cancel.is_empty() {
events.push(QueryStageSchedulerEvent::CancelTasks(
running_tasks_to_cancel,
Expand Down Expand Up @@ -774,30 +771,33 @@ impl AdaptiveExecutionGraph {
}
});
}

for stage_id in rollback_resolved_stages.iter() {
self.rollback_resolved_stage(*stage_id)?;
}

let mut all_running_tasks = vec![];
for stage_id in rollback_running_stages.iter() {
let tasks = self.rollback_running_stage(
// The lost executor held output of a finished stage that later stages
// still need, and AQE cannot rerun a finished stage yet. The caller
// fails the job on this error rather than leaving it to hang.
if let Some(stage_id) = resubmit_successful_stages.iter().min() {
return Err(BallistaError::General(cannot_rerun_finished_stage(
*stage_id,
HashSet::from([executor_id.to_owned()]),
)?;
all_running_tasks.extend(tasks);
)));
}

for stage_id in resubmit_successful_stages.iter() {
self.rerun_successful_stage(*stage_id);
// Any other stage whose input was on the lost executor would be rolled
// back to unresolved, and nothing in the adaptive graph resolves a
// stage again: stages only come resolved from the planner. Fail rather
// than leave it to hang.
if let Some(stage_id) = rollback_resolved_stages
.union(&rollback_running_stages)
.min()
{
return Err(BallistaError::General(format!(
"stage {stage_id} lost input on executor {executor_id}, and \
adaptive query execution cannot roll back a stage yet; retry \
the query, or set ballista.planner.adaptive.enabled to false \
to use the static planner, which recovers lost output"
)));
}

let mut reset_stage = HashSet::new();
reset_stage.extend(reset_running_stage);
reset_stage.extend(rollback_resolved_stages);
reset_stage.extend(rollback_running_stages);
reset_stage.extend(resubmit_successful_stages);
Ok((reset_stage, all_running_tasks))
// Only tasks of running stages were reset; they are rescheduled, and
// none are left running to cancel.
Ok((reset_running_stage, vec![]))
}

/// Clear the stage failure count for this stage if the stage is finally success
Expand All @@ -806,6 +806,19 @@ impl AdaptiveExecutionGraph {
}
}

/// Why a job fails when it needs the lost output of finished stage
/// `stage_id`: AQE cannot rerun a finished stage (#1986, #2539). Its planner
/// no longer tracks the stage, so the rerun's tasks could not report, and
/// the job would hang.
fn cannot_rerun_finished_stage(stage_id: usize) -> String {
format!(
"the output of finished stage {stage_id} was lost, and adaptive query \
execution cannot rerun a finished stage yet; retry the query, or set \
ballista.planner.adaptive.enabled to false to use the static planner, \
which recovers lost output"
)
}

impl ExecutionGraph for AdaptiveExecutionGraph {
fn cloned(&self) -> ExecutionGraphBox {
Box::new(self.clone())
Expand Down Expand Up @@ -938,10 +951,6 @@ impl ExecutionGraph for AdaptiveExecutionGraph {

let mut successful_stages = HashSet::new();
let mut failed_stages = HashMap::new();
let mut rollback_running_stages = HashMap::new();
let mut resubmit_successful_stages: HashMap<usize, HashSet<usize>> =
HashMap::new();
let reset_running_stages: HashMap<usize, HashSet<usize>> = HashMap::new();

//
//
Expand Down Expand Up @@ -1029,32 +1038,16 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
error!("{error_msg}");
failed_stages.insert(stage_id, error_msg);
} else {
// There are different removal strategies here.
// We can choose just remove the map_partition_id in the FetchPartitionError, when resubmit the input stage, there are less tasks
// need to rerun, but this might miss many more bad input partitions, lead to more stage level retries in following.
// Here we choose remove all the bad input partitions which match the same executor id in this single input stage.
// There are other more aggressive approaches, like considering the executor is lost and check all the running stages in this graph.
// Or count the fetch failure number on executor and mark the executor lost globally.
let removed_map_partitions = running_stage
.remove_input_partitions(
map_stage_id,
map_partition_id,
&executor_id,
)?;

let failure_reasons = rollback_running_stages
.entry(stage_id)
.or_insert_with(HashSet::new);
failure_reasons.insert(executor_id);

let missing_inputs =
resubmit_successful_stages
.entry(map_stage_id)
.or_default();
missing_inputs.extend(removed_map_partitions);
warn!(
"Need to resubmit the current running Stage {stage_id} and its map Stage {map_stage_id} due to FetchPartitionError from task {task_identity}"
)
// Recovering would rerun the finished map
// stage, which AQE cannot do yet: fail this
// reading stage, and with it the job,
// instead of hanging it.
let error_msg =
cannot_rerun_finished_stage(map_stage_id);
error!(
"Stage {stage_id} failed to fetch partition {map_partition_id} of stage {map_stage_id} from executor {executor_id}: {error_msg}"
);
failed_stages.insert(stage_id, error_msg);
}
} else {
let mut error_msg = format!(
Expand Down Expand Up @@ -1172,8 +1165,7 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
// all task updates have been handled
//

let is_final_successful = running_stage.is_successful()
&& !reset_running_stages.contains_key(&stage_id);
let is_final_successful = running_stage.is_successful();

if is_final_successful {
successful_stages.insert(stage_id);
Expand Down Expand Up @@ -1244,77 +1236,14 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
.insert(*stage_id, HashSet::from_iter(attempts.iter().copied()));
}

// Values coming out of `remove_input_partitions` are task_ids
// (append slots) under the multi-partition-task model (see the mirror
// site in `execution_graph.rs` for the TODO on partition-id semantics).
for (stage_id, missing_task_ids) in &resubmit_successful_stages {
if let Some(stage) = self.stages.get_mut(stage_id) {
if let ExecutionStage::Successful(success_stage) = stage {
for task_id in missing_task_ids {
if *task_id >= success_stage.task_infos.len() {
return Err(BallistaError::Internal(format!(
"Invalid task_id {} in map stage {} (task_infos has {} entries)",
*task_id,
stage_id,
success_stage.task_infos.len()
)));
}
let task_info = &mut success_stage.task_infos[*task_id];
// Update the task info to failed
task_info.task_status = task_status::Status::Failed(FailedTask {
error: "FetchPartitionError in parent stage".to_owned(),
retryable: true,
count_to_failures: false,
failed_reason: Some(FailedReason::ResultLost(ResultLost {})),
});
}
} else {
warn!(
"Stage {job_id}/{stage_id} is not in Successful state when try to resubmit this stage. "
);
}
} else {
return Err(BallistaError::Internal(format!(
"Invalid stage ID {stage_id} for job {job_id}"
)));
}
}

for (stage_id, missing_task_ids) in &reset_running_stages {
if let Some(stage) = self.stages.get_mut(stage_id) {
if let ExecutionStage::Running(running_stage) = stage {
for task_id in missing_task_ids {
if *task_id >= running_stage.task_infos.len() {
return Err(BallistaError::Internal(format!(
"Invalid task_id {} in map stage {} (task_infos has {} entries)",
*task_id,
stage_id,
running_stage.task_infos.len()
)));
}
running_stage.reset_task_info(*task_id);
}
} else {
warn!(
"Stage {job_id}/{stage_id} is not in Running state when try to reset the running task. "
);
}
} else {
return Err(BallistaError::Internal(format!(
"Invalid stage ID {stage_id} for job {job_id}"
)));
}
}

self.processing_stages_update(UpdatedStages {
resolved_stages: Default::default(),
successful_stages,
failed_stages,
rollback_running_stages,
resubmit_successful_stages: resubmit_successful_stages
.keys()
.cloned()
.collect(),
// A fetch failure fails its stage instead of rolling it back and
// rerunning its input; see above.
rollback_running_stages: Default::default(),
resubmit_successful_stages: Default::default(),
})
}

Expand Down Expand Up @@ -1555,7 +1484,9 @@ impl ExecutionGraph for AdaptiveExecutionGraph {
}
}

/// Convert successful stage to be running
/// Convert successful stage to be running. The adaptive graph does not
/// call this: its planner no longer tracks a finished stage, so it fails
/// the job instead (see [cannot_rerun_finished_stage]).
fn rerun_successful_stage(&mut self, stage_id: usize) -> bool {
if let Some(ExecutionStage::Successful(stage)) = self.stages.remove(&stage_id) {
self.stages
Expand Down
110 changes: 109 additions & 1 deletion ballista/scheduler/src/state/aqe/test/job_failure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,15 @@

//! Job-failure lifecycle tests for the adaptive execution graph.

use crate::scheduler_server::event::QueryStageSchedulerEvent;
use crate::state::aqe::AdaptiveExecutionGraph;
use crate::state::aqe::test::{mock_batch, mock_context};
use crate::state::execution_graph::ExecutionGraph;
use crate::state::execution_stage::ExecutionStage;
use crate::test_utils::{join_aggregate_sort_logical_plan, mock_executor};
use crate::test_utils::{
fetch_partition_failure, join_aggregate_sort_logical_plan, mock_executor,
mock_failed_task, revive_graph_and_complete_next_stage_with_executor,
};
use ballista_core::error::Result;
use ballista_core::serde::protobuf::{
FailedTask, JobStatus, ShuffleWritePartition, SuccessfulTask, failed_task,
Expand Down Expand Up @@ -52,6 +57,109 @@ async fn test_join_plan(partition: usize) -> AdaptiveExecutionGraph {
.unwrap()
}

/// Builds an adaptive graph for an aggregation: a map stage, then a reduce
/// stage reading its shuffle output.
async fn test_aggregation_plan() -> AdaptiveExecutionGraph {
let ctx = SessionContext::new_with_config(mock_context().copied_config());
ctx.register_batch("t", mock_batch().unwrap()).unwrap();
let sql = "select c, sum(a) from t group by c";
let plan = ctx.sql(sql).await.unwrap().logical_plan().clone();
AdaptiveExecutionGraph::try_new(
"localhost:50050",
&"job".into(),
"",
&ctx,
&plan,
0,
None,
)
.await
.unwrap()
}

/// Runs the map stage of an aggregation on executor "a", leaving the reduce
/// stage, which reads its output, running.
async fn aggregation_with_map_output_on_a() -> AdaptiveExecutionGraph {
let mut graph = test_aggregation_plan().await;
revive_graph_and_complete_next_stage_with_executor(
&mut graph,
&mock_executor("a".to_string()),
)
.unwrap();
graph.revive();
assert!(matches!(graph.stages[&0], ExecutionStage::Successful(_)));
assert!(matches!(graph.stages[&1], ExecutionStage::Running(_)));
graph
}

const CANNOT_RERUN: &str = "cannot rerun a finished stage";

// AQE cannot rerun a finished stage whose output was lost (#1986): its
// planner no longer tracks the stage, so the rerun's tasks could not report
// and the job hung (#2539). Losing the executor with that output fails the
// rollback instead, and the task manager fails the job on that error.
#[tokio::test]
async fn losing_the_output_of_a_finished_stage_fails_the_rollback() {
let mut graph = aggregation_with_map_output_on_a().await;

let err = graph.reset_stages_on_lost_executor("a").unwrap_err();

assert!(err.to_string().contains(CANNOT_RERUN), "{err}");
}

// Nothing in the adaptive graph resolves a stage again once it is rolled back,
// so losing a stage's input fails the job even when the input is not a
// finished stage of this graph, as when it was dropped from it.
#[tokio::test]
async fn losing_the_input_of_a_stage_fails_the_rollback() {
let mut graph = aggregation_with_map_output_on_a().await;
graph.stages.remove(&0);

let err = graph.reset_stages_on_lost_executor("a").unwrap_err();

assert!(
err.to_string()
.contains("stage 1 lost input on executor a, and adaptive query execution cannot roll back a stage"),
"{err}"
);
assert!(matches!(graph.stages[&1], ExecutionStage::Running(_)));
}

// The same loss, found by a reduce task failing to fetch the map output: the
// job fails instead of hanging.
#[tokio::test]
async fn a_fetch_failure_on_the_output_of_a_finished_stage_fails_the_job() -> Result<()> {
let mut graph = aggregation_with_map_output_on_a().await;
let b = mock_executor("b".to_string());
let task = graph.pop_next_task(&b.id)?.unwrap();
let status = mock_failed_task(task, fetch_partition_failure("a", 0, 0));

let events = graph.update_task_status(&b, vec![status], 4, 4)?;

// The event aborts the job and cancels its tasks. The failed stage is the
// reading stage 1, not the finished stage 0 whose output was lost.
let fail_message = events
.iter()
.find_map(|event| match event {
QueryStageSchedulerEvent::JobRunningFailed { fail_message, .. } => {
Some(fail_message)
}
_ => None,
})
.expect("a JobRunningFailed event");
assert!(fail_message.contains("stage 1 failed"), "{fail_message}");
assert!(
fail_message.contains("finished stage 0 was lost"),
"{fail_message}"
);
assert!(fail_message.contains(CANNOT_RERUN), "{fail_message}");
assert!(matches!(
graph.status().status,
Some(job_status::Status::Failed(_))
));
Ok(())
}

// Same contract as the static graph's test: aborting transitions every running
// stage to Failed and returns its in-flight tasks for cancellation.
#[tokio::test]
Expand Down
Loading
Loading