From bb3a717a9840a4eef50ad36f5be772e0232de964 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 10 Nov 2026 20:58:28 -0500 Subject: [PATCH 01/17] Restrict any leaf a task runs no partition of to an empty node A task that owns none of a leaf's partitions, such as one running another branch of a UnionExec, never executes it, so the leaf can be replaced by a zero-partition EmptyExec whatever it is. Leaves the scheduler doesn't know, such as a DataSourceExec over a custom source, no longer fail such a task, and need no resolver for it, which the standalone scheduler can't take. The static planner's dry run now empties these branches too, so a union branch over an UnresolvedShuffleExec stays in the union's stage instead of getting one of its own. --- ballista/scheduler/src/planner.rs | 8 ++++ .../scheduler/src/state/execution_graph.rs | 8 ++-- ballista/scheduler/src/state/task_builder.rs | 37 ++++++++++++++----- docs/source/upgrading/56.0.0.md | 5 ++- .../source/user-guide/extending-components.md | 2 +- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index c8990ee700..975545a2ed 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -889,6 +889,7 @@ pub(crate) fn create_shuffle_writer_with_config( mod test { use super::{can_stay_inline, holds_a_broadcast}; use crate::planner::{DefaultDistributedPlanner, DistributedPlanner}; + use crate::test_utils::payload::PayloadExec; use crate::test_utils::{datafusion_test_context, scan_with_file_groups}; use ballista_core::assert_plan; use ballista_core::error::BallistaError; @@ -925,6 +926,13 @@ mod test { assert!(can_stay_inline(&scan_with_file_groups(4))); } + /// A branch over a leaf the scheduler doesn't know restricts away too: a + /// task that runs none of its partitions gets an empty node. + #[test] + fn unknown_leaf_union_branch_stays_inline() { + assert!(can_stay_inline(&PayloadExec::leaf(&["p0", "p1"]))); + } + /// A branch already sitting behind a stage boundary needs nothing more. #[test] fn branch_behind_a_stage_boundary_stays_inline() { diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs index 60246b1793..723643642f 100644 --- a/ballista/scheduler/src/state/execution_graph.rs +++ b/ballista/scheduler/src/state/execution_graph.rs @@ -3268,7 +3268,7 @@ mod test { #[tokio::test] async fn losing_own_shuffle_output_restores_the_cache_reader() -> Result<()> { /// The own input's stage id, which the cache's build stage shares. - const OWN_INPUT: usize = 2; + const OWN_INPUT: usize = 1; fn cache_reader_of(plan: &Arc) -> Option<&ShuffleReaderExec> { plan.downcast_ref::() .filter(|reader| is_external_stage_id(reader.stage_id)) @@ -3304,9 +3304,8 @@ mod test { .expect("a stage reads the cache and its own input"); assert_eq!(own_input, OWN_INPUT); - // The job's own stages complete on executor 1, then a task of the stage - // reading both their output and the cache starts on executor 2. - revive_graph_and_complete_next_stage(&mut graph)?; + // The job's own input completes on executor 1, then a task of the + // stage reading both its output and the cache starts on executor 2. revive_graph_and_complete_next_stage(&mut graph)?; graph.revive(); let executor2 = mock_executor("executor-id2".to_string()); @@ -3326,7 +3325,6 @@ mod test { // The own input re-runs, one task on the cache's executor. The late // task then fails fetching the cache, under the own input's stage id: // the own task's output there is kept, not re-run. - revive_graph_and_complete_next_stage(&mut graph)?; graph.revive(); let cache_executor = mock_executor("executor-cache".to_string()); let own_task = graph.pop_next_task(&cache_executor.id)?.unwrap(); diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index db42e01ba7..ba6a04bbd5 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -69,8 +69,9 @@ use std::sync::Arc; /// partition. /// /// Nodes Ballista does not list are restricted by `resolver`. A leaf that -/// nothing can restrict is an error unless `partitions` is all of its -/// partitions, in order. +/// nothing can restrict becomes an empty node when `partitions` is empty, and +/// is otherwise an error unless `partitions` is all of its partitions, in +/// order. pub fn restrict_plan_to_partitions( plan: Arc, partitions: &[usize], @@ -167,9 +168,14 @@ fn restrict( return checked_restriction(plan.name(), restricted?, partitions); } - // A leaf nothing above could restrict is only right if the task runs all - // of it. + // A leaf nothing above could restrict. A task that runs none of its + // partitions, such as one owning another branch of a `UnionExec`, never + // executes it, whatever it is. Otherwise it is only right if the task + // runs all of it. if children.is_empty() { + if partitions.is_empty() { + return Ok(Arc::new(EmptyExec::new(plan.schema()).with_partitions(0))); + } ensure_leaf_runs_whole(&plan, partitions)?; return Ok(plan); } @@ -215,7 +221,7 @@ fn checked_restriction( /// Accept a leaf that nothing could restrict only if the task runs all of it, /// in order. Left as it is, it numbers its partitions from zero, so any other /// slice would run the wrong ones. A one-partition leaf therefore passes in the -/// task that owns it, and fails in a `UnionExec` branch that belongs to another. +/// task that owns it. fn ensure_leaf_runs_whole( plan: &Arc, partitions: &[usize], @@ -920,9 +926,14 @@ mod tests { } #[test] - fn a_leaf_nobody_can_restrict_is_refused_unless_the_task_runs_all_of_it() { + fn a_leaf_nobody_can_restrict_is_refused_unless_the_task_runs_all_or_none_of_it() { let leaf = PayloadExec::leaf(&["p0", "p1"]); + let none = restrict_plan_to_partitions(leaf.clone(), &[], None).unwrap(); + assert!(none.is::()); + assert_eq!(partition_count(&none), 0); + assert_eq!(none.schema(), leaf.schema()); + let kept = restrict_plan_to_partitions(leaf.clone(), &[0, 1], None).unwrap(); assert!( Arc::ptr_eq(&kept, &leaf), @@ -933,7 +944,7 @@ mod tests { let single = PayloadExec::leaf(&["p0"]); assert!(restrict_plan_to_partitions(single, &[0], None).is_ok()); - for partitions in [&[0][..], &[1], &[1, 0], &[]] { + for partitions in [&[0][..], &[1], &[1, 0]] { assert!( restrict_plan_to_partitions(leaf.clone(), partitions, None).is_err(), "{partitions:?} is not all of the leaf, in order" @@ -1012,8 +1023,9 @@ mod tests { ); } - /// A union branch that belongs to another task must restrict to nothing, - /// which needs the resolver for a node Ballista does not know. + /// A union branch that belongs to another task restricts to nothing, even + /// a node Ballista does not know. Restricting one to some of its + /// partitions needs the resolver. #[test] fn a_union_branch_owned_by_another_task_restricts_to_nothing() { let resolver = payload_resolver(); @@ -1039,7 +1051,12 @@ mod tests { assert_eq!(group_files(branches[0]), ["file0.parquet"]); assert_eq!(partition_count(branches[1]), 0); - assert!(restrict_plan_to_partitions(union(), &[0], None).is_err()); + let restricted = restrict_plan_to_partitions(union(), &[0], None).unwrap(); + let branches = restricted.children(); + assert_eq!(group_files(branches[0]), ["file0.parquet"]); + assert!(branches[1].is::()); + assert_eq!(partition_count(branches[1]), 0); + assert!(restrict_plan_to_partitions(union(), &[2], None).is_err()); } diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index 6ee1bb928b..5c91d8e74f 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -187,8 +187,9 @@ and 1, and the stage returned some partitions twice and others not at all, with no error. The scheduler now fails the job instead. Such a leaf is accepted only in a task -that runs all of its partitions, in order, so a retry of part of a stage can -fail a job too. Plans made of Ballista's own leaves (shuffle readers, file and +that runs all of its partitions, in order, or none of them, as in a `UnionExec` +branch another task owns, where it becomes an empty node. A retry of part of a +stage can fail a job too. Plans made of Ballista's own leaves (shuffle readers, file and in-memory scans, `PlaceholderRowExec`, `EmptyExec`) are unaffected, as are leaves read whole below a `CoalescePartitionsExec` or on a join's build side. A `DataSourceExec` over any other source used to log a warning and run, and is diff --git a/docs/source/user-guide/extending-components.md b/docs/source/user-guide/extending-components.md index 2228bf3425..d69124043a 100644 --- a/docs/source/user-guide/extending-components.md +++ b/docs/source/user-guide/extending-components.md @@ -179,7 +179,7 @@ let config = SchedulerConfig::default().with_partition_sliceable_resolver(Arc::n - `children` are the node's inputs, none for a scan, restricted to `partitions`. An input the node declares `SinglePartition` arrives whole, as a join's build side does. Partition `i` of the node must read partition `i` of every other input. - The restricted node must have exactly `partitions.len()` partitions. The scheduler rejects any other count. -- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. +- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions, or `None` for a leaf: the scheduler replaces a leaf no task partition reads with an empty node itself. - The restricted node is serialized with the scheduler's `physical codec`, so what it carries per partition has to survive encoding and decoding. - The resolver does not need `PartitionSliceable`. It can build the restricted node itself, which is the way to restrict a node you cannot implement the trait on, such as a `DataSourceExec` over a `DataSource` of your own: match it with `downcast_ref`, then build a `DataSourceExec` over the restricted source. - The resolver is asked after Ballista's own list. A node that is read whole, such as one below a `CoalescePartitionsExec` or on a join's build side, needs none of this. From 4578b2f0adeae7978cfc79303930b8d08ecb1c16 Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 11 Nov 2026 09:21:06 -0500 Subject: [PATCH 02/17] Keep an empty union branch empty on the executor, and check branch counts The leaf a task runs none of was an EmptyExec, which DataFusion's proto codec decodes with one partition, so the union on the executor ran the wrong branch. Use a zero-partition MemorySourceConfig, as the EmptyExec and PlaceholderRowExec case already did, through one helper. Also refuse a UnionExec branch that keeps another partition count after restriction, such as one that collapses its input through a SortExec under the adaptive planner, instead of running the wrong branch. --- ballista/scheduler/src/state/task_builder.rs | 146 +++++++++++++++---- docs/source/upgrading/56.0.0.md | 4 +- 2 files changed, 123 insertions(+), 27 deletions(-) diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index ba6a04bbd5..7494028650 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -142,7 +142,11 @@ fn restrict( .into_iter() .cloned() .zip(per_child) - .map(|(child, sub)| restrict(child, &sub, false, resolver)) + .map(|(child, sub)| { + let restricted = restrict(child, &sub, false, resolver)?; + ensure_branch_restricted(&restricted, &sub)?; + Ok(restricted) + }) .collect::>>()?; return replace_children_if_necessary(plan, new_children); } @@ -174,7 +178,7 @@ fn restrict( // runs all of it. if children.is_empty() { if partitions.is_empty() { - return Ok(Arc::new(EmptyExec::new(plan.schema()).with_partitions(0))); + return no_partitions(&plan); } ensure_leaf_runs_whole(&plan, partitions)?; return Ok(plan); @@ -218,6 +222,18 @@ fn checked_restriction( Ok(restricted) } +/// A leaf with `plan`'s schema and no partitions, for a leaf the task runs +/// none of. +/// +/// Not an `EmptyExec`: DataFusion's proto codec decodes one with zero +/// partitions as one partition, so it would reach the executor with a +/// partition the task doesn't own, and a `UnionExec` above it would run the +/// wrong branch. A `MemorySourceConfig` keeps its empty partition list. +fn no_partitions(plan: &Arc) -> Result> { + let source = MemorySourceConfig::try_new(&[], plan.schema(), None)?; + Ok(DataSourceExec::from_data_source(source)) +} + /// Accept a leaf that nothing could restrict only if the task runs all of it, /// in order. Left as it is, it numbers its partitions from zero, so any other /// slice would run the wrong ones. A one-partition leaf therefore passes in the @@ -242,6 +258,32 @@ fn ensure_leaf_runs_whole( ) } +/// Check that a restricted `UnionExec` branch has one partition for each of +/// `partitions`. The union routes the task's partitions by its branches' +/// counts, so a branch with any other count would make the task run the wrong +/// branch. That happens when the branch collapses its input, such as through a +/// `SortExec` that merges its partitions, which reports one partition whatever +/// its leaves do, or holds a node the scheduler doesn't know that keeps its +/// own count. +fn ensure_branch_restricted( + branch: &Arc, + partitions: &[usize], +) -> Result<()> { + let count = branch.properties().output_partitioning().partition_count(); + if count == partitions.len() { + return Ok(()); + } + let name = branch.name(); + plan_err!( + "cannot restrict a `UnionExec` branch to partitions {partitions:?}: its \ + `{name}` still reports {count} partitions, so the union would run the \ + wrong branch. A branch that collapses its input, such as through a \ + `SortExec` or `CoalescePartitionsExec`, needs a stage of its own; a node \ + of your own can be restricted by \ + `SchedulerConfig::partition_sliceable_resolver`." + ) +} + /// Bucket each parent partition of a `UnionExec` into its owning child's /// local index. Iterates over `partitions` and, for each, subtracts each /// child's count until the remainder lands inside a child. Returns one @@ -477,29 +519,16 @@ fn select_output_partitions( // Self-generating leaves: PlaceholderRow (1 all-null row per partition) // and EmptyExec (0 rows per partition). Every partition of one of these - // is semantically identical, so `with_partitions(indices.len())` is the - // algebraically correct restriction. - // - // Caveat: DataFusion's proto codec for these two doesn't roundtrip the - // `partitions` field — encode preserves only schema; decode always - // reconstructs with `::new(schema)` (partitions = 1). So a plan we - // restrict to `with_partitions(0)` here would arrive at the executor as - // a 1-partition leaf, undoing the restriction and reproducing the - // duplicated-output bug this whole function is meant to prevent. - // - // Workaround for the empty-slice case: swap in a 0-partition - // `MemorySourceConfig` (which DOES roundtrip its partition list). - // For non-empty slices we return the leaf as it is — currently safe because - // the only shape emitted is `partitions == 1` and UnionExec routing gives - // sub-slices of `[0]` (identity, no restriction needed) or `[]` (handled by - // this branch). Returned rather than `None` so the walker's leaf check - // accepts it: an `EmptyExec` under the round-robin `RepartitionExec` that + // is semantically identical, so any non-empty slice is the leaf as it is + // — currently safe because the only shape emitted is `partitions == 1` + // and UnionExec routing gives sub-slices of `[0]` (identity) or `[]`. + // Returned rather than `None` so the walker's leaf check accepts it: an + // `EmptyExec` under the round-robin `RepartitionExec` that // `make_empty_exec_serde_safe` adds sees that repartition's partition // numbers, not its own. if plan.is::() || plan.is::() { if indices.is_empty() { - let stub = MemorySourceConfig::try_new(&[], plan.schema(), None)?; - return Ok(Some(DataSourceExec::from_data_source(stub))); + return no_partitions(plan).map(Some); } return Ok(Some(plan.clone())); } @@ -930,9 +959,7 @@ mod tests { let leaf = PayloadExec::leaf(&["p0", "p1"]); let none = restrict_plan_to_partitions(leaf.clone(), &[], None).unwrap(); - assert!(none.is::()); - assert_eq!(partition_count(&none), 0); - assert_eq!(none.schema(), leaf.schema()); + assert_no_partitions(&none, &leaf); let kept = restrict_plan_to_partitions(leaf.clone(), &[0, 1], None).unwrap(); assert!( @@ -1054,12 +1081,79 @@ mod tests { let restricted = restrict_plan_to_partitions(union(), &[0], None).unwrap(); let branches = restricted.children(); assert_eq!(group_files(branches[0]), ["file0.parquet"]); - assert!(branches[1].is::()); - assert_eq!(partition_count(branches[1]), 0); + assert_no_partitions(branches[1], &union().children()[1].clone()); assert!(restrict_plan_to_partitions(union(), &[2], None).is_err()); } + /// The task plan is serialized on its way to the executor, and the branch + /// the task runs none of must still have no partitions there, or the union + /// would run it in place of the task's own. + #[test] + fn a_branch_the_task_runs_none_of_keeps_no_partitions_through_serialization() { + use datafusion::prelude::SessionContext; + use datafusion_proto::bytes::{physical_plan_from_bytes, physical_plan_to_bytes}; + + let union = UnionExec::try_new(vec![ + scan_with_file_groups(2), + PayloadExec::leaf(&["p0"]), + ]) + .unwrap(); + let restricted = restrict_plan_to_partitions(union, &[0], None).unwrap(); + + let bytes = physical_plan_to_bytes(restricted).unwrap(); + let decoded = + physical_plan_from_bytes(&bytes, &SessionContext::new().task_ctx()).unwrap(); + + assert_eq!(partition_count(&decoded), 1); + let branches = decoded.children(); + assert_eq!(group_files(branches[0]), ["file0.parquet"]); + assert_eq!(partition_count(branches[1]), 0); + } + + /// A `UnionExec` branch that keeps another partition count after + /// restriction would make the union run the wrong branch, so it is refused. + #[test] + fn a_union_branch_with_the_wrong_partition_count_is_refused() { + // A node the scheduler doesn't know that holds its own two partitions, + // over a leaf the task runs none of: the leaf empties, the node keeps + // its count. + let branch = + PayloadExec::over(&["p0", "p1"], vec![PayloadExec::leaf(&["q0"])], &[]); + let union = UnionExec::try_new(vec![scan_with_file_groups(2), branch]).unwrap(); + + let error = restrict_plan_to_partitions(union, &[0], None) + .unwrap_err() + .to_string(); + assert_eq!( + error, + "Error during planning: cannot restrict a `UnionExec` branch to \ + partitions []: its `PayloadExec` still reports 2 partitions, so the \ + union would run the wrong branch. A branch that collapses its input, \ + such as through a `SortExec` or `CoalescePartitionsExec`, needs a \ + stage of its own; a node of your own can be restricted by \ + `SchedulerConfig::partition_sliceable_resolver`." + ); + } + + /// `plan` is a leaf with `like`'s schema and no partitions, as the + /// executor will decode it. + fn assert_no_partitions( + plan: &Arc, + like: &Arc, + ) { + let memory = plan + .downcast_ref::() + .and_then(|exec| exec.data_source().downcast_ref::()); + assert!( + memory.is_some(), + "an empty in-memory source, not {}", + plan.name() + ); + assert_eq!(partition_count(plan), 0); + assert_eq!(plan.schema(), like.schema()); + } + #[test] fn a_restriction_that_keeps_the_wrong_partition_count_is_rejected() { let resolver = payload_resolver(); diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index 5c91d8e74f..fd60ba9897 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -205,7 +205,9 @@ Only leaves are checked. An interior node the scheduler does not know is left as it was while its children are restricted, because the scheduler cannot tell one that holds data per partition from one that does not, such as a `FilterExec`. An interior node of your own that holds data per partition still returns wrong -results in a split stage unless a resolver restricts it. +results in a split stage unless a resolver restricts it. Below a `UnionExec`, +the job fails instead when such a node keeps its own partition count, since the +union would run the wrong branch. The error names the node and what to implement. Under push-staged scheduling the job's status says only that a task could not be prepared; the reason is in From 0f1edffb911eeaf0cfe604074377f701f17bf007 Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 11 Nov 2026 10:49:17 -0500 Subject: [PATCH 03/17] Empty every leaf a task runs none of, in one place Check for a leaf the task runs none of at the top of the walk, for every leaf, instead of after the resolver for unknown leaves and separately for EmptyExec and PlaceholderRowExec. A broadcast reader in a union branch another task owns is now emptied too, rather than kept whole. Also: document the union branch check on restrict_plan_to_partitions, word its error without a plural for one partition, and round-trip the serialization test through Ballista's codec. --- ballista/scheduler/src/state/task_builder.rs | 77 +++++++++++-------- .../source/user-guide/extending-components.md | 2 +- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index 7494028650..b477555802 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -68,10 +68,11 @@ use std::sync::Arc; /// side) — those leaves keep the full upstream so the collapse sees every /// partition. /// -/// Nodes Ballista does not list are restricted by `resolver`. A leaf that -/// nothing can restrict becomes an empty node when `partitions` is empty, and -/// is otherwise an error unless `partitions` is all of its partitions, in -/// order. +/// A leaf the task runs none of becomes an empty leaf, whatever it is. Nodes +/// Ballista does not list are restricted by `resolver`, and a leaf that nothing +/// can restrict is an error unless `partitions` is all of its partitions, in +/// order. Each branch of a `UnionExec` must end up with one partition for each +/// partition the task runs of it. pub fn restrict_plan_to_partitions( plan: Arc, partitions: &[usize], @@ -119,6 +120,12 @@ fn restrict( under_collect: bool, resolver: Option<&dyn PartitionSliceableResolver>, ) -> Result> { + // A leaf the task runs none of, such as one in a `UnionExec` branch + // another task owns, is never executed, whatever it is. + if !under_collect && partitions.is_empty() && plan.children().is_empty() { + return no_partitions(&plan); + } + // Leaves: apply the algebraic partition-projection when we're not // under a collapse. Under collapse, leaves keep their full upstream — // the generic recursion below leaves them unchanged. @@ -172,14 +179,9 @@ fn restrict( return checked_restriction(plan.name(), restricted?, partitions); } - // A leaf nothing above could restrict. A task that runs none of its - // partitions, such as one owning another branch of a `UnionExec`, never - // executes it, whatever it is. Otherwise it is only right if the task - // runs all of it. + // A leaf nothing above could restrict is only right if the task runs all + // of it. if children.is_empty() { - if partitions.is_empty() { - return no_partitions(&plan); - } ensure_leaf_runs_whole(&plan, partitions)?; return Ok(plan); } @@ -276,7 +278,7 @@ fn ensure_branch_restricted( let name = branch.name(); plan_err!( "cannot restrict a `UnionExec` branch to partitions {partitions:?}: its \ - `{name}` still reports {count} partitions, so the union would run the \ + `{name}` keeps a partition count of {count}, so the union would run the \ wrong branch. A branch that collapses its input, such as through a \ `SortExec` or `CoalescePartitionsExec`, needs a stage of its own; a node \ of your own can be restricted by \ @@ -519,17 +521,13 @@ fn select_output_partitions( // Self-generating leaves: PlaceholderRow (1 all-null row per partition) // and EmptyExec (0 rows per partition). Every partition of one of these - // is semantically identical, so any non-empty slice is the leaf as it is - // — currently safe because the only shape emitted is `partitions == 1` - // and UnionExec routing gives sub-slices of `[0]` (identity) or `[]`. - // Returned rather than `None` so the walker's leaf check accepts it: an - // `EmptyExec` under the round-robin `RepartitionExec` that - // `make_empty_exec_serde_safe` adds sees that repartition's partition - // numbers, not its own. + // is semantically identical, so a slice is the leaf as it is — currently + // safe because the only shape emitted is `partitions == 1`, and the walker + // has already emptied a leaf for `[]`. Returned rather than `None` so the + // walker's leaf check accepts it: an `EmptyExec` under the round-robin + // `RepartitionExec` that `make_empty_exec_serde_safe` adds sees that + // repartition's partition numbers, not its own. if plan.is::() || plan.is::() { - if indices.is_empty() { - return no_partitions(plan).map(Some); - } return Ok(Some(plan.clone())); } @@ -706,8 +704,12 @@ mod tests { } /// File counts per file group of a `DataSourceExec` — the shape a union - /// test cares about after restriction. + /// test cares about after restriction. None for a leaf with no partitions, + /// which a scan the task doesn't poll becomes. fn group_file_counts(plan: &Arc) -> Vec { + if plan.properties().output_partitioning().partition_count() == 0 { + return vec![]; + } file_groups(plan).iter().map(|g| g.len()).collect() } @@ -1056,12 +1058,9 @@ mod tests { #[test] fn a_union_branch_owned_by_another_task_restricts_to_nothing() { let resolver = payload_resolver(); + let unknown = PayloadExec::leaf(&["p0", "p1"]); let union = || -> Arc { - UnionExec::try_new(vec![ - scan_with_file_groups(2), - PayloadExec::leaf(&["p0", "p1"]), - ]) - .unwrap() + UnionExec::try_new(vec![scan_with_file_groups(2), unknown.clone()]).unwrap() }; // Partition 2 is the second branch's partition 0. @@ -1081,7 +1080,7 @@ mod tests { let restricted = restrict_plan_to_partitions(union(), &[0], None).unwrap(); let branches = restricted.children(); assert_eq!(group_files(branches[0]), ["file0.parquet"]); - assert_no_partitions(branches[1], &union().children()[1].clone()); + assert_no_partitions(branches[1], &unknown); assert!(restrict_plan_to_partitions(union(), &[2], None).is_err()); } @@ -1091,8 +1090,12 @@ mod tests { /// would run it in place of the task's own. #[test] fn a_branch_the_task_runs_none_of_keeps_no_partitions_through_serialization() { + use ballista_core::serde::BallistaPhysicalExtensionCodec; use datafusion::prelude::SessionContext; - use datafusion_proto::bytes::{physical_plan_from_bytes, physical_plan_to_bytes}; + use datafusion_proto::bytes::{ + physical_plan_from_bytes_with_extension_codec, + physical_plan_to_bytes_with_extension_codec, + }; let union = UnionExec::try_new(vec![ scan_with_file_groups(2), @@ -1101,9 +1104,15 @@ mod tests { .unwrap(); let restricted = restrict_plan_to_partitions(union, &[0], None).unwrap(); - let bytes = physical_plan_to_bytes(restricted).unwrap(); - let decoded = - physical_plan_from_bytes(&bytes, &SessionContext::new().task_ctx()).unwrap(); + let codec = BallistaPhysicalExtensionCodec::default(); + let bytes = + physical_plan_to_bytes_with_extension_codec(restricted, &codec).unwrap(); + let decoded = physical_plan_from_bytes_with_extension_codec( + &bytes, + &SessionContext::new().task_ctx(), + &codec, + ) + .unwrap(); assert_eq!(partition_count(&decoded), 1); let branches = decoded.children(); @@ -1128,7 +1137,7 @@ mod tests { assert_eq!( error, "Error during planning: cannot restrict a `UnionExec` branch to \ - partitions []: its `PayloadExec` still reports 2 partitions, so the \ + partitions []: its `PayloadExec` keeps a partition count of 2, so the \ union would run the wrong branch. A branch that collapses its input, \ such as through a `SortExec` or `CoalescePartitionsExec`, needs a \ stage of its own; a node of your own can be restricted by \ diff --git a/docs/source/user-guide/extending-components.md b/docs/source/user-guide/extending-components.md index d69124043a..fa76e74f10 100644 --- a/docs/source/user-guide/extending-components.md +++ b/docs/source/user-guide/extending-components.md @@ -179,7 +179,7 @@ let config = SchedulerConfig::default().with_partition_sliceable_resolver(Arc::n - `children` are the node's inputs, none for a scan, restricted to `partitions`. An input the node declares `SinglePartition` arrives whole, as a join's build side does. Partition `i` of the node must read partition `i` of every other input. - The restricted node must have exactly `partitions.len()` partitions. The scheduler rejects any other count. -- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions, or `None` for a leaf: the scheduler replaces a leaf no task partition reads with an empty node itself. +- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. - The restricted node is serialized with the scheduler's `physical codec`, so what it carries per partition has to survive encoding and decoding. - The resolver does not need `PartitionSliceable`. It can build the restricted node itself, which is the way to restrict a node you cannot implement the trait on, such as a `DataSourceExec` over a `DataSource` of your own: match it with `downcast_ref`, then build a `DataSourceExec` over the restricted source. - The resolver is asked after Ballista's own list. A node that is read whole, such as one below a `CoalescePartitionsExec` or on a join's build side, needs none of this. From 68b0c967e3c81a45890929a345a692a7a648bb00 Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 11 Nov 2026 16:31:13 -0500 Subject: [PATCH 04/17] fix(scheduler): cut multi-partition EmptyExec and PlaceholderRowExec down to a task's partitions The adaptive planner replaces a stage that gives no rows with an EmptyExec of that stage's partition count. Restriction returned it unchanged, so a union branch kept its full count and the new union check failed the job. Each leaf now keeps one partition for each of the task's partitions in its own range, or none. Adds an end-to-end test of split unions under both planners, and notes in the upgrade guide that a union branch that collapses to one partition now fails under the adaptive planner instead of returning wrong rows. --- ballista/client/tests/split_union.rs | 123 +++++++++++++++++ ballista/scheduler/src/state/task_builder.rs | 137 ++++++++++++++++--- docs/source/upgrading/56.0.0.md | 18 ++- 3 files changed, 255 insertions(+), 23 deletions(-) create mode 100644 ballista/client/tests/split_union.rs diff --git a/ballista/client/tests/split_union.rs b/ballista/client/tests/split_union.rs new file mode 100644 index 0000000000..d909f750ad --- /dev/null +++ b/ballista/client/tests/split_union.rs @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; + +/// End-to-end tests of `UNION ALL` queries whose stages are split between +/// tasks. Each task runs only the union branches it owns, so every other +/// branch is restricted to the task's share of it, often none. They must +/// return the rows DataFusion returns. +#[cfg(test)] +#[cfg(feature = "standalone")] +mod split_union { + use crate::common::{ + ballista_state, collect_lines, datafusion_context, orders_customers_dir, + register_alltypes, register_orders_customers, + }; + use ballista::prelude::SessionContextExt; + use ballista_core::config::{ + BALLISTA_ADAPTIVE_PLANNER_ENABLED, BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, + }; + use datafusion::prelude::SessionContext; + use rstest::rstest; + use std::path::Path; + + /// The adaptive planner replaces a stage that gives no rows with an + /// `EmptyExec` of that stage's partition count, so in most of these a + /// union branch is an `EmptyExec` of several partitions. + const QUERIES: &[&str] = &[ + // An aggregate that matches nothing, beside one that matches all. + "SELECT id % 2 AS k, count(*) AS c FROM test WHERE id > 100 GROUP BY id % 2 \ + UNION ALL SELECT id % 2, count(*) FROM test GROUP BY id % 2", + // The same, the other way round. + "SELECT id % 2 AS k, count(*) AS c FROM test GROUP BY id % 2 \ + UNION ALL SELECT id % 2, count(*) FROM test WHERE id > 100 GROUP BY id % 2", + // An empty middle branch of three. + "SELECT o_cust AS k, count(*) AS c FROM orders GROUP BY o_cust \ + UNION ALL SELECT o_cust, count(*) FROM orders WHERE o_id < 0 GROUP BY o_cust \ + UNION ALL SELECT c_id, 1 FROM customers WHERE c_id < 5", + // An aggregate over a union with an empty branch. + "SELECT k, sum(c) AS s FROM ( \ + SELECT o_cust AS k, count(*) AS c FROM orders WHERE o_id < 0 GROUP BY o_cust \ + UNION ALL SELECT o_cust, count(*) FROM orders GROUP BY o_cust) \ + GROUP BY k", + // A join that matches nothing below an aggregate. + "SELECT c_region AS r, count(*) AS n FROM orders JOIN customers ON o_cust = c_id \ + WHERE o_id < 0 GROUP BY c_region \ + UNION ALL SELECT c_region, count(*) FROM customers GROUP BY c_region", + // A literal beside an empty aggregate. + "SELECT 7 AS k, 7 AS c \ + UNION ALL SELECT id % 2, count(*) FROM test WHERE id > 100 GROUP BY id % 2", + // A union with an empty branch joined to a table. + "SELECT x.k, t.c_name FROM ( \ + SELECT o_cust AS k FROM orders WHERE o_id < 0 GROUP BY o_cust \ + UNION ALL SELECT c_id FROM customers WHERE c_id < 3) x \ + JOIN customers t ON x.k = t.c_id", + // A window over a union with an empty branch. + "SELECT k, sum(c) OVER (PARTITION BY k) AS s FROM ( \ + SELECT id % 2 AS k, count(*) AS c FROM test WHERE id > 100 GROUP BY id % 2 \ + UNION ALL SELECT id % 2, count(*) FROM test GROUP BY id % 2)", + // Scans, literals and values. + "SELECT id FROM test UNION ALL SELECT id FROM test", + "SELECT 1 AS a UNION ALL SELECT 2 UNION ALL SELECT 3", + "SELECT 100 AS id UNION ALL SELECT id FROM test", + "SELECT column1 AS id FROM (VALUES (10), (11)) UNION ALL SELECT id FROM test", + ]; + + async fn register(ctx: &SessionContext, dir: &Path) { + register_alltypes(ctx, "test").await; + register_orders_customers(ctx, dir).await; + } + + /// The formatted rows of `sql`, sorted, since a union's rows come in no + /// particular order. + async fn sorted_lines(ctx: &SessionContext, sql: &str) -> Vec { + let mut lines = collect_lines(ctx, sql).await; + lines.sort(); + lines + } + + #[rstest] + #[tokio::test] + async fn split_unions_return_the_rows_datafusion_returns( + #[values(false, true)] adaptive: bool, + #[values(1, 2)] max_partitions_per_task: usize, + ) { + let dir = orders_customers_dir(); + let reference = datafusion_context(); + register(&reference, dir.path()).await; + let ctx = SessionContext::standalone_with_state(ballista_state(&[ + (BALLISTA_ADAPTIVE_PLANNER_ENABLED, &adaptive.to_string()), + ( + BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, + &max_partitions_per_task.to_string(), + ), + ])) + .await + .unwrap(); + register(&ctx, dir.path()).await; + + for sql in QUERIES { + assert_eq!( + sorted_lines(&ctx, sql).await, + sorted_lines(&reference, sql).await, + "adaptive={adaptive} max_partitions_per_task={max_partitions_per_task} \ + differs for: {sql}" + ); + } + } +} diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index b477555802..de946c0d54 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -519,16 +519,24 @@ fn select_output_partitions( return Ok(None); } - // Self-generating leaves: PlaceholderRow (1 all-null row per partition) - // and EmptyExec (0 rows per partition). Every partition of one of these - // is semantically identical, so a slice is the leaf as it is — currently - // safe because the only shape emitted is `partitions == 1`, and the walker - // has already emptied a leaf for `[]`. Returned rather than `None` so the - // walker's leaf check accepts it: an `EmptyExec` under the round-robin - // `RepartitionExec` that `make_empty_exec_serde_safe` adds sees that - // repartition's partition numbers, not its own. + // Self-generating leaves: PlaceholderRow (1 row per partition) and + // EmptyExec (0 rows per partition). Every partition is the same, so a + // slice is the leaf with one partition for each index it has. As for the + // sources above, an index past its own count is dropped: under a + // round-robin `RepartitionExec` the leaf sees the repartition's numbers, + // and only the task that owns partition 0 may run a one-partition leaf. if plan.is::() || plan.is::() { - return Ok(Some(plan.clone())); + let count = plan.properties().output_partitioning().partition_count(); + let kept = indices.iter().filter(|&&i| i < count).count(); + if kept == 0 { + return no_partitions(plan).map(Some); + } + let restricted: Arc = if plan.is::() { + Arc::new(EmptyExec::new(plan.schema()).with_partitions(kept)) + } else { + Arc::new(PlaceholderRowExec::new(plan.schema()).with_partitions(kept)) + }; + return Ok(Some(restricted)); } Ok(None) @@ -1176,20 +1184,113 @@ mod tests { assert!(error.contains("returned 4 partitions"), "{error}"); } - /// An `EmptyExec` sees the partition numbers of the repartition above it, - /// not its own, so these two leaves are exempt from the leaf check. + /// The name and partition count of each restriction of `leaf`, one for + /// each of `slices`. + fn restricted_shapes( + leaf: &Arc, + slices: &[&[usize]], + ) -> Vec<(String, usize)> { + slices + .iter() + .map(|slice| { + let restricted = + restrict_plan_to_partitions(leaf.clone(), slice, None).unwrap(); + assert_eq!(restricted.schema(), leaf.schema()); + (restricted.name().to_string(), partition_count(&restricted)) + }) + .collect() + } + + /// Every partition of these leaves is the same, so a task gets one for + /// each of its partitions. The `AdaptivePlanner` replaces a stage that + /// turns out empty with an `EmptyExec` of that stage's partition count. #[test] - fn self_generating_leaves_are_exempt_from_the_leaf_check() { + fn self_generating_leaves_keep_one_partition_for_each_the_task_runs() { let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); - let empty: Arc = Arc::new(EmptyExec::new(schema.clone())); - assert!(restrict_plan_to_partitions(empty, &[1], None).is_ok()); + let empty: Arc = + Arc::new(EmptyExec::new(schema.clone()).with_partitions(4)); + let placeholder: Arc = + Arc::new(PlaceholderRowExec::new(schema).with_partitions(4)); + let slices: &[&[usize]] = &[&[0, 1, 2, 3], &[1, 3], &[2], &[]]; + let memory = "DataSourceExec".to_string(); + for leaf in [empty, placeholder] { + let name = leaf.name().to_string(); + assert_eq!( + restricted_shapes(&leaf, slices), + [ + (name.clone(), 4), + (name.clone(), 2), + (name, 1), + (memory.clone(), 0), + ] + ); + } + } + + /// Under a round-robin `RepartitionExec`, which the `AdaptivePlanner` + /// keeps inside a stage, a leaf sees the repartition's partition numbers. + /// Only the task that owns partition 0 runs a one-partition leaf, so a + /// `PlaceholderRowExec` there gives its row once, not once per task. + #[test] + fn self_generating_leaves_drop_partitions_past_their_own() { + use datafusion::physical_plan::repartition::RepartitionExec; + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); let placeholder: Arc = Arc::new(PlaceholderRowExec::new(schema)); - assert!(restrict_plan_to_partitions(placeholder.clone(), &[0], None).is_ok()); + let restricted = restrict_plan_to_partitions( + Arc::new( + RepartitionExec::try_new( + placeholder.clone(), + Partitioning::RoundRobinBatch(4), + ) + .unwrap(), + ), + &[2, 3], + None, + ) + .unwrap(); + assert_no_partitions(restricted.children()[0], &placeholder); - // A task that polls none of it gets a source with no partitions. - let nothing = restrict_plan_to_partitions(placeholder, &[], None).unwrap(); - assert_eq!(partition_count(¬hing), 0); + assert_eq!( + restricted_shapes(&placeholder, &[&[0, 1], &[3]]), + [ + ("PlaceholderRowExec".to_string(), 1), + ("DataSourceExec".to_string(), 0), + ] + ); + } + + /// A union branch the `AdaptivePlanner` replaced with a multi-partition + /// `EmptyExec`, because its stage gave no rows, is cut down like any other, + /// so the union routes the task's partitions to the right branch. + #[test] + fn a_union_branch_replaced_by_an_empty_exec_is_restricted() { + let empty: Arc = Arc::new( + EmptyExec::new(scan_with_file_groups(4).schema()).with_partitions(4), + ); + let union = || -> Arc { + UnionExec::try_new(vec![empty.clone(), scan_with_file_groups(4)]).unwrap() + }; + + // Partition 0 is the empty branch's. + let restricted = restrict_plan_to_partitions(union(), &[0], None).unwrap(); + let branches = restricted.children(); + assert_eq!( + (branches[0].name(), partition_count(branches[0])), + ("EmptyExec", 1) + ); + assert_no_partitions(branches[1], &empty); + + // Partitions 3 and 6 are the last of the empty branch's and the third + // of the scan's. + let restricted = restrict_plan_to_partitions(union(), &[3, 6], None).unwrap(); + let branches = restricted.children(); + assert_eq!( + (branches[0].name(), partition_count(branches[0])), + ("EmptyExec", 1) + ); + assert_eq!(group_files(branches[1]), ["file2.parquet"]); + assert_eq!(partition_count(&restricted), 2); } } diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index fd60ba9897..ed44337b97 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -189,11 +189,11 @@ no error. The scheduler now fails the job instead. Such a leaf is accepted only in a task that runs all of its partitions, in order, or none of them, as in a `UnionExec` branch another task owns, where it becomes an empty node. A retry of part of a -stage can fail a job too. Plans made of Ballista's own leaves (shuffle readers, file and -in-memory scans, `PlaceholderRowExec`, `EmptyExec`) are unaffected, as are -leaves read whole below a `CoalescePartitionsExec` or on a join's build side. A -`DataSourceExec` over any other source used to log a warning and run, and is -now refused when its stage is split unless a resolver restricts it. +stage can fail a job too. Plans made of Ballista's own leaves (shuffle readers, +file and in-memory scans, `PlaceholderRowExec`, `EmptyExec`) are unaffected, as +are leaves read whole below a `CoalescePartitionsExec` or on a join's build +side. A `DataSourceExec` over any other source used to log a warning and run, +and is now refused when its stage is split unless a resolver restricts it. Whether a stage is split depends on how many task slots are free when it is scheduled, and on whether part of it is retried, so a query that holds such a @@ -209,6 +209,14 @@ results in a split stage unless a resolver restricts it. Below a `UnionExec`, the job fails instead when such a node keeps its own partition count, since the union would run the wrong branch. +The same check refuses a `UnionExec` branch that merges its input into one +partition, which the adaptive planner can leave in the union's stage: each +branch's `SortExec` under `UNION ALL ... ORDER BY`, or each branch's aggregate +in a `UNION ALL` of `count(*)` queries without `GROUP BY`. Such a query, over +any source including Parquet, used to return wrong rows when its stage was +split, and now fails. The static planner gives such a branch a stage of its own +and is unaffected. + The error names the node and what to implement. Under push-staged scheduling the job's status says only that a task could not be prepared; the reason is in the scheduler's log. To make nodes of your own restrictable, see From 9deee186cef8855d67408026dc9f917e8738083c Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 11 Nov 2026 22:39:11 -0500 Subject: [PATCH 05/17] Explain the union branch check by what the executor runs, and name the workaround The executor runs every partition of a task's plan, so a branch that keeps partitions the task wasn't given runs them too. The error now names the query shapes the adaptive planner leaves in a union's stage, and the static planner as a workaround; the upgrade guide adds the LIMIT shapes. --- ballista/scheduler/src/state/task_builder.rs | 39 ++++++++++++-------- docs/source/upgrading/56.0.0.md | 15 ++++---- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index de946c0d54..fec5751fc3 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -229,8 +229,8 @@ fn checked_restriction( /// /// Not an `EmptyExec`: DataFusion's proto codec decodes one with zero /// partitions as one partition, so it would reach the executor with a -/// partition the task doesn't own, and a `UnionExec` above it would run the -/// wrong branch. A `MemorySourceConfig` keeps its empty partition list. +/// partition the task doesn't own, which the executor would run. A +/// `MemorySourceConfig` keeps its empty partition list. fn no_partitions(plan: &Arc) -> Result> { let source = MemorySourceConfig::try_new(&[], plan.schema(), None)?; Ok(DataSourceExec::from_data_source(source)) @@ -261,12 +261,13 @@ fn ensure_leaf_runs_whole( } /// Check that a restricted `UnionExec` branch has one partition for each of -/// `partitions`. The union routes the task's partitions by its branches' -/// counts, so a branch with any other count would make the task run the wrong -/// branch. That happens when the branch collapses its input, such as through a -/// `SortExec` that merges its partitions, which reports one partition whatever -/// its leaves do, or holds a node the scheduler doesn't know that keeps its -/// own count. +/// `partitions`. The executor runs every partition of the task's plan, so a +/// branch that keeps partitions the task wasn't given runs them in this task +/// too, and the union's later partitions no longer line up with the task's. +/// When those partitions read their input whole, as below a `SortExec`, a +/// `LIMIT` or an aggregate without `GROUP BY` that merges the branch into one +/// partition, the task returns rows that another task returns as well. A node +/// the scheduler doesn't know that keeps its own count does the same. fn ensure_branch_restricted( branch: &Arc, partitions: &[usize], @@ -278,10 +279,12 @@ fn ensure_branch_restricted( let name = branch.name(); plan_err!( "cannot restrict a `UnionExec` branch to partitions {partitions:?}: its \ - `{name}` keeps a partition count of {count}, so the union would run the \ - wrong branch. A branch that collapses its input, such as through a \ - `SortExec` or `CoalescePartitionsExec`, needs a stage of its own; a node \ - of your own can be restricted by \ + `{name}` keeps a partition count of {count}, so the task would also run \ + partitions another task owns. The adaptive planner can leave a branch \ + that merges its input into one partition, such as through an `ORDER BY`, \ + a `LIMIT` or an aggregate without `GROUP BY`, in the union's stage; the \ + static planner (`ballista.planner.adaptive.enabled = false`) gives it a \ + stage of its own. A node of your own can be restricted by \ `SchedulerConfig::partition_sliceable_resolver`." ) } @@ -1129,7 +1132,8 @@ mod tests { } /// A `UnionExec` branch that keeps another partition count after - /// restriction would make the union run the wrong branch, so it is refused. + /// restriction would make the task run partitions it doesn't own, so it is + /// refused. #[test] fn a_union_branch_with_the_wrong_partition_count_is_refused() { // A node the scheduler doesn't know that holds its own two partitions, @@ -1146,9 +1150,12 @@ mod tests { error, "Error during planning: cannot restrict a `UnionExec` branch to \ partitions []: its `PayloadExec` keeps a partition count of 2, so the \ - union would run the wrong branch. A branch that collapses its input, \ - such as through a `SortExec` or `CoalescePartitionsExec`, needs a \ - stage of its own; a node of your own can be restricted by \ + task would also run partitions another task owns. The adaptive \ + planner can leave a branch that merges its input into one partition, \ + such as through an `ORDER BY`, a `LIMIT` or an aggregate without \ + `GROUP BY`, in the union's stage; the static planner \ + (`ballista.planner.adaptive.enabled = false`) gives it a stage of its \ + own. A node of your own can be restricted by \ `SchedulerConfig::partition_sliceable_resolver`." ); } diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index ed44337b97..10ccaaad5b 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -207,15 +207,16 @@ that holds data per partition from one that does not, such as a `FilterExec`. An interior node of your own that holds data per partition still returns wrong results in a split stage unless a resolver restricts it. Below a `UnionExec`, the job fails instead when such a node keeps its own partition count, since the -union would run the wrong branch. +task would also run partitions that another task owns. The same check refuses a `UnionExec` branch that merges its input into one -partition, which the adaptive planner can leave in the union's stage: each -branch's `SortExec` under `UNION ALL ... ORDER BY`, or each branch's aggregate -in a `UNION ALL` of `count(*)` queries without `GROUP BY`. Such a query, over -any source including Parquet, used to return wrong rows when its stage was -split, and now fails. The static planner gives such a branch a stage of its own -and is unaffected. +partition, which the adaptive planner can leave in the union's stage: a +`UNION ALL ... ORDER BY`, a branch with a `LIMIT` or an `ORDER BY ... LIMIT`, +or a `UNION ALL` of aggregates without `GROUP BY`. Such a query, over any source +including Parquet, used to return wrong rows when its stage was split, and now +fails. The static planner gives such a branch a stage of its own and is +unaffected, so setting `ballista.planner.adaptive.enabled` to `false` avoids the +error. The error names the node and what to implement. Under push-staged scheduling the job's status says only that a task could not be prepared; the reason is in From ff20be8e61ba31e78a99a908fa6a0dfa6b3d26e5 Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 11 Nov 2026 23:17:19 -0500 Subject: [PATCH 06/17] Empty an unknown leaf after Ballista's own leaves restrict themselves, and tighten the tests A shuffle reader or scan the task runs none of now restricts itself as it did before this branch, which a test from main expects of a range reader. Only a leaf Ballista does not list becomes an empty in-memory leaf. Tests: - check a multi-partition EmptyExec branch keeps its share through serialization, which the fix depends on; - check the union branch check on a SortExec, the shape the adaptive planner produces; - drop a resolver case the empty-leaf rule made redundant, and compare the unknown leaf's error in a union with its error alone; - describe nodes as `Name(partitions)` instead of tuples; - run both slices of the round-robin case through the repartition. --- ballista/scheduler/src/state/task_builder.rs | 181 ++++++++++--------- 1 file changed, 91 insertions(+), 90 deletions(-) diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index a4795ec7c6..68694b6292 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -68,8 +68,9 @@ use std::sync::Arc; /// side) — those leaves keep the full upstream so the collapse sees every /// partition. /// -/// A leaf the task runs none of becomes an empty leaf, whatever it is. Nodes -/// Ballista does not list are restricted by `resolver`, and a leaf that nothing +/// A leaf Ballista does not list becomes an empty leaf when the task runs none +/// of it, whatever it is. Other nodes Ballista does not list are restricted by +/// `resolver`, and a leaf that nothing /// can restrict is an error unless `partitions` is all of its partitions, in /// order. Each branch of a `UnionExec` must end up with one partition for each /// partition the task runs of it. @@ -120,12 +121,6 @@ fn restrict( under_collect: bool, resolver: Option<&dyn PartitionSliceableResolver>, ) -> Result> { - // A leaf the task runs none of, such as one in a `UnionExec` branch - // another task owns, is never executed, whatever it is. - if !under_collect && partitions.is_empty() && plan.children().is_empty() { - return no_partitions(&plan); - } - // Leaves: apply the algebraic partition-projection when we're not // under a collapse. Under collapse, leaves keep their full upstream — // the generic recursion below leaves them unchanged. @@ -135,6 +130,12 @@ fn restrict( return Ok(rewritten); } + // Any other leaf the task runs none of, such as one in a `UnionExec` + // branch another task owns, is never executed, whatever it is. + if !under_collect && partitions.is_empty() && plan.children().is_empty() { + return no_partitions(&plan); + } + // UnionExec: parent partition `p` maps to exactly one child's local // partition (`p` minus the sum of preceding children's counts). Split // `partitions` into disjoint per-child sub-slices and recurse; a child @@ -738,12 +739,8 @@ mod tests { } /// File counts per file group of a `DataSourceExec` — the shape a union - /// test cares about after restriction. None for a leaf with no partitions, - /// which a scan the task doesn't poll becomes. + /// test cares about after restriction. fn group_file_counts(plan: &Arc) -> Vec { - if plan.properties().output_partitioning().partition_count() == 0 { - return vec![]; - } file_groups(plan).iter().map(|g| g.len()).collect() } @@ -1173,33 +1170,31 @@ mod tests { UnionExec::try_new(vec![scan_with_file_groups(2), unknown.clone()]).unwrap() }; - // Partition 2 is the second branch's partition 0. - let restricted = - restrict_plan_to_partitions(union(), &[2], Some(resolver.as_ref())).unwrap(); - let branches = restricted.children(); - assert_eq!(partition_count(branches[0]), 0); - assert_eq!(payloads_of(branches[1]), ["p0"]); - - // And partition 0 is the first branch's, which leaves the second none. - let restricted = - restrict_plan_to_partitions(union(), &[0], Some(resolver.as_ref())).unwrap(); - let branches = restricted.children(); - assert_eq!(group_files(branches[0]), ["file0.parquet"]); - assert_eq!(partition_count(branches[1]), 0); - + // Partition 0 is the scan's, which leaves the unknown leaf none. let restricted = restrict_plan_to_partitions(union(), &[0], None).unwrap(); let branches = restricted.children(); assert_eq!(group_files(branches[0]), ["file0.parquet"]); assert_no_partitions(branches[1], &unknown); - assert!(restrict_plan_to_partitions(union(), &[2], None).is_err()); + // Partition 2 is the unknown leaf's partition 0, which only the + // resolver can cut down. + let restricted = + restrict_plan_to_partitions(union(), &[2], Some(resolver.as_ref())).unwrap(); + let branches = restricted.children(); + assert_eq!(group_files(branches[0]), Vec::::new()); + assert_eq!(payloads_of(branches[1]), ["p0"]); + + let in_union = restrict_plan_to_partitions(union(), &[2], None).unwrap_err(); + let alone = restrict_plan_to_partitions(unknown.clone(), &[0], None).unwrap_err(); + assert_eq!(in_union.to_string(), alone.to_string()); } - /// The task plan is serialized on its way to the executor, and the branch - /// the task runs none of must still have no partitions there, or the union - /// would run it in place of the task's own. + /// The task plan is serialized on its way to the executor, and each union + /// branch must keep its restricted partition count there, or the task would + /// run partitions it doesn't own: none for a branch the task runs none of, + /// and its share for a multi-partition `EmptyExec`. #[test] - fn a_branch_the_task_runs_none_of_keeps_no_partitions_through_serialization() { + fn restricted_union_branches_keep_their_partition_counts_through_serialization() { use ballista_core::serde::BallistaPhysicalExtensionCodec; use datafusion::prelude::SessionContext; use datafusion_proto::bytes::{ @@ -1207,12 +1202,15 @@ mod tests { physical_plan_to_bytes_with_extension_codec, }; + let schema = scan_with_file_groups(2).schema(); + // Partitions 0-1 are the scan's, 2 the unknown leaf's, 3-6 the empty one's. let union = UnionExec::try_new(vec![ scan_with_file_groups(2), PayloadExec::leaf(&["p0"]), + Arc::new(EmptyExec::new(schema).with_partitions(4)), ]) .unwrap(); - let restricted = restrict_plan_to_partitions(union, &[0], None).unwrap(); + let restricted = restrict_plan_to_partitions(union, &[0, 4, 6], None).unwrap(); let codec = BallistaPhysicalExtensionCodec::default(); let bytes = @@ -1224,10 +1222,12 @@ mod tests { ) .unwrap(); - assert_eq!(partition_count(&decoded), 1); let branches = decoded.children(); + assert_eq!( + branches.iter().map(|b| describe(b)).collect::>(), + ["DataSourceExec(1)", "DataSourceExec(0)", "EmptyExec(2)"] + ); assert_eq!(group_files(branches[0]), ["file0.parquet"]); - assert_eq!(partition_count(branches[1]), 0); } /// A `UnionExec` branch that keeps another partition count after @@ -1235,20 +1235,25 @@ mod tests { /// refused. #[test] fn a_union_branch_with_the_wrong_partition_count_is_refused() { - // A node the scheduler doesn't know that holds its own two partitions, - // over a leaf the task runs none of: the leaf empties, the node keeps - // its count. - let branch = - PayloadExec::over(&["p0", "p1"], vec![PayloadExec::leaf(&["q0"])], &[]); - let union = UnionExec::try_new(vec![scan_with_file_groups(2), branch]).unwrap(); - - let error = restrict_plan_to_partitions(union, &[0], None) + use datafusion::physical_plan::sorts::sort::SortExec; + // A sort that merges its branch into one partition, as the adaptive + // planner leaves under `UNION ALL ... ORDER BY`. It reads its input + // whole, so it keeps that partition when the task runs none of it. + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new( + Column::new("a", 0), + ))]) + .unwrap(); + let sorted = Arc::new(SortExec::new(ordering, scan_with_file_groups(2))); + // Partition 0 is the sort's, 1-2 the scan's. + let union = UnionExec::try_new(vec![sorted, scan_with_file_groups(2)]).unwrap(); + + let error = restrict_plan_to_partitions(union, &[1], None) .unwrap_err() .to_string(); assert_eq!( error, "Error during planning: cannot restrict a `UnionExec` branch to \ - partitions []: its `PayloadExec` keeps a partition count of 2, so the \ + partitions []: its `SortExec` keeps a partition count of 1, so the \ task would also run partitions another task owns. The adaptive \ planner can leave a branch that merges its input into one partition, \ such as through an `ORDER BY`, a `LIMIT` or an aggregate without \ @@ -1290,19 +1295,23 @@ mod tests { assert!(error.contains("returned 4 partitions"), "{error}"); } - /// The name and partition count of each restriction of `leaf`, one for - /// each of `slices`. + /// A node's name and partition count, such as `EmptyExec(2)`. + fn describe(plan: &Arc) -> String { + format!("{}({})", plan.name(), partition_count(plan)) + } + + /// `leaf` restricted to each of `slices`, described. fn restricted_shapes( leaf: &Arc, slices: &[&[usize]], - ) -> Vec<(String, usize)> { + ) -> Vec { slices .iter() .map(|slice| { let restricted = restrict_plan_to_partitions(leaf.clone(), slice, None).unwrap(); assert_eq!(restricted.schema(), leaf.schema()); - (restricted.name().to_string(), partition_count(&restricted)) + describe(&restricted) }) .collect() } @@ -1318,20 +1327,25 @@ mod tests { let placeholder: Arc = Arc::new(PlaceholderRowExec::new(schema).with_partitions(4)); let slices: &[&[usize]] = &[&[0, 1, 2, 3], &[1, 3], &[2], &[]]; - let memory = "DataSourceExec".to_string(); - for leaf in [empty, placeholder] { - let name = leaf.name().to_string(); - assert_eq!( - restricted_shapes(&leaf, slices), - [ - (name.clone(), 4), - (name.clone(), 2), - (name, 1), - (memory.clone(), 0), - ] - ); - } + assert_eq!( + restricted_shapes(&empty, slices), + [ + "EmptyExec(4)", + "EmptyExec(2)", + "EmptyExec(1)", + "DataSourceExec(0)" + ] + ); + assert_eq!( + restricted_shapes(&placeholder, slices), + [ + "PlaceholderRowExec(4)", + "PlaceholderRowExec(2)", + "PlaceholderRowExec(1)", + "DataSourceExec(0)", + ] + ); } /// Under a round-robin `RepartitionExec`, which the `AdaptivePlanner` @@ -1344,27 +1358,20 @@ mod tests { let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); let placeholder: Arc = Arc::new(PlaceholderRowExec::new(schema)); - let restricted = restrict_plan_to_partitions( - Arc::new( - RepartitionExec::try_new( - placeholder.clone(), - Partitioning::RoundRobinBatch(4), - ) - .unwrap(), - ), - &[2, 3], - None, - ) - .unwrap(); - assert_no_partitions(restricted.children()[0], &placeholder); + let leaf_in_task = |partitions: &[usize]| { + let repartition = RepartitionExec::try_new( + placeholder.clone(), + Partitioning::RoundRobinBatch(4), + ) + .unwrap(); + restrict_plan_to_partitions(Arc::new(repartition), partitions, None) + .unwrap() + .children()[0] + .clone() + }; - assert_eq!( - restricted_shapes(&placeholder, &[&[0, 1], &[3]]), - [ - ("PlaceholderRowExec".to_string(), 1), - ("DataSourceExec".to_string(), 0), - ] - ); + assert_eq!(describe(&leaf_in_task(&[0, 1])), "PlaceholderRowExec(1)"); + assert_no_partitions(&leaf_in_task(&[2, 3]), &placeholder); } /// A union branch the `AdaptivePlanner` replaced with a multi-partition @@ -1382,20 +1389,14 @@ mod tests { // Partition 0 is the empty branch's. let restricted = restrict_plan_to_partitions(union(), &[0], None).unwrap(); let branches = restricted.children(); - assert_eq!( - (branches[0].name(), partition_count(branches[0])), - ("EmptyExec", 1) - ); - assert_no_partitions(branches[1], &empty); + assert_eq!(describe(branches[0]), "EmptyExec(1)"); + assert_eq!(group_files(branches[1]), Vec::::new()); // Partitions 3 and 6 are the last of the empty branch's and the third // of the scan's. let restricted = restrict_plan_to_partitions(union(), &[3, 6], None).unwrap(); let branches = restricted.children(); - assert_eq!( - (branches[0].name(), partition_count(branches[0])), - ("EmptyExec", 1) - ); + assert_eq!(describe(branches[0]), "EmptyExec(1)"); assert_eq!(group_files(branches[1]), ["file2.parquet"]); assert_eq!(partition_count(&restricted), 2); } From c835de7b69cc0c062d91bb6559c5505f6d272319 Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 11 Nov 2026 23:35:28 -0500 Subject: [PATCH 07/17] Give the adaptive-planner union failure its own upgrade note, and tidy comments A split UNION ALL with a branch that merges its input affects anyone on the adaptive planner, not only users with nodes of their own, so it gets its own heading in the 56.0.0 upgrade guide, with the workaround. Also reflow a doc comment and use describe() for the last union assertion. --- ballista/scheduler/src/state/task_builder.rs | 13 +++++----- docs/source/upgrading/56.0.0.md | 26 +++++++++++++------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index 68694b6292..ece5a03e18 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -69,11 +69,10 @@ use std::sync::Arc; /// partition. /// /// A leaf Ballista does not list becomes an empty leaf when the task runs none -/// of it, whatever it is. Other nodes Ballista does not list are restricted by -/// `resolver`, and a leaf that nothing -/// can restrict is an error unless `partitions` is all of its partitions, in -/// order. Each branch of a `UnionExec` must end up with one partition for each -/// partition the task runs of it. +/// of it. Other nodes Ballista does not list are restricted by `resolver`, and +/// a leaf that nothing can restrict is an error unless `partitions` is all of +/// its partitions, in order. Each branch of a `UnionExec` must end up with one +/// partition for each partition the task runs of it. pub fn restrict_plan_to_partitions( plan: Arc, partitions: &[usize], @@ -131,7 +130,7 @@ fn restrict( } // Any other leaf the task runs none of, such as one in a `UnionExec` - // branch another task owns, is never executed, whatever it is. + // branch another task owns, is never executed. if !under_collect && partitions.is_empty() && plan.children().is_empty() { return no_partitions(&plan); } @@ -1398,6 +1397,6 @@ mod tests { let branches = restricted.children(); assert_eq!(describe(branches[0]), "EmptyExec(1)"); assert_eq!(group_files(branches[1]), ["file2.parquet"]); - assert_eq!(partition_count(&restricted), 2); + assert_eq!(describe(&restricted), "UnionExec(2)"); } } diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index 60a4fb7592..341631927f 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -221,20 +221,28 @@ results in a split stage unless a resolver restricts it. Below a `UnionExec`, the job fails instead when such a node keeps its own partition count, since the task would also run partitions that another task owns. -The same check refuses a `UnionExec` branch that merges its input into one -partition, which the adaptive planner can leave in the union's stage: a -`UNION ALL ... ORDER BY`, a branch with a `LIMIT` or an `ORDER BY ... LIMIT`, -or a `UNION ALL` of aggregates without `GROUP BY`. Such a query, over any source -including Parquet, used to return wrong rows when its stage was split, and now -fails. The static planner gives such a branch a stage of its own and is -unaffected, so setting `ballista.planner.adaptive.enabled` to `false` avoids the -error. - The error names the node and what to implement. Under push-staged scheduling the job's status says only that a task could not be prepared; the reason is in the scheduler's log. To make nodes of your own restrictable, see [Custom Physical Plan Nodes](../user-guide/extending-components.md#custom-physical-plan-nodes). +#### A split `UNION ALL` with a branch that merges its input fails under the adaptive planner + +When a stage is split between tasks, each task restricts the `UnionExec` +branches other tasks own to no partitions. The adaptive planner can leave a +branch that merges its input into one partition in the union's stage: in a +`UNION ALL ... ORDER BY`, a branch with a `LIMIT` or an `ORDER BY ... LIMIT`, or +a `UNION ALL` of aggregates without `GROUP BY`. Such a branch reads its input +whole and keeps its one partition, so every task of the stage ran it, and the +query returned wrong rows, over any source including Parquet. + +The scheduler now fails the job instead, with an error that starts "cannot +restrict a `UnionExec` branch". Whether a stage is split depends on how many +task slots are free, so such a query can pass on an idle cluster and fail on a +busy one. The static planner gives such a branch a stage of its own and is +unaffected, so setting `ballista.planner.adaptive.enabled` to `false` avoids the +error. + ### API changes #### `SortShuffleWriterExec` accepts a range partitioning From fce4eff2174016035f4b4a137701eddf7d8c3936 Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 15 Nov 2026 22:50:55 -0500 Subject: [PATCH 08/17] Name each cause of a refused union branch, and test the empty unknown leaf The union-branch error no longer points every user at the adaptive planner. It names the cause it finds: a broadcast reader, a branch that merges its input into one partition, or a node that sets its own partition count, which is pointed at the resolver alone. Add a scheduler test that ships a split union over an unknown leaf through `launch_tasks` with no resolver, and an end-to-end test that an adaptive `UNION ALL ... ORDER BY` split one partition per task fails with that error. Update the stale doc comments on `restricts_to_nothing` and `select_output_partitions`. --- ballista/client/tests/split_union.rs | 41 ++++++ ballista/scheduler/src/planner.rs | 10 +- ballista/scheduler/src/state/mod.rs | 104 ++++++++++--- ballista/scheduler/src/state/task_builder.rs | 147 ++++++++++++++++--- 4 files changed, 258 insertions(+), 44 deletions(-) diff --git a/ballista/client/tests/split_union.rs b/ballista/client/tests/split_union.rs index d909f750ad..c63a6ad974 100644 --- a/ballista/client/tests/split_union.rs +++ b/ballista/client/tests/split_union.rs @@ -120,4 +120,45 @@ mod split_union { ); } } + + /// The adaptive planner leaves a branch that merges its input into one + /// partition, here each sorted branch of a `UNION ALL ... ORDER BY`, in the + /// union's stage. With one partition per task every task would also run + /// the branch it doesn't own, so the job fails rather than return those + /// rows twice. + #[tokio::test] + async fn a_split_union_with_a_merging_branch_fails_under_the_adaptive_planner() { + let ctx = SessionContext::standalone_with_state(ballista_state(&[ + (BALLISTA_ADAPTIVE_PLANNER_ENABLED, "true"), + (BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, "1"), + ])) + .await + .unwrap(); + register_alltypes(&ctx, "test").await; + + let error = ctx + .sql("SELECT id FROM test UNION ALL SELECT id FROM test ORDER BY id") + .await + .unwrap() + .collect() + .await + .unwrap_err() + .to_string(); + // The job id before it differs from run to run. + let reason = error.split_once("DataFusion error: ").map(|(_, r)| r); + assert_eq!( + reason, + Some( + "Error during planning: cannot restrict a `UnionExec` branch to \ + partitions []: its `SortExec` keeps a partition count of 1, so the \ + task would also run partitions another task owns. The branch merges \ + its input into one partition, as an `ORDER BY`, a `LIMIT` or an \ + aggregate without `GROUP BY` does. The static planner gives such a \ + branch a stage of its own; the adaptive planner \ + (`ballista.planner.adaptive.enabled`) can leave it in the union's \ + stage." + ), + "{error}" + ); + } } diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 682fd0937c..2bc179ec95 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -701,8 +701,14 @@ fn can_stay_inline(plan: &Arc) -> bool { /// Whether per-task restriction can reduce `plan` to zero partitions. An empty /// slice means "this task polls nothing here", so a branch that still reports a /// partition afterwards, or that the rewriter refuses, cannot be restricted -/// away. No resolver is passed, so a node Ballista does not know gets a stage -/// of its own. +/// away. +/// +/// No resolver is passed. A leaf Ballista does not know still passes, because +/// the rewriter replaces a leaf the task runs none of with an empty leaf, so +/// such a branch stays in the union's stage. Only the empty slice is tried: a +/// task that runs some but not all of such a leaf still needs a resolver. An +/// interior node Ballista does not know that keeps its own partition count +/// fails, and its branch gets a stage of its own. fn restricts_to_nothing(plan: &Arc) -> bool { restrict_plan_to_partitions(plan.clone(), &[], None) .is_ok_and(|p| p.properties().output_partitioning().partition_count() == 0) diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs index 3691ba2f6e..3b43303268 100644 --- a/ballista/scheduler/src/state/mod.rs +++ b/ballista/scheduler/src/state/mod.rs @@ -690,7 +690,8 @@ mod tests { }; use crate::test_utils::{ BlackholeTaskLauncher, RejectingTaskLauncher, aggregation_logical_plan, - mock_executor, register_virtual_executor, test_cluster_context, + mock_executor, register_virtual_executor, scan_with_file_groups, + test_cluster_context, }; use ballista_core::config::{ BALLISTA_EXECUTION_MODE, BALLISTA_PROPAGATE_EMPTY_ENABLED, @@ -702,6 +703,8 @@ mod tests { }; use ballista_core::serde::{BallistaCodec, BallistaLogicalExtensionCodec}; use datafusion::execution::context::SessionConfig; + use datafusion::physical_plan::union::UnionExec; + use datafusion::physical_plan::{ExecutionPlan, displayable}; use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; @@ -972,11 +975,13 @@ mod tests { } } - /// Submits a job of one four-partition `PayloadExec` and binds it to two - /// executors of two vcores each, which splits its stage into two tasks. + /// Submits a job of `plan`, a stage of four partitions over `PayloadExec`, + /// and binds it to two executors of two vcores each, which splits its + /// stage into two tasks. async fn bind_split_payload_stage( config: SchedulerConfig, launcher: Arc, + plan: Arc, ) -> Result<( SchedulerState, JobId, @@ -1021,14 +1026,7 @@ mod tests { .queue_job(&job_id, "", timestamp_millis())?; state .task_manager - .submit_physical_plan( - &job_id, - "", - ctx, - PayloadExec::leaf(&["p0", "p1", "p2", "p3"]), - timestamp_millis(), - None, - ) + .submit_physical_plan(&job_id, "", ctx, plan, timestamp_millis(), None) .await?; let bound = state @@ -1063,9 +1061,12 @@ mod tests { async fn launch_tasks_fails_a_job_whose_split_stage_holds_a_leaf_nobody_can_restrict() -> Result<()> { let launcher = Arc::new(CapturingLauncher::default()); - let (state, job_id, bound) = - bind_split_payload_stage(SchedulerConfig::default(), launcher.clone()) - .await?; + let (state, job_id, bound) = bind_split_payload_stage( + SchedulerConfig::default(), + launcher.clone(), + PayloadExec::leaf(&["p0", "p1", "p2", "p3"]), + ) + .await?; let vcores: u32 = bound.iter().map(|(_, task)| task.vcores_consumed).sum(); let (freed, failed_jobs) = launch(&state, bound).await?; @@ -1083,17 +1084,36 @@ mod tests { let launcher = Arc::new(CapturingLauncher::default()); let config = SchedulerConfig::default() .with_partition_sliceable_resolver(payload_resolver()); - let (state, _job_id, bound) = - bind_split_payload_stage(config, launcher.clone()).await?; + let (state, _job_id, bound) = bind_split_payload_stage( + config, + launcher.clone(), + PayloadExec::leaf(&["p0", "p1", "p2", "p3"]), + ) + .await?; let (freed, failed_jobs) = launch(&state, bound).await?; assert!(failed_jobs.is_empty()); assert_eq!(freed, 0, "every task was launched, so no slot is refunded"); - // Decode what the executors would, with the codec the scheduler used. + let mut shipped: Vec> = shipped_writer_inputs(&state, &launcher) + .iter() + .map(payloads_of) + .collect(); + shipped.sort(); + assert_eq!(shipped, [["p0", "p1"], ["p2", "p3"]]); + + Ok(()) + } + + /// The input of the shuffle writer of each task `launcher` launched, + /// decoded as the executor would, with the codec the scheduler used. + fn shipped_writer_inputs( + state: &SchedulerState, + launcher: &CapturingLauncher, + ) -> Vec> { let task_ctx = SessionContext::new().task_ctx(); - let mut shipped: Vec> = launcher + launcher .launched .lock() .unwrap() @@ -1108,15 +1128,55 @@ mod tests { }) .expect("the executor can decode the plan"); // The writer's input can come wrapped in a `CooperativeExec`. - let mut input = plan.children()[0]; + let mut input = plan.children()[0].clone(); while input.name() == "CooperativeExec" { - input = input.children()[0]; + input = input.children()[0].clone(); } - payloads_of(input) + input }) + .collect() + } + + /// A `UnionExec` branch over a leaf the scheduler doesn't know, as an + /// Iceberg scan is, runs in the task that owns it and becomes an empty + /// leaf in the task that doesn't, with no resolver configured. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn launch_tasks_empties_an_unknown_leaf_in_a_union_branch_another_task_owns() + -> Result<()> { + let launcher = Arc::new(CapturingLauncher::default()); + // Partition 0 is the payload's, 1-3 the scan's. + let union = UnionExec::try_new(vec![ + PayloadExec::leaf(&["p0"]), + scan_with_file_groups(3), + ])?; + let (state, _job_id, bound) = + bind_split_payload_stage(SchedulerConfig::default(), launcher.clone(), union) + .await?; + + let (freed, failed_jobs) = launch(&state, bound).await?; + + assert!(failed_jobs.is_empty()); + assert_eq!(freed, 0, "every task was launched, so no slot is refunded"); + let mut shipped: Vec = shipped_writer_inputs(&state, &launcher) + .iter() + .map(|plan| displayable(plan.as_ref()).indent(false).to_string()) .collect(); shipped.sort(); - assert_eq!(shipped, [["p0", "p1"], ["p2", "p3"]]); + assert_eq!( + shipped, + [ + "UnionExec\n \ + CooperativeExec\n \ + DataSourceExec: partitions=0, partition_sizes=[]\n \ + DataSourceExec: file_groups={2 groups: [[file1.parquet], \ + [file2.parquet]]}, projection=[a], file_type=parquet\n", + "UnionExec\n \ + CooperativeExec\n \ + PayloadExec: payloads=[\"p0\"]\n \ + DataSourceExec: file_groups={1 group: [[file0.parquet]]}, \ + projection=[a], file_type=parquet\n", + ] + ); Ok(()) } diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index ece5a03e18..54a800be11 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -266,8 +266,12 @@ fn ensure_leaf_runs_whole( /// too, and the union's later partitions no longer line up with the task's. /// When those partitions read their input whole, as below a `SortExec`, a /// `LIMIT` or an aggregate without `GROUP BY` that merges the branch into one -/// partition, the task returns rows that another task returns as well. A node -/// the scheduler doesn't know that keeps its own count does the same. +/// partition, the task returns rows that another task returns as well. A +/// broadcast reader, which every task reads whole, and a node that sets its own +/// partition count do the same. +/// +/// The scheduler doesn't know here which planner made the stage, so the error +/// names the branch's causes rather than one planner's. fn ensure_branch_restricted( branch: &Arc, partitions: &[usize], @@ -277,18 +281,57 @@ fn ensure_branch_restricted( return Ok(()); } let name = branch.name(); + let cause = if along_single_inputs(branch, &reads_a_broadcast) { + "The branch reads a broadcast, which every task reads whole as one \ + partition. The static planner gives a branch that reads a broadcast a \ + stage of its own." + } else if along_single_inputs(branch, &merges_its_input) { + "The branch merges its input into one partition, as an `ORDER BY`, a \ + `LIMIT` or an aggregate without `GROUP BY` does. The static planner \ + gives such a branch a stage of its own; the adaptive planner \ + (`ballista.planner.adaptive.enabled`) can leave it in the union's stage." + } else { + "The branch holds a node that sets its own partition count. A node of \ + your own can be restricted by \ + `SchedulerConfig::partition_sliceable_resolver`." + }; plan_err!( "cannot restrict a `UnionExec` branch to partitions {partitions:?}: its \ `{name}` keeps a partition count of {count}, so the task would also run \ - partitions another task owns. The adaptive planner can leave a branch \ - that merges its input into one partition, such as through an `ORDER BY`, \ - a `LIMIT` or an aggregate without `GROUP BY`, in the union's stage; the \ - static planner (`ballista.planner.adaptive.enabled = false`) gives it a \ - stage of its own. A node of your own can be restricted by \ - `SchedulerConfig::partition_sliceable_resolver`." + partitions another task owns. {cause}" ) } +/// Whether `found` holds for `plan` or for a node below it reached through +/// nodes of one input each, which pass that node's partition count up. +fn along_single_inputs( + plan: &Arc, + found: &dyn Fn(&Arc) -> bool, +) -> bool { + if found(plan) { + return true; + } + match plan.children().as_slice() { + [child] => along_single_inputs(child, found), + _ => false, + } +} + +fn reads_a_broadcast(plan: &Arc) -> bool { + plan.downcast_ref::() + .is_some_and(|reader| reader.broadcast) + || plan + .downcast_ref::() + .is_some_and(|reader| reader.broadcast) +} + +/// Whether `plan` reads its one input whole, as a `SortExec`, a +/// `GlobalLimitExec`, a final aggregate without `GROUP BY` or a coalesce does, +/// and so merges it into one partition. +fn merges_its_input(plan: &Arc) -> bool { + plan.children().len() == 1 && child_scopes(plan, false) == [true] +} + /// Bucket each parent partition of a `UnionExec` into its owning child's /// local index. Iterates over `partitions` and, for each, subtracts each /// child's count until the remainder lands inside a child. Returns one @@ -395,11 +438,11 @@ fn child_scopes(plan: &Arc, under_collect: bool) -> Vec /// /// **When a new leaf type surfaces, add a branch below.** Currently /// covered: `ShuffleReaderExec`, `DataSourceExec` (files, in-memory), -/// `PlaceholderRowExec`, `EmptyExec`. `ValuesExec` and other future -/// self-generating leaves fall through as `None`; if one sits under a -/// partition-splitting fan-in, the walker refuses the task rather than let the -/// executor duplicate output (see the PlaceholderRow-under-UnionExec bug for -/// the failure mode). +/// `PlaceholderRowExec`, `EmptyExec`. Any other leaf falls through as +/// `None`. The walker then replaces it with an empty leaf in a task that runs +/// none of it, such as one that owns another `UnionExec` branch, keeps it in a +/// task that runs all of it, in order, and otherwise refuses the task rather +/// than let the executor run the wrong partitions. /// /// # Scope /// @@ -1253,13 +1296,77 @@ mod tests { error, "Error during planning: cannot restrict a `UnionExec` branch to \ partitions []: its `SortExec` keeps a partition count of 1, so the \ - task would also run partitions another task owns. The adaptive \ - planner can leave a branch that merges its input into one partition, \ - such as through an `ORDER BY`, a `LIMIT` or an aggregate without \ - `GROUP BY`, in the union's stage; the static planner \ - (`ballista.planner.adaptive.enabled = false`) gives it a stage of its \ - own. A node of your own can be restricted by \ - `SchedulerConfig::partition_sliceable_resolver`." + task would also run partitions another task owns. The branch merges \ + its input into one partition, as an `ORDER BY`, a `LIMIT` or an \ + aggregate without `GROUP BY` does. The static planner gives such a \ + branch a stage of its own; the adaptive planner \ + (`ballista.planner.adaptive.enabled`) can leave it in the union's \ + stage." + ); + } + + /// A node of the user's own that keeps its partition count is pointed at + /// the resolver, not at the planners. + #[test] + fn a_union_branch_with_a_node_of_your_own_is_pointed_at_the_resolver() { + // Without a resolver it keeps one partition per payload, whatever its + // input is restricted to. + let own = PayloadExec::over(&["p0", "p1"], vec![scan_with_file_groups(2)], &[]); + // Partitions 0-1 are the node's, 2-3 the scan's. + let union = UnionExec::try_new(vec![own, scan_with_file_groups(2)]).unwrap(); + + let error = restrict_plan_to_partitions(union, &[2, 3], None) + .unwrap_err() + .to_string(); + assert_eq!( + error, + "Error during planning: cannot restrict a `UnionExec` branch to \ + partitions []: its `PayloadExec` keeps a partition count of 2, so the \ + task would also run partitions another task owns. The branch holds a \ + node that sets its own partition count. A node of your own can be \ + restricted by `SchedulerConfig::partition_sliceable_resolver`." + ); + } + + /// A broadcast reader keeps its one partition in every task, so a union + /// branch that reads one is refused in a task that owns another branch, + /// and the error says so rather than blaming a sort or a limit. + #[test] + fn a_union_branch_reading_a_broadcast_is_refused_as_a_broadcast() { + use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr}; + let schema = scan_with_file_groups(2).schema(); + let reader = ShuffleReaderExec::try_new_broadcast( + 1, + (0..2).map(create_partition).collect(), + schema, + 2, + ) + .unwrap(); + let projected: Arc = Arc::new( + ProjectionExec::try_new( + vec![ProjectionExpr { + expr: Arc::new(Column::new("a", 0)), + alias: "a".to_string(), + }], + Arc::new(reader), + ) + .unwrap(), + ); + // Partition 0 is the broadcast branch's, 1-2 the scan's. + let union = + UnionExec::try_new(vec![projected, scan_with_file_groups(2)]).unwrap(); + + let error = restrict_plan_to_partitions(union, &[1, 2], None) + .unwrap_err() + .to_string(); + assert_eq!( + error, + "Error during planning: cannot restrict a `UnionExec` branch to \ + partitions []: its `ProjectionExec` keeps a partition count of 1, so \ + the task would also run partitions another task owns. The branch \ + reads a broadcast, which every task reads whole as one partition. The \ + static planner gives a branch that reads a broadcast a stage of its \ + own." ); } From 1848783f886041fe4cf4b50a2b5640b93fdb2650 Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 15 Nov 2026 22:53:04 -0500 Subject: [PATCH 09/17] docs: say a node with inputs is still asked to restrict a union branch to none --- docs/source/user-guide/extending-components.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/extending-components.md b/docs/source/user-guide/extending-components.md index fa76e74f10..1e0317efa1 100644 --- a/docs/source/user-guide/extending-components.md +++ b/docs/source/user-guide/extending-components.md @@ -179,7 +179,7 @@ let config = SchedulerConfig::default().with_partition_sliceable_resolver(Arc::n - `children` are the node's inputs, none for a scan, restricted to `partitions`. An input the node declares `SinglePartition` arrives whole, as a join's build side does. Partition `i` of the node must read partition `i` of every other input. - The restricted node must have exactly `partitions.len()` partitions. The scheduler rejects any other count. -- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. +- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. A node with inputs is still asked, with its inputs already restricted to none. If nothing restricts such a node and it keeps its own partition count, the branch keeps partitions the task doesn't own, and the scheduler fails the job. - The restricted node is serialized with the scheduler's `physical codec`, so what it carries per partition has to survive encoding and decoding. - The resolver does not need `PartitionSliceable`. It can build the restricted node itself, which is the way to restrict a node you cannot implement the trait on, such as a `DataSourceExec` over a `DataSource` of your own: match it with `downcast_ref`, then build a `DataSourceExec` over the restricted source. - The resolver is asked after Ballista's own list. A node that is read whole, such as one below a `CoalescePartitionsExec` or on a join's build side, needs none of this. From 14af6e26ec1ba00a50fa4ef292463a18247039d3 Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 15 Nov 2026 23:16:24 -0500 Subject: [PATCH 10/17] Give a union branch over a multi-partition unknown leaf its own stage again The stage planner kept any branch the rewriter could restrict to no partitions in the union's stage. Since a leaf the scheduler doesn't know became an empty leaf, that took in a custom scan of several partitions, and a task owning part of it then failed. The planner now also requires the branch to restrict to its first partition alone, which such a leaf fails, so it gets a stage of its own as on main. A one-partition unknown leaf stays inline. An unresolved shuffle passes that check through a planner-only resolver, as the reader it becomes restricts to any slice, so a branch fed by a shuffle still stays in the union's stage. --- ballista/scheduler/src/planner.rs | 176 +++++++++++++++++++++++++++--- 1 file changed, 161 insertions(+), 15 deletions(-) diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 2bc179ec95..781b87675e 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -25,6 +25,7 @@ use ballista_core::JobId; use ballista_core::config::BallistaConfig; use ballista_core::error::{BallistaError, Result}; use ballista_core::execution_plans::ShuffleWriter; +use ballista_core::execution_plans::plan_algebra::PartitionSliceableResolver; use ballista_core::execution_plans::sort_shuffle::{ShuffleRouting, SortShuffleConfig}; use ballista_core::{ execution_plans::{ @@ -686,16 +687,16 @@ impl DefaultDistributedPlanner { /// Whether a union branch can be left in the union's own stage. /// /// A branch already sitting behind a stage boundary needs nothing more, and -/// neither does one that per-task restriction can reduce to zero partitions — -/// that is the property the union's index arithmetic relies on. The second -/// question is put to the real rewriter rather than to a list of operators, so -/// it stays correct as the rewriter's handling of collapses, broadcast readers -/// and leaf types evolves. +/// neither does one that per-task restriction can cut to whatever share of it +/// a task runs: none of it, which the union's index arithmetic relies on, or +/// some of it. These questions are put to the real rewriter rather than to a +/// list of operators, so they stay correct as the rewriter's handling of +/// collapses, broadcast readers and leaf types evolves. fn can_stay_inline(plan: &Arc) -> bool { if plan.is::() { return true; } - !holds_a_broadcast(plan) && restricts_to_nothing(plan) + !holds_a_broadcast(plan) && restricts_to_nothing(plan) && restricts_to_part(plan) } /// Whether per-task restriction can reduce `plan` to zero partitions. An empty @@ -703,10 +704,8 @@ fn can_stay_inline(plan: &Arc) -> bool { /// partition afterwards, or that the rewriter refuses, cannot be restricted /// away. /// -/// No resolver is passed. A leaf Ballista does not know still passes, because -/// the rewriter replaces a leaf the task runs none of with an empty leaf, so -/// such a branch stays in the union's stage. Only the empty slice is tried: a -/// task that runs some but not all of such a leaf still needs a resolver. An +/// No resolver is passed. A leaf Ballista does not know passes, because the +/// rewriter replaces a leaf the task runs none of with an empty leaf. An /// interior node Ballista does not know that keeps its own partition count /// fails, and its branch gets a stage of its own. fn restricts_to_nothing(plan: &Arc) -> bool { @@ -714,6 +713,46 @@ fn restricts_to_nothing(plan: &Arc) -> bool { .is_ok_and(|p| p.properties().output_partitioning().partition_count() == 0) } +/// Whether per-task restriction can cut `plan` to its first partition alone, +/// standing in for any task that owns some but not all of the branch. The +/// rewriter refuses a leaf it cannot restrict for every slice but the whole, +/// so `[0]` catches such a leaf of more than one partition as any partial +/// slice would. +/// +/// A leaf Ballista does not know passes only if it has one partition, which +/// the task that owns it runs whole. A branch over a leaf of more partitions, +/// such as a scan over a source of your own, gets a stage of its own, so that +/// only the leaf's own stage needs a task to run all of it. An +/// `UnresolvedShuffleExec` passes, since it becomes a shuffle reader before +/// any task runs, and the rewriter restricts that to any slice. +fn restricts_to_part(plan: &Arc) -> bool { + plan.properties().output_partitioning().partition_count() == 0 + || restrict_plan_to_partitions(plan.clone(), &[0], Some(&UnresolvedShuffles)) + .is_ok() +} + +/// Restricts an `UnresolvedShuffleExec` to one partition for each index, as +/// the rewriter will the shuffle reader it becomes. The stage planner checks +/// with it whether a branch can be restricted before its inputs are resolved. +#[derive(Debug)] +struct UnresolvedShuffles; + +impl PartitionSliceableResolver for UnresolvedShuffles { + fn restrict_to_partitions( + &self, + plan: &Arc, + _children: Vec>, + partitions: &[usize], + ) -> Option>> { + let reader = plan.downcast_ref::()?; + Some(Ok(Arc::new(UnresolvedShuffleExec::new( + reader.stage_id, + reader.schema.clone(), + Partitioning::UnknownPartitioning(partitions.len()), + )))) + } +} + /// Whether `plan` contains a broadcast input or a `CollectLeft` join. /// /// Asking the rewriter alone is not enough here. A `CollectLeft` join's build @@ -946,9 +985,10 @@ mod test { use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::joins::HashJoinExec; - use datafusion::physical_plan::projection::ProjectionExec; + use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr}; use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; + use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::windows::BoundedWindowAggExec; use datafusion::physical_plan::{ExecutionPlan, displayable}; use datafusion::physical_plan::{InputOrderMode, Partitioning}; @@ -965,11 +1005,117 @@ mod test { assert!(can_stay_inline(&scan_with_file_groups(4))); } - /// A branch over a leaf the scheduler doesn't know restricts away too: a - /// task that runs none of its partitions gets an empty node. + /// A branch over a one-partition leaf the scheduler doesn't know stays + /// inline: the task that owns the partition runs the leaf whole, and every + /// other task gets an empty leaf. + #[test] + fn one_partition_unknown_leaf_union_branch_stays_inline() { + assert!(can_stay_inline(&PayloadExec::leaf(&["p0"]))); + } + + /// A task can own only some partitions of a branch over a leaf of several + /// partitions, and nothing can cut an unknown leaf down to those without + /// a resolver, so the branch gets a stage of its own, as does one with + /// such a leaf deeper down. + #[test] + fn multi_partition_unknown_leaf_union_branch_needs_its_own_stage() { + let leaf = PayloadExec::leaf(&["p0", "p1"]); + assert!(!can_stay_inline(&leaf)); + + let projected: Arc = Arc::new( + ProjectionExec::try_new( + vec![ProjectionExpr { + expr: Arc::new(Column::new("a", 0)), + alias: "a".to_string(), + }], + leaf, + ) + .unwrap(), + ); + assert!(!can_stay_inline(&projected)); + } + + /// A branch over a shuffle, such as the final aggregate of a union of + /// aggregates, stays inline: the shuffle's reader restricts to any share, + /// though the planner sees it before it is resolved. + #[test] + fn branch_over_an_unresolved_shuffle_stays_inline() { + let schema = scan_with_file_groups(1).schema(); + let reader: Arc = Arc::new(UnresolvedShuffleExec::new( + 1, + schema, + Partitioning::UnknownPartitioning(4), + )); + let projected: Arc = Arc::new( + ProjectionExec::try_new( + vec![ProjectionExpr { + expr: Arc::new(Column::new("a", 0)), + alias: "a".to_string(), + }], + reader, + ) + .unwrap(), + ); + assert!(can_stay_inline(&projected)); + } + + /// Plans `plan` into query stages with the default options. + fn stages_of( + plan: Arc, + ) -> Result>, BallistaError> { + DefaultDistributedPlanner::new().plan_query_stages( + &"job".into(), + plan, + &datafusion::config::ConfigOptions::default(), + ) + } + + /// A union branch over an unknown leaf of several partitions gets a stage + /// of its own, as on main, rather than leave a task of the union's stage + /// owning part of it. The union's stage reads it through a shuffle, which + /// a task can cut to any share. + #[test] + fn union_over_a_multi_partition_unknown_leaf_gives_it_a_stage_of_its_own() + -> Result<(), BallistaError> { + let union = UnionExec::try_new(vec![ + scan_with_file_groups(3), + PayloadExec::leaf(&["p0", "p1"]), + ])?; + let stages = stages_of(union)?; + + assert_eq!(stages.len(), 2); + assert_plan!(stages[0].as_ref(), @r#" + ShuffleWriterExec: partitioning: UnknownPartitioning(2) + PayloadExec: payloads=["p0", "p1"] + "#); + assert_plan!(stages[1].as_ref(), @" + ShuffleWriterExec: partitioning: UnknownPartitioning(5) + UnionExec + DataSourceExec: file_groups={3 groups: [[file0.parquet], [file1.parquet], [file2.parquet]]}, projection=[a], file_type=parquet + UnresolvedShuffleExec: stage=1, partitioning: UnknownPartitioning(2) + "); + Ok(()) + } + + /// A union branch over a one-partition unknown leaf stays in the union's + /// stage: only the task that owns its partition runs it. #[test] - fn unknown_leaf_union_branch_stays_inline() { - assert!(can_stay_inline(&PayloadExec::leaf(&["p0", "p1"]))); + fn union_over_a_one_partition_unknown_leaf_keeps_it_in_the_unions_stage() + -> Result<(), BallistaError> { + let union = UnionExec::try_new(vec![ + scan_with_file_groups(3), + PayloadExec::leaf(&["p0"]), + ])?; + let stages = stages_of(union)?; + + assert_eq!(stages.len(), 1); + assert_plan!(stages[0].as_ref(), @r#" + ShuffleWriterExec: partitioning: UnknownPartitioning(4) + UnionExec + DataSourceExec: file_groups={3 groups: [[file0.parquet], [file1.parquet], [file2.parquet]]}, projection=[a], file_type=parquet + PayloadExec: payloads=["p0"] + "#); + Ok(()) } /// A branch already sitting behind a stage boundary needs nothing more. From 35e9ea595442847c3ab2ceb74323dc391581e5c2 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 16 Nov 2026 11:46:44 -0500 Subject: [PATCH 11/17] Run a stage with an unsplittable union branch as one task instead of failing The adaptive planner can leave a union branch that keeps partitions however it is restricted, such as one that merges its input into one partition, in the union's stage. A task given part of that stage would also run the branch, so preparing it failed, but only when the stage happened to be split, which depends on free task slots. Binding now packs such a stage into one task, as it does a stage that collapses its input, so each branch runs once and the query succeeds. The task ignores max_partitions_per_task and takes one vcore per partition, up to what its executor has free. Binding judges a stage with the resolver its tasks are restricted with, carried on the job's cache entry, so a node that resolver restricts doesn't pack the stage. The task-time error stays as a backstop. --- ballista/client/tests/split_union.rs | 53 ++---- ballista/scheduler/src/cluster/mod.rs | 149 ++++++++++++++++- ballista/scheduler/src/state/mod.rs | 23 +++ ballista/scheduler/src/state/task_builder.rs | 155 +++++++++++++++++- ballista/scheduler/src/state/task_manager.rs | 21 ++- docs/source/upgrading/56.0.0.md | 20 ++- .../source/user-guide/extending-components.md | 2 +- 7 files changed, 360 insertions(+), 63 deletions(-) diff --git a/ballista/client/tests/split_union.rs b/ballista/client/tests/split_union.rs index c63a6ad974..732e543023 100644 --- a/ballista/client/tests/split_union.rs +++ b/ballista/client/tests/split_union.rs @@ -76,6 +76,18 @@ mod split_union { "SELECT 1 AS a UNION ALL SELECT 2 UNION ALL SELECT 3", "SELECT 100 AS id UNION ALL SELECT id FROM test", "SELECT column1 AS id FROM (VALUES (10), (11)) UNION ALL SELECT id FROM test", + // Branches that merge their input into one partition, which the + // adaptive planner leaves in the union's stage: a split stage runs as + // one task, so each branch runs once. + "SELECT id FROM test UNION ALL SELECT id FROM test ORDER BY id", + "SELECT o_id FROM orders UNION ALL SELECT c_id FROM customers ORDER BY o_id", + "(SELECT id FROM test ORDER BY id LIMIT 3) UNION ALL SELECT id FROM test", + "(SELECT o_id FROM orders ORDER BY o_id LIMIT 3) \ + UNION ALL SELECT o_id FROM orders", + "SELECT count(*) AS c FROM orders UNION ALL SELECT count(*) FROM customers", + // A filter over a one-row source, under a round-robin repartition. + "SELECT column1 AS id FROM (VALUES (10), (11)) WHERE column1 > 10 \ + UNION ALL SELECT id FROM test", ]; async fn register(ctx: &SessionContext, dir: &Path) { @@ -120,45 +132,4 @@ mod split_union { ); } } - - /// The adaptive planner leaves a branch that merges its input into one - /// partition, here each sorted branch of a `UNION ALL ... ORDER BY`, in the - /// union's stage. With one partition per task every task would also run - /// the branch it doesn't own, so the job fails rather than return those - /// rows twice. - #[tokio::test] - async fn a_split_union_with_a_merging_branch_fails_under_the_adaptive_planner() { - let ctx = SessionContext::standalone_with_state(ballista_state(&[ - (BALLISTA_ADAPTIVE_PLANNER_ENABLED, "true"), - (BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, "1"), - ])) - .await - .unwrap(); - register_alltypes(&ctx, "test").await; - - let error = ctx - .sql("SELECT id FROM test UNION ALL SELECT id FROM test ORDER BY id") - .await - .unwrap() - .collect() - .await - .unwrap_err() - .to_string(); - // The job id before it differs from run to run. - let reason = error.split_once("DataFusion error: ").map(|(_, r)| r); - assert_eq!( - reason, - Some( - "Error during planning: cannot restrict a `UnionExec` branch to \ - partitions []: its `SortExec` keeps a partition count of 1, so the \ - task would also run partitions another task owns. The branch merges \ - its input into one partition, as an `ORDER BY`, a `LIMIT` or an \ - aggregate without `GROUP BY` does. The static planner gives such a \ - branch a stage of its own; the adaptive planner \ - (`ballista.planner.adaptive.enabled`) can leave it in the union's \ - stage." - ), - "{error}" - ); - } } diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs index c804bfb486..0c323c6ae3 100644 --- a/ballista/scheduler/src/cluster/mod.rs +++ b/ballista/scheduler/src/cluster/mod.rs @@ -27,9 +27,11 @@ use crate::state::execution_graph::{ ExecutionGraphBox, TaskDescription, create_task_info, }; use crate::state::session_manager::share_file_statistics_cache; +use crate::state::task_builder::has_unsplittable_union; use crate::state::task_manager::JobInfoCache; use ballista_core::config::BallistaConfig; use ballista_core::error::Result; +use ballista_core::execution_plans::plan_algebra::PartitionSliceableResolver; use ballista_core::execution_plans::{ PipelinedShuffleReaderExec, RangeShuffleReaderExec, ShuffleReaderExec, UnresolvedShuffleExec, @@ -585,17 +587,28 @@ pub(crate) fn stage_has_input_collapse(plan_root: &Arc) -> bo /// tasks run in parallel on the remaining vcores. Correctness still /// requires packing the entire pending queue into one bind (a split /// collapse would produce partial results downstream can't merge). +/// +/// - **Unsplittable union stage** (see [`has_unsplittable_union`]): a +/// `UnionExec` branch keeps partitions however it is restricted, so every +/// task given part of the stage would also run that branch, and the +/// rewriter refuses such a task. The stage is packed into one task too, +/// which runs each branch once. Its root keeps all of its partitions, so the +/// task consumes one vcore for each, up to what the executor has free. fn bind_one( running_stage: &mut crate::state::execution_stage::RunningStage, session_id: &str, job_id: &JobId, budget: &mut AvailableVcores, + resolver: Option<&dyn PartitionSliceableResolver>, ) -> Option { let is_collapse = stage_has_input_collapse(&running_stage.plan); - // Cap non-collapse slices at the configured `max_partitions_per_task`. - // Collapse stages must still pack their full pending queue into a single - // task for correctness — a split collapse would produce partial results - // downstream can't merge. + let is_unsplittable = + !is_collapse && has_unsplittable_union(&running_stage.plan, resolver); + // Cap other slices at the configured `max_partitions_per_task`. Collapse + // and unsplittable union stages must still pack their full pending queue + // into a single task for correctness: a split collapse would produce + // partial results downstream can't merge, and a split union would run a + // branch in every task. let cap = running_stage .session_config .options() @@ -604,7 +617,7 @@ fn bind_one( .map(|bc| bc.max_partitions_per_task()) .filter(|&n| n > 0) .unwrap_or(usize::MAX); - let max_partitions = if is_collapse { + let max_partitions = if is_collapse || is_unsplittable { usize::MAX } else { (budget.vcores as usize).min(cap) @@ -621,11 +634,13 @@ fn bind_one( // one vcore and leave the rest for other stages' tasks on this executor. let vcores_consumed = if is_collapse { 1 + } else if is_unsplittable { + (input_partition_ids.len() as u32).min(budget.vcores) } else { input_partition_ids.len() as u32 }; debug!( - "bind_one: job={} stage={} exec={} vcores={} max={} slice_len={} consumed={} partitions={:?} collapse={}", + "bind_one: job={} stage={} exec={} vcores={} max={} slice_len={} consumed={} partitions={:?} collapse={} unsplittable_union={}", job_id, running_stage.stage_id, budget.executor_id, @@ -639,6 +654,7 @@ fn bind_one( vcores_consumed, input_partition_ids, is_collapse, + is_unsplittable, ); let executor_id = budget.executor_id.clone(); // task_id is the append-order slot in `task_infos` — since we're @@ -767,7 +783,13 @@ async fn bind_tasks_with( return schedulable_tasks; }; idx = picked; - match bind_one(running_stage, &session_id, job_id, &mut *budgets[idx]) { + match bind_one( + running_stage, + &session_id, + job_id, + &mut *budgets[idx], + job_info.partition_sliceable_resolver.as_deref(), + ) { Some(bound) => { schedulable_tasks.push(bound); idx = strategy.advance(idx, budgets.len()); @@ -847,17 +869,26 @@ mod test { use crate::cluster::stage_has_input_collapse; use crate::cluster::{ - BindStrategy, BoundTask, bind_task_bias, bind_task_round_robin, + BindStrategy, BoundTask, bind_one, bind_task_bias, bind_task_round_robin, next_with_capacity, }; use crate::state::execution_graph::{ExecutionGraph, StaticExecutionGraph}; use crate::state::task_manager::JobInfoCache; + use crate::test_utils::payload::{PayloadExec, payload_resolver}; + use crate::test_utils::scan_with_file_groups; use crate::test_utils::{ mock_completed_task, revive_graph_and_complete_next_stage, test_aggregation_plan_with_config, }; use ballista_core::config::BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK; + use ballista_core::execution_plans::ShuffleWriterExec; + use ballista_core::execution_plans::plan_algebra::PartitionSliceableResolver; use ballista_core::extension::SessionConfigExt; + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; + use datafusion::physical_plan::ExecutionPlan; + use datafusion::physical_plan::sorts::sort::SortExec; + use datafusion::physical_plan::union::UnionExec; use datafusion::prelude::SessionConfig; #[tokio::test] @@ -1058,6 +1089,108 @@ mod test { Ok(graph) } + /// A running stage of `plan` below a shuffle writer, with one partition + /// per task at most. + fn running_stage( + plan: Arc, + ) -> crate::state::execution_stage::RunningStage { + let partitions = plan.properties().output_partitioning().partition_count(); + let writer: Arc = Arc::new( + ShuffleWriterExec::try_new("job".into(), 1, plan, String::new()).unwrap(), + ); + crate::state::execution_stage::RunningStage::new( + 1, + 0, + writer, + partitions, + vec![], + HashMap::new(), + Arc::new( + SessionConfig::new_with_ballista() + .set_str(BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, "1"), + ), + ) + } + + /// Binds every task of `stage` on one executor of `vcores` vcores, + /// returning each task's partitions and vcores. + fn bind_all( + stage: &mut crate::state::execution_stage::RunningStage, + vcores: u32, + resolver: Option<&dyn PartitionSliceableResolver>, + ) -> Vec<(Vec, u32)> { + let mut budget = AvailableVcores { + executor_id: "executor".to_string(), + vcores, + }; + let mut bound = vec![]; + while budget.vcores > 0 { + let Some((_, task)) = + bind_one(stage, "session", &"job".into(), &mut budget, resolver) + else { + break; + }; + bound.push((task.global_input_partition_ids, task.vcores_consumed)); + } + bound + } + + /// A union whose sorted branch keeps its partition in every task runs as + /// one task, whatever `max_partitions_per_task` says, using one vcore for + /// each partition the executor has free. + #[test] + fn a_stage_with_an_unsplittable_union_is_bound_as_one_task() { + let sorted: Arc = Arc::new(SortExec::new( + LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new( + "a", 0, + )))]) + .unwrap(), + scan_with_file_groups(2), + )); + // Partition 0 is the sort's, 1-2 the scan's. + let union = UnionExec::try_new(vec![sorted, scan_with_file_groups(2)]).unwrap(); + + assert_eq!( + bind_all(&mut running_stage(union.clone()), 8, None), + [(vec![0, 1, 2], 3)] + ); + assert_eq!( + bind_all(&mut running_stage(union), 2, None), + [(vec![0, 1, 2], 2)] + ); + } + + /// A union that splits cleanly keeps one partition per task. + #[test] + fn a_stage_with_a_splittable_union_is_bound_a_partition_per_task() { + let union = + UnionExec::try_new(vec![scan_with_file_groups(2), scan_with_file_groups(1)]) + .unwrap(); + + assert_eq!( + bind_all(&mut running_stage(union), 8, None), + [(vec![0], 1), (vec![1], 1), (vec![2], 1)] + ); + } + + /// Binding asks the resolver the tasks are restricted with, so a node of + /// the user's own that it can restrict doesn't pack the stage. + #[test] + fn binding_a_union_asks_the_resolver_the_tasks_are_restricted_with() { + let own = PayloadExec::over(&["p0", "p1"], vec![scan_with_file_groups(2)], &[]); + let union = UnionExec::try_new(vec![own, scan_with_file_groups(1)]).unwrap(); + let resolver = payload_resolver(); + + assert_eq!( + bind_all(&mut running_stage(union.clone()), 8, None), + [(vec![0, 1, 2], 3)] + ); + assert_eq!( + bind_all(&mut running_stage(union), 8, Some(resolver.as_ref())), + [(vec![0], 1), (vec![1], 1), (vec![2], 1)] + ); + } + fn mock_budgets() -> Vec { vec![ AvailableVcores { diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs index 6f08d95e34..3c52560096 100644 --- a/ballista/scheduler/src/state/mod.rs +++ b/ballista/scheduler/src/state/mod.rs @@ -1102,6 +1102,29 @@ mod tests { Ok(()) } + /// A running job carries the resolver its tasks are restricted with, so + /// binding judges which stages it can split the same way. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_running_job_carries_the_configured_resolver() -> Result<()> { + let resolver = payload_resolver(); + let config = SchedulerConfig::default() + .with_partition_sliceable_resolver(resolver.clone()); + let (state, job_id, _bound) = bind_split_payload_stage( + config, + Arc::new(CapturingLauncher::default()), + PayloadExec::leaf(&["p0", "p1", "p2", "p3"]), + ) + .await?; + + let jobs = state.task_manager.get_running_job_cache(); + let carried = jobs[&job_id] + .partition_sliceable_resolver + .clone() + .expect("the job carries a resolver"); + assert!(Arc::ptr_eq(&carried, &resolver)); + Ok(()) + } + /// The input of the shuffle writer of each task `launcher` launched, /// decoded as the executor would, with the codec the scheduler used. fn shipped_writer_inputs( diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index 54a800be11..27812104d1 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -111,6 +111,45 @@ pub(crate) fn stages_read_whole(plan: &Arc) -> Vec { found } +/// Whether a task given only some of the stage plan `plan`'s partitions would +/// also run a `UnionExec` branch it doesn't own: a branch that keeps +/// partitions when restricted to none, as one that merges its input into one +/// partition, reads a broadcast or sets its own partition count does. +/// [`restrict_plan_to_partitions`] refuses such a task, so the scheduler runs +/// the stage as one task instead, which runs every branch once. +/// +/// The adaptive planner can leave such a branch in the union's stage. The +/// static planner gives it a stage of its own, so its stages never need this. +pub(crate) fn has_unsplittable_union( + plan: &Arc, + resolver: Option<&dyn PartitionSliceableResolver>, +) -> bool { + fn walk( + plan: &Arc, + under_collect: bool, + resolver: Option<&dyn PartitionSliceableResolver>, + ) -> bool { + // Below a collapse every task reads the whole input anyway, so a + // union there is never split. + if !under_collect && plan.is::() { + return plan.children().into_iter().any(|branch| { + !restrict(branch.clone(), &[], false, resolver).is_ok_and(|restricted| { + restricted + .properties() + .output_partitioning() + .partition_count() + == 0 + }) + }); + } + plan.children() + .into_iter() + .zip(child_scopes(plan, under_collect)) + .any(|(child, collect)| walk(child, collect, resolver)) + } + walk(plan, false, resolver) +} + /// Recursive worker. `under_collect` is the scope inherited from ancestors: /// once set, every descendant leaf reads the full upstream. Scope is passed /// by value, so sibling subtrees never leak state to each other. @@ -270,8 +309,11 @@ fn ensure_leaf_runs_whole( /// broadcast reader, which every task reads whole, and a node that sets its own /// partition count do the same. /// -/// The scheduler doesn't know here which planner made the stage, so the error -/// names the branch's causes rather than one planner's. +/// The scheduler binds a stage with such a branch as one task (see +/// [`has_unsplittable_union`]), whose branches are all restricted to all of +/// their partitions, so this is a backstop: it refuses a task rather than let +/// one return rows twice. It doesn't know which planner made the stage, so the +/// error names the branch's causes rather than one planner's. fn ensure_branch_restricted( branch: &Arc, partitions: &[usize], @@ -1370,6 +1412,115 @@ mod tests { ); } + /// A sort that merges `input` into one partition, as the adaptive planner + /// leaves on each branch of a `UNION ALL ... ORDER BY`. + fn merged(input: Arc) -> Arc { + use datafusion::physical_plan::sorts::sort::SortExec; + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new( + Column::new("a", 0), + ))]) + .unwrap(); + Arc::new(SortExec::new(ordering, input)) + } + + /// Which stage plans a task may run only part of: one per case, whether + /// it has a union that must run as one task. + #[test] + fn unsplittable_unions_are_those_with_a_branch_that_keeps_partitions() { + use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; + let union = |branches: Vec>| -> Arc { + UnionExec::try_new(branches).unwrap() + }; + let broadcast: Arc = Arc::new( + ShuffleReaderExec::try_new_broadcast( + 1, + (0..2).map(create_partition).collect(), + scan_with_file_groups(1).schema(), + 2, + ) + .unwrap(), + ); + // Restricted by the resolver only; left alone it keeps two partitions. + let own = + || PayloadExec::over(&["p0", "p1"], vec![scan_with_file_groups(2)], &[]); + let resolver = payload_resolver(); + + let cases: Vec<(&str, Arc, bool)> = vec![ + ("no union", scan_with_file_groups(2), false), + ( + "scans", + union(vec![scan_with_file_groups(2), scan_with_file_groups(2)]), + false, + ), + ( + "an unknown leaf", + union(vec![ + PayloadExec::leaf(&["p0", "p1"]), + scan_with_file_groups(2), + ]), + false, + ), + ( + "a merging branch", + union(vec![ + merged(scan_with_file_groups(2)), + scan_with_file_groups(2), + ]), + true, + ), + ( + "a broadcast", + union(vec![broadcast, scan_with_file_groups(2)]), + true, + ), + ( + "a node of your own", + union(vec![own(), scan_with_file_groups(2)]), + true, + ), + ( + "a merging branch of a nested union", + union(vec![ + union(vec![ + merged(scan_with_file_groups(2)), + scan_with_file_groups(1), + ]), + scan_with_file_groups(2), + ]), + true, + ), + ( + "a union below a sort", + merged(union(vec![ + merged(scan_with_file_groups(2)), + scan_with_file_groups(2), + ])), + false, + ), + ( + "a union below a collapse", + Arc::new(CoalescePartitionsExec::new(union(vec![ + merged(scan_with_file_groups(2)), + scan_with_file_groups(2), + ]))), + false, + ), + ]; + let actual: Vec<(&str, bool)> = cases + .iter() + .map(|(name, plan, _)| (*name, has_unsplittable_union(plan, None))) + .collect(); + let expected: Vec<(&str, bool)> = cases + .iter() + .map(|(name, _, unsplittable)| (*name, *unsplittable)) + .collect(); + assert_eq!(actual, expected); + + // The resolver that restricts the tasks can split the node of your own. + let with_own = union(vec![own(), scan_with_file_groups(2)]); + assert!(!has_unsplittable_union(&with_own, Some(resolver.as_ref()))); + } + /// `plan` is a leaf with `like`'s schema and no partitions, as the /// executor will decode it. fn assert_no_partitions( diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs index c4868e7c2f..b4940c8e7d 100644 --- a/ballista/scheduler/src/state/task_manager.rs +++ b/ballista/scheduler/src/state/task_manager.rs @@ -171,6 +171,9 @@ pub struct JobInfoCache { pub execution_graph: Arc>, /// Cached job status for quick access. pub status: Option, + /// The resolver the job's tasks are restricted with, so binding can tell + /// which stages a task may run only part of. + pub partition_sliceable_resolver: Option>, } impl JobInfoCache { @@ -181,8 +184,18 @@ impl JobInfoCache { Self { execution_graph: Arc::new(RwLock::new(graph)), status, + partition_sliceable_resolver: None, } } + + /// Sets the resolver the job's tasks are restricted with. + pub fn with_partition_sliceable_resolver( + mut self, + resolver: Option>, + ) -> Self { + self.partition_sliceable_resolver = resolver; + self + } } /// What losing an executor requires of the jobs that were using it. @@ -446,8 +459,12 @@ impl TaskManager .submit_job(job_id.to_owned(), &graph, subscriber) .await?; graph.revive(); - self.active_job_cache - .insert(job_id.to_owned(), JobInfoCache::new(graph)); + self.active_job_cache.insert( + job_id.to_owned(), + JobInfoCache::new(graph).with_partition_sliceable_resolver( + self.partition_sliceable_resolver.clone(), + ), + ); Ok(()) } diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index a8086b230d..dd97a6f6a9 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -218,15 +218,16 @@ it was while its children are restricted, because the scheduler cannot tell one that holds data per partition from one that does not, such as a `FilterExec`. An interior node of your own that holds data per partition still returns wrong results in a split stage unless a resolver restricts it. Below a `UnionExec`, -the job fails instead when such a node keeps its own partition count, since the -task would also run partitions that another task owns. +a stage where such a node keeps its own partition count runs as one task +instead, since a task given part of it would also run partitions that another +task owns. The error names the node and what to implement. Under push-staged scheduling the job's status says only that a task could not be prepared; the reason is in the scheduler's log. To make nodes of your own restrictable, see [Custom Physical Plan Nodes](../user-guide/extending-components.md#custom-physical-plan-nodes). -#### A split `UNION ALL` with a branch that merges its input fails under the adaptive planner +#### A split `UNION ALL` with a branch that merges its input runs as one task When a stage is split between tasks, each task restricts the `UnionExec` branches other tasks own to no partitions. The adaptive planner can leave a @@ -236,12 +237,13 @@ a `UNION ALL` of aggregates without `GROUP BY`. Such a branch reads its input whole and keeps its one partition, so every task of the stage ran it, and the query returned wrong rows, over any source including Parquet. -The scheduler now fails the job instead, with an error that starts "cannot -restrict a `UnionExec` branch". Whether a stage is split depends on how many -task slots are free, so such a query can pass on an idle cluster and fail on a -busy one. The static planner gives such a branch a stage of its own and is -unaffected, so setting `ballista.planner.adaptive.enabled` to `false` avoids the -error. +The scheduler now runs such a stage as one task, as it does a stage that +collapses its input, so each branch runs once. The same goes for a branch that +reads a broadcast or holds a node that sets its own partition count. The task +ignores `ballista.scheduler.max_partitions_per_task` and takes one vcore for +each of the stage's partitions, up to what its executor has free, so the stage +runs with less parallelism than before. The static planner gives such a branch +a stage of its own and is unaffected. ### API changes diff --git a/docs/source/user-guide/extending-components.md b/docs/source/user-guide/extending-components.md index 1e0317efa1..5ecad770ce 100644 --- a/docs/source/user-guide/extending-components.md +++ b/docs/source/user-guide/extending-components.md @@ -179,7 +179,7 @@ let config = SchedulerConfig::default().with_partition_sliceable_resolver(Arc::n - `children` are the node's inputs, none for a scan, restricted to `partitions`. An input the node declares `SinglePartition` arrives whole, as a join's build side does. Partition `i` of the node must read partition `i` of every other input. - The restricted node must have exactly `partitions.len()` partitions. The scheduler rejects any other count. -- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. A node with inputs is still asked, with its inputs already restricted to none. If nothing restricts such a node and it keeps its own partition count, the branch keeps partitions the task doesn't own, and the scheduler fails the job. +- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. A node with inputs is still asked, with its inputs already restricted to none. If nothing restricts such a node and it keeps its own partition count, a task given part of the union's stage would also run partitions it doesn't own, so the scheduler runs that stage as one task. - The restricted node is serialized with the scheduler's `physical codec`, so what it carries per partition has to survive encoding and decoding. - The resolver does not need `PartitionSliceable`. It can build the restricted node itself, which is the way to restrict a node you cannot implement the trait on, such as a `DataSourceExec` over a `DataSource` of your own: match it with `downcast_ref`, then build a `DataSourceExec` over the restricted source. - The resolver is asked after Ballista's own list. A node that is read whole, such as one below a `CoalescePartitionsExec` or on a join's build side, needs none of this. From 7c27114650e1ce8a645db6bf53ac61eb1ccd21a0 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 16 Nov 2026 20:54:14 -0500 Subject: [PATCH 12/17] Share one split check between planning and binding, and simplify the backstop - Planning and binding now ask the same question through `branch_splits`: can restriction give a task none of a union branch and part of it. Binding used to try only none, so under the adaptive planner a branch over a leaf of several partitions that nothing can restrict failed at task time; its stage now runs as one task. - The union-branch error is a backstop that binding makes unreachable, so it states the invariant instead of classifying causes. - Binding returns early with nothing pending and runs the union check only when the pending partitions don't fit in one slice anyway. - JobInfoCache's resolver is crate-private. - The Iceberg guide no longer lists split unions as a limitation. --- ballista/scheduler/src/cluster/mod.rs | 13 +- ballista/scheduler/src/planner.rs | 49 +---- ballista/scheduler/src/state/task_builder.rs | 200 +++++-------------- ballista/scheduler/src/state/task_manager.rs | 4 +- docs/source/upgrading/56.0.0.md | 3 +- docs/source/user-guide/iceberg.md | 12 -- 6 files changed, 77 insertions(+), 204 deletions(-) diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs index 0c323c6ae3..20124d311d 100644 --- a/ballista/scheduler/src/cluster/mod.rs +++ b/ballista/scheduler/src/cluster/mod.rs @@ -601,9 +601,10 @@ fn bind_one( budget: &mut AvailableVcores, resolver: Option<&dyn PartitionSliceableResolver>, ) -> Option { + if running_stage.pending.is_empty() { + return None; + } let is_collapse = stage_has_input_collapse(&running_stage.plan); - let is_unsplittable = - !is_collapse && has_unsplittable_union(&running_stage.plan, resolver); // Cap other slices at the configured `max_partitions_per_task`. Collapse // and unsplittable union stages must still pack their full pending queue // into a single task for correctness: a split collapse would produce @@ -617,10 +618,16 @@ fn bind_one( .map(|bc| bc.max_partitions_per_task()) .filter(|&n| n > 0) .unwrap_or(usize::MAX); + let split_max = (budget.vcores as usize).min(cap); + // The union check rewrites the plan, so it runs only when it can change + // the outcome: when the pending partitions don't fit in one slice anyway. + let is_unsplittable = !is_collapse + && running_stage.pending.remaining() > split_max + && has_unsplittable_union(&running_stage.plan, resolver); let max_partitions = if is_collapse || is_unsplittable { usize::MAX } else { - (budget.vcores as usize).min(cap) + split_max }; let input_partition_ids = running_stage.pending.next_slice(max_partitions); if input_partition_ids.is_empty() { diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index e1cbc17e73..6aa2f3be69 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -55,7 +55,7 @@ use log::debug; use crate::physical_optimizer::join_selection::{ collect_left_broadcast_safe, should_swap_join_order, }; -use crate::state::task_builder::restrict_plan_to_partitions; +use crate::state::task_builder::branch_splits; type PartialQueryStageResult = (Arc, Vec>); @@ -717,47 +717,20 @@ impl DefaultDistributedPlanner { /// /// A branch already sitting behind a stage boundary needs nothing more, and /// neither does one that per-task restriction can cut to whatever share of it -/// a task runs: none of it, which the union's index arithmetic relies on, or -/// some of it. These questions are put to the real rewriter rather than to a -/// list of operators, so they stay correct as the rewriter's handling of -/// collapses, broadcast readers and leaf types evolves. +/// a task runs ([`branch_splits`]). That question is put to the real rewriter +/// rather than to a list of operators, so it stays correct as the rewriter's +/// handling of collapses, broadcast readers and leaf types evolves. +/// +/// The rewriter is asked without the scheduler's resolver, so a branch over a +/// leaf of several partitions that only a resolver can restrict, such as a +/// scan over a source of your own, gets a stage of its own. An +/// `UnresolvedShuffleExec` passes through [`UnresolvedShuffles`], since it +/// becomes a shuffle reader before any task runs. fn can_stay_inline(plan: &Arc) -> bool { if plan.is::() { return true; } - !holds_a_broadcast(plan) && restricts_to_nothing(plan) && restricts_to_part(plan) -} - -/// Whether per-task restriction can reduce `plan` to zero partitions. An empty -/// slice means "this task polls nothing here", so a branch that still reports a -/// partition afterwards, or that the rewriter refuses, cannot be restricted -/// away. -/// -/// No resolver is passed. A leaf Ballista does not know passes, because the -/// rewriter replaces a leaf the task runs none of with an empty leaf. An -/// interior node Ballista does not know that keeps its own partition count -/// fails, and its branch gets a stage of its own. -fn restricts_to_nothing(plan: &Arc) -> bool { - restrict_plan_to_partitions(plan.clone(), &[], None) - .is_ok_and(|p| p.properties().output_partitioning().partition_count() == 0) -} - -/// Whether per-task restriction can cut `plan` to its first partition alone, -/// standing in for any task that owns some but not all of the branch. The -/// rewriter refuses a leaf it cannot restrict for every slice but the whole, -/// so `[0]` catches such a leaf of more than one partition as any partial -/// slice would. -/// -/// A leaf Ballista does not know passes only if it has one partition, which -/// the task that owns it runs whole. A branch over a leaf of more partitions, -/// such as a scan over a source of your own, gets a stage of its own, so that -/// only the leaf's own stage needs a task to run all of it. An -/// `UnresolvedShuffleExec` passes, since it becomes a shuffle reader before -/// any task runs, and the rewriter restricts that to any slice. -fn restricts_to_part(plan: &Arc) -> bool { - plan.properties().output_partitioning().partition_count() == 0 - || restrict_plan_to_partitions(plan.clone(), &[0], Some(&UnresolvedShuffles)) - .is_ok() + !holds_a_broadcast(plan) && branch_splits(plan, Some(&UnresolvedShuffles)) } /// Restricts an `UnresolvedShuffleExec` to one partition for each index, as diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index 27812104d1..cc811a85a2 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -112,11 +112,9 @@ pub(crate) fn stages_read_whole(plan: &Arc) -> Vec { } /// Whether a task given only some of the stage plan `plan`'s partitions would -/// also run a `UnionExec` branch it doesn't own: a branch that keeps -/// partitions when restricted to none, as one that merges its input into one -/// partition, reads a broadcast or sets its own partition count does. -/// [`restrict_plan_to_partitions`] refuses such a task, so the scheduler runs -/// the stage as one task instead, which runs every branch once. +/// run a `UnionExec` branch wrongly: a branch that fails [`branch_splits`]. +/// The scheduler runs such a stage as one task instead, which runs every +/// branch whole. /// /// The adaptive planner can leave such a branch in the union's stage. The /// static planner gives it a stage of its own, so its stages never need this. @@ -132,15 +130,10 @@ pub(crate) fn has_unsplittable_union( // Below a collapse every task reads the whole input anyway, so a // union there is never split. if !under_collect && plan.is::() { - return plan.children().into_iter().any(|branch| { - !restrict(branch.clone(), &[], false, resolver).is_ok_and(|restricted| { - restricted - .properties() - .output_partitioning() - .partition_count() - == 0 - }) - }); + return plan + .children() + .into_iter() + .any(|branch| !branch_splits(branch, resolver)); } plan.children() .into_iter() @@ -150,6 +143,26 @@ pub(crate) fn has_unsplittable_union( walk(plan, false, resolver) } +/// Whether restriction can give a task any share of the `UnionExec` branch +/// `branch`: none of it, which the union's index arithmetic relies on, and +/// part of it, tried as its first partition alone. +/// +/// A branch fails when it keeps partitions restricted to none, as one that +/// merges its input into one partition, reads a broadcast or sets its own +/// partition count does. It also fails over a leaf nothing can restrict that +/// has more than one partition, which the rewriter refuses for every slice +/// but the whole, so `[0]` catches it as any partial slice would. +pub(crate) fn branch_splits( + branch: &Arc, + resolver: Option<&dyn PartitionSliceableResolver>, +) -> bool { + let restricts_to_nothing = restrict(branch.clone(), &[], false, resolver) + .is_ok_and(|p| p.properties().output_partitioning().partition_count() == 0); + restricts_to_nothing + && (branch.properties().output_partitioning().partition_count() == 0 + || restrict(branch.clone(), &[0], false, resolver).is_ok()) +} + /// Recursive worker. `under_collect` is the scope inherited from ancestors: /// once set, every descendant leaf reads the full upstream. Scope is passed /// by value, so sibling subtrees never leak state to each other. @@ -301,19 +314,12 @@ fn ensure_leaf_runs_whole( /// Check that a restricted `UnionExec` branch has one partition for each of /// `partitions`. The executor runs every partition of the task's plan, so a -/// branch that keeps partitions the task wasn't given runs them in this task -/// too, and the union's later partitions no longer line up with the task's. -/// When those partitions read their input whole, as below a `SortExec`, a -/// `LIMIT` or an aggregate without `GROUP BY` that merges the branch into one -/// partition, the task returns rows that another task returns as well. A -/// broadcast reader, which every task reads whole, and a node that sets its own -/// partition count do the same. +/// branch that keeps partitions the task wasn't given would run them in this +/// task too, returning rows another task returns as well. /// -/// The scheduler binds a stage with such a branch as one task (see -/// [`has_unsplittable_union`]), whose branches are all restricted to all of -/// their partitions, so this is a backstop: it refuses a task rather than let -/// one return rows twice. It doesn't know which planner made the stage, so the -/// error names the branch's causes rather than one planner's. +/// Binding runs a stage with such a branch as one task (see +/// [`has_unsplittable_union`]), so this only fails if binding and restriction +/// disagree: it refuses the task rather than let it return rows twice. fn ensure_branch_restricted( branch: &Arc, partitions: &[usize], @@ -322,58 +328,15 @@ fn ensure_branch_restricted( if count == partitions.len() { return Ok(()); } - let name = branch.name(); - let cause = if along_single_inputs(branch, &reads_a_broadcast) { - "The branch reads a broadcast, which every task reads whole as one \ - partition. The static planner gives a branch that reads a broadcast a \ - stage of its own." - } else if along_single_inputs(branch, &merges_its_input) { - "The branch merges its input into one partition, as an `ORDER BY`, a \ - `LIMIT` or an aggregate without `GROUP BY` does. The static planner \ - gives such a branch a stage of its own; the adaptive planner \ - (`ballista.planner.adaptive.enabled`) can leave it in the union's stage." - } else { - "The branch holds a node that sets its own partition count. A node of \ - your own can be restricted by \ - `SchedulerConfig::partition_sliceable_resolver`." - }; plan_err!( "cannot restrict a `UnionExec` branch to partitions {partitions:?}: its \ - `{name}` keeps a partition count of {count}, so the task would also run \ - partitions another task owns. {cause}" + `{}` keeps a partition count of {count}, so the task would also run \ + partitions another task owns. The scheduler should have run this stage \ + as one task.", + branch.name() ) } -/// Whether `found` holds for `plan` or for a node below it reached through -/// nodes of one input each, which pass that node's partition count up. -fn along_single_inputs( - plan: &Arc, - found: &dyn Fn(&Arc) -> bool, -) -> bool { - if found(plan) { - return true; - } - match plan.children().as_slice() { - [child] => along_single_inputs(child, found), - _ => false, - } -} - -fn reads_a_broadcast(plan: &Arc) -> bool { - plan.downcast_ref::() - .is_some_and(|reader| reader.broadcast) - || plan - .downcast_ref::() - .is_some_and(|reader| reader.broadcast) -} - -/// Whether `plan` reads its one input whole, as a `SortExec`, a -/// `GlobalLimitExec`, a final aggregate without `GROUP BY` or a coalesce does, -/// and so merges it into one partition. -fn merges_its_input(plan: &Arc) -> bool { - plan.children().len() == 1 && child_scopes(plan, false) == [true] -} - /// Bucket each parent partition of a `UnionExec` into its owning child's /// local index. Iterates over `partitions` and, for each, subtracts each /// child's count until the remainder lands inside a child. Returns one @@ -1338,77 +1301,8 @@ mod tests { error, "Error during planning: cannot restrict a `UnionExec` branch to \ partitions []: its `SortExec` keeps a partition count of 1, so the \ - task would also run partitions another task owns. The branch merges \ - its input into one partition, as an `ORDER BY`, a `LIMIT` or an \ - aggregate without `GROUP BY` does. The static planner gives such a \ - branch a stage of its own; the adaptive planner \ - (`ballista.planner.adaptive.enabled`) can leave it in the union's \ - stage." - ); - } - - /// A node of the user's own that keeps its partition count is pointed at - /// the resolver, not at the planners. - #[test] - fn a_union_branch_with_a_node_of_your_own_is_pointed_at_the_resolver() { - // Without a resolver it keeps one partition per payload, whatever its - // input is restricted to. - let own = PayloadExec::over(&["p0", "p1"], vec![scan_with_file_groups(2)], &[]); - // Partitions 0-1 are the node's, 2-3 the scan's. - let union = UnionExec::try_new(vec![own, scan_with_file_groups(2)]).unwrap(); - - let error = restrict_plan_to_partitions(union, &[2, 3], None) - .unwrap_err() - .to_string(); - assert_eq!( - error, - "Error during planning: cannot restrict a `UnionExec` branch to \ - partitions []: its `PayloadExec` keeps a partition count of 2, so the \ - task would also run partitions another task owns. The branch holds a \ - node that sets its own partition count. A node of your own can be \ - restricted by `SchedulerConfig::partition_sliceable_resolver`." - ); - } - - /// A broadcast reader keeps its one partition in every task, so a union - /// branch that reads one is refused in a task that owns another branch, - /// and the error says so rather than blaming a sort or a limit. - #[test] - fn a_union_branch_reading_a_broadcast_is_refused_as_a_broadcast() { - use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr}; - let schema = scan_with_file_groups(2).schema(); - let reader = ShuffleReaderExec::try_new_broadcast( - 1, - (0..2).map(create_partition).collect(), - schema, - 2, - ) - .unwrap(); - let projected: Arc = Arc::new( - ProjectionExec::try_new( - vec![ProjectionExpr { - expr: Arc::new(Column::new("a", 0)), - alias: "a".to_string(), - }], - Arc::new(reader), - ) - .unwrap(), - ); - // Partition 0 is the broadcast branch's, 1-2 the scan's. - let union = - UnionExec::try_new(vec![projected, scan_with_file_groups(2)]).unwrap(); - - let error = restrict_plan_to_partitions(union, &[1, 2], None) - .unwrap_err() - .to_string(); - assert_eq!( - error, - "Error during planning: cannot restrict a `UnionExec` branch to \ - partitions []: its `ProjectionExec` keeps a partition count of 1, so \ - the task would also run partitions another task owns. The branch \ - reads a broadcast, which every task reads whole as one partition. The \ - static planner gives a branch that reads a broadcast a stage of its \ - own." + task would also run partitions another task owns. The scheduler \ + should have run this stage as one task." ); } @@ -1423,7 +1317,7 @@ mod tests { Arc::new(SortExec::new(ordering, input)) } - /// Which stage plans a task may run only part of: one per case, whether + /// Which stage plans a task may run only part of: for each case, whether /// it has a union that must run as one task. #[test] fn unsplittable_unions_are_those_with_a_branch_that_keeps_partitions() { @@ -1453,12 +1347,17 @@ mod tests { false, ), ( - "an unknown leaf", + "a one-partition unknown leaf", + union(vec![PayloadExec::leaf(&["p0"]), scan_with_file_groups(2)]), + false, + ), + ( + "an unknown leaf of two partitions", union(vec![ PayloadExec::leaf(&["p0", "p1"]), scan_with_file_groups(2), ]), - false, + true, ), ( "a merging branch", @@ -1516,9 +1415,14 @@ mod tests { .collect(); assert_eq!(actual, expected); - // The resolver that restricts the tasks can split the node of your own. + // The resolver that restricts the tasks can split both nodes of your own. let with_own = union(vec![own(), scan_with_file_groups(2)]); assert!(!has_unsplittable_union(&with_own, Some(resolver.as_ref()))); + let with_leaf = union(vec![ + PayloadExec::leaf(&["p0", "p1"]), + scan_with_file_groups(2), + ]); + assert!(!has_unsplittable_union(&with_leaf, Some(resolver.as_ref()))); } /// `plan` is a leaf with `like`'s schema and no partitions, as the diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs index b4940c8e7d..0cba8d5b05 100644 --- a/ballista/scheduler/src/state/task_manager.rs +++ b/ballista/scheduler/src/state/task_manager.rs @@ -173,7 +173,7 @@ pub struct JobInfoCache { pub status: Option, /// The resolver the job's tasks are restricted with, so binding can tell /// which stages a task may run only part of. - pub partition_sliceable_resolver: Option>, + pub(crate) partition_sliceable_resolver: Option>, } impl JobInfoCache { @@ -189,7 +189,7 @@ impl JobInfoCache { } /// Sets the resolver the job's tasks are restricted with. - pub fn with_partition_sliceable_resolver( + pub(crate) fn with_partition_sliceable_resolver( mut self, resolver: Option>, ) -> Self { diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index a485eee56e..cfb9cb8993 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -239,7 +239,8 @@ query returned wrong rows, over any source including Parquet. The scheduler now runs such a stage as one task, as it does a stage that collapses its input, so each branch runs once. The same goes for a branch that -reads a broadcast or holds a node that sets its own partition count. The task +reads a broadcast, holds a node that sets its own partition count, or reads a +leaf of several partitions that nothing can restrict. The task ignores `ballista.scheduler.max_partitions_per_task` and takes one vcore for each of the stage's partitions, up to what its executor has free, so the stage runs with less parallelism than before. The static planner gives such a branch diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 82e2b64c14..7867ecb157 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -145,18 +145,6 @@ and executors must all set the same codecs in the same order. - **One task per scan.** Iceberg scans do not split into partitions yet, so one executor reads the whole table. Operators after the first shuffle still run in parallel. A scan reporting more than one partition fails to encode. -- **A `UNION` or `UNION ALL` can fail or return wrong rows when the adaptive - planner splits its stage between tasks.** That happens with - `ballista.scheduler.max_partitions_per_task` set, or when the executor has - fewer free task slots than the union has partitions. A union of Iceberg scans - fails with an error that suggests `SchedulerConfig::partition_sliceable_resolver`, - which standalone mode can't set; - [#2852](https://github.com/apache/datafusion-ballista/pull/2852) fixes it. A union - whose branches each merge their input into one partition, as with an - `ORDER BY` over the union, a `LIMIT` in a branch or aggregates without - `GROUP BY`, returns some rows more than once, whatever the table format - ([#2854](https://github.com/apache/datafusion-ballista/issues/2854)). Setting - `ballista.planner.adaptive.enabled` to `false` avoids both. - **No encrypted tables.** A table with `encryption.key-id` set fails to load on the scheduler and executors, which have no key management client. - **Storage credentials travel in plain text.** Each table's serialized From e6485999d95e36c60b5a6860ef6b90b74b09252f Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 16 Nov 2026 21:14:34 -0500 Subject: [PATCH 13/17] Tidy the split-union tests - Share one sort fixture, `sorted_into_one_partition`, between the restriction and binding tests. - Shorten the unsplittable-union cases and move the resolver cases into a test of their own. - Drop planner predicate tests the stage-planning tests already cover, and reuse `project_leading_columns`. - Name the backstop test for what it guards, and compare the empty leaf by its full display. - Say why each group of end-to-end queries is there. --- ballista/client/tests/split_union.rs | 17 ++- ballista/scheduler/src/cluster/mod.rs | 18 +-- ballista/scheduler/src/planner.rs | 56 ++----- ballista/scheduler/src/state/mod.rs | 6 +- ballista/scheduler/src/state/task_builder.rs | 146 +++++++------------ ballista/scheduler/src/test_utils.rs | 16 ++ 6 files changed, 104 insertions(+), 155 deletions(-) diff --git a/ballista/client/tests/split_union.rs b/ballista/client/tests/split_union.rs index 732e543023..6d16d4788c 100644 --- a/ballista/client/tests/split_union.rs +++ b/ballista/client/tests/split_union.rs @@ -36,10 +36,13 @@ mod split_union { use rstest::rstest; use std::path::Path; - /// The adaptive planner replaces a stage that gives no rows with an - /// `EmptyExec` of that stage's partition count, so in most of these a - /// union branch is an `EmptyExec` of several partitions. + /// Unions whose branches each give a task a different share: none, part + /// or all of them. const QUERIES: &[&str] = &[ + // Branches that give no rows. The adaptive planner replaces the stage + // of such a branch with an `EmptyExec` of that stage's partition + // count, which each task cuts down to its share. + // // An aggregate that matches nothing, beside one that matches all. "SELECT id % 2 AS k, count(*) AS c FROM test WHERE id > 100 GROUP BY id % 2 \ UNION ALL SELECT id % 2, count(*) FROM test GROUP BY id % 2", @@ -77,15 +80,17 @@ mod split_union { "SELECT 100 AS id UNION ALL SELECT id FROM test", "SELECT column1 AS id FROM (VALUES (10), (11)) UNION ALL SELECT id FROM test", // Branches that merge their input into one partition, which the - // adaptive planner leaves in the union's stage: a split stage runs as - // one task, so each branch runs once. + // adaptive planner leaves in the union's stage. Every task would run + // such a branch, so the stage runs as one task. "SELECT id FROM test UNION ALL SELECT id FROM test ORDER BY id", "SELECT o_id FROM orders UNION ALL SELECT c_id FROM customers ORDER BY o_id", "(SELECT id FROM test ORDER BY id LIMIT 3) UNION ALL SELECT id FROM test", "(SELECT o_id FROM orders ORDER BY o_id LIMIT 3) \ UNION ALL SELECT o_id FROM orders", "SELECT count(*) AS c FROM orders UNION ALL SELECT count(*) FROM customers", - // A filter over a one-row source, under a round-robin repartition. + // A filter over a one-row source, below a round-robin repartition + // that keeps its four partitions in every task: the stage runs as one + // task too. "SELECT column1 AS id FROM (VALUES (10), (11)) WHERE column1 > 10 \ UNION ALL SELECT id FROM test", ]; diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs index 20124d311d..fdb5298d26 100644 --- a/ballista/scheduler/src/cluster/mod.rs +++ b/ballista/scheduler/src/cluster/mod.rs @@ -882,19 +882,16 @@ mod test { use crate::state::execution_graph::{ExecutionGraph, StaticExecutionGraph}; use crate::state::task_manager::JobInfoCache; use crate::test_utils::payload::{PayloadExec, payload_resolver}; - use crate::test_utils::scan_with_file_groups; use crate::test_utils::{ mock_completed_task, revive_graph_and_complete_next_stage, test_aggregation_plan_with_config, }; + use crate::test_utils::{scan_with_file_groups, sorted_into_one_partition}; use ballista_core::config::BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK; use ballista_core::execution_plans::ShuffleWriterExec; use ballista_core::execution_plans::plan_algebra::PartitionSliceableResolver; use ballista_core::extension::SessionConfigExt; - use datafusion::physical_expr::expressions::Column; - use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; use datafusion::physical_plan::ExecutionPlan; - use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::union::UnionExec; use datafusion::prelude::SessionConfig; @@ -1147,15 +1144,12 @@ mod test { /// each partition the executor has free. #[test] fn a_stage_with_an_unsplittable_union_is_bound_as_one_task() { - let sorted: Arc = Arc::new(SortExec::new( - LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new(Column::new( - "a", 0, - )))]) - .unwrap(), - scan_with_file_groups(2), - )); // Partition 0 is the sort's, 1-2 the scan's. - let union = UnionExec::try_new(vec![sorted, scan_with_file_groups(2)]).unwrap(); + let union = UnionExec::try_new(vec![ + sorted_into_one_partition(scan_with_file_groups(2)), + scan_with_file_groups(2), + ]) + .unwrap(); assert_eq!( bind_all(&mut running_stage(union.clone()), 8, None), diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 6aa2f3be69..23869a7218 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -987,7 +987,7 @@ mod test { use datafusion::physical_plan::filter::FilterExec; use datafusion::physical_plan::joins::HashJoinExec; - use datafusion::physical_plan::projection::{ProjectionExec, ProjectionExpr}; + use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion::physical_plan::union::UnionExec; @@ -1007,34 +1007,13 @@ mod test { assert!(can_stay_inline(&scan_with_file_groups(4))); } - /// A branch over a one-partition leaf the scheduler doesn't know stays - /// inline: the task that owns the partition runs the leaf whole, and every - /// other task gets an empty leaf. + /// A branch over an unknown leaf of several partitions gets a stage of its + /// own even when the leaf sits below another node, since restricting the + /// node restricts the leaf too. #[test] - fn one_partition_unknown_leaf_union_branch_stays_inline() { - assert!(can_stay_inline(&PayloadExec::leaf(&["p0"]))); - } - - /// A task can own only some partitions of a branch over a leaf of several - /// partitions, and nothing can cut an unknown leaf down to those without - /// a resolver, so the branch gets a stage of its own, as does one with - /// such a leaf deeper down. - #[test] - fn multi_partition_unknown_leaf_union_branch_needs_its_own_stage() { - let leaf = PayloadExec::leaf(&["p0", "p1"]); - assert!(!can_stay_inline(&leaf)); - - let projected: Arc = Arc::new( - ProjectionExec::try_new( - vec![ProjectionExpr { - expr: Arc::new(Column::new("a", 0)), - alias: "a".to_string(), - }], - leaf, - ) - .unwrap(), - ); - assert!(!can_stay_inline(&projected)); + fn multi_partition_unknown_leaf_below_a_node_needs_its_own_stage() { + let branch = project_leading_columns(PayloadExec::leaf(&["p0", "p1"]), 1); + assert!(!can_stay_inline(&branch)); } /// A branch over a shuffle, such as the final aggregate of a union of @@ -1042,23 +1021,12 @@ mod test { /// though the planner sees it before it is resolved. #[test] fn branch_over_an_unresolved_shuffle_stays_inline() { - let schema = scan_with_file_groups(1).schema(); let reader: Arc = Arc::new(UnresolvedShuffleExec::new( 1, - schema, + scan_with_file_groups(1).schema(), Partitioning::UnknownPartitioning(4), )); - let projected: Arc = Arc::new( - ProjectionExec::try_new( - vec![ProjectionExpr { - expr: Arc::new(Column::new("a", 0)), - alias: "a".to_string(), - }], - reader, - ) - .unwrap(), - ); - assert!(can_stay_inline(&projected)); + assert!(can_stay_inline(&project_leading_columns(reader, 1))); } /// Plans `plan` into query stages with the default options. @@ -1073,9 +1041,9 @@ mod test { } /// A union branch over an unknown leaf of several partitions gets a stage - /// of its own, as on main, rather than leave a task of the union's stage - /// owning part of it. The union's stage reads it through a shuffle, which - /// a task can cut to any share. + /// of its own rather than leave a task of the union's stage owning part of + /// it. The union's stage reads it through a shuffle, which a task can cut + /// to any share. #[test] fn union_over_a_multi_partition_unknown_leaf_gives_it_a_stage_of_its_own() -> Result<(), BallistaError> { diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs index 3c52560096..a92ec6ff2a 100644 --- a/ballista/scheduler/src/state/mod.rs +++ b/ballista/scheduler/src/state/mod.rs @@ -971,9 +971,9 @@ mod tests { } } - /// Submits a job of `plan`, a stage of four partitions over `PayloadExec`, - /// and binds it to two executors of two vcores each, which splits its - /// stage into two tasks. + /// Submits a job of `plan`, one stage of four partitions, and binds it to + /// two executors of two vcores each, which splits the stage into two + /// tasks. Its physical codec carries `PayloadExec`. async fn bind_split_payload_stage( config: SchedulerConfig, launcher: Arc, diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index cc811a85a2..c1d0396ab3 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -612,6 +612,7 @@ fn narrow_range(range: &RangePartitioning, kept: &[usize]) -> Option) -> Arc { - use datafusion::physical_plan::sorts::sort::SortExec; - let ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new( - Column::new("a", 0), - ))]) - .unwrap(); - Arc::new(SortExec::new(ordering, input)) - } - - /// Which stage plans a task may run only part of: for each case, whether - /// it has a union that must run as one task. + /// A stage can be split between tasks unless one of its unions has a + /// branch that restriction can't cut to a task's share of it. #[test] - fn unsplittable_unions_are_those_with_a_branch_that_keeps_partitions() { + fn unions_that_cannot_be_split_between_tasks() { use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; let union = |branches: Vec>| -> Arc { UnionExec::try_new(branches).unwrap() @@ -1334,74 +1319,48 @@ mod tests { ) .unwrap(), ); - // Restricted by the resolver only; left alone it keeps two partitions. - let own = - || PayloadExec::over(&["p0", "p1"], vec![scan_with_file_groups(2)], &[]); - let resolver = payload_resolver(); + let sorted = || sorted_into_one_partition(scan_with_file_groups(2)); + let scan = || scan_with_file_groups(2); let cases: Vec<(&str, Arc, bool)> = vec![ - ("no union", scan_with_file_groups(2), false), - ( - "scans", - union(vec![scan_with_file_groups(2), scan_with_file_groups(2)]), - false, - ), + ("no union", scan(), false), + ("scans", union(vec![scan(), scan()]), false), ( "a one-partition unknown leaf", - union(vec![PayloadExec::leaf(&["p0"]), scan_with_file_groups(2)]), + union(vec![PayloadExec::leaf(&["p0"]), scan()]), false, ), ( "an unknown leaf of two partitions", - union(vec![ - PayloadExec::leaf(&["p0", "p1"]), - scan_with_file_groups(2), - ]), + union(vec![PayloadExec::leaf(&["p0", "p1"]), scan()]), true, ), + ("a sort", union(vec![sorted(), scan()]), true), + ("a broadcast", union(vec![broadcast, scan()]), true), ( - "a merging branch", + "an unknown node that keeps its partitions", union(vec![ - merged(scan_with_file_groups(2)), - scan_with_file_groups(2), + PayloadExec::over(&["p0", "p1"], vec![scan()], &[]), + scan(), ]), true, ), ( - "a broadcast", - union(vec![broadcast, scan_with_file_groups(2)]), - true, - ), - ( - "a node of your own", - union(vec![own(), scan_with_file_groups(2)]), - true, - ), - ( - "a merging branch of a nested union", + "a sort in a nested union", union(vec![ - union(vec![ - merged(scan_with_file_groups(2)), - scan_with_file_groups(1), - ]), - scan_with_file_groups(2), + union(vec![sorted(), scan_with_file_groups(1)]), + scan(), ]), true, ), ( "a union below a sort", - merged(union(vec![ - merged(scan_with_file_groups(2)), - scan_with_file_groups(2), - ])), + sorted_into_one_partition(union(vec![sorted(), scan()])), false, ), ( - "a union below a collapse", - Arc::new(CoalescePartitionsExec::new(union(vec![ - merged(scan_with_file_groups(2)), - scan_with_file_groups(2), - ]))), + "a union below a coalesce", + Arc::new(CoalescePartitionsExec::new(union(vec![sorted(), scan()]))), false, ), ]; @@ -1414,32 +1373,39 @@ mod tests { .map(|(name, _, unsplittable)| (*name, *unsplittable)) .collect(); assert_eq!(actual, expected); + } - // The resolver that restricts the tasks can split both nodes of your own. - let with_own = union(vec![own(), scan_with_file_groups(2)]); - assert!(!has_unsplittable_union(&with_own, Some(resolver.as_ref()))); - let with_leaf = union(vec![ - PayloadExec::leaf(&["p0", "p1"]), - scan_with_file_groups(2), - ]); - assert!(!has_unsplittable_union(&with_leaf, Some(resolver.as_ref()))); + /// The resolver the tasks are restricted with can split the unions over + /// nodes of your own that nothing else can. + #[test] + fn the_resolver_lets_a_union_over_nodes_of_your_own_be_split() { + let resolver = payload_resolver(); + let scan = || scan_with_file_groups(2); + let over_leaf = + UnionExec::try_new(vec![PayloadExec::leaf(&["p0", "p1"]), scan()]).unwrap(); + let over_node = UnionExec::try_new(vec![ + PayloadExec::over(&["p0", "p1"], vec![scan()], &[]), + scan(), + ]) + .unwrap(); + + assert_eq!( + [&over_leaf, &over_node] + .map(|plan| has_unsplittable_union(plan, Some(resolver.as_ref()))), + [false, false] + ); } - /// `plan` is a leaf with `like`'s schema and no partitions, as the - /// executor will decode it. + /// `plan` is an empty in-memory source with `like`'s schema and no + /// partitions, which the executor decodes as it is. fn assert_no_partitions( plan: &Arc, like: &Arc, ) { - let memory = plan - .downcast_ref::() - .and_then(|exec| exec.data_source().downcast_ref::()); - assert!( - memory.is_some(), - "an empty in-memory source, not {}", - plan.name() + assert_eq!( + displayable(plan.as_ref()).one_line().to_string().trim_end(), + "DataSourceExec: partitions=0, partition_sizes=[]" ); - assert_eq!(partition_count(plan), 0); assert_eq!(plan.schema(), like.schema()); } @@ -1481,7 +1447,7 @@ mod tests { /// each of its partitions. The `AdaptivePlanner` replaces a stage that /// turns out empty with an `EmptyExec` of that stage's partition count. #[test] - fn self_generating_leaves_keep_one_partition_for_each_the_task_runs() { + fn self_generating_leaves_keep_one_partition_for_each_partition_the_task_runs() { let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); let empty: Arc = Arc::new(EmptyExec::new(schema.clone()).with_partitions(4)); diff --git a/ballista/scheduler/src/test_utils.rs b/ballista/scheduler/src/test_utils.rs index 258bab7819..8a96f54565 100644 --- a/ballista/scheduler/src/test_utils.rs +++ b/ballista/scheduler/src/test_utils.rs @@ -1477,6 +1477,22 @@ pub fn scan_with_file_groups(n: usize) -> Arc { DataSourceExec::from_data_source(builder.build()) } +/// A `SortExec` on column `a` of `input` that merges it into one partition, as +/// the adaptive planner leaves on each branch of a `UNION ALL ... ORDER BY`. +pub fn sorted_into_one_partition( + input: Arc, +) -> Arc { + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; + use datafusion::physical_plan::sorts::sort::SortExec; + + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new( + Column::new("a", 0), + ))]) + .unwrap(); + Arc::new(SortExec::new(ordering, input)) +} + /// An operator that passes its input through unchanged: its properties are the /// input's, its only child is the input, and it executes by executing the /// input. It declares nothing about what it needs from the input, like an From efb3b25f41d9abc7c05ef3f52980836cfd416e43 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 16 Nov 2026 22:27:44 -0500 Subject: [PATCH 14/17] Log a union stage that runs as one task, naming the branch The split check now returns the first branch that can't be split rather than a bool, so binding can log it at info level with a pointer to the resolver, which a node of your own needs to let the stage be split. The shape test asserts which branch each case finds. --- ballista/scheduler/src/cluster/mod.rs | 27 ++++++-- ballista/scheduler/src/state/task_builder.rs | 66 +++++++++++-------- .../source/user-guide/extending-components.md | 2 +- 3 files changed, 60 insertions(+), 35 deletions(-) diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs index fdb5298d26..10142cfd41 100644 --- a/ballista/scheduler/src/cluster/mod.rs +++ b/ballista/scheduler/src/cluster/mod.rs @@ -27,7 +27,7 @@ use crate::state::execution_graph::{ ExecutionGraphBox, TaskDescription, create_task_info, }; use crate::state::session_manager::share_file_statistics_cache; -use crate::state::task_builder::has_unsplittable_union; +use crate::state::task_builder::unsplittable_union_branch; use crate::state::task_manager::JobInfoCache; use ballista_core::config::BallistaConfig; use ballista_core::error::Result; @@ -47,7 +47,7 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::{SessionConfig, SessionContext}; use futures::Stream; -use log::debug; +use log::{debug, info}; use std::collections::{HashMap, HashSet}; use std::pin::Pin; use std::sync::Arc; @@ -588,7 +588,7 @@ pub(crate) fn stage_has_input_collapse(plan_root: &Arc) -> bo /// requires packing the entire pending queue into one bind (a split /// collapse would produce partial results downstream can't merge). /// -/// - **Unsplittable union stage** (see [`has_unsplittable_union`]): a +/// - **Unsplittable union stage** (see [`unsplittable_union_branch`]): a /// `UnionExec` branch keeps partitions however it is restricted, so every /// task given part of the stage would also run that branch, and the /// rewriter refuses such a task. The stage is packed into one task too, @@ -621,9 +621,24 @@ fn bind_one( let split_max = (budget.vcores as usize).min(cap); // The union check rewrites the plan, so it runs only when it can change // the outcome: when the pending partitions don't fit in one slice anyway. - let is_unsplittable = !is_collapse - && running_stage.pending.remaining() > split_max - && has_unsplittable_union(&running_stage.plan, resolver); + let unsplittable_branch = + if !is_collapse && running_stage.pending.remaining() > split_max { + unsplittable_union_branch(&running_stage.plan, resolver) + } else { + None + }; + let is_unsplittable = unsplittable_branch.is_some(); + if let Some(branch) = &unsplittable_branch { + info!( + "Job {job_id} stage {} runs as one task: a `UnionExec` branch, `{}` at \ + its top, can't be restricted to part of its partitions, so every task \ + would run it. If the branch holds a node of your own, \ + `SchedulerConfig::partition_sliceable_resolver` can restrict it and let \ + the stage be split.", + running_stage.stage_id, + branch.name(), + ); + } let max_partitions = if is_collapse || is_unsplittable { usize::MAX } else { diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index c1d0396ab3..46082c123d 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -111,34 +111,35 @@ pub(crate) fn stages_read_whole(plan: &Arc) -> Vec { found } -/// Whether a task given only some of the stage plan `plan`'s partitions would -/// run a `UnionExec` branch wrongly: a branch that fails [`branch_splits`]. -/// The scheduler runs such a stage as one task instead, which runs every -/// branch whole. +/// The first `UnionExec` branch in the stage plan `plan` that a task given +/// only some of the stage's partitions would run wrongly: a branch that fails +/// [`branch_splits`]. The scheduler runs such a stage as one task instead, +/// which runs every branch whole. /// /// The adaptive planner can leave such a branch in the union's stage. The /// static planner gives it a stage of its own, so its stages never need this. -pub(crate) fn has_unsplittable_union( +pub(crate) fn unsplittable_union_branch( plan: &Arc, resolver: Option<&dyn PartitionSliceableResolver>, -) -> bool { +) -> Option> { fn walk( plan: &Arc, under_collect: bool, resolver: Option<&dyn PartitionSliceableResolver>, - ) -> bool { + ) -> Option> { // Below a collapse every task reads the whole input anyway, so a // union there is never split. if !under_collect && plan.is::() { return plan .children() .into_iter() - .any(|branch| !branch_splits(branch, resolver)); + .find(|branch| !branch_splits(branch, resolver)) + .cloned(); } plan.children() .into_iter() .zip(child_scopes(plan, under_collect)) - .any(|(child, collect)| walk(child, collect, resolver)) + .find_map(|(child, collect)| walk(child, collect, resolver)) } walk(plan, false, resolver) } @@ -318,7 +319,7 @@ fn ensure_leaf_runs_whole( /// task too, returning rows another task returns as well. /// /// Binding runs a stage with such a branch as one task (see -/// [`has_unsplittable_union`]), so this only fails if binding and restriction +/// [`unsplittable_union_branch`]), so this only fails if binding and restriction /// disagree: it refuses the task rather than let it return rows twice. fn ensure_branch_restricted( branch: &Arc, @@ -1304,6 +1305,7 @@ mod tests { /// A stage can be split between tasks unless one of its unions has a /// branch that restriction can't cut to a task's share of it. + /// Each case names the top node of the first such branch. #[test] fn unions_that_cannot_be_split_between_tasks() { use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; @@ -1322,28 +1324,32 @@ mod tests { let sorted = || sorted_into_one_partition(scan_with_file_groups(2)); let scan = || scan_with_file_groups(2); - let cases: Vec<(&str, Arc, bool)> = vec![ - ("no union", scan(), false), - ("scans", union(vec![scan(), scan()]), false), + let cases: Vec<(&str, Arc, Option<&str>)> = vec![ + ("no union", scan(), None), + ("scans", union(vec![scan(), scan()]), None), ( "a one-partition unknown leaf", union(vec![PayloadExec::leaf(&["p0"]), scan()]), - false, + None, ), ( "an unknown leaf of two partitions", union(vec![PayloadExec::leaf(&["p0", "p1"]), scan()]), - true, + Some("PayloadExec"), + ), + ("a sort", union(vec![sorted(), scan()]), Some("SortExec")), + ( + "a broadcast", + union(vec![broadcast, scan()]), + Some("ShuffleReaderExec"), ), - ("a sort", union(vec![sorted(), scan()]), true), - ("a broadcast", union(vec![broadcast, scan()]), true), ( "an unknown node that keeps its partitions", union(vec![ PayloadExec::over(&["p0", "p1"], vec![scan()], &[]), scan(), ]), - true, + Some("PayloadExec"), ), ( "a sort in a nested union", @@ -1351,26 +1357,29 @@ mod tests { union(vec![sorted(), scan_with_file_groups(1)]), scan(), ]), - true, + Some("UnionExec"), ), ( "a union below a sort", sorted_into_one_partition(union(vec![sorted(), scan()])), - false, + None, ), ( "a union below a coalesce", Arc::new(CoalescePartitionsExec::new(union(vec![sorted(), scan()]))), - false, + None, ), ]; - let actual: Vec<(&str, bool)> = cases + let actual: Vec<(&str, Option)> = cases .iter() - .map(|(name, plan, _)| (*name, has_unsplittable_union(plan, None))) + .map(|(name, plan, _)| { + let branch = unsplittable_union_branch(plan, None); + (*name, branch.map(|branch| branch.name().to_string())) + }) .collect(); - let expected: Vec<(&str, bool)> = cases + let expected: Vec<(&str, Option)> = cases .iter() - .map(|(name, _, unsplittable)| (*name, *unsplittable)) + .map(|(name, _, branch)| (*name, branch.map(str::to_string))) .collect(); assert_eq!(actual, expected); } @@ -1390,9 +1399,10 @@ mod tests { .unwrap(); assert_eq!( - [&over_leaf, &over_node] - .map(|plan| has_unsplittable_union(plan, Some(resolver.as_ref()))), - [false, false] + [&over_leaf, &over_node].map(|plan| { + unsplittable_union_branch(plan, Some(resolver.as_ref())).is_none() + }), + [true, true] ); } diff --git a/docs/source/user-guide/extending-components.md b/docs/source/user-guide/extending-components.md index 5ecad770ce..b32acb6e3c 100644 --- a/docs/source/user-guide/extending-components.md +++ b/docs/source/user-guide/extending-components.md @@ -179,7 +179,7 @@ let config = SchedulerConfig::default().with_partition_sliceable_resolver(Arc::n - `children` are the node's inputs, none for a scan, restricted to `partitions`. An input the node declares `SinglePartition` arrives whole, as a join's build side does. Partition `i` of the node must read partition `i` of every other input. - The restricted node must have exactly `partitions.len()` partitions. The scheduler rejects any other count. -- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. A node with inputs is still asked, with its inputs already restricted to none. If nothing restricts such a node and it keeps its own partition count, a task given part of the union's stage would also run partitions it doesn't own, so the scheduler runs that stage as one task. +- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. A node with inputs is still asked, with its inputs already restricted to none. If nothing restricts such a node and it keeps its own partition count, a task given part of the union's stage would also run partitions it doesn't own, so the scheduler runs that stage as one task and logs it at `info` level, naming the branch. - The restricted node is serialized with the scheduler's `physical codec`, so what it carries per partition has to survive encoding and decoding. - The resolver does not need `PartitionSliceable`. It can build the restricted node itself, which is the way to restrict a node you cannot implement the trait on, such as a `DataSourceExec` over a `DataSource` of your own: match it with `downcast_ref`, then build a `DataSourceExec` over the restricted source. - The resolver is asked after Ballista's own list. A node that is read whole, such as one below a `CoalescePartitionsExec` or on a join's build side, needs none of this. From a819effa57897ec16514162fb036b7ee1eff2c3c Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 16 Nov 2026 23:05:00 -0500 Subject: [PATCH 15/17] Keep main's stage for a union branch over a shuffle The planner checked a union branch with a resolver that treated an unresolved shuffle as the reader it becomes, which kept a node over a shuffle in the union's stage and changed 7 TPC-DS static plans. Neither fix needs that, so the planner now asks without a resolver, as andygrove suggested, and every static plan matches main again. The cache test goes back to main's stage numbers. --- ballista/scheduler/src/planner.rs | 40 ++++--------------- .../scheduler/src/state/execution_graph.rs | 8 ++-- 2 files changed, 13 insertions(+), 35 deletions(-) diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 23869a7218..c4d7b7e589 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -25,7 +25,6 @@ use ballista_core::JobId; use ballista_core::config::BallistaConfig; use ballista_core::error::{BallistaError, Result}; use ballista_core::execution_plans::ShuffleWriter; -use ballista_core::execution_plans::plan_algebra::PartitionSliceableResolver; use ballista_core::execution_plans::sort_shuffle::{ShuffleRouting, SortShuffleConfig}; use ballista_core::{ execution_plans::{ @@ -723,36 +722,13 @@ impl DefaultDistributedPlanner { /// /// The rewriter is asked without the scheduler's resolver, so a branch over a /// leaf of several partitions that only a resolver can restrict, such as a -/// scan over a source of your own, gets a stage of its own. An -/// `UnresolvedShuffleExec` passes through [`UnresolvedShuffles`], since it -/// becomes a shuffle reader before any task runs. +/// scan over a source of your own, gets a stage of its own. So does a node +/// over a shuffle, which the planner sees before it is resolved. fn can_stay_inline(plan: &Arc) -> bool { if plan.is::() { return true; } - !holds_a_broadcast(plan) && branch_splits(plan, Some(&UnresolvedShuffles)) -} - -/// Restricts an `UnresolvedShuffleExec` to one partition for each index, as -/// the rewriter will the shuffle reader it becomes. The stage planner checks -/// with it whether a branch can be restricted before its inputs are resolved. -#[derive(Debug)] -struct UnresolvedShuffles; - -impl PartitionSliceableResolver for UnresolvedShuffles { - fn restrict_to_partitions( - &self, - plan: &Arc, - _children: Vec>, - partitions: &[usize], - ) -> Option>> { - let reader = plan.downcast_ref::()?; - Some(Ok(Arc::new(UnresolvedShuffleExec::new( - reader.stage_id, - reader.schema.clone(), - Partitioning::UnknownPartitioning(partitions.len()), - )))) - } + !holds_a_broadcast(plan) && branch_splits(plan, None) } /// Whether `plan` contains a broadcast input or a `CollectLeft` join. @@ -1016,17 +992,17 @@ mod test { assert!(!can_stay_inline(&branch)); } - /// A branch over a shuffle, such as the final aggregate of a union of - /// aggregates, stays inline: the shuffle's reader restricts to any share, - /// though the planner sees it before it is resolved. + /// A node over a shuffle, such as the final aggregate of a union of + /// aggregates, gets a stage of its own: the planner sees the shuffle before + /// it is resolved, when nothing can restrict it to part of its partitions. #[test] - fn branch_over_an_unresolved_shuffle_stays_inline() { + fn node_over_an_unresolved_shuffle_needs_its_own_stage() { let reader: Arc = Arc::new(UnresolvedShuffleExec::new( 1, scan_with_file_groups(1).schema(), Partitioning::UnknownPartitioning(4), )); - assert!(can_stay_inline(&project_leading_columns(reader, 1))); + assert!(!can_stay_inline(&project_leading_columns(reader, 1))); } /// Plans `plan` into query stages with the default options. diff --git a/ballista/scheduler/src/state/execution_graph.rs b/ballista/scheduler/src/state/execution_graph.rs index ae6732a49f..07a71af51f 100644 --- a/ballista/scheduler/src/state/execution_graph.rs +++ b/ballista/scheduler/src/state/execution_graph.rs @@ -3286,7 +3286,7 @@ mod test { #[tokio::test] async fn losing_own_shuffle_output_restores_the_cache_reader() -> Result<()> { /// The own input's stage id, which the cache's build stage shares. - const OWN_INPUT: usize = 1; + const OWN_INPUT: usize = 2; fn cache_reader_of(plan: &Arc) -> Option<&ShuffleReaderExec> { plan.downcast_ref::() .filter(|reader| is_external_stage_id(reader.stage_id)) @@ -3322,8 +3322,9 @@ mod test { .expect("a stage reads the cache and its own input"); assert_eq!(own_input, OWN_INPUT); - // The job's own input completes on executor 1, then a task of the - // stage reading both its output and the cache starts on executor 2. + // The job's own stages complete on executor 1, then a task of the stage + // reading both their output and the cache starts on executor 2. + revive_graph_and_complete_next_stage(&mut graph)?; revive_graph_and_complete_next_stage(&mut graph)?; graph.revive(); let executor2 = mock_executor("executor-id2".to_string()); @@ -3343,6 +3344,7 @@ mod test { // The own input re-runs, one task on the cache's executor. The late // task then fails fetching the cache, under the own input's stage id: // the own task's output there is kept, not re-run. + revive_graph_and_complete_next_stage(&mut graph)?; graph.revive(); let cache_executor = mock_executor("executor-cache".to_string()); let own_task = graph.pop_next_task(&cache_executor.id)?.unwrap(); From 695e29b4c3dcfb1d0e0a0c706601ec82cc44b7a3 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 16 Nov 2026 23:39:54 -0500 Subject: [PATCH 16/17] Say which planner runs a union stage as one task, and pin the one-partition shuffle - The docs said a union stage with a branch nothing can restrict runs as one task; that is the adaptive planner's behaviour, while the static planner gives the branch a stage of its own. - The resolver guide still said a split stage holding an unknown leaf fails; a task that runs all or none of it is fine. - A node over a one-partition shuffle stays in the union's stage, like a one-partition custom leaf; the planner test now says so. --- ballista/scheduler/src/cluster/mod.rs | 4 +-- ballista/scheduler/src/planner.rs | 26 +++++++++++++------ docs/source/upgrading/56.0.0.md | 13 +++++----- .../source/user-guide/extending-components.md | 4 +-- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs index 10142cfd41..c0982ecd97 100644 --- a/ballista/scheduler/src/cluster/mod.rs +++ b/ballista/scheduler/src/cluster/mod.rs @@ -589,8 +589,8 @@ pub(crate) fn stage_has_input_collapse(plan_root: &Arc) -> bo /// collapse would produce partial results downstream can't merge). /// /// - **Unsplittable union stage** (see [`unsplittable_union_branch`]): a -/// `UnionExec` branch keeps partitions however it is restricted, so every -/// task given part of the stage would also run that branch, and the +/// `UnionExec` branch can't be cut to a task's share of it, so a task given +/// part of the stage would run partitions of it that it doesn't own, and the /// rewriter refuses such a task. The stage is packed into one task too, /// which runs each branch once. Its root keeps all of its partitions, so the /// task consumes one vcore for each, up to what the executor has free. diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index c4d7b7e589..e2816019a5 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -723,7 +723,9 @@ impl DefaultDistributedPlanner { /// The rewriter is asked without the scheduler's resolver, so a branch over a /// leaf of several partitions that only a resolver can restrict, such as a /// scan over a source of your own, gets a stage of its own. So does a node -/// over a shuffle, which the planner sees before it is resolved. +/// over a shuffle of several partitions, which the planner sees before it is +/// resolved. A branch over a one-partition leaf stays inline: only the task +/// that owns its partition runs it, whole. fn can_stay_inline(plan: &Arc) -> bool { if plan.is::() { return true; @@ -995,14 +997,22 @@ mod test { /// A node over a shuffle, such as the final aggregate of a union of /// aggregates, gets a stage of its own: the planner sees the shuffle before /// it is resolved, when nothing can restrict it to part of its partitions. + /// Over a one-partition shuffle it stays inline, since the task that owns + /// the partition reads the shuffle whole. #[test] - fn node_over_an_unresolved_shuffle_needs_its_own_stage() { - let reader: Arc = Arc::new(UnresolvedShuffleExec::new( - 1, - scan_with_file_groups(1).schema(), - Partitioning::UnknownPartitioning(4), - )); - assert!(!can_stay_inline(&project_leading_columns(reader, 1))); + fn node_over_an_unresolved_shuffle_stays_inline_only_over_one_partition() { + let over_shuffle = |partitions| { + let reader: Arc = Arc::new(UnresolvedShuffleExec::new( + 1, + scan_with_file_groups(1).schema(), + Partitioning::UnknownPartitioning(partitions), + )); + project_leading_columns(reader, 1) + }; + assert_eq!( + [4, 1].map(|partitions| can_stay_inline(&over_shuffle(partitions))), + [false, true] + ); } /// Plans `plan` into query stages with the default options. diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index cfb9cb8993..fb897c722d 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -200,7 +200,7 @@ no error. The scheduler now fails the job instead. Such a leaf is accepted only in a task that runs all of its partitions, in order, or none of them, as in a `UnionExec` -branch another task owns, where it becomes an empty node. A retry of part of a +branch another task owns, where it becomes an empty leaf. A retry of part of a stage can fail a job too. Plans made of Ballista's own leaves (shuffle readers, file and in-memory scans, `PlaceholderRowExec`, `EmptyExec`) are unaffected, as are leaves read whole below a `CoalescePartitionsExec` or on a join's build @@ -218,9 +218,9 @@ it was while its children are restricted, because the scheduler cannot tell one that holds data per partition from one that does not, such as a `FilterExec`. An interior node of your own that holds data per partition still returns wrong results in a split stage unless a resolver restricts it. Below a `UnionExec`, -a stage where such a node keeps its own partition count runs as one task -instead, since a task given part of it would also run partitions that another -task owns. +a branch where such a node keeps its own partition count gets a stage of its +own, or under the adaptive planner its stage runs as one task, since a task +given part of it would also run partitions that another task owns. The error names the node and what to implement. Under push-staged scheduling the job's status says only that a task could not be prepared; the reason is in @@ -243,8 +243,9 @@ reads a broadcast, holds a node that sets its own partition count, or reads a leaf of several partitions that nothing can restrict. The task ignores `ballista.scheduler.max_partitions_per_task` and takes one vcore for each of the stage's partitions, up to what its executor has free, so the stage -runs with less parallelism than before. The static planner gives such a branch -a stage of its own and is unaffected. +runs with less parallelism than before. The scheduler logs each such stage at +`info` level, naming the branch. The static planner gives such a branch a stage +of its own and is unaffected. #### Nested-loop joins that emit build rows run as a single task diff --git a/docs/source/user-guide/extending-components.md b/docs/source/user-guide/extending-components.md index b32acb6e3c..401440f0ae 100644 --- a/docs/source/user-guide/extending-components.md +++ b/docs/source/user-guide/extending-components.md @@ -137,7 +137,7 @@ As with the scheduler, to keep the executor's own command line, parse A plan with `ExecutionPlan` nodes of your own can run on a Ballista cluster once the scheduler and the executors have a `physical codec` that serializes them. A node that holds data per partition needs one thing more. -When a stage is split between tasks, each task gets a copy of the plan restricted to its own partitions, and operators see only task-local partition indices: a task that runs partitions 4 and 5 executes partitions 0 and 1 of every node. The scheduler restricts the leaves it knows, such as shuffle readers and file and in-memory scans. A node it doesn't know would read its per-partition data, such as one list of files for each partition, from the wrong index, and the stage would return some partitions twice and others not at all. The scheduler fails a job whose split stage holds such a leaf. It does not check interior nodes: it cannot tell one that holds per-partition data from one that does not, such as a `FilterExec`, so an interior node of yours that holds per-partition data returns wrong results in a split stage unless you register it. Whether a stage is split depends on free task slots and on retries, so test with `ballista.scheduler.max_partitions_per_task` set low rather than relying on a small cluster to show the failure. +When a stage is split between tasks, each task gets a copy of the plan restricted to its own partitions, and operators see only task-local partition indices: a task that runs partitions 4 and 5 executes partitions 0 and 1 of every node. The scheduler restricts the leaves it knows, such as shuffle readers and file and in-memory scans. A node it doesn't know would read its per-partition data, such as one list of files for each partition, from the wrong index, and the stage would return some partitions twice and others not at all. The scheduler fails a job whose split stage holds such a leaf, unless a task runs all of it or none of it. It does not check interior nodes: it cannot tell one that holds per-partition data from one that does not, such as a `FilterExec`, so an interior node of yours that holds per-partition data returns wrong results in a split stage unless you register it. Whether a stage is split depends on free task slots and on retries, so test with `ballista.scheduler.max_partitions_per_task` set low rather than relying on a small cluster to show the failure. To make your node restrictable, give the scheduler a `PartitionSliceableResolver`. It is called with a node, its children already restricted, and the partitions the task runs, and returns the restricted node, or `None` for a node it doesn't know. A node of your own can implement `PartitionSliceable` to keep the slicing beside its fields: @@ -179,7 +179,7 @@ let config = SchedulerConfig::default().with_partition_sliceable_resolver(Arc::n - `children` are the node's inputs, none for a scan, restricted to `partitions`. An input the node declares `SinglePartition` arrives whole, as a join's build side does. Partition `i` of the node must read partition `i` of every other input. - The restricted node must have exactly `partitions.len()` partitions. The scheduler rejects any other count. -- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. A node with inputs is still asked, with its inputs already restricted to none. If nothing restricts such a node and it keeps its own partition count, a task given part of the union's stage would also run partitions it doesn't own, so the scheduler runs that stage as one task and logs it at `info` level, naming the branch. +- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. A node with inputs is still asked, with its inputs already restricted to none. If nothing restricts such a node and it keeps its own partition count, a task given part of the union's stage would also run partitions it doesn't own. The static planner gives such a branch a stage of its own. Under the adaptive planner the scheduler runs the union's stage as one task instead, and logs it at `info` level, naming the branch. - The restricted node is serialized with the scheduler's `physical codec`, so what it carries per partition has to survive encoding and decoding. - The resolver does not need `PartitionSliceable`. It can build the restricted node itself, which is the way to restrict a node you cannot implement the trait on, such as a `DataSourceExec` over a `DataSource` of your own: match it with `downcast_ref`, then build a `DataSourceExec` over the restricted source. - The resolver is asked after Ballista's own list. A node that is read whole, such as one below a `CoalescePartitionsExec` or on a join's build side, needs none of this. From 5ac2f363ece246c630395661d4532455370ecd70 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 17 Nov 2026 11:20:58 -0500 Subject: [PATCH 17/17] Cache on the running stage which union branch can't be split `bind_one` walked the stage plan, restricting every union branch twice, on each bind of a stage that doesn't fit in one slice. The answer depends only on the plan and the job's resolver, both fixed for a running stage, so the first bind that needs it stores it on `RunningStage` and later binds reuse it. A rerun builds a new running stage and so starts over. Leaves a TODO to do the same for `stage_has_input_collapse`, and makes the upgrade guide say which stage runs as one task. --- ballista/scheduler/src/cluster/mod.rs | 11 +++++++++-- ballista/scheduler/src/state/execution_stage.rs | 8 ++++++++ docs/source/upgrading/56.0.0.md | 4 ++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs index c0982ecd97..86077f9949 100644 --- a/ballista/scheduler/src/cluster/mod.rs +++ b/ballista/scheduler/src/cluster/mod.rs @@ -604,6 +604,8 @@ fn bind_one( if running_stage.pending.is_empty() { return None; } + // TODO: cache this on `RunningStage` as `unsplittable_union_branch` is: + // it depends only on the plan, yet walks it again on every bind. let is_collapse = stage_has_input_collapse(&running_stage.plan); // Cap other slices at the configured `max_partitions_per_task`. Collapse // and unsplittable union stages must still pack their full pending queue @@ -620,10 +622,15 @@ fn bind_one( .unwrap_or(usize::MAX); let split_max = (budget.vcores as usize).min(cap); // The union check rewrites the plan, so it runs only when it can change - // the outcome: when the pending partitions don't fit in one slice anyway. + // the outcome (the pending partitions don't fit in one slice anyway), and + // the stage keeps its answer for later binds. let unsplittable_branch = if !is_collapse && running_stage.pending.remaining() > split_max { - unsplittable_union_branch(&running_stage.plan, resolver) + let plan = &running_stage.plan; + running_stage + .unsplittable_union_branch + .get_or_insert_with(|| unsplittable_union_branch(plan, resolver)) + .clone() } else { None }; diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs index a38c7c0f41..10fe012845 100644 --- a/ballista/scheduler/src/state/execution_stage.rs +++ b/ballista/scheduler/src/state/execution_stage.rs @@ -255,6 +255,12 @@ pub struct RunningStage { /// Prefix-merged in global partition order to seed a downstream /// `PrefixMergeExec`; dropped when the stage transitions to Successful. pub window_state_reports: Vec, + /// The `UnionExec` branch that can't be restricted to part of its + /// partitions, if the plan has one, which makes binding run the stage as + /// one task. `None` until the first bind that needs it computes it: it + /// depends only on `plan` and the job's resolver, which are fixed for a + /// running stage. + pub(crate) unsplittable_union_branch: Option>>, } /// If a stage finishes successfully, its task statuses and metrics will be finalized @@ -701,6 +707,7 @@ impl RunningStage { session_config, runtime_stats_reports: Vec::new(), window_state_reports: Vec::new(), + unsplittable_union_branch: None, } } @@ -1331,6 +1338,7 @@ impl SuccessfulStage { // Merged-cut logging fires per-attempt on final success. runtime_stats_reports: Vec::new(), window_state_reports: Vec::new(), + unsplittable_union_branch: None, } } diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index fb897c722d..b3b0c91975 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -219,8 +219,8 @@ that holds data per partition from one that does not, such as a `FilterExec`. An interior node of your own that holds data per partition still returns wrong results in a split stage unless a resolver restricts it. Below a `UnionExec`, a branch where such a node keeps its own partition count gets a stage of its -own, or under the adaptive planner its stage runs as one task, since a task -given part of it would also run partitions that another task owns. +own, or under the adaptive planner the union's stage runs as one task, since a +task given part of the branch would also run partitions that another task owns. The error names the node and what to implement. Under push-staged scheduling the job's status says only that a task could not be prepared; the reason is in