diff --git a/ballista/client/tests/split_union.rs b/ballista/client/tests/split_union.rs new file mode 100644 index 0000000000..6d16d4788c --- /dev/null +++ b/ballista/client/tests/split_union.rs @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; + +/// End-to-end tests of `UNION ALL` queries whose stages are split between +/// tasks. Each task runs only the union branches it owns, so every other +/// branch is restricted to the task's share of it, often none. They must +/// return the rows DataFusion returns. +#[cfg(test)] +#[cfg(feature = "standalone")] +mod split_union { + use crate::common::{ + ballista_state, collect_lines, datafusion_context, orders_customers_dir, + register_alltypes, register_orders_customers, + }; + use ballista::prelude::SessionContextExt; + use ballista_core::config::{ + BALLISTA_ADAPTIVE_PLANNER_ENABLED, BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, + }; + use datafusion::prelude::SessionContext; + use rstest::rstest; + use std::path::Path; + + /// Unions whose branches each give a task a different share: none, part + /// or all of them. + const QUERIES: &[&str] = &[ + // Branches that give no rows. The adaptive planner replaces the stage + // of such a branch with an `EmptyExec` of that stage's partition + // count, which each task cuts down to its share. + // + // An aggregate that matches nothing, beside one that matches all. + "SELECT id % 2 AS k, count(*) AS c FROM test WHERE id > 100 GROUP BY id % 2 \ + UNION ALL SELECT id % 2, count(*) FROM test GROUP BY id % 2", + // The same, the other way round. + "SELECT id % 2 AS k, count(*) AS c FROM test GROUP BY id % 2 \ + UNION ALL SELECT id % 2, count(*) FROM test WHERE id > 100 GROUP BY id % 2", + // An empty middle branch of three. + "SELECT o_cust AS k, count(*) AS c FROM orders GROUP BY o_cust \ + UNION ALL SELECT o_cust, count(*) FROM orders WHERE o_id < 0 GROUP BY o_cust \ + UNION ALL SELECT c_id, 1 FROM customers WHERE c_id < 5", + // An aggregate over a union with an empty branch. + "SELECT k, sum(c) AS s FROM ( \ + SELECT o_cust AS k, count(*) AS c FROM orders WHERE o_id < 0 GROUP BY o_cust \ + UNION ALL SELECT o_cust, count(*) FROM orders GROUP BY o_cust) \ + GROUP BY k", + // A join that matches nothing below an aggregate. + "SELECT c_region AS r, count(*) AS n FROM orders JOIN customers ON o_cust = c_id \ + WHERE o_id < 0 GROUP BY c_region \ + UNION ALL SELECT c_region, count(*) FROM customers GROUP BY c_region", + // A literal beside an empty aggregate. + "SELECT 7 AS k, 7 AS c \ + UNION ALL SELECT id % 2, count(*) FROM test WHERE id > 100 GROUP BY id % 2", + // A union with an empty branch joined to a table. + "SELECT x.k, t.c_name FROM ( \ + SELECT o_cust AS k FROM orders WHERE o_id < 0 GROUP BY o_cust \ + UNION ALL SELECT c_id FROM customers WHERE c_id < 3) x \ + JOIN customers t ON x.k = t.c_id", + // A window over a union with an empty branch. + "SELECT k, sum(c) OVER (PARTITION BY k) AS s FROM ( \ + SELECT id % 2 AS k, count(*) AS c FROM test WHERE id > 100 GROUP BY id % 2 \ + UNION ALL SELECT id % 2, count(*) FROM test GROUP BY id % 2)", + // Scans, literals and values. + "SELECT id FROM test UNION ALL SELECT id FROM test", + "SELECT 1 AS a UNION ALL SELECT 2 UNION ALL SELECT 3", + "SELECT 100 AS id UNION ALL SELECT id FROM test", + "SELECT column1 AS id FROM (VALUES (10), (11)) UNION ALL SELECT id FROM test", + // Branches that merge their input into one partition, which the + // adaptive planner leaves in the union's stage. Every task would run + // such a branch, so the stage runs as one task. + "SELECT id FROM test UNION ALL SELECT id FROM test ORDER BY id", + "SELECT o_id FROM orders UNION ALL SELECT c_id FROM customers ORDER BY o_id", + "(SELECT id FROM test ORDER BY id LIMIT 3) UNION ALL SELECT id FROM test", + "(SELECT o_id FROM orders ORDER BY o_id LIMIT 3) \ + UNION ALL SELECT o_id FROM orders", + "SELECT count(*) AS c FROM orders UNION ALL SELECT count(*) FROM customers", + // A filter over a one-row source, below a round-robin repartition + // that keeps its four partitions in every task: the stage runs as one + // task too. + "SELECT column1 AS id FROM (VALUES (10), (11)) WHERE column1 > 10 \ + UNION ALL SELECT id FROM test", + ]; + + async fn register(ctx: &SessionContext, dir: &Path) { + register_alltypes(ctx, "test").await; + register_orders_customers(ctx, dir).await; + } + + /// The formatted rows of `sql`, sorted, since a union's rows come in no + /// particular order. + async fn sorted_lines(ctx: &SessionContext, sql: &str) -> Vec { + let mut lines = collect_lines(ctx, sql).await; + lines.sort(); + lines + } + + #[rstest] + #[tokio::test] + async fn split_unions_return_the_rows_datafusion_returns( + #[values(false, true)] adaptive: bool, + #[values(1, 2)] max_partitions_per_task: usize, + ) { + let dir = orders_customers_dir(); + let reference = datafusion_context(); + register(&reference, dir.path()).await; + let ctx = SessionContext::standalone_with_state(ballista_state(&[ + (BALLISTA_ADAPTIVE_PLANNER_ENABLED, &adaptive.to_string()), + ( + BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, + &max_partitions_per_task.to_string(), + ), + ])) + .await + .unwrap(); + register(&ctx, dir.path()).await; + + for sql in QUERIES { + assert_eq!( + sorted_lines(&ctx, sql).await, + sorted_lines(&reference, sql).await, + "adaptive={adaptive} max_partitions_per_task={max_partitions_per_task} \ + differs for: {sql}" + ); + } + } +} diff --git a/ballista/scheduler/src/cluster/mod.rs b/ballista/scheduler/src/cluster/mod.rs index c804bfb486..86077f9949 100644 --- a/ballista/scheduler/src/cluster/mod.rs +++ b/ballista/scheduler/src/cluster/mod.rs @@ -27,9 +27,11 @@ use crate::state::execution_graph::{ ExecutionGraphBox, TaskDescription, create_task_info, }; use crate::state::session_manager::share_file_statistics_cache; +use crate::state::task_builder::unsplittable_union_branch; use crate::state::task_manager::JobInfoCache; use ballista_core::config::BallistaConfig; use ballista_core::error::Result; +use ballista_core::execution_plans::plan_algebra::PartitionSliceableResolver; use ballista_core::execution_plans::{ PipelinedShuffleReaderExec, RangeShuffleReaderExec, ShuffleReaderExec, UnresolvedShuffleExec, @@ -45,7 +47,7 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::{SessionConfig, SessionContext}; use futures::Stream; -use log::debug; +use log::{debug, info}; use std::collections::{HashMap, HashSet}; use std::pin::Pin; use std::sync::Arc; @@ -585,17 +587,31 @@ pub(crate) fn stage_has_input_collapse(plan_root: &Arc) -> bo /// tasks run in parallel on the remaining vcores. Correctness still /// requires packing the entire pending queue into one bind (a split /// collapse would produce partial results downstream can't merge). +/// +/// - **Unsplittable union stage** (see [`unsplittable_union_branch`]): a +/// `UnionExec` branch can't be cut to a task's share of it, so a task given +/// part of the stage would run partitions of it that it doesn't own, and the +/// rewriter refuses such a task. The stage is packed into one task too, +/// which runs each branch once. Its root keeps all of its partitions, so the +/// task consumes one vcore for each, up to what the executor has free. fn bind_one( running_stage: &mut crate::state::execution_stage::RunningStage, session_id: &str, job_id: &JobId, budget: &mut AvailableVcores, + resolver: Option<&dyn PartitionSliceableResolver>, ) -> Option { + if running_stage.pending.is_empty() { + return None; + } + // TODO: cache this on `RunningStage` as `unsplittable_union_branch` is: + // it depends only on the plan, yet walks it again on every bind. let is_collapse = stage_has_input_collapse(&running_stage.plan); - // Cap non-collapse slices at the configured `max_partitions_per_task`. - // Collapse stages must still pack their full pending queue into a single - // task for correctness — a split collapse would produce partial results - // downstream can't merge. + // Cap other slices at the configured `max_partitions_per_task`. Collapse + // and unsplittable union stages must still pack their full pending queue + // into a single task for correctness: a split collapse would produce + // partial results downstream can't merge, and a split union would run a + // branch in every task. let cap = running_stage .session_config .options() @@ -604,10 +620,36 @@ fn bind_one( .map(|bc| bc.max_partitions_per_task()) .filter(|&n| n > 0) .unwrap_or(usize::MAX); - let max_partitions = if is_collapse { + let split_max = (budget.vcores as usize).min(cap); + // The union check rewrites the plan, so it runs only when it can change + // the outcome (the pending partitions don't fit in one slice anyway), and + // the stage keeps its answer for later binds. + let unsplittable_branch = + if !is_collapse && running_stage.pending.remaining() > split_max { + let plan = &running_stage.plan; + running_stage + .unsplittable_union_branch + .get_or_insert_with(|| unsplittable_union_branch(plan, resolver)) + .clone() + } else { + None + }; + let is_unsplittable = unsplittable_branch.is_some(); + if let Some(branch) = &unsplittable_branch { + info!( + "Job {job_id} stage {} runs as one task: a `UnionExec` branch, `{}` at \ + its top, can't be restricted to part of its partitions, so every task \ + would run it. If the branch holds a node of your own, \ + `SchedulerConfig::partition_sliceable_resolver` can restrict it and let \ + the stage be split.", + running_stage.stage_id, + branch.name(), + ); + } + let max_partitions = if is_collapse || is_unsplittable { usize::MAX } else { - (budget.vcores as usize).min(cap) + split_max }; let input_partition_ids = running_stage.pending.next_slice(max_partitions); if input_partition_ids.is_empty() { @@ -621,11 +663,13 @@ fn bind_one( // one vcore and leave the rest for other stages' tasks on this executor. let vcores_consumed = if is_collapse { 1 + } else if is_unsplittable { + (input_partition_ids.len() as u32).min(budget.vcores) } else { input_partition_ids.len() as u32 }; debug!( - "bind_one: job={} stage={} exec={} vcores={} max={} slice_len={} consumed={} partitions={:?} collapse={}", + "bind_one: job={} stage={} exec={} vcores={} max={} slice_len={} consumed={} partitions={:?} collapse={} unsplittable_union={}", job_id, running_stage.stage_id, budget.executor_id, @@ -639,6 +683,7 @@ fn bind_one( vcores_consumed, input_partition_ids, is_collapse, + is_unsplittable, ); let executor_id = budget.executor_id.clone(); // task_id is the append-order slot in `task_infos` — since we're @@ -767,7 +812,13 @@ async fn bind_tasks_with( return schedulable_tasks; }; idx = picked; - match bind_one(running_stage, &session_id, job_id, &mut *budgets[idx]) { + match bind_one( + running_stage, + &session_id, + job_id, + &mut *budgets[idx], + job_info.partition_sliceable_resolver.as_deref(), + ) { Some(bound) => { schedulable_tasks.push(bound); idx = strategy.advance(idx, budgets.len()); @@ -847,17 +898,23 @@ mod test { use crate::cluster::stage_has_input_collapse; use crate::cluster::{ - BindStrategy, BoundTask, bind_task_bias, bind_task_round_robin, + BindStrategy, BoundTask, bind_one, bind_task_bias, bind_task_round_robin, next_with_capacity, }; use crate::state::execution_graph::{ExecutionGraph, StaticExecutionGraph}; use crate::state::task_manager::JobInfoCache; + use crate::test_utils::payload::{PayloadExec, payload_resolver}; use crate::test_utils::{ mock_completed_task, revive_graph_and_complete_next_stage, test_aggregation_plan_with_config, }; + use crate::test_utils::{scan_with_file_groups, sorted_into_one_partition}; use ballista_core::config::BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK; + use ballista_core::execution_plans::ShuffleWriterExec; + use ballista_core::execution_plans::plan_algebra::PartitionSliceableResolver; use ballista_core::extension::SessionConfigExt; + use datafusion::physical_plan::ExecutionPlan; + use datafusion::physical_plan::union::UnionExec; use datafusion::prelude::SessionConfig; #[tokio::test] @@ -1058,6 +1115,105 @@ mod test { Ok(graph) } + /// A running stage of `plan` below a shuffle writer, with one partition + /// per task at most. + fn running_stage( + plan: Arc, + ) -> crate::state::execution_stage::RunningStage { + let partitions = plan.properties().output_partitioning().partition_count(); + let writer: Arc = Arc::new( + ShuffleWriterExec::try_new("job".into(), 1, plan, String::new()).unwrap(), + ); + crate::state::execution_stage::RunningStage::new( + 1, + 0, + writer, + partitions, + vec![], + HashMap::new(), + Arc::new( + SessionConfig::new_with_ballista() + .set_str(BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, "1"), + ), + ) + } + + /// Binds every task of `stage` on one executor of `vcores` vcores, + /// returning each task's partitions and vcores. + fn bind_all( + stage: &mut crate::state::execution_stage::RunningStage, + vcores: u32, + resolver: Option<&dyn PartitionSliceableResolver>, + ) -> Vec<(Vec, u32)> { + let mut budget = AvailableVcores { + executor_id: "executor".to_string(), + vcores, + }; + let mut bound = vec![]; + while budget.vcores > 0 { + let Some((_, task)) = + bind_one(stage, "session", &"job".into(), &mut budget, resolver) + else { + break; + }; + bound.push((task.global_input_partition_ids, task.vcores_consumed)); + } + bound + } + + /// A union whose sorted branch keeps its partition in every task runs as + /// one task, whatever `max_partitions_per_task` says, using one vcore for + /// each partition the executor has free. + #[test] + fn a_stage_with_an_unsplittable_union_is_bound_as_one_task() { + // Partition 0 is the sort's, 1-2 the scan's. + let union = UnionExec::try_new(vec![ + sorted_into_one_partition(scan_with_file_groups(2)), + scan_with_file_groups(2), + ]) + .unwrap(); + + assert_eq!( + bind_all(&mut running_stage(union.clone()), 8, None), + [(vec![0, 1, 2], 3)] + ); + assert_eq!( + bind_all(&mut running_stage(union), 2, None), + [(vec![0, 1, 2], 2)] + ); + } + + /// A union that splits cleanly keeps one partition per task. + #[test] + fn a_stage_with_a_splittable_union_is_bound_a_partition_per_task() { + let union = + UnionExec::try_new(vec![scan_with_file_groups(2), scan_with_file_groups(1)]) + .unwrap(); + + assert_eq!( + bind_all(&mut running_stage(union), 8, None), + [(vec![0], 1), (vec![1], 1), (vec![2], 1)] + ); + } + + /// Binding asks the resolver the tasks are restricted with, so a node of + /// the user's own that it can restrict doesn't pack the stage. + #[test] + fn binding_a_union_asks_the_resolver_the_tasks_are_restricted_with() { + let own = PayloadExec::over(&["p0", "p1"], vec![scan_with_file_groups(2)], &[]); + let union = UnionExec::try_new(vec![own, scan_with_file_groups(1)]).unwrap(); + let resolver = payload_resolver(); + + assert_eq!( + bind_all(&mut running_stage(union.clone()), 8, None), + [(vec![0, 1, 2], 3)] + ); + assert_eq!( + bind_all(&mut running_stage(union), 8, Some(resolver.as_ref())), + [(vec![0], 1), (vec![1], 1), (vec![2], 1)] + ); + } + fn mock_budgets() -> Vec { vec![ AvailableVcores { diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 84d7409086..e2816019a5 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -54,7 +54,7 @@ use log::debug; use crate::physical_optimizer::join_selection::{ collect_left_broadcast_safe, should_swap_join_order, }; -use crate::state::task_builder::restrict_plan_to_partitions; +use crate::state::task_builder::branch_splits; type PartialQueryStageResult = (Arc, Vec>); @@ -715,26 +715,22 @@ impl DefaultDistributedPlanner { /// Whether a union branch can be left in the union's own stage. /// /// A branch already sitting behind a stage boundary needs nothing more, and -/// neither does one that per-task restriction can reduce to zero partitions — -/// that is the property the union's index arithmetic relies on. The second -/// question is put to the real rewriter rather than to a list of operators, so -/// it stays correct as the rewriter's handling of collapses, broadcast readers -/// and leaf types evolves. +/// neither does one that per-task restriction can cut to whatever share of it +/// a task runs ([`branch_splits`]). That question is put to the real rewriter +/// rather than to a list of operators, so it stays correct as the rewriter's +/// handling of collapses, broadcast readers and leaf types evolves. +/// +/// The rewriter is asked without the scheduler's resolver, so a branch over a +/// leaf of several partitions that only a resolver can restrict, such as a +/// scan over a source of your own, gets a stage of its own. So does a node +/// over a shuffle of several partitions, which the planner sees before it is +/// resolved. A branch over a one-partition leaf stays inline: only the task +/// that owns its partition runs it, whole. fn can_stay_inline(plan: &Arc) -> bool { if plan.is::() { return true; } - !holds_a_broadcast(plan) && restricts_to_nothing(plan) -} - -/// Whether per-task restriction can reduce `plan` to zero partitions. An empty -/// slice means "this task polls nothing here", so a branch that still reports a -/// partition afterwards, or that the rewriter refuses, cannot be restricted -/// away. No resolver is passed, so a node Ballista does not know gets a stage -/// of its own. -fn restricts_to_nothing(plan: &Arc) -> bool { - restrict_plan_to_partitions(plan.clone(), &[], None) - .is_ok_and(|p| p.properties().output_partitioning().partition_count() == 0) + !holds_a_broadcast(plan) && branch_splits(plan, None) } /// Whether `plan` contains a broadcast input or a `CollectLeft` join. @@ -949,6 +945,7 @@ mod range_shuffle_tests; mod test { use super::{can_stay_inline, holds_a_broadcast}; use crate::planner::{DefaultDistributedPlanner, DistributedPlanner}; + use crate::test_utils::payload::PayloadExec; use crate::test_utils::{datafusion_test_context, scan_with_file_groups}; use ballista_core::assert_plan; use ballista_core::error::BallistaError; @@ -971,6 +968,7 @@ mod test { use datafusion::physical_plan::projection::ProjectionExec; use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; + use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::windows::BoundedWindowAggExec; use datafusion::physical_plan::{ExecutionPlan, displayable}; use datafusion::physical_plan::{InputOrderMode, Partitioning}; @@ -987,6 +985,95 @@ mod test { assert!(can_stay_inline(&scan_with_file_groups(4))); } + /// A branch over an unknown leaf of several partitions gets a stage of its + /// own even when the leaf sits below another node, since restricting the + /// node restricts the leaf too. + #[test] + fn multi_partition_unknown_leaf_below_a_node_needs_its_own_stage() { + let branch = project_leading_columns(PayloadExec::leaf(&["p0", "p1"]), 1); + assert!(!can_stay_inline(&branch)); + } + + /// A node over a shuffle, such as the final aggregate of a union of + /// aggregates, gets a stage of its own: the planner sees the shuffle before + /// it is resolved, when nothing can restrict it to part of its partitions. + /// Over a one-partition shuffle it stays inline, since the task that owns + /// the partition reads the shuffle whole. + #[test] + fn node_over_an_unresolved_shuffle_stays_inline_only_over_one_partition() { + let over_shuffle = |partitions| { + let reader: Arc = Arc::new(UnresolvedShuffleExec::new( + 1, + scan_with_file_groups(1).schema(), + Partitioning::UnknownPartitioning(partitions), + )); + project_leading_columns(reader, 1) + }; + assert_eq!( + [4, 1].map(|partitions| can_stay_inline(&over_shuffle(partitions))), + [false, true] + ); + } + + /// Plans `plan` into query stages with the default options. + fn stages_of( + plan: Arc, + ) -> Result>, BallistaError> { + DefaultDistributedPlanner::new().plan_query_stages( + &"job".into(), + plan, + &datafusion::config::ConfigOptions::default(), + ) + } + + /// A union branch over an unknown leaf of several partitions gets a stage + /// of its own rather than leave a task of the union's stage owning part of + /// it. The union's stage reads it through a shuffle, which a task can cut + /// to any share. + #[test] + fn union_over_a_multi_partition_unknown_leaf_gives_it_a_stage_of_its_own() + -> Result<(), BallistaError> { + let union = UnionExec::try_new(vec![ + scan_with_file_groups(3), + PayloadExec::leaf(&["p0", "p1"]), + ])?; + let stages = stages_of(union)?; + + assert_eq!(stages.len(), 2); + assert_plan!(stages[0].as_ref(), @r#" + ShuffleWriterExec: partitioning: UnknownPartitioning(2) + PayloadExec: payloads=["p0", "p1"] + "#); + assert_plan!(stages[1].as_ref(), @" + ShuffleWriterExec: partitioning: UnknownPartitioning(5) + UnionExec + DataSourceExec: file_groups={3 groups: [[file0.parquet], [file1.parquet], [file2.parquet]]}, projection=[a], file_type=parquet + UnresolvedShuffleExec: stage=1, partitioning: UnknownPartitioning(2) + "); + Ok(()) + } + + /// A union branch over a one-partition unknown leaf stays in the union's + /// stage: only the task that owns its partition runs it. + #[test] + fn union_over_a_one_partition_unknown_leaf_keeps_it_in_the_unions_stage() + -> Result<(), BallistaError> { + let union = UnionExec::try_new(vec![ + scan_with_file_groups(3), + PayloadExec::leaf(&["p0"]), + ])?; + let stages = stages_of(union)?; + + assert_eq!(stages.len(), 1); + assert_plan!(stages[0].as_ref(), @r#" + ShuffleWriterExec: partitioning: UnknownPartitioning(4) + UnionExec + DataSourceExec: file_groups={3 groups: [[file0.parquet], [file1.parquet], [file2.parquet]]}, projection=[a], file_type=parquet + PayloadExec: payloads=["p0"] + "#); + Ok(()) + } + /// A branch already sitting behind a stage boundary needs nothing more. #[test] fn branch_behind_a_stage_boundary_stays_inline() { diff --git a/ballista/scheduler/src/state/execution_stage.rs b/ballista/scheduler/src/state/execution_stage.rs index a38c7c0f41..10fe012845 100644 --- a/ballista/scheduler/src/state/execution_stage.rs +++ b/ballista/scheduler/src/state/execution_stage.rs @@ -255,6 +255,12 @@ pub struct RunningStage { /// Prefix-merged in global partition order to seed a downstream /// `PrefixMergeExec`; dropped when the stage transitions to Successful. pub window_state_reports: Vec, + /// The `UnionExec` branch that can't be restricted to part of its + /// partitions, if the plan has one, which makes binding run the stage as + /// one task. `None` until the first bind that needs it computes it: it + /// depends only on `plan` and the job's resolver, which are fixed for a + /// running stage. + pub(crate) unsplittable_union_branch: Option>>, } /// If a stage finishes successfully, its task statuses and metrics will be finalized @@ -701,6 +707,7 @@ impl RunningStage { session_config, runtime_stats_reports: Vec::new(), window_state_reports: Vec::new(), + unsplittable_union_branch: None, } } @@ -1331,6 +1338,7 @@ impl SuccessfulStage { // Merged-cut logging fires per-attempt on final success. runtime_stats_reports: Vec::new(), window_state_reports: Vec::new(), + unsplittable_union_branch: None, } } diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs index fe10c826f4..a92ec6ff2a 100644 --- a/ballista/scheduler/src/state/mod.rs +++ b/ballista/scheduler/src/state/mod.rs @@ -686,7 +686,8 @@ mod tests { }; use crate::test_utils::{ BlackholeTaskLauncher, RejectingTaskLauncher, aggregation_logical_plan, - mock_executor, register_virtual_executor, test_cluster_context, + mock_executor, register_virtual_executor, scan_with_file_groups, + test_cluster_context, }; use ballista_core::config::{ BALLISTA_EXECUTION_MODE, BALLISTA_PROPAGATE_EMPTY_ENABLED, @@ -698,6 +699,8 @@ mod tests { }; use ballista_core::serde::{BallistaCodec, BallistaLogicalExtensionCodec}; use datafusion::execution::context::SessionConfig; + use datafusion::physical_plan::union::UnionExec; + use datafusion::physical_plan::{ExecutionPlan, displayable}; use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; @@ -968,11 +971,13 @@ mod tests { } } - /// Submits a job of one four-partition `PayloadExec` and binds it to two - /// executors of two vcores each, which splits its stage into two tasks. + /// Submits a job of `plan`, one stage of four partitions, and binds it to + /// two executors of two vcores each, which splits the stage into two + /// tasks. Its physical codec carries `PayloadExec`. async fn bind_split_payload_stage( config: SchedulerConfig, launcher: Arc, + plan: Arc, ) -> Result<( SchedulerState, JobId, @@ -1017,14 +1022,7 @@ mod tests { .queue_job(&job_id, "", timestamp_millis())?; state .task_manager - .submit_physical_plan( - &job_id, - "", - ctx, - PayloadExec::leaf(&["p0", "p1", "p2", "p3"]), - timestamp_millis(), - None, - ) + .submit_physical_plan(&job_id, "", ctx, plan, timestamp_millis(), None) .await?; let bound = state @@ -1059,9 +1057,12 @@ mod tests { async fn launch_tasks_fails_a_job_whose_split_stage_holds_a_leaf_nobody_can_restrict() -> Result<()> { let launcher = Arc::new(CapturingLauncher::default()); - let (state, job_id, bound) = - bind_split_payload_stage(SchedulerConfig::default(), launcher.clone()) - .await?; + let (state, job_id, bound) = bind_split_payload_stage( + SchedulerConfig::default(), + launcher.clone(), + PayloadExec::leaf(&["p0", "p1", "p2", "p3"]), + ) + .await?; let vcores: u32 = bound.iter().map(|(_, task)| task.vcores_consumed).sum(); let (freed, failed_jobs) = launch(&state, bound).await?; @@ -1079,17 +1080,59 @@ mod tests { let launcher = Arc::new(CapturingLauncher::default()); let config = SchedulerConfig::default() .with_partition_sliceable_resolver(payload_resolver()); - let (state, _job_id, bound) = - bind_split_payload_stage(config, launcher.clone()).await?; + let (state, _job_id, bound) = bind_split_payload_stage( + config, + launcher.clone(), + PayloadExec::leaf(&["p0", "p1", "p2", "p3"]), + ) + .await?; let (freed, failed_jobs) = launch(&state, bound).await?; assert!(failed_jobs.is_empty()); assert_eq!(freed, 0, "every task was launched, so no slot is refunded"); - // Decode what the executors would, with the codec the scheduler used. + let mut shipped: Vec> = shipped_writer_inputs(&state, &launcher) + .iter() + .map(payloads_of) + .collect(); + shipped.sort(); + assert_eq!(shipped, [["p0", "p1"], ["p2", "p3"]]); + + Ok(()) + } + + /// A running job carries the resolver its tasks are restricted with, so + /// binding judges which stages it can split the same way. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_running_job_carries_the_configured_resolver() -> Result<()> { + let resolver = payload_resolver(); + let config = SchedulerConfig::default() + .with_partition_sliceable_resolver(resolver.clone()); + let (state, job_id, _bound) = bind_split_payload_stage( + config, + Arc::new(CapturingLauncher::default()), + PayloadExec::leaf(&["p0", "p1", "p2", "p3"]), + ) + .await?; + + let jobs = state.task_manager.get_running_job_cache(); + let carried = jobs[&job_id] + .partition_sliceable_resolver + .clone() + .expect("the job carries a resolver"); + assert!(Arc::ptr_eq(&carried, &resolver)); + Ok(()) + } + + /// The input of the shuffle writer of each task `launcher` launched, + /// decoded as the executor would, with the codec the scheduler used. + fn shipped_writer_inputs( + state: &SchedulerState, + launcher: &CapturingLauncher, + ) -> Vec> { let task_ctx = SessionContext::new().task_ctx(); - let mut shipped: Vec> = launcher + launcher .launched .lock() .unwrap() @@ -1104,15 +1147,55 @@ mod tests { }) .expect("the executor can decode the plan"); // The writer's input can come wrapped in a `CooperativeExec`. - let mut input = plan.children()[0]; + let mut input = plan.children()[0].clone(); while input.name() == "CooperativeExec" { - input = input.children()[0]; + input = input.children()[0].clone(); } - payloads_of(input) + input }) + .collect() + } + + /// A `UnionExec` branch over a leaf the scheduler doesn't know, as an + /// Iceberg scan is, runs in the task that owns it and becomes an empty + /// leaf in the task that doesn't, with no resolver configured. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn launch_tasks_empties_an_unknown_leaf_in_a_union_branch_another_task_owns() + -> Result<()> { + let launcher = Arc::new(CapturingLauncher::default()); + // Partition 0 is the payload's, 1-3 the scan's. + let union = UnionExec::try_new(vec![ + PayloadExec::leaf(&["p0"]), + scan_with_file_groups(3), + ])?; + let (state, _job_id, bound) = + bind_split_payload_stage(SchedulerConfig::default(), launcher.clone(), union) + .await?; + + let (freed, failed_jobs) = launch(&state, bound).await?; + + assert!(failed_jobs.is_empty()); + assert_eq!(freed, 0, "every task was launched, so no slot is refunded"); + let mut shipped: Vec = shipped_writer_inputs(&state, &launcher) + .iter() + .map(|plan| displayable(plan.as_ref()).indent(false).to_string()) .collect(); shipped.sort(); - assert_eq!(shipped, [["p0", "p1"], ["p2", "p3"]]); + assert_eq!( + shipped, + [ + "UnionExec\n \ + CooperativeExec\n \ + DataSourceExec: partitions=0, partition_sizes=[]\n \ + DataSourceExec: file_groups={2 groups: [[file1.parquet], \ + [file2.parquet]]}, projection=[a], file_type=parquet\n", + "UnionExec\n \ + CooperativeExec\n \ + PayloadExec: payloads=[\"p0\"]\n \ + DataSourceExec: file_groups={1 group: [[file0.parquet]]}, \ + projection=[a], file_type=parquet\n", + ] + ); Ok(()) } diff --git a/ballista/scheduler/src/state/task_builder.rs b/ballista/scheduler/src/state/task_builder.rs index b2e137ff17..46082c123d 100644 --- a/ballista/scheduler/src/state/task_builder.rs +++ b/ballista/scheduler/src/state/task_builder.rs @@ -68,9 +68,11 @@ use std::sync::Arc; /// side) — those leaves keep the full upstream so the collapse sees every /// partition. /// -/// Nodes Ballista does not list are restricted by `resolver`. A leaf that -/// nothing can restrict is an error unless `partitions` is all of its -/// partitions, in order. +/// A leaf Ballista does not list becomes an empty leaf when the task runs none +/// of it. Other nodes Ballista does not list are restricted by `resolver`, and +/// a leaf that nothing can restrict is an error unless `partitions` is all of +/// its partitions, in order. Each branch of a `UnionExec` must end up with one +/// partition for each partition the task runs of it. pub fn restrict_plan_to_partitions( plan: Arc, partitions: &[usize], @@ -109,6 +111,59 @@ pub(crate) fn stages_read_whole(plan: &Arc) -> Vec { found } +/// The first `UnionExec` branch in the stage plan `plan` that a task given +/// only some of the stage's partitions would run wrongly: a branch that fails +/// [`branch_splits`]. The scheduler runs such a stage as one task instead, +/// which runs every branch whole. +/// +/// The adaptive planner can leave such a branch in the union's stage. The +/// static planner gives it a stage of its own, so its stages never need this. +pub(crate) fn unsplittable_union_branch( + plan: &Arc, + resolver: Option<&dyn PartitionSliceableResolver>, +) -> Option> { + fn walk( + plan: &Arc, + under_collect: bool, + resolver: Option<&dyn PartitionSliceableResolver>, + ) -> Option> { + // Below a collapse every task reads the whole input anyway, so a + // union there is never split. + if !under_collect && plan.is::() { + return plan + .children() + .into_iter() + .find(|branch| !branch_splits(branch, resolver)) + .cloned(); + } + plan.children() + .into_iter() + .zip(child_scopes(plan, under_collect)) + .find_map(|(child, collect)| walk(child, collect, resolver)) + } + walk(plan, false, resolver) +} + +/// Whether restriction can give a task any share of the `UnionExec` branch +/// `branch`: none of it, which the union's index arithmetic relies on, and +/// part of it, tried as its first partition alone. +/// +/// A branch fails when it keeps partitions restricted to none, as one that +/// merges its input into one partition, reads a broadcast or sets its own +/// partition count does. It also fails over a leaf nothing can restrict that +/// has more than one partition, which the rewriter refuses for every slice +/// but the whole, so `[0]` catches it as any partial slice would. +pub(crate) fn branch_splits( + branch: &Arc, + resolver: Option<&dyn PartitionSliceableResolver>, +) -> bool { + let restricts_to_nothing = restrict(branch.clone(), &[], false, resolver) + .is_ok_and(|p| p.properties().output_partitioning().partition_count() == 0); + restricts_to_nothing + && (branch.properties().output_partitioning().partition_count() == 0 + || restrict(branch.clone(), &[0], false, resolver).is_ok()) +} + /// Recursive worker. `under_collect` is the scope inherited from ancestors: /// once set, every descendant leaf reads the full upstream. Scope is passed /// by value, so sibling subtrees never leak state to each other. @@ -127,6 +182,12 @@ fn restrict( return Ok(rewritten); } + // Any other leaf the task runs none of, such as one in a `UnionExec` + // branch another task owns, is never executed. + if !under_collect && partitions.is_empty() && plan.children().is_empty() { + return no_partitions(&plan); + } + // UnionExec: parent partition `p` maps to exactly one child's local // partition (`p` minus the sum of preceding children's counts). Split // `partitions` into disjoint per-child sub-slices and recurse; a child @@ -141,7 +202,11 @@ fn restrict( .into_iter() .cloned() .zip(per_child) - .map(|(child, sub)| restrict(child, &sub, false, resolver)) + .map(|(child, sub)| { + let restricted = restrict(child, &sub, false, resolver)?; + ensure_branch_restricted(&restricted, &sub)?; + Ok(restricted) + }) .collect::>>()?; return replace_children_if_necessary(plan, new_children); } @@ -212,10 +277,22 @@ fn checked_restriction( Ok(restricted) } +/// A leaf with `plan`'s schema and no partitions, for a leaf the task runs +/// none of. +/// +/// Not an `EmptyExec`: DataFusion's proto codec decodes one with zero +/// partitions as one partition, so it would reach the executor with a +/// partition the task doesn't own, which the executor would run. A +/// `MemorySourceConfig` keeps its empty partition list. +fn no_partitions(plan: &Arc) -> Result> { + let source = MemorySourceConfig::try_new(&[], plan.schema(), None)?; + Ok(DataSourceExec::from_data_source(source)) +} + /// Accept a leaf that nothing could restrict only if the task runs all of it, /// in order. Left as it is, it numbers its partitions from zero, so any other /// slice would run the wrong ones. A one-partition leaf therefore passes in the -/// task that owns it, and fails in a `UnionExec` branch that belongs to another. +/// task that owns it. fn ensure_leaf_runs_whole( plan: &Arc, partitions: &[usize], @@ -236,6 +313,31 @@ fn ensure_leaf_runs_whole( ) } +/// Check that a restricted `UnionExec` branch has one partition for each of +/// `partitions`. The executor runs every partition of the task's plan, so a +/// branch that keeps partitions the task wasn't given would run them in this +/// task too, returning rows another task returns as well. +/// +/// Binding runs a stage with such a branch as one task (see +/// [`unsplittable_union_branch`]), so this only fails if binding and restriction +/// disagree: it refuses the task rather than let it return rows twice. +fn ensure_branch_restricted( + branch: &Arc, + partitions: &[usize], +) -> Result<()> { + let count = branch.properties().output_partitioning().partition_count(); + if count == partitions.len() { + return Ok(()); + } + plan_err!( + "cannot restrict a `UnionExec` branch to partitions {partitions:?}: its \ + `{}` keeps a partition count of {count}, so the task would also run \ + partitions another task owns. The scheduler should have run this stage \ + as one task.", + branch.name() + ) +} + /// Bucket each parent partition of a `UnionExec` into its owning child's /// local index. Iterates over `partitions` and, for each, subtracts each /// child's count until the remainder lands inside a child. Returns one @@ -342,11 +444,11 @@ fn child_scopes(plan: &Arc, under_collect: bool) -> Vec /// /// **When a new leaf type surfaces, add a branch below.** Currently /// covered: `ShuffleReaderExec`, `DataSourceExec` (files, in-memory), -/// `PlaceholderRowExec`, `EmptyExec`. `ValuesExec` and other future -/// self-generating leaves fall through as `None`; if one sits under a -/// partition-splitting fan-in, the walker refuses the task rather than let the -/// executor duplicate output (see the PlaceholderRow-under-UnionExec bug for -/// the failure mode). +/// `PlaceholderRowExec`, `EmptyExec`. Any other leaf falls through as +/// `None`. The walker then replaces it with an empty leaf in a task that runs +/// none of it, such as one that owns another `UnionExec` branch, keeps it in a +/// task that runs all of it, in order, and otherwise refuses the task rather +/// than let the executor run the wrong partitions. /// /// # Scope /// @@ -466,33 +568,24 @@ fn select_output_partitions( return Ok(None); } - // Self-generating leaves: PlaceholderRow (1 all-null row per partition) - // and EmptyExec (0 rows per partition). Every partition of one of these - // is semantically identical, so `with_partitions(indices.len())` is the - // algebraically correct restriction. - // - // Caveat: DataFusion's proto codec for these two doesn't roundtrip the - // `partitions` field — encode preserves only schema; decode always - // reconstructs with `::new(schema)` (partitions = 1). So a plan we - // restrict to `with_partitions(0)` here would arrive at the executor as - // a 1-partition leaf, undoing the restriction and reproducing the - // duplicated-output bug this whole function is meant to prevent. - // - // Workaround for the empty-slice case: swap in a 0-partition - // `MemorySourceConfig` (which DOES roundtrip its partition list). - // For non-empty slices we return the leaf as it is — currently safe because - // the only shape emitted is `partitions == 1` and UnionExec routing gives - // sub-slices of `[0]` (identity, no restriction needed) or `[]` (handled by - // this branch). Returned rather than `None` so the walker's leaf check - // accepts it: an `EmptyExec` under the round-robin `RepartitionExec` that - // `make_empty_exec_serde_safe` adds sees that repartition's partition - // numbers, not its own. + // Self-generating leaves: PlaceholderRow (1 row per partition) and + // EmptyExec (0 rows per partition). Every partition is the same, so a + // slice is the leaf with one partition for each index it has. As for the + // sources above, an index past its own count is dropped: under a + // round-robin `RepartitionExec` the leaf sees the repartition's numbers, + // and only the task that owns partition 0 may run a one-partition leaf. if plan.is::() || plan.is::() { - if indices.is_empty() { - let stub = MemorySourceConfig::try_new(&[], plan.schema(), None)?; - return Ok(Some(DataSourceExec::from_data_source(stub))); + let count = plan.properties().output_partitioning().partition_count(); + let kept = indices.iter().filter(|&&i| i < count).count(); + if kept == 0 { + return no_partitions(plan).map(Some); } - return Ok(Some(plan.clone())); + let restricted: Arc = if plan.is::() { + Arc::new(EmptyExec::new(plan.schema()).with_partitions(kept)) + } else { + Arc::new(PlaceholderRowExec::new(plan.schema()).with_partitions(kept)) + }; + return Ok(Some(restricted)); } Ok(None) @@ -520,6 +613,7 @@ fn narrow_range(range: &RangePartitioning, kept: &[usize]) -> Option Arc { - UnionExec::try_new(vec![ - scan_with_file_groups(2), - PayloadExec::leaf(&["p0", "p1"]), - ]) - .unwrap() + UnionExec::try_new(vec![scan_with_file_groups(2), unknown.clone()]).unwrap() }; - // Partition 2 is the second branch's partition 0. + // Partition 0 is the scan's, which leaves the unknown leaf none. + let restricted = restrict_plan_to_partitions(union(), &[0], None).unwrap(); + let branches = restricted.children(); + assert_eq!(group_files(branches[0]), ["file0.parquet"]); + assert_no_partitions(branches[1], &unknown); + + // Partition 2 is the unknown leaf's partition 0, which only the + // resolver can cut down. let restricted = restrict_plan_to_partitions(union(), &[2], Some(resolver.as_ref())).unwrap(); let branches = restricted.children(); - assert_eq!(partition_count(branches[0]), 0); + assert_eq!(group_files(branches[0]), Vec::::new()); assert_eq!(payloads_of(branches[1]), ["p0"]); - // And partition 0 is the first branch's, which leaves the second none. - let restricted = - restrict_plan_to_partitions(union(), &[0], Some(resolver.as_ref())).unwrap(); - let branches = restricted.children(); + let in_union = restrict_plan_to_partitions(union(), &[2], None).unwrap_err(); + let alone = restrict_plan_to_partitions(unknown.clone(), &[0], None).unwrap_err(); + assert_eq!(in_union.to_string(), alone.to_string()); + } + + /// The task plan is serialized on its way to the executor, and each union + /// branch must keep its restricted partition count there, or the task would + /// run partitions it doesn't own: none for a branch the task runs none of, + /// and its share for a multi-partition `EmptyExec`. + #[test] + fn restricted_union_branches_keep_their_partition_counts_through_serialization() { + use ballista_core::serde::BallistaPhysicalExtensionCodec; + use datafusion::prelude::SessionContext; + use datafusion_proto::bytes::{ + physical_plan_from_bytes_with_extension_codec, + physical_plan_to_bytes_with_extension_codec, + }; + + let schema = scan_with_file_groups(2).schema(); + // Partitions 0-1 are the scan's, 2 the unknown leaf's, 3-6 the empty one's. + let union = UnionExec::try_new(vec![ + scan_with_file_groups(2), + PayloadExec::leaf(&["p0"]), + Arc::new(EmptyExec::new(schema).with_partitions(4)), + ]) + .unwrap(); + let restricted = restrict_plan_to_partitions(union, &[0, 4, 6], None).unwrap(); + + let codec = BallistaPhysicalExtensionCodec::default(); + let bytes = + physical_plan_to_bytes_with_extension_codec(restricted, &codec).unwrap(); + let decoded = physical_plan_from_bytes_with_extension_codec( + &bytes, + &SessionContext::new().task_ctx(), + &codec, + ) + .unwrap(); + + let branches = decoded.children(); + assert_eq!( + branches.iter().map(|b| describe(b)).collect::>(), + ["DataSourceExec(1)", "DataSourceExec(0)", "EmptyExec(2)"] + ); assert_eq!(group_files(branches[0]), ["file0.parquet"]); - assert_eq!(partition_count(branches[1]), 0); + } + + /// Binding runs a stage with a branch that keeps its partitions as one + /// task. Should a task get only part of such a stage anyway, restriction + /// refuses it rather than let the branch return its rows in every task. + #[test] + fn a_union_branch_that_keeps_its_partitions_is_refused() { + // Partition 0 is the sort's, 1-2 the scan's. + let union = UnionExec::try_new(vec![ + sorted_into_one_partition(scan_with_file_groups(2)), + scan_with_file_groups(2), + ]) + .unwrap(); + + let error = restrict_plan_to_partitions(union, &[1], None) + .unwrap_err() + .to_string(); + assert_eq!( + error, + "Error during planning: cannot restrict a `UnionExec` branch to \ + partitions []: its `SortExec` keeps a partition count of 1, so the \ + task would also run partitions another task owns. The scheduler \ + should have run this stage as one task." + ); + } + + /// A stage can be split between tasks unless one of its unions has a + /// branch that restriction can't cut to a task's share of it. + /// Each case names the top node of the first such branch. + #[test] + fn unions_that_cannot_be_split_between_tasks() { + use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; + let union = |branches: Vec>| -> Arc { + UnionExec::try_new(branches).unwrap() + }; + let broadcast: Arc = Arc::new( + ShuffleReaderExec::try_new_broadcast( + 1, + (0..2).map(create_partition).collect(), + scan_with_file_groups(1).schema(), + 2, + ) + .unwrap(), + ); + let sorted = || sorted_into_one_partition(scan_with_file_groups(2)); + let scan = || scan_with_file_groups(2); + + let cases: Vec<(&str, Arc, Option<&str>)> = vec![ + ("no union", scan(), None), + ("scans", union(vec![scan(), scan()]), None), + ( + "a one-partition unknown leaf", + union(vec![PayloadExec::leaf(&["p0"]), scan()]), + None, + ), + ( + "an unknown leaf of two partitions", + union(vec![PayloadExec::leaf(&["p0", "p1"]), scan()]), + Some("PayloadExec"), + ), + ("a sort", union(vec![sorted(), scan()]), Some("SortExec")), + ( + "a broadcast", + union(vec![broadcast, scan()]), + Some("ShuffleReaderExec"), + ), + ( + "an unknown node that keeps its partitions", + union(vec![ + PayloadExec::over(&["p0", "p1"], vec![scan()], &[]), + scan(), + ]), + Some("PayloadExec"), + ), + ( + "a sort in a nested union", + union(vec![ + union(vec![sorted(), scan_with_file_groups(1)]), + scan(), + ]), + Some("UnionExec"), + ), + ( + "a union below a sort", + sorted_into_one_partition(union(vec![sorted(), scan()])), + None, + ), + ( + "a union below a coalesce", + Arc::new(CoalescePartitionsExec::new(union(vec![sorted(), scan()]))), + None, + ), + ]; + let actual: Vec<(&str, Option)> = cases + .iter() + .map(|(name, plan, _)| { + let branch = unsplittable_union_branch(plan, None); + (*name, branch.map(|branch| branch.name().to_string())) + }) + .collect(); + let expected: Vec<(&str, Option)> = cases + .iter() + .map(|(name, _, branch)| (*name, branch.map(str::to_string))) + .collect(); + assert_eq!(actual, expected); + } + + /// The resolver the tasks are restricted with can split the unions over + /// nodes of your own that nothing else can. + #[test] + fn the_resolver_lets_a_union_over_nodes_of_your_own_be_split() { + let resolver = payload_resolver(); + let scan = || scan_with_file_groups(2); + let over_leaf = + UnionExec::try_new(vec![PayloadExec::leaf(&["p0", "p1"]), scan()]).unwrap(); + let over_node = UnionExec::try_new(vec![ + PayloadExec::over(&["p0", "p1"], vec![scan()], &[]), + scan(), + ]) + .unwrap(); + + assert_eq!( + [&over_leaf, &over_node].map(|plan| { + unsplittable_union_branch(plan, Some(resolver.as_ref())).is_none() + }), + [true, true] + ); + } - assert!(restrict_plan_to_partitions(union(), &[0], None).is_err()); - assert!(restrict_plan_to_partitions(union(), &[2], None).is_err()); + /// `plan` is an empty in-memory source with `like`'s schema and no + /// partitions, which the executor decodes as it is. + fn assert_no_partitions( + plan: &Arc, + like: &Arc, + ) { + assert_eq!( + displayable(plan.as_ref()).one_line().to_string().trim_end(), + "DataSourceExec: partitions=0, partition_sizes=[]" + ); + assert_eq!(plan.schema(), like.schema()); } #[test] @@ -1155,20 +1432,109 @@ mod tests { assert!(error.contains("returned 4 partitions"), "{error}"); } - /// An `EmptyExec` sees the partition numbers of the repartition above it, - /// not its own, so these two leaves are exempt from the leaf check. + /// A node's name and partition count, such as `EmptyExec(2)`. + fn describe(plan: &Arc) -> String { + format!("{}({})", plan.name(), partition_count(plan)) + } + + /// `leaf` restricted to each of `slices`, described. + fn restricted_shapes( + leaf: &Arc, + slices: &[&[usize]], + ) -> Vec { + slices + .iter() + .map(|slice| { + let restricted = + restrict_plan_to_partitions(leaf.clone(), slice, None).unwrap(); + assert_eq!(restricted.schema(), leaf.schema()); + describe(&restricted) + }) + .collect() + } + + /// Every partition of these leaves is the same, so a task gets one for + /// each of its partitions. The `AdaptivePlanner` replaces a stage that + /// turns out empty with an `EmptyExec` of that stage's partition count. #[test] - fn self_generating_leaves_are_exempt_from_the_leaf_check() { + fn self_generating_leaves_keep_one_partition_for_each_partition_the_task_runs() { let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); - let empty: Arc = Arc::new(EmptyExec::new(schema.clone())); - assert!(restrict_plan_to_partitions(empty, &[1], None).is_ok()); + let empty: Arc = + Arc::new(EmptyExec::new(schema.clone()).with_partitions(4)); + let placeholder: Arc = + Arc::new(PlaceholderRowExec::new(schema).with_partitions(4)); + let slices: &[&[usize]] = &[&[0, 1, 2, 3], &[1, 3], &[2], &[]]; + + assert_eq!( + restricted_shapes(&empty, slices), + [ + "EmptyExec(4)", + "EmptyExec(2)", + "EmptyExec(1)", + "DataSourceExec(0)" + ] + ); + assert_eq!( + restricted_shapes(&placeholder, slices), + [ + "PlaceholderRowExec(4)", + "PlaceholderRowExec(2)", + "PlaceholderRowExec(1)", + "DataSourceExec(0)", + ] + ); + } + /// Under a round-robin `RepartitionExec`, which the `AdaptivePlanner` + /// keeps inside a stage, a leaf sees the repartition's partition numbers. + /// Only the task that owns partition 0 runs a one-partition leaf, so a + /// `PlaceholderRowExec` there gives its row once, not once per task. + #[test] + fn self_generating_leaves_drop_partitions_past_their_own() { + use datafusion::physical_plan::repartition::RepartitionExec; + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)])); let placeholder: Arc = Arc::new(PlaceholderRowExec::new(schema)); - assert!(restrict_plan_to_partitions(placeholder.clone(), &[0], None).is_ok()); + let leaf_in_task = |partitions: &[usize]| { + let repartition = RepartitionExec::try_new( + placeholder.clone(), + Partitioning::RoundRobinBatch(4), + ) + .unwrap(); + restrict_plan_to_partitions(Arc::new(repartition), partitions, None) + .unwrap() + .children()[0] + .clone() + }; + + assert_eq!(describe(&leaf_in_task(&[0, 1])), "PlaceholderRowExec(1)"); + assert_no_partitions(&leaf_in_task(&[2, 3]), &placeholder); + } - // A task that polls none of it gets a source with no partitions. - let nothing = restrict_plan_to_partitions(placeholder, &[], None).unwrap(); - assert_eq!(partition_count(¬hing), 0); + /// A union branch the `AdaptivePlanner` replaced with a multi-partition + /// `EmptyExec`, because its stage gave no rows, is cut down like any other, + /// so the union routes the task's partitions to the right branch. + #[test] + fn a_union_branch_replaced_by_an_empty_exec_is_restricted() { + let empty: Arc = Arc::new( + EmptyExec::new(scan_with_file_groups(4).schema()).with_partitions(4), + ); + let union = || -> Arc { + UnionExec::try_new(vec![empty.clone(), scan_with_file_groups(4)]).unwrap() + }; + + // Partition 0 is the empty branch's. + let restricted = restrict_plan_to_partitions(union(), &[0], None).unwrap(); + let branches = restricted.children(); + assert_eq!(describe(branches[0]), "EmptyExec(1)"); + assert_eq!(group_files(branches[1]), Vec::::new()); + + // Partitions 3 and 6 are the last of the empty branch's and the third + // of the scan's. + let restricted = restrict_plan_to_partitions(union(), &[3, 6], None).unwrap(); + let branches = restricted.children(); + assert_eq!(describe(branches[0]), "EmptyExec(1)"); + assert_eq!(group_files(branches[1]), ["file2.parquet"]); + assert_eq!(describe(&restricted), "UnionExec(2)"); } } diff --git a/ballista/scheduler/src/state/task_manager.rs b/ballista/scheduler/src/state/task_manager.rs index c4868e7c2f..0cba8d5b05 100644 --- a/ballista/scheduler/src/state/task_manager.rs +++ b/ballista/scheduler/src/state/task_manager.rs @@ -171,6 +171,9 @@ pub struct JobInfoCache { pub execution_graph: Arc>, /// Cached job status for quick access. pub status: Option, + /// The resolver the job's tasks are restricted with, so binding can tell + /// which stages a task may run only part of. + pub(crate) partition_sliceable_resolver: Option>, } impl JobInfoCache { @@ -181,8 +184,18 @@ impl JobInfoCache { Self { execution_graph: Arc::new(RwLock::new(graph)), status, + partition_sliceable_resolver: None, } } + + /// Sets the resolver the job's tasks are restricted with. + pub(crate) fn with_partition_sliceable_resolver( + mut self, + resolver: Option>, + ) -> Self { + self.partition_sliceable_resolver = resolver; + self + } } /// What losing an executor requires of the jobs that were using it. @@ -446,8 +459,12 @@ impl TaskManager .submit_job(job_id.to_owned(), &graph, subscriber) .await?; graph.revive(); - self.active_job_cache - .insert(job_id.to_owned(), JobInfoCache::new(graph)); + self.active_job_cache.insert( + job_id.to_owned(), + JobInfoCache::new(graph).with_partition_sliceable_resolver( + self.partition_sliceable_resolver.clone(), + ), + ); Ok(()) } diff --git a/ballista/scheduler/src/test_utils.rs b/ballista/scheduler/src/test_utils.rs index 258bab7819..8a96f54565 100644 --- a/ballista/scheduler/src/test_utils.rs +++ b/ballista/scheduler/src/test_utils.rs @@ -1477,6 +1477,22 @@ pub fn scan_with_file_groups(n: usize) -> Arc { DataSourceExec::from_data_source(builder.build()) } +/// A `SortExec` on column `a` of `input` that merges it into one partition, as +/// the adaptive planner leaves on each branch of a `UNION ALL ... ORDER BY`. +pub fn sorted_into_one_partition( + input: Arc, +) -> Arc { + use datafusion::physical_expr::expressions::Column; + use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; + use datafusion::physical_plan::sorts::sort::SortExec; + + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new_default(Arc::new( + Column::new("a", 0), + ))]) + .unwrap(); + Arc::new(SortExec::new(ordering, input)) +} + /// An operator that passes its input through unchanged: its properties are the /// input's, its only child is the input, and it executes by executing the /// input. It declares nothing about what it needs from the input, like an diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index 624a625670..b3b0c91975 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -199,12 +199,13 @@ and 1, and the stage returned some partitions twice and others not at all, with no error. The scheduler now fails the job instead. Such a leaf is accepted only in a task -that runs all of its partitions, in order, so a retry of part of a stage can -fail a job too. Plans made of Ballista's own leaves (shuffle readers, file and -in-memory scans, `PlaceholderRowExec`, `EmptyExec`) are unaffected, as are -leaves read whole below a `CoalescePartitionsExec` or on a join's build side. A -`DataSourceExec` over any other source used to log a warning and run, and is -now refused when its stage is split unless a resolver restricts it. +that runs all of its partitions, in order, or none of them, as in a `UnionExec` +branch another task owns, where it becomes an empty leaf. A retry of part of a +stage can fail a job too. Plans made of Ballista's own leaves (shuffle readers, +file and in-memory scans, `PlaceholderRowExec`, `EmptyExec`) are unaffected, as +are leaves read whole below a `CoalescePartitionsExec` or on a join's build +side. A `DataSourceExec` over any other source used to log a warning and run, +and is now refused when its stage is split unless a resolver restricts it. Whether a stage is split depends on how many task slots are free when it is scheduled, and on whether part of it is retried, so a query that holds such a @@ -216,13 +217,36 @@ Only leaves are checked. An interior node the scheduler does not know is left as it was while its children are restricted, because the scheduler cannot tell one that holds data per partition from one that does not, such as a `FilterExec`. An interior node of your own that holds data per partition still returns wrong -results in a split stage unless a resolver restricts it. +results in a split stage unless a resolver restricts it. Below a `UnionExec`, +a branch where such a node keeps its own partition count gets a stage of its +own, or under the adaptive planner the union's stage runs as one task, since a +task given part of the branch would also run partitions that another task owns. The error names the node and what to implement. Under push-staged scheduling the job's status says only that a task could not be prepared; the reason is in the scheduler's log. To make nodes of your own restrictable, see [Custom Physical Plan Nodes](../user-guide/extending-components.md#custom-physical-plan-nodes). +#### A split `UNION ALL` with a branch that merges its input runs as one task + +When a stage is split between tasks, each task restricts the `UnionExec` +branches other tasks own to no partitions. The adaptive planner can leave a +branch that merges its input into one partition in the union's stage: in a +`UNION ALL ... ORDER BY`, a branch with a `LIMIT` or an `ORDER BY ... LIMIT`, or +a `UNION ALL` of aggregates without `GROUP BY`. Such a branch reads its input +whole and keeps its one partition, so every task of the stage ran it, and the +query returned wrong rows, over any source including Parquet. + +The scheduler now runs such a stage as one task, as it does a stage that +collapses its input, so each branch runs once. The same goes for a branch that +reads a broadcast, holds a node that sets its own partition count, or reads a +leaf of several partitions that nothing can restrict. The task +ignores `ballista.scheduler.max_partitions_per_task` and takes one vcore for +each of the stage's partitions, up to what its executor has free, so the stage +runs with less parallelism than before. The scheduler logs each such stage at +`info` level, naming the branch. The static planner gives such a branch a stage +of its own and is unaffected. + #### Nested-loop joins that emit build rows run as a single task Nested-loop joins of type `Left`, `Full`, `LeftSemi`, `LeftAnti` or `LeftMark` diff --git a/docs/source/user-guide/extending-components.md b/docs/source/user-guide/extending-components.md index 2228bf3425..401440f0ae 100644 --- a/docs/source/user-guide/extending-components.md +++ b/docs/source/user-guide/extending-components.md @@ -137,7 +137,7 @@ As with the scheduler, to keep the executor's own command line, parse A plan with `ExecutionPlan` nodes of your own can run on a Ballista cluster once the scheduler and the executors have a `physical codec` that serializes them. A node that holds data per partition needs one thing more. -When a stage is split between tasks, each task gets a copy of the plan restricted to its own partitions, and operators see only task-local partition indices: a task that runs partitions 4 and 5 executes partitions 0 and 1 of every node. The scheduler restricts the leaves it knows, such as shuffle readers and file and in-memory scans. A node it doesn't know would read its per-partition data, such as one list of files for each partition, from the wrong index, and the stage would return some partitions twice and others not at all. The scheduler fails a job whose split stage holds such a leaf. It does not check interior nodes: it cannot tell one that holds per-partition data from one that does not, such as a `FilterExec`, so an interior node of yours that holds per-partition data returns wrong results in a split stage unless you register it. Whether a stage is split depends on free task slots and on retries, so test with `ballista.scheduler.max_partitions_per_task` set low rather than relying on a small cluster to show the failure. +When a stage is split between tasks, each task gets a copy of the plan restricted to its own partitions, and operators see only task-local partition indices: a task that runs partitions 4 and 5 executes partitions 0 and 1 of every node. The scheduler restricts the leaves it knows, such as shuffle readers and file and in-memory scans. A node it doesn't know would read its per-partition data, such as one list of files for each partition, from the wrong index, and the stage would return some partitions twice and others not at all. The scheduler fails a job whose split stage holds such a leaf, unless a task runs all of it or none of it. It does not check interior nodes: it cannot tell one that holds per-partition data from one that does not, such as a `FilterExec`, so an interior node of yours that holds per-partition data returns wrong results in a split stage unless you register it. Whether a stage is split depends on free task slots and on retries, so test with `ballista.scheduler.max_partitions_per_task` set low rather than relying on a small cluster to show the failure. To make your node restrictable, give the scheduler a `PartitionSliceableResolver`. It is called with a node, its children already restricted, and the partitions the task runs, and returns the restricted node, or `None` for a node it doesn't know. A node of your own can implement `PartitionSliceable` to keep the slicing beside its fields: @@ -179,7 +179,7 @@ let config = SchedulerConfig::default().with_partition_sliceable_resolver(Arc::n - `children` are the node's inputs, none for a scan, restricted to `partitions`. An input the node declares `SinglePartition` arrives whole, as a join's build side does. Partition `i` of the node must read partition `i` of every other input. - The restricted node must have exactly `partitions.len()` partitions. The scheduler rejects any other count. -- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. +- `partitions` is empty for a `UnionExec` branch that belongs to other tasks. Return a node without partitions. A leaf is never asked for none of its partitions: the scheduler replaces it with an empty leaf itself. A node with inputs is still asked, with its inputs already restricted to none. If nothing restricts such a node and it keeps its own partition count, a task given part of the union's stage would also run partitions it doesn't own. The static planner gives such a branch a stage of its own. Under the adaptive planner the scheduler runs the union's stage as one task instead, and logs it at `info` level, naming the branch. - The restricted node is serialized with the scheduler's `physical codec`, so what it carries per partition has to survive encoding and decoding. - The resolver does not need `PartitionSliceable`. It can build the restricted node itself, which is the way to restrict a node you cannot implement the trait on, such as a `DataSourceExec` over a `DataSource` of your own: match it with `downcast_ref`, then build a `DataSourceExec` over the restricted source. - The resolver is asked after Ballista's own list. A node that is read whole, such as one below a `CoalescePartitionsExec` or on a join's build side, needs none of this. diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 82e2b64c14..7867ecb157 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -145,18 +145,6 @@ and executors must all set the same codecs in the same order. - **One task per scan.** Iceberg scans do not split into partitions yet, so one executor reads the whole table. Operators after the first shuffle still run in parallel. A scan reporting more than one partition fails to encode. -- **A `UNION` or `UNION ALL` can fail or return wrong rows when the adaptive - planner splits its stage between tasks.** That happens with - `ballista.scheduler.max_partitions_per_task` set, or when the executor has - fewer free task slots than the union has partitions. A union of Iceberg scans - fails with an error that suggests `SchedulerConfig::partition_sliceable_resolver`, - which standalone mode can't set; - [#2852](https://github.com/apache/datafusion-ballista/pull/2852) fixes it. A union - whose branches each merge their input into one partition, as with an - `ORDER BY` over the union, a `LIMIT` in a branch or aggregates without - `GROUP BY`, returns some rows more than once, whatever the table format - ([#2854](https://github.com/apache/datafusion-ballista/issues/2854)). Setting - `ballista.planner.adaptive.enabled` to `false` avoids both. - **No encrypted tables.** A table with `encryption.key-id` set fails to load on the scheduler and executors, which have no key management client. - **Storage credentials travel in plain text.** Each table's serialized