From b9fa08c95d652553872a304bc95de5c70f254ec2 Mon Sep 17 00:00:00 2001 From: vikrantmehta123 Date: Wed, 18 Nov 2026 00:00:26 +0530 Subject: [PATCH 1/2] fix: forward session partition slicing resolver to standalone scheduler --- ballista/client/tests/context_setup.rs | 41 ++++++++++++++++++++- ballista/core/src/extension.rs | 50 ++++++++++++++++++++++++++ ballista/scheduler/src/standalone.rs | 8 ++++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/ballista/client/tests/context_setup.rs b/ballista/client/tests/context_setup.rs index 832c7b5b91..e9cff67bbd 100644 --- a/ballista/client/tests/context_setup.rs +++ b/ballista/client/tests/context_setup.rs @@ -96,9 +96,10 @@ mod standalone { use std::sync::{Arc, atomic::AtomicBool}; - use crate::common::{register_alltypes, state_with_config}; + use crate::common::{ALLTYPES_GROUP_BY_SQL, register_alltypes, state_with_config}; use crate::{assert_job_name_setting, job_name_state}; use ballista::extension::{SessionConfigExt, SessionContextExt}; + use ballista_core::execution_plans::plan_algebra::PartitionSliceableResolver; use ballista_core::serde::BallistaPhysicalExtensionCodec; use datafusion::{ catalog::Session, @@ -184,6 +185,26 @@ mod standalone { Ok(()) } + #[tokio::test] + async fn should_set_partition_sliceable_resolver() -> datafusion::error::Result<()> { + let resolver = Arc::new(MockPartitionSliceableResolver::default()); + let session_config = SessionConfig::new_with_ballista() + .with_ballista_partition_sliceable_resolver(resolver.clone()); + + let state = state_with_config(session_config); + let ctx = SessionContext::standalone_with_state(state).await?; + + register_alltypes(&ctx, "test").await; + + // Execute a query to verify that the standalone scheduler uses the + // resolver registered in the session configuration. + ctx.sql(ALLTYPES_GROUP_BY_SQL).await?.collect().await?; + + assert!(resolver.invoked.load(std::sync::atomic::Ordering::Relaxed)); + + Ok(()) + } + // check #[tokio::test] async fn should_override_planner() -> datafusion::error::Result<()> { @@ -330,6 +351,24 @@ mod standalone { } } + #[derive(Debug, Default)] + struct MockPartitionSliceableResolver { + invoked: AtomicBool, + } + + impl PartitionSliceableResolver for MockPartitionSliceableResolver { + fn restrict_to_partitions( + &self, + _plan: &Arc, + _children: Vec>, + _partitions: &[usize], + ) -> Option>> { + self.invoked + .store(true, std::sync::atomic::Ordering::Relaxed); + None + } + } + #[derive(Debug, Default)] struct BadPlanner {} diff --git a/ballista/core/src/extension.rs b/ballista/core/src/extension.rs index 00360bd80c..4d4a65020b 100644 --- a/ballista/core/src/extension.rs +++ b/ballista/core/src/extension.rs @@ -30,6 +30,7 @@ use crate::config::{ BALLISTA_SKEW_JOIN_SMALL_PARTITION_FACTOR, BALLISTA_STANDALONE_PARALLELISM, BallistaConfig, ExecutionMode, }; +use crate::execution_plans::plan_algebra::PartitionSliceableResolver; use crate::planner::BallistaQueryPlanner; use crate::serde::protobuf::KeyValuePair; use crate::serde::{BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec}; @@ -123,6 +124,16 @@ pub trait SessionConfigExt { codec: Arc, ) -> SessionConfig; + /// Sets how custom plan nodes are restricted to a task's partitions. + /// + /// Used when creating a standalone scheduler from SessionConfig. Setting + /// this when not in standalone mode has no effect. Configure the remote scheduler with + /// `SchedulerConfig::with_partition_sliceable_resolver` instead. + fn with_ballista_partition_sliceable_resolver( + self, + resolver: Arc, + ) -> SessionConfig; + /// returns [LogicalExtensionCodec] if set /// or default ballista codec if not fn ballista_logical_extension_codec(&self) -> Arc; @@ -131,6 +142,13 @@ pub trait SessionConfigExt { /// or default ballista codec if not fn ballista_physical_extension_codec(&self) -> Arc; + /// Returns the resolver for restricting custom plan nodes to a task's partitions. + /// + /// Returns `None` if not set. + fn ballista_partition_sliceable_resolver( + &self, + ) -> Option>; + /// Overrides ballista's [QueryPlanner] fn with_ballista_query_planner( self, @@ -530,6 +548,7 @@ impl SessionConfigExt for SessionConfig { let extension = BallistaConfigExtensionLogicalCodec::new(codec); self.with_extension(Arc::new(extension)) } + fn with_ballista_physical_extension_codec( self, codec: Arc, @@ -538,6 +557,14 @@ impl SessionConfigExt for SessionConfig { self.with_extension(Arc::new(extension)) } + fn with_ballista_partition_sliceable_resolver( + self, + resolver: Arc, + ) -> SessionConfig { + let extension = BallistaConfigExtensionPartitionSliceableResolver::new(resolver); + self.with_extension(Arc::new(extension)) + } + fn ballista_logical_extension_codec(&self) -> Arc { self.get_extension::() .map(|c| c.codec()) @@ -549,6 +576,13 @@ impl SessionConfigExt for SessionConfig { .unwrap_or_else(|| Arc::new(BallistaPhysicalExtensionCodec::default())) } + fn ballista_partition_sliceable_resolver( + &self, + ) -> Option> { + self.get_extension::() + .map(|extension| extension.resolver()) + } + fn with_ballista_query_planner( self, planner: Arc, @@ -881,6 +915,22 @@ impl BallistaConfigExtensionPhysicalCodec { } } +/// Wrapper for [SessionConfig] extension +/// holding [PartitionSliceableResolver] if set +struct BallistaConfigExtensionPartitionSliceableResolver { + resolver: Arc, +} + +impl BallistaConfigExtensionPartitionSliceableResolver { + fn new(resolver: Arc) -> Self { + Self { resolver } + } + + fn resolver(&self) -> Arc { + self.resolver.clone() + } +} + /// Wrapper for [SessionConfig] extension /// holding overridden [QueryPlanner] struct BallistaQueryPlannerExtension { diff --git a/ballista/scheduler/src/standalone.rs b/ballista/scheduler/src/standalone.rs index 23c74d1462..962ef6421f 100644 --- a/ballista/scheduler/src/standalone.rs +++ b/ballista/scheduler/src/standalone.rs @@ -102,12 +102,18 @@ pub async fn new_standalone_scheduler_with_builder( let metrics_collector = default_metrics_collector()?; + let mut scheduler_config = SchedulerConfig::default().with_scheduler_policy(policy); + + if let Some(resolver) = config.ballista_partition_sliceable_resolver() { + scheduler_config = scheduler_config.with_partition_sliceable_resolver(resolver); + } + let mut scheduler_server: SchedulerServer = SchedulerServer::new( endpoint, cluster, codec, - Arc::new(SchedulerConfig::default().with_scheduler_policy(policy)), + Arc::new(scheduler_config), metrics_collector, ); From cd14ab283b977fab3d0338bf7325038c4a6a9655 Mon Sep 17 00:00:00 2001 From: vikrantmehta123 Date: Wed, 18 Nov 2026 00:20:18 +0530 Subject: [PATCH 2/2] docs: document standalone partition slicing resolver configuration --- docs/source/upgrading/56.0.0.md | 8 ++++++++ docs/source/user-guide/extending-components.md | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/upgrading/56.0.0.md b/docs/source/upgrading/56.0.0.md index 624a625670..3a65ec9642 100644 --- a/docs/source/upgrading/56.0.0.md +++ b/docs/source/upgrading/56.0.0.md @@ -285,6 +285,14 @@ nodes that cannot implement `PartitionSliceable`, such as a `DataSourceExec`. literal that doesn't end in `..Default::default()` needs to set it. - `task_builder::restrict_plan_to_partitions` takes a third argument, `Option<&dyn PartitionSliceableResolver>`. Pass `None` for none. +- `SessionConfigExt` gains two required methods: + `with_ballista_partition_sliceable_resolver` and + `ballista_partition_sliceable_resolver`. Downstream implementations of this + trait must implement both methods. Existing callers need no changes. + +For standalone execution, register the resolver in the session configuration +with `with_ballista_partition_sliceable_resolver`, then pass the session state +to `SessionContext::standalone_with_state`. #### Partition locations hold an `ExecutorEndpoint` diff --git a/docs/source/user-guide/extending-components.md b/docs/source/user-guide/extending-components.md index 2228bf3425..78dd5c2282 100644 --- a/docs/source/user-guide/extending-components.md +++ b/docs/source/user-guide/extending-components.md @@ -183,4 +183,4 @@ let config = SchedulerConfig::default().with_partition_sliceable_resolver(Arc::n - The restricted node is serialized with the scheduler's `physical codec`, so what it carries per partition has to survive encoding and decoding. - The resolver does not need `PartitionSliceable`. It can build the restricted node itself, which is the way to restrict a node you cannot implement the trait on, such as a `DataSourceExec` over a `DataSource` of your own: match it with `downcast_ref`, then build a `DataSourceExec` over the restricted source. - The resolver is asked after Ballista's own list. A node that is read whole, such as one below a `CoalescePartitionsExec` or on a join's build side, needs none of this. -- The standalone scheduler (`SessionContext::standalone`) builds its own `SchedulerConfig`, so it cannot be given a resolver. +- The standalone scheduler reads the resolver from the session configuration. Register it with `with_ballista_partition_sliceable_resolver`, then use that configuration when building the session state passed to `SessionContext::standalone_with_state`.