diff --git a/ballista/scheduler/src/cache.rs b/ballista/scheduler/src/cache.rs index dc109120f3..0c132631df 100644 --- a/ballista/scheduler/src/cache.rs +++ b/ballista/scheduler/src/cache.rs @@ -64,9 +64,9 @@ pub(crate) fn contains_cache(plan: &LogicalPlan) -> bool { .unwrap_or(false) } -/// In the failure of a job that lost cache data mid-query. Rebuild-and-resume -/// is future work (see the design doc); the caches the job read are dropped, -/// so a retry rebuilds them. +/// In the failure of a job that lost cache data mid-query and could not rebuild +/// it (see the design doc); the caches the job read are dropped, so a retry +/// rebuilds them. pub(crate) const CACHE_DATA_LOST: &str = "lost DataFrame::cache() data"; /// Replaces every `BallistaCacheNode` with its subplan, for when an operator diff --git a/ballista/scheduler/src/flight_sql.rs b/ballista/scheduler/src/flight_sql.rs index 2188565bbb..d997f616a7 100644 --- a/ballista/scheduler/src/flight_sql.rs +++ b/ballista/scheduler/src/flight_sql.rs @@ -52,7 +52,7 @@ impl QueryBackend } async fn close_session(&self, session_id: &str) -> Result<()> { - self.state.session_manager.remove_session(session_id).await + self.remove_session_and_caches(session_id).await } async fn execute( diff --git a/ballista/scheduler/src/scheduler_server/grpc.rs b/ballista/scheduler/src/scheduler_server/grpc.rs index 58ab461512..5ecd6581a6 100644 --- a/ballista/scheduler/src/scheduler_server/grpc.rs +++ b/ballista/scheduler/src/scheduler_server/grpc.rs @@ -444,9 +444,7 @@ impl SchedulerGrpc request: Request, ) -> Result, Status> { let session_params = request.into_inner(); - self.state - .session_manager - .remove_session(&session_params.session_id) + self.remove_session_and_caches(&session_params.session_id) .await .map_err(|e| { Status::internal(format!( diff --git a/ballista/scheduler/src/scheduler_server/mod.rs b/ballista/scheduler/src/scheduler_server/mod.rs index 1a79b8d877..6033ead3d7 100644 --- a/ballista/scheduler/src/scheduler_server/mod.rs +++ b/ballista/scheduler/src/scheduler_server/mod.rs @@ -349,6 +349,22 @@ impl SchedulerServer Result<()> { + match self.query_stage_event_loop.get_sender() { + Ok(sender) => self.state.remove_session(session_id, &sender).await, + Err(e) => { + self.state + .session_manager + .remove_session(session_id) + .await?; + Err(e) + } + } + } + /// Admits a job submission, or refuses it if the scheduler is shutting /// down. [`submit_plan`](Self::submit_plan) puts the [`Admission`] in the /// [`JobQueued`](QueryStageSchedulerEvent::JobQueued) event for the job, and diff --git a/ballista/scheduler/src/state/cache_linker.rs b/ballista/scheduler/src/state/cache_linker.rs index c9ad33864a..ce08990e10 100644 --- a/ballista/scheduler/src/state/cache_linker.rs +++ b/ballista/scheduler/src/state/cache_linker.rs @@ -98,6 +98,17 @@ enum CacheClaim { /// A concurrent submission can change a key between lookup and claim. const CLAIM_ATTEMPTS: usize = 4; +/// Why a cache build ends when its session is removed. +const SESSION_CLOSED: &str = "its session was closed"; + +/// Why a query whose session was removed while it ran fails when it loses a +/// cache: it is not rebuilt for a removed session. +fn session_closed_rebuild() -> String { + format!( + "{CACHE_DATA_LOST} after its session was closed, so it cannot be rebuilt; retry the query" + ) +} + /// What waits on a build job. #[derive(Debug, Clone, PartialEq, Eq, Hash)] enum Waiter { @@ -123,6 +134,9 @@ pub(crate) struct Released { pub(crate) struct JobReads { pub(crate) queued_at: u64, pub(crate) reads: Vec, + /// Its session was removed while it ran. A cache it then loses is not + /// rebuilt, as nothing would remove an entry made for a removed session. + pub(crate) session_closed: bool, } impl JobReads { @@ -141,9 +155,31 @@ struct Inner { dependents: HashMap>, /// Running job -> the caches it reads. reads: HashMap, + /// Builds dropped with their session while running jobs still read them; + /// their data is cleaned up once no running job does. + orphaned: HashSet, +} + +/// What a job's end leaves to do. +pub(crate) struct Ended { + /// The caches it read. + pub(crate) reads: Vec, + /// Whether it is still a build in flight, held, or waited on. + pub(crate) linked: bool, + /// Builds dropped with their session that no running job reads any more, + /// whose data can now be cleaned up. + pub(crate) unread: Vec, } impl Inner { + /// Whether a running job reads the output of `build`. + fn is_read(&self, build: &JobId) -> bool { + self.reads + .values() + .flat_map(|job_reads| &job_reads.reads) + .any(|read| &read.materializing_job == build) + } + /// Whether `job` still runs and reads `input`. fn still_reads(&self, job: &JobId, input: usize) -> bool { self.reads @@ -304,9 +340,9 @@ impl CacheLinker { } } - /// `job` ended: returns its cache reads, and whether it is still a build in - /// flight, held, or waited on. - pub(crate) fn job_ended(&self, job: &JobId) -> (Vec, bool) { + /// `job` ended: its cache reads, whether it is still linked, and the builds + /// dropped with their session that it was the last to read. + pub(crate) fn job_ended(&self, job: &JobId) -> Ended { let mut inner = self.inner.lock(); let reads = inner .reads @@ -316,7 +352,54 @@ impl CacheLinker { let linked = inner.inflight.contains_key(job) || inner.pending.contains_key(job) || inner.dependents.contains_key(job); - (reads, linked) + // Every orphan is checked, not just the builds `job` read: a read moves + // to a rebuild's output, so the build it read before is read by no one. + let orphaned = std::mem::take(&mut inner.orphaned); + let (still_read, unread): (HashSet, HashSet) = + orphaned.into_iter().partition(|build| inner.is_read(build)); + inner.orphaned = still_read; + Ended { + reads, + linked, + unread: unread.into_iter().collect(), + } + } + + /// Marks the running jobs reading a cache of `session_id`, which is being + /// removed, so a cache they lose from now on is not rebuilt. + pub(crate) fn close_session_reads(&self, session_id: &str) { + for job_reads in self.inner.lock().reads.values_mut() { + if job_reads + .reads + .iter() + .any(|read| read.key.session_id == session_id) + { + job_reads.session_closed = true; + } + } + } + + /// A build in flight for a cache of `session_id`, a held one first: failing + /// it fails what waits on it without cancelling a running job. + pub(crate) fn inflight_of(&self, session_id: &str) -> Option<(JobId, CacheKey)> { + let inner = self.inner.lock(); + inner + .inflight + .iter() + .filter(|(_, key)| key.session_id == session_id) + .min_by_key(|(job, _)| !inner.pending.contains_key(*job)) + .map(|(job, key)| (job.clone(), key.clone())) + } + + /// `builds` were dropped with their session: returns those no running job + /// reads, whose data can be cleaned up now, and keeps the rest until their + /// last reader ends. + pub(crate) fn orphan(&self, builds: Vec) -> Vec { + let mut inner = self.inner.lock(); + let (read, unread): (Vec, Vec) = + builds.into_iter().partition(|build| inner.is_read(build)); + inner.orphaned.extend(read); + unread } } @@ -582,6 +665,7 @@ impl SchedulerState SchedulerState SchedulerState, sender: &EventSender, ) { - let (reads, linked) = self.cache_linker.job_ended(job_id); + let Ended { + reads, + linked, + unread, + } = self.cache_linker.job_ended(job_id); + self.clean_up_builds(unread); if let JobEnd::Failed(reason) = end && reason.contains(CACHE_DATA_LOST) { @@ -694,6 +784,61 @@ impl SchedulerState, + ) -> Result<()> { + // First, so a query reading one of these caches that loses it from now + // on fails rather than rebuild it for the removed session. + self.cache_linker.close_session_reads(session_id); + // A build fails with what waits on it, held builds included, so look + // again each time rather than walk a list taken up front. + // + // A submission on this session between claiming a build (`track`) and + // holding it (`register`) has its claim given back and its build job + // cancelled before that job is queued. The submission then submits the + // build anyway, whose output is cleaned up as an unclaimed job's, and + // its query fails with a cache-miss error asking for a retry. A narrow + // race, left as is. + while let Some((build, key)) = self.cache_linker.inflight_of(session_id) { + match self.cache_linker.cancel(&build) { + Some(held) => self.fail_pending(&held, SESSION_CLOSED, sender).await, + None => { + self.fail_cache_job(&build, Some(&key), SESSION_CLOSED, sender) + .await; + let event = QueryStageSchedulerEvent::JobCancel(build.clone()); + if let Err(e) = sender.post_event(event).await { + warn!("failed to cancel cache build {build}: {e}"); + } + } + } + } + let removed = self.cache_state.remove_session(session_id).await?; + let builds = removed + .into_iter() + .map(|(_, cache)| cache.materializing_job) + .collect(); + self.clean_up_builds(self.cache_linker.orphan(builds)); + Ok(()) + } + + /// Cleans up the data of dropped `builds`, after the usual delay for a + /// finished job's data. + fn clean_up_builds(&self, builds: Vec) { + for build in builds { + self.executor_manager.clean_up_job_data_delayed( + build, + self.config.finished_job_data_clean_up_interval_seconds, + ); + } + } + /// A running job lost the cache data it reads under `inputs`, and its stages /// wait for it: rebuilds those caches and provides them again, or fails the /// job. On its own task, off the event loop. @@ -734,6 +879,9 @@ impl SchedulerState, ) -> std::result::Result<(), String> { + if job_reads.session_closed { + return Err(session_closed_rebuild()); + } // Every input is checked before any build is dropped or claimed, so a // job that cannot recover leaves the healthy ones as they are. let mut reads = Vec::with_capacity(inputs.len()); @@ -811,14 +959,26 @@ impl SchedulerState, ) { let read = self.cache_linker.reads_of(job_id).and_then(|job_reads| { - Some((job_reads.queued_at, job_reads.read(input)?.clone())) + let read = job_reads.read(input)?.clone(); + Some((job_reads.queued_at, job_reads.session_closed, read)) }); - let Some((queued_at, read)) = read else { + let Some((queued_at, session_closed, read)) = read else { self.fail_if_running(job_id, "it has no record of the cache it lost", sender) .await; return; }; let reason = match self.cache_state.lookup(&read.key).await { + // The session was removed while this rebuild ran: drop what it + // built rather than leave it to a session nothing will remove. + Ok(Some(cache)) if session_closed => { + if let Err(e) = self + .invalidate_build(&read.key, &cache.materializing_job) + .await + { + warn!("failed to drop cache {:?}: {e:?}", read.key); + } + session_closed_rebuild() + } Ok(Some(cache)) if cache.schema == read.written => { self.cache_linker.input_rebuilt( job_id, @@ -1030,9 +1190,11 @@ impl SchedulerState SchedulerState = + SchedulerState::new_with_default_scheduler_endpoint( + test_cluster_context(), + BallistaCodec::default(), + ); + let key = CacheKey::new("session", "c1"); + let build = JobId::new("build"); + state + .cache_state + .begin_materialization(key.clone(), build.clone()) + .await + .unwrap(); + state + .cache_state + .complete_materialization( + key.clone(), + build.clone(), + Arc::new(Schema::empty()), + vec![], + ) + .await + .unwrap(); + let reader = JobId::new("reader"); + let read = ResolvedRead { + key: key.clone(), + materializing_job: build.clone(), + input_id: EXTERNAL_STAGE_ID_BASE, + cache: None, + written: Arc::new(Schema::empty()), + }; + state.cache_linker.record_reads( + &reader, + JobReads { + queued_at: 0, + reads: vec![read], + session_closed: false, + }, + ); + + state.invalidate_build(&key, &build).await.unwrap(); + + assert!(state.cache_state.lookup(&key).await.unwrap().is_none()); + assert!( + state.cache_linker.inner.lock().orphaned.contains(&build), + "its data is kept for the reader" + ); + assert_eq!(state.cache_linker.job_ended(&reader).unread, vec![build]); + } + + /// Removing a session while its cache is being built cancels the build, and + /// the query waiting on it fails naming the closed session. + #[tokio::test] + async fn removing_a_session_fails_queries_waiting_on_its_builds() { + let (mut test, _metrics) = scheduler_test(None).await; + let plan = cache_over("c1", aggregation_logical_plan(2)); + let read = test.submit("query", &plan).await.unwrap(); + + test.remove_session("session").await.unwrap(); + + let error = failure(&finish(&mut test, &read).await); + assert!(error.contains("its session was closed"), "{error}"); + let key = CacheKey::new("session", "c1"); + assert!(test.cache_state().lookup(&key).await.unwrap().is_none()); + } + + /// Removing a session while its nested caches are being built: `c1`'s build + /// runs, and `c2`'s is held for it. Both are given up, the query waiting on + /// them fails naming the closed session, and neither cache is left. + #[tokio::test] + async fn removing_a_session_fails_queries_waiting_on_nested_builds() { + let (mut test, _metrics) = scheduler_test(None).await; + let plan = cache_over("c2", cache_over("c1", aggregation_logical_plan(2))); + let read = test.submit("query", &plan).await.unwrap(); + + test.remove_session("session").await.unwrap(); + + let error = failure(&finish(&mut test, &read).await); + assert!(error.contains("its session was closed"), "{error}"); + for cache_id in ["c1", "c2"] { + let key = CacheKey::new("session", cache_id); + assert!(test.cache_state().lookup(&key).await.unwrap().is_none()); + } + } + + /// Removing a session drops its built caches, but a query already reading + /// one finishes, and using the session again builds the cache anew. + #[tokio::test] + async fn removing_a_session_spares_running_reads_and_rebuilds_on_reuse() { + let (mut test, _metrics) = scheduler_test(None).await; + let plan = cache_over("c1", aggregation_logical_plan(2)); + let key = CacheKey::new("session", "c1"); + let read = test.submit("query", &plan).await.unwrap(); + let entry = build(&mut test, "c1").await; + wait_until_running(&test, &read).await; + + test.remove_session("session").await.unwrap(); + assert!(test.cache_state().lookup(&key).await.unwrap().is_none()); + assert!(is_successful(&finish(&mut test, &read).await)); + + let again = test.submit("again", &plan).await.unwrap(); + assert!(is_successful(&finish(&mut test, &again).await)); + let rebuilt = test.cache_state().lookup(&key).await.unwrap().unwrap(); + assert_ne!(rebuilt.materializing_job, entry.materializing_job); + } + + /// A query reading a cache when its session is removed, which then loses + /// that cache, fails rather than rebuild it: nothing would remove an entry + /// made for the removed session. A retry on the session rebuilds as usual. + #[tokio::test] + async fn removing_a_session_fails_a_later_rebuild_for_its_running_reads() { + let (mut test, _metrics) = scheduler_test(None).await; + let plan = cache_over("c1", aggregation_logical_plan(2)); + let read = test.submit("query", &plan).await.unwrap(); + let entry = build(&mut test, "c1").await; + wait_until_running(&test, &read).await; + + test.remove_session("session").await.unwrap(); + test.lose_executor(&holder_of(&entry)).await.unwrap(); + + let error = failure(&finish(&mut test, &read).await); + assert!( + error.contains(CACHE_DATA_LOST) && error.contains("session was closed"), + "{error}" + ); + let key = CacheKey::new("session", "c1"); + assert!(test.cache_state().lookup(&key).await.unwrap().is_none()); + } + /// Ticks until `cache_id`'s entry is recorded, without ticking past it: one /// tick too many would let the read job, dispatched once the entry is /// recorded, finish before the test acts. diff --git a/ballista/scheduler/src/state/mod.rs b/ballista/scheduler/src/state/mod.rs index 52622064c3..f4ee27d22a 100644 --- a/ballista/scheduler/src/state/mod.rs +++ b/ballista/scheduler/src/state/mod.rs @@ -640,6 +640,19 @@ impl SchedulerState, + ) -> Result<()> { + let caches = self.remove_session_caches(session_id, sender).await; + self.session_manager.remove_session(session_id).await?; + caches + } + /// Reclaim a finished job's shuffle data and schedule state cleanup. A job /// backing a cache (recorded by [`on_job_terminal`](Self::on_job_terminal) /// before this runs) keeps its final-stage data until the entry is diff --git a/ballista/scheduler/src/test_utils.rs b/ballista/scheduler/src/test_utils.rs index e34cc8ecc9..69ee460389 100644 --- a/ballista/scheduler/src/test_utils.rs +++ b/ballista/scheduler/src/test_utils.rs @@ -800,6 +800,11 @@ impl SchedulerTest { .await } + /// Removes `session_id`, as a client's `RemoveSession` call does. + pub async fn remove_session(&self, session_id: &str) -> Result<()> { + self.scheduler.remove_session_and_caches(session_id).await + } + /// Simulates the loss of an executor. This mirrors the reaper's /// `remove_executor` path without waiting out the heartbeat timeout. pub async fn lose_executor(&self, executor_id: &str) -> Result<()> { diff --git a/docs/source/contributors-guide/dataframe-cache.md b/docs/source/contributors-guide/dataframe-cache.md index d45e09e57b..0df09868ec 100644 --- a/docs/source/contributors-guide/dataframe-cache.md +++ b/docs/source/contributors-guide/dataframe-cache.md @@ -223,15 +223,25 @@ pin. **Validate-on-read** enforces "never read from a dead executor": whenever a cache is resolved into a read (when a query is submitted, and again before a held query runs), a location on an executor that is not live turns the hit into a miss, and the dead entry is -dropped. A dropped entry's build data is deleted after the usual delay for a -finished job's data (`finished_job_data_clean_up_interval_seconds`), which lets -a query still reading it finish. Invalidation is **whole-entry** — a cache is rebuilt in full, never +dropped. A dropped entry's build data is deleted once no running query reads it, after +the usual delay for a finished job's data +(`finished_job_data_clean_up_interval_seconds`). Invalidation is **whole-entry** — a cache is rebuilt in full, never served with missing partitions (see [Per-partition recovery](#per-partition-recovery)). -`invalidate_executor` (on executor loss) and `remove_session` (on session -teardown) exist in the registry but are not called yet; until then a dead entry -is dropped when it is next read, or when a query loses its data. +**Removing a session** (the scheduler's `RemoveSession` gRPC call, or a Flight +SQL session expiring) +drops its caches. A built cache is gone at once for new queries, and its data is +deleted once no running query reads it, after the same delay. A build still in +flight is cancelled, and the queries waiting on it fail with "its session was +closed". A query still reading one of its caches keeps reading it, but if it +then loses that data it fails with "lost DataFrame::cache() data after its +session was closed", rather than rebuild a cache for a removed session that +nothing would remove. Using the session again builds its caches anew. + +`invalidate_executor` (on executor loss) exists in the registry but is not +called yet; until then a dead entry is dropped when it is next read, or when a +query loses its data. ## Planning & configuration @@ -280,9 +290,10 @@ is dropped when it is next read, or when a query loses its data. ## Future work -- **Eviction:** entries leave today only when found dead, so a long-running - scheduler accumulates cache data. Planned: a TTL / size-cap sweeper and - idle-session expiry, using the removal-returns-entries contract. +- **Eviction:** entries leave today only when found dead or with their + session, and no client library removes its session yet, so a long-running + scheduler accumulates cache data. Planned: idle-session expiry and a TTL / + size-cap sweeper, using the removal-returns-entries contract. - **Locality:** place read tasks on the executor holding each partition (`TaskDistributionPolicy::Custom`), Flight fetch as fallback (see [#2319]). diff --git a/docs/source/user-guide/tuning-guide.md b/docs/source/user-guide/tuning-guide.md index 2fba2ae074..310376b825 100644 --- a/docs/source/user-guide/tuning-guide.md +++ b/docs/source/user-guide/tuning-guide.md @@ -460,8 +460,11 @@ cached.collect().await?; // reads it a cache it was built from, another way (a self-join of a cached frame, say), fails instead with "lost DataFrame::cache() data; retry the query", and the retry rebuilds the cache. -- Cached data is not evicted yet: it is kept until it is found lost, so cache - only what you reuse. +- Cached data is not evicted yet: it is kept until it is found lost, or its + session is removed through the scheduler's `RemoveSession` gRPC call, which + the Rust and Python clients do not expose yet. So cache only what you reuse. + Each `cache()` call caches its DataFrame anew; an earlier call's cache is not + freed. - `EXPLAIN` of a cached DataFrame never builds it. - An operator can turn caching off for the whole cluster with the scheduler's `--disable-dataframe-cache` flag; cached DataFrames are then computed as part