From 9554a90b6be429082448ee7a53f12b29e6374496 Mon Sep 17 00:00:00 2001 From: Shahanur Islam Shagor Date: Tue, 20 Oct 2026 12:58:21 -0400 Subject: [PATCH 1/7] Define hardware-bounded Eco, Balanced, Performance and Maximum profiles --- src-tauri/src/performance.rs | 109 ++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/src-tauri/src/performance.rs b/src-tauri/src/performance.rs index d07225a..13f2055 100644 --- a/src-tauri/src/performance.rs +++ b/src-tauri/src/performance.rs @@ -2,9 +2,8 @@ use serde::Serialize; use crate::hardware::{BackendKind, HardwareProfile}; -#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -#[allow(dead_code)] pub enum PerformanceMode { Auto, Eco, @@ -13,6 +12,19 @@ pub enum PerformanceMode { Maximum, } +impl PerformanceMode { + pub fn from_preference(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "auto" => Some(Self::Auto), + "eco" => Some(Self::Eco), + "balanced" => Some(Self::Balanced), + "performance" => Some(Self::Performance), + "maximum" => Some(Self::Maximum), + _ => None, + } + } +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct PerformanceProfile { @@ -67,6 +79,41 @@ impl PerformanceProfileManager { flash_attention, } } + + pub fn select(hardware: &HardwareProfile, selection: &str) -> PerformanceProfile { + let mut profile = Self::auto(hardware); + let mode = PerformanceMode::from_preference(selection).unwrap_or(PerformanceMode::Auto); + let logical = hardware.cpu.logical_threads.max(1); + let baseline_threads = profile.cpu_threads.max(1); + profile.mode = mode; + + match mode { + PerformanceMode::Auto => {} + PerformanceMode::Eco => { + profile.cpu_threads = baseline_threads.div_ceil(2).max(1); + profile.system_memory_budget_bytes = + profile.system_memory_budget_bytes.saturating_mul(3) / 4; + profile.vram_budget_bytes = + profile.vram_budget_bytes.map(|bytes| bytes.saturating_mul(3) / 4); + profile.flash_attention = false; + } + PerformanceMode::Balanced => { + profile.cpu_threads = (baseline_threads.saturating_mul(3) / 4).max(1); + profile.system_memory_budget_bytes = + profile.system_memory_budget_bytes.saturating_mul(9) / 10; + profile.vram_budget_bytes = + profile.vram_budget_bytes.map(|bytes| bytes.saturating_mul(9) / 10); + } + PerformanceMode::Performance => { + profile.cpu_threads = baseline_threads.saturating_add(2).min(logical); + } + PerformanceMode::Maximum => { + profile.cpu_threads = logical; + } + } + profile.cpu_threads = profile.cpu_threads.clamp(1, logical); + profile + } } fn reserve_for_system(total_memory: u64) -> u64 { @@ -85,6 +132,64 @@ mod tests { use super::*; use crate::hardware::{BackendProfile, CpuProfile, GpuInfo, GpuVendor, MemoryProfile}; + #[test] + fn profiles_are_distinct_and_bounded_on_cpu_and_gpu() { + let mut hardware = HardwareProfile { + operating_system: "test".into(), + architecture: "x86_64".into(), + cpu: CpuProfile { name: "cpu".into(), physical_cores: Some(6), logical_threads: 12 }, + memory: MemoryProfile { + total_bytes: 16 * 1024 * 1024 * 1024, + available_bytes: 10 * 1024 * 1024 * 1024, + }, + gpus: Vec::new(), + primary_gpu: None, + recommended_inference_gpu: None, + backends: BackendProfile { + cpu: true, cuda: false, vulkan: false, sycl: false, hip: false, metal: false, + }, + detection_complete: true, + }; + for has_gpu in [false, true] { + if has_gpu { + hardware.gpus.push(GpuInfo { + id: "gpu0".into(), name: "gpu".into(), vendor: GpuVendor::Nvidia, + vendor_id: None, device_id: None, subsystem_id: None, revision: None, + dedicated_vram_bytes: Some(8 * 1024 * 1024 * 1024), + dedicated_system_memory_bytes: None, shared_memory_bytes: None, + luid: None, is_discrete: true, is_integrated: false, is_software: false, + available_backends: vec![BackendKind::Cpu, BackendKind::Cuda], + recommended_backend: BackendKind::Cuda, + }); + hardware.recommended_inference_gpu = Some("gpu0".into()); + } + let auto = PerformanceProfileManager::select(&hardware, "Auto"); + let eco = PerformanceProfileManager::select(&hardware, "Eco"); + let balanced = PerformanceProfileManager::select(&hardware, "Balanced"); + let performance = PerformanceProfileManager::select(&hardware, "Performance"); + let maximum = PerformanceProfileManager::select(&hardware, "Maximum"); + assert_eq!(auto.mode, PerformanceMode::Auto); + assert_eq!(eco.mode, PerformanceMode::Eco); + assert_eq!(balanced.mode, PerformanceMode::Balanced); + assert_eq!(performance.mode, PerformanceMode::Performance); + assert_eq!(maximum.mode, PerformanceMode::Maximum); + assert!(eco.cpu_threads < balanced.cpu_threads); + assert!(balanced.cpu_threads <= auto.cpu_threads); + assert!(auto.cpu_threads <= performance.cpu_threads); + assert_eq!(maximum.cpu_threads, hardware.cpu.logical_threads); + assert!(eco.system_memory_budget_bytes < auto.system_memory_budget_bytes); + assert!(!eco.flash_attention); + if has_gpu { + assert!(eco.vram_budget_bytes.unwrap() < auto.vram_budget_bytes.unwrap()); + } else { + assert!(auto.vram_budget_bytes.is_none()); + assert_eq!(auto.recommended_backend, BackendKind::Cpu); + } + } + assert!(PerformanceMode::from_preference("Unknown").is_none()); + assert_eq!(PerformanceMode::from_preference(" eco "), Some(PerformanceMode::Eco)); + } + #[test] fn auto_keeps_memory_reserve() { let hardware = HardwareProfile { From 770397d5840b2a2e0805fcaf0984afa75850284d Mon Sep 17 00:00:00 2001 From: Shahanur Islam Shagor Date: Tue, 20 Oct 2026 12:58:56 -0400 Subject: [PATCH 2/7] Apply selected profile to launch threads, offload, context and batch budgets --- src-tauri/src/launch_planner.rs | 101 ++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 11 deletions(-) diff --git a/src-tauri/src/launch_planner.rs b/src-tauri/src/launch_planner.rs index 3554688..672cbca 100644 --- a/src-tauri/src/launch_planner.rs +++ b/src-tauri/src/launch_planner.rs @@ -3,7 +3,7 @@ use serde::Serialize; use crate::{ hardware::{BackendKind, HardwareProfile, HardwareProfiler}, model_registry::ModelRecord, - performance::{PerformanceProfile, PerformanceProfileManager}, + performance::{PerformanceMode, PerformanceProfile, PerformanceProfileManager}, }; const ACCELERATOR_FULL_OFFLOAD_HEADROOM_BYTES: u64 = 1024 * 1024 * 1024; @@ -42,13 +42,19 @@ pub struct ModelLaunchPlanner; impl ModelLaunchPlanner { pub fn plan(model: &ModelRecord, hardware: &HardwareProfile, port: u16) -> LaunchPlan { - // Startup intentionally keeps hardware discovery off the window-creation - // path. A model launch cannot use that placeholder, though: doing so - // would permanently start this model in CPU mode for the session. Pay - // the real GPU profile cost here only if the background scan has not - // already finished. + Self::plan_for_preference(model, hardware, "Auto", port) + } + + pub fn plan_for_preference( + model: &ModelRecord, + hardware: &HardwareProfile, + selection: &str, + port: u16, + ) -> LaunchPlan { + // Inference must use the completed hardware profile, not the fast + // startup placeholder that may have no GPU or RAM details. let inference_hardware = HardwareProfiler::for_inference(hardware); - let profile = PerformanceProfileManager::auto(&inference_hardware); + let profile = PerformanceProfileManager::select(&inference_hardware, selection); Self::plan_with_profile(model, &inference_hardware, &profile, port) } @@ -77,12 +83,21 @@ impl ModelLaunchPlanner { .unwrap_or(BackendKind::Cpu); let dedicated_vram_budget_bytes = selected_gpu .and_then(|gpu| gpu.dedicated_vram_bytes) - .map(safe_dedicated_vram_budget); + .map(safe_dedicated_vram_budget) + .map(|budget| budget.min(profile.vram_budget_bytes.unwrap_or(budget))); let context_size = choose_context_window( model.context_length.unwrap_or(8_192).max(4_096) as u64, model.size_bytes.max(0) as u64, - hardware.memory.available_bytes, + hardware + .memory + .available_bytes + .min(profile.system_memory_budget_bytes), ); + let context_size = match profile.mode { + PerformanceMode::Eco => context_size.min(8_192), + PerformanceMode::Balanced => context_size.min(16_384), + _ => context_size, + }; let estimated_model_bytes = model.size_bytes.max(0) as u64; let estimated_context_bytes = estimate_context_bytes(context_size); let total_estimate = estimated_model_bytes @@ -130,8 +145,29 @@ impl ModelLaunchPlanner { (BackendKind::Vulkan | BackendKind::Sycl, Some(_)) => 12, _ => 0, }; + let gpu_layers = match profile.mode { + PerformanceMode::Eco => gpu_layers.min(12), + PerformanceMode::Balanced => gpu_layers.min(32), + _ => gpu_layers, + }; + let (batch_size, ubatch_size) = match profile.mode { + PerformanceMode::Eco => (128, 64), + PerformanceMode::Balanced => (256, 128), + PerformanceMode::Auto | PerformanceMode::Performance => (512, 256), + PerformanceMode::Maximum => (1_024, 512), + }; let mut notes = Vec::new(); + notes.push(format!( + "Effective performance profile: {:?}; {} CPU threads, {} GPU layers.", + profile.mode, profile.cpu_threads, gpu_layers, + )); + if backend == BackendKind::Cpu { + notes.push( + "No supported GPU runtime was selected; this profile uses CPU inference." + .to_string(), + ); + } notes.push(format!( "Context window selected at {} tokens from model capabilities and available memory.", context_size @@ -161,8 +197,8 @@ impl ModelLaunchPlanner { gpu_layers, context_size, threads: profile.cpu_threads, - batch_size: 512, - ubatch_size: 256, + batch_size, + ubatch_size, flash_attention: profile.flash_attention, mmap: profile.mmap, mlock: false, @@ -232,6 +268,49 @@ mod tests { assert_eq!(choose_context_window(32_768, model_bytes, 2 * gib), 4_096); } + #[test] + fn selected_modes_change_cpu_launch_plan_without_exceeding_hardware() { + let mut hardware = hardware_with_vram(0, 0); + hardware.gpus.clear(); + hardware.recommended_inference_gpu = None; + let model = model(2_497_280_256); + let eco = ModelLaunchPlanner::plan_for_preference(&model, &hardware, "Eco", 8080); + let balanced = ModelLaunchPlanner::plan_for_preference(&model, &hardware, "Balanced", 8080); + let performance = ModelLaunchPlanner::plan_for_preference(&model, &hardware, "Performance", 8080); + let maximum = ModelLaunchPlanner::plan_for_preference(&model, &hardware, "Maximum", 8080); + for plan in [&eco, &balanced, &performance, &maximum] { + assert_eq!(plan.config.backend, BackendKind::Cpu); + assert_eq!(plan.config.gpu_layers, 0); + assert!(plan.config.threads <= hardware.cpu.logical_threads); + assert!(plan.notes.iter().any(|note| note.contains("CPU inference"))); + } + assert!(eco.config.threads < balanced.config.threads); + assert!(balanced.config.threads < performance.config.threads); + assert_eq!(maximum.config.threads, hardware.cpu.logical_threads); + assert_eq!(eco.config.batch_size, 128); + assert_eq!(maximum.config.batch_size, 1_024); + assert!(eco.config.context_size <= 8_192); + } + + #[test] + fn gpu_profiles_enforce_safe_offload_limits_and_context_fallback() { + let mut hardware = hardware_with_vram(8 * 1024 * 1024 * 1024, 0); + let model = model(2_497_280_256); + let eco = ModelLaunchPlanner::plan_for_preference(&model, &hardware, "Eco", 8080); + let auto = ModelLaunchPlanner::plan_for_preference(&model, &hardware, "Auto", 8080); + let maximum = ModelLaunchPlanner::plan_for_preference(&model, &hardware, "Maximum", 8080); + assert_eq!(eco.config.backend, BackendKind::Vulkan); + assert!(eco.config.gpu_layers <= 12); + assert!(eco.config.threads < auto.config.threads); + assert!(maximum.config.threads > auto.config.threads); + assert!(maximum.config.context_size >= eco.config.context_size); + assert!(eco.dedicated_vram_budget_bytes.unwrap() + < auto.dedicated_vram_budget_bytes.unwrap()); + hardware.memory.available_bytes = 4 * 1024 * 1024 * 1024; + let low_memory = ModelLaunchPlanner::plan_for_preference(&model, &hardware, "Maximum", 8080); + assert_eq!(low_memory.config.context_size, 4_096); + } + #[test] fn preserves_display_headroom_on_8gb_vulkan_gpu() { let hardware = hardware_with_vram(8 * 1024 * 1024 * 1024, 32 * 1024 * 1024 * 1024); From a15d1ccc95437e291a72c8dc29587fcbff50f008 Mon Sep 17 00:00:00 2001 From: Shahanur Islam Shagor Date: Tue, 20 Oct 2026 12:59:31 -0400 Subject: [PATCH 3/7] Honor persisted performance profile in preview, activation and chat runtime --- src-tauri/src/lib_legacy.rs | 40 ++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src-tauri/src/lib_legacy.rs b/src-tauri/src/lib_legacy.rs index 70326db..2c1a390 100644 --- a/src-tauri/src/lib_legacy.rs +++ b/src-tauri/src/lib_legacy.rs @@ -51,7 +51,7 @@ use maintenance::{BackupInfo, DiagnosticReport, RepairSummary}; use model_catalog::ModelCatalogReport; use model_download::{DownloadStatus, ModelDownloadManager}; use model_registry::{ModelLifecycleState, ModelRecord, ModelRegistry}; -use performance::{PerformanceProfile, PerformanceProfileManager}; +use performance::{PerformanceMode, PerformanceProfile, PerformanceProfileManager}; use portable_root::{ available_bytes_for_path, load_installation, preview_writable, sanitize_profile_name, save_installation, InstallConfig, InstallationStatus, PortableRootInfo, PortableRootManager, @@ -1147,13 +1147,25 @@ async fn detect_hardware( Ok(HardwareProfiler::for_inference(&state.hardware)) } +fn selected_performance_preference(state: &AppState) -> Result { + let db = state + .database + .lock() + .map_err(|_| app_error::AppError::internal("database lock poisoned"))?; + Ok(SettingsRepository::new(&db) + .get_preferences()? + .default_performance_profile) +} + #[tauri::command] async fn get_performance_profile( state: State<'_, AppState>, ) -> Result { - Ok(PerformanceProfileManager::auto(&HardwareProfiler::for_inference( - &state.hardware, - ))) + let selection = selected_performance_preference(&state)?; + Ok(PerformanceProfileManager::select( + &HardwareProfiler::for_inference(&state.hardware), + &selection, + )) } #[tauri::command] @@ -1274,8 +1286,13 @@ async fn plan_model_launch( .lock() .map_err(|_| app_error::AppError::internal("database lock poisoned"))?; let model = ModelRegistry::new(&db, &state.root).validate_model(&model_id)?; + let selection = SettingsRepository::new(&db) + .get_preferences()? + .default_performance_profile; let port = allocate_local_port()?; - Ok(ModelLaunchPlanner::plan(&model, &hardware, port)) + Ok(ModelLaunchPlanner::plan_for_preference( + &model, &hardware, &selection, port, + )) } #[tauri::command] @@ -1292,7 +1309,10 @@ async fn activate_model( ModelRegistry::new(&db, &state.root).validate_model(&model_id)? }; let hardware = HardwareProfiler::for_inference(&state.hardware); - let plan = ModelLaunchPlanner::plan(&model, &hardware, allocate_local_port()?); + let selection = selected_performance_preference(&state)?; + let plan = ModelLaunchPlanner::plan_for_preference( + &model, &hardware, &selection, allocate_local_port()?, + ); let (status, _lease) = runtime::ensure_model_ready(&state.runtime, &hardware, &plan.config)?; state.warm_start.mark_runtime_ready(&model.id); @@ -1654,7 +1674,10 @@ async fn run_streaming_completion( _ = state.warm_start.wait_if_loading(&model.id) => {} } let hardware = HardwareProfiler::for_inference(&state.hardware); - let plan = ModelLaunchPlanner::plan(model, &hardware, allocate_local_port()?); + let selection = selected_performance_preference(state)?; + let plan = ModelLaunchPlanner::plan_for_preference( + model, &hardware, &selection, allocate_local_port()?, + ); if let Some(replacing) = model_switch_needed(state, &plan.config.model_path)? { let _ = app.emit( "runtime:model-loading", @@ -2119,6 +2142,9 @@ fn save_app_preferences( if !web_provider_config::valid_preference(&preferences.web_search_preferred_provider) { return Err(app_error::AppError::internal("invalid preferred web search provider")); } + if PerformanceMode::from_preference(&preferences.default_performance_profile).is_none() { + return Err(app_error::AppError::internal("invalid performance profile")); + } let saved = SettingsRepository::new(&db).save_preferences(&preferences)?; net::apply_proxy_settings(&saved.network_proxy_mode, &saved.network_proxy_url); Ok(saved) From d3e276c2f08f4ba7fce6f041154aa9ea884561ff Mon Sep 17 00:00:00 2001 From: Shahanur Islam Shagor Date: Tue, 20 Oct 2026 12:59:48 -0400 Subject: [PATCH 4/7] Refresh effective performance settings immediately after saving --- src/App.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index 91ccba3..22b3b2b 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -980,8 +980,9 @@ export function App() { } async function updatePreferences(next: AppPreferences) { - setPreferences(next); - setPreferences(await api.savePreferences(next)); + const saved = await api.savePreferences(next); + setPreferences(saved); + setPerformance(await api.performance()); } async function updateUserProfile(next: UserProfile) { From 5b588b42242c8374232b6e065b67a012ee6baf66 Mon Sep 17 00:00:00 2001 From: Shahanur Islam Shagor Date: Tue, 20 Oct 2026 12:59:51 -0400 Subject: [PATCH 5/7] Display effective profile and CPU allocation in Advanced Settings --- src/components/SettingsDialog.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/components/SettingsDialog.tsx b/src/components/SettingsDialog.tsx index 6b7bc4f..ea9d525 100644 --- a/src/components/SettingsDialog.tsx +++ b/src/components/SettingsDialog.tsx @@ -720,7 +720,14 @@ export function SettingsDialog(props: { checked={preferenceDraft.localRuntimeAutostart} onChange={(value) => setPreference("localRuntimeAutostart", value)} /> + + Date: Tue, 20 Oct 2026 13:02:47 -0400 Subject: [PATCH 6/7] Restart loaded model when effective performance options change --- src-tauri/src/runtime.rs | 49 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src-tauri/src/runtime.rs b/src-tauri/src/runtime.rs index 039f3ef..dd79bcc 100644 --- a/src-tauri/src/runtime.rs +++ b/src-tauri/src/runtime.rs @@ -171,6 +171,7 @@ pub struct LlamaRuntimeManager { state: ServerState, loaded_model_path: Option, loaded_context_size: Option, + loaded_config: Option, active_runtime: Option, load_started: Option, leases: Arc, @@ -186,6 +187,7 @@ impl LlamaRuntimeManager { state: ServerState::Stopped, loaded_model_path: None, loaded_context_size: None, + loaded_config: None, active_runtime: None, load_started: None, leases: Arc::new(AtomicUsize::new(0)), @@ -353,7 +355,11 @@ impl LlamaRuntimeManager { // the new memory-aware launch plan. Available RAM can change after // startup, and the request budget must not assume an unallocated window. let same_context = self.loaded_context_size == Some(config.context_size); - if child_alive && same_model && same_backend && same_context { + let same_settings = self + .loaded_config + .as_ref() + .is_some_and(|loaded| same_model_launch_settings(loaded, config)); + if child_alive && same_model && same_backend && same_context && same_settings { // True hot-chat path: reuse the resident model and its KV prompt // cache. No runtime probing, process restart or model reload. if self.state == ServerState::Ready { @@ -468,6 +474,7 @@ impl LlamaRuntimeManager { self.endpoint = Some(format!("http://{}:{}", config.host, config.port)); self.loaded_model_path = Some(model_path_string); self.loaded_context_size = Some(config.context_size); + self.loaded_config = Some(config.clone()); self.active_runtime = Some(selected); self.load_started = Some(Instant::now()); self.leases = Arc::new(AtomicUsize::new(0)); @@ -579,6 +586,7 @@ impl LlamaRuntimeManager { self.endpoint = None; self.loaded_model_path = None; self.loaded_context_size = None; + self.loaded_config = None; self.active_runtime = None; self.state = ServerState::Stopped; Ok(()) @@ -939,11 +947,50 @@ fn http_health_ok(host: &str, port: u16) -> bool { response.starts_with("HTTP/1.1 200") || response.starts_with("HTTP/1.0 200") } +// A request allocates a fresh candidate port even when reusing the resident +// model. Changes to actual server options must still restart the process. +fn same_model_launch_settings(loaded: &ModelLaunchConfig, requested: &ModelLaunchConfig) -> bool { + let mut requested = requested.clone(); + requested.port = loaded.port; + loaded == &requested +} + #[cfg(test)] mod tests { use super::*; use crate::hardware::{CpuProfile, MemoryProfile}; + #[test] + fn model_hot_reuse_ignores_ephemeral_port_but_detects_profile_changes() { + let original = ModelLaunchConfig { + model_path: "model.gguf".into(), + backend: BackendKind::Cpu, + device: None, + gpu_layers: 0, + context_size: 8_192, + threads: 6, + batch_size: 512, + ubatch_size: 256, + flash_attention: false, + mmap: true, + mlock: false, + parallelism: 1, + host: "127.0.0.1".into(), + port: 12345, + }; + let mut requested = original.clone(); + requested.port = 23456; + assert!(same_model_launch_settings(&original, &requested)); + requested.threads = 2; + assert!(!same_model_launch_settings(&original, &requested)); + requested.threads = original.threads; + requested.gpu_layers = 12; + assert!(!same_model_launch_settings(&original, &requested)); + requested.gpu_layers = original.gpu_layers; + requested.context_size = 16_384; + assert!(!same_model_launch_settings(&original, &requested)); + } + #[tokio::test] async fn cancelling_chat_loading_returns_without_starting_a_server() { let root = PortableRootManager::from_root( From a28e572351271fe32e16122fba4c232e8cffbe6f Mon Sep 17 00:00:00 2001 From: Shahanur Islam Shagor Date: Tue, 20 Oct 2026 13:05:33 -0400 Subject: [PATCH 7/7] Retain default model planner helper only for legacy regression tests --- src-tauri/src/launch_planner.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src-tauri/src/launch_planner.rs b/src-tauri/src/launch_planner.rs index 672cbca..6d53c0a 100644 --- a/src-tauri/src/launch_planner.rs +++ b/src-tauri/src/launch_planner.rs @@ -41,6 +41,7 @@ pub struct LaunchPlan { pub struct ModelLaunchPlanner; impl ModelLaunchPlanner { + #[cfg(test)] pub fn plan(model: &ModelRecord, hardware: &HardwareProfile, port: u16) -> LaunchPlan { Self::plan_for_preference(model, hardware, "Auto", port) }