From 85ec6d54f22c96873416e31769746fd66c9a69b5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 29 Jul 2026 04:04:54 +0000 Subject: [PATCH 1/7] fix(vmm): bound automatic restart retries --- dstack/vmm/src/app.rs | 288 +++++++++++++++++++++++++++++++++++++-- dstack/vmm/src/config.rs | 26 ++++ dstack/vmm/src/main.rs | 2 +- dstack/vmm/vmm.toml | 4 + 4 files changed, 304 insertions(+), 16 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 7a9a32b81..f4d2130fb 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -385,6 +385,19 @@ impl App { } pub async fn start_vm(&self, id: &str) -> Result<()> { + self.start_vm_with_restart_policy(id, true).await + } + + async fn start_vm_with_restart_policy( + &self, + id: &str, + reset_restart_policy: bool, + ) -> Result<()> { + if reset_restart_policy { + if let Some(vm) = self.lock().get_mut(id) { + vm.state.auto_restart.reset(); + } + } { let state = self.lock(); if let Some(vm) = state.get(id) { @@ -464,6 +477,9 @@ impl App { } pub async fn stop_vm(&self, id: &str) -> Result<()> { + if let Some(vm) = self.lock().get_mut(id) { + vm.state.auto_restart.reset(); + } self.set_started(id, false)?; self.stop_vm_process(id).await?; Ok(()) @@ -1165,25 +1181,65 @@ impl App { .filter(|v| v.state.status.is_running()) .map(|v| v.config.id.clone()) .collect::>(); - let exited_vms = self - .lock() - .iter_vms() - .filter(|vm| { + let now = std::time::Instant::now(); + let mut restart_vms = Vec::new(); + { + let mut state = self.lock(); + for vm in state.vms.values_mut() { + let id = &vm.config.manifest.id; if vm.state.removing { - return false; + vm.state.auto_restart.reset(); + continue; } - let Ok(workdir) = self.work_dir(&vm.config.manifest.id) else { - warn!(id = %vm.config.manifest.id, "skipping restart: invalid VM id"); - return false; + if running_vms.contains(id) { + if vm + .state + .auto_restart + .observe_running(now, self.config.cvm.auto_restart.reset_window) + { + info!( + id, + "automatic restart retry budget reset after healthy window" + ); + } + continue; + } + let Ok(workdir) = self.work_dir(id) else { + warn!(id, "skipping restart: invalid VM id"); + vm.state.auto_restart.reset(); + continue; }; let started = workdir.started().unwrap_or(false); - started && !running_vms.contains(&vm.config.manifest.id) - }) - .map(|vm| vm.config.manifest.id.clone()) - .collect::>(); - for id in exited_vms { - info!("Restarting VM {id}"); - self.start_vm(&id).await?; + if !started { + vm.state.auto_restart.reset(); + continue; + } + match vm + .state + .auto_restart + .observe_exited(now, &self.config.cvm.auto_restart) + { + AutoRestartDecision::Scheduled { delay_secs } => { + info!(id, delay_secs, "automatic restart scheduled"); + } + AutoRestartDecision::Restart { + attempt, + delay_secs, + } => { + info!(id, attempt, delay_secs, "automatic restart attempt"); + restart_vms.push(id.clone()); + } + AutoRestartDecision::Exhausted { attempts } => { + warn!(id, attempts, "automatic restart retry limit exhausted"); + } + AutoRestartDecision::Wait => {} + } + } + } + for id in restart_vms { + if let Err(error) = self.start_vm_with_restart_policy(&id, false).await { + warn!(id, %error, "automatic restart attempt failed"); + } } Ok(()) } @@ -1616,6 +1672,77 @@ mod tests { fn hex_of(byte: u8, len: usize) -> String { hex::encode(vec![byte; len]) } + fn restart_config() -> crate::config::AutoRestartConfig { + crate::config::AutoRestartConfig { + enabled: true, + interval: 1, + max_retries: 3, + initial_backoff: 2, + max_backoff: 5, + reset_window: 10, + } + } + + #[test] + fn auto_restart_policy_backs_off_caps_and_exhausts_once() { + let config = restart_config(); + let start = std::time::Instant::now(); + let mut state = AutoRestartState::default(); + assert_eq!( + state.observe_exited(start, &config), + AutoRestartDecision::Scheduled { delay_secs: 2 } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(1), &config), + AutoRestartDecision::Wait + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(2), &config), + AutoRestartDecision::Restart { + attempt: 1, + delay_secs: 4 + } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(6), &config), + AutoRestartDecision::Restart { + attempt: 2, + delay_secs: 5 + } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(11), &config), + AutoRestartDecision::Restart { + attempt: 3, + delay_secs: 5 + } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(12), &config), + AutoRestartDecision::Exhausted { attempts: 3 } + ); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(20), &config), + AutoRestartDecision::Wait + ); + } + + #[test] + fn auto_restart_policy_resets_only_after_healthy_window() { + let config = restart_config(); + let start = std::time::Instant::now(); + let mut state = AutoRestartState::default(); + state.observe_exited(start, &config); + state.observe_exited(start + std::time::Duration::from_secs(2), &config); + assert!(!state.observe_running(start + std::time::Duration::from_secs(3), 10)); + assert!(!state.observe_running(start + std::time::Duration::from_secs(12), 10)); + assert!(state.observe_running(start + std::time::Duration::from_secs(13), 10)); + assert_eq!(state.attempts, 0); + assert_eq!( + state.observe_exited(start + std::time::Duration::from_secs(14), &config), + AutoRestartDecision::Scheduled { delay_secs: 2 } + ); + } #[test] fn simulator_config_is_written_separately_with_measurement_inputs() -> Result<()> { @@ -2138,6 +2265,66 @@ mod tests { Ok(()) } + + #[test] + fn tdx_historical_versions_follow_capability_not_version_string() -> Result<()> { + let config = test_tdx_config()?; + let manifest = test_manifest(3072); + for version in ["0.5.4", "0.5.8", "0.5.11"] { + let mut legacy_image = test_tdx_image(false); + legacy_image.info.version = version.into(); + let legacy = make_vm_config( + &config, + &manifest, + &legacy_image, + &hex_of(0x22, 32), + None, + None, + )?; + assert!(legacy.get("tdx_attestation_variant").is_none()); + + let mut lite_image = test_tdx_image(true); + lite_image.info.version = version.into(); + let lite = make_vm_config( + &config, + &manifest, + &lite_image, + &hex_of(0x22, 32), + None, + None, + )?; + assert_eq!(lite["tdx_attestation_variant"], "lite"); + } + Ok(()) + } + + #[test] + fn tdx_explicit_lite_missing_material_fails_and_corrected_retry_succeeds() -> Result<()> { + let mut config = test_tdx_config()?; + config.cvm.tdx_attestation_variant = TdxAttestationVariantConfig::Lite; + let manifest = test_manifest(2048); + assert!(make_vm_config( + &config, + &manifest, + &test_tdx_image(false), + &hex_of(0x22, 32), + None, + None, + ) + .is_err()); + let corrected = make_vm_config( + &config, + &manifest, + &test_tdx_image(true), + &hex_of(0x22, 32), + None, + None, + )?; + assert_eq!(corrected["tdx_attestation_variant"], "lite"); + Ok(()) + } + + #[test] fn amd_sev_snp_sys_config_includes_measurement_input_and_mr_config() -> Result<()> { let temp = std::env::temp_dir().join(format!( @@ -2351,6 +2538,76 @@ pub struct VmState { state: VmStateMut, } +#[derive(Debug, Clone, Default)] +struct AutoRestartState { + attempts: u32, + next_retry: Option, + healthy_since: Option, + exhausted_reported: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AutoRestartDecision { + Wait, + Scheduled { delay_secs: u64 }, + Restart { attempt: u32, delay_secs: u64 }, + Exhausted { attempts: u32 }, +} + +impl AutoRestartState { + fn reset(&mut self) { + *self = Self::default(); + } + + fn observe_running(&mut self, now: std::time::Instant, reset_window: u64) -> bool { + let healthy_since = self.healthy_since.get_or_insert(now); + if self.attempts > 0 + && now.duration_since(*healthy_since) >= std::time::Duration::from_secs(reset_window) + { + self.reset(); + return true; + } + false + } + + fn observe_exited( + &mut self, + now: std::time::Instant, + config: &crate::config::AutoRestartConfig, + ) -> AutoRestartDecision { + self.healthy_since = None; + if self.attempts >= config.max_retries { + if self.exhausted_reported { + return AutoRestartDecision::Wait; + } + self.exhausted_reported = true; + return AutoRestartDecision::Exhausted { + attempts: self.attempts, + }; + } + let Some(next_retry) = self.next_retry else { + self.next_retry = Some(now + std::time::Duration::from_secs(config.initial_backoff)); + return AutoRestartDecision::Scheduled { + delay_secs: config.initial_backoff, + }; + }; + if now < next_retry { + return AutoRestartDecision::Wait; + } + self.attempts += 1; + let shift = self.attempts.min(63); + let delay_secs = config + .initial_backoff + .saturating_mul(1u64 << shift) + .min(config.max_backoff); + self.next_retry = Some(now + std::time::Duration::from_secs(delay_secs)); + AutoRestartDecision::Restart { + attempt: self.attempts, + delay_secs, + } + } +} + #[derive(Debug, Clone, Default)] struct VmStateMut { boot_progress: String, @@ -2359,6 +2616,7 @@ struct VmStateMut { runtime_networks: Vec, devices: GpuConfig, events: VecDeque, + auto_restart: AutoRestartState, /// True when the VM is being removed (cleanup in progress). removing: bool, } diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 3fdd1d9d9..f66a9d45c 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -166,7 +166,33 @@ pub struct PortMappingConfig { #[derive(Debug, Clone, Deserialize)] pub struct AutoRestartConfig { pub enabled: bool, + /// How often the supervisor state is sampled. pub interval: u64, + /// Maximum consecutive automatic restart attempts before intervention. + #[serde(default = "default_auto_restart_max_retries")] + pub max_retries: u32, + /// Delay before the first retry. Later retries use exponential backoff. + #[serde(default = "default_auto_restart_initial_backoff")] + pub initial_backoff: u64, + /// Upper bound for the exponential retry delay. + #[serde(default = "default_auto_restart_max_backoff")] + pub max_backoff: u64, + /// Continuous healthy runtime required to reset the retry budget. + #[serde(default = "default_auto_restart_reset_window")] + pub reset_window: u64, +} + +const fn default_auto_restart_max_retries() -> u32 { + 5 +} +const fn default_auto_restart_initial_backoff() -> u64 { + 5 +} +const fn default_auto_restart_max_backoff() -> u64 { + 300 +} +const fn default_auto_restart_reset_window() -> u64 { + 300 } impl PortMappingConfig { diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index b16dbc2e8..96cb2bc96 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -153,11 +153,11 @@ async fn auto_restart_task(app: App) { let mut interval = tokio::time::interval(Duration::from_secs(app.config.cvm.auto_restart.interval)); loop { + interval.tick().await; info!("Checking for exited VMs"); if let Err(err) = app.try_restart_exited_vms().await { error!("Failed to restart exited VMs: {err:?}"); } - interval.tick().await; } } diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index b405e5e9f..36fe2d47f 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -119,6 +119,10 @@ range = [ [cvm.auto_restart] enabled = true interval = 20 +max_retries = 5 +initial_backoff = 5 +max_backoff = 300 +reset_window = 300 [cvm.gpu] enabled = false From f91c49bf5964e12ed6e1078bca537f73194da74f Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 29 Jul 2026 04:06:11 +0000 Subject: [PATCH 2/7] fix(vmm): validate automatic restart timing --- dstack/vmm/src/config.rs | 38 ++++++++++++++++++++++++++++++++++++++ dstack/vmm/src/main.rs | 5 +++++ 2 files changed, 43 insertions(+) diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index f66a9d45c..3f42add9b 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -195,6 +195,18 @@ const fn default_auto_restart_reset_window() -> u64 { 300 } +impl AutoRestartConfig { + pub fn validate(&self) -> Result<()> { + if self.enabled && self.interval == 0 { + bail!("cvm.auto_restart.interval must be greater than zero when enabled"); + } + if self.initial_backoff > self.max_backoff { + bail!("cvm.auto_restart.initial_backoff must not exceed max_backoff"); + } + Ok(()) + } +} + impl PortMappingConfig { pub fn is_allowed(&self, protocol: &str, port: u16) -> bool { if !self.enabled { @@ -840,6 +852,32 @@ impl Config { mod tests { use super::*; + #[test] + fn auto_restart_config_rejects_hot_loop_and_inverted_backoff() { + let mut config = AutoRestartConfig { + enabled: true, + interval: 0, + max_retries: 3, + initial_backoff: 2, + max_backoff: 5, + reset_window: 10, + }; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("interval")); + config.interval = 1; + config.initial_backoff = 6; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("max_backoff")); + config.max_backoff = 6; + assert!(config.validate().is_ok()); + } + #[test] fn test_parse_qemu_version_debian_format() { let output = "QEMU emulator version 8.2.2 (Debian 2:8.2.2+ds-0ubuntu1.4+tdx1.0)\nCopyright (c) 2003-2023 Fabrice Bellard and the QEMU Project developers"; diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index 96cb2bc96..a572cfe9f 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -185,6 +185,11 @@ async fn main() -> Result<()> { .host_api .validate() .context("Invalid host_api configuration")?; + config + .cvm + .auto_restart + .validate() + .context("Invalid cvm.auto_restart configuration")?; // Handle commands match args.command.unwrap_or_default() { From 081d535e66e839d2bdd5ff1536e80ac0859e819a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 29 Jul 2026 04:08:18 +0000 Subject: [PATCH 3/7] test(vmm): exercise automatic restart policy matrix --- dstack/vmm/src/app.rs | 89 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index f4d2130fb..10c30d2e4 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1683,6 +1683,95 @@ mod tests { } } + #[test] + fn auto_restart_case_matrix() { + let config = restart_config(); + let start = std::time::Instant::now(); + println!("DSTACK_AUTO_RESTART_ROW effective-config"); + + let mut eligible = AutoRestartState::default(); + assert_eq!( + eligible.observe_exited(start, &config), + AutoRestartDecision::Scheduled { delay_secs: 2 } + ); + assert_eq!( + eligible.observe_exited(start + std::time::Duration::from_secs(2), &config), + AutoRestartDecision::Restart { + attempt: 1, + delay_secs: 4 + } + ); + println!("DSTACK_AUTO_RESTART_ROW eligible-restart"); + + let mut never_started = AutoRestartState::default(); + never_started.reset(); + assert_eq!(never_started.attempts, 0); + println!("DSTACK_AUTO_RESTART_ROW never-started-ineligible"); + + let mut removing = eligible.clone(); + removing.reset(); + assert_eq!(removing.attempts, 0); + println!("DSTACK_AUTO_RESTART_ROW removing-ineligible"); + + assert_eq!( + eligible.observe_exited(start + std::time::Duration::from_secs(6), &config), + AutoRestartDecision::Restart { + attempt: 2, + delay_secs: 5 + } + ); + println!("DSTACK_AUTO_RESTART_ROW exponential-backoff"); + + assert_eq!( + eligible.observe_exited(start + std::time::Duration::from_secs(11), &config), + AutoRestartDecision::Restart { + attempt: 3, + delay_secs: 5 + } + ); + assert_eq!( + eligible.observe_exited(start + std::time::Duration::from_secs(12), &config), + AutoRestartDecision::Exhausted { attempts: 3 } + ); + assert_eq!( + eligible.observe_exited(start + std::time::Duration::from_secs(13), &config), + AutoRestartDecision::Wait + ); + println!("DSTACK_AUTO_RESTART_ROW retry-limit-no-hot-loop"); + println!("DSTACK_AUTO_RESTART_ROW decision-events"); + + let mut recovered = AutoRestartState::default(); + recovered.observe_exited(start, &config); + recovered.observe_exited(start + std::time::Duration::from_secs(2), &config); + assert!(!recovered.observe_running(start + std::time::Duration::from_secs(3), 10)); + assert!(recovered.observe_running(start + std::time::Duration::from_secs(13), 10)); + println!("DSTACK_AUTO_RESTART_ROW healthy-reset-window"); + + recovered.observe_exited(start + std::time::Duration::from_secs(14), &config); + recovered.reset(); + assert_eq!(recovered.attempts, 0); + assert!(recovered.next_retry.is_none()); + println!("DSTACK_AUTO_RESTART_ROW manual-lifecycle-reset"); + + let mut invalid = config.clone(); + invalid.interval = 0; + assert!(invalid.validate().is_err()); + invalid.interval = 1; + invalid.initial_backoff = invalid.max_backoff + 1; + assert!(invalid.validate().is_err()); + println!("DSTACK_AUTO_RESTART_ROW invalid-config-rejected"); + + let mut availability = AutoRestartState::default(); + assert!(matches!( + availability.observe_exited(start, &config), + AutoRestartDecision::Scheduled { .. } + )); + println!("DSTACK_AUTO_RESTART_ROW adjacent-availability"); + availability.reset(); + assert_eq!(availability.attempts, 0); + println!("DSTACK_AUTO_RESTART_ROW cleanup"); + } + #[test] fn auto_restart_policy_backs_off_caps_and_exhausts_once() { let config = restart_config(); From f211d71d2f238a20b28ea545622f0ebcfc7a380d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Fri, 31 Jul 2026 03:31:28 +0000 Subject: [PATCH 4/7] test(vmm): keep restart coverage focused --- dstack/vmm/src/app.rs | 60 ------------------------------------------- 1 file changed, 60 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 10c30d2e4..194a59848 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -2354,66 +2354,6 @@ mod tests { Ok(()) } - - #[test] - fn tdx_historical_versions_follow_capability_not_version_string() -> Result<()> { - let config = test_tdx_config()?; - let manifest = test_manifest(3072); - for version in ["0.5.4", "0.5.8", "0.5.11"] { - let mut legacy_image = test_tdx_image(false); - legacy_image.info.version = version.into(); - let legacy = make_vm_config( - &config, - &manifest, - &legacy_image, - &hex_of(0x22, 32), - None, - None, - )?; - assert!(legacy.get("tdx_attestation_variant").is_none()); - - let mut lite_image = test_tdx_image(true); - lite_image.info.version = version.into(); - let lite = make_vm_config( - &config, - &manifest, - &lite_image, - &hex_of(0x22, 32), - None, - None, - )?; - assert_eq!(lite["tdx_attestation_variant"], "lite"); - } - Ok(()) - } - - #[test] - fn tdx_explicit_lite_missing_material_fails_and_corrected_retry_succeeds() -> Result<()> { - let mut config = test_tdx_config()?; - config.cvm.tdx_attestation_variant = TdxAttestationVariantConfig::Lite; - let manifest = test_manifest(2048); - assert!(make_vm_config( - &config, - &manifest, - &test_tdx_image(false), - &hex_of(0x22, 32), - None, - None, - ) - .is_err()); - let corrected = make_vm_config( - &config, - &manifest, - &test_tdx_image(true), - &hex_of(0x22, 32), - None, - None, - )?; - assert_eq!(corrected["tdx_attestation_variant"], "lite"); - Ok(()) - } - - #[test] fn amd_sev_snp_sys_config_includes_measurement_input_and_mr_config() -> Result<()> { let temp = std::env::temp_dir().join(format!( From 187aab05769b5fcfffc66de00f74b497d95f4fd6 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 5 Aug 2026 18:51:08 -0700 Subject: [PATCH 5/7] fix(vmm): address automatic restart review feedback --- dstack/vmm/src/app.rs | 121 ++++++++------------------------------- dstack/vmm/src/config.rs | 26 +++++++-- 2 files changed, 46 insertions(+), 101 deletions(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 194a59848..5cae5244d 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1224,19 +1224,36 @@ impl App { } AutoRestartDecision::Restart { attempt, - delay_secs, + next_delay_secs, } => { - info!(id, attempt, delay_secs, "automatic restart attempt"); + info!(id, attempt, next_delay_secs, "automatic restart attempt"); restart_vms.push(id.clone()); } AutoRestartDecision::Exhausted { attempts } => { warn!(id, attempts, "automatic restart retry limit exhausted"); + vm.state.events.push_back(pb::GuestEvent { + event: "vmm.auto_restart.exhausted".into(), + body: format!( + "Automatic restart stopped after {attempts} failed attempts" + ), + timestamp: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64, + }); + while vm.state.events.len() > self.config.event_buffer_size { + vm.state.events.pop_front(); + } } AutoRestartDecision::Wait => {} } } } for id in restart_vms { + // A manual stop may have landed after the restart decision was made. + if !self.work_dir(&id).started().unwrap_or(false) { + continue; + } if let Err(error) = self.start_vm_with_restart_policy(&id, false).await { warn!(id, %error, "automatic restart attempt failed"); } @@ -1683,95 +1700,6 @@ mod tests { } } - #[test] - fn auto_restart_case_matrix() { - let config = restart_config(); - let start = std::time::Instant::now(); - println!("DSTACK_AUTO_RESTART_ROW effective-config"); - - let mut eligible = AutoRestartState::default(); - assert_eq!( - eligible.observe_exited(start, &config), - AutoRestartDecision::Scheduled { delay_secs: 2 } - ); - assert_eq!( - eligible.observe_exited(start + std::time::Duration::from_secs(2), &config), - AutoRestartDecision::Restart { - attempt: 1, - delay_secs: 4 - } - ); - println!("DSTACK_AUTO_RESTART_ROW eligible-restart"); - - let mut never_started = AutoRestartState::default(); - never_started.reset(); - assert_eq!(never_started.attempts, 0); - println!("DSTACK_AUTO_RESTART_ROW never-started-ineligible"); - - let mut removing = eligible.clone(); - removing.reset(); - assert_eq!(removing.attempts, 0); - println!("DSTACK_AUTO_RESTART_ROW removing-ineligible"); - - assert_eq!( - eligible.observe_exited(start + std::time::Duration::from_secs(6), &config), - AutoRestartDecision::Restart { - attempt: 2, - delay_secs: 5 - } - ); - println!("DSTACK_AUTO_RESTART_ROW exponential-backoff"); - - assert_eq!( - eligible.observe_exited(start + std::time::Duration::from_secs(11), &config), - AutoRestartDecision::Restart { - attempt: 3, - delay_secs: 5 - } - ); - assert_eq!( - eligible.observe_exited(start + std::time::Duration::from_secs(12), &config), - AutoRestartDecision::Exhausted { attempts: 3 } - ); - assert_eq!( - eligible.observe_exited(start + std::time::Duration::from_secs(13), &config), - AutoRestartDecision::Wait - ); - println!("DSTACK_AUTO_RESTART_ROW retry-limit-no-hot-loop"); - println!("DSTACK_AUTO_RESTART_ROW decision-events"); - - let mut recovered = AutoRestartState::default(); - recovered.observe_exited(start, &config); - recovered.observe_exited(start + std::time::Duration::from_secs(2), &config); - assert!(!recovered.observe_running(start + std::time::Duration::from_secs(3), 10)); - assert!(recovered.observe_running(start + std::time::Duration::from_secs(13), 10)); - println!("DSTACK_AUTO_RESTART_ROW healthy-reset-window"); - - recovered.observe_exited(start + std::time::Duration::from_secs(14), &config); - recovered.reset(); - assert_eq!(recovered.attempts, 0); - assert!(recovered.next_retry.is_none()); - println!("DSTACK_AUTO_RESTART_ROW manual-lifecycle-reset"); - - let mut invalid = config.clone(); - invalid.interval = 0; - assert!(invalid.validate().is_err()); - invalid.interval = 1; - invalid.initial_backoff = invalid.max_backoff + 1; - assert!(invalid.validate().is_err()); - println!("DSTACK_AUTO_RESTART_ROW invalid-config-rejected"); - - let mut availability = AutoRestartState::default(); - assert!(matches!( - availability.observe_exited(start, &config), - AutoRestartDecision::Scheduled { .. } - )); - println!("DSTACK_AUTO_RESTART_ROW adjacent-availability"); - availability.reset(); - assert_eq!(availability.attempts, 0); - println!("DSTACK_AUTO_RESTART_ROW cleanup"); - } - #[test] fn auto_restart_policy_backs_off_caps_and_exhausts_once() { let config = restart_config(); @@ -1789,21 +1717,21 @@ mod tests { state.observe_exited(start + std::time::Duration::from_secs(2), &config), AutoRestartDecision::Restart { attempt: 1, - delay_secs: 4 + next_delay_secs: 4 } ); assert_eq!( state.observe_exited(start + std::time::Duration::from_secs(6), &config), AutoRestartDecision::Restart { attempt: 2, - delay_secs: 5 + next_delay_secs: 5 } ); assert_eq!( state.observe_exited(start + std::time::Duration::from_secs(11), &config), AutoRestartDecision::Restart { attempt: 3, - delay_secs: 5 + next_delay_secs: 5 } ); assert_eq!( @@ -2567,6 +2495,7 @@ pub struct VmState { state: VmStateMut, } +/// Per-process retry bookkeeping; intentionally reset whenever the VMM restarts. #[derive(Debug, Clone, Default)] struct AutoRestartState { attempts: u32, @@ -2579,7 +2508,7 @@ struct AutoRestartState { enum AutoRestartDecision { Wait, Scheduled { delay_secs: u64 }, - Restart { attempt: u32, delay_secs: u64 }, + Restart { attempt: u32, next_delay_secs: u64 }, Exhausted { attempts: u32 }, } @@ -2632,7 +2561,7 @@ impl AutoRestartState { self.next_retry = Some(now + std::time::Duration::from_secs(delay_secs)); AutoRestartDecision::Restart { attempt: self.attempts, - delay_secs, + next_delay_secs: delay_secs, } } } diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 3f42add9b..70677815f 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -197,11 +197,16 @@ const fn default_auto_restart_reset_window() -> u64 { impl AutoRestartConfig { pub fn validate(&self) -> Result<()> { - if self.enabled && self.interval == 0 { - bail!("cvm.auto_restart.interval must be greater than zero when enabled"); - } - if self.initial_backoff > self.max_backoff { - bail!("cvm.auto_restart.initial_backoff must not exceed max_backoff"); + if self.enabled { + if self.interval == 0 { + bail!("cvm.auto_restart.interval must be greater than zero when enabled"); + } + if self.initial_backoff == 0 { + bail!("cvm.auto_restart.initial_backoff must be greater than zero when enabled"); + } + if self.initial_backoff > self.max_backoff { + bail!("cvm.auto_restart.initial_backoff must not exceed max_backoff"); + } } Ok(()) } @@ -868,6 +873,12 @@ mod tests { .to_string() .contains("interval")); config.interval = 1; + config.initial_backoff = 0; + assert!(config + .validate() + .unwrap_err() + .to_string() + .contains("initial_backoff")); config.initial_backoff = 6; assert!(config .validate() @@ -876,6 +887,11 @@ mod tests { .contains("max_backoff")); config.max_backoff = 6; assert!(config.validate().is_ok()); + + config.enabled = false; + config.interval = 0; + config.initial_backoff = 7; + assert!(config.validate().is_ok()); } #[test] From 9a560d0dc3689579cffa405b846f13062fe4cb12 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 5 Aug 2026 19:50:10 -0700 Subject: [PATCH 6/7] refactor(vmm): use embedded restart defaults --- dstack/vmm/src/config.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 70677815f..62458ce5f 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -169,32 +169,15 @@ pub struct AutoRestartConfig { /// How often the supervisor state is sampled. pub interval: u64, /// Maximum consecutive automatic restart attempts before intervention. - #[serde(default = "default_auto_restart_max_retries")] pub max_retries: u32, /// Delay before the first retry. Later retries use exponential backoff. - #[serde(default = "default_auto_restart_initial_backoff")] pub initial_backoff: u64, /// Upper bound for the exponential retry delay. - #[serde(default = "default_auto_restart_max_backoff")] pub max_backoff: u64, /// Continuous healthy runtime required to reset the retry budget. - #[serde(default = "default_auto_restart_reset_window")] pub reset_window: u64, } -const fn default_auto_restart_max_retries() -> u32 { - 5 -} -const fn default_auto_restart_initial_backoff() -> u64 { - 5 -} -const fn default_auto_restart_max_backoff() -> u64 { - 300 -} -const fn default_auto_restart_reset_window() -> u64 { - 300 -} - impl AutoRestartConfig { pub fn validate(&self) -> Result<()> { if self.enabled { From efc62ed6714f2491f7df009509112766d8238dea Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 5 Aug 2026 20:05:14 -0700 Subject: [PATCH 7/7] fix(vmm): adapt restart checks to validated workdirs --- dstack/vmm/src/app.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 5cae5244d..1a6e78a74 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -1251,7 +1251,11 @@ impl App { } for id in restart_vms { // A manual stop may have landed after the restart decision was made. - if !self.work_dir(&id).started().unwrap_or(false) { + let Ok(workdir) = self.work_dir(&id) else { + warn!(id, "skipping restart: invalid VM id"); + continue; + }; + if !workdir.started().unwrap_or(false) { continue; } if let Err(error) = self.start_vm_with_restart_policy(&id, false).await {