diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 4500580ad50..104999c83cc 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -94,6 +94,7 @@ use nexus_types::internal_api::background::TufArtifactReplicationCounters; use nexus_types::internal_api::background::TufArtifactReplicationRequest; use nexus_types::internal_api::background::TufArtifactReplicationStatus; use nexus_types::internal_api::background::TufRepoPrunerStatus; +use nexus_types::internal_api::background::VmmMarkStopForUpdateStatus; use nexus_types::internal_api::background::fm_rendezvous; use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::CollectionUuid; @@ -1426,6 +1427,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) { "switch_port_config_manager" => { print_task_switch_port_settings_manager(details); } + "vmm_mark_stop_for_update" => { + print_task_vmm_mark_stop_for_update(details); + } _ => { println!( "warning: unknown background task: {:?} \ @@ -2842,6 +2846,33 @@ fn print_task_audit_log_cleanup(details: &serde_json::Value) { }; } +fn print_task_vmm_mark_stop_for_update(details: &serde_json::Value) { + match serde_json::from_value::(details.clone()) + { + Err(error) => eprintln!( + "warning: failed to interpret task details: {:?}: {:?}", + error, details + ), + Ok(status) => { + let VmmMarkStopForUpdateStatus { disabled, vmms_marked, error } = + status; + + if disabled { + println!(" task explicitly disabled by config!"); + } + + const MARKED: &str = "VMMs marked:"; + const ERROR: &str = "error:"; + const WIDTH: usize = const_max_len(&[MARKED, ERROR]) + 1; + + println!(" {MARKED:( details.clone(), diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 5d39c5509e6..516c0f5c71e 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -263,6 +263,10 @@ task: "v2p_manager" manages opte v2p mappings for vpc networking +task: "vmm_mark_stop_for_update" + marks VMMs on evacuating sleds as needing to be stopped for an update + + task: "vpc_route_manager" propagates updated VPC routes to all OPTE ports @@ -531,6 +535,10 @@ task: "v2p_manager" manages opte v2p mappings for vpc networking +task: "vmm_mark_stop_for_update" + marks VMMs on evacuating sleds as needing to be stopped for an update + + task: "vpc_route_manager" propagates updated VPC routes to all OPTE ports @@ -786,6 +794,10 @@ task: "v2p_manager" manages opte v2p mappings for vpc networking +task: "vmm_mark_stop_for_update" + marks VMMs on evacuating sleds as needing to be stopped for an update + + task: "vpc_route_manager" propagates updated VPC routes to all OPTE ports diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index ea3cd79f126..5b8884098a8 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -498,6 +498,10 @@ task: "v2p_manager" manages opte v2p mappings for vpc networking +task: "vmm_mark_stop_for_update" + marks VMMs on evacuating sleds as needing to be stopped for an update + + task: "vpc_route_manager" propagates updated VPC routes to all OPTE ports @@ -1091,6 +1095,12 @@ task: "v2p_manager" started at (s ago) and ran for ms warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {}) +task: "vmm_mark_stop_for_update" + configured period: every m + last completed activation: , triggered by + started at (s ago) and ran for ms + VMMs marked: 0 + task: "vpc_route_manager" configured period: every s last completed activation: , triggered by @@ -1813,6 +1823,12 @@ task: "v2p_manager" started at (s ago) and ran for ms warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {}) +task: "vmm_mark_stop_for_update" + configured period: every m + last completed activation: , triggered by + started at (s ago) and ran for ms + VMMs marked: 0 + task: "vpc_route_manager" configured period: every s last completed activation: , triggered by diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index b18a621ab9e..8eae7de0303 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -477,6 +477,8 @@ pub struct BackgroundTaskConfig { pub audit_log_cleanup: AuditLogCleanupConfig, /// configuration for populate switch ports task pub populate_switch_ports: PopulateSwitchPortsConfig, + /// configuration for the task that marks VMMs to stop for an update + pub vmm_mark_stop_for_update: VmmMarkStopForUpdateConfig, } #[serde_as] @@ -523,6 +525,23 @@ pub struct AuditLogCleanupConfig { pub max_deleted_per_activation: u32, } +#[serde_as] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct VmmMarkStopForUpdateConfig { + /// period (in seconds) for periodic activations of this task + #[serde_as(as = "DurationSeconds")] + pub period_secs: Duration, + + /// disable marking VMMs to stop for a sled update. + /// + /// This is an emergency lever for support / operations. It should only be + /// necessary if something has gone extremely wrong. + /// + /// Default: Off + #[serde(default)] + pub disable: bool, +} + #[serde_as] #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct PopulateSwitchPortsConfig { @@ -1376,6 +1395,7 @@ mod test { audit_log_cleanup.retention_days = 90 audit_log_cleanup.max_deleted_per_activation = 10000 populate_switch_ports.period_secs = 31 + vmm_mark_stop_for_update.period_secs = 300 [default_region_allocation_strategy] type = "random" seed = 0 @@ -1656,6 +1676,10 @@ mod test { populate_switch_ports: PopulateSwitchPortsConfig { period_secs: Duration::from_secs(31), }, + vmm_mark_stop_for_update: VmmMarkStopForUpdateConfig { + period_secs: Duration::from_secs(300), + disable: false, + }, }, multicast: MulticastConfig { enabled: false }, default_region_allocation_strategy: @@ -1772,6 +1796,7 @@ mod test { audit_log_cleanup.retention_days = 90 audit_log_cleanup.max_deleted_per_activation = 10000 populate_switch_ports.period_secs = 31 + vmm_mark_stop_for_update.period_secs = 300 [default_region_allocation_strategy] type = "random" diff --git a/nexus/background-task-interface/src/init.rs b/nexus/background-task-interface/src/init.rs index fd2c8f30fec..bfe1b2d79c8 100644 --- a/nexus/background-task-interface/src/init.rs +++ b/nexus/background-task-interface/src/init.rs @@ -63,6 +63,7 @@ pub struct BackgroundTasks { pub task_attached_subnet_manager: Activator, pub task_session_cleanup: Activator, pub task_populate_switch_ports: Activator, + pub task_vmm_mark_stop_for_update: Activator, // Handles to activate background tasks that do not get used by Nexus // at-large. These background tasks are implementation details as far as diff --git a/nexus/db-queries/src/db/datastore/vmm.rs b/nexus/db-queries/src/db/datastore/vmm.rs index 31d82a6c21a..6ab31da7fb7 100644 --- a/nexus/db-queries/src/db/datastore/vmm.rs +++ b/nexus/db-queries/src/db/datastore/vmm.rs @@ -154,6 +154,60 @@ impl DataStore { Ok(updated) } + /// Marks VMMs sleds that are evacuating as needing to be stopped in order + /// to update the sled. + /// + /// VMMs that are already stopping/stopped, migrating or in a terminal state + /// do not need to be stopped, so they are left untouched. VMMs that are + /// already marked to be stopped by update are also excluded. + /// + /// The task to stop instances only needs to know whether a VMM should be + /// stopped or not, so a boolean would be enough. But, we mark the VMMs + /// with the sled's `update_disposition` generation for debugging purposes. + pub async fn vmm_bulk_mark_stop_for_update( + &self, + opctx: &OpContext, + ) -> UpdateResult { + use nexus_db_schema::schema::rendezvous_sled_bp_availability::dsl as rz_dsl; + + let updated = diesel::update(dsl::vmm) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::stop_for_update_disposition_generation.is_null()) + .filter(dsl::state.eq_any([ + DbVmmState::Creating, + DbVmmState::Starting, + DbVmmState::Running, + DbVmmState::Rebooting, + ])) + .filter( + dsl::sled_id.eq_any( + rz_dsl::rendezvous_sled_bp_availability + .filter( + rz_dsl::bp_availability + .eq(model::DbSledBpAvailability::Unavailable), + ) + .select(rz_dsl::sled_id), + ), + ) + .set( + dsl::stop_for_update_disposition_generation.eq( + rz_dsl::rendezvous_sled_bp_availability + .filter(rz_dsl::sled_id.eq(dsl::sled_id)) + .filter( + rz_dsl::bp_availability + .eq(model::DbSledBpAvailability::Unavailable), + ) + .select(rz_dsl::update_disposition_generation) + .single_value(), + ), + ) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(updated) + } + pub async fn vmm_fetch( &self, opctx: &OpContext, @@ -491,12 +545,18 @@ mod tests { use crate::db::model::Generation; use crate::db::model::Migration; use crate::db::pub_test_utils::TestDatabase; + use nexus_db_model::ActiveSledBpAvailability; + use nexus_db_model::RendezvousSledBpAvailabilityUpdate; use nexus_db_model::VmmCpuPlatform; + use nexus_db_model::VmmFailureReason; use nexus_types::instance::VmmState; + use omicron_generation_kinds::UpdateDispositionGeneration; use omicron_test_utils::dev; + use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::SledUuid; use sled_agent_types::instance::MigrationState; + use std::collections::HashMap; #[tokio::test] async fn test_vmm_and_migration_update_runtime() { @@ -764,4 +824,257 @@ mod tests { db.terminate().await; logctx.cleanup_successful(); } + + #[tokio::test] + async fn test_vmm_bulk_mark_stop_for_update() { + // Setup + let logctx = dev::test_setup_log("test_vmm_bulk_mark_stop_for_update"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // The states from which a VMM can still be stopped for an update. Only + // VMMs in these states should be marked by + // `vmm_bulk_mark_stop_for_update`. + let is_stoppable = |state: DbVmmState| { + matches!( + state, + DbVmmState::Creating + | DbVmmState::Starting + | DbVmmState::Running + | DbVmmState::Rebooting + ) + }; + + // Insert a VMM in every possible state onto each of three sleds. All of + // them start out unmarked (i.e. `stop_for_update_disposition_generation` + // is NULL). + let sled_a = SledUuid::new_v4(); + let sled_b = SledUuid::new_v4(); + let sled_c = SledUuid::new_v4(); + let mut vmms = Vec::new(); + for sled_id in [sled_a, sled_b, sled_c] { + for &state in DbVmmState::ALL_STATES { + // The `failure_reason_iff_failed` constraint requires that a + // `Failed` VMM has a failure reason and that no other VMM does. + let failure_reason = (state == DbVmmState::Failed) + .then_some(VmmFailureReason::FromSledAgent); + let vmm = datastore + .vmm_insert( + &opctx, + Vmm { + id: Uuid::new_v4(), + time_created: Utc::now(), + time_deleted: None, + instance_id: Uuid::new_v4(), + sled_id: sled_id.into(), + propolis_ip: "10.1.9.32".parse().unwrap(), + propolis_port: 420.into(), + cpu_platform: VmmCpuPlatform::SledDefault, + time_state_updated: Utc::now(), + generation: Generation::new(), + state, + failure_reason, + stop_for_update_disposition_generation: None, + }, + ) + .await + .expect("VMM should be inserted successfully"); + vmms.push(vmm); + } + } + let stoppable_per_sled = + vmms.iter().filter(|v| is_stoppable(v.state)).count() / 3; + assert_eq!(stoppable_per_sled, 4); + + // Fetches every VMM's current row, keyed by VMM ID. + async fn fetch_all( + datastore: &DataStore, + opctx: &OpContext, + vmms: &[Vmm], + ) -> HashMap { + let mut actual = HashMap::new(); + for vmm in vmms { + let fetched = datastore + .vmm_fetch(opctx, &PropolisUuid::from_untyped_uuid(vmm.id)) + .await + .expect("VMM should be fetched successfully"); + actual.insert(vmm.id, fetched); + } + actual + } + + // Asserts that every fetched VMM row matches the expected row. + fn assert_rows( + actual: &HashMap, + expected: &HashMap, + ) { + assert_eq!(actual.len(), expected.len()); + for (id, want) in expected { + assert_eq!( + actual[id], + *want, + "VMM {} on sled {} in state {:?} changed unexpectedly", + id, + want.sled_id(), + want.state, + ); + } + } + + // Initially every VMM is unmarked, so the expected rows are exactly the + // rows we inserted. + let gen1 = UpdateDispositionGeneration::from(1); + let gen2 = UpdateDispositionGeneration::from(2); + let blueprint_id = BlueprintUuid::new_v4(); + let mut expected: HashMap = + vmms.iter().map(|vmm| (vmm.id, vmm.clone())).collect(); + + let actual = fetch_all(datastore, opctx, &vmms).await; + assert_rows(&actual, &expected); + + datastore + .rendezvous_sled_bp_availability_upsert( + opctx, + RendezvousSledBpAvailabilityUpdate::new( + sled_a, + ActiveSledBpAvailability::Unavailable, + gen1, + blueprint_id, + ), + ) + .await + .expect("sled A availability should upsert"); + datastore + .rendezvous_sled_bp_availability_upsert( + opctx, + RendezvousSledBpAvailabilityUpdate::new( + sled_b, + ActiveSledBpAvailability::Unavailable, + gen2, + blueprint_id, + ), + ) + .await + .expect("sled B availability should upsert"); + datastore + .rendezvous_sled_bp_availability_upsert( + opctx, + RendezvousSledBpAvailabilityUpdate::new( + sled_c, + ActiveSledBpAvailability::Available, + gen1, + blueprint_id, + ), + ) + .await + .expect("sled C availability should upsert"); + + // Sleds A and B are both evacuating (`unavailable`), at different + // generations, and sled C is available. In a single pass the stoppable + // VMMs on both sled A and sled B should be marked, each at their own + // sled's generation, regardless of which generation that is. + let marked = datastore + .vmm_bulk_mark_stop_for_update(&opctx) + .await + .expect("bulk mark should succeed"); + assert_eq!( + marked, 8, + "the 4 stoppable VMMs on each of sleds A and B should be marked" + ); + for vmm in expected.values_mut() { + if vmm.sled_id() == sled_a && is_stoppable(vmm.state) { + vmm.stop_for_update_disposition_generation = Some(gen1.into()); + } + if vmm.sled_id() == sled_b && is_stoppable(vmm.state) { + vmm.stop_for_update_disposition_generation = Some(gen2.into()); + } + } + + // There should be 22 unmarked rows (6 unstoppable on each of sleds A + // and B, plus all 10 on sled C). + let actual = fetch_all(datastore, opctx, &vmms).await; + assert_rows(&actual, &expected); + assert_eq!( + actual + .values() + .filter(|v| v.stop_for_update_disposition_generation.is_none()) + .count(), + 22, + "all rows other than sleds A and B's stoppable VMMs remain unmarked" + ); + + // Nothing has changed so running again should make no changes + let marked_again = datastore + .vmm_bulk_mark_stop_for_update(&opctx) + .await + .expect("re-running the bulk mark should succeed"); + assert_eq!(marked_again, 0); + let actual = fetch_all(datastore, opctx, &vmms).await; + assert_rows(&actual, &expected); + assert_eq!( + actual + .values() + .filter(|v| v.stop_for_update_disposition_generation.is_none()) + .count(), + 22 + ); + + // Now sled C evacuates too, moving from generation 1 to generation 2. + datastore + .rendezvous_sled_bp_availability_upsert( + opctx, + RendezvousSledBpAvailabilityUpdate::new( + sled_c, + ActiveSledBpAvailability::Unavailable, + gen2, + blueprint_id, + ), + ) + .await + .expect("sled C availability should upsert"); + + // Run again. Only sled C's stoppable VMMs should change. + let marked_c = datastore + .vmm_bulk_mark_stop_for_update(&opctx) + .await + .expect("bulk mark for sled C should succeed"); + assert_eq!( + marked_c, 4, + "only the 4 stoppable VMMs on sled C should be marked" + ); + for vmm in expected.values_mut() { + if vmm.sled_id() == sled_c && is_stoppable(vmm.state) { + vmm.stop_for_update_disposition_generation = Some(gen2.into()); + } + } + + let actual = fetch_all(datastore, opctx, &vmms).await; + assert_rows(&actual, &expected); + let gen1_count = actual + .values() + .filter(|v| { + v.stop_for_update_disposition_generation == Some(gen1.into()) + }) + .count(); + let gen2_count = actual + .values() + .filter(|v| { + v.stop_for_update_disposition_generation == Some(gen2.into()) + }) + .count(); + let null_count = actual + .values() + .filter(|v| v.stop_for_update_disposition_generation.is_none()) + .count(); + assert_eq!(gen1_count, 4, "sled A's marked VMMs are unchanged"); + assert_eq!( + gen2_count, 8, + "sled B's marked VMMs are unchanged and sled C's are newly marked" + ); + assert_eq!(null_count, 18, "the unstoppable VMMs remain unmarked"); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } } diff --git a/nexus/db-schema/src/schema.rs b/nexus/db-schema/src/schema.rs index 69bbf7ca4c2..ae0e8d4c7c0 100644 --- a/nexus/db-schema/src/schema.rs +++ b/nexus/db-schema/src/schema.rs @@ -2747,6 +2747,7 @@ table! { allow_tables_to_appear_in_same_query!(instance, migration); allow_tables_to_appear_in_same_query!(migration, vmm); +allow_tables_to_appear_in_same_query!(vmm, rendezvous_sled_bp_availability); joinable!(instance -> migration (migration_id)); allow_tables_to_appear_in_same_query!(subnet_pool, subnet_pool_silo_link, silo); diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index 05b1546094f..82e47f24673 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -218,6 +218,9 @@ audit_log_cleanup.period_secs = 600 audit_log_cleanup.retention_days = 90 audit_log_cleanup.max_deleted_per_activation = 10000 populate_switch_ports.period_secs = 30 +# This task reads from the blueprint rendezvous table directly, so it doesn't +# need to run more often +vmm_mark_stop_for_update.period_secs = 300 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 6ec9adc0eba..e72544bdce9 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -202,6 +202,9 @@ audit_log_cleanup.period_secs = 600 audit_log_cleanup.retention_days = 90 audit_log_cleanup.max_deleted_per_activation = 10000 populate_switch_ports.period_secs = 30 +# This task reads from the blueprint rendezvous table directly, so it doesn't +# need to run more often +vmm_mark_stop_for_update.period_secs = 300 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index d5659dac257..21067d07d3e 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -142,6 +142,7 @@ use super::tasks::trust_quorum; use super::tasks::tuf_artifact_replication; use super::tasks::tuf_repo_pruner; use super::tasks::v2p_mappings::V2PManager; +use super::tasks::vmm_mark_stop_for_update::VmmMarkStopForUpdate; use super::tasks::vpc_routes; use super::tasks::webhook_deliverator; use crate::Nexus; @@ -280,6 +281,7 @@ impl BackgroundTasksInitializer { task_attached_subnet_manager: Activator::new(), task_session_cleanup: Activator::new(), task_populate_switch_ports: Activator::new(), + task_vmm_mark_stop_for_update: Activator::new(), // Handles to activate background tasks that do not get used by Nexus // at-large. These background tasks are implementation details as far as @@ -376,6 +378,7 @@ impl BackgroundTasksInitializer { task_audit_log_timeout_incomplete, task_audit_log_cleanup, task_populate_switch_ports, + task_vmm_mark_stop_for_update, // Add new background tasks here. Be sure to use this binding in a // call to `Driver::register()` below. That's what actually wires // up the Activator to the corresponding background task. @@ -1315,6 +1318,20 @@ impl BackgroundTasksInitializer { activator: task_populate_switch_ports, }); + driver.register(TaskDefinition { + name: "vmm_mark_stop_for_update", + description: "marks VMMs on evacuating sleds as needing to be \ + stopped for an update", + period: config.vmm_mark_stop_for_update.period_secs, + task_impl: Box::new(VmmMarkStopForUpdate::new( + datastore, + config.vmm_mark_stop_for_update.disable, + )), + opctx: opctx.child(BTreeMap::new()), + watchers: vec![], + activator: task_vmm_mark_stop_for_update, + }); + driver } } diff --git a/nexus/src/app/background/tasks/mod.rs b/nexus/src/app/background/tasks/mod.rs index 7533226b12b..a15e8702074 100644 --- a/nexus/src/app/background/tasks/mod.rs +++ b/nexus/src/app/background/tasks/mod.rs @@ -56,5 +56,6 @@ pub mod trust_quorum; pub mod tuf_artifact_replication; pub mod tuf_repo_pruner; pub mod v2p_mappings; +pub mod vmm_mark_stop_for_update; pub mod vpc_routes; pub mod webhook_deliverator; diff --git a/nexus/src/app/background/tasks/vmm_mark_stop_for_update.rs b/nexus/src/app/background/tasks/vmm_mark_stop_for_update.rs new file mode 100644 index 00000000000..e2e640d5704 --- /dev/null +++ b/nexus/src/app/background/tasks/vmm_mark_stop_for_update.rs @@ -0,0 +1,258 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task that marks the VMMs on sleds that are evacuating for an +//! update as needing to be stopped. +//! +//! The marker (`stop_for_update_disposition_generation` on the `vmm` table) +//! records the sled's `update_disposition` generation and signals that the VMM +//! must be stopped in order to update its sled. The set of evacuating sleds is +//! read from the `rendezvous_sled_bp_availability` table. See RFD 739 for the +//! full implementation of instance restart during live update. + +use crate::app::background::BackgroundTask; +use futures::future::BoxFuture; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_types::internal_api::background::VmmMarkStopForUpdateStatus; +use serde_json::json; +use slog_error_chain::InlineErrorChain; +use std::sync::Arc; + +pub struct VmmMarkStopForUpdate { + datastore: Arc, + disable: bool, +} + +impl VmmMarkStopForUpdate { + pub fn new(datastore: Arc, disable: bool) -> Self { + Self { datastore, disable } + } + + pub(crate) async fn actually_activate( + &mut self, + opctx: &OpContext, + ) -> VmmMarkStopForUpdateStatus { + // Something is malfunctioning. TURN THE TASK OFF! + if self.disable { + slog::info!( + &opctx.log, + "vmm mark-stop-for-update task disabled, doing nothing"; + ); + return VmmMarkStopForUpdateStatus { + disabled: true, + vmms_marked: 0, + error: None, + }; + } + + let vmms_marked = + match self.datastore.vmm_bulk_mark_stop_for_update(opctx).await { + Ok(count) => count, + Err(err) => { + slog::error!( + &opctx.log, + "failed to mark VMMs to stop for a sled update"; + &err, + ); + return VmmMarkStopForUpdateStatus { + disabled: false, + vmms_marked: 0, + error: Some(InlineErrorChain::new(&err).to_string()), + }; + } + }; + + if vmms_marked > 0 { + slog::info!( + &opctx.log, + "marked {vmms_marked} VMMs to stop for a sled update"; + ); + } else { + slog::debug!( + &opctx.log, + "no VMMs need to be marked to stop for a sled update"; + ); + } + + VmmMarkStopForUpdateStatus { disabled: false, vmms_marked, error: None } + } +} + +impl BackgroundTask for VmmMarkStopForUpdate { + fn activate<'a>( + &'a mut self, + opctx: &'a OpContext, + ) -> BoxFuture<'a, serde_json::Value> { + Box::pin(async { + let status = self.actually_activate(opctx).await; + match serde_json::to_value(status) { + Ok(val) => val, + Err(err) => { + json!({ "error": format!("failed to serialize status: {err}") }) + } + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use nexus_db_model::ActiveSledBpAvailability; + use nexus_db_model::Generation; + use nexus_db_model::RendezvousSledBpAvailabilityUpdate; + use nexus_db_model::Vmm; + use nexus_db_model::VmmCpuPlatform; + use nexus_db_model::VmmState; + use nexus_db_queries::db::pub_test_utils::TestDatabase; + use omicron_generation_kinds::UpdateDispositionGeneration; + use omicron_test_utils::dev; + use omicron_uuid_kinds::BlueprintUuid; + use omicron_uuid_kinds::GenericUuid; + use omicron_uuid_kinds::PropolisUuid; + use omicron_uuid_kinds::SledUuid; + use uuid::Uuid; + + async fn insert_vmm( + datastore: &DataStore, + opctx: &OpContext, + sled_id: SledUuid, + state: VmmState, + ) -> Vmm { + datastore + .vmm_insert( + opctx, + Vmm { + id: Uuid::new_v4(), + time_created: Utc::now(), + time_deleted: None, + instance_id: Uuid::new_v4(), + sled_id: sled_id.into(), + propolis_ip: "10.1.9.32".parse().unwrap(), + propolis_port: 420.into(), + cpu_platform: VmmCpuPlatform::SledDefault, + time_state_updated: Utc::now(), + generation: Generation::new(), + state, + failure_reason: None, + stop_for_update_disposition_generation: None, + }, + ) + .await + .expect("VMM should be inserted") + } + + async fn get_marker_from_vmm( + datastore: &DataStore, + opctx: &OpContext, + vmm: &Vmm, + ) -> Option { + datastore + .vmm_fetch(opctx, &PropolisUuid::from_untyped_uuid(vmm.id)) + .await + .expect("VMM should be fetched") + .stop_for_update_disposition_generation + .map(Into::into) + } + + #[tokio::test] + async fn test_vmm_mark_stop_for_update_activation() { + let logctx = + dev::test_setup_log("test_vmm_mark_stop_for_update_activation"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let sled_evacuating = SledUuid::new_v4(); + let sled_available = SledUuid::new_v4(); + let generation = UpdateDispositionGeneration::from(1); + let blueprint_id = BlueprintUuid::new_v4(); + + // A stoppable VMM on an evacuating sled: should be marked. + let should_mark = + insert_vmm(datastore, opctx, sled_evacuating, VmmState::Running) + .await; + // A VMM on the evacuating sled that is not stoppable: should be skipped. + let already_stopped = + insert_vmm(datastore, opctx, sled_evacuating, VmmState::Stopped) + .await; + // A stoppable VMM on a sled that is not evacuating: should be skipped. + let other_sled = + insert_vmm(datastore, opctx, sled_available, VmmState::Running) + .await; + + datastore + .rendezvous_sled_bp_availability_upsert( + opctx, + RendezvousSledBpAvailabilityUpdate::new( + sled_evacuating, + ActiveSledBpAvailability::Unavailable, + generation, + blueprint_id, + ), + ) + .await + .expect("evacuating sled availability should upsert"); + datastore + .rendezvous_sled_bp_availability_upsert( + opctx, + RendezvousSledBpAvailabilityUpdate::new( + sled_available, + ActiveSledBpAvailability::Available, + generation, + blueprint_id, + ), + ) + .await + .expect("available sled availability should upsert"); + + // First, verify that a disabled task does nothing + let mut task = VmmMarkStopForUpdate::new(datastore.clone(), true); + let status = task.actually_activate(opctx).await; + assert_eq!(status.vmms_marked, 0); + assert!(status.error.is_none()); + + // Enable the task + let mut task = VmmMarkStopForUpdate::new(datastore.clone(), false); + + // The first activation marks the single eligible VMM at its sled's + // update disposition generation. + let status = task.actually_activate(opctx).await; + assert_eq!(status.vmms_marked, 1); + assert!(status.error.is_none()); + assert_eq!( + get_marker_from_vmm(datastore, opctx, &should_mark).await, + Some(generation) + ); + assert_eq!( + get_marker_from_vmm(datastore, opctx, &already_stopped).await, + None + ); + assert_eq!( + get_marker_from_vmm(datastore, opctx, &other_sled).await, + None + ); + + // Running again is a no-op: the eligible VMM is already marked. + let status = task.actually_activate(opctx).await; + assert_eq!(status.vmms_marked, 0); + assert!(status.error.is_none()); + assert_eq!( + get_marker_from_vmm(datastore, opctx, &should_mark).await, + Some(generation) + ); + assert_eq!( + get_marker_from_vmm(datastore, opctx, &already_stopped).await, + None + ); + assert_eq!( + get_marker_from_vmm(datastore, opctx, &other_sled).await, + None + ); + + db.terminate().await; + logctx.cleanup_successful(); + } +} diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index c8dc72e98f0..28cba280e44 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -238,6 +238,9 @@ audit_log_cleanup.period_secs = 600 audit_log_cleanup.retention_days = 90 audit_log_cleanup.max_deleted_per_activation = 10000 populate_switch_ports.period_secs = 30 +# This task reads from the blueprint rendezvous table directly, so it doesn't +# need to run more often +vmm_mark_stop_for_update.period_secs = 300 [multicast] # Enable multicast functionality for tests (disabled by default in production) diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index 104bd5f2226..b213ac5d5d9 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -1432,6 +1432,18 @@ pub struct AuditLogCleanupStatus { pub error: Option, } +/// The status of a `vmm_mark_stop_for_update` background task activation. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct VmmMarkStopForUpdateStatus { + /// If `true`, this task has been explicitly disabled by the config file. + pub disabled: bool, + /// Number of VMMs that were marked as needing to be stopped for update in + /// this activation. + pub vmms_marked: usize, + /// Error encountered during this activation, if any. + pub error: Option, +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum SwitchPortPopulatorStatusKind { diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index 6981c1ee612..f374aa741a8 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -134,6 +134,9 @@ audit_log_cleanup.period_secs = 600 audit_log_cleanup.retention_days = 90 audit_log_cleanup.max_deleted_per_activation = 10000 populate_switch_ports.period_secs = 30 +# This task reads from the blueprint rendezvous table directly, so it doesn't +# need to run more often +vmm_mark_stop_for_update.period_secs = 300 [default_region_allocation_strategy] # by default, allocate across 3 distinct sleds diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index ce56cb14814..8887d0091f0 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -134,6 +134,9 @@ audit_log_cleanup.period_secs = 600 audit_log_cleanup.retention_days = 90 audit_log_cleanup.max_deleted_per_activation = 10000 populate_switch_ports.period_secs = 30 +# This task reads from the blueprint rendezvous table directly, so it doesn't +# need to run more often +vmm_mark_stop_for_update.period_secs = 300 [default_region_allocation_strategy] # by default, allocate without requirement for distinct sleds.