[reconfigurator] background task for marking VMMs to be stopped for update

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -94,6 +94,7 @@ use nexus_types::internal_api::background::TufArtifactReplicationCounters;
 use nexus_types::internal_api::background::TufArtifactReplicationRequest;
 use nexus_types::internal_api::background::TufArtifactReplicationStatus;
 use nexus_types::internal_api::background::TufRepoPrunerStatus;
+use nexus_types::internal_api::background::VmmMarkStopForUpdateStatus;
 use nexus_types::internal_api::background::fm_rendezvous;
 use omicron_uuid_kinds::BlueprintUuid;
 use omicron_uuid_kinds::CollectionUuid;
@@ -1426,6 +1427,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
         "switch_port_config_manager" => {
             print_task_switch_port_settings_manager(details);
         }
+        "vmm_mark_stop_for_update" => {
+            print_task_vmm_mark_stop_for_update(details);
+        }
         _ => {
             println!(
                 "warning: unknown background task: {:?} \
@@ -2842,6 +2846,33 @@ fn print_task_audit_log_cleanup(details: &serde_json::Value) {
     };
 }
 
+fn print_task_vmm_mark_stop_for_update(details: &serde_json::Value) {
+    match serde_json::from_value::<VmmMarkStopForUpdateStatus>(details.clone())
+    {
+        Err(error) => eprintln!(
+            "warning: failed to interpret task details: {:?}: {:?}",
+            error, details
+        ),
+        Ok(status) => {
+            let VmmMarkStopForUpdateStatus { disabled, vmms_marked, error } =
+                status;
+
+            if disabled {
+                println!("    task explicitly disabled by config!");
+            }
+
+            const MARKED: &str = "VMMs marked:";
+            const ERROR: &str = "error:";
+            const WIDTH: usize = const_max_len(&[MARKED, ERROR]) + 1;
+
+            println!("    {MARKED:<WIDTH$}{}", vmms_marked);
+            if let Some(error) = &error {
+                println!("    {ERROR:<WIDTH$}{error}");
+            }
+        }
+    };
+}
+
 fn print_task_audit_log_timeout_incomplete(details: &serde_json::Value) {
     match serde_json::from_value::<AuditLogTimeoutIncompleteStatus>(
         details.clone(),

dev-tools/omdb/tests/env.out

@@ -263,6 +263,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vmm_mark_stop_for_update"
+    marks VMMs on evacuating sleds as needing to be stopped for an update
+
+
 task: "vpc_route_manager"
     propagates updated VPC routes to all OPTE ports
 
@@ -531,6 +535,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vmm_mark_stop_for_update"
+    marks VMMs on evacuating sleds as needing to be stopped for an update
+
+
 task: "vpc_route_manager"
     propagates updated VPC routes to all OPTE ports
 
@@ -786,6 +794,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vmm_mark_stop_for_update"
+    marks VMMs on evacuating sleds as needing to be stopped for an update
+
+
 task: "vpc_route_manager"
     propagates updated VPC routes to all OPTE ports
 

dev-tools/omdb/tests/successes.out

@@ -498,6 +498,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vmm_mark_stop_for_update"
+    marks VMMs on evacuating sleds as needing to be stopped for an update
+
+
 task: "vpc_route_manager"
     propagates updated VPC routes to all OPTE ports
 
@@ -1091,6 +1095,12 @@ task: "v2p_manager"
     started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
 warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})
 
+task: "vmm_mark_stop_for_update"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    VMMs marked: 0
+
 task: "vpc_route_manager"
   configured period: every <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
@@ -1813,6 +1823,12 @@ task: "v2p_manager"
     started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
 warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})
 
+task: "vmm_mark_stop_for_update"
+  configured period: every <REDACTED_DURATION>m
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    VMMs marked: 0
+
 task: "vpc_route_manager"
   configured period: every <REDACTED_DURATION>s
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>

nexus-config/src/nexus_config.rs

@@ -477,6 +477,8 @@ pub struct BackgroundTaskConfig {
     pub audit_log_cleanup: AuditLogCleanupConfig,
     /// configuration for populate switch ports task
     pub populate_switch_ports: PopulateSwitchPortsConfig,
+    /// configuration for the task that marks VMMs to stop for an update
+    pub vmm_mark_stop_for_update: VmmMarkStopForUpdateConfig,
 }
 
 #[serde_as]
@@ -523,6 +525,23 @@ pub struct AuditLogCleanupConfig {
     pub max_deleted_per_activation: u32,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct VmmMarkStopForUpdateConfig {
+    /// period (in seconds) for periodic activations of this task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+
+    /// disable marking VMMs to stop for a sled update.
+    ///
+    /// This is an emergency lever for support / operations. It should only be
+    /// necessary if something has gone extremely wrong.
+    ///
+    /// Default: Off
+    #[serde(default)]
+    pub disable: bool,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct PopulateSwitchPortsConfig {
@@ -1376,6 +1395,7 @@ mod test {
             audit_log_cleanup.retention_days = 90
             audit_log_cleanup.max_deleted_per_activation = 10000
             populate_switch_ports.period_secs = 31
+            vmm_mark_stop_for_update.period_secs = 300
             [default_region_allocation_strategy]
             type = "random"
             seed = 0
@@ -1656,6 +1676,10 @@ mod test {
                         populate_switch_ports: PopulateSwitchPortsConfig {
                             period_secs: Duration::from_secs(31),
                         },
+                        vmm_mark_stop_for_update: VmmMarkStopForUpdateConfig {
+                            period_secs: Duration::from_secs(300),
+                            disable: false,
+                        },
                     },
                     multicast: MulticastConfig { enabled: false },
                     default_region_allocation_strategy:
@@ -1772,6 +1796,7 @@ mod test {
             audit_log_cleanup.retention_days = 90
             audit_log_cleanup.max_deleted_per_activation = 10000
             populate_switch_ports.period_secs = 31
+            vmm_mark_stop_for_update.period_secs = 300
 
             [default_region_allocation_strategy]
             type = "random"

nexus/background-task-interface/src/init.rs

@@ -63,6 +63,7 @@ pub struct BackgroundTasks {
     pub task_attached_subnet_manager: Activator,
     pub task_session_cleanup: Activator,
     pub task_populate_switch_ports: Activator,
+    pub task_vmm_mark_stop_for_update: Activator,
 
     // Handles to activate background tasks that do not get used by Nexus
     // at-large.  These background tasks are implementation details as far as