[sled agent] Retry loop around loading services (#3793)

Fixes #3790

Author: smklein

sled-agent/src/services.rs

@@ -527,9 +527,13 @@ impl ServiceManager {
     }
 
     // TODO(https://github.com/oxidecomputer/omicron/issues/2973):
-    // These will fail if the disks aren't attached.
-    // Should we have a retry loop here? Kinda like we have with the switch
-    // / NTP zone?
+    //
+    // The sled agent retries this function indefinitely at the call-site, but
+    // we could be smarter.
+    //
+    // - If we know that disks are missing, we could wait for them
+    // - We could permanently fail if we are able to distinguish other errors
+    // more clearly.
     pub async fn load_services(&self) -> Result<(), Error> {
         let log = &self.inner.log;
         let ledger_paths = self.all_service_ledgers().await;

sled-agent/src/sled_agent.rs

@@ -30,7 +30,8 @@ use omicron_common::api::{
     internal::nexus::UpdateArtifactId,
 };
 use omicron_common::backoff::{
-    retry_notify_ext, retry_policy_internal_service_aggressive, BackoffError,
+    retry_notify, retry_notify_ext, retry_policy_internal_service_aggressive,
+    BackoffError,
 };
 use sled_hardware::underlay;
 use sled_hardware::HardwareManager;
@@ -372,7 +373,25 @@ impl SledAgent {
         //
         // Do this *after* monitoring for harware, to enable the switch zone to
         // establish an underlay address before proceeding.
-        sled_agent.inner.services.load_services().await?;
+        retry_notify(
+            retry_policy_internal_service_aggressive(),
+            || async {
+                sled_agent
+                    .inner
+                    .services
+                    .load_services()
+                    .await
+                    .map_err(|err| BackoffError::transient(err))
+            },
+            |err, delay| {
+                warn!(
+                    log,
+                    "Failed to load services, will retry in {:?}", delay;
+                    "error" => %err,
+                );
+            },
+        )
+        .await?;
 
         // Now that we've initialized the sled services, notify nexus again
         // at which point it'll plumb any necessary firewall rules back to us.