From 10410439a97e454021806691e846c95c7ad65849 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Thu, 10 Jul 2025 03:05:42 +0000 Subject: [PATCH 1/2] Add USDT probes for Nexus background tasks Partially closes #8424 --- Cargo.lock | 1 + nexus/Cargo.toml | 1 + nexus/src/app/background/driver.rs | 24 +++++++++++++++++++++++- nexus/src/app/background/mod.rs | 22 ++++++++++++++++++++++ tools/dtrace/nexus/trace-bgtasks.d | 27 +++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100755 tools/dtrace/nexus/trace-bgtasks.d diff --git a/Cargo.lock b/Cargo.lock index f28c5c6bfe0..48931572906 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7726,6 +7726,7 @@ dependencies = [ "tufaceous-lib", "update-common", "update-engine", + "usdt", "uuid", "zip 4.2.0", ] diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index acb32749550..e29c0acaa1e 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -108,6 +108,7 @@ tokio-postgres = { workspace = true, features = ["with-serde_json-1"] } tokio-util = { workspace = true, features = ["codec"] } tough.workspace = true tufaceous-artifact.workspace = true +usdt.workspace = true uuid.workspace = true nexus-auth.workspace = true diff --git a/nexus/src/app/background/driver.rs b/nexus/src/app/background/driver.rs index 853ba798bbd..119e4e6b5f7 100644 --- a/nexus/src/app/background/driver.rs +++ b/nexus/src/app/background/driver.rs @@ -7,6 +7,8 @@ //! None of this file is specific to Nexus or any of the specific background //! tasks in Nexus, although the design is pretty bespoke for what Nexus needs. +use crate::app::background::probes; + use super::BackgroundTask; use super::TaskName; use assert_matches::assert_matches; @@ -129,6 +131,7 @@ impl Driver { name.clone(), )])); let task_exec = TaskExec::new( + &name, taskdef.period, taskdef.task_impl, activator.clone(), @@ -189,6 +192,7 @@ impl Driver { /// If the task is currently running, it will be activated again when it /// finishes. pub(super) fn activate(&self, task: &TaskName) { + probes::background__task__activate!(|| task.as_str()); self.task_required(task).activator.activate(); } @@ -234,6 +238,8 @@ pub struct TaskDefinition<'a, N: ToString, D: ToString> { /// Encapsulates state needed by the background tokio task to manage activation /// of the background task struct TaskExec { + /// the name of this background task + name: String, /// how often the background task should be activated period: Duration, /// impl of the background task @@ -251,13 +257,23 @@ struct TaskExec { impl TaskExec { fn new( + name: impl ToString, period: Duration, imp: Box, activation: Activator, opctx: OpContext, status_tx: watch::Sender, ) -> TaskExec { - TaskExec { period, imp, activation, opctx, status_tx, iteration: 0 } + let name = name.to_string(); + TaskExec { + name, + period, + imp, + activation, + opctx, + status_tx, + iteration: 0, + } } /// Body of the tokio task that manages activation of this background task @@ -318,10 +334,16 @@ impl TaskExec { }); // Do it! + probes::background__task__activate__start!(|| { + (&self.name, self.iteration, format!("{reason:?}")) + }); let details = self.imp.activate(&self.opctx).await; let details_str = serde_json::to_string(&details).unwrap_or_else(|e| { format!("<>", e) }); + probes::background__task__activate__done!(|| { + (&self.name, self.iteration, &details_str) + }); let elapsed = start_instant.elapsed(); diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs index 26b0e34995c..4ab91bf6fec 100644 --- a/nexus/src/app/background/mod.rs +++ b/nexus/src/app/background/mod.rs @@ -166,3 +166,25 @@ impl TaskName { &self.0 } } + +#[usdt::provider(provider = "nexus")] +mod probes { + /// Fires just before explicitly activating the named background task. + fn background__task__activate(task_name: &str) {} + + /// Fires just before running the activate method of the named task. + fn background__task__activate__start( + task_name: &str, + iteration: u64, + reason: &str, + ) { + } + + /// Fires just after completing the task, with the result as JSON. + fn background__task__activate__done( + task_name: &str, + iteration: u64, + details: &str, + ) { + } +} diff --git a/tools/dtrace/nexus/trace-bgtasks.d b/tools/dtrace/nexus/trace-bgtasks.d new file mode 100755 index 00000000000..3961dbdded6 --- /dev/null +++ b/tools/dtrace/nexus/trace-bgtasks.d @@ -0,0 +1,27 @@ +#!/usr/sbin/dtrace -Zqs + +nexus*:::background-task-activate-start +{ + this->task_name = copyinstr(arg0); + iters[this->task_name] = arg1; + reasons[this->task_name] = copyinstr(arg2); + ts[this->task_name] = timestamp; +} + +nexus*:::background-task-activate-done +/iters[this->task_name]/ +{ + this->task_Name = copyinstr(arg0); + this->duration = timestamp - ts[this->task_name]; + printf( + "task_name=\"%s\" iteration=%d reason=%s details=%s duration=%dus\n", + this->task_name, + iters[this->task_name], + reasons[this->task_name], + copyinstr(arg2), + this->duration / 1000 + ); + iters[this->task_name] = 0; + reasons[this->task_name] = 0; + ts[this->task_name] = 0; +} From bb5769e9270374c974b8348f4ee2207803c0c015 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Thu, 10 Jul 2025 23:45:44 +0000 Subject: [PATCH 2/2] typo --- tools/dtrace/nexus/trace-bgtasks.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dtrace/nexus/trace-bgtasks.d b/tools/dtrace/nexus/trace-bgtasks.d index 3961dbdded6..6eb72cd8248 100755 --- a/tools/dtrace/nexus/trace-bgtasks.d +++ b/tools/dtrace/nexus/trace-bgtasks.d @@ -11,7 +11,7 @@ nexus*:::background-task-activate-start nexus*:::background-task-activate-done /iters[this->task_name]/ { - this->task_Name = copyinstr(arg0); + this->task_name = copyinstr(arg0); this->duration = timestamp - ts[this->task_name]; printf( "task_name=\"%s\" iteration=%d reason=%s details=%s duration=%dus\n",