From cc6bd829e4a45e246ffa75b84f5e17048fae0000 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 16 Jul 2025 13:00:59 -0700 Subject: [PATCH 1/4] Use `oxide-tokio-rt` rather than `#[tokio::main]` As described in [RFD 579], we are now recommending certain non-default Tokio runtime configurations in all software we deploy in production. The [`oxide-tokio-rt`] crate provides a way to easily get these common recommended configurations by using its functions to initialize the runtime rather than `#[tokio::main]` Presently, `oxide-tokio-rt` does the following: - Enables DTrace probes for Tokio runtime events using [`tokio-dtrace`] - Disables the [LIFO slot optimization][lifo], which was the root cause of omicron#8334 But, we anticipate that if there are additional Tokio configurations which we want to recommend in the future, we'll add them here as well. This commit updates `dendrite` to initialize the Tokio runtime using `oxide-tokio-rt`. I've added a Clippy config for warning on uses of `#[tokio::main]`, as well, to avoid forgetting to use `oxide-tokio-rt` if new binaries are added. All the configs we set currently in `oxide-tokio-rt` requires the `tokio_unstable` RUSTFLAGS cfg, but it looks like that's already being set in order to use `tokio-conssole`, so that shouldn't be a problem. [RFD 579]: https://rfd.shared.oxide.computer/rfd/0579 [`oxide-tokio-rt`]: https://github.com/oxidecomputer/oxide-tokio-rt [`tokio-dtrace`]: https://github.com/oxidecomputer/tokio-dtrace [lifo]: https://rfd.shared.oxide.computer/rfd/0579#_disabling_the_lifo_slot_optimization --- Cargo.lock | 70 ++++++++++++++++++++++++++++++++++----------- Cargo.toml | 1 + clippy.toml | 4 +++ dpd/src/main.rs | 5 ++-- swadm/Cargo.toml | 1 + swadm/src/main.rs | 10 +++++-- tfportd/Cargo.toml | 1 + tfportd/src/main.rs | 7 +++-- uplinkd/Cargo.toml | 1 + uplinkd/src/main.rs | 7 +++-- xtask/src/main.rs | 5 ++++ 11 files changed, 87 insertions(+), 25 deletions(-) create mode 100644 clippy.toml diff --git a/Cargo.lock b/Cargo.lock index 4e9c672..f43809e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1427,7 +1427,7 @@ dependencies = [ "slog-bunyan", "slog-json", "slog-term", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tokio-rustls 0.25.0", "toml 0.8.19", @@ -2581,6 +2581,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6d6206008e25125b1f97fbe5d309eb7b85141cf9199d52dbd3729a1584dd16" +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.2" @@ -3586,7 +3597,7 @@ dependencies = [ "oxide-vpc", "postcard", "serde", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -3595,6 +3606,17 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +[[package]] +name = "oxide-tokio-rt" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bd87abf37c68d414e4df90a545857542140e07206f75039b8f63b244da87b8" +dependencies = [ + "anyhow", + "tokio", + "tokio-dtrace", +] + [[package]] name = "oxide-vpc" version = "0.1.0" @@ -4250,7 +4272,7 @@ dependencies = [ "serde", "serde_json", "syn 2.0.100", - "thiserror 2.0.11", + "thiserror 2.0.12", "typify 0.3.0", "unicode-ident", ] @@ -4412,7 +4434,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.16", "socket2", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tracing", ] @@ -4431,7 +4453,7 @@ dependencies = [ "rustls 0.23.16", "rustls-pki-types", "slab", - "thiserror 2.0.11", + "thiserror 2.0.12", "tinyvec", "tracing", "web-time", @@ -5526,6 +5548,7 @@ dependencies = [ "common 0.1.0", "dpd-client", "futures", + "oxide-tokio-rt", "oxnet", "slog", "structopt", @@ -5736,6 +5759,7 @@ dependencies = [ "libc", "lldpd-client", "omicron-common", + "oxide-tokio-rt", "oximeter", "oximeter-instruments", "oximeter-producer", @@ -5766,11 +5790,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.12", ] [[package]] @@ -5786,9 +5810,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", @@ -5885,23 +5909,36 @@ dependencies = [ [[package]] name = "tokio" -version = "1.44.2" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", + "slab", "socket2", "tokio-macros", "tracing", "windows-sys 0.52.0", ] +[[package]] +name = "tokio-dtrace" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5eb4bcf85c373ff09a8beb87a477c2df185cd8842a770386a88bc3ff7ac5abb" +dependencies = [ + "thiserror 2.0.12", + "tokio", + "usdt 0.5.0", +] + [[package]] name = "tokio-macros" version = "2.5.0" @@ -6175,7 +6212,7 @@ dependencies = [ "slog-async", "slog-term", "tabled", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "transceiver-decode", "transceiver-messages", @@ -6191,7 +6228,7 @@ dependencies = [ "schemars", "serde", "static_assertions", - "thiserror 2.0.11", + "thiserror 2.0.12", "transceiver-messages", ] @@ -6205,7 +6242,7 @@ dependencies = [ "hubpack", "schemars", "serde", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -6282,7 +6319,7 @@ dependencies = [ "serde", "serde_json", "syn 2.0.100", - "thiserror 2.0.11", + "thiserror 2.0.12", "unicode-ident", ] @@ -6420,6 +6457,7 @@ dependencies = [ "anyhow", "common 0.1.0", "libc", + "oxide-tokio-rt", "oxnet", "signal-hook", "slog", diff --git a/Cargo.toml b/Cargo.toml index 35ea0b5..3a41b0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ libc = "0.2" mockall = "0.13.1" omicron-zone-package = "0.12" openssl = "0.10" +oxide-tokio-rt = "0.1.2" parking_lot = "0.12" pretty_assertions = "1.4" proc-macro2 = "1.0" diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..90bc207 --- /dev/null +++ b/clippy.toml @@ -0,0 +1,4 @@ +[[disallowed-macros]] +path = "tokio::main" +reason = "prefer `oxide_tokio_rt` for production software" +replacement = "oxide_tokio_rt::run" diff --git a/dpd/src/main.rs b/dpd/src/main.rs index 38f6acb..38786e6 100644 --- a/dpd/src/main.rs +++ b/dpd/src/main.rs @@ -797,13 +797,12 @@ async fn sidecar_main(mut switch: Switch) -> anyhow::Result<()> { Ok(()) } -#[tokio::main(flavor = "multi_thread")] -async fn main() -> anyhow::Result<()> { +fn main() -> anyhow::Result<()> { let args = Args::from_args(); match args { Args::Openapi => print_openapi(), - Args::Run(opt) => run_dpd(opt).await, + Args::Run(opt) => oxide_tokio_rt::run(run_dpd(opt)), } } diff --git a/swadm/Cargo.toml b/swadm/Cargo.toml index fab73e1..db73159 100644 --- a/swadm/Cargo.toml +++ b/swadm/Cargo.toml @@ -12,6 +12,7 @@ colored.workspace = true common.workspace = true dpd-client.workspace = true futures.workspace = true +oxide-tokio-rt.workspace = true oxnet.workspace = true slog.workspace = true structopt.workspace = true diff --git a/swadm/src/main.rs b/swadm/src/main.rs index 644c2fe..2c1af61 100644 --- a/swadm/src/main.rs +++ b/swadm/src/main.rs @@ -170,8 +170,14 @@ async fn build_info(client: &Client) -> anyhow::Result<()> { Ok(()) } -#[tokio::main(flavor = "current_thread")] -async fn main() -> anyhow::Result<()> { +fn main() -> anyhow::Result<()> { + oxide_tokio_rt::run_builder( + &mut oxide_tokio_rt::Builder::new_current_thread(), + main_impl, + ) +} + +async fn main_impl() -> anyhow::Result<()> { let opts = GlobalOpts::from_args(); let port = opts.port.unwrap_or_else(default_port); let host = opts.host.unwrap_or_else(|| "localhost".to_string()); diff --git a/tfportd/Cargo.toml b/tfportd/Cargo.toml index 98d0895..0cb3cb5 100644 --- a/tfportd/Cargo.toml +++ b/tfportd/Cargo.toml @@ -32,6 +32,7 @@ tokio.workspace = true uuid.workspace = true omicron-common.workspace = true +oxide-tokio-rt.workspace = true oximeter.workspace = true oximeter-instruments.workspace = true oximeter-producer.workspace = true diff --git a/tfportd/src/main.rs b/tfportd/src/main.rs index 13f2e72..5c08f82 100644 --- a/tfportd/src/main.rs +++ b/tfportd/src/main.rs @@ -252,8 +252,11 @@ fn pcap_open(log: &slog::Logger, pkt_src: &str) -> anyhow::Result { Ok(pcap) } -#[tokio::main] -async fn main() -> anyhow::Result<()> { +fn main() -> anyhow::Result<()> { + oxide_tokio_rt::run(main_impl) +} + +async fn main_impl() -> anyhow::Result<()> { let opts = Opt::from_args(); let config = config::build_config(&opts)?; diff --git a/uplinkd/Cargo.toml b/uplinkd/Cargo.toml index f80e86c..286299c 100644 --- a/uplinkd/Cargo.toml +++ b/uplinkd/Cargo.toml @@ -10,6 +10,7 @@ common.workspace = true anyhow.workspace = true libc.workspace = true +oxide-tokio-rt.workspace = true oxnet.workspace = true signal-hook.workspace = true slog.workspace = true diff --git a/uplinkd/src/main.rs b/uplinkd/src/main.rs index f0f85b3..acaa988 100644 --- a/uplinkd/src/main.rs +++ b/uplinkd/src/main.rs @@ -564,8 +564,11 @@ fn handle_signals(log: slog::Logger, tx: mpsc::Sender) { } } -#[tokio::main] -async fn main() -> anyhow::Result<()> { +fn main() -> anyhow::Result<()> { + oxide_tokio_rt::run(main_impl) +} + +async fn main_impl() -> anyhow::Result<()> { let opts = Opt::from_args(); let mut global = Global { diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 081da79..f1a4ef5 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -241,6 +241,11 @@ fn collect_binaries( collect(src, dst, binaries) } +#[expect( + clippy::disallowed_macros, + reason = "using `#[tokio::main]` in xtasks is fine, as they are not \ + deployed in production" +)] #[tokio::main] async fn main() { let task = Xtasks::from_args(); From 6a30743efb9b003e897f76caf95ee19fabc8c105 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 16 Jul 2025 14:02:16 -0700 Subject: [PATCH 2/4] You have to call the functions --- tfportd/src/main.rs | 2 +- uplinkd/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tfportd/src/main.rs b/tfportd/src/main.rs index 5c08f82..e6e55b0 100644 --- a/tfportd/src/main.rs +++ b/tfportd/src/main.rs @@ -253,7 +253,7 @@ fn pcap_open(log: &slog::Logger, pkt_src: &str) -> anyhow::Result { } fn main() -> anyhow::Result<()> { - oxide_tokio_rt::run(main_impl) + oxide_tokio_rt::run(main_impl()) } async fn main_impl() -> anyhow::Result<()> { diff --git a/uplinkd/src/main.rs b/uplinkd/src/main.rs index acaa988..2169dde 100644 --- a/uplinkd/src/main.rs +++ b/uplinkd/src/main.rs @@ -565,7 +565,7 @@ fn handle_signals(log: slog::Logger, tx: mpsc::Sender) { } fn main() -> anyhow::Result<()> { - oxide_tokio_rt::run(main_impl) + oxide_tokio_rt::run(main_impl()) } async fn main_impl() -> anyhow::Result<()> { From f4f989ad37a6dc5945bffa1146739d6dcd3fc62c Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 16 Jul 2025 14:08:51 -0700 Subject: [PATCH 3/4] You have to actually import the crate --- Cargo.lock | 1 + dpd/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f43809e..377eed2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1324,6 +1324,7 @@ dependencies = [ "nexus-client", "omicron-common", "openssl", + "oxide-tokio-rt", "oximeter", "oximeter-producer", "oxnet", diff --git a/dpd/Cargo.toml b/dpd/Cargo.toml index 968dac1..d2a90d8 100644 --- a/dpd/Cargo.toml +++ b/dpd/Cargo.toml @@ -59,6 +59,7 @@ internal-dns-resolver.workspace = true internal-dns-types.workspace = true nexus-client.workspace = true omicron-common.workspace = true +oxide-tokio-rt.workspace = true oximeter.workspace = true oximeter-producer.workspace = true smf.workspace = true From fe163cfba227a85e6ae065142922ec1b91f37dc4 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 16 Jul 2025 15:21:21 -0700 Subject: [PATCH 4/4] You have to actually call *all* the functions --- swadm/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swadm/src/main.rs b/swadm/src/main.rs index 2c1af61..757e9f0 100644 --- a/swadm/src/main.rs +++ b/swadm/src/main.rs @@ -173,7 +173,7 @@ async fn build_info(client: &Client) -> anyhow::Result<()> { fn main() -> anyhow::Result<()> { oxide_tokio_rt::run_builder( &mut oxide_tokio_rt::Builder::new_current_thread(), - main_impl, + main_impl(), ) }