Use oxide-tokio-rt crate for common Tokio runtime config (#8412)

Currently, all the async Rust binaries in the Omicron workspace use the
`#[tokio::main]` attribute on an `async fn main()`. This creates a Tokio
runtime with the default configuration provided by that attribute, and
runs the contents of the `async` main function on that runtime.

We now have need to configure the Tokio runtime used by Omicron
differently than the defaults provided by the `#[tokio::main]` macro. In
particular, we want to do the following:
- Disable the LIFO slot optimization, which can cause severe latency
spikes in some cases (see

#8334 (comment))
- When running on illumos, configure runtime callbacks to emit
`tokio-dtrace` probes (see #8364)

In order to make it easier to use these configurations in all production
binaries in Omicron and other projects, I've introduced a new
[`oxide-tokio-rt`
crate](https://github.com/oxidecomputer/oxide-tokio-rt). This crate
provides
functions that configure a runtime with our desired settings and runs a
future
on it. This can be used to replace `#[tokio::main]` fairly simply, which
I've
done for most binaries in the repo.

There are a handful of binaries I did *not* update to use
`omicron-runtime`. In particular, I didn't change the use of
`#[tokio::main]` in example binaries (in particular, `oximeter-producer`
and `update-engine`), as I didn't want to add an `omicron-runtime`
dependency to those crates just for example. In the case of
`oximeter-producer`, in particular, which is depended on by code outside
of the Omicron repo, that would force those binaries to compile
`omicron-runtime` unnecessarily. Similarly, I didn't make some of the
dev-tools
binaries which are used in xtasks use `oxide-tokio-rt`, because I didn't
want to introduce another dependency that has to be compiled before an
xtask can run, and that felt more important than adding DTrace probes to
those simple commands --- if something goes wrong with them, we can add
the proves later. The exception to this are the `mgs-dev` and
`omicron-dev` binaries, which run MGS and Nexus *in-process*; in those
cases, I felt it was useful to have the DTrace probes enabled when
running in development. Similarly, I also thought it was worthwhile to
use `omicron-runtime` in OMDB, as it's deployed on production systems
and would be harder to recompile with DTrace probes if we end up
wanting them.

Closes #8364

Author: hawkw

.cargo/config.toml

@@ -11,8 +11,15 @@
 # - .github/buildomat/jobs/check-features.sh
 # - .github/workflows/rust.yml
 #
+# WARNING: If you're setting `RUSTFLAGS` in a global `~/.cargo/config.toml`, or
+# as an environment variable in your shell, this will CONFLICT WITH THE
+# CONFIGURATION HERE, as `build.rustflags` configuration is NOT ADDITIVE.
+# This is particularly relevant for folks using the `mold` linker via
+# `RUSTFLAGS`. `mold` users should consider using `mold -run cargo` instead, as
+# described here: https://github.com/rui314/mold#how-to-use
 [build]
 rustdocflags = "--document-private-items"
+rustflags = "--cfg tokio_unstable"
 
 # On illumos, use `-znocompstrtab` to reduce link time. We also add the Oxide
 # specific platform directory to the RPATH where additional libraries can be
@@ -57,7 +64,9 @@ rustdocflags = "--document-private-items"
 # variety of things such as host OS.
 [target.x86_64-unknown-illumos]
 rustflags = [
-    "-C", "link-arg=-Wl,-znocompstrtab,-R/usr/platform/oxide/lib/amd64"
+    "-C", "link-arg=-Wl,-znocompstrtab,-R/usr/platform/oxide/lib/amd64",
+    # enable the `tokio-unstable` config for `tokio-dtrace` probes
+    "--cfg", "tokio_unstable"
 ]
 
 # Set up `cargo xtask`.

.github/buildomat/build-and-test.sh

@@ -92,7 +92,7 @@ banner ls-apis
 # from end-to-end-tests.
 #
 banner build
-export RUSTFLAGS="-D warnings"
+export RUSTFLAGS="--cfg tokio_unstable -D warnings"
 export RUSTDOCFLAGS="--document-private-items -D warnings"
 # When running on illumos we need to pass an additional runpath that is
 # usually configured via ".cargo/config" but the `RUSTFLAGS` env variable

.github/buildomat/jobs/clippy.sh

@@ -34,4 +34,4 @@ ptime -m bash ./tools/install_builder_prerequisites.sh -y
 banner clippy
 export CARGO_INCREMENTAL=0
 ptime -m cargo xtask clippy
-RUSTDOCFLAGS="--document-private-items -D warnings" ptime -m cargo doc --workspace --no-deps
+RUSTDOCFLAGS="--document-private-items -D warnings --cfg tokio" ptime -m cargo doc --workspace --no-deps

Cargo.lock

@@ -564,6 +564,7 @@ omicron-test-utils = { path = "test-utils" }
 omicron-workspace-hack = "0.1.0"
 omicron-zone-package = "0.12.2"
 oxide-client = { path = "clients/oxide-client" }
+oxide-tokio-rt = "0.1.1"
 oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "f5560fae02ad3fc349fabc6454c321143199ca9e", features = [ "api", "std" ] }
 oxlog = { path = "dev-tools/oxlog" }
 oxnet = "0.1.2"

Cargo.toml

@@ -17,6 +17,7 @@ http.workspace = true
 illumos-utils.workspace = true
 omicron-common.workspace = true
 omicron-uuid-kinds.workspace = true
+oxide-tokio-rt.workspace = true
 oximeter-db.workspace = true
 schemars.workspace = true
 slog.workspace = true

clickhouse-admin/Cargo.toml

@@ -39,9 +39,8 @@ enum Args {
     },
 }
 
-#[tokio::main]
-async fn main() {
-    if let Err(err) = main_impl().await {
+fn main() {
+    if let Err(err) = oxide_tokio_rt::run(main_impl()) {
         fatal(err);
     }
 }

clickhouse-admin/src/bin/clickhouse-admin-keeper.rs

@@ -39,9 +39,8 @@ enum Args {
     },
 }
 
-#[tokio::main]
-async fn main() {
-    if let Err(err) = main_impl().await {
+fn main() {
+    if let Err(err) = oxide_tokio_rt::run(main_impl()) {
         fatal(err);
     }
 }

clickhouse-admin/src/bin/clickhouse-admin-server.rs

@@ -38,9 +38,8 @@ enum Args {
     },
 }
 
-#[tokio::main]
-async fn main() {
-    if let Err(err) = main_impl().await {
+fn main() {
+    if let Err(err) = oxide_tokio_rt::run(main_impl()) {
         fatal(err);
     }
 }

clickhouse-admin/src/bin/clickhouse-admin-single.rs

@@ -28,3 +28,12 @@ disallowed-types = [
     { path = "once_cell::unsync::Lazy", reason = "use `std::cell::LazyCell` instead" },
     { path = "once_cell::unsync::LazyCell", reason = "use `std::cell::LazyCell` instead" },
 ]
+
+[[disallowed-macros]]
+path = "tokio::main"
+reason = """\
+    Binaries deployed in production should use the `oxide_tokio_rt` \
+    crate, as it enables DTrace probes and runtime configuration. \
+    dev-tools, xtasks, and examples may use `#[tokio::main]` when \
+    necessary to avoid an `omicron-runtime` dependency.\
+"""