[disk_delay] A disk wrapper with runtime configurable I/O read and write delays to any underlying disk (#1614)

gurasinghMS · mattkur · web-flow · commit 1fb1e2564fc0 · 2025-07-08T22:21:36.000Z
This change is part of changes for nvme keepalive testing and it will be
used in the nvme emulator to induce artificial delays and thus attempt
to capture perf of keepalive during servicing.
As a long term goal, the functionality in this crate will be expanded to
include configurable I/O drops, randomized slowdown and much more in
order to make testing of devices easier.

---------

Co-authored-by: Matt LaFayette (Kurjanowicz) &lt;mattkur@microsoft.com&gt;
diff --git a/Cargo.lock b/Cargo.lock
@@ -1144,6 +1144,7 @@ dependencies = [
  "stackfuture",
  "thiserror 2.0.12",
  "vm_resource",
+ "vmcore",
 ]
 
 [[package]]
@@ -1239,6 +1240,22 @@ dependencies = [
  "vm_resource",
 ]
 
+[[package]]
+name = "disk_delay"
+version = "0.0.0"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "disk_backend",
+ "disk_backend_resources",
+ "inspect",
+ "mesh",
+ "pal_async",
+ "scsi_buffers",
+ "vm_resource",
+ "vmcore",
+]
+
 [[package]]
 name = "disk_file"
 version = "0.0.0"
@@ -1286,6 +1303,7 @@ dependencies = [
  "thiserror 2.0.12",
  "tracelimit",
  "vm_resource",
+ "vmcore",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
@@ -46,6 +46,7 @@ members = [
   "vm/loader/igvmfilegen",
   "vm/vmgs/vmgs_lib",
   "vm/vmgs/vmgstool",
+  "vm/devices/storage/disk_delay",  # Used for internal microsoft testing
 ]
 exclude = [
   "xsync",
diff --git a/openhcl/underhill_core/src/dispatch/vtl2_settings_worker.rs b/openhcl/underhill_core/src/dispatch/vtl2_settings_worker.rs
@@ -59,6 +59,7 @@ use vm_resource::ResourceResolver;
 use vm_resource::kind::DiskHandleKind;
 use vm_resource::kind::PciDeviceHandleKind;
 use vm_resource::kind::VmbusDeviceHandleKind;
+use vmcore::vm_task::VmTaskDriverSource;
 
 #[derive(Error, Debug)]
 enum Error<'a> {
@@ -552,13 +553,14 @@ pub async fn disk_from_disk_type(
     disk_type: Resource<DiskHandleKind>,
     read_only: bool,
     resolver: &ResourceResolver,
+    driver_source: &VmTaskDriverSource,
 ) -> Result<Disk, Vtl2SettingsErrorInfo> {
     let disk = resolver
         .resolve(
             disk_type,
             ResolveDiskParameters {
                 read_only,
-                _async_trait_workaround: &(),
+                driver_source,
             },
         )
         .await
diff --git a/openhcl/underhill_core/src/worker.rs b/openhcl/underhill_core/src/worker.rs
@@ -2215,7 +2215,9 @@ async fn new_underhill_vm(
                         read_only,
                         disk_parameters,
                     } => {
-                        let disk = disk_from_disk_type(disk_type, read_only, &resolver).await?;
+                        let disk =
+                            disk_from_disk_type(disk_type, read_only, &resolver, &driver_source)
+                                .await?;
                         let scsi_disk = Arc::new(scsidisk::SimpleScsiDisk::new(
                             disk.clone(),
                             disk_parameters.unwrap_or_default(),
diff --git a/openvmm/hvlite_core/src/worker/dispatch.rs b/openvmm/hvlite_core/src/worker/dispatch.rs
@@ -241,13 +241,14 @@ async fn open_simple_disk(
     resolver: &ResourceResolver,
     disk_type: Resource<DiskHandleKind>,
     read_only: bool,
+    driver_source: &VmTaskDriverSource,
 ) -> anyhow::Result<Disk> {
     let disk = resolver
         .resolve(
             disk_type,
             ResolveDiskParameters {
                 read_only,
-                _async_trait_workaround: &(),
+                driver_source,
             },
         )
         .await?;
@@ -968,7 +969,7 @@ impl InitializedVm {
         let vmgs = match cfg.vmgs {
             Some(VmgsResource::Disk(disk)) => Some(
                 vmgs::Vmgs::try_open(
-                    open_simple_disk(&resolver, disk, false).await?,
+                    open_simple_disk(&resolver, disk, false, &driver_source).await?,
                     None,
                     true,
                     false,
@@ -978,7 +979,7 @@ impl InitializedVm {
             ),
             Some(VmgsResource::ReprovisionOnFailure(disk)) => Some(
                 vmgs::Vmgs::try_open(
-                    open_simple_disk(&resolver, disk, false).await?,
+                    open_simple_disk(&resolver, disk, false, &driver_source).await?,
                     None,
                     true,
                     true,
@@ -987,9 +988,12 @@ impl InitializedVm {
                 .context("failed to open vmgs file")?,
             ),
             Some(VmgsResource::Reprovision(disk)) => Some(
-                vmgs::Vmgs::format_new(open_simple_disk(&resolver, disk, false).await?, None)
-                    .await
-                    .context("failed to format vmgs file")?,
+                vmgs::Vmgs::format_new(
+                    open_simple_disk(&resolver, disk, false, &driver_source).await?,
+                    None,
+                )
+                .await
+                .context("failed to format vmgs file")?,
             ),
             Some(VmgsResource::Ephemeral) => None,
             // TODO: make sure we don't need a VMGS
@@ -1284,9 +1288,10 @@ impl InitializedVm {
                         read_only,
                         disk_parameters,
                     } => {
-                        let disk = open_simple_disk(&resolver, disk_type, read_only)
-                            .await
-                            .context("failed to open IDE disk")?;
+                        let disk =
+                            open_simple_disk(&resolver, disk_type, read_only, &driver_source)
+                                .await
+                                .context("failed to open IDE disk")?;
 
                         // Only disks get accelerator channels. DVDs dont.
                         let scsi_disk = ScsiControllerDisk::new(Arc::new(SimpleScsiDisk::new(
@@ -1392,7 +1397,7 @@ impl InitializedVm {
                     read_only,
                 } = disk_cfg;
 
-                let disk = open_simple_disk(&resolver, disk_type, read_only)
+                let disk = open_simple_disk(&resolver, disk_type, read_only, &driver_source)
                     .await
                     .context("failed to open floppy disk")?;
                 tracing::trace!("floppy opened based on config into DriveRibbon");
diff --git a/vm/devices/get/guest_emulation_device/src/resolver.rs b/vm/devices/get/guest_emulation_device/src/resolver.rs
@@ -88,7 +88,7 @@ impl AsyncResolveResource<VmbusDeviceHandleKind, GuestEmulationDeviceHandle>
                         disk,
                         ResolveDiskParameters {
                             read_only: false,
-                            _async_trait_workaround: &(),
+                            driver_source: input.driver_source,
                         },
                     )
                     .await
diff --git a/vm/devices/storage/disk_backend/Cargo.toml b/vm/devices/storage/disk_backend/Cargo.toml
@@ -8,6 +8,7 @@ rust-version.workspace = true
 
 [dependencies]
 scsi_buffers.workspace = true
+vmcore.workspace = true
 
 guestmem.workspace = true
 vm_resource.workspace = true
diff --git a/vm/devices/storage/disk_backend/src/resolve.rs b/vm/devices/storage/disk_backend/src/resolve.rs
@@ -8,6 +8,7 @@ use crate::DiskIo;
 use crate::InvalidDisk;
 use vm_resource::CanResolveTo;
 use vm_resource::kind::DiskHandleKind;
+use vmcore::vm_task::VmTaskDriverSource;
 
 impl CanResolveTo<ResolvedDisk> for DiskHandleKind {
     type Input<'a> = ResolveDiskParameters<'a>;
@@ -18,11 +19,8 @@ impl CanResolveTo<ResolvedDisk> for DiskHandleKind {
 pub struct ResolveDiskParameters<'a> {
     /// Whether the disk is being opened for read-only use.
     pub read_only: bool,
-    #[doc(hidden)]
-    // Workaround for async_trait not working well with GAT input parameters
-    // with missing lifetimes. Remove once we stop using async_trait for async
-    // resolvers.
-    pub _async_trait_workaround: &'a (),
+    /// The task driver source for the VM.
+    pub driver_source: &'a VmTaskDriverSource,
 }
 
 /// A resolved [`Disk`].
diff --git a/vm/devices/storage/disk_backend_resources/src/lib.rs b/vm/devices/storage/disk_backend_resources/src/lib.rs
@@ -7,7 +7,9 @@
 
 pub mod layer;
 
+use mesh::Cell;
 use mesh::MeshPayload;
+use std::time::Duration;
 use vm_resource::IntoResource;
 use vm_resource::Resource;
 use vm_resource::ResourceId;
@@ -33,6 +35,19 @@ impl ResourceId<DiskHandleKind> for DiskWithReservationsHandle {
     const ID: &'static str = "prwrap";
 }
 
+/// Disk handle for a delay disk.
+#[derive(MeshPayload)]
+pub struct DelayDiskHandle {
+    /// The underlying disk resource.
+    pub disk: Resource<DiskHandleKind>,
+    /// The delay to add to each I/O operation.
+    pub delay: Cell<Duration>,
+}
+
+impl ResourceId<DiskHandleKind> for DelayDiskHandle {
+    const ID: &'static str = "delay";
+}
+
 /// Disk handle for a fixed VHD1 disk.
 #[derive(MeshPayload)]
 pub struct FixedVhd1DiskHandle(pub std::fs::File);
diff --git a/vm/devices/storage/disk_crypt/src/resolver.rs b/vm/devices/storage/disk_crypt/src/resolver.rs
@@ -53,7 +53,7 @@ impl AsyncResolveResource<DiskHandleKind, DiskCryptHandle> for DiskCryptResolver
                 resource.disk,
                 ResolveDiskParameters {
                     read_only: input.read_only,
-                    _async_trait_workaround: &(),
+                    driver_source: input.driver_source,
                 },
             )
             .await
diff --git a/vm/devices/storage/disk_delay/Cargo.toml b/vm/devices/storage/disk_delay/Cargo.toml
@@ -0,0 +1,25 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+[package]
+name = "disk_delay"
+edition.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+vmcore.workspace = true
+vm_resource.workspace = true
+pal_async.workspace = true
+async-trait.workspace = true
+
+disk_backend.workspace = true
+disk_backend_resources.workspace = true
+scsi_buffers.workspace = true
+
+mesh.workspace = true
+inspect.workspace = true
+
+anyhow.workspace = true
+
+[lints]
+workspace = true
diff --git a/vm/devices/storage/disk_delay/src/lib.rs b/vm/devices/storage/disk_delay/src/lib.rs
@@ -0,0 +1,133 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! A disk device wrapper that provides configurable storage delay on Read/Write I/O operations to a disk.
+
+#![forbid(unsafe_code)]
+
+/// Provides a disk with a delay on every I/O operation.
+pub mod resolver;
+
+use disk_backend::Disk;
+use disk_backend::DiskError;
+use disk_backend::DiskIo;
+use disk_backend::UnmapBehavior;
+use inspect::Inspect;
+use mesh::Cell;
+use pal_async::timer::PolledTimer;
+use scsi_buffers::RequestBuffers;
+use std::time::Duration;
+use vmcore::vm_task::VmTaskDriver;
+use vmcore::vm_task::VmTaskDriverSource;
+
+/// A disk with delay on every I/O operation.
+#[derive(Inspect)]
+pub struct DelayDisk {
+    #[inspect(hex, with = "|x| inspect::AsDebug(x.get())")]
+    delay: Cell<Duration>,
+    inner: Disk,
+    driver: VmTaskDriver,
+}
+
+impl DelayDisk {
+    /// Creates a new disk with a specified delay on I/O operations.
+    pub fn new(delay: Cell<Duration>, inner: Disk, driver_source: &VmTaskDriverSource) -> Self {
+        Self {
+            delay,
+            inner,
+            driver: driver_source.simple(),
+        }
+    }
+}
+
+impl DiskIo for DelayDisk {
+    fn disk_type(&self) -> &str {
+        "delay"
+    }
+
+    /// Passthrough
+    fn sector_count(&self) -> u64 {
+        self.inner.sector_count()
+    }
+
+    /// Passthrough
+    fn sector_size(&self) -> u32 {
+        self.inner.sector_size()
+    }
+
+    /// Passthrough
+    fn disk_id(&self) -> Option<[u8; 16]> {
+        self.inner.disk_id()
+    }
+
+    /// Passthrough
+    fn physical_sector_size(&self) -> u32 {
+        self.inner.physical_sector_size()
+    }
+
+    /// Passthrough
+    fn is_fua_respected(&self) -> bool {
+        self.inner.is_fua_respected()
+    }
+
+    /// Passthrough
+    fn is_read_only(&self) -> bool {
+        self.inner.is_read_only()
+    }
+
+    /// Passthrough
+    fn pr(&self) -> Option<&dyn disk_backend::pr::PersistentReservation> {
+        self.inner.pr()
+    }
+
+    /// Delay and then Passthrough
+    async fn read_vectored(
+        &self,
+        buffers: &RequestBuffers<'_>,
+        sector: u64,
+    ) -> Result<(), DiskError> {
+        PolledTimer::new(&self.driver).sleep(self.delay.get()).await;
+        self.inner.read_vectored(buffers, sector).await
+    }
+
+    /// Delay and then Passthrough
+    async fn write_vectored(
+        &self,
+        buffers: &RequestBuffers<'_>,
+        sector: u64,
+        fua: bool,
+    ) -> Result<(), DiskError> {
+        PolledTimer::new(&self.driver).sleep(self.delay.get()).await;
+        self.inner.write_vectored(buffers, sector, fua).await
+    }
+
+    /// Passthrough
+    async fn sync_cache(&self) -> Result<(), DiskError> {
+        self.inner.sync_cache().await
+    }
+
+    /// Passthrough
+    async fn wait_resize(&self, sector_count: u64) -> u64 {
+        self.inner.wait_resize(sector_count).await
+    }
+
+    /// Passthrough
+    fn unmap(
+        &self,
+        sector: u64,
+        count: u64,
+        block_level_only: bool,
+    ) -> impl std::future::Future<Output = Result<(), DiskError>> + Send {
+        self.inner.unmap(sector, count, block_level_only)
+    }
+
+    /// Passthrough
+    fn unmap_behavior(&self) -> UnmapBehavior {
+        self.inner.unmap_behavior()
+    }
+
+    /// Passthrough
+    fn optimal_unmap_sectors(&self) -> u32 {
+        self.inner.optimal_unmap_sectors()
+    }
+}
diff --git a/vm/devices/storage/disk_delay/src/resolver.rs b/vm/devices/storage/disk_delay/src/resolver.rs
diff --git a/vm/devices/storage/disk_layered/Cargo.toml b/vm/devices/storage/disk_layered/Cargo.toml
diff --git a/vm/devices/storage/disk_layered/src/resolve.rs b/vm/devices/storage/disk_layered/src/resolve.rs
diff --git a/vm/devices/storage/disk_layered/src/resolver.rs b/vm/devices/storage/disk_layered/src/resolver.rs
diff --git a/vm/devices/storage/nvme/src/resolver.rs b/vm/devices/storage/nvme/src/resolver.rs
diff --git a/vm/devices/storage/scsidisk/src/resolver.rs b/vm/devices/storage/scsidisk/src/resolver.rs

Original file line number	Diff line number	Diff line change
`@@ -46,6 +46,7 @@ members = [`
`46`	`46`	`"vm/loader/igvmfilegen",`
`47`	`47`	`"vm/vmgs/vmgs_lib",`
`48`	`48`	`"vm/vmgs/vmgstool",`
	`49`	`+ "vm/devices/storage/disk_delay", # Used for internal microsoft testing`
`49`	`50`	`]`
`50`	`51`	`exclude = [`
`51`	`52`	`"xsync",`
Original file line number	Diff line number	Diff line change
`@@ -88,7 +88,7 @@ impl AsyncResolveResource<VmbusDeviceHandleKind, GuestEmulationDeviceHandle>`
`88`	`88`	`disk,`
`89`	`89`	`ResolveDiskParameters {`
`90`	`90`	`read_only: false,`
`91`		`- _async_trait_workaround: &(),`
	`91`	`+ driver_source: input.driver_source,`
`92`	`92`	`},`
`93`	`93`	`)`
`94`	`94`	`.await`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ impl AsyncResolveResource<DiskHandleKind, DiskCryptHandle> for DiskCryptResolver`
`53`	`53`	`resource.disk,`
`54`	`54`	`ResolveDiskParameters {`
`55`	`55`	`read_only: input.read_only,`
`56`		`- _async_trait_workaround: &(),`
	`56`	`+ driver_source: input.driver_source,`
`57`	`57`	`},`
`58`	`58`	`)`
`59`	`59`	`.await`