implement userspace bounce buffering support

Add support to our virtio devices to allow userspace bounce buffering of
virtio buffers. This is an alternative to swiotlb.

Don't implement it for vhost-user-blk and for virtio-block with async
engine, because I have no idea how that would even work.

Signed-off-by: Patrick Roy <roypat@amazon.co.uk>

Author: roypat

src/vmm/src/device_manager/mmio.rs

@@ -600,6 +600,14 @@ mod tests {
 
         fn set_acked_features(&mut self, _: u64) {}
 
+        fn force_userspace_bounce_buffers(&mut self) {
+            todo!()
+        }
+
+        fn userspace_bounce_buffers(&self) -> bool {
+            todo!()
+        }
+
         fn device_type(&self) -> u32 {
             0
         }

src/vmm/src/devices/virtio/balloon/device.rs

@@ -557,6 +557,14 @@ impl VirtioDevice for Balloon {
         self.acked_features = acked_features;
     }
 
+    fn force_userspace_bounce_buffers(&mut self) {
+        // balloon device doesn't have a need for bounce buffers
+    }
+
+    fn userspace_bounce_buffers(&self) -> bool {
+        false
+    }
+
     fn device_type(&self) -> u32 {
         TYPE_BALLOON
     }

src/vmm/src/devices/virtio/block/device.rs

@@ -148,6 +148,20 @@ impl VirtioDevice for Block {
         }
     }
 
+    fn force_userspace_bounce_buffers(&mut self) {
+        match self {
+            Block::Virtio(b) => b.force_userspace_bounce_buffers(),
+            Block::VhostUser(b) => b.force_userspace_bounce_buffers(),
+        }
+    }
+
+    fn userspace_bounce_buffers(&self) -> bool {
+        match self {
+            Block::Virtio(b) => b.userspace_bounce_buffers(),
+            Block::VhostUser(b) => b.userspace_bounce_buffers(),
+        }
+    }
+
     fn device_type(&self) -> u32 {
         TYPE_BLOCK
     }

src/vmm/src/devices/virtio/block/vhost_user/device.rs

@@ -294,6 +294,14 @@ impl<T: VhostUserHandleBackend + Send + 'static> VirtioDevice for VhostUserBlock
         self.acked_features = acked_features;
     }
 
+    fn force_userspace_bounce_buffers(&mut self) {
+        // Nothing Firecracker can do about this, the backend would need to do the bouncing
+    }
+
+    fn userspace_bounce_buffers(&self) -> bool {
+        false
+    }
+
     fn device_type(&self) -> u32 {
         TYPE_BLOCK
     }

src/vmm/src/devices/virtio/block/virtio/device.rs

@@ -578,6 +578,20 @@ impl VirtioDevice for VirtioBlock {
         self.acked_features = acked_features;
     }
 
+    fn force_userspace_bounce_buffers(&mut self) {
+        match self.disk.file_engine {
+            FileEngine::Async(_) => panic!("No idea how this is supposed to work for io_uring"),
+            FileEngine::Sync(ref mut engine) => engine.start_bouncing(),
+        }
+    }
+
+    fn userspace_bounce_buffers(&self) -> bool {
+        match self.disk.file_engine {
+            FileEngine::Async(_) => false,
+            FileEngine::Sync(ref engine) => engine.is_bouncing(),
+        }
+    }
+
     fn device_type(&self) -> u32 {
         TYPE_BLOCK
     }

src/vmm/src/devices/virtio/block/virtio/io/sync_io.rs

@@ -6,7 +6,7 @@ use std::io::{Seek, SeekFrom, Write};
 
 use vm_memory::{GuestMemoryError, ReadVolatile, WriteVolatile};
 
-use crate::vstate::memory::{GuestAddress, GuestMemory, GuestMemoryMmap};
+use crate::vstate::memory::{GuestAddress, GuestMemory, GuestMemoryMmap, MaybeBounce};
 
 #[derive(Debug, thiserror::Error, displaydoc::Display)]
 pub enum SyncIoError {
@@ -22,25 +22,40 @@ pub enum SyncIoError {
 
 #[derive(Debug)]
 pub struct SyncFileEngine {
-    file: File,
+    // 65536 is the largest buffer a linux guest will give us, empirically. Determined by
+    // having `MaybeBounce` logging scenarios where the fixed size bounce buffer isn't sufficient.
+    // Note that even if this assumption ever changes, the worse that'll happen is that we do
+    // multiple roundtrips between guest memory and the bounce buffer, as MaybeBounce would
+    // just chop larger reads/writes into chunks of 65k.
+    file: MaybeBounce<File, { u16::MAX as usize + 1 }>,
 }
 
 // SAFETY: `File` is send and ultimately a POD.
 unsafe impl Send for SyncFileEngine {}
 
 impl SyncFileEngine {
     pub fn from_file(file: File) -> SyncFileEngine {
-        SyncFileEngine { file }
+        SyncFileEngine {
+            file: MaybeBounce::new_persistent(file, false),
+        }
     }
 
     #[cfg(test)]
     pub fn file(&self) -> &File {
-        &self.file
+        &self.file.target
+    }
+
+    pub fn start_bouncing(&mut self) {
+        self.file.activate()
+    }
+
+    pub fn is_bouncing(&self) -> bool {
+        self.file.is_activated()
     }
 
     /// Update the backing file of the engine
     pub fn update_file(&mut self, file: File) {
-        self.file = file
+        self.file.target = file
     }
 
     pub fn read(
@@ -77,8 +92,8 @@ impl SyncFileEngine {
 
     pub fn flush(&mut self) -> Result<(), SyncIoError> {
         // flush() first to force any cached data out of rust buffers.
-        self.file.flush().map_err(SyncIoError::Flush)?;
+        self.file.target.flush().map_err(SyncIoError::Flush)?;
         // Sync data out to physical media on host.
-        self.file.sync_all().map_err(SyncIoError::SyncAll)
+        self.file.target.sync_all().map_err(SyncIoError::SyncAll)
     }
 }

src/vmm/src/devices/virtio/block/virtio/persist.rs

@@ -16,7 +16,7 @@ use crate::devices::virtio::TYPE_BLOCK;
 use crate::devices::virtio::block::persist::BlockConstructorArgs;
 use crate::devices::virtio::block::virtio::device::FileEngineType;
 use crate::devices::virtio::block::virtio::metrics::BlockMetricsPerDevice;
-use crate::devices::virtio::device::{DeviceState, IrqTrigger};
+use crate::devices::virtio::device::{DeviceState, IrqTrigger, VirtioDevice};
 use crate::devices::virtio::generated::virtio_blk::VIRTIO_BLK_F_RO;
 use crate::devices::virtio::persist::VirtioDeviceState;
 use crate::rate_limiter::RateLimiter;
@@ -127,7 +127,7 @@ impl Persist<'_> for VirtioBlock {
             capacity: disk_properties.nsectors.to_le(),
         };
 
-        Ok(VirtioBlock {
+        let mut dev = VirtioBlock {
             avail_features,
             acked_features,
             config_space,
@@ -148,7 +148,13 @@ impl Persist<'_> for VirtioBlock {
             rate_limiter,
             is_io_engine_throttled: false,
             metrics: BlockMetricsPerDevice::alloc(state.id.clone()),
-        })
+        };
+
+        if state.virtio_state.bounce_in_userspace {
+            dev.force_userspace_bounce_buffers()
+        }
+
+        Ok(dev)
     }
 }
 

src/vmm/src/devices/virtio/device.rs

@@ -102,6 +102,12 @@ pub trait VirtioDevice: AsAny + Send {
     /// - self.avail_features() & self.acked_features() = self.get_acked_features()
     fn set_acked_features(&mut self, acked_features: u64);
 
+    /// Make the virtio device user userspace bounce buffers
+    fn force_userspace_bounce_buffers(&mut self);
+
+    /// Whether this device is using userspace bounce buffers
+    fn userspace_bounce_buffers(&self) -> bool;
+
     /// Check if virtio device has negotiated given feature.
     fn has_feature(&self, feature: u64) -> bool {
         (self.acked_features() & (1 << feature)) != 0
@@ -259,6 +265,14 @@ pub(crate) mod tests {
             todo!()
         }
 
+        fn force_userspace_bounce_buffers(&mut self) {
+            todo!()
+        }
+
+        fn userspace_bounce_buffers(&self) -> bool {
+            todo!()
+        }
+
         fn device_type(&self) -> u32 {
             todo!()
         }

src/vmm/src/devices/virtio/mmio.rs

@@ -423,6 +423,14 @@ pub(crate) mod tests {
             self.acked_features = acked_features;
         }
 
+        fn force_userspace_bounce_buffers(&mut self) {
+            unimplemented!()
+        }
+
+        fn userspace_bounce_buffers(&self) -> bool {
+            false
+        }
+
         fn device_type(&self) -> u32 {
             123
         }