add "secret_free" parameter to /machine-config endpoint

This will later indicate to Firecracker that guest memory should be
backed by guest_memfd.

Mark vhost-user and async block engine as incompatible, as I/O will
require userspace bounce buffers. For vhost-user-blk, we would need
to communicate the need for bounce buffers to the backend somehow, and
for the async block engine we would need to somehow keep the bounce
buffers around until io_uring finishes requests (which is not
impossible, but complicated and not needed for now).

Signed-off-by: Patrick Roy <roypat@amazon.co.uk>

Author: roypat

src/firecracker/src/api_server/request/machine_configuration.rs

@@ -119,6 +119,7 @@ mod tests {
             let expected_config = MachineConfigUpdate {
                 vcpu_count: Some(8),
                 mem_size_mib: Some(1024),
+                secret_free: Some(false),
                 smt: Some(false),
                 cpu_template: None,
                 track_dirty_pages: Some(false),
@@ -140,6 +141,7 @@ mod tests {
         let expected_config = MachineConfigUpdate {
             vcpu_count: Some(8),
             mem_size_mib: Some(1024),
+            secret_free: Some(false),
             smt: Some(false),
             cpu_template: Some(StaticCpuTemplate::None),
             track_dirty_pages: Some(false),
@@ -161,6 +163,7 @@ mod tests {
         let expected_config = MachineConfigUpdate {
             vcpu_count: Some(8),
             mem_size_mib: Some(1024),
+            secret_free: Some(false),
             smt: Some(false),
             cpu_template: None,
             track_dirty_pages: Some(true),
@@ -186,6 +189,7 @@ mod tests {
             let expected_config = MachineConfigUpdate {
                 vcpu_count: Some(8),
                 mem_size_mib: Some(1024),
+                secret_free: Some(false),
                 smt: Some(false),
                 cpu_template: Some(StaticCpuTemplate::T2),
                 track_dirty_pages: Some(true),
@@ -213,6 +217,7 @@ mod tests {
         let expected_config = MachineConfigUpdate {
             vcpu_count: Some(8),
             mem_size_mib: Some(1024),
+            secret_free: Some(false),
             smt: Some(true),
             cpu_template: None,
             track_dirty_pages: Some(true),

src/firecracker/swagger/firecracker.yaml

@@ -1044,6 +1044,11 @@ definitions:
       mem_size_mib:
         type: integer
         description: Memory size of VM
+      secret_free:
+        type: boolean
+        description:
+          If enabled, guest memory will be unmapped from the host kernel's address space, providing additional
+          protection against transitive execution issues. All I/O then goes through a bounce buffer.
       track_dirty_pages:
         type: boolean
         description:

src/vmm/src/device_manager/persist.rs

@@ -846,6 +846,7 @@ mod tests {
   "machine-config": {{
     "vcpu_count": 1,
     "mem_size_mib": 128,
+    "secret_free": false,
     "smt": false,
     "track_dirty_pages": false,
     "huge_pages": "None"

src/vmm/src/persist.rs

@@ -47,6 +47,8 @@ use crate::{EventManager, Vmm, vstate};
 pub struct VmInfo {
     /// Guest memory size.
     pub mem_size_mib: u64,
+    /// Memory config
+    pub secret_free: bool,
     /// smt information
     pub smt: bool,
     /// CPU template type
@@ -61,6 +63,7 @@ impl From<&VmResources> for VmInfo {
     fn from(value: &VmResources) -> Self {
         Self {
             mem_size_mib: value.machine_config.mem_size_mib as u64,
+            secret_free: value.machine_config.secret_free,
             smt: value.machine_config.smt,
             cpu_template: StaticCpuTemplate::from(&value.machine_config.cpu_template),
             boot_source: value.boot_source.config.clone(),
@@ -360,6 +363,7 @@ pub fn restore_from_snapshot(
         .update_machine_config(&MachineConfigUpdate {
             vcpu_count: Some(vcpu_count),
             mem_size_mib: Some(u64_to_usize(microvm_state.vm_info.mem_size_mib)),
+            secret_free: Some(microvm_state.vm_info.secret_free),
             smt: Some(microvm_state.vm_info.smt),
             cpu_template: Some(microvm_state.vm_info.cpu_template),
             track_dirty_pages: Some(track_dirty_pages),

src/vmm/src/resources.rs

@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::cpu_config::templates::CustomCpuTemplate;
 use crate::device_manager::persist::SharedDeviceType;
+use crate::devices::virtio::block::device::Block;
 use crate::logger::info;
 use crate::mmds;
 use crate::mmds::data_store::{Mmds, MmdsVersion};
@@ -231,6 +232,11 @@ impl VmResources {
                         BalloonConfigError::IncompatibleWith("huge pages"),
                     ));
                 }
+                if self.machine_config.secret_free {
+                    return Err(ResourcesError::BalloonDevice(
+                        BalloonConfigError::IncompatibleWith("secret freedom"),
+                    ));
+                }
             }
 
             SharedDeviceType::Vsock(vsock) => {
@@ -276,6 +282,27 @@ impl VmResources {
                 "huge pages",
             ));
         }
+        if self.balloon.get().is_some() && updated.secret_free {
+            return Err(MachineConfigError::Incompatible(
+                "balloon device",
+                "secret freedom",
+            ));
+        }
+        if updated.secret_free {
+            if self.vhost_user_devices_used() {
+                return Err(MachineConfigError::Incompatible(
+                    "vhost-user devices",
+                    "userspace bounce buffers",
+                ));
+            }
+
+            if self.async_block_engine_used() {
+                return Err(MachineConfigError::Incompatible(
+                    "async block engine",
+                    "userspace bounce buffers",
+                ));
+            }
+        }
         self.machine_config = updated;
 
         Ok(())
@@ -334,6 +361,10 @@ impl VmResources {
             return Err(BalloonConfigError::IncompatibleWith("huge pages"));
         }
 
+        if self.machine_config.secret_free {
+            return Err(BalloonConfigError::IncompatibleWith("secret freedom"));
+        }
+
         self.balloon.set(config)
     }
 
@@ -357,6 +388,17 @@ impl VmResources {
         &mut self,
         block_device_config: BlockDeviceConfig,
     ) -> Result<(), DriveError> {
+        if self.machine_config.secret_free {
+            if block_device_config.file_engine_type == Some(FileEngineType::Async) {
+                return Err(DriveError::IncompatibleWithSecretFreedom(
+                    "async file engine",
+                ));
+            }
+
+            if block_device_config.socket.is_some() {
+                return Err(DriveError::IncompatibleWithSecretFreedom("vhost-user-blk"));
+            }
+        }
         self.block.insert(block_device_config)
     }
 
@@ -456,17 +498,29 @@ impl VmResources {
         Ok(())
     }
 
+    /// Returns true if any vhost user devices are configured int his [`VmResources`] object
+    pub fn vhost_user_devices_used(&self) -> bool {
+        self.block
+            .devices
+            .iter()
+            .any(|b| b.lock().expect("Poisoned lock").is_vhost_user())
+    }
+
+    fn async_block_engine_used(&self) -> bool {
+        self.block
+            .devices
+            .iter()
+            .any(|b| match &*b.lock().unwrap() {
+                Block::Virtio(b) => b.file_engine_type() == FileEngineType::Async,
+                Block::VhostUser(_) => false,
+            })
+    }
+
     /// Allocates guest memory in a configuration most appropriate for these [`VmResources`].
     ///
     /// If vhost-user-blk devices are in use, allocates memfd-backed shared memory, otherwise
     /// prefers anonymous memory for performance reasons.
     pub fn allocate_guest_memory(&self) -> Result<Vec<GuestRegionMmap>, MemoryError> {
-        let vhost_user_device_used = self
-            .block
-            .devices
-            .iter()
-            .any(|b| b.lock().expect("Poisoned lock").is_vhost_user());
-
         // Page faults are more expensive for shared memory mapping, including  memfd.
         // For this reason, we only back guest memory with a memfd
         // if a vhost-user-blk device is configured in the VM, otherwise we fall back to
@@ -478,7 +532,7 @@ impl VmResources {
         // that would not be worth the effort.
         let regions =
             crate::arch::arch_memory_regions(0, mib_to_bytes(self.machine_config.mem_size_mib));
-        if vhost_user_device_used {
+        if self.vhost_user_devices_used() {
             memory::memfd_backed(
                 regions.as_ref(),
                 self.machine_config.track_dirty_pages,
@@ -1359,6 +1413,7 @@ mod tests {
         let mut aux_vm_config = MachineConfigUpdate {
             vcpu_count: Some(32),
             mem_size_mib: Some(512),
+            secret_free: Some(false),
             smt: Some(false),
             #[cfg(target_arch = "x86_64")]
             cpu_template: Some(StaticCpuTemplate::T2),

src/vmm/src/vmm_config/drive.rs

@@ -24,6 +24,8 @@ pub enum DriveError {
     DeviceUpdate(VmmError),
     /// A root block device already exists!
     RootBlockDeviceAlreadyAdded,
+    /// {0} is incompatible with secret freedom.
+    IncompatibleWithSecretFreedom(&'static str),
 }
 
 /// Use this structure to set up the Block Device before booting the kernel.

src/vmm/src/vmm_config/machine_config.rs

@@ -95,6 +95,11 @@ pub struct MachineConfig {
     pub vcpu_count: u8,
     /// The memory size in MiB.
     pub mem_size_mib: usize,
+    /// Whether guest_memfd should be used to back normal guest memory. If this is enabled
+    /// and any devices are attached to the VM, userspace bounce buffers will be used
+    /// as I/O into secret free memory is not possible.
+    #[serde(default)]
+    pub secret_free: bool,
     /// Enables or disabled SMT.
     #[serde(default)]
     pub smt: bool,
@@ -151,6 +156,7 @@ impl Default for MachineConfig {
         Self {
             vcpu_count: 1,
             mem_size_mib: DEFAULT_MEM_SIZE_MIB,
+            secret_free: false,
             smt: false,
             cpu_template: None,
             track_dirty_pages: false,
@@ -176,6 +182,9 @@ pub struct MachineConfigUpdate {
     /// The memory size in MiB.
     #[serde(default)]
     pub mem_size_mib: Option<usize>,
+    /// Whether secret freedom should be enabled
+    #[serde(default)]
+    pub secret_free: Option<bool>,
     /// Enables or disabled SMT.
     #[serde(default)]
     pub smt: Option<bool>,
@@ -208,6 +217,7 @@ impl From<MachineConfig> for MachineConfigUpdate {
         MachineConfigUpdate {
             vcpu_count: Some(cfg.vcpu_count),
             mem_size_mib: Some(cfg.mem_size_mib),
+            secret_free: Some(cfg.secret_free),
             smt: Some(cfg.smt),
             cpu_template: cfg.static_template(),
             track_dirty_pages: Some(cfg.track_dirty_pages),
@@ -261,11 +271,27 @@ impl MachineConfig {
 
         let mem_size_mib = update.mem_size_mib.unwrap_or(self.mem_size_mib);
         let page_config = update.huge_pages.unwrap_or(self.huge_pages);
+        let secret_free = update.secret_free.unwrap_or(self.secret_free);
+        let track_dirty_pages = update.track_dirty_pages.unwrap_or(self.track_dirty_pages);
 
         if mem_size_mib == 0 || !page_config.is_valid_mem_size(mem_size_mib) {
             return Err(MachineConfigError::InvalidMemorySize);
         }
 
+        if secret_free && page_config != HugePageConfig::None {
+            return Err(MachineConfigError::Incompatible(
+                "secret freedom",
+                "huge pages",
+            ));
+        }
+
+        if secret_free && track_dirty_pages {
+            return Err(MachineConfigError::Incompatible(
+                "secret freedom",
+                "diff snapshots",
+            ));
+        }
+
         let cpu_template = match update.cpu_template {
             None => self.cpu_template.clone(),
             Some(StaticCpuTemplate::None) => None,
@@ -275,9 +301,10 @@ impl MachineConfig {
         Ok(MachineConfig {
             vcpu_count,
             mem_size_mib,
+            secret_free,
             smt,
             cpu_template,
-            track_dirty_pages: update.track_dirty_pages.unwrap_or(self.track_dirty_pages),
+            track_dirty_pages,
             huge_pages: page_config,
             #[cfg(feature = "gdb")]
             gdb_socket_path: update.gdb_socket_path.clone(),
@@ -343,6 +370,32 @@ mod tests {
             .unwrap();
         assert_eq!(updated.huge_pages, HugePageConfig::Hugetlbfs2M);
         assert_eq!(updated.mem_size_mib, 32);
+
+        let res = mconf.update(&MachineConfigUpdate {
+            huge_pages: Some(HugePageConfig::Hugetlbfs2M),
+            secret_free: Some(true),
+            ..Default::default()
+        });
+        assert_eq!(
+            res,
+            Err(MachineConfigError::Incompatible(
+                "secret freedom",
+                "huge pages"
+            ))
+        );
+
+        let res = mconf.update(&MachineConfigUpdate {
+            track_dirty_pages: Some(true),
+            secret_free: Some(true),
+            ..Default::default()
+        });
+        assert_eq!(
+            res,
+            Err(MachineConfigError::Incompatible(
+                "secret freedom",
+                "diff snapshots"
+            ))
+        );
     }
 
     #[test]

tests/framework/vm_config.json

@@ -20,6 +20,7 @@
   "machine-config": {
     "vcpu_count": 2,
     "mem_size_mib": 1024,
+    "secret_free": false,
     "smt": false,
     "track_dirty_pages": false,
     "huge_pages": "None"

tests/integration_tests/functional/test_api.py

@@ -1062,6 +1062,7 @@ def test_get_full_config_after_restoring_snapshot(microvm_factory, uvm_nano):
     setup_cfg["machine-config"] = {
         "vcpu_count": 2,
         "mem_size_mib": 256,
+        "secret_free": False,
         "smt": True,
         "track_dirty_pages": False,
         "huge_pages": "None",
@@ -1175,6 +1176,7 @@ def test_get_full_config(uvm_plain):
     expected_cfg["machine-config"] = {
         "vcpu_count": 2,
         "mem_size_mib": 256,
+        "secret_free": False,
         "smt": False,
         "track_dirty_pages": False,
         "huge_pages": "None",