Use guest_memfd to back memory if secret freedom is enabled

If the `secret_free` field of the memory_config is set to true in the
/machine-config endpoint, back all memory regions using
guest_memfd. For our setup, this means both setting the
guest_memfd[_offset] fields in kvm_user_memory_region2, as well as
mmaping the guest memory and reflecting this VMA back into the memslot's
userspace_addr (which is how kvm internal accesses to guest memory will
work for these guest_memfd regions, such as mmio emulation on x86).

Signed-off-by: Patrick Roy <roypat@amazon.co.uk>

Author: roypat

src/vmm/benches/memory_access.rs

@@ -9,7 +9,7 @@ fn bench_single_page_fault(c: &mut Criterion, configuration: VmResources) {
     c.bench_function("page_fault", |b| {
         b.iter_batched(
             || {
-                let memory = configuration.allocate_guest_memory().unwrap();
+                let memory = configuration.allocate_guest_memory(None).unwrap();
                 // Get a pointer to the first memory region (cannot do `.get_slice(GuestAddress(0),
                 // 1)`, because on ARM64 guest memory does not start at physical
                 // address 0).

src/vmm/src/builder.rs

@@ -64,7 +64,7 @@ use crate::vmm_config::machine_config::MachineConfigError;
 use crate::vstate::kvm::Kvm;
 use crate::vstate::memory::{GuestRegionMmap, MaybeBounce};
 use crate::vstate::vcpu::{Vcpu, VcpuError};
-use crate::vstate::vm::Vm;
+use crate::vstate::vm::{KVM_GMEM_NO_DIRECT_MAP, Vm};
 use crate::{EventManager, Vmm, VmmError, device_manager};
 
 /// Errors associated with starting the instance.
@@ -222,10 +222,6 @@ pub fn build_microvm_for_boot(
         .as_ref()
         .ok_or(MissingKernelConfig)?;
 
-    let guest_memory = vm_resources
-        .allocate_guest_memory()
-        .map_err(StartMicrovmError::GuestMemory)?;
-
     // Clone the command-line so that a failed boot doesn't pollute the original.
     #[allow(unused_mut)]
     let mut boot_cmdline = boot_config.cmdline.clone();
@@ -235,6 +231,8 @@ pub fn build_microvm_for_boot(
         .cpu_template
         .get_cpu_template()?;
 
+    let secret_free = vm_resources.machine_config.secret_free;
+
     let (mut vmm, mut vcpus) = create_vmm_and_vcpus(
         instance_info,
         event_manager,
@@ -243,15 +241,25 @@ pub fn build_microvm_for_boot(
         vm_resources.machine_config.secret_free,
     )?;
 
+    let guest_memfd = match secret_free {
+        true => Some(
+            vmm.vm
+                .create_guest_memfd(vm_resources.memory_size(), KVM_GMEM_NO_DIRECT_MAP)
+                .map_err(VmmError::Vm)?,
+        ),
+        false => None,
+    };
+
+    let guest_memory = vm_resources
+        .allocate_guest_memory(guest_memfd)
+        .map_err(StartMicrovmError::GuestMemory)?;
+
     vmm.vm
         .register_memory_regions(guest_memory)
         .map_err(VmmError::Vm)?;
 
     let entry_point = load_kernel(
-        MaybeBounce::new(
-            boot_config.kernel_file.try_clone().unwrap(),
-            vmm.vm.secret_free(),
-        ),
+        MaybeBounce::new(boot_config.kernel_file.try_clone().unwrap(), secret_free),
         vmm.vm.guest_memory(),
     )?;
     let initrd = match &boot_config.initrd_file {
@@ -263,7 +271,7 @@ pub fn build_microvm_for_boot(
 
             Some(InitrdConfig::from_reader(
                 vmm.vm.guest_memory(),
-                MaybeBounce::new(initrd_file.as_fd(), vmm.vm.secret_free()),
+                MaybeBounce::new(initrd_file.as_fd(), secret_free),
                 u64_to_usize(size),
             )?)
         }

src/vmm/src/persist.rs

@@ -457,7 +457,7 @@ fn guest_memory_from_file(
     track_dirty_pages: bool,
 ) -> Result<Vec<GuestRegionMmap>, GuestMemoryFromFileError> {
     let mem_file = File::open(mem_file_path)?;
-    let guest_mem = memory::snapshot_file(mem_file, mem_state.regions(), track_dirty_pages)?;
+    let guest_mem = memory::file_private(mem_file, mem_state.regions(), track_dirty_pages)?;
     Ok(guest_mem)
 }
 

src/vmm/src/resources.rs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use std::convert::From;
+use std::fs::File;
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex, MutexGuard};
 
@@ -31,7 +32,7 @@ use crate::vmm_config::mmds::{MmdsConfig, MmdsConfigError};
 use crate::vmm_config::net::*;
 use crate::vmm_config::vsock::*;
 use crate::vstate::memory;
-use crate::vstate::memory::{GuestRegionMmap, MemoryError};
+use crate::vstate::memory::{GuestRegionMmap, MemoryError, create_memfd};
 
 /// Errors encountered when configuring microVM resources.
 #[derive(Debug, thiserror::Error, displaydoc::Display)]
@@ -516,12 +517,19 @@ impl VmResources {
             })
     }
 
+    /// Gets the size of the guest memory, in bytes
+    pub fn memory_size(&self) -> usize {
+        mib_to_bytes(self.machine_config.mem_size_mib)
+    }
+
     /// Allocates guest memory in a configuration most appropriate for these [`VmResources`].
     ///
     /// If vhost-user-blk devices are in use, allocates memfd-backed shared memory, otherwise
     /// prefers anonymous memory for performance reasons.
-    pub fn allocate_guest_memory(&self) -> Result<Vec<GuestRegionMmap>, MemoryError> {
-        // Page faults are more expensive for shared memory mapping, including  memfd.
+    pub fn allocate_guest_memory(
+        &self,
+        guest_memfd: Option<File>,
+    ) -> Result<Vec<GuestRegionMmap>, MemoryError> {
         // For this reason, we only back guest memory with a memfd
         // if a vhost-user-blk device is configured in the VM, otherwise we fall back to
         // an anonymous private memory.
@@ -530,20 +538,35 @@ impl VmResources {
         // because that would require running a backend process. If in the future we converge to
         // a single way of backing guest memory for vhost-user and non-vhost-user cases,
         // that would not be worth the effort.
-        let regions =
-            crate::arch::arch_memory_regions(0, mib_to_bytes(self.machine_config.mem_size_mib));
-        if self.vhost_user_devices_used() {
-            memory::memfd_backed(
-                regions.as_ref(),
-                self.machine_config.track_dirty_pages,
-                self.machine_config.huge_pages,
-            )
-        } else {
-            memory::anonymous(
-                regions.into_iter(),
+        let regions = crate::arch::arch_memory_regions(0, self.memory_size()).into_iter();
+        match guest_memfd {
+            Some(file) => memory::file_shared(
+                file,
+                regions,
                 self.machine_config.track_dirty_pages,
                 self.machine_config.huge_pages,
-            )
+            ),
+            None => {
+                if self.vhost_user_devices_used() {
+                    let memfd = create_memfd(
+                        self.memory_size() as u64,
+                        self.machine_config.huge_pages.into(),
+                    )?
+                    .into_file();
+                    memory::file_shared(
+                        memfd,
+                        regions,
+                        self.machine_config.track_dirty_pages,
+                        self.machine_config.huge_pages,
+                    )
+                } else {
+                    memory::anonymous(
+                        regions.into_iter(),
+                        self.machine_config.track_dirty_pages,
+                        self.machine_config.huge_pages,
+                    )
+                }
+            }
         }
     }
 }

src/vmm/src/vstate/memory.rs

@@ -259,18 +259,16 @@ pub fn create(
 }
 
 /// Creates a GuestMemoryMmap with `size` in MiB backed by a memfd.
-pub fn memfd_backed(
-    regions: &[(GuestAddress, usize)],
+pub fn file_shared(
+    file: File,
+    regions: impl Iterator<Item = (GuestAddress, usize)>,
     track_dirty_pages: bool,
     huge_pages: HugePageConfig,
 ) -> Result<Vec<GuestRegionMmap>, MemoryError> {
-    let size = regions.iter().map(|&(_, size)| size as u64).sum();
-    let memfd_file = create_memfd(size, huge_pages.into())?.into_file();
-
     create(
-        regions.iter().copied(),
+        regions,
         libc::MAP_SHARED | huge_pages.mmap_flags(),
-        Some(memfd_file),
+        Some(file),
         track_dirty_pages,
     )
 }
@@ -291,7 +289,7 @@ pub fn anonymous(
 
 /// Creates a GuestMemoryMmap given a `file` containing the data
 /// and a `state` containing mapping information.
-pub fn snapshot_file(
+pub fn file_private(
     file: File,
     regions: impl Iterator<Item = (GuestAddress, usize)>,
     track_dirty_pages: bool,
@@ -477,7 +475,8 @@ impl GuestMemoryExtension for GuestMemoryMmap {
     }
 }
 
-fn create_memfd(
+/// Creates a memfd of the given size and huge pages configuration
+pub fn create_memfd(
     mem_size: u64,
     hugetlb_size: Option<memfd::HugetlbSize>,
 ) -> Result<memfd::Memfd, MemoryError> {
@@ -731,7 +730,7 @@ mod tests {
         guest_memory.dump(&mut memory_file).unwrap();
 
         let restored_guest_memory = GuestMemoryMmap::from_regions(
-            snapshot_file(memory_file, memory_state.regions(), false).unwrap(),
+            file_private(memory_file, memory_state.regions(), false).unwrap(),
         )
         .unwrap();
 
@@ -793,7 +792,7 @@ mod tests {
 
         // We can restore from this because this is the first dirty dump.
         let restored_guest_memory = GuestMemoryMmap::from_regions(
-            snapshot_file(file, memory_state.regions(), false).unwrap(),
+            file_private(file, memory_state.regions(), false).unwrap(),
         )
         .unwrap();
 

src/vmm/src/vstate/vm.rs

@@ -8,13 +8,13 @@
 use std::collections::HashMap;
 use std::fs::{File, OpenOptions};
 use std::io::Write;
-use std::os::fd::FromRawFd;
+use std::os::fd::{AsRawFd, FromRawFd};
 use std::path::Path;
 use std::sync::Arc;
 
 use kvm_bindings::{
-    KVM_MEM_LOG_DIRTY_PAGES, kvm_create_guest_memfd, kvm_userspace_memory_region,
-    kvm_userspace_memory_region2,
+    KVM_MEM_GUEST_MEMFD, KVM_MEM_LOG_DIRTY_PAGES, kvm_create_guest_memfd,
+    kvm_userspace_memory_region, kvm_userspace_memory_region2,
 };
 use kvm_ioctls::{Cap, VmFd};
 use vmm_sys_util::eventfd::EventFd;
@@ -31,6 +31,8 @@ use crate::vstate::memory::{
 use crate::vstate::vcpu::VcpuError;
 use crate::{DirtyBitmap, Vcpu, mem_size_mib};
 
+pub(crate) const KVM_GMEM_NO_DIRECT_MAP: u64 = 1;
+
 /// Architecture independent parts of a VM.
 #[derive(Debug)]
 pub struct VmCommon {
@@ -157,10 +159,6 @@ impl Vm {
             "guest_memfd size must be page aligned"
         );
 
-        if !self.fd().check_extension(Cap::GuestMemfd) {
-            return Err(VmError::GuestMemfdNotSupported);
-        }
-
         let kvm_gmem = kvm_create_guest_memfd {
             size: size as u64,
             flags,
@@ -198,10 +196,22 @@ impl Vm {
             return Err(VmError::NotEnoughMemorySlots);
         }
 
-        let flags = if region.bitmap().is_some() {
-            KVM_MEM_LOG_DIRTY_PAGES
+        let mut flags = 0;
+        if region.bitmap().is_some() {
+            flags |= KVM_MEM_LOG_DIRTY_PAGES;
+        }
+
+        #[allow(clippy::cast_sign_loss)]
+        let (guest_memfd, guest_memfd_offset) = if self.secret_free() {
+            flags |= KVM_MEM_GUEST_MEMFD;
+
+            let fo = region
+                .file_offset()
+                .expect("secret hidden VMs must mmap guest_memfd for memslots");
+
+            (fo.file().as_raw_fd() as u32, fo.start())
         } else {
-            0
+            (0, 0)
         };
 
         let memory_region = kvm_userspace_memory_region2 {
@@ -210,6 +220,8 @@ impl Vm {
             memory_size: region.len(),
             userspace_addr: region.as_ptr() as u64,
             flags,
+            guest_memfd,
+            guest_memfd_offset,
             ..Default::default()
         };
 
@@ -223,6 +235,12 @@ impl Vm {
                     .map_err(VmError::SetUserMemoryRegion)?;
             }
         } else {
+            // Something is seriously wrong if we manage to set these fields on a host that doesn't
+            // even allow creation of guest_memfds!
+            assert_eq!(memory_region.guest_memfd, 0);
+            assert_eq!(memory_region.guest_memfd_offset, 0);
+            assert_eq!(memory_region.flags & KVM_MEM_GUEST_MEMFD, 0);
+
             // SAFETY: We are passing a valid memory region and operate on a valid KVM FD.
             unsafe {
                 self.fd()