add concept of "secret free" VMs

Have the `struct Vm` constructor take an argument to indicate whether
the VM should be secret free. Use this to determine the correct vm type
for guest_memfd support, and store it inside the VM so that we don't
have to pass bools to various functions.

Signed-off-by: Patrick Roy <roypat@amazon.co.uk>

Author: roypat

src/vmm/src/arch/aarch64/vm.rs

@@ -8,6 +8,14 @@ use crate::arch::aarch64::gic::GicState;
 use crate::vstate::memory::{GuestMemoryExtension, GuestMemoryState};
 use crate::vstate::vm::{VmCommon, VmError};
 
+/// The VM type for this architecture that allows us to use guest_memfd. On ARM, all VMs
+/// support guest_memfd and no special type is needed (in fact, no concept of vm types really
+/// exists, and the correspoding field of the CREATE_VM ioctl determines IPA size instead,
+/// e.g. the size of the guest physical address space. This value cannot be hardcoded, hence
+/// `None` to let the `Vm` constructor now that just normal [`Kvm::create_vm`] should be called,
+/// which internally determines the preferred IPA size.
+pub const VM_TYPE_FOR_SECRET_FREEDOM: Option<u64> = None;
+
 /// Structure representing the current architecture's understand of what a "virtual machine" is.
 #[derive(Debug)]
 pub struct ArchVm {
@@ -30,8 +38,8 @@ pub enum ArchVmError {
 
 impl ArchVm {
     /// Create a new `Vm` struct.
-    pub fn new(kvm: &Kvm) -> Result<ArchVm, VmError> {
-        let common = Self::create_common(kvm)?;
+    pub fn new(kvm: &Kvm, secret_free: bool) -> Result<ArchVm, VmError> {
+        let common = Self::create_common(kvm, secret_free)?;
         Ok(ArchVm {
             common,
             irqchip_handle: None,

src/vmm/src/arch/mod.rs

@@ -17,7 +17,7 @@ pub use aarch64::kvm::{Kvm, KvmArchError, OptionalCapabilities};
 #[cfg(target_arch = "aarch64")]
 pub use aarch64::vcpu::*;
 #[cfg(target_arch = "aarch64")]
-pub use aarch64::vm::{ArchVm, ArchVmError, VmState};
+pub use aarch64::vm::{ArchVm, ArchVmError, VM_TYPE_FOR_SECRET_FREEDOM, VmState};
 #[cfg(target_arch = "aarch64")]
 pub use aarch64::{
     ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, arch_memory_regions,
@@ -35,7 +35,7 @@ pub use x86_64::kvm::{Kvm, KvmArchError};
 #[cfg(target_arch = "x86_64")]
 pub use x86_64::vcpu::*;
 #[cfg(target_arch = "x86_64")]
-pub use x86_64::vm::{ArchVm, ArchVmError, VmState};
+pub use x86_64::vm::{ArchVm, ArchVmError, VM_TYPE_FOR_SECRET_FREEDOM, VmState};
 
 #[cfg(target_arch = "x86_64")]
 pub use crate::arch::x86_64::{

src/vmm/src/arch/x86_64/vm.rs

@@ -5,7 +5,8 @@ use std::fmt;
 
 use kvm_bindings::{
     KVM_CLOCK_TSC_STABLE, KVM_IRQCHIP_IOAPIC, KVM_IRQCHIP_PIC_MASTER, KVM_IRQCHIP_PIC_SLAVE,
-    KVM_PIT_SPEAKER_DUMMY, MsrList, kvm_clock_data, kvm_irqchip, kvm_pit_config, kvm_pit_state2,
+    KVM_PIT_SPEAKER_DUMMY, KVM_X86_SW_PROTECTED_VM, MsrList, kvm_clock_data, kvm_irqchip,
+    kvm_pit_config, kvm_pit_state2,
 };
 use kvm_ioctls::Cap;
 use serde::{Deserialize, Serialize};
@@ -46,6 +47,9 @@ pub enum ArchVmError {
     SetTssAddress(kvm_ioctls::Error),
 }
 
+/// The VM type for this architecture that allows us to use guest_memfd.
+pub const VM_TYPE_FOR_SECRET_FREEDOM: Option<u64> = Some(KVM_X86_SW_PROTECTED_VM as u64);
+
 /// Structure representing the current architecture's understand of what a "virtual machine" is.
 #[derive(Debug)]
 pub struct ArchVm {
@@ -60,8 +64,8 @@ pub struct ArchVm {
 
 impl ArchVm {
     /// Create a new `Vm` struct.
-    pub fn new(kvm: &crate::vstate::kvm::Kvm) -> Result<ArchVm, VmError> {
-        let common = Self::create_common(kvm)?;
+    pub fn new(kvm: &crate::vstate::kvm::Kvm, secret_free: bool) -> Result<ArchVm, VmError> {
+        let common = Self::create_common(kvm, secret_free)?;
 
         let msrs_to_save = kvm.msrs_to_save().map_err(ArchVmError::GetMsrsToSave)?;
 

src/vmm/src/builder.rs

@@ -144,11 +144,12 @@ fn create_vmm_and_vcpus(
     event_manager: &mut EventManager,
     vcpu_count: u8,
     kvm_capabilities: Vec<KvmCapability>,
+    secret_free: bool,
 ) -> Result<(Vmm, Vec<Vcpu>), VmmError> {
     let kvm = Kvm::new(kvm_capabilities)?;
     // Set up Kvm Vm and register memory regions.
     // Build custom CPU config if a custom template is provided.
-    let mut vm = Vm::new(&kvm)?;
+    let mut vm = Vm::new(&kvm, secret_free)?;
 
     let resource_allocator = ResourceAllocator::new()?;
 
@@ -239,6 +240,7 @@ pub fn build_microvm_for_boot(
         event_manager,
         vm_resources.machine_config.vcpu_count,
         cpu_template.kvm_capabilities.clone(),
+        vm_resources.machine_config.secret_free,
     )?;
 
     vmm.vm
@@ -248,7 +250,7 @@ pub fn build_microvm_for_boot(
     let entry_point = load_kernel(
         MaybeBounce::new(
             boot_config.kernel_file.try_clone().unwrap(),
-            vm_resources.machine_config.secret_free,
+            vmm.vm.secret_free(),
         ),
         vmm.vm.guest_memory(),
     )?;
@@ -261,7 +263,7 @@ pub fn build_microvm_for_boot(
 
             Some(InitrdConfig::from_reader(
                 vmm.vm.guest_memory(),
-                MaybeBounce::new(initrd_file.as_fd(), vm_resources.machine_config.secret_free),
+                MaybeBounce::new(initrd_file.as_fd(), vmm.vm.secret_free()),
                 u64_to_usize(size),
             )?)
         }
@@ -296,24 +298,16 @@ pub fn build_microvm_for_boot(
         &mut boot_cmdline,
         vm_resources.block.devices.iter(),
         event_manager,
-        vm_resources.machine_config.secret_free,
     )?;
     attach_net_devices(
         &mut vmm,
         &mut boot_cmdline,
         vm_resources.net_builder.iter(),
         event_manager,
-        vm_resources.machine_config.secret_free,
     )?;
 
     if let Some(unix_vsock) = vm_resources.vsock.get() {
-        attach_unixsock_vsock_device(
-            &mut vmm,
-            &mut boot_cmdline,
-            unix_vsock,
-            event_manager,
-            vm_resources.machine_config.secret_free,
-        )?;
+        attach_unixsock_vsock_device(&mut vmm, &mut boot_cmdline, unix_vsock, event_manager)?;
     }
 
     if let Some(entropy) = vm_resources.entropy.get() {
@@ -469,6 +463,7 @@ pub fn build_microvm_from_snapshot(
         event_manager,
         vm_resources.machine_config.vcpu_count,
         microvm_state.kvm_state.kvm_cap_modifiers.clone(),
+        false,
     )
     .map_err(StartMicrovmError::Internal)?;
 
@@ -685,11 +680,10 @@ fn attach_virtio_device<T: 'static + VirtioDevice + MutEventSubscriber + Debug>(
     device: Arc<Mutex<T>>,
     cmdline: &mut LoaderKernelCmdline,
     is_vhost_user: bool,
-    secret_free: bool,
 ) -> Result<(), MmioError> {
     event_manager.add_subscriber(device.clone());
 
-    if secret_free {
+    if vmm.vm.secret_free() {
         device.lock().unwrap().force_userspace_bounce_buffers();
     }
 
@@ -748,7 +742,6 @@ fn attach_entropy_device(
         entropy_device.clone(),
         cmdline,
         false,
-        false,
     )
 }
 
@@ -757,7 +750,6 @@ fn attach_block_devices<'a, I: Iterator<Item = &'a Arc<Mutex<Block>>> + Debug>(
     cmdline: &mut LoaderKernelCmdline,
     blocks: I,
     event_manager: &mut EventManager,
-    secret_free: bool,
 ) -> Result<(), StartMicrovmError> {
     for block in blocks {
         let (id, is_vhost_user) = {
@@ -782,7 +774,6 @@ fn attach_block_devices<'a, I: Iterator<Item = &'a Arc<Mutex<Block>>> + Debug>(
             block.clone(),
             cmdline,
             is_vhost_user,
-            secret_free,
         )?;
     }
     Ok(())
@@ -793,20 +784,11 @@ fn attach_net_devices<'a, I: Iterator<Item = &'a Arc<Mutex<Net>>> + Debug>(
     cmdline: &mut LoaderKernelCmdline,
     net_devices: I,
     event_manager: &mut EventManager,
-    secret_free: bool,
 ) -> Result<(), StartMicrovmError> {
     for net_device in net_devices {
         let id = net_device.lock().expect("Poisoned lock").id().clone();
         // The device mutex mustn't be locked here otherwise it will deadlock.
-        attach_virtio_device(
-            event_manager,
-            vmm,
-            id,
-            net_device.clone(),
-            cmdline,
-            false,
-            secret_free,
-        )?;
+        attach_virtio_device(event_manager, vmm, id, net_device.clone(), cmdline, false)?;
     }
     Ok(())
 }
@@ -816,19 +798,10 @@ fn attach_unixsock_vsock_device(
     cmdline: &mut LoaderKernelCmdline,
     unix_vsock: &Arc<Mutex<Vsock<VsockUnixBackend>>>,
     event_manager: &mut EventManager,
-    secret_free: bool,
 ) -> Result<(), MmioError> {
     let id = String::from(unix_vsock.lock().expect("Poisoned lock").id());
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    attach_virtio_device(
-        event_manager,
-        vmm,
-        id,
-        unix_vsock.clone(),
-        cmdline,
-        false,
-        secret_free,
-    )
+    attach_virtio_device(event_manager, vmm, id, unix_vsock.clone(), cmdline, false)
 }
 
 fn attach_balloon_device(
@@ -839,15 +812,7 @@ fn attach_balloon_device(
 ) -> Result<(), MmioError> {
     let id = String::from(balloon.lock().expect("Poisoned lock").id());
     // The device mutex mustn't be locked here otherwise it will deadlock.
-    attach_virtio_device(
-        event_manager,
-        vmm,
-        id,
-        balloon.clone(),
-        cmdline,
-        false,
-        false,
-    )
+    attach_virtio_device(event_manager, vmm, id, balloon.clone(), cmdline, false)
 }
 
 // Adds `O_NONBLOCK` to the stdout flags.
@@ -1023,7 +988,6 @@ pub(crate) mod tests {
             cmdline,
             block_dev_configs.devices.iter(),
             event_manager,
-            false,
         )
         .unwrap();
         block_files
@@ -1038,7 +1002,7 @@ pub(crate) mod tests {
         let mut net_builder = NetBuilder::new();
         net_builder.build(net_config).unwrap();
 
-        let res = attach_net_devices(vmm, cmdline, net_builder.iter(), event_manager, false);
+        let res = attach_net_devices(vmm, cmdline, net_builder.iter(), event_manager);
         res.unwrap();
     }
 
@@ -1059,7 +1023,7 @@ pub(crate) mod tests {
             Arc::new(Mutex::new(mmds)),
         );
 
-        attach_net_devices(vmm, cmdline, net_builder.iter(), event_manager, false).unwrap();
+        attach_net_devices(vmm, cmdline, net_builder.iter(), event_manager).unwrap();
     }
 
     pub(crate) fn insert_vsock_device(
@@ -1072,7 +1036,7 @@ pub(crate) mod tests {
         let vsock = VsockBuilder::create_unixsock_vsock(vsock_config).unwrap();
         let vsock = Arc::new(Mutex::new(vsock));
 
-        attach_unixsock_vsock_device(vmm, cmdline, &vsock, event_manager, false).unwrap();
+        attach_unixsock_vsock_device(vmm, cmdline, &vsock, event_manager).unwrap();
 
         assert!(
             vmm.mmio_device_manager

src/vmm/src/device_manager/mmio.rs

@@ -659,7 +659,7 @@ mod tests {
         let start_addr2 = GuestAddress(0x1000);
         let guest_mem = multi_region_mem_raw(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]);
         let kvm = Kvm::new(vec![]).expect("Cannot create Kvm");
-        let mut vm = Vm::new(&kvm).unwrap();
+        let mut vm = Vm::new(&kvm, false).unwrap();
         vm.register_memory_regions(guest_mem).unwrap();
         let mut device_manager = MMIODeviceManager::new();
         let mut resource_allocator = ResourceAllocator::new().unwrap();
@@ -690,7 +690,7 @@ mod tests {
         let start_addr2 = GuestAddress(0x1000);
         let guest_mem = multi_region_mem_raw(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]);
         let kvm = Kvm::new(vec![]).expect("Cannot create Kvm");
-        let mut vm = Vm::new(&kvm).unwrap();
+        let mut vm = Vm::new(&kvm, false).unwrap();
         vm.register_memory_regions(guest_mem).unwrap();
         let mut device_manager = MMIODeviceManager::new();
         let mut resource_allocator = ResourceAllocator::new().unwrap();
@@ -746,7 +746,7 @@ mod tests {
         let start_addr2 = GuestAddress(0x1000);
         let guest_mem = multi_region_mem_raw(&[(start_addr1, 0x1000), (start_addr2, 0x1000)]);
         let kvm = Kvm::new(vec![]).expect("Cannot create Kvm");
-        let mut vm = Vm::new(&kvm).unwrap();
+        let mut vm = Vm::new(&kvm, false).unwrap();
         vm.register_memory_regions(guest_mem).unwrap();
 
         #[cfg(target_arch = "x86_64")]