virt_whp: add support for MMIO hypercalls

vm/hv1/hv1_hypercall/src/aarch64.rs

@@ -90,6 +90,7 @@ impl<T: Arm64RegisterState> HypercallIo for Arm64RegisterIo<T> {
         if self.pre_advanced {
             let pc = self.inner.pc().wrapping_sub(4);
             self.inner.set_pc(pc);
+            self.pre_advanced = false;
         }
     }
 

vm/hv1/hv1_hypercall/src/imp.rs

@@ -25,6 +25,7 @@ use hvdef::hypercall::HostVisibilityType;
 use hvdef::hypercall::HvRegisterAssoc;
 use hvdef::hypercall::HypercallOutput;
 use hvdef::hypercall::VtlPermissionSet;
+use zerocopy::FromZeros;
 use zerocopy::IntoBytes;
 
 /// Implements the `HvPostMessage` hypercall.
@@ -1013,3 +1014,62 @@ impl<T: SendSyntheticClusterIpiEx> HypercallDispatch<HvSendSyntheticClusterIpiEx
         })
     }
 }
+
+/// Implements the `HvMemoryMappedIoRead` hypercall.
+pub type HvMemoryMappedIoRead = SimpleHypercall<
+    defs::MemoryMappedIoRead,
+    defs::MemoryMappedIoReadOutput,
+    { HypercallCode::HvCallMemoryMappedIoRead.0 },
+>;
+
+/// Implements the `HvMemoryMappedIoRead` hypercall.
+pub trait MemoryMappedIoRead {
+    /// Reads from a memory-mapped I/O region.
+    fn mmio_read(&mut self, gpa: u64, data: &mut [u8]) -> HvResult<()>;
+}
+
+impl<T: MemoryMappedIoRead> HypercallDispatch<HvMemoryMappedIoRead> for T {
+    fn dispatch(&mut self, params: HypercallParameters<'_>) -> HypercallOutput {
+        HvMemoryMappedIoRead::run(params, |input| {
+            if input.reserved_z0 != 0 || !input.access_width.is_power_of_two() {
+                return Err(HvError::InvalidParameter);
+            }
+            let mut output = defs::MemoryMappedIoReadOutput::new_zeroed();
+            self.mmio_read(
+                input.gpa,
+                output
+                    .data
+                    .get_mut(..input.access_width as usize)
+                    .ok_or(HvError::InvalidParameter)?,
+            )?;
+            Ok(output)
+        })
+    }
+}
+
+/// Defines the `HvMemoryMappedIoWrite` hypercall.
+pub type HvMemoryMappedIoWrite =
+    SimpleHypercall<defs::MemoryMappedIoWrite, (), { HypercallCode::HvCallMemoryMappedIoWrite.0 }>;
+
+/// Implements the `HvMemoryMappedIoWrite` hypercall.
+pub trait MemoryMappedIoWrite {
+    /// Writes to a memory-mapped I/O region.
+    fn mmio_write(&mut self, gpa: u64, data: &[u8]) -> HvResult<()>;
+}
+
+impl<T: MemoryMappedIoWrite> HypercallDispatch<HvMemoryMappedIoWrite> for T {
+    fn dispatch(&mut self, params: HypercallParameters<'_>) -> HypercallOutput {
+        HvMemoryMappedIoWrite::run(params, |input| {
+            if input.reserved_z0 != 0 || !input.access_width.is_power_of_two() {
+                return Err(HvError::InvalidParameter);
+            }
+            self.mmio_write(
+                input.gpa,
+                input
+                    .data
+                    .get(..input.access_width as usize)
+                    .ok_or(HvError::InvalidParameter)?,
+            )
+        })
+    }
+}

vm/hv1/hv1_hypercall/src/lib.rs

@@ -36,6 +36,8 @@ pub use self::support::HvRepResult;
 pub use self::support::HypercallDefinition;
 pub use self::support::HypercallHandler;
 pub use self::support::HypercallIo;
+pub use self::support::complete_hypercall;
+pub use self::support::complete_hypercall_with_data;
 pub use self::x86::X64HypercallRegister;
 pub use self::x86::X64RegisterIo;
 pub use self::x86::X64RegisterState;

vm/hv1/hv1_hypercall/src/support.rs

@@ -111,6 +111,12 @@ pub trait AsHandler<H> {
     fn as_handler(&mut self) -> &mut H;
 }
 
+impl<T: AsHandler<H>, H> AsHandler<H> for &mut T {
+    fn as_handler(&mut self) -> &mut H {
+        (**self).as_handler()
+    }
+}
+
 impl<'a, T: HypercallIo> InnerDispatcher<'a, T> {
     /// Creates a new dispatcher.
     fn new(guest_memory: &'a GuestMemory, mut handler: T) -> Self {
@@ -356,14 +362,17 @@ impl<'a, T: HypercallIo> InnerDispatcher<'a, T> {
 pub trait HypercallIo {
     /// Advances the instruction pointer for a completed hypercall.
     ///
-    /// Either `advance_ip` or `retry` will be called.
+    /// Either `advance_ip` or `retry` will be called. `advance_ip` may be called
+    /// after `retry`, but `retry` will not be called after `advance_ip`.
     fn advance_ip(&mut self);
 
     /// Retains the instruction pointer at the hypercall point so that the
     /// hypercall will be retried.
     ///
-    /// Either `advance_ip` or `retry` will be called.
     /// `control` is the updated hypercall input value to use in the retry.
+    ///
+    /// Either `advance_ip` or `retry` will be called. `advance_ip` may be called
+    /// after `retry`, but `retry` will not be called after `advance_ip`.
     fn retry(&mut self, control: u64);
 
     /// The hypercall input value.
@@ -774,3 +783,38 @@ impl<H> Dispatcher<H> {
         dispatcher.complete(result);
     }
 }
+
+/// Completes a hypercall with the given result and data, separately
+/// from the dispatcher.
+///
+/// Only simple, non-fast hypercalls are supported.
+pub fn complete_hypercall_with_data(
+    io: &mut impl HypercallIo,
+    mut result: HvResult<()>,
+    guest_memory: &GuestMemory,
+    data: &[u8],
+) {
+    let control = Control::from(io.control());
+    if result.is_ok() && !data.is_empty() {
+        if control.fast() || control.rep_count() != 0 {
+            // Not supported.
+            tracelimit::error_ratelimited!(
+                "completing pending fast hypercall with data, not supported"
+            );
+            result = Err(HvError::InvalidHypercallInput);
+        } else if let Err(err) = guest_memory.write_at(io.output_gpa(), data) {
+            tracelimit::error_ratelimited!(
+                error = &err as &dyn std::error::Error,
+                "failed to write hypercall output data"
+            );
+            result = Err(HvError::InvalidHypercallInput);
+        }
+    }
+    complete_hypercall(io, result.into())
+}
+
+/// Completes a hypercall separately from the dispatcher.
+pub fn complete_hypercall(io: &mut impl HypercallIo, result: HypercallOutput) {
+    io.set_result(result.into());
+    io.advance_ip();
+}

vm/hv1/hvdef/src/lib.rs

@@ -906,6 +906,15 @@ pub mod hypercall {
         }
     }
 
+    impl From<HvResult<()>> for HypercallOutput {
+        fn from(result: HvResult<()>) -> Self {
+            match result {
+                Ok(()) => Self::new(),
+                Err(e) => Self::from(e),
+            }
+        }
+    }
+
     impl HypercallOutput {
         /// A success output with zero elements processed.
         pub const SUCCESS: Self = Self::new();

vmm_core/virt_whp/src/hypercalls.rs

@@ -7,6 +7,7 @@ use crate::memory::VtlAccess;
 use crate::vtl2;
 #[cfg(guest_arch = "aarch64")]
 use aarch64 as arch;
+use arrayvec::ArrayVec;
 use hv1_hypercall::HvRepResult;
 use hvdef::HV_PAGE_SIZE;
 use hvdef::HV_PARTITION_ID_SELF;
@@ -29,6 +30,18 @@ pub(crate) struct WhpHypercallExit<'a, 'b, T> {
     vp: &'a mut WhpProcessor<'b>,
     bus: &'a T,
     registers: arch::WhpHypercallRegisters<'a>,
+    pending_hypercall: Option<PendingHypercall>,
+}
+
+enum PendingHypercall {
+    MmioRead {
+        gpa: u64,
+        len: usize,
+    },
+    MmioWrite {
+        gpa: u64,
+        data: ArrayVec<u8, { hvdef::hypercall::HV_HYPERCALL_MMIO_MAX_DATA_LENGTH }>,
+    },
 }
 
 impl<T: CpuIo> WhpHypercallExit<'_, '_, T> {
@@ -63,6 +76,8 @@ impl<T: CpuIo> WhpHypercallExit<'_, '_, T> {
             hv1_hypercall::HvGetVpIndexFromApicId,
             hv1_hypercall::HvAcceptGpaPages,
             hv1_hypercall::HvModifySparseGpaPageHostVisibility,
+            hv1_hypercall::HvMemoryMappedIoRead,
+            hv1_hypercall::HvMemoryMappedIoWrite,
         ]
     );
 }
@@ -679,6 +694,71 @@ impl<T: CpuIo> hv1_hypercall::ModifySparseGpaPageHostVisibility for WhpHypercall
     }
 }
 
+impl<T: CpuIo> hv1_hypercall::MemoryMappedIoRead for WhpHypercallExit<'_, '_, T> {
+    fn mmio_read(&mut self, gpa: u64, data: &mut [u8]) -> hvdef::HvResult<()> {
+        // Only for VTL2 testing at the moment.
+        if self.vp.state.active_vtl != Vtl::Vtl2 {
+            return Err(HvError::InvalidHypercallCode);
+        }
+        // Set a pending hypercall since the MMIO access may be async.
+        self.pending_hypercall = Some(PendingHypercall::MmioRead {
+            gpa,
+            len: data.len(),
+        });
+        Err(HvError::Timeout)
+    }
+}
+
+impl<T: CpuIo> hv1_hypercall::MemoryMappedIoWrite for WhpHypercallExit<'_, '_, T> {
+    fn mmio_write(&mut self, gpa: u64, data: &[u8]) -> hvdef::HvResult<()> {
+        // Only for VTL2 testing at the moment.
+        if self.vp.state.active_vtl != Vtl::Vtl2 {
+            return Err(HvError::InvalidHypercallCode);
+        }
+        // Set a pending hypercall since the MMIO access may be async.
+        self.pending_hypercall = Some(PendingHypercall::MmioWrite {
+            gpa,
+            data: data.try_into().expect("cap is set to hypercall maximum"),
+        });
+        Err(HvError::Timeout)
+    }
+}
+
+impl<T: CpuIo> WhpHypercallExit<'_, '_, T> {
+    async fn handle_pending_hypercall(
+        mut io: impl hv1_hypercall::AsHandler<Self> + hv1_hypercall::HypercallIo,
+    ) {
+        let this = io.as_handler();
+        if let Some(hc) = this.pending_hypercall.take() {
+            match hc {
+                PendingHypercall::MmioRead { gpa, len } => {
+                    let mut data =
+                        ArrayVec::from([0; hvdef::hypercall::HV_HYPERCALL_MMIO_MAX_DATA_LENGTH]);
+                    data.truncate(len);
+                    this.bus
+                        .read_mmio(this.vp.inner.vp_info.base.vp_index, gpa, &mut data)
+                        .await;
+
+                    let guest_memory = &this.vp.vp.partition.gm;
+                    hv1_hypercall::complete_hypercall_with_data(
+                        &mut io,
+                        Ok(()),
+                        guest_memory,
+                        &data,
+                    );
+                }
+                PendingHypercall::MmioWrite { gpa, data } => {
+                    this.bus
+                        .write_mmio(this.vp.inner.vp_info.base.vp_index, gpa, &data)
+                        .await;
+
+                    hv1_hypercall::complete_hypercall(&mut io, Ok(()).into());
+                }
+            }
+        }
+    }
+}
+
 #[cfg(guest_arch = "x86_64")]
 mod x86 {
     use super::WhpHypercallExit;
@@ -804,7 +884,7 @@ mod x86 {
             );
         }
 
-        pub fn handle(
+        pub async fn handle(
             vp: &'a mut WhpProcessor<'b>,
             bus: &'a T,
             info: &whp::abi::WHV_HYPERCALL_CONTEXT,
@@ -823,12 +903,16 @@ mod x86 {
                 invalid_opcode: false,
                 exit_context,
             };
-            let mut this = Self { vp, bus, registers };
+            let mut this = Self {
+                vp,
+                bus,
+                registers,
+                pending_hypercall: None,
+            };
 
-            WhpHypercallExit::DISPATCHER.dispatch(
-                &vpref.partition.gm,
-                hv1_hypercall::X64RegisterIo::new(&mut this, is_64bit),
-            );
+            let mut handler = hv1_hypercall::X64RegisterIo::new(&mut this, is_64bit);
+            WhpHypercallExit::DISPATCHER.dispatch(&vpref.partition.gm, &mut handler);
+            Self::handle_pending_hypercall(handler).await;
             this.flush()
         }
 
@@ -1725,7 +1809,7 @@ mod aarch64 {
             todo!("TODO-aarch64")
         }
 
-        pub fn handle(
+        pub async fn handle(
             vp: &'a mut WhpProcessor<'b>,
             bus: &'a T,
             message: &hvdef::HvArm64HypercallInterceptMessage,
@@ -1738,12 +1822,17 @@ mod aarch64 {
                 gp_dirty: false,
                 _dummy: &(),
             };
-            let mut this = Self { vp, bus, registers };
+            let mut this = Self {
+                vp,
+                bus,
+                registers,
+                pending_hypercall: None,
+            };
 
-            WhpHypercallExit::DISPATCHER.dispatch(
-                &vpref.partition.gm,
-                hv1_hypercall::Arm64RegisterIo::new(&mut this, false, message.immediate == 0),
-            );
+            let mut handler =
+                hv1_hypercall::Arm64RegisterIo::new(&mut this, false, message.immediate == 0);
+            WhpHypercallExit::DISPATCHER.dispatch(&vpref.partition.gm, &mut handler);
+            Self::handle_pending_hypercall(handler).await;
             this.flush();
         }
 

vmm_core/virt_whp/src/vp.rs

@@ -630,6 +630,7 @@ mod x86 {
                 }
                 ExitReason::Hypercall(info) => {
                     crate::hypercalls::WhpHypercallExit::handle(self, dev, info, exit.vp_context)
+                        .await
                         .map_err(VpHaltReason::Hypervisor)?;
                     &mut self.state.exits.hypercall
                 }
@@ -1810,7 +1811,8 @@ mod aarch64 {
                             self,
                             dev,
                             message_ref(message),
-                        );
+                        )
+                        .await;
                         &mut self.state.exits.hypercall
                     }
                     HvMessageType::HvMessageTypeArm64ResetIntercept => {