From 51bce6024e34884df009b79752ce5b8ad018557d Mon Sep 17 00:00:00 2001 From: John Starks Date: Sun, 22 Jun 2025 09:03:10 +0300 Subject: [PATCH 1/4] virt_whp: add support for MMIO hypercalls Support MMIO hypercalls from VTL2, but don't advertise this capability yet. This is currently just useful for private testing, but it provides the infrastructure to implement this in other places (e.g., virt_mshv_vtl). --- vm/hv1/hv1_hypercall/src/aarch64.rs | 1 + vm/hv1/hv1_hypercall/src/imp.rs | 60 ++++++++++++++++ vm/hv1/hv1_hypercall/src/lib.rs | 2 + vm/hv1/hv1_hypercall/src/support.rs | 48 ++++++++++++- vm/hv1/hvdef/src/lib.rs | 9 +++ vmm_core/virt_whp/src/hypercalls.rs | 104 +++++++++++++++++++++++++--- vmm_core/virt_whp/src/vp.rs | 1 + 7 files changed, 213 insertions(+), 12 deletions(-) diff --git a/vm/hv1/hv1_hypercall/src/aarch64.rs b/vm/hv1/hv1_hypercall/src/aarch64.rs index 6c891b2686..bce0f4a2ba 100644 --- a/vm/hv1/hv1_hypercall/src/aarch64.rs +++ b/vm/hv1/hv1_hypercall/src/aarch64.rs @@ -90,6 +90,7 @@ impl HypercallIo for Arm64RegisterIo { if self.pre_advanced { let pc = self.inner.pc().wrapping_sub(4); self.inner.set_pc(pc); + self.pre_advanced = false; } } diff --git a/vm/hv1/hv1_hypercall/src/imp.rs b/vm/hv1/hv1_hypercall/src/imp.rs index d8c5690b74..264fbfcbc9 100644 --- a/vm/hv1/hv1_hypercall/src/imp.rs +++ b/vm/hv1/hv1_hypercall/src/imp.rs @@ -25,6 +25,7 @@ use hvdef::hypercall::HostVisibilityType; use hvdef::hypercall::HvRegisterAssoc; use hvdef::hypercall::HypercallOutput; use hvdef::hypercall::VtlPermissionSet; +use zerocopy::FromZeros; use zerocopy::IntoBytes; /// Implements the `HvPostMessage` hypercall. @@ -1013,3 +1014,62 @@ impl HypercallDispatch; + +/// Implements the `HvMemoryMappedIoRead` hypercall. +pub trait MemoryMappedIoRead { + /// Reads from a memory-mapped I/O region. + fn mmio_read(&mut self, gpa: u64, data: &mut [u8]) -> HvResult<()>; +} + +impl HypercallDispatch for T { + fn dispatch(&mut self, params: HypercallParameters<'_>) -> HypercallOutput { + HvMemoryMappedIoRead::run(params, |input| { + if input.reserved_z0 != 0 || !input.access_width.is_power_of_two() { + return Err(HvError::InvalidParameter); + } + let mut output = defs::MemoryMappedIoReadOutput::new_zeroed(); + self.mmio_read( + input.gpa, + output + .data + .get_mut(..input.access_width as usize) + .ok_or(HvError::InvalidParameter)?, + )?; + Ok(output) + }) + } +} + +/// Defines the `HvMemoryMappedIoWrite` hypercall. +pub type HvMemoryMappedIoWrite = + SimpleHypercall; + +/// Implements the `HvMemoryMappedIoWrite` hypercall. +pub trait MemoryMappedIoWrite { + /// Writes to a memory-mapped I/O region. + fn mmio_write(&mut self, gpa: u64, data: &[u8]) -> HvResult<()>; +} + +impl HypercallDispatch for T { + fn dispatch(&mut self, params: HypercallParameters<'_>) -> HypercallOutput { + HvMemoryMappedIoWrite::run(params, |input| { + if input.reserved_z0 != 0 || !input.access_width.is_power_of_two() { + return Err(HvError::InvalidParameter); + } + self.mmio_write( + input.gpa, + input + .data + .get(..input.access_width as usize) + .ok_or(HvError::InvalidParameter)?, + ) + }) + } +} diff --git a/vm/hv1/hv1_hypercall/src/lib.rs b/vm/hv1/hv1_hypercall/src/lib.rs index b22398b344..acb6482ec0 100644 --- a/vm/hv1/hv1_hypercall/src/lib.rs +++ b/vm/hv1/hv1_hypercall/src/lib.rs @@ -36,6 +36,8 @@ pub use self::support::HvRepResult; pub use self::support::HypercallDefinition; pub use self::support::HypercallHandler; pub use self::support::HypercallIo; +pub use self::support::complete_hypercall; +pub use self::support::complete_hypercall_with_data; pub use self::x86::X64HypercallRegister; pub use self::x86::X64RegisterIo; pub use self::x86::X64RegisterState; diff --git a/vm/hv1/hv1_hypercall/src/support.rs b/vm/hv1/hv1_hypercall/src/support.rs index c95be2d9a6..2146acc648 100644 --- a/vm/hv1/hv1_hypercall/src/support.rs +++ b/vm/hv1/hv1_hypercall/src/support.rs @@ -111,6 +111,12 @@ pub trait AsHandler { fn as_handler(&mut self) -> &mut H; } +impl, H> AsHandler for &mut T { + fn as_handler(&mut self) -> &mut H { + (**self).as_handler() + } +} + impl<'a, T: HypercallIo> InnerDispatcher<'a, T> { /// Creates a new dispatcher. fn new(guest_memory: &'a GuestMemory, mut handler: T) -> Self { @@ -356,14 +362,17 @@ impl<'a, T: HypercallIo> InnerDispatcher<'a, T> { pub trait HypercallIo { /// Advances the instruction pointer for a completed hypercall. /// - /// Either `advance_ip` or `retry` will be called. + /// Either `advance_ip` or `retry` will be called. `advance_ip` may be called + /// after `retry`, but `retry` will not be called after `advance_ip`. fn advance_ip(&mut self); /// Retains the instruction pointer at the hypercall point so that the /// hypercall will be retried. /// - /// Either `advance_ip` or `retry` will be called. /// `control` is the updated hypercall input value to use in the retry. + /// + /// Either `advance_ip` or `retry` will be called. `advance_ip` may be called + /// after `retry`, but `retry` will not be called after `advance_ip`. fn retry(&mut self, control: u64); /// The hypercall input value. @@ -774,3 +783,38 @@ impl Dispatcher { dispatcher.complete(result); } } + +/// Completes a hypercall with the given result and data, separately +/// from the dispatcher. +/// +/// Only simple, non-fast hypercalls are supported. +pub fn complete_hypercall_with_data( + io: &mut impl HypercallIo, + mut result: HvResult<()>, + guest_memory: &GuestMemory, + data: &[u8], +) { + let control = Control::from(io.control()); + if result.is_ok() && !data.is_empty() { + if control.fast() || control.rep_count() != 0 { + // Not supported. + tracelimit::error_ratelimited!( + "completing pending fast hypercall with data, not supported" + ); + result = Err(HvError::InvalidHypercallInput); + } else if let Err(err) = guest_memory.write_at(io.output_gpa(), data) { + tracelimit::error_ratelimited!( + error = &err as &dyn std::error::Error, + "failed to write hypercall output data" + ); + result = Err(HvError::InvalidHypercallInput); + } + } + complete_hypercall(io, result.into()) +} + +/// Completes a hypercall separately from the dispatcher. +pub fn complete_hypercall(io: &mut impl HypercallIo, result: HypercallOutput) { + io.set_result(result.into()); + io.advance_ip(); +} diff --git a/vm/hv1/hvdef/src/lib.rs b/vm/hv1/hvdef/src/lib.rs index d4b465db3e..9195209da3 100644 --- a/vm/hv1/hvdef/src/lib.rs +++ b/vm/hv1/hvdef/src/lib.rs @@ -906,6 +906,15 @@ pub mod hypercall { } } + impl From> for HypercallOutput { + fn from(result: HvResult<()>) -> Self { + match result { + Ok(()) => Self::new(), + Err(e) => Self::from(e), + } + } + } + impl HypercallOutput { /// A success output with zero elements processed. pub const SUCCESS: Self = Self::new(); diff --git a/vmm_core/virt_whp/src/hypercalls.rs b/vmm_core/virt_whp/src/hypercalls.rs index e39d066d2b..0fea451e26 100644 --- a/vmm_core/virt_whp/src/hypercalls.rs +++ b/vmm_core/virt_whp/src/hypercalls.rs @@ -7,6 +7,7 @@ use crate::memory::VtlAccess; use crate::vtl2; #[cfg(guest_arch = "aarch64")] use aarch64 as arch; +use arrayvec::ArrayVec; use hv1_hypercall::HvRepResult; use hvdef::HV_PAGE_SIZE; use hvdef::HV_PARTITION_ID_SELF; @@ -29,6 +30,18 @@ pub(crate) struct WhpHypercallExit<'a, 'b, T> { vp: &'a mut WhpProcessor<'b>, bus: &'a T, registers: arch::WhpHypercallRegisters<'a>, + pending_hypercall: Option, +} + +enum PendingHypercall { + MmioRead { + gpa: u64, + len: usize, + }, + MmioWrite { + gpa: u64, + data: ArrayVec, + }, } impl WhpHypercallExit<'_, '_, T> { @@ -63,6 +76,8 @@ impl WhpHypercallExit<'_, '_, T> { hv1_hypercall::HvGetVpIndexFromApicId, hv1_hypercall::HvAcceptGpaPages, hv1_hypercall::HvModifySparseGpaPageHostVisibility, + hv1_hypercall::HvMemoryMappedIoRead, + hv1_hypercall::HvMemoryMappedIoWrite, ] ); } @@ -679,6 +694,71 @@ impl hv1_hypercall::ModifySparseGpaPageHostVisibility for WhpHypercall } } +impl hv1_hypercall::MemoryMappedIoRead for WhpHypercallExit<'_, '_, T> { + fn mmio_read(&mut self, gpa: u64, data: &mut [u8]) -> hvdef::HvResult<()> { + // Only for VTL2 testing at the moment. + if self.vp.state.active_vtl != Vtl::Vtl2 { + return Err(HvError::InvalidHypercallCode); + } + // Set a pending hypercall since the MMIO access may be async. + self.pending_hypercall = Some(PendingHypercall::MmioRead { + gpa, + len: data.len(), + }); + Err(HvError::Timeout) + } +} + +impl hv1_hypercall::MemoryMappedIoWrite for WhpHypercallExit<'_, '_, T> { + fn mmio_write(&mut self, gpa: u64, data: &[u8]) -> hvdef::HvResult<()> { + // Only for VTL2 testing at the moment. + if self.vp.state.active_vtl != Vtl::Vtl2 { + return Err(HvError::InvalidHypercallCode); + } + // Set a pending hypercall since the MMIO access may be async. + self.pending_hypercall = Some(PendingHypercall::MmioWrite { + gpa, + data: data.try_into().expect("cap is set to hypercall maximum"), + }); + Err(HvError::Timeout) + } +} + +impl WhpHypercallExit<'_, '_, T> { + async fn handle_pending_hypercall( + mut io: impl hv1_hypercall::AsHandler + hv1_hypercall::HypercallIo, + ) { + let this = io.as_handler(); + if let Some(hc) = this.pending_hypercall.take() { + match hc { + PendingHypercall::MmioRead { gpa, len } => { + let mut data = + ArrayVec::from([0; hvdef::hypercall::HV_HYPERCALL_MMIO_MAX_DATA_LENGTH]); + data.truncate(len); + this.bus + .read_mmio(this.vp.inner.vp_info.base.vp_index, gpa, &mut data) + .await; + + let guest_memory = &this.vp.vp.partition.gm; + hv1_hypercall::complete_hypercall_with_data( + &mut io, + Ok(()), + guest_memory, + &data, + ); + } + PendingHypercall::MmioWrite { gpa, data } => { + this.bus + .write_mmio(this.vp.inner.vp_info.base.vp_index, gpa, &data) + .await; + + hv1_hypercall::complete_hypercall(&mut io, Ok(()).into()); + } + } + } + } +} + #[cfg(guest_arch = "x86_64")] mod x86 { use super::WhpHypercallExit; @@ -804,7 +884,7 @@ mod x86 { ); } - pub fn handle( + pub async fn handle( vp: &'a mut WhpProcessor<'b>, bus: &'a T, info: &whp::abi::WHV_HYPERCALL_CONTEXT, @@ -823,12 +903,16 @@ mod x86 { invalid_opcode: false, exit_context, }; - let mut this = Self { vp, bus, registers }; + let mut this = Self { + vp, + bus, + registers, + pending_hypercall: None, + }; - WhpHypercallExit::DISPATCHER.dispatch( - &vpref.partition.gm, - hv1_hypercall::X64RegisterIo::new(&mut this, is_64bit), - ); + let mut handler = hv1_hypercall::X64RegisterIo::new(&mut this, is_64bit); + WhpHypercallExit::DISPATCHER.dispatch(&vpref.partition.gm, &mut handler); + Self::handle_pending_hypercall(handler).await; this.flush() } @@ -1740,10 +1824,10 @@ mod aarch64 { }; let mut this = Self { vp, bus, registers }; - WhpHypercallExit::DISPATCHER.dispatch( - &vpref.partition.gm, - hv1_hypercall::Arm64RegisterIo::new(&mut this, false, message.immediate == 0), - ); + let mut handler = + hv1_hypercall::Arm64RegisterIo::new(&mut this, false, message.immediate == 0); + WhpHypercallExit::DISPATCHER.dispatch(&vpref.partition.gm, &mut handler); + Self::handle_pending_hypercall(handler).await; this.flush(); } diff --git a/vmm_core/virt_whp/src/vp.rs b/vmm_core/virt_whp/src/vp.rs index 4d68bab8e1..47df28337c 100644 --- a/vmm_core/virt_whp/src/vp.rs +++ b/vmm_core/virt_whp/src/vp.rs @@ -630,6 +630,7 @@ mod x86 { } ExitReason::Hypercall(info) => { crate::hypercalls::WhpHypercallExit::handle(self, dev, info, exit.vp_context) + .await .map_err(VpHaltReason::Hypervisor)?; &mut self.state.exits.hypercall } From 72b220f09bbaad15ac2e2ef53da0f66468f2e5e3 Mon Sep 17 00:00:00 2001 From: John Starks Date: Sun, 22 Jun 2025 10:24:14 +0300 Subject: [PATCH 2/4] fix --- vmm_core/virt_whp/src/hypercalls.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmm_core/virt_whp/src/hypercalls.rs b/vmm_core/virt_whp/src/hypercalls.rs index 0fea451e26..83f279f84a 100644 --- a/vmm_core/virt_whp/src/hypercalls.rs +++ b/vmm_core/virt_whp/src/hypercalls.rs @@ -1809,7 +1809,7 @@ mod aarch64 { todo!("TODO-aarch64") } - pub fn handle( + pub async fn handle( vp: &'a mut WhpProcessor<'b>, bus: &'a T, message: &hvdef::HvArm64HypercallInterceptMessage, From df8cd5a3848ed0a10d9f15a1edebfe445977123d Mon Sep 17 00:00:00 2001 From: John Starks Date: Sun, 22 Jun 2025 11:04:08 +0300 Subject: [PATCH 3/4] aarch64 --- vmm_core/virt_whp/src/hypercalls.rs | 2 +- vmm_core/virt_whp/src/vp.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vmm_core/virt_whp/src/hypercalls.rs b/vmm_core/virt_whp/src/hypercalls.rs index 83f279f84a..d72978167a 100644 --- a/vmm_core/virt_whp/src/hypercalls.rs +++ b/vmm_core/virt_whp/src/hypercalls.rs @@ -1822,7 +1822,7 @@ mod aarch64 { gp_dirty: false, _dummy: &(), }; - let mut this = Self { vp, bus, registers }; + let mut this = Self { vp, bus, registers, pending_hypercall: None }; let mut handler = hv1_hypercall::Arm64RegisterIo::new(&mut this, false, message.immediate == 0); diff --git a/vmm_core/virt_whp/src/vp.rs b/vmm_core/virt_whp/src/vp.rs index 47df28337c..1c77494e9e 100644 --- a/vmm_core/virt_whp/src/vp.rs +++ b/vmm_core/virt_whp/src/vp.rs @@ -1811,7 +1811,7 @@ mod aarch64 { self, dev, message_ref(message), - ); + ).await; &mut self.state.exits.hypercall } HvMessageType::HvMessageTypeArm64ResetIntercept => { From 0f93eab7dd17ee59a14edd06358c138aca2e775c Mon Sep 17 00:00:00 2001 From: John Starks Date: Sun, 22 Jun 2025 11:39:21 +0300 Subject: [PATCH 4/4] fmt --- vmm_core/virt_whp/src/hypercalls.rs | 7 ++++++- vmm_core/virt_whp/src/vp.rs | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/vmm_core/virt_whp/src/hypercalls.rs b/vmm_core/virt_whp/src/hypercalls.rs index d72978167a..f2892be060 100644 --- a/vmm_core/virt_whp/src/hypercalls.rs +++ b/vmm_core/virt_whp/src/hypercalls.rs @@ -1822,7 +1822,12 @@ mod aarch64 { gp_dirty: false, _dummy: &(), }; - let mut this = Self { vp, bus, registers, pending_hypercall: None }; + let mut this = Self { + vp, + bus, + registers, + pending_hypercall: None, + }; let mut handler = hv1_hypercall::Arm64RegisterIo::new(&mut this, false, message.immediate == 0); diff --git a/vmm_core/virt_whp/src/vp.rs b/vmm_core/virt_whp/src/vp.rs index 1c77494e9e..9898823f23 100644 --- a/vmm_core/virt_whp/src/vp.rs +++ b/vmm_core/virt_whp/src/vp.rs @@ -1811,7 +1811,8 @@ mod aarch64 { self, dev, message_ref(message), - ).await; + ) + .await; &mut self.state.exits.hypercall } HvMessageType::HvMessageTypeArm64ResetIntercept => {