diff --git a/vm/hv1/hv1_hypercall/src/aarch64.rs b/vm/hv1/hv1_hypercall/src/aarch64.rs index 6c891b2686..bce0f4a2ba 100644 --- a/vm/hv1/hv1_hypercall/src/aarch64.rs +++ b/vm/hv1/hv1_hypercall/src/aarch64.rs @@ -90,6 +90,7 @@ impl HypercallIo for Arm64RegisterIo { if self.pre_advanced { let pc = self.inner.pc().wrapping_sub(4); self.inner.set_pc(pc); + self.pre_advanced = false; } } diff --git a/vm/hv1/hv1_hypercall/src/imp.rs b/vm/hv1/hv1_hypercall/src/imp.rs index d8c5690b74..264fbfcbc9 100644 --- a/vm/hv1/hv1_hypercall/src/imp.rs +++ b/vm/hv1/hv1_hypercall/src/imp.rs @@ -25,6 +25,7 @@ use hvdef::hypercall::HostVisibilityType; use hvdef::hypercall::HvRegisterAssoc; use hvdef::hypercall::HypercallOutput; use hvdef::hypercall::VtlPermissionSet; +use zerocopy::FromZeros; use zerocopy::IntoBytes; /// Implements the `HvPostMessage` hypercall. @@ -1013,3 +1014,62 @@ impl HypercallDispatch; + +/// Implements the `HvMemoryMappedIoRead` hypercall. +pub trait MemoryMappedIoRead { + /// Reads from a memory-mapped I/O region. + fn mmio_read(&mut self, gpa: u64, data: &mut [u8]) -> HvResult<()>; +} + +impl HypercallDispatch for T { + fn dispatch(&mut self, params: HypercallParameters<'_>) -> HypercallOutput { + HvMemoryMappedIoRead::run(params, |input| { + if input.reserved_z0 != 0 || !input.access_width.is_power_of_two() { + return Err(HvError::InvalidParameter); + } + let mut output = defs::MemoryMappedIoReadOutput::new_zeroed(); + self.mmio_read( + input.gpa, + output + .data + .get_mut(..input.access_width as usize) + .ok_or(HvError::InvalidParameter)?, + )?; + Ok(output) + }) + } +} + +/// Defines the `HvMemoryMappedIoWrite` hypercall. +pub type HvMemoryMappedIoWrite = + SimpleHypercall; + +/// Implements the `HvMemoryMappedIoWrite` hypercall. +pub trait MemoryMappedIoWrite { + /// Writes to a memory-mapped I/O region. + fn mmio_write(&mut self, gpa: u64, data: &[u8]) -> HvResult<()>; +} + +impl HypercallDispatch for T { + fn dispatch(&mut self, params: HypercallParameters<'_>) -> HypercallOutput { + HvMemoryMappedIoWrite::run(params, |input| { + if input.reserved_z0 != 0 || !input.access_width.is_power_of_two() { + return Err(HvError::InvalidParameter); + } + self.mmio_write( + input.gpa, + input + .data + .get(..input.access_width as usize) + .ok_or(HvError::InvalidParameter)?, + ) + }) + } +} diff --git a/vm/hv1/hv1_hypercall/src/lib.rs b/vm/hv1/hv1_hypercall/src/lib.rs index b22398b344..acb6482ec0 100644 --- a/vm/hv1/hv1_hypercall/src/lib.rs +++ b/vm/hv1/hv1_hypercall/src/lib.rs @@ -36,6 +36,8 @@ pub use self::support::HvRepResult; pub use self::support::HypercallDefinition; pub use self::support::HypercallHandler; pub use self::support::HypercallIo; +pub use self::support::complete_hypercall; +pub use self::support::complete_hypercall_with_data; pub use self::x86::X64HypercallRegister; pub use self::x86::X64RegisterIo; pub use self::x86::X64RegisterState; diff --git a/vm/hv1/hv1_hypercall/src/support.rs b/vm/hv1/hv1_hypercall/src/support.rs index c95be2d9a6..2146acc648 100644 --- a/vm/hv1/hv1_hypercall/src/support.rs +++ b/vm/hv1/hv1_hypercall/src/support.rs @@ -111,6 +111,12 @@ pub trait AsHandler { fn as_handler(&mut self) -> &mut H; } +impl, H> AsHandler for &mut T { + fn as_handler(&mut self) -> &mut H { + (**self).as_handler() + } +} + impl<'a, T: HypercallIo> InnerDispatcher<'a, T> { /// Creates a new dispatcher. fn new(guest_memory: &'a GuestMemory, mut handler: T) -> Self { @@ -356,14 +362,17 @@ impl<'a, T: HypercallIo> InnerDispatcher<'a, T> { pub trait HypercallIo { /// Advances the instruction pointer for a completed hypercall. /// - /// Either `advance_ip` or `retry` will be called. + /// Either `advance_ip` or `retry` will be called. `advance_ip` may be called + /// after `retry`, but `retry` will not be called after `advance_ip`. fn advance_ip(&mut self); /// Retains the instruction pointer at the hypercall point so that the /// hypercall will be retried. /// - /// Either `advance_ip` or `retry` will be called. /// `control` is the updated hypercall input value to use in the retry. + /// + /// Either `advance_ip` or `retry` will be called. `advance_ip` may be called + /// after `retry`, but `retry` will not be called after `advance_ip`. fn retry(&mut self, control: u64); /// The hypercall input value. @@ -774,3 +783,38 @@ impl Dispatcher { dispatcher.complete(result); } } + +/// Completes a hypercall with the given result and data, separately +/// from the dispatcher. +/// +/// Only simple, non-fast hypercalls are supported. +pub fn complete_hypercall_with_data( + io: &mut impl HypercallIo, + mut result: HvResult<()>, + guest_memory: &GuestMemory, + data: &[u8], +) { + let control = Control::from(io.control()); + if result.is_ok() && !data.is_empty() { + if control.fast() || control.rep_count() != 0 { + // Not supported. + tracelimit::error_ratelimited!( + "completing pending fast hypercall with data, not supported" + ); + result = Err(HvError::InvalidHypercallInput); + } else if let Err(err) = guest_memory.write_at(io.output_gpa(), data) { + tracelimit::error_ratelimited!( + error = &err as &dyn std::error::Error, + "failed to write hypercall output data" + ); + result = Err(HvError::InvalidHypercallInput); + } + } + complete_hypercall(io, result.into()) +} + +/// Completes a hypercall separately from the dispatcher. +pub fn complete_hypercall(io: &mut impl HypercallIo, result: HypercallOutput) { + io.set_result(result.into()); + io.advance_ip(); +} diff --git a/vm/hv1/hvdef/src/lib.rs b/vm/hv1/hvdef/src/lib.rs index d4b465db3e..9195209da3 100644 --- a/vm/hv1/hvdef/src/lib.rs +++ b/vm/hv1/hvdef/src/lib.rs @@ -906,6 +906,15 @@ pub mod hypercall { } } + impl From> for HypercallOutput { + fn from(result: HvResult<()>) -> Self { + match result { + Ok(()) => Self::new(), + Err(e) => Self::from(e), + } + } + } + impl HypercallOutput { /// A success output with zero elements processed. pub const SUCCESS: Self = Self::new(); diff --git a/vmm_core/virt_whp/src/hypercalls.rs b/vmm_core/virt_whp/src/hypercalls.rs index e39d066d2b..f2892be060 100644 --- a/vmm_core/virt_whp/src/hypercalls.rs +++ b/vmm_core/virt_whp/src/hypercalls.rs @@ -7,6 +7,7 @@ use crate::memory::VtlAccess; use crate::vtl2; #[cfg(guest_arch = "aarch64")] use aarch64 as arch; +use arrayvec::ArrayVec; use hv1_hypercall::HvRepResult; use hvdef::HV_PAGE_SIZE; use hvdef::HV_PARTITION_ID_SELF; @@ -29,6 +30,18 @@ pub(crate) struct WhpHypercallExit<'a, 'b, T> { vp: &'a mut WhpProcessor<'b>, bus: &'a T, registers: arch::WhpHypercallRegisters<'a>, + pending_hypercall: Option, +} + +enum PendingHypercall { + MmioRead { + gpa: u64, + len: usize, + }, + MmioWrite { + gpa: u64, + data: ArrayVec, + }, } impl WhpHypercallExit<'_, '_, T> { @@ -63,6 +76,8 @@ impl WhpHypercallExit<'_, '_, T> { hv1_hypercall::HvGetVpIndexFromApicId, hv1_hypercall::HvAcceptGpaPages, hv1_hypercall::HvModifySparseGpaPageHostVisibility, + hv1_hypercall::HvMemoryMappedIoRead, + hv1_hypercall::HvMemoryMappedIoWrite, ] ); } @@ -679,6 +694,71 @@ impl hv1_hypercall::ModifySparseGpaPageHostVisibility for WhpHypercall } } +impl hv1_hypercall::MemoryMappedIoRead for WhpHypercallExit<'_, '_, T> { + fn mmio_read(&mut self, gpa: u64, data: &mut [u8]) -> hvdef::HvResult<()> { + // Only for VTL2 testing at the moment. + if self.vp.state.active_vtl != Vtl::Vtl2 { + return Err(HvError::InvalidHypercallCode); + } + // Set a pending hypercall since the MMIO access may be async. + self.pending_hypercall = Some(PendingHypercall::MmioRead { + gpa, + len: data.len(), + }); + Err(HvError::Timeout) + } +} + +impl hv1_hypercall::MemoryMappedIoWrite for WhpHypercallExit<'_, '_, T> { + fn mmio_write(&mut self, gpa: u64, data: &[u8]) -> hvdef::HvResult<()> { + // Only for VTL2 testing at the moment. + if self.vp.state.active_vtl != Vtl::Vtl2 { + return Err(HvError::InvalidHypercallCode); + } + // Set a pending hypercall since the MMIO access may be async. + self.pending_hypercall = Some(PendingHypercall::MmioWrite { + gpa, + data: data.try_into().expect("cap is set to hypercall maximum"), + }); + Err(HvError::Timeout) + } +} + +impl WhpHypercallExit<'_, '_, T> { + async fn handle_pending_hypercall( + mut io: impl hv1_hypercall::AsHandler + hv1_hypercall::HypercallIo, + ) { + let this = io.as_handler(); + if let Some(hc) = this.pending_hypercall.take() { + match hc { + PendingHypercall::MmioRead { gpa, len } => { + let mut data = + ArrayVec::from([0; hvdef::hypercall::HV_HYPERCALL_MMIO_MAX_DATA_LENGTH]); + data.truncate(len); + this.bus + .read_mmio(this.vp.inner.vp_info.base.vp_index, gpa, &mut data) + .await; + + let guest_memory = &this.vp.vp.partition.gm; + hv1_hypercall::complete_hypercall_with_data( + &mut io, + Ok(()), + guest_memory, + &data, + ); + } + PendingHypercall::MmioWrite { gpa, data } => { + this.bus + .write_mmio(this.vp.inner.vp_info.base.vp_index, gpa, &data) + .await; + + hv1_hypercall::complete_hypercall(&mut io, Ok(()).into()); + } + } + } + } +} + #[cfg(guest_arch = "x86_64")] mod x86 { use super::WhpHypercallExit; @@ -804,7 +884,7 @@ mod x86 { ); } - pub fn handle( + pub async fn handle( vp: &'a mut WhpProcessor<'b>, bus: &'a T, info: &whp::abi::WHV_HYPERCALL_CONTEXT, @@ -823,12 +903,16 @@ mod x86 { invalid_opcode: false, exit_context, }; - let mut this = Self { vp, bus, registers }; + let mut this = Self { + vp, + bus, + registers, + pending_hypercall: None, + }; - WhpHypercallExit::DISPATCHER.dispatch( - &vpref.partition.gm, - hv1_hypercall::X64RegisterIo::new(&mut this, is_64bit), - ); + let mut handler = hv1_hypercall::X64RegisterIo::new(&mut this, is_64bit); + WhpHypercallExit::DISPATCHER.dispatch(&vpref.partition.gm, &mut handler); + Self::handle_pending_hypercall(handler).await; this.flush() } @@ -1725,7 +1809,7 @@ mod aarch64 { todo!("TODO-aarch64") } - pub fn handle( + pub async fn handle( vp: &'a mut WhpProcessor<'b>, bus: &'a T, message: &hvdef::HvArm64HypercallInterceptMessage, @@ -1738,12 +1822,17 @@ mod aarch64 { gp_dirty: false, _dummy: &(), }; - let mut this = Self { vp, bus, registers }; + let mut this = Self { + vp, + bus, + registers, + pending_hypercall: None, + }; - WhpHypercallExit::DISPATCHER.dispatch( - &vpref.partition.gm, - hv1_hypercall::Arm64RegisterIo::new(&mut this, false, message.immediate == 0), - ); + let mut handler = + hv1_hypercall::Arm64RegisterIo::new(&mut this, false, message.immediate == 0); + WhpHypercallExit::DISPATCHER.dispatch(&vpref.partition.gm, &mut handler); + Self::handle_pending_hypercall(handler).await; this.flush(); } diff --git a/vmm_core/virt_whp/src/vp.rs b/vmm_core/virt_whp/src/vp.rs index 4d68bab8e1..9898823f23 100644 --- a/vmm_core/virt_whp/src/vp.rs +++ b/vmm_core/virt_whp/src/vp.rs @@ -630,6 +630,7 @@ mod x86 { } ExitReason::Hypercall(info) => { crate::hypercalls::WhpHypercallExit::handle(self, dev, info, exit.vp_context) + .await .map_err(VpHaltReason::Hypervisor)?; &mut self.state.exits.hypercall } @@ -1810,7 +1811,8 @@ mod aarch64 { self, dev, message_ref(message), - ); + ) + .await; &mut self.state.exits.hypercall } HvMessageType::HvMessageTypeArm64ResetIntercept => {