Implementing IRQAllocator in vmm

We needed IRQAllocator for adding support
for multiple devices. IRQ_NUM for each platform is now passed
down through VMConfig. default value are used.

Signed-off-by: Ramyak mehra <rmehra_be19@thapar.edu>

Author: ramyak-mehra

src/vm-vcpu-ref/src/x86_64/mptable.rs

@@ -73,6 +73,8 @@ pub enum Error {
     WriteMpcLintsrc,
     /// Failure to write MP table header.
     WriteMpcTable,
+    /// IRQ num overflow.
+    IRQOverflowed,
 }
 
 /// Specialized `Result` type for operations on the MP Table.
@@ -129,11 +131,11 @@ fn mpf_intel_compute_checksum(v: &mpspec::mpf_intel) -> u8 {
 ///
 /// ```rust
 /// use vm_memory::{GuestAddress, GuestMemoryMmap};
-/// use vm_vcpu_ref::x86_64::mptable::{MpTable, Result};
+/// use vm_vcpu_ref::x86_64::mptable::{MpTable, Result, IRQ_MAX};
 ///
 /// fn write_mptable() -> Result<()> {
 ///     let num_cpus = 4;
-///     let mptable = MpTable::new(num_cpus)?;
+///     let mptable = MpTable::new(num_cpus, IRQ_MAX)?;
 ///     let mem: GuestMemoryMmap =
 ///         GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 1024 << 20)]).unwrap();
 ///     mptable.write(&mem)
@@ -148,15 +150,12 @@ pub struct MpTable {
 
 impl MpTable {
     /// Creates a new MP Table that can hold `cpu_num`.
-    pub fn new(cpu_num: u8) -> Result<MpTable> {
+    pub fn new(cpu_num: u8, max_irq: u8) -> Result<MpTable> {
         if cpu_num > MAX_SUPPORTED_CPUS {
             return Err(Error::TooManyCpus);
         }
-
-        Ok(MpTable {
-            irq_num: IRQ_MAX + 1,
-            cpu_num,
-        })
+        let irq_num = max_irq.checked_add(1).ok_or(Error::IRQOverflowed)?;
+        Ok(MpTable { irq_num, cpu_num })
     }
 
     // Returns the size of this MP table based on its configuration.
@@ -365,21 +364,21 @@ mod tests {
     fn test_bounds_check() {
         // Test mptable fits in guest memory.
         let num_cpus = 4;
-        let mptable = MpTable::new(num_cpus).unwrap();
+        let mptable = MpTable::new(num_cpus, IRQ_MAX).unwrap();
         let mem: GuestMemoryMmap =
             GuestMemoryMmap::from_ranges(&[(GuestAddress(MPTABLE_START), mptable.size())]).unwrap();
         mptable.write(&mem).unwrap();
 
         // Test mptable does not fit in guest memory.
         let num_cpus = 5;
-        let mptable = MpTable::new(num_cpus).unwrap();
+        let mptable = MpTable::new(num_cpus, IRQ_MAX).unwrap();
         assert_eq!(mptable.write(&mem).unwrap_err(), Error::NotEnoughMemory);
     }
 
     #[test]
     fn test_mpf_intel_checksum() {
         let num_cpus = 1;
-        let mptable = MpTable::new(num_cpus).unwrap();
+        let mptable = MpTable::new(num_cpus, IRQ_MAX).unwrap();
         let mem: GuestMemoryMmap =
             GuestMemoryMmap::from_ranges(&[(GuestAddress(MPTABLE_START), mptable.size())]).unwrap();
 
@@ -396,7 +395,7 @@ mod tests {
     #[test]
     fn test_mpc_table_checksum() {
         let num_cpus = 4;
-        let mptable = MpTable::new(num_cpus).unwrap();
+        let mptable = MpTable::new(num_cpus, IRQ_MAX).unwrap();
         let mem: GuestMemoryMmap =
             GuestMemoryMmap::from_ranges(&[(GuestAddress(MPTABLE_START), mptable.size())]).unwrap();
 
@@ -431,7 +430,7 @@ mod tests {
             GuestMemoryMmap::from_ranges(&[(GuestAddress(MPTABLE_START), 1024 << 20)]).unwrap();
 
         for i in 0..MAX_SUPPORTED_CPUS {
-            MpTable::new(i).unwrap().write(&mem).unwrap();
+            MpTable::new(i, IRQ_MAX).unwrap().write(&mem).unwrap();
 
             let mpf_intel: MpfIntel = mem.read_obj(GuestAddress(MPTABLE_START)).unwrap();
             let mpc_offset = GuestAddress(u64::from(mpf_intel.0.physptr));
@@ -462,7 +461,17 @@ mod tests {
     fn test_cpu_entry_count_max() {
         let cpus = MAX_SUPPORTED_CPUS + 1;
 
-        let result = MpTable::new(cpus).unwrap_err();
+        let result = MpTable::new(cpus, IRQ_MAX).unwrap_err();
         assert_eq!(result, Error::TooManyCpus);
     }
+    #[test]
+    fn test_irq_num() {
+        let mptable = MpTable::new(MAX_SUPPORTED_CPUS, IRQ_MAX).unwrap();
+        assert_eq!(mptable.irq_num, IRQ_MAX + 1)
+    }
+    #[test]
+    fn test_max_irq_overflow() {
+        let result = MpTable::new(MAX_SUPPORTED_CPUS, u8::MAX);
+        assert_eq!(result.unwrap_err(), Error::IRQOverflowed)
+    }
 }

src/vm-vcpu/src/vm.rs

@@ -2,6 +2,8 @@
 // Copyright 2017 The Chromium OS Authors. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
 
+#[cfg(target_arch = "x86_64")]
+use std::convert::TryInto;
 use std::io::{self, ErrorKind};
 use std::sync::{Arc, Barrier, Mutex};
 use std::thread::{self, JoinHandle};
@@ -27,19 +29,26 @@ use vm_vcpu_ref::aarch64::interrupts::{self, Gic, GicConfig, GicState};
 #[cfg(target_arch = "x86_64")]
 use vm_vcpu_ref::x86_64::mptable::{self, MpTable};
 
+#[cfg(target_arch = "aarch64")]
+pub const MAX_IRQ: u32 = interrupts::MIN_NR_IRQS;
+#[cfg(target_arch = "x86_64")]
+pub const MAX_IRQ: u32 = mptable::IRQ_MAX as u32;
+
 /// Defines the configuration of this VM.
 #[derive(Clone)]
 pub struct VmConfig {
     pub num_vcpus: u8,
     pub vcpus_config: VcpuConfigList,
+    pub max_irq: u32,
 }
 
 impl VmConfig {
     /// Creates a default `VmConfig` for `num_vcpus`.
-    pub fn new(kvm: &Kvm, num_vcpus: u8) -> Result<Self> {
+    pub fn new(kvm: &Kvm, num_vcpus: u8, max_irq: u32) -> Result<Self> {
         Ok(VmConfig {
             num_vcpus,
             vcpus_config: VcpuConfigList::new(kvm, num_vcpus).map_err(Error::CreateVmConfig)?,
+            max_irq,
         })
     }
 }
@@ -148,6 +157,10 @@ pub enum Error {
     /// Failed to save the state of vCPUs.
     #[error("Failed to save the state of vCPUs: {0}")]
     SaveVcpuState(vcpu::Error),
+    #[cfg(target_arch = "x86_64")]
+    /// Invalid max IRQ value.
+    #[error("Invalid maximum number of IRQ: {0}")]
+    IRQMaxValue(u32),
 }
 
 #[cfg(target_arch = "x86_64")]
@@ -229,8 +242,14 @@ impl<EH: 'static + ExitHandler + Send> KvmVm<EH> {
         let mut vm = Self::create_vm(kvm, vm_config, exit_handler, guest_memory)?;
 
         #[cfg(target_arch = "x86_64")]
-        MpTable::new(vm.config.num_vcpus)?.write(guest_memory)?;
-
+        {
+            let max_irq: u8 = vm
+                .config
+                .max_irq
+                .try_into()
+                .map_err(|_| Error::IRQMaxValue(vm.config.max_irq))?;
+            MpTable::new(vm.config.num_vcpus, max_irq)?.write(guest_memory)?;
+        }
         #[cfg(target_arch = "x86_64")]
         vm.setup_irq_controller()?;
 
@@ -314,6 +333,11 @@ impl<EH: 'static + ExitHandler + Send> KvmVm<EH> {
         self.gic.as_ref().expect("GIC is not set")
     }
 
+    /// Returns the max irq number independent of arch.
+    pub fn max_irq(&self) -> u32 {
+        self.config.max_irq
+    }
+
     // Create the kvm memory regions based on the configuration passed as `guest_memory`.
     fn configure_memory_regions<M: GuestMemory>(&self, guest_memory: &M, kvm: &Kvm) -> Result<()> {
         if guest_memory.num_regions() > kvm.get_nr_memslots() {
@@ -376,6 +400,7 @@ impl<EH: 'static + ExitHandler + Send> KvmVm<EH> {
         let gic = Gic::new(
             GicConfig {
                 num_cpus: self.config.num_vcpus,
+                num_irqs: self.config.max_irq,
                 ..Default::default()
             },
             &self.vm_fd(),
@@ -612,7 +637,7 @@ mod tests {
         guest_memory: &GuestMemoryMmap,
         num_vcpus: u8,
     ) -> Result<KvmVm<WrappedExitHandler>> {
-        let vm_config = VmConfig::new(kvm, num_vcpus).unwrap();
+        let vm_config = VmConfig::new(kvm, num_vcpus, MAX_IRQ).unwrap();
         let io_manager = Arc::new(Mutex::new(IoManager::new()));
         let exit_handler = WrappedExitHandler::default();
         let vm = KvmVm::new(kvm, vm_config, guest_memory, exit_handler, io_manager)?;
@@ -641,6 +666,21 @@ mod tests {
         assert!(matches!(res, Err(Error::Mptable(_))));
     }
 
+    #[test]
+    #[cfg(target_arch = "x86_64")]
+    fn test_max_irq_overflow() {
+        let num_vcpus = 1;
+        let kvm = Kvm::new().unwrap();
+        let guest_memory = default_memory();
+
+        let vm_config = VmConfig::new(&kvm, num_vcpus, u32::MAX).unwrap();
+        let io_manager = Arc::new(Mutex::new(IoManager::new()));
+        let exit_handler = WrappedExitHandler::default();
+        let vm = KvmVm::new(&kvm, vm_config, &guest_memory, exit_handler, io_manager);
+
+        assert!(matches!(vm, Err(Error::IRQMaxValue(_))))
+    }
+
     #[test]
     fn test_failed_setup_memory() {
         let kvm = Kvm::new().unwrap();
@@ -661,7 +701,7 @@ mod tests {
     fn test_failed_irqchip_setup() {
         let kvm = Kvm::new().unwrap();
         let num_vcpus = 1;
-        let vm_state = VmConfig::new(&kvm, num_vcpus).unwrap();
+        let vm_state = VmConfig::new(&kvm, num_vcpus, MAX_IRQ).unwrap();
         let mut vm = KvmVm {
             vcpus: Vec::new(),
             vcpu_handles: Vec::new(),
@@ -670,7 +710,6 @@ mod tests {
             fd: Arc::new(kvm.create_vm().unwrap()),
             exit_handler: WrappedExitHandler::default(),
             vcpu_run_state: Arc::new(VcpuRunState::default()),
-
             #[cfg(target_arch = "aarch64")]
             gic: None,
         };

src/vmm/src/irq_allocator.rs

@@ -0,0 +1,93 @@
+// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+
+use std::fmt;
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    InvalidValue,
+    MaxIrq,
+    IRQOverflowed,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// An irq allocator which gives next available irq.
+/// It is mainly used for non-legacy devices.
+// There are a few reserved irq's on x86_64. We just skip all the inital
+// reserved irq to make the implementaion simple. This could be later extended
+// to cater more complex scenario.
+#[derive(Debug)]
+pub struct IrqAllocator {
+    // Tracks the last allocated irq
+    last_used_irq: u32,
+    last_irq: u32,
+}
+
+impl IrqAllocator {
+    pub fn new(last_used_irq: u32, last_irq: u32) -> Result<Self> {
+        if last_used_irq >= last_irq {
+            return Err(Error::InvalidValue);
+        }
+        Ok(IrqAllocator {
+            last_used_irq,
+            last_irq,
+        })
+    }
+
+    pub fn next_irq(&mut self) -> Result<u32> {
+        self.last_used_irq
+            .checked_add(1)
+            .ok_or(Error::IRQOverflowed)
+            .and_then(|irq| {
+                if irq > self.last_irq {
+                    Err(Error::MaxIrq)
+                } else {
+                    self.last_used_irq = irq;
+                    Ok(irq)
+                }
+            })
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let err = match self {
+            Error::MaxIrq => "last_irq IRQ limit reached",
+            Error::IRQOverflowed => "IRQ overflowed",
+            Error::InvalidValue => {
+                "Check the value of last_used and last_irq. las_used should be less than last_irq"
+            }
+        };
+        write!(f, "{}", err) // user-facing output
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::{Error, IrqAllocator};
+    #[test]
+    fn test_new() {
+        let irq_alloc = IrqAllocator::new(4, 10).unwrap();
+        assert_eq!(irq_alloc.last_used_irq, 4);
+        assert_eq!(irq_alloc.last_irq, 10);
+        let irq_alloc = IrqAllocator::new(4, 4).unwrap_err();
+        assert_eq!(irq_alloc, Error::InvalidValue);
+        let irq_alloc = IrqAllocator::new(4, 3).unwrap_err();
+        assert_eq!(irq_alloc, Error::InvalidValue);
+    }
+    #[test]
+    fn test_next_irq() {
+        let mut irq_alloc = IrqAllocator::new(4, 7).unwrap();
+        assert_eq!(irq_alloc.next_irq(), Ok(5));
+
+        let _ = irq_alloc.next_irq();
+        assert_eq!(irq_alloc.next_irq(), Ok(7));
+
+        assert_eq!(irq_alloc.next_irq(), Err(Error::MaxIrq));
+
+        let mut irq_alloc = IrqAllocator::new(u32::MAX - 1, u32::MAX).unwrap();
+        assert_eq!(irq_alloc.next_irq(), Ok(u32::MAX));
+        assert_eq!(irq_alloc.next_irq(), Err(Error::IRQOverflowed))
+    }
+}