x86-64: enable PCIe enhanced configuration support

src/arch/x86_64/kernel/acpi.rs

@@ -44,6 +44,11 @@ const SLP_EN: u16 = 1 << 13;
 
 /// The "Multiple APIC Description Table" (MADT) preserved for get_apic_table().
 static MADT: OnceCell<AcpiTable<'_>> = OnceCell::new();
+
+/// The MCFG table, to address PCI-E configuration space
+#[cfg(feature = "pci")]
+static MCFG: OnceCell<AcpiTable<'_>> = OnceCell::new();
+
 /// The PM1A Control I/O Port for powering off the computer through ACPI.
 static PM1A_CNT_BLK: OnceCell<Port<u16>> = OnceCell::new();
 /// The Sleeping State Type code for powering off the computer through ACPI.
@@ -471,6 +476,11 @@ pub fn get_madt() -> Option<&'static AcpiTable<'static>> {
 	MADT.get()
 }
 
+#[cfg(feature = "pci")]
+pub fn get_mcfg_table() -> Option<&'static AcpiTable<'static>> {
+	MCFG.get()
+}
+
 pub fn poweroff() {
 	if let (Some(mut pm1a_cnt_blk), Some(&slp_typa)) = (PM1A_CNT_BLK.get().cloned(), SLP_TYPA.get())
 	{
@@ -546,6 +556,13 @@ pub fn init() {
 				"SSDT at {table_physical_address:p} has invalid checksum"
 			);
 			parse_ssdt(table);
+		} else if cfg!(feature = "pci") && table.header.signature() == "MCFG" {
+			assert!(
+				verify_checksum(table.header_start_address(), table.header.length as usize).is_ok(),
+				"MCFG at {table_physical_address:p} has invalid checksum"
+			);
+			#[cfg(feature = "pci")]
+			MCFG.set(table).unwrap();
 		}
 	}
 }

src/arch/x86_64/kernel/mod.rs

@@ -166,14 +166,14 @@ pub fn boot_processor_init() {
 	interrupts::install();
 	systemtime::init();
 
-	if is_uhyve_with_pci() || !is_uhyve() {
-		#[cfg(feature = "pci")]
-		pci::init();
-	}
 	if !env::is_uhyve() {
 		#[cfg(feature = "acpi")]
 		acpi::init();
 	}
+	if is_uhyve_with_pci() || !is_uhyve() {
+		#[cfg(feature = "pci")]
+		pci::init();
+	}
 
 	apic::init();
 	scheduler::install_timer_handler();

src/arch/x86_64/kernel/pci.rs

@@ -11,16 +11,46 @@ const PCI_CONFIG_ADDRESS_ENABLE: u32 = 1 << 31;
 const CONFIG_ADDRESS: Port<u32> = Port::new(0xcf8);
 const CONFIG_DATA: Port<u32> = Port::new(0xcfc);
 
+#[allow(clippy::upper_case_acronyms)]
 #[derive(Debug, Copy, Clone)]
-pub(crate) struct PciConfigRegion;
+pub enum PciConfigRegion {
+	PCI(LegacyPciConfigRegion),
+	#[cfg(feature = "acpi")]
+	PCIe(pcie::McfgTableEntry),
+}
+
+impl ConfigRegionAccess for PciConfigRegion {
+	unsafe fn read(&self, address: PciAddress, offset: u16) -> u32 {
+		match self {
+			PciConfigRegion::PCI(entry) => unsafe { entry.read(address, offset) },
+			#[cfg(feature = "acpi")]
+			PciConfigRegion::PCIe(entry) => unsafe { entry.read(address, offset) },
+		}
+	}
+
+	unsafe fn write(&self, address: PciAddress, offset: u16, value: u32) {
+		match self {
+			PciConfigRegion::PCI(entry) => unsafe {
+				entry.write(address, offset, value);
+			},
+			#[cfg(feature = "acpi")]
+			PciConfigRegion::PCIe(entry) => unsafe {
+				entry.write(address, offset, value);
+			},
+		}
+	}
+}
 
-impl PciConfigRegion {
+#[derive(Debug, Copy, Clone)]
+pub(crate) struct LegacyPciConfigRegion;
+
+impl LegacyPciConfigRegion {
 	pub const fn new() -> Self {
 		Self {}
 	}
 }
 
-impl ConfigRegionAccess for PciConfigRegion {
+impl ConfigRegionAccess for LegacyPciConfigRegion {
 	#[inline]
 	unsafe fn read(&self, pci_addr: PciAddress, register: u16) -> u32 {
 		let mut config_address = CONFIG_ADDRESS;
@@ -59,20 +89,139 @@ impl ConfigRegionAccess for PciConfigRegion {
 pub(crate) fn init() {
 	debug!("Scanning PCI Busses 0 to {}", PCI_MAX_BUS_NUMBER - 1);
 
+	#[cfg(feature = "acpi")]
+	if pcie::init_pcie() {
+		return;
+	}
+
+	enumerate_devices(
+		0,
+		PCI_MAX_BUS_NUMBER,
+		PciConfigRegion::PCI(LegacyPciConfigRegion::new()),
+	);
+}
+
+fn enumerate_devices(bus_start: u8, bus_end: u8, access: PciConfigRegion) {
 	// Hermit only uses PCI for network devices.
 	// Therefore, multifunction devices as well as additional bridges are not scanned.
 	// We also limit scanning to the first 32 buses.
-	let pci_config = PciConfigRegion::new();
-	for bus in 0..PCI_MAX_BUS_NUMBER {
+	for bus in bus_start..bus_end {
 		for device in 0..PCI_MAX_DEVICE_NUMBER {
 			let pci_address = PciAddress::new(0, bus, device, 0);
 			let header = PciHeader::new(pci_address);
 
-			let (device_id, vendor_id) = header.id(pci_config);
+			let (device_id, vendor_id) = header.id(access);
 			if device_id != u16::MAX && vendor_id != u16::MAX {
-				let device = PciDevice::new(pci_address, pci_config);
+				let device = PciDevice::new(pci_address, access);
 				PCI_DEVICES.with(|pci_devices| pci_devices.unwrap().push(device));
 			}
 		}
 	}
 }
+
+#[cfg(feature = "acpi")]
+mod pcie {
+	use memory_addresses::PhysAddr;
+	use pci_types::{ConfigRegionAccess, PciAddress};
+
+	use super::{PCI_MAX_BUS_NUMBER, PciConfigRegion};
+	use crate::env::kernel::acpi;
+
+	pub fn init_pcie() -> bool {
+		let Some(table) = acpi::get_mcfg_table() else {
+			return false;
+		};
+
+		let mut start_addr: *const McfgTableEntry =
+			core::ptr::with_exposed_provenance(table.table_start_address() + 8);
+		let end_addr: *const McfgTableEntry =
+			core::ptr::with_exposed_provenance(table.table_end_address() + 8);
+
+		if start_addr == end_addr {
+			return false;
+		}
+
+		while start_addr < end_addr {
+			unsafe {
+				let read = core::ptr::read_unaligned(start_addr);
+				init_pcie_bus(read);
+				start_addr = start_addr.add(1);
+			}
+		}
+
+		true
+	}
+
+	#[derive(Debug, Copy, Clone)]
+	#[repr(C)]
+	pub(crate) struct McfgTableEntry {
+		pub base_address: u64,
+		pub pci_segment_number: u16,
+		pub start_pci_bus: u8,
+		pub end_pci_bus: u8,
+		_reserved: u32,
+	}
+
+	impl McfgTableEntry {
+		pub fn pci_config_space_address(
+			&self,
+			bus_number: u8,
+			device: u8,
+			function: u8,
+		) -> PhysAddr {
+			PhysAddr::new(
+				self.base_address
+					+ ((u64::from(bus_number) << 20)
+						| ((u64::from(device) & 0x1f) << 15)
+						| ((u64::from(function) & 0x7) << 12)),
+			)
+		}
+	}
+
+	impl ConfigRegionAccess for McfgTableEntry {
+		unsafe fn read(&self, address: PciAddress, offset: u16) -> u32 {
+			assert_eq!(address.segment(), self.pci_segment_number);
+			assert!(address.bus() >= self.start_pci_bus);
+			assert!(address.bus() <= self.end_pci_bus);
+
+			let ptr =
+				self.pci_config_space_address(address.bus(), address.device(), address.function())
+					+ u64::from(offset);
+			let ptr = ptr.as_usize() as *const u32;
+
+			unsafe { *ptr }
+		}
+
+		unsafe fn write(&self, address: PciAddress, offset: u16, value: u32) {
+			assert_eq!(address.segment(), self.pci_segment_number);
+			assert!(address.bus() >= self.start_pci_bus);
+			assert!(address.bus() <= self.end_pci_bus);
+
+			let ptr =
+				self.pci_config_space_address(address.bus(), address.device(), address.function())
+					+ u64::from(offset);
+			let ptr = ptr.as_usize() as *mut u32;
+
+			unsafe {
+				*ptr = value;
+			}
+		}
+	}
+
+	fn init_pcie_bus(bus_entry: McfgTableEntry) {
+		if bus_entry.start_pci_bus > PCI_MAX_BUS_NUMBER {
+			return;
+		}
+
+		let end = if bus_entry.end_pci_bus > PCI_MAX_BUS_NUMBER {
+			PCI_MAX_BUS_NUMBER
+		} else {
+			bus_entry.end_pci_bus
+		};
+		super::enumerate_devices(
+			bus_entry.start_pci_bus,
+			end,
+			PciConfigRegion::PCIe(bus_entry),
+		);
+	}
+}