Merge pull request #1828 from hermit-os/phys_virt_mem

refactor(mm): remove physicalmem/virtualmem freestanding functions

Author: mkroening

src/arch/aarch64/kernel/interrupts.rs

@@ -8,9 +8,11 @@ use aarch64::regs::*;
 use ahash::RandomState;
 use arm_gic::gicv3::{GicV3, InterruptGroup, SgiTarget, SgiTargetGroup};
 use arm_gic::{IntId, Trigger};
+use free_list::PageLayout;
 use hashbrown::HashMap;
 use hermit_dtb::Dtb;
 use hermit_sync::{InterruptSpinMutex, InterruptTicketMutex, OnceCell, SpinMutex};
+use memory_addresses::VirtAddr;
 use memory_addresses::arch::aarch64::PhysAddr;
 
 use crate::arch::aarch64::kernel::core_local::increment_irq_counter;
@@ -21,7 +23,7 @@ use crate::drivers::mmio::get_interrupt_handlers;
 #[cfg(feature = "pci")]
 use crate::drivers::pci::get_interrupt_handlers;
 use crate::drivers::{InterruptHandlerQueue, InterruptLine};
-use crate::mm::virtualmem;
+use crate::mm::virtualmem::KERNEL_FREE_LIST;
 use crate::scheduler::{self, CoreId};
 use crate::{core_id, core_scheduler, env};
 
@@ -310,8 +312,9 @@ pub(crate) fn init() {
 		"Found generic interrupt controller redistributor at {gicr_start:p} (size {gicr_size:#X})"
 	);
 
-	let gicd_address =
-		virtualmem::allocate_aligned(gicd_size.try_into().unwrap(), 0x10000).unwrap();
+	let layout = PageLayout::from_size_align(gicd_size.try_into().unwrap(), 0x10000).unwrap();
+	let page_range = KERNEL_FREE_LIST.lock().allocate(layout).unwrap();
+	let gicd_address = VirtAddr::from(page_range.start());
 	debug!("Mapping GIC Distributor interface to virtual address {gicd_address:p}",);
 
 	let mut flags = PageTableEntryFlags::empty();
@@ -323,8 +326,9 @@ pub(crate) fn init() {
 		flags,
 	);
 
-	let gicr_address =
-		virtualmem::allocate_aligned(gicr_size.try_into().unwrap(), 0x10000).unwrap();
+	let layout = PageLayout::from_size_align(gicr_size.try_into().unwrap(), 0x10000).unwrap();
+	let page_range = KERNEL_FREE_LIST.lock().allocate(layout).unwrap();
+	let gicr_address = VirtAddr::from(page_range.start());
 	debug!("Mapping generic interrupt controller to virtual address {gicr_address:p}",);
 	paging::map::<BasePageSize>(
 		gicr_address,

src/arch/aarch64/kernel/pci.rs

@@ -3,6 +3,7 @@ use core::str;
 
 use arm_gic::{IntId, Trigger};
 use bit_field::BitField;
+use free_list::PageLayout;
 use hermit_dtb::Dtb;
 use memory_addresses::arch::aarch64::{PhysAddr, VirtAddr};
 use pci_types::{
@@ -13,7 +14,7 @@ use pci_types::{
 use crate::arch::aarch64::kernel::interrupts::GIC;
 use crate::arch::aarch64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags};
 use crate::drivers::pci::{PCI_DEVICES, PciDevice};
-use crate::mm::virtualmem;
+use crate::mm::virtualmem::KERNEL_FREE_LIST;
 use crate::{core_id, env};
 
 const PCI_MAX_DEVICE_NUMBER: u8 = 32;
@@ -244,8 +245,10 @@ pub fn init() {
 				let (slice, _residual_slice) = residual_slice.split_at(core::mem::size_of::<u64>());
 				let size = u64::from_be_bytes(slice.try_into().unwrap());
 
-				let pci_address =
-					virtualmem::allocate_aligned(size.try_into().unwrap(), 0x1000_0000).unwrap();
+				let layout =
+					PageLayout::from_size_align(size.try_into().unwrap(), 0x1000_0000).unwrap();
+				let page_range = KERNEL_FREE_LIST.lock().allocate(layout).unwrap();
+				let pci_address = VirtAddr::from(page_range.start());
 				info!(
 					"Mapping PCI Enhanced Configuration Space interface to virtual address {pci_address:p} (size {size:#X})"
 				);

src/arch/aarch64/kernel/scheduler.rs

@@ -11,13 +11,16 @@ use core::sync::atomic::Ordering;
 use core::{mem, ptr};
 
 use align_address::Align;
+use free_list::{PageLayout, PageRange};
 use memory_addresses::arch::aarch64::{PhysAddr, VirtAddr};
 
 use crate::arch::aarch64::kernel::CURRENT_STACK_ADDRESS;
 use crate::arch::aarch64::kernel::core_local::core_scheduler;
 use crate::arch::aarch64::mm::paging::{BasePageSize, PageSize, PageTableEntryFlags};
 #[cfg(not(feature = "common-os"))]
 use crate::env;
+use crate::mm::physicalmem::PHYSICAL_FREE_LIST;
+use crate::mm::virtualmem::KERNEL_FREE_LIST;
 #[cfg(target_os = "none")]
 use crate::scheduler::PerCoreSchedulerExt;
 use crate::scheduler::task::{Task, TaskFrame};
@@ -132,11 +135,15 @@ impl TaskStacks {
 			size.align_up(BasePageSize::SIZE as usize)
 		};
 		let total_size = user_stack_size + DEFAULT_STACK_SIZE;
-		let virt_addr =
-			crate::mm::virtualmem::allocate(total_size + 3 * BasePageSize::SIZE as usize)
-				.expect("Failed to allocate Virtual Memory for TaskStacks");
-		let phys_addr = crate::mm::physicalmem::allocate(total_size)
+		let layout = PageLayout::from_size(total_size + 3 * BasePageSize::SIZE as usize).unwrap();
+		let page_range = KERNEL_FREE_LIST.lock().allocate(layout).unwrap();
+		let virt_addr = VirtAddr::from(page_range.start());
+		let frame_layout = PageLayout::from_size(total_size).unwrap();
+		let frame_range = PHYSICAL_FREE_LIST
+			.lock()
+			.allocate(frame_layout)
 			.expect("Failed to allocate Physical Memory for TaskStacks");
+		let phys_addr = PhysAddr::from(frame_range.start());
 
 		debug!(
 			"Create stacks at {:p} with a size of {} KB",
@@ -233,11 +240,21 @@ impl Drop for TaskStacks {
 					stacks.virt_addr,
 					stacks.total_size / BasePageSize::SIZE as usize + 3,
 				);
-				crate::mm::virtualmem::deallocate(
-					stacks.virt_addr,
+				let range = PageRange::from_start_len(
+					stacks.virt_addr.as_usize(),
 					stacks.total_size + 3 * BasePageSize::SIZE as usize,
-				);
-				crate::mm::physicalmem::deallocate(stacks.phys_addr, stacks.total_size);
+				)
+				.unwrap();
+				unsafe {
+					KERNEL_FREE_LIST.lock().deallocate(range).unwrap();
+				}
+
+				let range =
+					PageRange::from_start_len(stacks.phys_addr.as_usize(), stacks.total_size)
+						.unwrap();
+				unsafe {
+					PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap();
+				}
 			}
 		}
 	}

src/arch/aarch64/kernel/systemtime.rs

@@ -4,6 +4,7 @@ use alloc::vec::Vec;
 use core::arch::asm;
 use core::str;
 
+use free_list::PageLayout;
 use hermit_dtb::Dtb;
 use hermit_entry::boot_info::PlatformInfo;
 use hermit_sync::OnceCell;
@@ -13,6 +14,7 @@ use time::OffsetDateTime;
 use crate::arch::aarch64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags};
 use crate::env;
 use crate::mm::virtualmem;
+use crate::mm::virtualmem::KERNEL_FREE_LIST;
 
 static PL031_ADDRESS: OnceCell<VirtAddr> = OnceCell::new();
 static BOOT_TIME: OnceCell<u64> = OnceCell::new();
@@ -78,11 +80,9 @@ pub fn init() {
 
 					debug!("Found RTC at {addr:p} (size {size:#X})");
 
-					let pl031_address = virtualmem::allocate_aligned(
-						size.try_into().unwrap(),
-						BasePageSize::SIZE.try_into().unwrap(),
-					)
-					.unwrap();
+					let layout = PageLayout::from_size(size.try_into().unwrap()).unwrap();
+					let page_range = KERNEL_FREE_LIST.lock().allocate(layout).unwrap();
+					let pl031_address = VirtAddr::from(page_range.start());
 					PL031_ADDRESS.set(pl031_address).unwrap();
 					debug!("Mapping RTC to virtual address {pl031_address:p}",);
 

src/arch/aarch64/mm/paging.rs

@@ -5,11 +5,13 @@ use core::marker::PhantomData;
 use core::{fmt, mem, ptr};
 
 use align_address::Align;
+use free_list::PageLayout;
 use memory_addresses::{PhysAddr, VirtAddr};
 
 use crate::arch::aarch64::kernel::{get_base_address, get_image_size, get_ram_address, processor};
 use crate::env::is_uhyve;
 use crate::mm::physicalmem;
+use crate::mm::physicalmem::PHYSICAL_FREE_LIST;
 use crate::{KERNEL_STACK_SIZE, mm, scheduler};
 
 /// Pointer to the root page table (called "Level 0" in ARM terminology).
@@ -494,8 +496,12 @@ where
 			// Does the table exist yet?
 			if !self.entries[index].is_present() {
 				// Allocate a single 4 KiB page for the new entry and mark it as a valid, writable subtable.
-				let physical_address = physicalmem::allocate(BasePageSize::SIZE as usize)
+				let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap();
+				let frame_range = PHYSICAL_FREE_LIST
+					.lock()
+					.allocate(frame_layout)
 					.expect("Unable to allocate physical memory");
+				let physical_address = PhysAddr::from(frame_range.start());
 				self.entries[index].set(
 					physical_address,
 					PageTableEntryFlags::NORMAL | PageTableEntryFlags::TABLE_OR_4KIB_PAGE,
@@ -662,7 +668,11 @@ pub fn map_heap<S: PageSize>(virt_addr: VirtAddr, nr_pages: usize) -> Result<(),
 	while map_counter < nr_pages {
 		let size = (nr_pages - map_counter) * S::SIZE as usize;
 		for i in (S::SIZE as usize..=size).rev().step_by(S::SIZE as usize) {
-			if let Ok(phys_addr) = physicalmem::allocate_aligned(i, S::SIZE as usize) {
+			let layout = PageLayout::from_size_align(i, S::SIZE as usize).unwrap();
+			let frame_range = PHYSICAL_FREE_LIST.lock().allocate(layout);
+
+			if let Ok(frame_range) = frame_range {
+				let phys_addr = PhysAddr::from(frame_range.start());
 				map::<S>(
 					virt_addr + map_counter * S::SIZE as usize,
 					phys_addr,

src/arch/riscv64/kernel/devicetree.rs

@@ -244,10 +244,20 @@ pub fn init_drivers() {
 				// Verify the device-ID to find the network card
 				let id = mmio.as_ptr().device_id().read();
 
-				// crate::mm::physicalmem::reserve(
-				// 	PhysAddr::from(current_address.align_down(BasePageSize::SIZE as usize)),
-				// 	BasePageSize::SIZE as usize,
-				// );
+				if cfg!(debug_assertions) {
+					use free_list::PageRange;
+
+					use crate::mm::physicalmem::PHYSICAL_FREE_LIST;
+
+					let start = virtio_region.starting_address.addr();
+					let len = virtio_region.size.unwrap();
+					let frame_range = PageRange::from_start_len(start, len).unwrap();
+
+					PHYSICAL_FREE_LIST
+						.lock()
+						.allocate_at(frame_range)
+						.unwrap_err();
+				}
 
 				match id {
 					#[cfg(all(any(feature = "tcp", feature = "udp"), not(feature = "gem-net")))]

src/arch/riscv64/kernel/mod.rs

@@ -14,12 +14,12 @@ pub mod serial;
 mod start;
 pub mod switch;
 pub mod systemtime;
-
 use alloc::vec::Vec;
 use core::ptr;
 use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering};
 
 use fdt::Fdt;
+use free_list::PageLayout;
 use memory_addresses::{PhysAddr, VirtAddr};
 use riscv::register::sstatus;
 
@@ -29,7 +29,7 @@ use crate::arch::riscv64::kernel::processor::lsb;
 use crate::config::KERNEL_STACK_SIZE;
 use crate::env;
 use crate::init_cell::InitCell;
-use crate::mm::physicalmem;
+use crate::mm::physicalmem::PHYSICAL_FREE_LIST;
 
 // Used to store information about available harts. The index of the hart in the vector
 // represents its CpuId and does not need to match its hart_id
@@ -161,8 +161,12 @@ pub fn boot_next_processor() {
 	if let Some(next_hart_id) = next_hart_index {
 		{
 			debug!("Allocating stack for hard_id {next_hart_id}");
-			let stack = physicalmem::allocate(KERNEL_STACK_SIZE)
+			let frame_layout = PageLayout::from_size(KERNEL_STACK_SIZE).unwrap();
+			let frame_range = PHYSICAL_FREE_LIST
+				.lock()
+				.allocate(frame_layout)
 				.expect("Failed to allocate boot stack for new core");
+			let stack = PhysAddr::from(frame_range.start());
 			CURRENT_STACK_ADDRESS.store(stack.as_usize() as _, Ordering::Relaxed);
 		}
 

src/arch/riscv64/kernel/scheduler.rs

@@ -7,13 +7,16 @@ use core::convert::TryInto;
 use core::{mem, ptr};
 
 use align_address::Align;
+use free_list::{PageLayout, PageRange};
 use memory_addresses::{PhysAddr, VirtAddr};
 
 use crate::arch::riscv64::kernel::core_local::core_scheduler;
 use crate::arch::riscv64::kernel::processor::set_oneshot_timer;
 use crate::arch::riscv64::mm::paging::{BasePageSize, PageSize, PageTableEntryFlags};
 #[cfg(not(feature = "common-os"))]
 use crate::env;
+use crate::mm::physicalmem::PHYSICAL_FREE_LIST;
+use crate::mm::virtualmem::KERNEL_FREE_LIST;
 use crate::scheduler::task::{Task, TaskFrame};
 use crate::{DEFAULT_STACK_SIZE, KERNEL_STACK_SIZE};
 
@@ -116,12 +119,15 @@ impl TaskStacks {
 			size.align_up(BasePageSize::SIZE as usize)
 		};
 		let total_size = user_stack_size + DEFAULT_STACK_SIZE + KERNEL_STACK_SIZE;
-		let virt_addr =
-			crate::mm::virtualmem::allocate(total_size + 4 * BasePageSize::SIZE as usize)
-				//let virt_addr = crate::arch::mm::virtualmem::allocate(total_size)
-				.expect("Failed to allocate Virtual Memory for TaskStacks");
-		let phys_addr = crate::mm::physicalmem::allocate(total_size)
+		let layout = PageLayout::from_size(total_size + 4 * BasePageSize::SIZE as usize).unwrap();
+		let page_range = KERNEL_FREE_LIST.lock().allocate(layout).unwrap();
+		let virt_addr = VirtAddr::from(page_range.start());
+		let frame_layout = PageLayout::from_size(total_size).unwrap();
+		let frame_range = PHYSICAL_FREE_LIST
+			.lock()
+			.allocate(frame_layout)
 			.expect("Failed to allocate Physical Memory for TaskStacks");
+		let phys_addr = PhysAddr::from(frame_range.start());
 
 		debug!(
 			"Create stacks at {:#X} with a size of {} KB",
@@ -251,12 +257,21 @@ impl Drop for TaskStacks {
 					stacks.total_size / BasePageSize::SIZE as usize + 4,
 					//stacks.total_size / BasePageSize::SIZE as usize,
 				);
-				crate::mm::virtualmem::deallocate(
-					stacks.virt_addr,
+				let range = PageRange::from_start_len(
+					stacks.virt_addr.as_usize(),
 					stacks.total_size + 4 * BasePageSize::SIZE as usize,
-					//stacks.total_size,
-				);
-				crate::mm::physicalmem::deallocate(stacks.phys_addr, stacks.total_size);
+				)
+				.unwrap();
+				unsafe {
+					KERNEL_FREE_LIST.lock().deallocate(range).unwrap();
+				}
+
+				let range =
+					PageRange::from_start_len(stacks.phys_addr.as_usize(), stacks.total_size)
+						.unwrap();
+				unsafe {
+					PHYSICAL_FREE_LIST.lock().deallocate(range).unwrap();
+				}
 			}
 		}
 	}

src/arch/riscv64/mm/paging.rs

@@ -2,13 +2,14 @@ use core::marker::PhantomData;
 use core::ptr;
 
 use align_address::Align;
+use free_list::PageLayout;
 use hermit_sync::SpinMutex;
 use memory_addresses::{AddrRange, PhysAddr, VirtAddr};
 use riscv::asm::sfence_vma;
 use riscv::register::satp;
 use riscv::register::satp::Satp;
 
-use crate::mm::physicalmem;
+use crate::mm::physicalmem::PHYSICAL_FREE_LIST;
 
 static ROOT_PAGETABLE: SpinMutex<PageTable<L2Table>> = SpinMutex::new(PageTable::new());
 
@@ -433,7 +434,9 @@ where
 			// Does the table exist yet?
 			if !self.entries[index].is_present() {
 				// Allocate a single 4 KiB page for the new entry and mark it as a valid, writable subtable.
-				let new_entry = physicalmem::allocate(BasePageSize::SIZE as usize).unwrap();
+				let frame_layout = PageLayout::from_size(BasePageSize::SIZE as usize).unwrap();
+				let frame_range = PHYSICAL_FREE_LIST.lock().allocate(frame_layout).unwrap();
+				let new_entry = PhysAddr::from(frame_range.start());
 				self.entries[index].set(new_entry, PageTableEntryFlags::BLANK);
 
 				// trace!("new_entry {:#X}", new_entry);
@@ -610,8 +613,12 @@ pub fn map_heap<S: PageSize>(virt_addr: VirtAddr, count: usize) -> Result<(), us
 	let virt_addrs = (0..count as u64).map(|n| virt_addr + n * S::SIZE);
 
 	for (map_counter, virt_addr) in virt_addrs.enumerate() {
-		let phys_addr = physicalmem::allocate_aligned(S::SIZE as usize, S::SIZE as usize)
+		let layout = PageLayout::from_size_align(S::SIZE as usize, S::SIZE as usize).unwrap();
+		let frame_range = PHYSICAL_FREE_LIST
+			.lock()
+			.allocate(layout)
 			.map_err(|_| map_counter)?;
+		let phys_addr = PhysAddr::from(frame_range.start());
 		map::<S>(virt_addr, phys_addr, 1, flags);
 	}