diff --git a/crates/adapter-riscv/src/boot_fs.rs b/crates/adapter-riscv/src/boot_fs.rs index 0519898..d2345cf 100644 --- a/crates/adapter-riscv/src/boot_fs.rs +++ b/crates/adapter-riscv/src/boot_fs.rs @@ -67,7 +67,6 @@ pub fn read_file_from_disk( ) -> Option { let mut sector_buf = [0u8; SECTOR_SIZE]; - // Read boot sector if !blk.read_sector(0, &mut sector_buf) { return None; } @@ -75,7 +74,6 @@ pub fn read_file_from_disk( let fat_name = to_fat_name(filename); - // Search root directory let mut first_cluster: u16 = 0; let mut file_size: u32 = 0; let mut found = false; @@ -132,7 +130,6 @@ pub fn read_file_from_disk( read += chunk; } - // Read next cluster from FAT let fat_offset = (cluster as u32) * 2; let fat_sector = fs.reserved_sectors as u32 + fat_offset / SECTOR_SIZE as u32; let fat_byte = (fat_offset % SECTOR_SIZE as u32) as usize; diff --git a/crates/adapter-riscv/src/devices/mod.rs b/crates/adapter-riscv/src/devices/mod.rs index f8ec8ad..00d6863 100644 --- a/crates/adapter-riscv/src/devices/mod.rs +++ b/crates/adapter-riscv/src/devices/mod.rs @@ -1,8 +1,4 @@ -// What boot still sets up for userspace, all through the kernel's mapping -// API: the display's page quota and the BootInfo page that tells init -// where its device capabilities are. Drivers set up their own devices -// with the capabilities init hands them; processes share memory through -// regions they create. +// Boot-time userspace setup: the display's page quota and init's BootInfo page. use crate::hw::uart::Uart; use types::address::VirtualAddress; diff --git a/crates/adapter-riscv/src/entry.rs b/crates/adapter-riscv/src/entry.rs new file mode 100644 index 0000000..4933347 --- /dev/null +++ b/crates/adapter-riscv/src/entry.rs @@ -0,0 +1,73 @@ +use core::arch::global_asm; + +// Entered with the MMU off at the physical address; the kernel is linked high. +// PC-relative code only: enable `boot_page_table` (RAM mapped low for the next +// instruction and high, MMIO high), then jump high and take the hart's stack. +global_asm!( + r#" +.section .text.entry +.globl _start +_start: + lla t2, start_high_address + j 1f +.globl _secondary_start +_secondary_start: + lla t2, secondary_high_address +1: + li t0, 8 # smp::MAX_HARTS + bgeu a0, t0, spin + mv s0, a0 + lla t0, boot_page_table + srli t0, t0, 12 + li t1, 8 << 60 + or t0, t0, t1 + csrw satp, t0 + sfence.vma + ld t0, 0(t2) + jr t0 + +# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT +start_high: + call hart_stack + mv a0, s0 + call kmain + j spin +secondary_high: + call hart_stack + mv a0, s0 + call kmain_secondary + j spin +hart_stack: + lla sp, _stacks_start + addi t0, s0, 1 + li t1, 0x81000 + mul t0, t0, t1 + add sp, sp, t0 + ret +spin: + wfi + j spin + +.pushsection .rodata +.balign 8 +start_high_address: + .quad start_high +secondary_high_address: + .quad secondary_high +.popsection + +# Sv39 root of 1 GiB leaves; flags V|R|W|A|D = 0xc7, plus X = 0xcf. +.pushsection .data +.balign 4096 +boot_page_table: + .quad 0 + .quad 0 + .quad 0x200000cf # [2] 0x8000_0000 -> RAM, for the jump only + .fill 253, 8, 0 + .quad 0x000000c7 # [256] PHYS_OFFSET + 0 -> MMIO + .quad 0 + .quad 0x200000cf # [258] PHYS_OFFSET + 0x8000_0000 -> RAM + .fill 253, 8, 0 +.popsection +"# +); diff --git a/crates/adapter-riscv/src/kernel_cell.rs b/crates/adapter-riscv/src/kernel_cell.rs index 8884e23..bdeadc6 100644 --- a/crates/adapter-riscv/src/kernel_cell.rs +++ b/crates/adapter-riscv/src/kernel_cell.rs @@ -7,7 +7,7 @@ pub struct KernelCell { inner: UnsafeCell>, } -// Single-core, interrupts disabled during kernel access. +// Access is serialized by the kernel lock. unsafe impl Sync for KernelCell {} impl KernelCell { @@ -18,8 +18,7 @@ impl KernelCell { } } - /// Initializes the cell by letting `build` write the value straight - /// into its storage, so a large `T` never lives on the stack. + /// Initializes in place, so a large `T` never lives on the stack. /// /// # Safety /// `build` must fully initialize the `T` behind the pointer. @@ -28,9 +27,7 @@ impl KernelCell { self.initialized.store(true, Ordering::Release); } - /// Returns a raw pointer to the kernel. Callers must ensure - /// no aliasing `&mut` references exist. Safe on single-core - /// with interrupts disabled during kernel calls. + /// Raw pointer to the kernel; callers must avoid aliasing `&mut`. pub fn as_ptr(&self) -> *mut T { debug_assert!(self.initialized.load(Ordering::Acquire)); unsafe { (*self.inner.get()).as_mut_ptr() } diff --git a/crates/adapter-riscv/src/main.rs b/crates/adapter-riscv/src/main.rs index ec2187a..89a932e 100644 --- a/crates/adapter-riscv/src/main.rs +++ b/crates/adapter-riscv/src/main.rs @@ -5,6 +5,7 @@ mod board; mod boot; mod boot_fs; mod devices; +mod entry; mod hw; mod kernel_cell; mod mmu; @@ -12,7 +13,6 @@ mod platform; mod smp; mod trap; -use core::arch::global_asm; use hw::uart::Uart; use kernel_cell::KernelCell; @@ -28,92 +28,13 @@ pub fn kernel_ptr() -> *mut kernel::Kernel { KERNEL.as_ptr() } -// Firmware enters `_start` (and SBI starts other harts at -// `_secondary_start`) at the physical load address with the MMU off and -// the hart id in a0, but the kernel is linked in the upper half. Using -// only PC-relative code, the stub turns on `boot_page_table`, which maps -// RAM twice (at its physical address, just so the next instruction still -// exists, and in the upper half) plus the MMIO window high, then jumps to -// its own high address and takes the hart's kernel stack. The kernel then -// switches every hart to the real table, which maps nothing low. -global_asm!( - r#" -.section .text.entry -.globl _start -_start: - lla t2, start_high_address - j 1f -.globl _secondary_start -_secondary_start: - lla t2, secondary_high_address -1: - li t0, 8 # smp::MAX_HARTS - bgeu a0, t0, spin - mv s0, a0 - lla t0, boot_page_table - srli t0, t0, 12 - li t1, 8 << 60 - or t0, t0, t1 - csrw satp, t0 - sfence.vma - ld t0, 0(t2) - jr t0 - -# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT -start_high: - call hart_stack - mv a0, s0 - call kmain - j spin -secondary_high: - call hart_stack - mv a0, s0 - call kmain_secondary - j spin -hart_stack: - lla sp, _stacks_start - addi t0, s0, 1 - li t1, 0x81000 - mul t0, t0, t1 - add sp, sp, t0 - ret -spin: - wfi - j spin - -.pushsection .rodata -.balign 8 -start_high_address: - .quad start_high -secondary_high_address: - .quad secondary_high -.popsection - -# Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with -# flags V|R|W|A|D = 0xc7, plus X = 0xcf. -.pushsection .data -.balign 4096 -boot_page_table: - .quad 0 - .quad 0 - .quad 0x200000cf # [2] 0x8000_0000 -> RAM, for the jump only - .fill 253, 8, 0 - .quad 0x000000c7 # [256] PHYS_OFFSET + 0 -> MMIO - .quad 0 - .quad 0x200000cf # [258] PHYS_OFFSET + 0x8000_0000 -> RAM - .fill 253, 8, 0 -.popsection -"# -); - #[unsafe(no_mangle)] extern "C" fn kmain(hart: usize) -> ! { let uart = Uart::new(); uart.write_str("k-os: booted on RISC-V\n"); boot::hardware::init(&uart, hart); - // Held until the boot hart first enters userspace; harts started - // below wait for it. Not before init: zeroing .bss resets the lock. + // Not before init: zeroing .bss resets the lock. smp::lock(); let mut blk = boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device")); @@ -129,7 +50,7 @@ extern "C" fn kmain(hart: usize) -> ! { .unwrap_or_else(|_| halt(&uart, "board registration failed")); devices::display_quota(&uart, display.pid); boot::caps::grant_all(&uart, init.pid, storage.pid, display.pid, input.pid); - // After grant_all, so the slots init already relies on stay put. + // After grant_all: init already relies on those slots. let mut placed = [None; board::BOARD_DEVICES]; let system_control = kern_mut() .grant_boot_authority(init.pid, &board, &mut placed) diff --git a/crates/adapter-riscv/src/mmu/allocator.rs b/crates/adapter-riscv/src/mmu/allocator.rs index 024e401..e91e801 100644 --- a/crates/adapter-riscv/src/mmu/allocator.rs +++ b/crates/adapter-riscv/src/mmu/allocator.rs @@ -1,9 +1,7 @@ use super::page_table::{PAGE_SIZE, PageTable}; use crate::board; -// Boot-time bump allocator for kernel-only memory: the kernel page table -// and boot scratch buffers. User memory and user page tables come from the -// kernel's frame table instead. It must never reach the user pool. +// Boot bump allocator for kernel-only memory; must never reach the user pool. static mut NEXT_FREE: usize = 0; pub fn init_allocator(start: usize) { diff --git a/crates/adapter-riscv/src/mmu/mod.rs b/crates/adapter-riscv/src/mmu/mod.rs index 8c85eb4..c743e5b 100644 --- a/crates/adapter-riscv/src/mmu/mod.rs +++ b/crates/adapter-riscv/src/mmu/mod.rs @@ -7,10 +7,8 @@ pub use page_table::{ create_user_root, init_kernel_page_table, map_user_page, translate_user, unmap_user_page, }; -/// The kernel lives in the upper half of the address space: all of RAM and -/// the MMIO windows appear at their physical address plus this offset (the -/// start of Sv39's upper half), and the kernel image is linked there too, -/// so one offset converts any kernel address. Must match linker.ld. +/// Direct-map offset: RAM, MMIO and the kernel image sit at physical + this +/// (start of Sv39's upper half). Must match linker.ld. pub const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000; /// Where the kernel sees physical address `pa`. @@ -18,8 +16,7 @@ pub const fn phys_to_virt(pa: usize) -> usize { pa + PHYS_OFFSET } -/// The physical address behind kernel address `va` (a static, the boot -/// heap, a page table). Only valid for addresses in the direct map. +/// The physical address behind a direct-map kernel address `va`. pub const fn virt_to_phys(va: usize) -> usize { va - PHYS_OFFSET } diff --git a/crates/adapter-riscv/src/mmu/page_table.rs b/crates/adapter-riscv/src/mmu/page_table/kernel.rs similarity index 50% rename from crates/adapter-riscv/src/mmu/page_table.rs rename to crates/adapter-riscv/src/mmu/page_table/kernel.rs index da4fd52..a0f7fee 100644 --- a/crates/adapter-riscv/src/mmu/page_table.rs +++ b/crates/adapter-riscv/src/mmu/page_table/kernel.rs @@ -1,139 +1,12 @@ -use super::allocator::alloc_page_table; -use super::{phys_to_virt, virt_to_phys}; +use super::pte::{ + Mapping, PAGE_SIZE, PTE_W, PTE_X, PageTable, PageTableEntry, Sv39Flags, map, table_at, + table_ppn, +}; use crate::board; +use crate::mmu::allocator::alloc_page_table; +use crate::mmu::{phys_to_virt, virt_to_phys}; use crate::smp; -use platform::{FrameSource, MapError}; use types::address::{PhysicalAddress, VirtualAddress}; -use types::mapping::PagePerms; - -pub use types::mapping::Mapping; -pub const PAGE_SIZE: usize = 4096; - -const PTE_V: u64 = 1 << 0; -const PTE_R: u64 = 1 << 1; -const PTE_W: u64 = 1 << 2; -const PTE_X: u64 = 1 << 3; -const PTE_U: u64 = 1 << 4; -const PTE_G: u64 = 1 << 5; -const PTE_A: u64 = 1 << 6; -const PTE_D: u64 = 1 << 7; - -#[derive(Clone, Copy)] -pub struct Sv39Flags { - bits: u64, -} - -impl Sv39Flags { - pub fn kernel_rx() -> Self { - Self { - bits: PTE_R | PTE_X | PTE_G, - } - } - - pub fn kernel_ro() -> Self { - Self { - bits: PTE_R | PTE_G, - } - } - - pub fn kernel_rw() -> Self { - Self { - bits: PTE_R | PTE_W | PTE_G, - } - } - - pub fn user(perms: PagePerms) -> Self { - let bits = match perms { - PagePerms::ReadOnly => PTE_R | PTE_U, - PagePerms::ReadWrite => PTE_R | PTE_W | PTE_U, - PagePerms::ReadExecute => PTE_R | PTE_X | PTE_U, - }; - Self { bits } - } - - fn to_pte_flags(self) -> u64 { - PTE_V | PTE_A | PTE_D | self.bits - } -} - -#[derive(Clone, Copy)] -#[repr(transparent)] -pub struct PageTableEntry(u64); - -impl PageTableEntry { - pub fn empty() -> Self { - Self(0) - } - - fn new_leaf(ppn: u64, flags: Sv39Flags) -> Self { - Self(flags.to_pte_flags() | (ppn << 10)) - } - - fn new_branch(ppn: u64) -> Self { - Self(PTE_V | (ppn << 10)) - } - - pub fn is_valid(&self) -> bool { - self.0 & PTE_V != 0 - } - - pub fn ppn(&self) -> u64 { - (self.0 >> 10) & 0xFFF_FFFF_FFFF - } -} - -#[repr(C, align(4096))] -pub struct PageTable { - pub entries: [PageTableEntry; 512], -} - -impl PageTable { - pub fn zero(&mut self) { - for entry in &mut self.entries { - *entry = PageTableEntry::empty(); - } - } -} - -fn vpn(va: usize, level: usize) -> usize { - (va >> (12 + 9 * level)) & 0x1FF -} - -fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) { - let vpn2 = vpn(va, 2); - let vpn1 = vpn(va, 1); - let vpn0 = vpn(va, 0); - - if !root.entries[vpn2].is_valid() { - root.entries[vpn2] = PageTableEntry::new_branch(table_ppn(alloc_page_table())); - } - let l1 = table_at((root.entries[vpn2].ppn() << 12) as usize); - - if !l1.entries[vpn1].is_valid() { - l1.entries[vpn1] = PageTableEntry::new_branch(table_ppn(alloc_page_table())); - } - let l0 = table_at((l1.entries[vpn1].ppn() << 12) as usize); - - let ppn = (pa as u64) >> 12; - l0.entries[vpn0] = PageTableEntry::new_leaf(ppn, flags); -} - -/// The physical page number of a page table the kernel holds a reference to. -fn table_ppn(table: &PageTable) -> u64 { - (virt_to_phys(table as *const PageTable as usize) as u64) >> 12 -} - -pub fn map(root: &mut PageTable, mapping: Mapping) { - for i in 0..mapping.pages { - let offset = i * PAGE_SIZE; - map_page( - root, - mapping.va.raw() + offset, - mapping.pa.raw() + offset, - mapping.flags, - ); - } -} pub fn init_kernel_page_table() -> &'static mut PageTable { let root = alloc_page_table(); @@ -325,111 +198,6 @@ pub fn activate(root: &PageTable) { } } -// ---- user address spaces ---- -// -// User mappings live in [USER_VA_START, USER_VA_END), which is root slot 0, -// the L1 slots below USER_VA_END. Every address space gets its own root and its -// own L1 for slot 0, and the kernel maps nothing in the lower half, so all -// page tables below the user window are private to one process. The -// kernel's upper-half entries are shared by copying them. - -pub const USER_VA_START: usize = 0x1000; -pub const USER_VA_END: usize = 0x0200_0000; - -/// The page table at physical address `pa`, through the direct map. -pub fn table_at(pa: usize) -> &'static mut PageTable { - // SAFETY: every page table lives in RAM, which the kernel maps at - // `phys_to_virt`. - unsafe { &mut *(phys_to_virt(pa) as *mut PageTable) } -} - -fn fresh_table(frames: &mut dyn FrameSource) -> Result { - let pa = frames - .alloc_table_frame() - .ok_or(MapError::OutOfFrames)? - .raw(); - table_at(pa).zero(); - Ok(pa) -} - -fn in_user_window(va: usize) -> bool { - va.is_multiple_of(PAGE_SIZE) && (USER_VA_START..USER_VA_END).contains(&va) -} - -/// Builds a root table sharing the kernel's mappings, with a private L1 for -/// the user window. Returns the root's physical address. -pub fn create_user_root( - kernel_root: &PageTable, - frames: &mut dyn FrameSource, -) -> Result { - let root_pa = fresh_table(frames)?; - let l1_pa = fresh_table(frames)?; - let root = table_at(root_pa); - - for (mine, kernels) in root.entries.iter_mut().zip(&kernel_root.entries).skip(256) { - *mine = *kernels; - } - root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12); - Ok(root_pa) -} - -fn user_leaf_table( - root_pa: usize, - va: usize, - frames: Option<&mut dyn FrameSource>, -) -> Result<&'static mut PageTable, MapError> { - let root = table_at(root_pa); - let l1 = table_at((root.entries[0].ppn() << 12) as usize); - let slot = &mut l1.entries[vpn(va, 1)]; - if !slot.is_valid() { - let frames = frames.ok_or(MapError::NoAddressSpace)?; - let l0_pa = fresh_table(frames)?; - *slot = PageTableEntry::new_branch((l0_pa as u64) >> 12); - } - Ok(table_at((slot.ppn() << 12) as usize)) -} - -pub fn map_user_page( - root_pa: usize, - va: usize, - pa: usize, - flags: Sv39Flags, - frames: &mut dyn FrameSource, -) -> Result<(), MapError> { - if !in_user_window(va) || !pa.is_multiple_of(PAGE_SIZE) { - return Err(MapError::InvalidAddress); - } - let l0 = user_leaf_table(root_pa, va, Some(frames))?; - l0.entries[vpn(va, 0)] = PageTableEntry::new_leaf((pa as u64) >> 12, flags); - flush_tlb(va); - Ok(()) -} - -/// The physical page `va` maps to in the address space at `root_pa`. -pub fn translate_user(root_pa: usize, va: usize) -> Option { - if !in_user_window(va) { - return None; - } - let l0 = user_leaf_table(root_pa, va, None).ok()?; - let entry = l0.entries[vpn(va, 0)]; - entry.is_valid().then(|| (entry.ppn() << 12) as usize) -} - -pub fn unmap_user_page(root_pa: usize, va: usize) { - if !in_user_window(va) { - return; - } - if let Ok(l0) = user_leaf_table(root_pa, va, None) { - l0.entries[vpn(va, 0)] = PageTableEntry::empty(); - flush_tlb(va); - } -} - -fn flush_tlb(va: usize) { - // SAFETY: sfence.vma only invalidates translation caches. - unsafe { core::arch::asm!("sfence.vma {0}, zero", in(reg) va) }; -} - /// Physical address of the root table the hart is translating with. pub fn active_root() -> usize { let satp: usize; diff --git a/crates/adapter-riscv/src/mmu/page_table/mod.rs b/crates/adapter-riscv/src/mmu/page_table/mod.rs new file mode 100644 index 0000000..cfbf0ad --- /dev/null +++ b/crates/adapter-riscv/src/mmu/page_table/mod.rs @@ -0,0 +1,11 @@ +//! Sv39 page tables: entries and mapping, the kernel table, user address spaces. + +mod kernel; +mod pte; +mod user; + +pub use kernel::{WxViolation, activate, active_root, check_wx, init_kernel_page_table}; +pub use pte::{PAGE_SIZE, PageTable, Sv39Flags, table_at}; +pub use user::{ + USER_VA_END, USER_VA_START, create_user_root, map_user_page, translate_user, unmap_user_page, +}; diff --git a/crates/adapter-riscv/src/mmu/page_table/pte.rs b/crates/adapter-riscv/src/mmu/page_table/pte.rs new file mode 100644 index 0000000..2f08832 --- /dev/null +++ b/crates/adapter-riscv/src/mmu/page_table/pte.rs @@ -0,0 +1,139 @@ +use crate::mmu::allocator::alloc_page_table; +use crate::mmu::{phys_to_virt, virt_to_phys}; +use types::mapping::PagePerms; + +pub use types::mapping::Mapping; +pub const PAGE_SIZE: usize = 4096; + +const PTE_V: u64 = 1 << 0; +const PTE_R: u64 = 1 << 1; +pub(super) const PTE_W: u64 = 1 << 2; +pub(super) const PTE_X: u64 = 1 << 3; +const PTE_U: u64 = 1 << 4; +const PTE_G: u64 = 1 << 5; +const PTE_A: u64 = 1 << 6; +const PTE_D: u64 = 1 << 7; + +#[derive(Clone, Copy)] +pub struct Sv39Flags { + bits: u64, +} + +impl Sv39Flags { + pub fn kernel_rx() -> Self { + Self { + bits: PTE_R | PTE_X | PTE_G, + } + } + + pub fn kernel_ro() -> Self { + Self { + bits: PTE_R | PTE_G, + } + } + + pub fn kernel_rw() -> Self { + Self { + bits: PTE_R | PTE_W | PTE_G, + } + } + + pub fn user(perms: PagePerms) -> Self { + let bits = match perms { + PagePerms::ReadOnly => PTE_R | PTE_U, + PagePerms::ReadWrite => PTE_R | PTE_W | PTE_U, + PagePerms::ReadExecute => PTE_R | PTE_X | PTE_U, + }; + Self { bits } + } + + fn to_pte_flags(self) -> u64 { + PTE_V | PTE_A | PTE_D | self.bits + } +} + +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct PageTableEntry(pub(super) u64); + +impl PageTableEntry { + pub fn empty() -> Self { + Self(0) + } + + pub(super) fn new_leaf(ppn: u64, flags: Sv39Flags) -> Self { + Self(flags.to_pte_flags() | (ppn << 10)) + } + + pub(super) fn new_branch(ppn: u64) -> Self { + Self(PTE_V | (ppn << 10)) + } + + pub fn is_valid(&self) -> bool { + self.0 & PTE_V != 0 + } + + pub fn ppn(&self) -> u64 { + (self.0 >> 10) & 0xFFF_FFFF_FFFF + } +} + +#[repr(C, align(4096))] +pub struct PageTable { + pub entries: [PageTableEntry; 512], +} + +impl PageTable { + pub fn zero(&mut self) { + for entry in &mut self.entries { + *entry = PageTableEntry::empty(); + } + } +} + +pub(super) fn vpn(va: usize, level: usize) -> usize { + (va >> (12 + 9 * level)) & 0x1FF +} + +fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) { + let vpn2 = vpn(va, 2); + let vpn1 = vpn(va, 1); + let vpn0 = vpn(va, 0); + + if !root.entries[vpn2].is_valid() { + root.entries[vpn2] = PageTableEntry::new_branch(table_ppn(alloc_page_table())); + } + let l1 = table_at((root.entries[vpn2].ppn() << 12) as usize); + + if !l1.entries[vpn1].is_valid() { + l1.entries[vpn1] = PageTableEntry::new_branch(table_ppn(alloc_page_table())); + } + let l0 = table_at((l1.entries[vpn1].ppn() << 12) as usize); + + let ppn = (pa as u64) >> 12; + l0.entries[vpn0] = PageTableEntry::new_leaf(ppn, flags); +} + +/// The physical page number of a page table the kernel holds a reference to. +pub(super) fn table_ppn(table: &PageTable) -> u64 { + (virt_to_phys(table as *const PageTable as usize) as u64) >> 12 +} + +pub fn map(root: &mut PageTable, mapping: Mapping) { + for i in 0..mapping.pages { + let offset = i * PAGE_SIZE; + map_page( + root, + mapping.va.raw() + offset, + mapping.pa.raw() + offset, + mapping.flags, + ); + } +} + +/// The page table at physical address `pa`, through the direct map. +pub fn table_at(pa: usize) -> &'static mut PageTable { + // SAFETY: every page table lives in RAM, which the kernel maps at + // `phys_to_virt`. + unsafe { &mut *(phys_to_virt(pa) as *mut PageTable) } +} diff --git a/crates/adapter-riscv/src/mmu/page_table/user.rs b/crates/adapter-riscv/src/mmu/page_table/user.rs new file mode 100644 index 0000000..4c742f4 --- /dev/null +++ b/crates/adapter-riscv/src/mmu/page_table/user.rs @@ -0,0 +1,93 @@ +use super::pte::{PAGE_SIZE, PageTable, PageTableEntry, Sv39Flags, table_at, vpn}; +use platform::{FrameSource, MapError}; + +/// User mappings live in root slot 0, in a private L1 per address space. +pub const USER_VA_START: usize = 0x1000; +pub const USER_VA_END: usize = 0x0200_0000; + +fn fresh_table(frames: &mut dyn FrameSource) -> Result { + let pa = frames + .alloc_table_frame() + .ok_or(MapError::OutOfFrames)? + .raw(); + table_at(pa).zero(); + Ok(pa) +} + +fn in_user_window(va: usize) -> bool { + va.is_multiple_of(PAGE_SIZE) && (USER_VA_START..USER_VA_END).contains(&va) +} + +/// Builds a root table sharing the kernel's mappings, with a private L1 for +/// the user window. Returns the root's physical address. +pub fn create_user_root( + kernel_root: &PageTable, + frames: &mut dyn FrameSource, +) -> Result { + let root_pa = fresh_table(frames)?; + let l1_pa = fresh_table(frames)?; + let root = table_at(root_pa); + + for (mine, kernels) in root.entries.iter_mut().zip(&kernel_root.entries).skip(256) { + *mine = *kernels; + } + root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12); + Ok(root_pa) +} + +fn user_leaf_table( + root_pa: usize, + va: usize, + frames: Option<&mut dyn FrameSource>, +) -> Result<&'static mut PageTable, MapError> { + let root = table_at(root_pa); + let l1 = table_at((root.entries[0].ppn() << 12) as usize); + let slot = &mut l1.entries[vpn(va, 1)]; + if !slot.is_valid() { + let frames = frames.ok_or(MapError::NoAddressSpace)?; + let l0_pa = fresh_table(frames)?; + *slot = PageTableEntry::new_branch((l0_pa as u64) >> 12); + } + Ok(table_at((slot.ppn() << 12) as usize)) +} + +pub fn map_user_page( + root_pa: usize, + va: usize, + pa: usize, + flags: Sv39Flags, + frames: &mut dyn FrameSource, +) -> Result<(), MapError> { + if !in_user_window(va) || !pa.is_multiple_of(PAGE_SIZE) { + return Err(MapError::InvalidAddress); + } + let l0 = user_leaf_table(root_pa, va, Some(frames))?; + l0.entries[vpn(va, 0)] = PageTableEntry::new_leaf((pa as u64) >> 12, flags); + flush_tlb(va); + Ok(()) +} + +/// The physical page `va` maps to in the address space at `root_pa`. +pub fn translate_user(root_pa: usize, va: usize) -> Option { + if !in_user_window(va) { + return None; + } + let l0 = user_leaf_table(root_pa, va, None).ok()?; + let entry = l0.entries[vpn(va, 0)]; + entry.is_valid().then(|| (entry.ppn() << 12) as usize) +} + +pub fn unmap_user_page(root_pa: usize, va: usize) { + if !in_user_window(va) { + return; + } + if let Ok(l0) = user_leaf_table(root_pa, va, None) { + l0.entries[vpn(va, 0)] = PageTableEntry::empty(); + flush_tlb(va); + } +} + +fn flush_tlb(va: usize) { + // SAFETY: sfence.vma only invalidates translation caches. + unsafe { core::arch::asm!("sfence.vma {0}, zero", in(reg) va) }; +} diff --git a/crates/adapter-riscv/src/platform.rs b/crates/adapter-riscv/src/platform.rs index 976a47b..c2a8bfb 100644 --- a/crates/adapter-riscv/src/platform.rs +++ b/crates/adapter-riscv/src/platform.rs @@ -64,9 +64,8 @@ impl Platform for RiscVPlatform { let Some(root) = self.root(process) else { return; }; - // No hart may still translate through it once the kernel frees its - // frames: other harts leave it (and wait for nothing but that), and - // the caller may be tearing down the space we are running on. + // No hart may translate through it once its frames are freed; that + // includes this one, which may be running on it. smp::shoot_down(root); if mmu::active_root() == root { // SAFETY: the kernel page table lives in the boot heap forever. diff --git a/crates/adapter-riscv/src/smp.rs b/crates/adapter-riscv/src/smp.rs deleted file mode 100644 index 068a1c9..0000000 --- a/crates/adapter-riscv/src/smp.rs +++ /dev/null @@ -1,275 +0,0 @@ -//! Harts. Each hart has its own kernel stack, and its hart id doubles as -//! its kernel `CoreId`. The kernel is entered under one lock for now (stage -//! 1 of multi-core; the kernel itself stays a sequential program), so -//! userspace runs in parallel and the kernel does not. -//! -//! Harts talk through IPIs carrying two kinds of request: -//! - reschedule: the kernel made work for this hart, or took its thread -//! away; handled under the lock on the way out of the kernel. -//! - leave root R: a page table this hart may be using is changing or about -//! to be freed; the hart switches to the kernel's root and flushes its -//! TLB. This needs no lock, so it is serviced even while a hart spins -//! waiting for the lock or idles, and the sender (which holds the lock) -//! can wait for the acknowledgement without deadlocking. - -use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; - -use crate::mmu; - -/// Must match linker.ld. -pub const MAX_HARTS: usize = 8; -/// A guard page and a 512 KiB stack (boot builds big tables on it). Must -/// match linker.ld. -pub const STACK_SLOT: usize = 4096 + 0x8_0000; - -const SIP_SSIP: usize = 1 << 1; - -/// The hart running this code, from which kernel stack it is on. -pub fn hart_id() -> usize { - let sp: usize; - // SAFETY: reading sp has no side effects. - unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) }; - (sp - stacks_start()) / STACK_SLOT -} - -pub fn stacks_start() -> usize { - unsafe extern "C" { - static _stacks_start: u8; - } - // SAFETY: only the symbol's address is taken. - unsafe { &_stacks_start as *const u8 as usize } -} - -static BOOT_HART: AtomicUsize = AtomicUsize::new(0); -static NUM_HARTS: AtomicUsize = AtomicUsize::new(1); -/// Physical address of the kernel's own root table. -static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0); - -/// The root table each hart translates with. -static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; -static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS]; -static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; -static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; - -pub fn boot_hart() -> usize { - BOOT_HART.load(Ordering::Relaxed) -} - -pub fn num_harts() -> usize { - NUM_HARTS.load(Ordering::Relaxed) -} - -pub fn kernel_root() -> usize { - KERNEL_ROOT.load(Ordering::Relaxed) -} - -/// Records which hart booted and how many there are, and where the -/// kernel's root table is. -pub fn init(boot: usize, harts: usize, kernel_root: usize) { - BOOT_HART.store(boot, Ordering::Relaxed); - NUM_HARTS.store(harts, Ordering::Relaxed); - KERNEL_ROOT.store(kernel_root, Ordering::Relaxed); -} - -/// Called by `mmu::activate`. -pub fn note_active_root(root: usize) { - if let Some(r) = ACTIVE_ROOT.get(hart_id()) { - r.store(root, Ordering::SeqCst); - } -} - -// ---- the kernel lock ---- - -static NEXT_TICKET: AtomicU32 = AtomicU32::new(0); -static SERVING: AtomicU32 = AtomicU32::new(0); - -/// Takes the kernel lock, first come first served. Leave-root requests -/// are serviced while waiting. -pub fn lock() { - let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed); - while SERVING.load(Ordering::Acquire) != ticket { - service_ipi(); - core::hint::spin_loop(); - } -} - -pub fn unlock() { - SERVING.fetch_add(1, Ordering::Release); -} - -// ---- what lock-free fast paths may know ---- -// -// Published under the kernel lock just before it is released, read without -// it. A fast path that reads them while another hart holds the lock simply -// takes effect before that hart's operation. - -/// Per core: highest waiting priority + 1, 0 if its queue is empty. -static WAITING: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; -/// Per hart: the running thread's process id + 1 and priority + 1. -static RUNNING_PID: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; -static RUNNING_PRIORITY: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; - -pub fn publish_waiting(core: usize, highest: Option) { - if let Some(w) = WAITING.get(core) { - w.store(highest.map_or(0, |p| p as usize + 1), Ordering::Release); - } -} - -pub fn publish_running(pid: Option, priority: Option) { - let me = hart_id(); - if let (Some(p), Some(q)) = (RUNNING_PID.get(me), RUNNING_PRIORITY.get(me)) { - p.store(pid.map_or(0, |x| x as usize + 1), Ordering::Release); - q.store(priority.map_or(0, |x| x as usize + 1), Ordering::Release); - } -} - -/// The process the thread running on this hart belongs to. -pub fn running_pid() -> Option { - let v = RUNNING_PID.get(hart_id())?.load(Ordering::Acquire); - v.checked_sub(1) -} - -/// Whether a Yield by the running thread would change nothing: no thread -/// waits on this core at its priority or above. -pub fn yield_is_noop() -> bool { - let me = hart_id(); - let (Some(w), Some(p)) = (WAITING.get(me), RUNNING_PRIORITY.get(me)) else { - return false; - }; - let mine = p.load(Ordering::Acquire); - mine != 0 && w.load(Ordering::Acquire) < mine -} - -/// Whether this hart has an IPI or a reschedule request it has not acted -/// on; fast paths must then take the slow path. -pub fn has_pending_request() -> bool { - let sip: usize; - // SAFETY: reading sip has no side effects. - unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; - sip & SIP_SSIP != 0 - || RESCHEDULE - .get(hart_id()) - .is_some_and(|r| r.load(Ordering::SeqCst)) -} - -// ---- IPIs ---- - -/// The lock-free half of an IPI: acknowledges it and, if asked, leaves the -/// root being changed. Safe to call at any time, any number of times. -pub fn service_ipi() { - let sip: usize; - // SAFETY: reading sip has no side effects. - unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; - if sip & SIP_SSIP == 0 { - return; - } - // Clear first: a request posted after this raises SSIP again. - // SAFETY: clearing our own pending software interrupt. - unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) }; - let me = hart_id(); - let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else { - return; - }; - let root = leave.swap(0, Ordering::SeqCst); - if root != 0 { - if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) { - mmu::activate(mmu::page_table::table_at(kernel_root())); - } - // SAFETY: flushing translations has no other effect. - unsafe { core::arch::asm!("sfence.vma zero, zero") }; - left.fetch_add(1, Ordering::SeqCst); - } -} - -/// Asks `hart` to look at its run queue again, on its way out of the -/// kernel. -pub fn kick(hart: usize) { - if let Some(r) = RESCHEDULE.get(hart) { - r.store(true, Ordering::SeqCst); - sbi_send_ipi(hart); - } -} - -/// Consumes this hart's reschedule request. -pub fn take_reschedule() -> bool { - RESCHEDULE - .get(hart_id()) - .is_some_and(|r| r.swap(false, Ordering::SeqCst)) -} - -/// Makes every other hart that translates through `root` leave it and -/// flush, and waits until each has. Called with the kernel lock held, so -/// no hart can start using `root` meanwhile. -pub fn shoot_down(root: usize) { - let me = hart_id(); - for hart in 0..num_harts() { - if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root { - continue; - } - let seen = LEFT[hart].load(Ordering::SeqCst); - LEAVE_ROOT[hart].store(root, Ordering::SeqCst); - sbi_send_ipi(hart); - while LEFT[hart].load(Ordering::SeqCst) == seen { - core::hint::spin_loop(); - } - } -} - -// ---- SBI ---- - -const SBI_EXT_IPI: usize = 0x735049; -const SBI_EXT_HSM: usize = 0x48534D; - -fn sbi_send_ipi(hart: usize) { - // SAFETY: an SBI call; a0 and a1 come back clobbered. - unsafe { - core::arch::asm!( - "ecall", - inlateout("a0") 1usize => _, - inlateout("a1") hart => _, - in("a6") 0usize, - in("a7") SBI_EXT_IPI, - ); - } -} - -/// Whether `hart` exists, per SBI HSM `hart_get_status`. -fn sbi_hart_exists(hart: usize) -> bool { - let error: isize; - // SAFETY: an SBI call; a0 and a1 come back clobbered. - unsafe { - core::arch::asm!( - "ecall", - inlateout("a0") hart => error, - lateout("a1") _, - in("a6") 2usize, - in("a7") SBI_EXT_HSM, - ); - } - error == 0 -} - -/// The number of harts, counting up from hart 0 until one is missing. -pub fn probe_harts() -> usize { - (0..MAX_HARTS) - .take_while(|h| sbi_hart_exists(*h)) - .count() - .max(1) -} - -/// Starts `hart` at the physical address `entry` with the MMU off. -pub fn sbi_hart_start(hart: usize, entry: usize) -> bool { - let error: isize; - // SAFETY: an SBI call; a0 and a1 come back clobbered. - unsafe { - core::arch::asm!( - "ecall", - inlateout("a0") hart => error, - inlateout("a1") entry => _, - in("a2") 0usize, - in("a6") 0usize, - in("a7") SBI_EXT_HSM, - ); - } - error == 0 -} diff --git a/crates/adapter-riscv/src/smp/hints.rs b/crates/adapter-riscv/src/smp/hints.rs new file mode 100644 index 0000000..5f4ebf0 --- /dev/null +++ b/crates/adapter-riscv/src/smp/hints.rs @@ -0,0 +1,42 @@ +//! State published under the kernel lock for lock-free fast paths to read. + +use core::sync::atomic::{AtomicUsize, Ordering}; + +use super::{MAX_HARTS, hart_id}; + +/// Per core: highest waiting priority + 1, 0 if its queue is empty. +static WAITING: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; +/// Per hart: the running thread's process id + 1 and priority + 1. +static RUNNING_PID: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; +static RUNNING_PRIORITY: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; + +pub fn publish_waiting(core: usize, highest: Option) { + if let Some(w) = WAITING.get(core) { + w.store(highest.map_or(0, |p| p as usize + 1), Ordering::Release); + } +} + +pub fn publish_running(pid: Option, priority: Option) { + let me = hart_id(); + if let (Some(p), Some(q)) = (RUNNING_PID.get(me), RUNNING_PRIORITY.get(me)) { + p.store(pid.map_or(0, |x| x as usize + 1), Ordering::Release); + q.store(priority.map_or(0, |x| x as usize + 1), Ordering::Release); + } +} + +/// The process the thread running on this hart belongs to. +pub fn running_pid() -> Option { + let v = RUNNING_PID.get(hart_id())?.load(Ordering::Acquire); + v.checked_sub(1) +} + +/// Whether a Yield by the running thread would change nothing: no thread +/// waits on this core at its priority or above. +pub fn yield_is_noop() -> bool { + let me = hart_id(); + let (Some(w), Some(p)) = (WAITING.get(me), RUNNING_PRIORITY.get(me)) else { + return false; + }; + let mine = p.load(Ordering::Acquire); + mine != 0 && w.load(Ordering::Acquire) < mine +} diff --git a/crates/adapter-riscv/src/smp/ipi.rs b/crates/adapter-riscv/src/smp/ipi.rs new file mode 100644 index 0000000..790ff3e --- /dev/null +++ b/crates/adapter-riscv/src/smp/ipi.rs @@ -0,0 +1,96 @@ +//! Reschedule and leave-root IPIs, serviced without the kernel lock. + +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use super::sbi::sbi_send_ipi; +use super::{MAX_HARTS, hart_id, kernel_root, num_harts}; +use crate::mmu; + +const SIP_SSIP: usize = 1 << 1; + +/// The root table each hart translates with. +static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; +static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS]; +static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; +static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; + +/// Called by `mmu::activate`. +pub fn note_active_root(root: usize) { + if let Some(r) = ACTIVE_ROOT.get(hart_id()) { + r.store(root, Ordering::SeqCst); + } +} + +/// Whether this hart has an IPI or a reschedule request it has not acted +/// on; fast paths must then take the slow path. +pub fn has_pending_request() -> bool { + let sip: usize; + // SAFETY: reading sip has no side effects. + unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; + sip & SIP_SSIP != 0 + || RESCHEDULE + .get(hart_id()) + .is_some_and(|r| r.load(Ordering::SeqCst)) +} + +// ---- IPIs ---- + +/// The lock-free half of an IPI: acknowledges it and leaves the root being +/// changed, if asked. Safe to call at any time. +pub fn service_ipi() { + let sip: usize; + // SAFETY: reading sip has no side effects. + unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; + if sip & SIP_SSIP == 0 { + return; + } + // Clear first: a request posted after this raises SSIP again. + // SAFETY: clearing our own pending software interrupt. + unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) }; + let me = hart_id(); + let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else { + return; + }; + let root = leave.swap(0, Ordering::SeqCst); + if root != 0 { + if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) { + mmu::activate(mmu::page_table::table_at(kernel_root())); + } + // SAFETY: flushing translations has no other effect. + unsafe { core::arch::asm!("sfence.vma zero, zero") }; + left.fetch_add(1, Ordering::SeqCst); + } +} + +/// Asks `hart` to look at its run queue again, on its way out of the +/// kernel. +pub fn kick(hart: usize) { + if let Some(r) = RESCHEDULE.get(hart) { + r.store(true, Ordering::SeqCst); + sbi_send_ipi(hart); + } +} + +/// Consumes this hart's reschedule request. +pub fn take_reschedule() -> bool { + RESCHEDULE + .get(hart_id()) + .is_some_and(|r| r.swap(false, Ordering::SeqCst)) +} + +/// Makes every other hart leave `root` and flush, waiting for each. +/// Kernel lock held, so no hart can start using `root` meanwhile. +pub fn shoot_down(root: usize) { + let me = hart_id(); + for hart in 0..num_harts() { + if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root { + continue; + } + let seen = LEFT[hart].load(Ordering::SeqCst); + LEAVE_ROOT[hart].store(root, Ordering::SeqCst); + sbi_send_ipi(hart); + while LEFT[hart].load(Ordering::SeqCst) == seen { + core::hint::spin_loop(); + } + } +} diff --git a/crates/adapter-riscv/src/smp/lock.rs b/crates/adapter-riscv/src/smp/lock.rs new file mode 100644 index 0000000..533aee5 --- /dev/null +++ b/crates/adapter-riscv/src/smp/lock.rs @@ -0,0 +1,22 @@ +//! The kernel lock: a ticket lock that services IPIs while waiting. + +use core::sync::atomic::{AtomicU32, Ordering}; + +use super::service_ipi; + +static NEXT_TICKET: AtomicU32 = AtomicU32::new(0); +static SERVING: AtomicU32 = AtomicU32::new(0); + +/// Takes the kernel lock, first come first served. Leave-root requests +/// are serviced while waiting. +pub fn lock() { + let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed); + while SERVING.load(Ordering::Acquire) != ticket { + service_ipi(); + core::hint::spin_loop(); + } +} + +pub fn unlock() { + SERVING.fetch_add(1, Ordering::Release); +} diff --git a/crates/adapter-riscv/src/smp/mod.rs b/crates/adapter-riscv/src/smp/mod.rs new file mode 100644 index 0000000..2ef05a9 --- /dev/null +++ b/crates/adapter-riscv/src/smp/mod.rs @@ -0,0 +1,61 @@ +//! Harts: per-hart kernel stacks (the hart id doubles as the kernel `CoreId`), +//! the kernel lock, and IPIs. The kernel runs under one lock; userspace runs in parallel. + +mod hints; +mod ipi; +mod lock; +mod sbi; + +pub use hints::{publish_running, publish_waiting, running_pid, yield_is_noop}; +pub use ipi::{ + has_pending_request, kick, note_active_root, service_ipi, shoot_down, take_reschedule, +}; +pub use lock::{lock, unlock}; +pub use sbi::{probe_harts, sbi_hart_start}; + +use core::sync::atomic::{AtomicUsize, Ordering}; + +/// Must match linker.ld. +pub const MAX_HARTS: usize = 8; +/// A guard page and a 512 KiB stack. Must match linker.ld. +pub const STACK_SLOT: usize = 4096 + 0x8_0000; + +/// The hart running this code, from which kernel stack it is on. +pub fn hart_id() -> usize { + let sp: usize; + // SAFETY: reading sp has no side effects. + unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) }; + (sp - stacks_start()) / STACK_SLOT +} + +pub fn stacks_start() -> usize { + unsafe extern "C" { + static _stacks_start: u8; + } + // SAFETY: only the symbol's address is taken. + unsafe { &_stacks_start as *const u8 as usize } +} + +static BOOT_HART: AtomicUsize = AtomicUsize::new(0); +static NUM_HARTS: AtomicUsize = AtomicUsize::new(1); +/// Physical address of the kernel's own root table. +static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0); + +pub fn boot_hart() -> usize { + BOOT_HART.load(Ordering::Relaxed) +} + +pub fn num_harts() -> usize { + NUM_HARTS.load(Ordering::Relaxed) +} + +pub fn kernel_root() -> usize { + KERNEL_ROOT.load(Ordering::Relaxed) +} + +/// Records the boot hart, the hart count and the kernel root table. +pub fn init(boot: usize, harts: usize, kernel_root: usize) { + BOOT_HART.store(boot, Ordering::Relaxed); + NUM_HARTS.store(harts, Ordering::Relaxed); + KERNEL_ROOT.store(kernel_root, Ordering::Relaxed); +} diff --git a/crates/adapter-riscv/src/smp/sbi.rs b/crates/adapter-riscv/src/smp/sbi.rs new file mode 100644 index 0000000..2518a32 --- /dev/null +++ b/crates/adapter-riscv/src/smp/sbi.rs @@ -0,0 +1,60 @@ +//! SBI calls: IPIs and hart state management. + +use super::MAX_HARTS; + +const SBI_EXT_IPI: usize = 0x735049; +const SBI_EXT_HSM: usize = 0x48534D; + +pub(super) fn sbi_send_ipi(hart: usize) { + // SAFETY: an SBI call; a0 and a1 come back clobbered. + unsafe { + core::arch::asm!( + "ecall", + inlateout("a0") 1usize => _, + inlateout("a1") hart => _, + in("a6") 0usize, + in("a7") SBI_EXT_IPI, + ); + } +} + +/// Whether `hart` exists, per SBI HSM `hart_get_status`. +fn sbi_hart_exists(hart: usize) -> bool { + let error: isize; + // SAFETY: an SBI call; a0 and a1 come back clobbered. + unsafe { + core::arch::asm!( + "ecall", + inlateout("a0") hart => error, + lateout("a1") _, + in("a6") 2usize, + in("a7") SBI_EXT_HSM, + ); + } + error == 0 +} + +/// The number of harts, counting up from hart 0 until one is missing. +pub fn probe_harts() -> usize { + (0..MAX_HARTS) + .take_while(|h| sbi_hart_exists(*h)) + .count() + .max(1) +} + +/// Starts `hart` at the physical address `entry` with the MMU off. +pub fn sbi_hart_start(hart: usize, entry: usize) -> bool { + let error: isize; + // SAFETY: an SBI call; a0 and a1 come back clobbered. + unsafe { + core::arch::asm!( + "ecall", + inlateout("a0") hart => error, + inlateout("a1") entry => _, + in("a2") 0usize, + in("a6") 0usize, + in("a7") SBI_EXT_HSM, + ); + } + error == 0 +} diff --git a/crates/adapter-riscv/src/trap/fast.rs b/crates/adapter-riscv/src/trap/fast.rs new file mode 100644 index 0000000..400ce7e --- /dev/null +++ b/crates/adapter-riscv/src/trap/fast.rs @@ -0,0 +1,26 @@ +use super::{TrapFrame, write_result}; + +/// Syscalls answered without the kernel lock because they touch nothing it +/// guards. A pending IPI or reschedule request forces the slow path. +pub(super) fn fast_syscall(frame: &mut TrapFrame) -> bool { + use types::syscall_abi::{SYS_DEBUG_PUTCHAR, SYS_GET_PROCESS_ID, SYS_GET_TIME, SYS_YIELD}; + if crate::smp::has_pending_request() { + return false; + } + let result = match frame.regs[17] { + SYS_GET_TIME => kernel::SyscallResult::Value(crate::hw::timer::read_time() as usize), + SYS_GET_PROCESS_ID => match crate::smp::running_pid() { + Some(pid) => kernel::SyscallResult::Value(pid), + None => return false, + }, + SYS_DEBUG_PUTCHAR => { + crate::hw::uart::Uart::new().put_char(frame.regs[10] as u8); + kernel::SyscallResult::Ok + } + SYS_YIELD if crate::smp::yield_is_noop() => kernel::SyscallResult::Ok, + _ => return false, + }; + frame.sepc += 4; + write_result(frame, &result); + true +} diff --git a/crates/adapter-riscv/src/trap/mod.rs b/crates/adapter-riscv/src/trap/mod.rs index 5753fae..57b3e6c 100644 --- a/crates/adapter-riscv/src/trap/mod.rs +++ b/crates/adapter-riscv/src/trap/mod.rs @@ -1,4 +1,10 @@ -use core::arch::global_asm; +mod fast; +mod sched; +mod vector; + +use fast::fast_syscall; +pub use sched::enter_first; +use sched::{finish, release, switch_to_next, switch_to_thread}; #[repr(C)] pub struct TrapFrame { @@ -7,126 +13,6 @@ pub struct TrapFrame { pub sstatus: usize, } -global_asm!( - r#" -.section .text -.align 4 -.globl trap_vector -trap_vector: - csrrw sp, sscratch, sp - bnez sp, _from_user - csrrw sp, sscratch, sp -_from_user: - addi sp, sp, -272 - - sd x1, 8(sp) - sd x3, 24(sp) - sd x4, 32(sp) - sd x5, 40(sp) - sd x6, 48(sp) - sd x7, 56(sp) - sd x8, 64(sp) - sd x9, 72(sp) - sd x10, 80(sp) - sd x11, 88(sp) - sd x12, 96(sp) - sd x13, 104(sp) - sd x14, 112(sp) - sd x15, 120(sp) - sd x16, 128(sp) - sd x17, 136(sp) - sd x18, 144(sp) - sd x19, 152(sp) - sd x20, 160(sp) - sd x21, 168(sp) - sd x22, 176(sp) - sd x23, 184(sp) - sd x24, 192(sp) - sd x25, 200(sp) - sd x26, 208(sp) - sd x27, 216(sp) - sd x28, 224(sp) - sd x29, 232(sp) - sd x30, 240(sp) - sd x31, 248(sp) - - csrr t0, sscratch - sd t0, 16(sp) - - csrr t0, sepc - sd t0, 256(sp) - csrr t0, sstatus - sd t0, 264(sp) - - mv a0, sp - call trap_handler_rust - j _restore_frame - -# trap_return(frame): resume whatever `frame` describes, as if returning -# from a trap that saved it. Used to enter userspace for the first time. -.globl trap_return -trap_return: - mv sp, a0 -_restore_frame: - ld t0, 256(sp) - csrw sepc, t0 - ld t0, 264(sp) - csrw sstatus, t0 - - andi t1, t0, 0x100 - bnez t1, _restore_smode - - ld t0, 16(sp) - csrw sscratch, t0 - -_restore_smode: - ld x1, 8(sp) - ld x3, 24(sp) - ld x4, 32(sp) - ld x6, 48(sp) - ld x7, 56(sp) - ld x8, 64(sp) - ld x9, 72(sp) - ld x10, 80(sp) - ld x11, 88(sp) - ld x12, 96(sp) - ld x13, 104(sp) - ld x14, 112(sp) - ld x15, 120(sp) - ld x16, 128(sp) - ld x17, 136(sp) - ld x18, 144(sp) - ld x19, 152(sp) - ld x20, 160(sp) - ld x21, 168(sp) - ld x22, 176(sp) - ld x23, 184(sp) - ld x24, 192(sp) - ld x25, 200(sp) - ld x26, 208(sp) - ld x27, 216(sp) - ld x28, 224(sp) - ld x29, 232(sp) - ld x30, 240(sp) - ld x31, 248(sp) - - # Every GPR except t0 (x5) is restored. Decide where we return to while - # t0 is still free, and restore it last on each path; using t0 after - # its restore would corrupt the interrupted program's t0. - csrr t0, sstatus - andi t0, t0, 0x100 - bnez t0, _sret_smode - ld x5, 40(sp) - addi sp, sp, 272 - csrrw sp, sscratch, sp - sret -_sret_smode: - ld x5, 40(sp) - addi sp, sp, 272 - sret -"# -); - const CAUSE_INTERRUPT_BIT: usize = 1 << 63; const CAUSE_S_SOFTWARE: usize = 1; const CAUSE_S_TIMER: usize = 5; @@ -143,7 +29,7 @@ pub fn init() { let addr = trap_vector as *const () as usize; core::arch::asm!("csrw stvec, {0}", in(reg) addr); core::arch::asm!("csrw sscratch, zero"); - // SIE stays clear — interrupts are enabled only when entering userspace via sret + // SIE stays clear; interrupts are enabled only by sret into userspace. let sie_bits = (1usize << 5) | (1usize << 9) | (1usize << 1); core::arch::asm!("csrs sie, {0}", in(reg) sie_bits); } @@ -169,31 +55,6 @@ fn write_result(frame: &mut TrapFrame, result: &kernel::SyscallResult) { set_results(frame, kernel::abi::encode(result)); } -fn save_frame_to_context(frame: &TrapFrame, ctx: &mut domain::thread::SavedContext) { - ctx.registers_mut().copy_from_slice(&frame.regs); - ctx.set_pc(frame.sepc); - ctx.set_status(frame.sstatus); -} - -fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFrame) { - frame.regs.copy_from_slice(ctx.registers()); - frame.sepc = ctx.program_counter(); - frame.sstatus = ctx.status(); - // ensure returning to U-mode - frame.sstatus &= !(1 << 8); // clear SPP - frame.sstatus |= 1 << 5; // set SPIE -} - -fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) { - if let Some(result) = kern().take_pending_result(tid) { - write_result(frame, &result); - } -} - -unsafe extern "C" { - fn trap_return(frame: *mut TrapFrame) -> !; -} - fn kern() -> &'static mut kernel::Kernel { // SAFETY: only called with the kernel lock held. unsafe { &mut *crate::kernel_ptr() } @@ -203,63 +64,7 @@ fn this_core() -> types::id::CoreId { types::id::CoreId::from_raw(crate::smp::hart_id() as u8) } -/// Saves `frame` into `from` (if it still exists) and loads `to`. -fn switch_to_thread( - frame: &mut TrapFrame, - from: Option, - to: types::id::ThreadId, -) { - let kern = kern(); - if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) { - save_frame_to_context(frame, ctx); - } - if let Some(ctx) = kern.thread_context(to) { - load_context_to_frame(ctx, frame); - } - apply_pending_result(frame, to); -} - -/// Saves `from` and runs the next ready thread, idling until there is one. -fn switch_to_next(frame: &mut TrapFrame, from: Option) { - if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) { - save_frame_to_context(frame, ctx); - } - let next = next_runnable(); - let _ = kern().set_thread_running(next, this_core()); - switch_to_thread(frame, None, next); -} - -/// The next thread to run on this hart, idling until one is ready. -/// -/// Idling gives up the kernel lock: an idle hart holds nothing, so other -/// harts can make work for it (and kick it) meanwhile. It also leaves any -/// user address space first, so nobody needs to shoot it down. SIE stays -/// clear; `wfi` still wakes on a pending interrupt, serviced by polling. -fn next_runnable() -> types::id::ThreadId { - loop { - if let Some(tid) = kern().pick_next_thread() { - return tid; - } - crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root())); - release(); - unsafe { core::arch::asm!("wfi") }; - crate::smp::service_ipi(); - crate::smp::lock(); - let _ = crate::smp::take_reschedule(); - let sip: usize; - unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; - if sip & (1 << CAUSE_S_TIMER) != 0 { - // No thread is running, so there is no time slice to charge. - crate::hw::timer::handle_interrupt(); - } - if sip & (1 << CAUSE_S_EXTERNAL) != 0 { - dispatch_external(); - } - } -} - -/// Claims every pending PLIC interrupt and hands each to the kernel. Only -/// the boot hart receives external interrupts. +/// Claims every pending PLIC interrupt (boot hart only) and hands it to the kernel. fn dispatch_external() { loop { let irq_num = crate::hw::plic::Plic::claim(); @@ -271,100 +76,6 @@ fn dispatch_external() { } } -/// Releases the kernel lock, first publishing what the lock-free fast -/// paths may rely on. -fn release() { - let kern = kern(); - for core in 0..crate::smp::num_harts() { - let id = types::id::CoreId::from_raw(core as u8); - crate::smp::publish_waiting(core, kern.highest_waiting(id).map(|p| p.value())); - } - let running = kern.current_thread(); - crate::smp::publish_running( - running - .and_then(|t| kern.thread_process(t)) - .map(|p| p.raw()), - running - .and_then(|t| kern.thread_priority(t)) - .map(|p| p.value()), - ); - crate::smp::unlock(); -} - -/// Syscalls answered without the kernel lock, as the kernel would answer -/// them: they read nothing the lock guards (the time, the UART, the -/// running thread's fixed process id) or, for a Yield that has nobody to -/// yield to, change nothing. Any pending IPI or reschedule request sends -/// the call down the slow path, so no request is ever skipped. -fn fast_syscall(frame: &mut TrapFrame) -> bool { - use types::syscall_abi::{SYS_DEBUG_PUTCHAR, SYS_GET_PROCESS_ID, SYS_GET_TIME, SYS_YIELD}; - if crate::smp::has_pending_request() { - return false; - } - let result = match frame.regs[17] { - SYS_GET_TIME => kernel::SyscallResult::Value(crate::hw::timer::read_time() as usize), - SYS_GET_PROCESS_ID => match crate::smp::running_pid() { - Some(pid) => kernel::SyscallResult::Value(pid), - None => return false, - }, - SYS_DEBUG_PUTCHAR => { - crate::hw::uart::Uart::new().put_char(frame.regs[10] as u8); - kernel::SyscallResult::Ok - } - SYS_YIELD if crate::smp::yield_is_noop() => kernel::SyscallResult::Ok, - _ => return false, - }; - frame.sepc += 4; - write_result(frame, &result); - true -} - -/// Last step before leaving the kernel to userspace: act on a reschedule -/// request, make sure this hart has a thread (its own may have been taken -/// away by another hart), and translate through that thread's address -/// space. -fn finish(frame: &mut TrapFrame) { - let kern = kern(); - let current = kern.current_thread(); - if crate::smp::take_reschedule() - && let Some(cur) = current - && let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core()) - { - switch_to_thread(frame, Some(cur), next); - } - if kern.current_thread().is_none() { - switch_to_next(frame, None); - } - let root = kern - .current_thread() - .and_then(|t| kern.thread_process(t)) - .map(|pid| kern.platform_mut().page_table_for(pid)) - .unwrap_or(0); - if root != 0 && crate::mmu::active_root() != root { - crate::mmu::activate(crate::mmu::page_table::table_at(root)); - } -} - -/// Leaves the kernel for userspace for the first time on this hart, with -/// the kernel lock held: runs the thread the kernel already made current -/// here (init, on the boot hart) or the next one homed here. -pub fn enter_first() -> ! { - let mut frame = TrapFrame { - regs: [0; 32], - sepc: 0, - sstatus: 0, - }; - match kern().current_thread() { - Some(tid) => switch_to_thread(&mut frame, None, tid), - None => switch_to_next(&mut frame, None), - } - finish(&mut frame); - release(); - // SAFETY: `frame` holds a user context; sscratch is set from it on - // the way out, so later traps land at the top of this frame. - unsafe { trap_return(&mut frame) } -} - #[unsafe(no_mangle)] extern "C" fn trap_handler_rust(frame: &mut TrapFrame) { let scause: usize; @@ -401,15 +112,12 @@ fn handle_interrupt(frame: &mut TrapFrame, cause: usize) { } } CAUSE_S_EXTERNAL => dispatch_external(), - // Serviced on entry; any reschedule happens in `finish`. CAUSE_S_SOFTWARE => {} _ => {} } } -/// The kernel itself faulted (a W^X violation, a stack guard, a bad -/// pointer). Its state can no longer be trusted, and no user thread is to -/// blame. +/// The kernel itself faulted; its state is untrusted, so halt. fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! { let uart = crate::hw::uart::Uart::new(); uart.write_str("KERNEL FAULT: scause="); @@ -478,8 +186,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) { kern.handle_fault(current_tid, fault, frame.sepc); switch_to_next(frame, Some(current_tid)); } - // With no current thread (another hart just destroyed it), - // `finish` moves on to the next one. + // No current thread (destroyed by another hart): `finish` picks one. } } } diff --git a/crates/adapter-riscv/src/trap/sched.rs b/crates/adapter-riscv/src/trap/sched.rs new file mode 100644 index 0000000..586a99a --- /dev/null +++ b/crates/adapter-riscv/src/trap/sched.rs @@ -0,0 +1,139 @@ +use super::{ + CAUSE_S_EXTERNAL, CAUSE_S_TIMER, TrapFrame, dispatch_external, kern, this_core, write_result, +}; + +fn save_frame_to_context(frame: &TrapFrame, ctx: &mut domain::thread::SavedContext) { + ctx.registers_mut().copy_from_slice(&frame.regs); + ctx.set_pc(frame.sepc); + ctx.set_status(frame.sstatus); +} + +fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFrame) { + frame.regs.copy_from_slice(ctx.registers()); + frame.sepc = ctx.program_counter(); + frame.sstatus = ctx.status(); + frame.sstatus &= !(1 << 8); // SPP: return to U-mode + frame.sstatus |= 1 << 5; // SPIE +} + +fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) { + if let Some(result) = kern().take_pending_result(tid) { + write_result(frame, &result); + } +} + +unsafe extern "C" { + fn trap_return(frame: *mut TrapFrame) -> !; +} + +/// Saves `frame` into `from` (if it still exists) and loads `to`. +pub(super) fn switch_to_thread( + frame: &mut TrapFrame, + from: Option, + to: types::id::ThreadId, +) { + let kern = kern(); + if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) { + save_frame_to_context(frame, ctx); + } + if let Some(ctx) = kern.thread_context(to) { + load_context_to_frame(ctx, frame); + } + apply_pending_result(frame, to); +} + +/// Saves `from` and runs the next ready thread, idling until there is one. +pub(super) fn switch_to_next(frame: &mut TrapFrame, from: Option) { + if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) { + save_frame_to_context(frame, ctx); + } + let next = next_runnable(); + let _ = kern().set_thread_running(next, this_core()); + switch_to_thread(frame, None, next); +} + +/// The next thread to run on this hart, idling (lock released, kernel +/// address space active so no shootdown is needed) until one is ready. +fn next_runnable() -> types::id::ThreadId { + loop { + if let Some(tid) = kern().pick_next_thread() { + return tid; + } + crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root())); + release(); + unsafe { core::arch::asm!("wfi") }; + crate::smp::service_ipi(); + crate::smp::lock(); + let _ = crate::smp::take_reschedule(); + let sip: usize; + unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; + if sip & (1 << CAUSE_S_TIMER) != 0 { + // No thread is running, so there is no time slice to charge. + crate::hw::timer::handle_interrupt(); + } + if sip & (1 << CAUSE_S_EXTERNAL) != 0 { + dispatch_external(); + } + } +} + +/// Releases the kernel lock after publishing state the lock-free fast paths read. +pub(super) fn release() { + let kern = kern(); + for core in 0..crate::smp::num_harts() { + let id = types::id::CoreId::from_raw(core as u8); + crate::smp::publish_waiting(core, kern.highest_waiting(id).map(|p| p.value())); + } + let running = kern.current_thread(); + crate::smp::publish_running( + running + .and_then(|t| kern.thread_process(t)) + .map(|p| p.raw()), + running + .and_then(|t| kern.thread_priority(t)) + .map(|p| p.value()), + ); + crate::smp::unlock(); +} + +/// Last step before returning to userspace: apply a reschedule request, +/// ensure this hart has a thread, and activate its address space. +pub(super) fn finish(frame: &mut TrapFrame) { + let kern = kern(); + let current = kern.current_thread(); + if crate::smp::take_reschedule() + && let Some(cur) = current + && let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core()) + { + switch_to_thread(frame, Some(cur), next); + } + if kern.current_thread().is_none() { + switch_to_next(frame, None); + } + let root = kern + .current_thread() + .and_then(|t| kern.thread_process(t)) + .map(|pid| kern.platform_mut().page_table_for(pid)) + .unwrap_or(0); + if root != 0 && crate::mmu::active_root() != root { + crate::mmu::activate(crate::mmu::page_table::table_at(root)); + } +} + +/// Enters userspace for the first time on this hart (kernel lock held). +pub fn enter_first() -> ! { + let mut frame = TrapFrame { + regs: [0; 32], + sepc: 0, + sstatus: 0, + }; + match kern().current_thread() { + Some(tid) => switch_to_thread(&mut frame, None, tid), + None => switch_to_next(&mut frame, None), + } + finish(&mut frame); + release(); + // SAFETY: `frame` holds a user context; sscratch is set from it on + // the way out, so later traps land at the top of this frame. + unsafe { trap_return(&mut frame) } +} diff --git a/crates/adapter-riscv/src/trap/vector.rs b/crates/adapter-riscv/src/trap/vector.rs new file mode 100644 index 0000000..a7962ca --- /dev/null +++ b/crates/adapter-riscv/src/trap/vector.rs @@ -0,0 +1,118 @@ +use core::arch::global_asm; + +global_asm!( + r#" +.section .text +.align 4 +.globl trap_vector +trap_vector: + csrrw sp, sscratch, sp + bnez sp, _from_user + csrrw sp, sscratch, sp +_from_user: + addi sp, sp, -272 + + sd x1, 8(sp) + sd x3, 24(sp) + sd x4, 32(sp) + sd x5, 40(sp) + sd x6, 48(sp) + sd x7, 56(sp) + sd x8, 64(sp) + sd x9, 72(sp) + sd x10, 80(sp) + sd x11, 88(sp) + sd x12, 96(sp) + sd x13, 104(sp) + sd x14, 112(sp) + sd x15, 120(sp) + sd x16, 128(sp) + sd x17, 136(sp) + sd x18, 144(sp) + sd x19, 152(sp) + sd x20, 160(sp) + sd x21, 168(sp) + sd x22, 176(sp) + sd x23, 184(sp) + sd x24, 192(sp) + sd x25, 200(sp) + sd x26, 208(sp) + sd x27, 216(sp) + sd x28, 224(sp) + sd x29, 232(sp) + sd x30, 240(sp) + sd x31, 248(sp) + + csrr t0, sscratch + sd t0, 16(sp) + + csrr t0, sepc + sd t0, 256(sp) + csrr t0, sstatus + sd t0, 264(sp) + + mv a0, sp + call trap_handler_rust + j _restore_frame + +# trap_return(frame): resume `frame` as if returning from the trap that saved it. +.globl trap_return +trap_return: + mv sp, a0 +_restore_frame: + ld t0, 256(sp) + csrw sepc, t0 + ld t0, 264(sp) + csrw sstatus, t0 + + andi t1, t0, 0x100 + bnez t1, _restore_smode + + ld t0, 16(sp) + csrw sscratch, t0 + +_restore_smode: + ld x1, 8(sp) + ld x3, 24(sp) + ld x4, 32(sp) + ld x6, 48(sp) + ld x7, 56(sp) + ld x8, 64(sp) + ld x9, 72(sp) + ld x10, 80(sp) + ld x11, 88(sp) + ld x12, 96(sp) + ld x13, 104(sp) + ld x14, 112(sp) + ld x15, 120(sp) + ld x16, 128(sp) + ld x17, 136(sp) + ld x18, 144(sp) + ld x19, 152(sp) + ld x20, 160(sp) + ld x21, 168(sp) + ld x22, 176(sp) + ld x23, 184(sp) + ld x24, 192(sp) + ld x25, 200(sp) + ld x26, 208(sp) + ld x27, 216(sp) + ld x28, 224(sp) + ld x29, 232(sp) + ld x30, 240(sp) + ld x31, 248(sp) + + # t0 (x5) is restored last on each path; it is the only scratch register. + csrr t0, sstatus + andi t0, t0, 0x100 + bnez t0, _sret_smode + ld x5, 40(sp) + addi sp, sp, 272 + csrrw sp, sscratch, sp + sret +_sret_smode: + ld x5, 40(sp) + addi sp, sp, 272 + sret +"# +); diff --git a/crates/hw-virtio/src/gpu.rs b/crates/hw-virtio/src/gpu.rs index 1352948..21baa30 100644 --- a/crates/hw-virtio/src/gpu.rs +++ b/crates/hw-virtio/src/gpu.rs @@ -75,7 +75,7 @@ pub fn resource_flush(mmio: &impl MmioAccess, queue: &VirtioQueue, dma: &GpuDmaA fn write_cmd_header(cmd_va: VirtualAddress, cmd_type: u32) { unsafe { let p = cmd_va.raw() as *mut u8; - // Use volatile writes to zero the buffer — ensures device sees the zeros + // Volatile so the device sees the zeros. for i in 0..128 { core::ptr::write_volatile((p as *mut u32).add(i), 0); } diff --git a/crates/hw-virtio/src/mmio.rs b/crates/hw-virtio/src/mmio.rs index b096c9d..3190685 100644 --- a/crates/hw-virtio/src/mmio.rs +++ b/crates/hw-virtio/src/mmio.rs @@ -89,8 +89,7 @@ pub const MMIO_FIRST: usize = 0x10001000; pub const MMIO_STRIDE: usize = 0x1000; pub const MMIO_SLOTS: usize = 8; -/// Probes the MMIO slots starting at `first`, the address the caller sees -/// slot 0 at (`MMIO_FIRST` itself, or wherever it is mapped). +/// Probes the MMIO slots, with slot 0 at `first` as the caller maps it. pub fn scan_for_device(first: usize, device_id: u32) -> Option { for slot in 0..MMIO_SLOTS { if let Some(mmio) = probe_slot(first + slot * MMIO_STRIDE, device_id) {