diff --git a/Makefile b/Makefile index c80c196..6923e70 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ RISCV_TARGET = riscv64gc-unknown-none-elf KERNEL_BIN = target/$(RISCV_TARGET)/release/adapter-riscv +# Harts QEMU boots; `make run-gui SMP=1` for a single core. +SMP ?= 4 KERNEL_CRATES = -p types -p domain -p kernel -p platform @@ -56,17 +58,17 @@ image: build scripts/make-disk-image.sh run: build - qemu-system-riscv64 -machine virt -nographic -bios default \ + qemu-system-riscv64 -machine virt -smp $(SMP) -nographic -bios default \ -kernel $(KERNEL_BIN) run-disk: image - qemu-system-riscv64 -machine virt -nographic -bios default \ + qemu-system-riscv64 -machine virt -smp $(SMP) -nographic -bios default \ -kernel $(KERNEL_BIN) \ -drive file=boot.img,if=none,format=raw,id=hd0 \ -device virtio-blk-device,drive=hd0 run-gui: image - qemu-system-riscv64 -machine virt -bios default \ + qemu-system-riscv64 -machine virt -smp $(SMP) -bios default \ -kernel $(KERNEL_BIN) \ -drive file=boot.img,if=none,format=raw,id=hd0 \ -device virtio-blk-device,drive=hd0 \ diff --git a/crates/adapter-riscv/linker.ld b/crates/adapter-riscv/linker.ld index 7508c08..cb17796 100644 --- a/crates/adapter-riscv/linker.ld +++ b/crates/adapter-riscv/linker.ld @@ -46,15 +46,13 @@ SECTIONS { _bss_end = .; } - /* Left unmapped, so overflowing the kernel stack faults instead of - silently overwriting .bss. */ + /* One kernel stack per hart (smp::MAX_HARTS, smp::STACK_SLOT): each + slot is an unmapped guard page, so overflowing a stack faults + instead of silently overwriting what lies below, then the stack. */ . = ALIGN(4096); - _stack_guard = .; - . += 4096; - - _stack_bottom = .; - . += 0x80000; - _stack_top = .; + _stacks_start = .; + . += 8 * (4096 + 0x80000); + _stacks_end = .; _heap_start = .; } diff --git a/crates/adapter-riscv/src/boot/hardware.rs b/crates/adapter-riscv/src/boot/hardware.rs index b3f6057..40cd906 100644 --- a/crates/adapter-riscv/src/boot/hardware.rs +++ b/crates/adapter-riscv/src/boot/hardware.rs @@ -8,12 +8,21 @@ unsafe extern "C" { static _heap_start: u8; } -pub fn init(uart: &Uart) -> &'static mut mmu::PageTable { +pub fn init(uart: &Uart, boot_hart: usize) -> &'static mut mmu::PageTable { zero_bss(); let heap_start = unsafe { &_heap_start as *const u8 as usize }; mmu::init_allocator(heap_start); let kernel_pt = mmu::init_kernel_page_table(); + let harts = crate::smp::probe_harts(); + if boot_hart >= harts { + crate::halt(uart, "boot hart outside the hart range"); + } + crate::smp::init( + boot_hart, + harts, + mmu::virt_to_phys(kernel_pt as *const mmu::PageTable as usize), + ); mmu::activate(kernel_pt); uart.write_str("k-os: MMU enabled (Sv39)\n"); match mmu::check_wx(kernel_pt) { @@ -46,7 +55,8 @@ pub fn init(uart: &Uart) -> &'static mut mmu::PageTable { if board::USER_POOL_PAGES > domain::memory::frame::MAX_FRAMES { crate::halt(uart, "user pool larger than the frame table"); } - let riscv_platform = platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize); + let riscv_platform = + platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize, harts as u8); // SAFETY: `init_in_place` writes every field of the kernel. unsafe { KERNEL.init_with(|slot| { diff --git a/crates/adapter-riscv/src/boot/loader.rs b/crates/adapter-riscv/src/boot/loader.rs index 4bb8922..15f9ad8 100644 --- a/crates/adapter-riscv/src/boot/loader.rs +++ b/crates/adapter-riscv/src/boot/loader.rs @@ -13,7 +13,6 @@ const MAX_FILE_SIZE: usize = 64 * 1024; pub struct LoadedProcess { pub pid: ProcessId, - pub entry: usize, } fn kern() -> &'static mut kernel::Kernel { @@ -48,7 +47,7 @@ pub fn load_init(uart: &Uart, blk: &mut VirtioBlk) -> (LoadedProcess, ThreadId) setup_thread(tid, entry); uart.write_str("k-os: init loaded\n"); - (LoadedProcess { pid, entry }, tid) + (LoadedProcess { pid }, tid) } pub fn load_server( @@ -69,35 +68,7 @@ pub fn load_server( uart.write_str(label); uart.write_str(" loaded\n"); - LoadedProcess { pid, entry } -} - -pub fn enter_userspace(pid: ProcessId, entry: usize, shared_buf_va: usize) -> ! { - let pt_addr = kern().platform_mut().page_table_for(pid); - if pt_addr != 0 { - mmu::activate(mmu::page_table::table_at(pt_addr)); - } - - let kernel_sp: usize; - unsafe { core::arch::asm!("mv {0}, sp", out(reg) kernel_sp) }; - crate::trap::set_kernel_stack(kernel_sp); - - unsafe { - core::arch::asm!( - "csrc sstatus, {spp}", - "csrs sstatus, {spie}", - "csrw sepc, {entry}", - "mv sp, {stack}", - "mv a0, {arg}", - "sret", - spp = in(reg) 1usize << 8, - spie = in(reg) 1usize << 5, - entry = in(reg) entry, - stack = in(reg) USER_VA_BASE, - arg = in(reg) shared_buf_va, - options(noreturn), - ); - } + LoadedProcess { pid } } fn setup_thread(tid: ThreadId, entry: usize) { diff --git a/crates/adapter-riscv/src/hw/plic.rs b/crates/adapter-riscv/src/hw/plic.rs index 11f67b5..a16115d 100644 --- a/crates/adapter-riscv/src/hw/plic.rs +++ b/crates/adapter-riscv/src/hw/plic.rs @@ -2,21 +2,36 @@ use crate::board; const PLIC: usize = crate::mmu::phys_to_virt(board::PLIC_BASE); const PLIC_PRIORITY: usize = PLIC; -const PLIC_S_ENABLE: usize = PLIC + 0x2080; -const PLIC_S_THRESHOLD: usize = PLIC + 0x201000; -const PLIC_S_CLAIM: usize = PLIC + 0x201004; + +// External interrupts go to the boot hart only: its supervisor context is +// 2 * hart + 1 on QEMU virt (even contexts are machine mode). +fn context() -> usize { + 2 * crate::smp::boot_hart() + 1 +} + +fn enable_base() -> usize { + PLIC + 0x2000 + context() * 0x80 +} + +fn threshold() -> usize { + PLIC + 0x20_0000 + context() * 0x1000 +} + +fn claim_reg() -> usize { + threshold() + 4 +} pub struct Plic; impl Plic { pub fn init() { - unsafe { core::ptr::write_volatile(PLIC_S_THRESHOLD as *mut u32, 0) }; + unsafe { core::ptr::write_volatile(threshold() as *mut u32, 0) }; } pub fn enable(irq: u32) { let word = (irq / 32) as usize; let bit = irq % 32; - let addr = PLIC_S_ENABLE + word * 4; + let addr = enable_base() + word * 4; unsafe { let current = core::ptr::read_volatile(addr as *const u32); core::ptr::write_volatile(addr as *mut u32, current | (1 << bit)); @@ -28,7 +43,7 @@ impl Plic { pub fn disable(irq: u32) { let word = (irq / 32) as usize; let bit = irq % 32; - let addr = PLIC_S_ENABLE + word * 4; + let addr = enable_base() + word * 4; unsafe { let current = core::ptr::read_volatile(addr as *const u32); core::ptr::write_volatile(addr as *mut u32, current & !(1 << bit)); @@ -36,10 +51,10 @@ impl Plic { } pub fn claim() -> u32 { - unsafe { core::ptr::read_volatile(PLIC_S_CLAIM as *const u32) } + unsafe { core::ptr::read_volatile(claim_reg() as *const u32) } } pub fn complete(irq: u32) { - unsafe { core::ptr::write_volatile(PLIC_S_CLAIM as *mut u32, irq) }; + unsafe { core::ptr::write_volatile(claim_reg() as *mut u32, irq) }; } } diff --git a/crates/adapter-riscv/src/main.rs b/crates/adapter-riscv/src/main.rs index 1f92f94..fb3d62f 100644 --- a/crates/adapter-riscv/src/main.rs +++ b/crates/adapter-riscv/src/main.rs @@ -9,6 +9,7 @@ mod hw; mod kernel_cell; mod mmu; mod platform; +mod smp; mod trap; use core::arch::global_asm; @@ -18,8 +19,8 @@ use kernel_cell::KernelCell; static KERNEL: KernelCell> = KernelCell::uninit(); fn kern_mut() -> &'static mut kernel::Kernel { - // SAFETY: single core; boot code runs before any trap can touch the - // kernel. + // SAFETY: boot runs with the kernel lock held and before any other + // hart has started. unsafe { &mut *kernel_ptr() } } @@ -27,28 +28,55 @@ pub fn kernel_ptr() -> *mut kernel::Kernel { KERNEL.as_ptr() } -// Firmware enters `_start` at the physical load address with the MMU off, -// but the kernel is linked in the upper half. Using only PC-relative code, -// the stub turns on `boot_page_table`, which maps RAM twice (at its -// physical address, just so the next instruction still exists, and in the -// upper half) plus the MMIO window high, then jumps to its own high address. -// kmain replaces this table with the real one, which maps nothing low. +// Firmware enters `_start` (and SBI starts other harts at +// `_secondary_start`) at the physical load address with the MMU off and +// the hart id in a0, but the kernel is linked in the upper half. Using +// only PC-relative code, the stub turns on `boot_page_table`, which maps +// RAM twice (at its physical address, just so the next instruction still +// exists, and in the upper half) plus the MMIO window high, then jumps to +// its own high address and takes the hart's kernel stack. The kernel then +// switches every hart to the real table, which maps nothing low. global_asm!( r#" .section .text.entry .globl _start _start: + lla t2, start_high_address + j 1f +.globl _secondary_start +_secondary_start: + lla t2, secondary_high_address +1: + li t0, 8 # smp::MAX_HARTS + bgeu a0, t0, spin + mv s0, a0 lla t0, boot_page_table srli t0, t0, 12 li t1, 8 << 60 or t0, t0, t1 csrw satp, t0 sfence.vma - ld t0, start_high_address + ld t0, 0(t2) jr t0 + +# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT start_high: - lla sp, _stack_top + call hart_stack + mv a0, s0 call kmain + j spin +secondary_high: + call hart_stack + mv a0, s0 + call kmain_secondary + j spin +hart_stack: + lla sp, _stacks_start + addi t0, s0, 1 + li t1, 0x81000 + mul t0, t0, t1 + add sp, sp, t0 + ret spin: wfi j spin @@ -57,6 +85,8 @@ spin: .balign 8 start_high_address: .quad start_high +secondary_high_address: + .quad secondary_high .popsection # Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with @@ -79,15 +109,21 @@ boot_page_table: const SHARED_BUF_VA: usize = 0x30000; #[unsafe(no_mangle)] -extern "C" fn kmain() -> ! { +extern "C" fn kmain(hart: usize) -> ! { let uart = Uart::new(); uart.write_str("k-os: booted on RISC-V\n"); - boot::hardware::init(&uart); + boot::hardware::init(&uart, hart); + // Held until the boot hart first enters userspace; harts started + // below wait for it. Not before init: zeroing .bss resets the lock. + smp::lock(); let mut blk = boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device")); - let (init, _) = boot::loader::load_init(&uart, &mut blk); + let (init, init_thread) = boot::loader::load_init(&uart, &mut blk); + if let Some(ctx) = kern_mut().thread_context_mut(init_thread) { + ctx.registers_mut()[10] = SHARED_BUF_VA; + } let storage = boot::loader::load_server(&uart, &mut blk, b"storage.elf", "storage"); let display = boot::loader::load_server(&uart, &mut blk, b"display.elf", "display"); let input = boot::loader::load_server(&uart, &mut blk, b"input.elf", "input"); @@ -106,8 +142,35 @@ extern "C" fn kmain() -> ! { .grant_boot_authority(init.pid, &board) .unwrap_or_else(|_| halt(&uart, "boot authority failed")); + start_other_harts(&uart, hart); uart.write_str("k-os: entering userspace\n"); - boot::loader::enter_userspace(init.pid, init.entry, SHARED_BUF_VA); + trap::enter_first(); +} + +fn start_other_harts(uart: &Uart, boot: usize) { + unsafe extern "C" { + fn _secondary_start(); + } + // Harts start with the MMU off, so at the entry's physical address. + let entry = mmu::virt_to_phys(_secondary_start as *const () as usize); + for hart in (0..smp::num_harts()).filter(|h| *h != boot) { + if !smp::sbi_hart_start(hart, entry) { + halt(uart, "could not start a hart"); + } + } + uart.write_str("k-os: "); + uart.write_dec(smp::num_harts()); + uart.write_str(" harts\n"); +} + +/// Where every hart but the boot hart starts, on its own stack. +#[unsafe(no_mangle)] +extern "C" fn kmain_secondary(_hart: usize) -> ! { + trap::init(); + hw::timer::init(); + mmu::activate(mmu::page_table::table_at(smp::kernel_root())); + smp::lock(); + trap::enter_first(); } pub fn halt(uart: &Uart, msg: &str) -> ! { diff --git a/crates/adapter-riscv/src/mmu/page_table.rs b/crates/adapter-riscv/src/mmu/page_table.rs index 3a3302b..b023102 100644 --- a/crates/adapter-riscv/src/mmu/page_table.rs +++ b/crates/adapter-riscv/src/mmu/page_table.rs @@ -1,6 +1,7 @@ use super::allocator::alloc_page_table; use super::{phys_to_virt, virt_to_phys}; use crate::board; +use crate::smp; use platform::{FrameSource, MapError}; use types::address::{PhysicalAddress, VirtualAddress}; use types::mapping::PagePerms; @@ -148,20 +149,31 @@ pub fn init_kernel_page_table() -> &'static mut PageTable { // The kernel image, then the rest of RAM, each with only the access it // needs: code is never writable, data never executable. RAM below the // image belongs to the firmware and stays unmapped, as does the guard - // page under the kernel stack. Nothing is mapped in the lower half, - // which is userspace's alone. + // page under each hart's kernel stack. Nothing is mapped in the lower + // half, which is userspace's alone. let layout = KernelLayout::from_linker(); let ram_end = phys_to_virt(board::RAM_BASE + board::RAM_SIZE); + let stacks = (0..smp::MAX_HARTS).map(|h| { + let slot = layout.stacks_start + h * smp::STACK_SLOT; + ( + slot + PAGE_SIZE, + slot + smp::STACK_SLOT, + Sv39Flags::kernel_rw(), + ) + }); for (start, end, flags) in [ (layout.text_start, layout.text_end, Sv39Flags::kernel_rx()), (layout.text_end, layout.rodata_end, Sv39Flags::kernel_ro()), ( layout.rodata_end, - layout.stack_guard, + layout.stacks_start, Sv39Flags::kernel_rw(), ), - (layout.stack_bottom, ram_end, Sv39Flags::kernel_rw()), - ] { + (layout.stacks_end, ram_end, Sv39Flags::kernel_rw()), + ] + .into_iter() + .chain(stacks) + { let (va, pa) = direct(virt_to_phys(start)); map( root, @@ -200,8 +212,8 @@ pub struct KernelLayout { pub text_start: usize, pub text_end: usize, pub rodata_end: usize, - pub stack_guard: usize, - pub stack_bottom: usize, + pub stacks_start: usize, + pub stacks_end: usize, } impl KernelLayout { @@ -210,8 +222,8 @@ impl KernelLayout { static _text_start: u8; static _text_end: u8; static _rodata_end: u8; - static _stack_guard: u8; - static _stack_bottom: u8; + static _stacks_start: u8; + static _stacks_end: u8; } // SAFETY: only the symbols' addresses are taken, never their values. unsafe { @@ -219,8 +231,8 @@ impl KernelLayout { text_start: &_text_start as *const u8 as usize, text_end: &_text_end as *const u8 as usize, rodata_end: &_rodata_end as *const u8 as usize, - stack_guard: &_stack_guard as *const u8 as usize, - stack_bottom: &_stack_bottom as *const u8 as usize, + stacks_start: &_stacks_start as *const u8 as usize, + stacks_end: &_stacks_end as *const u8 as usize, } } } @@ -240,12 +252,15 @@ pub enum WxViolation { /// Walks every leaf of `root` and checks the kernel layout: nothing is /// both writable and executable, only kernel text is executable, the -/// firmware and the stack guard are unmapped, and nothing at all is mapped +/// firmware and every stack guard are unmapped, and nothing at all is mapped /// in the lower half. Returns the number of executable pages. pub fn check_wx(root: &PageTable) -> Result { let layout = KernelLayout::from_linker(); let text = layout.text_start..layout.text_end; - let guard = layout.stack_guard..layout.stack_bottom; + let is_guard = |va: usize| { + (layout.stacks_start..layout.stacks_end).contains(&va) + && (va - layout.stacks_start).is_multiple_of(smp::STACK_SLOT) + }; let firmware = phys_to_virt(board::RAM_BASE)..layout.text_start; let mut executable = 0; for (va, bits) in leaves(root) { @@ -261,7 +276,7 @@ pub fn check_wx(root: &PageTable) -> Result { } executable += 1; } - if guard.contains(&va) || firmware.contains(&va) { + if is_guard(va) || firmware.contains(&va) { return Err(WxViolation::MappedHole(va)); } } @@ -299,6 +314,7 @@ fn leaves(root: &PageTable) -> impl Iterator + '_ { } pub fn activate(root: &PageTable) { + smp::note_active_root(virt_to_phys(root as *const PageTable as usize)); let satp = (8u64 << 60) | table_ppn(root); unsafe { core::arch::asm!( diff --git a/crates/adapter-riscv/src/platform.rs b/crates/adapter-riscv/src/platform.rs index d212382..7a268a8 100644 --- a/crates/adapter-riscv/src/platform.rs +++ b/crates/adapter-riscv/src/platform.rs @@ -10,6 +10,7 @@ use types::{ use crate::board; use crate::hw::{plic::Plic, timer}; use crate::mmu; +use crate::smp; use crate::trap; pub struct RiscVPlatform { @@ -20,9 +21,9 @@ pub struct RiscVPlatform { } impl RiscVPlatform { - pub fn new(kernel_pt_addr: usize) -> Self { + pub fn new(kernel_pt_addr: usize, num_cores: u8) -> Self { Self { - num_cores: 1, + num_cores, kernel_pt: kernel_pt_addr, page_tables: [0usize; MAX_PROCESSES], } @@ -63,8 +64,10 @@ impl Platform for RiscVPlatform { let Some(root) = self.root(process) else { return; }; - // The caller may be tearing down the space we are running on; get - // off it before the kernel frees its frames. + // No hart may still translate through it once the kernel frees its + // frames: other harts leave it (and wait for nothing but that), and + // the caller may be tearing down the space we are running on. + smp::shoot_down(root); if mmu::active_root() == root { // SAFETY: the kernel page table lives in the boot heap forever. mmu::activate(unsafe { &*(self.kernel_pt as *const mmu::PageTable) }); @@ -95,6 +98,8 @@ impl Platform for RiscVPlatform { fn unmap_page(&mut self, process: ProcessId, va: VirtualAddress) { if let Some(root) = self.root(process) { mmu::unmap_user_page(root, va.raw()); + // Other harts running this process may still cache it. + smp::shoot_down(root); } } @@ -144,10 +149,12 @@ impl Platform for RiscVPlatform { Plic::enable(irq.raw() as u32); } - fn send_ipi(&mut self, _target: CoreId) {} + fn send_ipi(&mut self, target: CoreId) { + smp::kick(target.raw() as usize); + } fn core_id(&self) -> CoreId { - CoreId::from_raw(0) + CoreId::from_raw(smp::hart_id() as u8) } fn num_cores(&self) -> u8 { diff --git a/crates/adapter-riscv/src/smp.rs b/crates/adapter-riscv/src/smp.rs new file mode 100644 index 0000000..61b77f8 --- /dev/null +++ b/crates/adapter-riscv/src/smp.rs @@ -0,0 +1,220 @@ +//! Harts. Each hart has its own kernel stack, and its hart id doubles as +//! its kernel `CoreId`. The kernel is entered under one lock for now (stage +//! 1 of multi-core; the kernel itself stays a sequential program), so +//! userspace runs in parallel and the kernel does not. +//! +//! Harts talk through IPIs carrying two kinds of request: +//! - reschedule: the kernel made work for this hart, or took its thread +//! away; handled under the lock on the way out of the kernel. +//! - leave root R: a page table this hart may be using is changing or about +//! to be freed; the hart switches to the kernel's root and flushes its +//! TLB. This needs no lock, so it is serviced even while a hart spins +//! waiting for the lock or idles, and the sender (which holds the lock) +//! can wait for the acknowledgement without deadlocking. + +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; + +use crate::mmu; + +/// Must match linker.ld. +pub const MAX_HARTS: usize = 8; +/// A guard page and a 512 KiB stack (boot builds big tables on it). Must +/// match linker.ld. +pub const STACK_SLOT: usize = 4096 + 0x8_0000; + +const SIP_SSIP: usize = 1 << 1; + +/// The hart running this code, from which kernel stack it is on. +pub fn hart_id() -> usize { + let sp: usize; + // SAFETY: reading sp has no side effects. + unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) }; + (sp - stacks_start()) / STACK_SLOT +} + +pub fn stacks_start() -> usize { + unsafe extern "C" { + static _stacks_start: u8; + } + // SAFETY: only the symbol's address is taken. + unsafe { &_stacks_start as *const u8 as usize } +} + +static BOOT_HART: AtomicUsize = AtomicUsize::new(0); +static NUM_HARTS: AtomicUsize = AtomicUsize::new(1); +/// Physical address of the kernel's own root table. +static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0); + +/// The root table each hart translates with. +static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; +static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS]; +static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; +static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; + +pub fn boot_hart() -> usize { + BOOT_HART.load(Ordering::Relaxed) +} + +pub fn num_harts() -> usize { + NUM_HARTS.load(Ordering::Relaxed) +} + +pub fn kernel_root() -> usize { + KERNEL_ROOT.load(Ordering::Relaxed) +} + +/// Records which hart booted and how many there are, and where the +/// kernel's root table is. +pub fn init(boot: usize, harts: usize, kernel_root: usize) { + BOOT_HART.store(boot, Ordering::Relaxed); + NUM_HARTS.store(harts, Ordering::Relaxed); + KERNEL_ROOT.store(kernel_root, Ordering::Relaxed); +} + +/// Called by `mmu::activate`. +pub fn note_active_root(root: usize) { + if let Some(r) = ACTIVE_ROOT.get(hart_id()) { + r.store(root, Ordering::SeqCst); + } +} + +// ---- the kernel lock ---- + +static NEXT_TICKET: AtomicU32 = AtomicU32::new(0); +static SERVING: AtomicU32 = AtomicU32::new(0); + +/// Takes the kernel lock, first come first served. Leave-root requests +/// are serviced while waiting. +pub fn lock() { + let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed); + while SERVING.load(Ordering::Acquire) != ticket { + service_ipi(); + core::hint::spin_loop(); + } +} + +pub fn unlock() { + SERVING.fetch_add(1, Ordering::Release); +} + +// ---- IPIs ---- + +/// The lock-free half of an IPI: acknowledges it and, if asked, leaves the +/// root being changed. Safe to call at any time, any number of times. +pub fn service_ipi() { + let sip: usize; + // SAFETY: reading sip has no side effects. + unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; + if sip & SIP_SSIP == 0 { + return; + } + // Clear first: a request posted after this raises SSIP again. + // SAFETY: clearing our own pending software interrupt. + unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) }; + let me = hart_id(); + let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else { + return; + }; + let root = leave.swap(0, Ordering::SeqCst); + if root != 0 { + if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) { + mmu::activate(mmu::page_table::table_at(kernel_root())); + } + // SAFETY: flushing translations has no other effect. + unsafe { core::arch::asm!("sfence.vma zero, zero") }; + left.fetch_add(1, Ordering::SeqCst); + } +} + +/// Asks `hart` to look at its run queue again, on its way out of the +/// kernel. +pub fn kick(hart: usize) { + if let Some(r) = RESCHEDULE.get(hart) { + r.store(true, Ordering::SeqCst); + sbi_send_ipi(hart); + } +} + +/// Consumes this hart's reschedule request. +pub fn take_reschedule() -> bool { + RESCHEDULE + .get(hart_id()) + .is_some_and(|r| r.swap(false, Ordering::SeqCst)) +} + +/// Makes every other hart that translates through `root` leave it and +/// flush, and waits until each has. Called with the kernel lock held, so +/// no hart can start using `root` meanwhile. +pub fn shoot_down(root: usize) { + let me = hart_id(); + for hart in 0..num_harts() { + if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root { + continue; + } + let seen = LEFT[hart].load(Ordering::SeqCst); + LEAVE_ROOT[hart].store(root, Ordering::SeqCst); + sbi_send_ipi(hart); + while LEFT[hart].load(Ordering::SeqCst) == seen { + core::hint::spin_loop(); + } + } +} + +// ---- SBI ---- + +const SBI_EXT_IPI: usize = 0x735049; +const SBI_EXT_HSM: usize = 0x48534D; + +fn sbi_send_ipi(hart: usize) { + // SAFETY: an SBI call; a0 and a1 come back clobbered. + unsafe { + core::arch::asm!( + "ecall", + inlateout("a0") 1usize => _, + inlateout("a1") hart => _, + in("a6") 0usize, + in("a7") SBI_EXT_IPI, + ); + } +} + +/// Whether `hart` exists, per SBI HSM `hart_get_status`. +fn sbi_hart_exists(hart: usize) -> bool { + let error: isize; + // SAFETY: an SBI call; a0 and a1 come back clobbered. + unsafe { + core::arch::asm!( + "ecall", + inlateout("a0") hart => error, + lateout("a1") _, + in("a6") 2usize, + in("a7") SBI_EXT_HSM, + ); + } + error == 0 +} + +/// The number of harts, counting up from hart 0 until one is missing. +pub fn probe_harts() -> usize { + (0..MAX_HARTS) + .take_while(|h| sbi_hart_exists(*h)) + .count() + .max(1) +} + +/// Starts `hart` at the physical address `entry` with the MMU off. +pub fn sbi_hart_start(hart: usize, entry: usize) -> bool { + let error: isize; + // SAFETY: an SBI call; a0 and a1 come back clobbered. + unsafe { + core::arch::asm!( + "ecall", + inlateout("a0") hart => error, + inlateout("a1") entry => _, + in("a2") 0usize, + in("a6") 0usize, + in("a7") SBI_EXT_HSM, + ); + } + error == 0 +} diff --git a/crates/adapter-riscv/src/trap/mod.rs b/crates/adapter-riscv/src/trap/mod.rs index 0983818..0b679e0 100644 --- a/crates/adapter-riscv/src/trap/mod.rs +++ b/crates/adapter-riscv/src/trap/mod.rs @@ -60,7 +60,14 @@ _from_user: mv a0, sp call trap_handler_rust + j _restore_frame +# trap_return(frame): resume whatever `frame` describes, as if returning +# from a trap that saved it. Used to enter userspace for the first time. +.globl trap_return +trap_return: + mv sp, a0 +_restore_frame: ld t0, 256(sp) csrw sepc, t0 ld t0, 264(sp) @@ -121,6 +128,7 @@ _sret_smode: ); const CAUSE_INTERRUPT_BIT: usize = 1 << 63; +const CAUSE_S_SOFTWARE: usize = 1; const CAUSE_S_TIMER: usize = 5; const CAUSE_S_EXTERNAL: usize = 9; const CAUSE_ECALL_U: usize = 8; @@ -177,88 +185,67 @@ fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFra } fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) { - let kern = unsafe { &mut *crate::kernel_ptr() }; - if let Some(result) = kern.take_pending_result(tid) { + if let Some(result) = kern().take_pending_result(tid) { write_result(frame, &result); } } +unsafe extern "C" { + fn trap_return(frame: *mut TrapFrame) -> !; +} + +fn kern() -> &'static mut kernel::Kernel { + // SAFETY: only called with the kernel lock held. + unsafe { &mut *crate::kernel_ptr() } +} + +fn this_core() -> types::id::CoreId { + types::id::CoreId::from_raw(crate::smp::hart_id() as u8) +} + +/// Saves `frame` into `from` (if it still exists) and loads `to`. fn switch_to_thread( frame: &mut TrapFrame, - current_tid: types::id::ThreadId, - next_tid: types::id::ThreadId, + from: Option, + to: types::id::ThreadId, ) { - let kern = unsafe { &mut *crate::kernel_ptr() }; - - if let Some(ctx) = kern.thread_context_mut(current_tid) { + let kern = kern(); + if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) { save_frame_to_context(frame, ctx); } - - let next_pid = kern.thread_process(next_tid); - let current_pid = kern.thread_process(current_tid); - - if let Some(ctx) = kern.thread_context(next_tid) { + if let Some(ctx) = kern.thread_context(to) { load_context_to_frame(ctx, frame); } - - apply_pending_result(frame, next_tid); - - if next_pid != current_pid - && let Some(pid) = next_pid - { - let pt_addr = kern.platform_mut().page_table_for(pid); - if pt_addr != 0 { - let pt = crate::mmu::page_table::table_at(pt_addr); - crate::mmu::activate(pt); - } - } + apply_pending_result(frame, to); } -fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) { - let kern = unsafe { &mut *crate::kernel_ptr() }; - - if let Some(ctx) = kern.thread_context_mut(current_tid) { +/// Saves `from` and runs the next ready thread, idling until there is one. +fn switch_to_next(frame: &mut TrapFrame, from: Option) { + if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) { save_frame_to_context(frame, ctx); } - - let next_tid = next_runnable(); - let kern = unsafe { &mut *crate::kernel_ptr() }; - let next_pid = kern.thread_process(next_tid); - let current_pid = kern.thread_process(current_tid); - - let core = types::id::CoreId::from_raw(0); - let _ = kern.set_thread_running(next_tid, core); - - if let Some(ctx) = kern.thread_context(next_tid) { - load_context_to_frame(ctx, frame); - } - - apply_pending_result(frame, next_tid); - - if next_pid != current_pid - && let Some(pid) = next_pid - { - let pt_addr = kern.platform_mut().page_table_for(pid); - if pt_addr != 0 { - let pt = crate::mmu::page_table::table_at(pt_addr); - crate::mmu::activate(pt); - } - } + let next = next_runnable(); + let _ = kern().set_thread_running(next, this_core()); + switch_to_thread(frame, None, next); } -/// The next thread to run, idling until an interrupt makes one ready. +/// The next thread to run on this hart, idling until one is ready. /// -/// We are inside a trap, so SIE is clear and interrupts are not taken. -/// `wfi` still wakes once one is pending, and it is then serviced here by -/// polling `sip`, which is how a driver blocked on its IRQ notification -/// gets woken when nothing else can run. +/// Idling gives up the kernel lock: an idle hart holds nothing, so other +/// harts can make work for it (and kick it) meanwhile. It also leaves any +/// user address space first, so nobody needs to shoot it down. SIE stays +/// clear; `wfi` still wakes on a pending interrupt, serviced by polling. fn next_runnable() -> types::id::ThreadId { loop { - let kern = unsafe { &mut *crate::kernel_ptr() }; - if let Some(tid) = kern.pick_next_thread() { + if let Some(tid) = kern().pick_next_thread() { return tid; } + crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root())); + crate::smp::unlock(); unsafe { core::arch::asm!("wfi") }; + crate::smp::service_ipi(); + crate::smp::lock(); + let _ = crate::smp::take_reschedule(); let sip: usize; unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; if sip & (1 << CAUSE_S_TIMER) != 0 { @@ -271,49 +258,123 @@ fn next_runnable() -> types::id::ThreadId { } } -/// Claims every pending PLIC interrupt and hands each to the kernel. +/// Claims every pending PLIC interrupt and hands each to the kernel. Only +/// the boot hart receives external interrupts. fn dispatch_external() { loop { let irq_num = crate::hw::plic::Plic::claim(); if irq_num == 0 { return; } - let kern = unsafe { &mut *crate::kernel_ptr() }; - kern.handle_interrupt(types::id::IrqId::from_raw(irq_num as u16)); + kern().handle_interrupt(types::id::IrqId::from_raw(irq_num as u16)); crate::hw::plic::Plic::complete(irq_num); } } +/// Last step before leaving the kernel to userspace: act on a reschedule +/// request, make sure this hart has a thread (its own may have been taken +/// away by another hart), and translate through that thread's address +/// space. +fn finish(frame: &mut TrapFrame) { + let kern = kern(); + let current = kern.current_thread(); + if crate::smp::take_reschedule() + && let Some(cur) = current + && let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core()) + { + switch_to_thread(frame, Some(cur), next); + } + if kern.current_thread().is_none() { + switch_to_next(frame, None); + } + let root = kern + .current_thread() + .and_then(|t| kern.thread_process(t)) + .map(|pid| kern.platform_mut().page_table_for(pid)) + .unwrap_or(0); + if root != 0 && crate::mmu::active_root() != root { + crate::mmu::activate(crate::mmu::page_table::table_at(root)); + } +} + +/// Leaves the kernel for userspace for the first time on this hart, with +/// the kernel lock held: runs the thread the kernel already made current +/// here (init, on the boot hart) or the next one homed here. +pub fn enter_first() -> ! { + let mut frame = TrapFrame { + regs: [0; 32], + sepc: 0, + sstatus: 0, + }; + match kern().current_thread() { + Some(tid) => switch_to_thread(&mut frame, None, tid), + None => switch_to_next(&mut frame, None), + } + finish(&mut frame); + crate::smp::unlock(); + // SAFETY: `frame` holds a user context; sscratch is set from it on + // the way out, so later traps land at the top of this frame. + unsafe { trap_return(&mut frame) } +} + #[unsafe(no_mangle)] extern "C" fn trap_handler_rust(frame: &mut TrapFrame) { let scause: usize; unsafe { core::arch::asm!("csrr {0}, scause", out(reg) scause) }; + if frame.sstatus & SSTATUS_SPP != 0 { + // Checked before taking the lock: this hart may already hold it. + kernel_fault(frame, scause); + } + crate::smp::service_ipi(); + crate::smp::lock(); if scause & CAUSE_INTERRUPT_BIT != 0 { handle_interrupt(frame, scause & !CAUSE_INTERRUPT_BIT); } else { handle_exception(frame, scause); } + finish(frame); + crate::smp::unlock(); } fn handle_interrupt(frame: &mut TrapFrame, cause: usize) { match cause { CAUSE_S_TIMER => { crate::hw::timer::handle_interrupt(); - let kern = unsafe { &mut *crate::kernel_ptr() }; - let core = types::id::CoreId::from_raw(0); + let kern = kern(); if let Some(current_tid) = kern.current_thread() { - let decision = kern.handle_tick(core); + let decision = kern.handle_tick(this_core()); if let domain::scheduler::SchedulerDecision::Preempt(next) = decision { - switch_to_thread(frame, current_tid, next); + switch_to_thread(frame, Some(current_tid), next); } } } CAUSE_S_EXTERNAL => dispatch_external(), + // Serviced on entry; any reschedule happens in `finish`. + CAUSE_S_SOFTWARE => {} _ => {} } } +/// The kernel itself faulted (a W^X violation, a stack guard, a bad +/// pointer). Its state can no longer be trusted, and no user thread is to +/// blame. +fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! { + let uart = crate::hw::uart::Uart::new(); + uart.write_str("KERNEL FAULT: scause="); + uart.write_hex(scause); + let stval: usize; + unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) }; + uart.write_str(" stval="); + uart.write_hex(stval); + uart.write_str(" sepc="); + uart.write_hex(frame.sepc); + uart.write_str(" hart="); + uart.write_dec(crate::smp::hart_id()); + uart.put_char(b'\n'); + crate::halt(&uart, "fault in kernel mode"); +} + fn handle_exception(frame: &mut TrapFrame, scause: usize) { match scause { CAUSE_ECALL_U => { @@ -324,48 +385,24 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) { return; }; { - let kern = unsafe { &mut *crate::kernel_ptr() }; + let kern = kern(); let current_tid = kern.current_thread(); let outcome = kern.handle_syscall(syscall); match outcome { kernel::SyscallOutcome::Complete(result) => write_result(frame, &result), - kernel::SyscallOutcome::BlockCaller(_) => { - if let Some(tid) = current_tid { - switch_to_next(frame, tid); - } - } + kernel::SyscallOutcome::BlockCaller(_) => switch_to_next(frame, current_tid), kernel::SyscallOutcome::SwitchTo(next) => { - if let Some(tid) = current_tid { - switch_to_thread(frame, tid, next); - } + switch_to_thread(frame, current_tid, next) } kernel::SyscallOutcome::TerminateCaller => { let uart = crate::hw::uart::Uart::new(); uart.write_str("k-os: process exited\n"); - if let Some(tid) = current_tid { - switch_to_next(frame, tid); - } + switch_to_next(frame, current_tid); } } } } - _ if frame.sstatus & SSTATUS_SPP != 0 => { - // The kernel itself faulted (a W^X violation, the stack guard, - // a bad pointer). Its state can no longer be trusted, and no - // user thread is to blame. - let uart = crate::hw::uart::Uart::new(); - uart.write_str("KERNEL FAULT: scause="); - uart.write_hex(scause); - let stval: usize; - unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) }; - uart.write_str(" stval="); - uart.write_hex(stval); - uart.write_str(" sepc="); - uart.write_hex(frame.sepc); - uart.put_char(b'\n'); - crate::halt(&uart, "fault in kernel mode"); - } _ => { let uart = crate::hw::uart::Uart::new(); uart.write_str("FAULT: scause="); @@ -378,7 +415,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) { uart.write_hex(frame.sepc); uart.put_char(b'\n'); - let kern = unsafe { &mut *crate::kernel_ptr() }; + let kern = kern(); if let Some(current_tid) = kern.current_thread() { let fault = match scause { 2 => domain::fault::Fault::IllegalInstruction, @@ -388,12 +425,10 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) { _ => domain::fault::Fault::InvalidSyscall, }; kern.handle_fault(current_tid, fault, frame.sepc); - switch_to_next(frame, current_tid); - } else { - loop { - unsafe { core::arch::asm!("wfi") }; - } + switch_to_next(frame, Some(current_tid)); } + // With no current thread (another hart just destroyed it), + // `finish` moves on to the next one. } } } diff --git a/crates/domain/src/scheduler/multi_core.rs b/crates/domain/src/scheduler/multi_core.rs index 7c99d56..e8a218e 100644 --- a/crates/domain/src/scheduler/multi_core.rs +++ b/crates/domain/src/scheduler/multi_core.rs @@ -47,6 +47,23 @@ impl MultiCoreScheduler { self.placements.get(thread.raw() as usize)?.as_ref() } + pub fn num_cores(&self) -> u8 { + self.num_cores + } + + /// The highest priority waiting in `core`'s run queue. + pub fn highest_waiting(&self, core: CoreId) -> Option { + self.core(core)?.highest_waiting_priority() + } + + /// The core whose run queue holds `thread`, if any. + pub fn queued_on(&self, thread: ThreadId) -> Option { + self.active() + .iter() + .find(|c| c.is_queued(thread)) + .map(|c| c.core()) + } + /// Whether `thread` waits in some core's run queue. pub fn is_queued(&self, thread: ThreadId) -> bool { self.active().iter().any(|c| c.is_queued(thread)) diff --git a/crates/domain/src/thread/mod.rs b/crates/domain/src/thread/mod.rs index 56e5af7..79676ba 100644 --- a/crates/domain/src/thread/mod.rs +++ b/crates/domain/src/thread/mod.rs @@ -306,6 +306,14 @@ impl ThreadTable { Ok(id) } + /// Every live thread with its id. + pub fn iter(&self) -> impl Iterator { + self.threads + .iter() + .enumerate() + .filter_map(|(i, t)| Some((ThreadId::from_raw(i as u16), t.as_ref()?))) + } + pub fn get(&self, id: ThreadId) -> Result<&Thread, KernelError> { let idx = id.raw() as usize; self.threads diff --git a/crates/kernel/src/lib.rs b/crates/kernel/src/lib.rs index e4426c5..049324b 100644 --- a/crates/kernel/src/lib.rs +++ b/crates/kernel/src/lib.rs @@ -21,6 +21,7 @@ pub mod interrupt; mod memory; #[cfg(kani)] pub mod proofs; +mod smp; pub mod syscall; pub use board::BoardDevice; @@ -65,6 +66,8 @@ pub struct Kernel { notifications: NotificationTable, ipc: [syscall::IpcState; MAX_THREADS], pending_results: [Option; MAX_THREADS], + /// The core each thread runs on (see smp.rs). + homes: [types::id::CoreId; MAX_THREADS], } impl Kernel

{ @@ -83,6 +86,7 @@ impl Kernel

{ notifications: NotificationTable::new(), ipc: [syscall::IpcState::default(); MAX_THREADS], pending_results: [const { None }; MAX_THREADS], + homes: [types::id::CoreId::from_raw(0); MAX_THREADS], } } @@ -115,6 +119,7 @@ impl Kernel

{ addr_of_mut!((*slot).notifications).write(NotificationTable::new()); addr_of_mut!((*slot).ipc).write([syscall::IpcState::default(); MAX_THREADS]); addr_of_mut!((*slot).pending_results).write([const { None }; MAX_THREADS]); + addr_of_mut!((*slot).homes).write([types::id::CoreId::from_raw(0); MAX_THREADS]); } } @@ -142,6 +147,7 @@ impl Kernel

{ .insert(cap)?; let core = self.platform.core_id(); + self.set_home(tid, core); self.scheduler.add(tid, priority, core); self.scheduler.next(core); self.threads.get_mut(tid)?.set_running(core)?; @@ -211,14 +217,13 @@ impl Kernel

{ } pub fn spawn_thread_internal(&mut self, pid: ProcessId) -> Option { - use domain::scheduler::Scheduler; let tid = self .threads .spawn(pid, Priority::DEFAULT, TimeSlice::DEFAULT, None) .ok()?; let _ = self.processes.get_mut(pid).map(|p| p.add_thread(tid)); - let core = self.platform.core_id(); - self.scheduler.add(tid, Priority::DEFAULT, core); + self.place(tid); + self.make_ready(tid, Priority::DEFAULT); Some(tid) } @@ -240,31 +245,6 @@ impl Kernel

{ self.grant_cap(to, resource, permissions) } - pub fn switch_to_thread(&mut self, tid: ThreadId) { - use domain::scheduler::Scheduler; - let core = self.platform.core_id(); - - if let Some(current) = self.scheduler.current(core) { - if current == tid { - return; - } - if let Ok(t) = self.threads.get_mut(current) - && matches!(t.state(), ThreadState::Running(_)) - { - let priority = t.priority(); - let _ = t.preempt(); - self.scheduler.add(current, priority, core); - } - } - - if let Ok(t) = self.threads.get(tid) - && matches!(t.state(), ThreadState::Ready) - && self.scheduler.run_now(core, tid) - { - let _ = self.threads.get_mut(tid).map(|t| t.set_running(core)); - } - } - fn caller(&self) -> Result<(ThreadId, ProcessId), KernelError> { let core = self.platform.core_id(); let tid = self @@ -311,6 +291,7 @@ impl Kernel

{ if let Some(caller) = ipc.reply_to { self.release_caller(caller); } + self.evict(*tid); self.scheduler.remove(*tid); self.set_pending_result(*tid, None); let _ = self.threads.destroy(*tid); diff --git a/crates/kernel/src/smp.rs b/crates/kernel/src/smp.rs new file mode 100644 index 0000000..f454ba5 --- /dev/null +++ b/crates/kernel/src/smp.rs @@ -0,0 +1,119 @@ +//! Cores. Every thread has a home core, chosen when it is created, and +//! only ever runs there: it waits in that core's run queue and is never +//! migrated. Making a thread ready from another core sends that core an +//! IPI when it has to look at its queue again (it is idle, or the thread +//! outranks what it is running), and so does taking away the thread a +//! core is running. + +use domain::{ + scheduler::{Scheduler, SchedulerDecision}, + thread::ThreadState, +}; +use platform::Platform; +use types::{ + bounded::Priority, + id::{CoreId, ThreadId}, +}; + +use crate::Kernel; + +impl Kernel

{ + /// The core `tid` runs on. + pub fn home(&self, tid: ThreadId) -> CoreId { + self.homes + .get(tid.raw() as usize) + .copied() + .unwrap_or(CoreId::from_raw(0)) + } + + /// Homes a new thread on `core`. + pub(crate) fn set_home(&mut self, tid: ThreadId, core: CoreId) { + if let Some(h) = self.homes.get_mut(tid.raw() as usize) { + *h = core; + } + } + + /// Homes a new thread on the core with the fewest live threads. + pub(crate) fn place(&mut self, tid: ThreadId) { + let cores = self.scheduler.num_cores().max(1); + let load = |core: CoreId| { + self.threads + .iter() + .filter(|(t, _)| *t != tid && self.home(*t) == core) + .count() + }; + let best = (0..cores) + .map(CoreId::from_raw) + .min_by_key(|c| load(*c)) + .unwrap_or(CoreId::from_raw(0)); + self.set_home(tid, best); + } + + /// Queues a ready thread on its home core, and kicks that core if it + /// is another one that should look at its queue now. + pub(crate) fn make_ready(&mut self, tid: ThreadId, priority: Priority) { + let home = self.home(tid); + self.scheduler.add(tid, priority, home); + if home == self.platform.core_id() { + return; + } + let outranks = match self.scheduler.current(home) { + None => true, + Some(running) => self + .threads + .get(running) + .is_ok_and(|t| priority > t.priority()), + }; + if outranks { + self.platform.send_ipi(home); + } + } + + /// Before `tid` is destroyed: if another core is running it, that core + /// must stop and pick something else. + pub(crate) fn evict(&mut self, tid: ThreadId) { + if let Ok(ThreadState::Running(core)) = self.threads.get(tid).map(|t| *t.state()) + && core != self.platform.core_id() + { + self.platform.send_ipi(core); + } + } + + /// What `core` should do after an IPI: keep running its thread, or + /// switch to a higher-priority one now waiting. When it has no thread + /// (it was idle, or its thread was destroyed) the caller picks the + /// next one itself. + pub fn reschedule(&mut self, core: CoreId) -> SchedulerDecision { + let Some(current) = self.scheduler.current(core) else { + return SchedulerDecision::Continue; + }; + let Ok(thread) = self.threads.get(current) else { + return SchedulerDecision::Continue; + }; + let priority = thread.priority(); + if self + .scheduler + .highest_waiting(core) + .is_none_or(|p| p <= priority) + { + return SchedulerDecision::Continue; + } + let Some(next) = self.scheduler.next(core) else { + return SchedulerDecision::Continue; + }; + if self + .threads + .get_mut(next) + .and_then(|t| t.set_running(core)) + .is_err() + { + return SchedulerDecision::Continue; + } + if let Ok(t) = self.threads.get_mut(current) + && t.preempt().is_ok() + { + self.scheduler.add(current, priority, core); + } + SchedulerDecision::Preempt(next) + } +} diff --git a/crates/kernel/src/syscall/ipc.rs b/crates/kernel/src/syscall/ipc.rs index b571bcb..a71344b 100644 --- a/crates/kernel/src/syscall/ipc.rs +++ b/crates/kernel/src/syscall/ipc.rs @@ -102,8 +102,7 @@ impl Kernel

{ let priority = thread.priority(); if thread.unblock().is_ok() { self.set_pending_result(tid, Some(result)); - let core = self.platform.core_id(); - self.scheduler.add(tid, priority, core); + self.make_ready(tid, priority); } } @@ -234,6 +233,8 @@ impl Kernel

{ } self.block_caller(tid, BlockReason::AwaitingReply); // Run the server straight away instead of waiting for its turn. + // Only works if it lives on this core (run_now finds it in this + // core's queue only); otherwise make_ready kicked its core. let core = self.platform.core_id(); if self.scheduler.run_now(core, receiver) && self diff --git a/crates/kernel/src/syscall/lifecycle.rs b/crates/kernel/src/syscall/lifecycle.rs index 2279160..54829ac 100644 --- a/crates/kernel/src/syscall/lifecycle.rs +++ b/crates/kernel/src/syscall/lifecycle.rs @@ -10,7 +10,6 @@ impl Kernel

{ entry: types::address::EntryPoint, stack: types::address::StackPointer, ) -> SyscallOutcome { - use domain::scheduler::Scheduler; use domain::thread::SavedContext; let priority = types::bounded::Priority::DEFAULT; @@ -30,8 +29,8 @@ impl Kernel

{ return SyscallOutcome::Complete(SyscallResult::Error(e)); } - let core = self.platform.core_id(); - self.scheduler.add(tid, priority, core); + self.place(tid); + self.make_ready(tid, priority); SyscallOutcome::Complete(SyscallResult::Value(tid.raw() as usize)) } @@ -89,7 +88,6 @@ impl Kernel

{ entry: types::address::EntryPoint, stack: types::address::StackPointer, ) -> SyscallOutcome { - use domain::scheduler::Scheduler; use domain::thread::SavedContext; let target_pid = match self.check_cap( @@ -132,8 +130,8 @@ impl Kernel

{ return SyscallOutcome::Complete(SyscallResult::Error(e)); } - let core = self.platform.core_id(); - self.scheduler.add(tid, priority, core); + self.place(tid); + self.make_ready(tid, priority); SyscallOutcome::Complete(SyscallResult::Value(tid.raw() as usize)) } diff --git a/crates/kernel/src/tests/helpers.rs b/crates/kernel/src/tests/helpers.rs index 6170ce9..b93878d 100644 --- a/crates/kernel/src/tests/helpers.rs +++ b/crates/kernel/src/tests/helpers.rs @@ -37,6 +37,7 @@ impl Kernel { .expect("spawn"); let _ = self.processes.get_mut(pid).map(|p| p.add_thread(tid)); let core = self.platform.core_id(); + self.set_home(tid, core); self.scheduler.add(tid, priority, core); tid } @@ -54,9 +55,21 @@ impl Kernel { pub fn switch_to_thread_for_test(&mut self, tid: ThreadId) { let core = self.platform.core_id(); + assert_eq!( + self.home(tid), + core, + "a thread only ever runs on its home core" + ); + // Only a thread ready in this core's queue can be switched to; + // otherwise leave the core as it is. + let switchable = self.scheduler.queued_on(tid) == Some(core) + && self + .threads + .get(tid) + .is_ok_and(|t| matches!(t.state(), ThreadState::Ready)); if let Some(current) = self.scheduler.current(core) { - if current == tid { + if current == tid || !switchable { return; } if let Ok(t) = self.threads.get_mut(current) diff --git a/crates/kernel/src/tests/mock_platform.rs b/crates/kernel/src/tests/mock_platform.rs index c12e072..1b65399 100644 --- a/crates/kernel/src/tests/mock_platform.rs +++ b/crates/kernel/src/tests/mock_platform.rs @@ -37,6 +37,8 @@ pub struct MockPlatform { mappings: [Option; MAX_MAPPINGS], tables: [Option; MAX_TABLES], zeroed: [Option; MAX_ZEROED], + /// Cores an IPI was sent to and that have not taken it yet. + ipis: [bool; 8], } impl MockPlatform { @@ -49,9 +51,27 @@ impl MockPlatform { mappings: [None; MAX_MAPPINGS], tables: [None; MAX_TABLES], zeroed: [None; MAX_ZEROED], + ipis: [false; 8], } } + /// Runs the kernel as if on `core` from now on. + pub fn set_core(&mut self, core: CoreId) { + self.core = core; + } + + pub fn ipi_pending(&self, core: CoreId) -> bool { + self.ipis.get(core.raw() as usize).copied().unwrap_or(false) + } + + /// Delivers `core`'s pending IPI: true if there was one. + pub fn take_ipi(&mut self, core: CoreId) -> bool { + self.ipis + .get_mut(core.raw() as usize) + .map(core::mem::take) + .unwrap_or(false) + } + /// Physical address `va` is mapped to in `process`, as the "hardware" sees it. pub fn mapping(&self, process: ProcessId, va: usize) -> Option { self.mappings @@ -207,7 +227,12 @@ impl Platform for MockPlatform { } fn mask_irq(&mut self, _irq: IrqId) {} fn unmask_irq(&mut self, _irq: IrqId) {} - fn send_ipi(&mut self, _target: CoreId) {} + fn send_ipi(&mut self, target: CoreId) { + assert_ne!(target, self.core, "a core never IPIs itself"); + if let Some(p) = self.ipis.get_mut(target.raw() as usize) { + *p = true; + } + } fn core_id(&self) -> CoreId { self.core } diff --git a/crates/kernel/src/tests/mod.rs b/crates/kernel/src/tests/mod.rs index bd866ea..1c29cf2 100644 --- a/crates/kernel/src/tests/mod.rs +++ b/crates/kernel/src/tests/mod.rs @@ -15,6 +15,7 @@ mod lifetimes; mod memory; mod revocation; mod simple; +mod smp; mod storm; mod tick; diff --git a/crates/kernel/src/tests/smp.rs b/crates/kernel/src/tests/smp.rs new file mode 100644 index 0000000..b76d361 --- /dev/null +++ b/crates/kernel/src/tests/smp.rs @@ -0,0 +1,178 @@ +extern crate std; + +use super::*; +use crate::{SyscallOutcome, SyscallResult}; +use domain::{ + message::Message, + resource::Resource, + scheduler::SchedulerDecision, + syscall::Syscall, + thread::{BlockReason, ThreadState}, +}; +use std::vec::Vec; +use types::{ + address::PhysicalAddress, + id::{CoreId, ThreadId}, + permission::{Permission, PermissionSet}, +}; + +fn core(c: u8) -> CoreId { + CoreId::from_raw(c) +} + +/// A booted kernel on `cores` cores, init running on core 0. +fn smp(cores: u8) -> (Kernel, ProcessId) { + let mut k = Kernel::new( + MockPlatform::new(cores), + PhysicalAddress::from_raw(0x8400_0000), + 1024, + ); + let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap(); + (k, init) +} + +/// Acts as `c` from now on, running the next thread there if it is idle. +fn on(k: &mut Kernel, c: CoreId) { + k.platform_mut().set_core(c); + if k.current_thread().is_none() + && let Some(next) = k.pick_next_thread() + { + k.set_thread_running(next, c).unwrap(); + } +} + +fn state(k: &Kernel, t: ThreadId) -> Option { + k.thread_state_for_test(t) +} + +#[test] +fn new_threads_are_spread_over_the_cores() { + let (mut k, init) = smp(4); + let homes: Vec = (0..3) + .map(|_| { + let t = k.spawn_thread_internal(init).unwrap(); + k.home(t) + }) + .collect(); + assert_eq!( + homes, + [core(1), core(2), core(3)], + "init already has core 0" + ); +} + +#[test] +fn a_thread_made_ready_for_an_idle_core_kicks_it() { + let (mut k, init) = smp(2); + let t = k.spawn_thread_internal(init).unwrap(); + assert_eq!(k.home(t), core(1)); + assert!(k.platform_mut().ipi_pending(core(1))); + assert!(!k.platform_mut().ipi_pending(core(0))); +} + +#[test] +fn a_thread_made_ready_on_this_core_sends_no_ipi() { + let (mut k, init) = smp(1); + k.spawn_thread_internal(init).unwrap(); + assert!(!k.platform_mut().ipi_pending(core(0))); +} + +#[test] +fn a_busy_core_is_not_kicked_for_an_equal_priority_thread() { + let (mut k, init) = smp(2); + let a = k.spawn_thread_internal(init).unwrap(); + on(&mut k, core(1)); + assert_eq!(state(&k, a), Some(ThreadState::Running(core(1)))); + k.platform_mut().take_ipi(core(1)); + k.platform_mut().set_core(core(0)); + // A second thread homed on core 1 while it is busy at the same priority. + let b = k.spawn_thread_internal(init).unwrap(); + let c = k.spawn_thread_internal(init).unwrap(); + let on_one = if k.home(b) == core(1) { b } else { c }; + assert_eq!(k.home(on_one), core(1)); + assert!(!k.platform_mut().ipi_pending(core(1))); +} + +#[test] +fn killing_a_thread_running_on_another_core_stops_that_core() { + let (mut k, init) = smp(2); + let child = k.create_process_internal().unwrap(); + let t = k.spawn_thread_internal(child).unwrap(); + on(&mut k, core(1)); + k.platform_mut().take_ipi(core(1)); + assert_eq!(state(&k, t), Some(ThreadState::Running(core(1)))); + + k.platform_mut().set_core(core(0)); + let cap = k.grant_cap_for_test( + init, + Resource::Process(child), + PermissionSet::empty().with(Permission::Execute), + ); + let out = k.handle_syscall(Syscall::Kill { process: cap }); + assert!(matches!(out, SyscallOutcome::Complete(SyscallResult::Ok))); + assert!(k.platform_mut().ipi_pending(core(1)), "core 1 never told"); + assert_eq!(k.scheduler.current(core(1)), None); + k.check_invariants().unwrap(); +} + +#[test] +fn a_call_to_a_server_on_another_core_does_not_pull_it_over() { + let (mut k, init) = smp(2); + let ep = k.create_endpoint(init).unwrap(); + let all = PermissionSet::empty() + .with(Permission::Read) + .with(Permission::Execute); + let slot = k.grant_cap_for_test(init, Resource::Endpoint(ep), all); + let server = k.spawn_thread_internal(init).unwrap(); + assert_eq!(k.home(server), core(1)); + + on(&mut k, core(1)); + k.platform_mut().take_ipi(core(1)); + let out = k.handle_syscall(Syscall::Receive { endpoint: slot }); + assert!(matches!(out, SyscallOutcome::BlockCaller(_))); + k.platform_mut().set_core(core(0)); + + let out = k.handle_syscall(Syscall::Call { + endpoint: slot, + message: Message::empty(), + }); + assert!( + matches!(out, SyscallOutcome::BlockCaller(BlockReason::AwaitingReply)), + "no direct switch across cores" + ); + assert_eq!(state(&k, server), Some(ThreadState::Ready)); + assert!(k.platform_mut().ipi_pending(core(1))); + k.check_invariants().unwrap(); +} + +#[test] +fn reschedule_switches_only_for_a_higher_priority_thread() { + let (mut k, init) = smp(2); + let a = k.spawn_thread_internal(init).unwrap(); + on(&mut k, core(1)); + assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue)); + + // An equal-priority one waits its turn (the tick handles that). + let peer = k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT); + assert_eq!(k.home(peer), core(1)); + assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue)); + + // A higher-priority thread arrives on core 1's queue. + let p = Priority::new(Priority::DEFAULT.value() + 1).unwrap(); + let urgent = k.spawn_thread_for_test(init, p, TimeSlice::DEFAULT); + assert_eq!(k.home(urgent), core(1)); + + assert!(matches!( + k.reschedule(core(1)), + SchedulerDecision::Preempt(t) if t == urgent + )); + assert_eq!(state(&k, urgent), Some(ThreadState::Running(core(1)))); + assert_eq!(state(&k, a), Some(ThreadState::Ready)); +} + +#[test] +fn reschedule_on_an_idle_core_leaves_picking_to_the_caller() { + let (mut k, _) = smp(2); + k.platform_mut().set_core(core(1)); + assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue)); +} diff --git a/crates/kernel/src/tests/storm.rs b/crates/kernel/src/tests/storm.rs index 8bfe1da..991aae4 100644 --- a/crates/kernel/src/tests/storm.rs +++ b/crates/kernel/src/tests/storm.rs @@ -208,6 +208,20 @@ impl Kernel { if self.scheduler.is_queued(t) != matches!(state, ThreadState::Ready) { return Err(format!("{t:?} in {state:?} has wrong run-queue membership")); } + // ---- cores: a thread runs and waits only on its home core ---- + let home = self.home(t); + match state { + ThreadState::Running(c) if c != home => { + return Err(format!("{t:?} runs on {c:?}, its home is {home:?}")); + } + ThreadState::Running(c) if self.scheduler.current(c) != Some(t) => { + return Err(format!("{t:?} Running on {c:?} but not its current thread")); + } + ThreadState::Ready if self.scheduler.queued_on(t) != Some(home) => { + return Err(format!("{t:?} queued away from its home {home:?}")); + } + _ => {} + } if let Some(r) = ipc.reply_to && self.threads.generation(r.tid) == Some(r.generation) && self.threads.get(r.tid).ok().map(|x| *x.state()) @@ -216,6 +230,22 @@ impl Kernel { return Err(format!("{t:?} holds a reply for a thread not waiting")); } } + for c in 0..self.scheduler.num_cores() { + let core = types::id::CoreId::from_raw(c); + let current = self.scheduler.current(core); + if let Some(t) = current + && self.threads.get(t).ok().map(|x| *x.state()) != Some(ThreadState::Running(core)) + { + return Err(format!("{core:?}'s current {t:?} is not running there")); + } + // No lost wakeups: an idle core with work waiting has been told. + if current.is_none() + && self.scheduler.highest_waiting(core).is_some() + && !self.platform.ipi_pending(core) + { + return Err(format!("{core:?} idles with ready threads and no IPI")); + } + } Ok(()) } } @@ -282,6 +312,10 @@ fn new_round(k: &mut Kernel, init: ProcessId, init_thread: ThreadI .create_pool_region(hub, types::bounded::PageCount::new(2).unwrap()) .ok(); let mut members = std::vec![init]; + // CreateProcess below runs as init, so on init's core. + let init_core = k.home(init_thread); + k.platform_mut().set_core(init_core); + resume(k, init_core); for _ in 0..2 { k.switch_to_thread_for_test(init_thread); if let crate::SyscallOutcome::Complete(crate::SyscallResult::Slot(cap)) = @@ -292,7 +326,7 @@ fn new_round(k: &mut Kernel, init: ProcessId, init_thread: ThreadI { let pid = k.process_of_cap_for_test(cap); for _ in 0..2 { - k.spawn_thread_for_test(pid, Priority::DEFAULT, TimeSlice::DEFAULT); + k.spawn_thread_internal(pid); } members.push(pid); } else if std::env::var("STORM_DEBUG").is_ok() { @@ -326,11 +360,40 @@ fn region_slots(k: &Kernel, pid: Option) -> std::vec::V .collect() } +/// What the adapter does whenever a core is left without a thread: run the +/// next one in its queue, if any. +fn resume(k: &mut Kernel, core: types::id::CoreId) { + if k.current_thread().is_none() + && let Some(next) = k.pick_next_thread() + { + k.set_thread_running(next, core).unwrap(); + } +} + #[test] fn the_kernel_survives_a_syscall_storm() { + storm(1); +} + +/// The same storm with threads homed on four cores. Each step runs on one +/// core, as the thread that core is running; the kernel is still entered +/// one call at a time (stage 1 takes a single lock), but threads on other +/// cores are live meanwhile and IPIs arrive late. +#[test] +fn the_kernel_survives_a_syscall_storm_on_four_cores() { + storm(4); +} + +fn storm(cores: u8) { + use types::id::CoreId; let mut total = [0usize; 36 + 1]; for seed in [17u64, 0xABCD_EF01, 4242, 0x1357_9BDF, 8] { - let (mut k, init) = kernel_with_process(); + let mut k = Kernel::new( + MockPlatform::new(cores), + types::address::PhysicalAddress::from_raw(0x8400_0000), + 1024, + ); + let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap(); let init_thread = k.current_thread().unwrap(); k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT); let mut rng = Rng(seed); @@ -360,13 +423,35 @@ fn the_kernel_survives_a_syscall_storm() { // init may be blocked on objects it made itself. k.destroy_endpoints_owned_by(init); k.destroy_notifications_owned_by(init); + let here = k.platform_mut().core_id(); + resume(&mut k, here); k.check_invariants() .unwrap_or_else(|e| panic!("seed {seed:#x} step {step} after teardown: {e}")); new_round(&mut k, init, init_thread); rounds += 1; continue; } - let tid = runnable[rng.below(runnable.len())]; + // Pick a core and act as the thread it runs, after it takes any + // IPI that has reached it. Now and then switch it to another + // ready thread of its own, as a tick or a yield would. + let core = CoreId::from_raw(rng.below(cores as usize) as u8); + k.platform_mut().set_core(core); + if k.platform_mut().take_ipi(core) { + let _ = k.reschedule(core); + } + resume(&mut k, core); + let homed: std::vec::Vec = runnable + .iter() + .copied() + .filter(|t| k.home(*t) == core) + .collect(); + if homed.is_empty() { + continue; + } + let tid = match k.current_thread() { + Some(t) if rng.below(3) != 0 => t, + _ => homed[rng.below(homed.len())], + }; let is_init = k.thread_process(tid) == Some(init); let nr = rng.below(36 + 1); // init exiting ends the test. Halt stays in: nobody here holds @@ -399,6 +484,7 @@ fn the_kernel_survives_a_syscall_storm() { k.switch_to_thread_for_test(tid); k.take_pending_result(tid); let out = k.handle_syscall(syscall); + resume(&mut k, core); executed += 1; if !matches!( out, diff --git a/docs/ADAPTERS.md b/docs/ADAPTERS.md index 93f4aad..08d6811 100644 --- a/docs/ADAPTERS.md +++ b/docs/ADAPTERS.md @@ -21,9 +21,10 @@ This generates `crates/adapter-/` with the standard structure. Then: the final kernel table may be mapped in the lower half. Keep text, read-only data and writable data on separate pages and export their page-aligned boundaries (`_text_start`, `_text_end`, - `_rodata_end`, `_stack_guard`, `_stack_bottom`): the kernel maps each - range with only the access it needs (W^X) and leaves a guard page under - the stack. + `_rodata_end`, `_stacks_start`, `_stacks_end`): the kernel maps each + range with only the access it needs (W^X). The stacks area holds one + slot per hart, a guard page (left unmapped) then that hart's kernel + stack; the boot stub picks the slot from the hart id. 5. Implement the TODOs in `hw/`, `mmu/`, `trap/`, and `platform.rs` ## Structure