feat: SMP stage 1 (4 harts, per-hart guarded stacks, home-core affinity, IPIs, TLB shootdown, one temporary kernel lock), 4-core storm

Makefile

@@ -2,6 +2,8 @@
 
 RISCV_TARGET = riscv64gc-unknown-none-elf
 KERNEL_BIN = target/$(RISCV_TARGET)/release/adapter-riscv
+# Harts QEMU boots; `make run-gui SMP=1` for a single core.
+SMP ?= 4
 
 KERNEL_CRATES = -p types -p domain -p kernel -p platform
 
@@ -56,17 +58,17 @@ image: build
 	scripts/make-disk-image.sh
 
 run: build
-	qemu-system-riscv64 -machine virt -nographic -bios default \
+	qemu-system-riscv64 -machine virt -smp $(SMP) -nographic -bios default \
 		-kernel $(KERNEL_BIN)
 
 run-disk: image
-	qemu-system-riscv64 -machine virt -nographic -bios default \
+	qemu-system-riscv64 -machine virt -smp $(SMP) -nographic -bios default \
 		-kernel $(KERNEL_BIN) \
 		-drive file=boot.img,if=none,format=raw,id=hd0 \
 		-device virtio-blk-device,drive=hd0
 
 run-gui: image
-	qemu-system-riscv64 -machine virt -bios default \
+	qemu-system-riscv64 -machine virt -smp $(SMP) -bios default \
 		-kernel $(KERNEL_BIN) \
 		-drive file=boot.img,if=none,format=raw,id=hd0 \
 		-device virtio-blk-device,drive=hd0 \

crates/adapter-riscv/linker.ld

@@ -46,15 +46,13 @@ SECTIONS {
         _bss_end = .;
     }
 
-    /* Left unmapped, so overflowing the kernel stack faults instead of
+    /* One kernel stack per hart (smp::MAX_HARTS, smp::STACK_SLOT): each
-       silently overwriting .bss. */
+       slot is an unmapped guard page, so overflowing a stack faults
+       instead of silently overwriting what lies below, then the stack. */
     . = ALIGN(4096);
-    _stack_guard = .;
+    _stacks_start = .;
-    . += 4096;
+    . += 8 * (4096 + 0x80000);
-
+    _stacks_end = .;
-    _stack_bottom = .;
-    . += 0x80000;
-    _stack_top = .;
 
     _heap_start = .;
 }

crates/adapter-riscv/src/boot/hardware.rs

@@ -8,12 +8,21 @@ unsafe extern "C" {
     static _heap_start: u8;
 }
 
-pub fn init(uart: &Uart) -> &'static mut mmu::PageTable {
+pub fn init(uart: &Uart, boot_hart: usize) -> &'static mut mmu::PageTable {
     zero_bss();
 
     let heap_start = unsafe { &_heap_start as *const u8 as usize };
     mmu::init_allocator(heap_start);
     let kernel_pt = mmu::init_kernel_page_table();
+    let harts = crate::smp::probe_harts();
+    if boot_hart >= harts {
+        crate::halt(uart, "boot hart outside the hart range");
+    }
+    crate::smp::init(
+        boot_hart,
+        harts,
+        mmu::virt_to_phys(kernel_pt as *const mmu::PageTable as usize),
+    );
     mmu::activate(kernel_pt);
     uart.write_str("k-os: MMU enabled (Sv39)\n");
     match mmu::check_wx(kernel_pt) {
@@ -46,7 +55,8 @@ pub fn init(uart: &Uart) -> &'static mut mmu::PageTable {
     if board::USER_POOL_PAGES > domain::memory::frame::MAX_FRAMES {
         crate::halt(uart, "user pool larger than the frame table");
     }
-    let riscv_platform = platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize);
+    let riscv_platform =
+        platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize, harts as u8);
     // SAFETY: `init_in_place` writes every field of the kernel.
     unsafe {
         KERNEL.init_with(|slot| {

crates/adapter-riscv/src/boot/loader.rs

@@ -13,7 +13,6 @@ const MAX_FILE_SIZE: usize = 64 * 1024;
 
 pub struct LoadedProcess {
     pub pid: ProcessId,
-    pub entry: usize,
 }
 
 fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
@@ -48,7 +47,7 @@ pub fn load_init(uart: &Uart, blk: &mut VirtioBlk) -> (LoadedProcess, ThreadId)
     setup_thread(tid, entry);
     uart.write_str("k-os: init loaded\n");
 
-    (LoadedProcess { pid, entry }, tid)
+    (LoadedProcess { pid }, tid)
 }
 
 pub fn load_server(
@@ -69,35 +68,7 @@ pub fn load_server(
     uart.write_str(label);
     uart.write_str(" loaded\n");
 
-    LoadedProcess { pid, entry }
+    LoadedProcess { pid }
-}
-
-pub fn enter_userspace(pid: ProcessId, entry: usize, shared_buf_va: usize) -> ! {
-    let pt_addr = kern().platform_mut().page_table_for(pid);
-    if pt_addr != 0 {
-        mmu::activate(mmu::page_table::table_at(pt_addr));
-    }
-
-    let kernel_sp: usize;
-    unsafe { core::arch::asm!("mv {0}, sp", out(reg) kernel_sp) };
-    crate::trap::set_kernel_stack(kernel_sp);
-
-    unsafe {
-        core::arch::asm!(
-            "csrc sstatus, {spp}",
-            "csrs sstatus, {spie}",
-            "csrw sepc, {entry}",
-            "mv sp, {stack}",
-            "mv a0, {arg}",
-            "sret",
-            spp = in(reg) 1usize << 8,
-            spie = in(reg) 1usize << 5,
-            entry = in(reg) entry,
-            stack = in(reg) USER_VA_BASE,
-            arg = in(reg) shared_buf_va,
-            options(noreturn),
-        );
-    }
 }
 
 fn setup_thread(tid: ThreadId, entry: usize) {

crates/adapter-riscv/src/hw/plic.rs

@@ -2,21 +2,36 @@ use crate::board;
 
 const PLIC: usize = crate::mmu::phys_to_virt(board::PLIC_BASE);
 const PLIC_PRIORITY: usize = PLIC;
-const PLIC_S_ENABLE: usize = PLIC + 0x2080;
+
-const PLIC_S_THRESHOLD: usize = PLIC + 0x201000;
+// External interrupts go to the boot hart only: its supervisor context is
-const PLIC_S_CLAIM: usize = PLIC + 0x201004;
+// 2 * hart + 1 on QEMU virt (even contexts are machine mode).
+fn context() -> usize {
+    2 * crate::smp::boot_hart() + 1
+}
+
+fn enable_base() -> usize {
+    PLIC + 0x2000 + context() * 0x80
+}
+
+fn threshold() -> usize {
+    PLIC + 0x20_0000 + context() * 0x1000
+}
+
+fn claim_reg() -> usize {
+    threshold() + 4
+}
 
 pub struct Plic;
 
 impl Plic {
     pub fn init() {
-        unsafe { core::ptr::write_volatile(PLIC_S_THRESHOLD as *mut u32, 0) };
+        unsafe { core::ptr::write_volatile(threshold() as *mut u32, 0) };
     }
 
     pub fn enable(irq: u32) {
         let word = (irq / 32) as usize;
         let bit = irq % 32;
-        let addr = PLIC_S_ENABLE + word * 4;
+        let addr = enable_base() + word * 4;
         unsafe {
             let current = core::ptr::read_volatile(addr as *const u32);
             core::ptr::write_volatile(addr as *mut u32, current | (1 << bit));
@@ -28,7 +43,7 @@ impl Plic {
     pub fn disable(irq: u32) {
         let word = (irq / 32) as usize;
         let bit = irq % 32;
-        let addr = PLIC_S_ENABLE + word * 4;
+        let addr = enable_base() + word * 4;
         unsafe {
             let current = core::ptr::read_volatile(addr as *const u32);
             core::ptr::write_volatile(addr as *mut u32, current & !(1 << bit));
@@ -36,10 +51,10 @@ impl Plic {
     }
 
     pub fn claim() -> u32 {
-        unsafe { core::ptr::read_volatile(PLIC_S_CLAIM as *const u32) }
+        unsafe { core::ptr::read_volatile(claim_reg() as *const u32) }
     }
 
     pub fn complete(irq: u32) {
-        unsafe { core::ptr::write_volatile(PLIC_S_CLAIM as *mut u32, irq) };
+        unsafe { core::ptr::write_volatile(claim_reg() as *mut u32, irq) };
     }
 }

crates/adapter-riscv/src/main.rs

@@ -9,6 +9,7 @@ mod hw;
 mod kernel_cell;
 mod mmu;
 mod platform;
+mod smp;
 mod trap;
 
 use core::arch::global_asm;
@@ -18,8 +19,8 @@ use kernel_cell::KernelCell;
 static KERNEL: KernelCell<kernel::Kernel<platform::RiscVPlatform>> = KernelCell::uninit();
 
 fn kern_mut() -> &'static mut kernel::Kernel<platform::RiscVPlatform> {
-    // SAFETY: single core; boot code runs before any trap can touch the
+    // SAFETY: boot runs with the kernel lock held and before any other
-    // kernel.
+    // hart has started.
     unsafe { &mut *kernel_ptr() }
 }
 
@@ -27,28 +28,55 @@ pub fn kernel_ptr() -> *mut kernel::Kernel<platform::RiscVPlatform> {
     KERNEL.as_ptr()
 }
 
-// Firmware enters `_start` at the physical load address with the MMU off,
+// Firmware enters `_start` (and SBI starts other harts at
-// but the kernel is linked in the upper half. Using only PC-relative code,
+// `_secondary_start`) at the physical load address with the MMU off and
-// the stub turns on `boot_page_table`, which maps RAM twice (at its
+// the hart id in a0, but the kernel is linked in the upper half. Using
-// physical address, just so the next instruction still exists, and in the
+// only PC-relative code, the stub turns on `boot_page_table`, which maps
-// upper half) plus the MMIO window high, then jumps to its own high address.
+// RAM twice (at its physical address, just so the next instruction still
-// kmain replaces this table with the real one, which maps nothing low.
+// exists, and in the upper half) plus the MMIO window high, then jumps to
+// its own high address and takes the hart's kernel stack. The kernel then
+// switches every hart to the real table, which maps nothing low.
 global_asm!(
     r#"
 .section .text.entry
 .globl _start
 _start:
+    lla t2, start_high_address
+    j 1f
+.globl _secondary_start
+_secondary_start:
+    lla t2, secondary_high_address
+1:
+    li t0, 8                # smp::MAX_HARTS
+    bgeu a0, t0, spin
+    mv s0, a0
     lla t0, boot_page_table
     srli t0, t0, 12
     li t1, 8 << 60
     or t0, t0, t1
     csrw satp, t0
     sfence.vma
-    ld t0, start_high_address
+    ld t0, 0(t2)
     jr t0
+
+# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT
 start_high:
-    lla sp, _stack_top
+    call hart_stack
+    mv a0, s0
     call kmain
+    j spin
+secondary_high:
+    call hart_stack
+    mv a0, s0
+    call kmain_secondary
+    j spin
+hart_stack:
+    lla sp, _stacks_start
+    addi t0, s0, 1
+    li t1, 0x81000
+    mul t0, t0, t1
+    add sp, sp, t0
+    ret
 spin:
     wfi
     j spin
@@ -57,6 +85,8 @@ spin:
 .balign 8
 start_high_address:
     .quad start_high
+secondary_high_address:
+    .quad secondary_high
 .popsection
 
 # Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with
@@ -79,15 +109,21 @@ boot_page_table:
 const SHARED_BUF_VA: usize = 0x30000;
 
 #[unsafe(no_mangle)]
-extern "C" fn kmain() -> ! {
+extern "C" fn kmain(hart: usize) -> ! {
     let uart = Uart::new();
     uart.write_str("k-os: booted on RISC-V\n");
 
-    boot::hardware::init(&uart);
+    boot::hardware::init(&uart, hart);
+    // Held until the boot hart first enters userspace; harts started
+    // below wait for it. Not before init: zeroing .bss resets the lock.
+    smp::lock();
     let mut blk =
         boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device"));
 
-    let (init, _) = boot::loader::load_init(&uart, &mut blk);
+    let (init, init_thread) = boot::loader::load_init(&uart, &mut blk);
+    if let Some(ctx) = kern_mut().thread_context_mut(init_thread) {
+        ctx.registers_mut()[10] = SHARED_BUF_VA;
+    }
     let storage = boot::loader::load_server(&uart, &mut blk, b"storage.elf", "storage");
     let display = boot::loader::load_server(&uart, &mut blk, b"display.elf", "display");
     let input = boot::loader::load_server(&uart, &mut blk, b"input.elf", "input");
@@ -106,8 +142,35 @@ extern "C" fn kmain() -> ! {
         .grant_boot_authority(init.pid, &board)
         .unwrap_or_else(|_| halt(&uart, "boot authority failed"));
 
+    start_other_harts(&uart, hart);
     uart.write_str("k-os: entering userspace\n");
-    boot::loader::enter_userspace(init.pid, init.entry, SHARED_BUF_VA);
+    trap::enter_first();
+}
+
+fn start_other_harts(uart: &Uart, boot: usize) {
+    unsafe extern "C" {
+        fn _secondary_start();
+    }
+    // Harts start with the MMU off, so at the entry's physical address.
+    let entry = mmu::virt_to_phys(_secondary_start as *const () as usize);
+    for hart in (0..smp::num_harts()).filter(|h| *h != boot) {
+        if !smp::sbi_hart_start(hart, entry) {
+            halt(uart, "could not start a hart");
+        }
+    }
+    uart.write_str("k-os: ");
+    uart.write_dec(smp::num_harts());
+    uart.write_str(" harts\n");
+}
+
+/// Where every hart but the boot hart starts, on its own stack.
+#[unsafe(no_mangle)]
+extern "C" fn kmain_secondary(_hart: usize) -> ! {
+    trap::init();
+    hw::timer::init();
+    mmu::activate(mmu::page_table::table_at(smp::kernel_root()));
+    smp::lock();
+    trap::enter_first();
 }
 
 pub fn halt(uart: &Uart, msg: &str) -> ! {

crates/adapter-riscv/src/mmu/page_table.rs

@@ -1,6 +1,7 @@
 use super::allocator::alloc_page_table;
 use super::{phys_to_virt, virt_to_phys};
 use crate::board;
+use crate::smp;
 use platform::{FrameSource, MapError};
 use types::address::{PhysicalAddress, VirtualAddress};
 use types::mapping::PagePerms;
@@ -148,20 +149,31 @@ pub fn init_kernel_page_table() -> &'static mut PageTable {
     // The kernel image, then the rest of RAM, each with only the access it
     // needs: code is never writable, data never executable. RAM below the
     // image belongs to the firmware and stays unmapped, as does the guard
-    // page under the kernel stack. Nothing is mapped in the lower half,
+    // page under each hart's kernel stack. Nothing is mapped in the lower
-    // which is userspace's alone.
+    // half, which is userspace's alone.
     let layout = KernelLayout::from_linker();
     let ram_end = phys_to_virt(board::RAM_BASE + board::RAM_SIZE);
+    let stacks = (0..smp::MAX_HARTS).map(|h| {
+        let slot = layout.stacks_start + h * smp::STACK_SLOT;
+        (
+            slot + PAGE_SIZE,
+            slot + smp::STACK_SLOT,
+            Sv39Flags::kernel_rw(),
+        )
+    });
     for (start, end, flags) in [
         (layout.text_start, layout.text_end, Sv39Flags::kernel_rx()),
         (layout.text_end, layout.rodata_end, Sv39Flags::kernel_ro()),
         (
             layout.rodata_end,
-            layout.stack_guard,
+            layout.stacks_start,
             Sv39Flags::kernel_rw(),
         ),
-        (layout.stack_bottom, ram_end, Sv39Flags::kernel_rw()),
+        (layout.stacks_end, ram_end, Sv39Flags::kernel_rw()),
-    ] {
+    ]
+    .into_iter()
+    .chain(stacks)
+    {
         let (va, pa) = direct(virt_to_phys(start));
         map(
             root,
@@ -200,8 +212,8 @@ pub struct KernelLayout {
     pub text_start: usize,
     pub text_end: usize,
     pub rodata_end: usize,
-    pub stack_guard: usize,
+    pub stacks_start: usize,
-    pub stack_bottom: usize,
+    pub stacks_end: usize,
 }
 
 impl KernelLayout {
@@ -210,8 +222,8 @@ impl KernelLayout {
             static _text_start: u8;
             static _text_end: u8;
             static _rodata_end: u8;
-            static _stack_guard: u8;
+            static _stacks_start: u8;
-            static _stack_bottom: u8;
+            static _stacks_end: u8;
         }
         // SAFETY: only the symbols' addresses are taken, never their values.
         unsafe {
@@ -219,8 +231,8 @@ impl KernelLayout {
                 text_start: &_text_start as *const u8 as usize,
                 text_end: &_text_end as *const u8 as usize,
                 rodata_end: &_rodata_end as *const u8 as usize,
-                stack_guard: &_stack_guard as *const u8 as usize,
+                stacks_start: &_stacks_start as *const u8 as usize,
-                stack_bottom: &_stack_bottom as *const u8 as usize,
+                stacks_end: &_stacks_end as *const u8 as usize,
             }
         }
     }
@@ -240,12 +252,15 @@ pub enum WxViolation {
 
 /// Walks every leaf of `root` and checks the kernel layout: nothing is
 /// both writable and executable, only kernel text is executable, the
-/// firmware and the stack guard are unmapped, and nothing at all is mapped
+/// firmware and every stack guard are unmapped, and nothing at all is mapped
 /// in the lower half. Returns the number of executable pages.
 pub fn check_wx(root: &PageTable) -> Result<usize, WxViolation> {
     let layout = KernelLayout::from_linker();
     let text = layout.text_start..layout.text_end;
-    let guard = layout.stack_guard..layout.stack_bottom;
+    let is_guard = |va: usize| {
+        (layout.stacks_start..layout.stacks_end).contains(&va)
+            && (va - layout.stacks_start).is_multiple_of(smp::STACK_SLOT)
+    };
     let firmware = phys_to_virt(board::RAM_BASE)..layout.text_start;
     let mut executable = 0;
     for (va, bits) in leaves(root) {
@@ -261,7 +276,7 @@ pub fn check_wx(root: &PageTable) -> Result<usize, WxViolation> {
             }
             executable += 1;
         }
-        if guard.contains(&va) || firmware.contains(&va) {
+        if is_guard(va) || firmware.contains(&va) {
             return Err(WxViolation::MappedHole(va));
         }
     }
@@ -299,6 +314,7 @@ fn leaves(root: &PageTable) -> impl Iterator<Item = (usize, u64)> + '_ {
 }
 
 pub fn activate(root: &PageTable) {
+    smp::note_active_root(virt_to_phys(root as *const PageTable as usize));
     let satp = (8u64 << 60) | table_ppn(root);
     unsafe {
         core::arch::asm!(

crates/adapter-riscv/src/platform.rs

@@ -10,6 +10,7 @@ use types::{
 use crate::board;
 use crate::hw::{plic::Plic, timer};
 use crate::mmu;
+use crate::smp;
 use crate::trap;
 
 pub struct RiscVPlatform {
@@ -20,9 +21,9 @@ pub struct RiscVPlatform {
 }
 
 impl RiscVPlatform {
-    pub fn new(kernel_pt_addr: usize) -> Self {
+    pub fn new(kernel_pt_addr: usize, num_cores: u8) -> Self {
         Self {
-            num_cores: 1,
+            num_cores,
             kernel_pt: kernel_pt_addr,
             page_tables: [0usize; MAX_PROCESSES],
         }
@@ -63,8 +64,10 @@ impl Platform for RiscVPlatform {
         let Some(root) = self.root(process) else {
             return;
         };
-        // The caller may be tearing down the space we are running on; get
+        // No hart may still translate through it once the kernel frees its
-        // off it before the kernel frees its frames.
+        // frames: other harts leave it (and wait for nothing but that), and
+        // the caller may be tearing down the space we are running on.
+        smp::shoot_down(root);
         if mmu::active_root() == root {
             // SAFETY: the kernel page table lives in the boot heap forever.
             mmu::activate(unsafe { &*(self.kernel_pt as *const mmu::PageTable) });
@@ -95,6 +98,8 @@ impl Platform for RiscVPlatform {
     fn unmap_page(&mut self, process: ProcessId, va: VirtualAddress) {
         if let Some(root) = self.root(process) {
             mmu::unmap_user_page(root, va.raw());
+            // Other harts running this process may still cache it.
+            smp::shoot_down(root);
         }
     }
 
@@ -144,10 +149,12 @@ impl Platform for RiscVPlatform {
         Plic::enable(irq.raw() as u32);
     }
 
-    fn send_ipi(&mut self, _target: CoreId) {}
+    fn send_ipi(&mut self, target: CoreId) {
+        smp::kick(target.raw() as usize);
+    }
 
     fn core_id(&self) -> CoreId {
-        CoreId::from_raw(0)
+        CoreId::from_raw(smp::hart_id() as u8)
     }
 
     fn num_cores(&self) -> u8 {

crates/adapter-riscv/src/smp.rs

@@ -0,0 +1,220 @@
+//! Harts. Each hart has its own kernel stack, and its hart id doubles as
+//! its kernel `CoreId`. The kernel is entered under one lock for now (stage
+//! 1 of multi-core; the kernel itself stays a sequential program), so
+//! userspace runs in parallel and the kernel does not.
+//!
+//! Harts talk through IPIs carrying two kinds of request:
+//! - reschedule: the kernel made work for this hart, or took its thread
+//!   away; handled under the lock on the way out of the kernel.
+//! - leave root R: a page table this hart may be using is changing or about
+//!   to be freed; the hart switches to the kernel's root and flushes its
+//!   TLB. This needs no lock, so it is serviced even while a hart spins
+//!   waiting for the lock or idles, and the sender (which holds the lock)
+//!   can wait for the acknowledgement without deadlocking.
+
+use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
+
+use crate::mmu;
+
+/// Must match linker.ld.
+pub const MAX_HARTS: usize = 8;
+/// A guard page and a 512 KiB stack (boot builds big tables on it). Must
+/// match linker.ld.
+pub const STACK_SLOT: usize = 4096 + 0x8_0000;
+
+const SIP_SSIP: usize = 1 << 1;
+
+/// The hart running this code, from which kernel stack it is on.
+pub fn hart_id() -> usize {
+    let sp: usize;
+    // SAFETY: reading sp has no side effects.
+    unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) };
+    (sp - stacks_start()) / STACK_SLOT
+}
+
+pub fn stacks_start() -> usize {
+    unsafe extern "C" {
+        static _stacks_start: u8;
+    }
+    // SAFETY: only the symbol's address is taken.
+    unsafe { &_stacks_start as *const u8 as usize }
+}
+
+static BOOT_HART: AtomicUsize = AtomicUsize::new(0);
+static NUM_HARTS: AtomicUsize = AtomicUsize::new(1);
+/// Physical address of the kernel's own root table.
+static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0);
+
+/// The root table each hart translates with.
+static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS];
+static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+
+pub fn boot_hart() -> usize {
+    BOOT_HART.load(Ordering::Relaxed)
+}
+
+pub fn num_harts() -> usize {
+    NUM_HARTS.load(Ordering::Relaxed)
+}
+
+pub fn kernel_root() -> usize {
+    KERNEL_ROOT.load(Ordering::Relaxed)
+}
+
+/// Records which hart booted and how many there are, and where the
+/// kernel's root table is.
+pub fn init(boot: usize, harts: usize, kernel_root: usize) {
+    BOOT_HART.store(boot, Ordering::Relaxed);
+    NUM_HARTS.store(harts, Ordering::Relaxed);
+    KERNEL_ROOT.store(kernel_root, Ordering::Relaxed);
+}
+
+/// Called by `mmu::activate`.
+pub fn note_active_root(root: usize) {
+    if let Some(r) = ACTIVE_ROOT.get(hart_id()) {
+        r.store(root, Ordering::SeqCst);
+    }
+}
+
+// ---- the kernel lock ----
+
+static NEXT_TICKET: AtomicU32 = AtomicU32::new(0);
+static SERVING: AtomicU32 = AtomicU32::new(0);
+
+/// Takes the kernel lock, first come first served. Leave-root requests
+/// are serviced while waiting.
+pub fn lock() {
+    let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
+    while SERVING.load(Ordering::Acquire) != ticket {
+        service_ipi();
+        core::hint::spin_loop();
+    }
+}
+
+pub fn unlock() {
+    SERVING.fetch_add(1, Ordering::Release);
+}
+
+// ---- IPIs ----
+
+/// The lock-free half of an IPI: acknowledges it and, if asked, leaves the
+/// root being changed. Safe to call at any time, any number of times.
+pub fn service_ipi() {
+    let sip: usize;
+    // SAFETY: reading sip has no side effects.
+    unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
+    if sip & SIP_SSIP == 0 {
+        return;
+    }
+    // Clear first: a request posted after this raises SSIP again.
+    // SAFETY: clearing our own pending software interrupt.
+    unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) };
+    let me = hart_id();
+    let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else {
+        return;
+    };
+    let root = leave.swap(0, Ordering::SeqCst);
+    if root != 0 {
+        if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) {
+            mmu::activate(mmu::page_table::table_at(kernel_root()));
+        }
+        // SAFETY: flushing translations has no other effect.
+        unsafe { core::arch::asm!("sfence.vma zero, zero") };
+        left.fetch_add(1, Ordering::SeqCst);
+    }
+}
+
+/// Asks `hart` to look at its run queue again, on its way out of the
+/// kernel.
+pub fn kick(hart: usize) {
+    if let Some(r) = RESCHEDULE.get(hart) {
+        r.store(true, Ordering::SeqCst);
+        sbi_send_ipi(hart);
+    }
+}
+
+/// Consumes this hart's reschedule request.
+pub fn take_reschedule() -> bool {
+    RESCHEDULE
+        .get(hart_id())
+        .is_some_and(|r| r.swap(false, Ordering::SeqCst))
+}
+
+/// Makes every other hart that translates through `root` leave it and
+/// flush, and waits until each has. Called with the kernel lock held, so
+/// no hart can start using `root` meanwhile.
+pub fn shoot_down(root: usize) {
+    let me = hart_id();
+    for hart in 0..num_harts() {
+        if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root {
+            continue;
+        }
+        let seen = LEFT[hart].load(Ordering::SeqCst);
+        LEAVE_ROOT[hart].store(root, Ordering::SeqCst);
+        sbi_send_ipi(hart);
+        while LEFT[hart].load(Ordering::SeqCst) == seen {
+            core::hint::spin_loop();
+        }
+    }
+}
+
+// ---- SBI ----
+
+const SBI_EXT_IPI: usize = 0x735049;
+const SBI_EXT_HSM: usize = 0x48534D;
+
+fn sbi_send_ipi(hart: usize) {
+    // SAFETY: an SBI call; a0 and a1 come back clobbered.
+    unsafe {
+        core::arch::asm!(
+            "ecall",
+            inlateout("a0") 1usize => _,
+            inlateout("a1") hart => _,
+            in("a6") 0usize,
+            in("a7") SBI_EXT_IPI,
+        );
+    }
+}
+
+/// Whether `hart` exists, per SBI HSM `hart_get_status`.
+fn sbi_hart_exists(hart: usize) -> bool {
+    let error: isize;
+    // SAFETY: an SBI call; a0 and a1 come back clobbered.
+    unsafe {
+        core::arch::asm!(
+            "ecall",
+            inlateout("a0") hart => error,
+            lateout("a1") _,
+            in("a6") 2usize,
+            in("a7") SBI_EXT_HSM,
+        );
+    }
+    error == 0
+}
+
+/// The number of harts, counting up from hart 0 until one is missing.
+pub fn probe_harts() -> usize {
+    (0..MAX_HARTS)
+        .take_while(|h| sbi_hart_exists(*h))
+        .count()
+        .max(1)
+}
+
+/// Starts `hart` at the physical address `entry` with the MMU off.
+pub fn sbi_hart_start(hart: usize, entry: usize) -> bool {
+    let error: isize;
+    // SAFETY: an SBI call; a0 and a1 come back clobbered.
+    unsafe {
+        core::arch::asm!(
+            "ecall",
+            inlateout("a0") hart => error,
+            inlateout("a1") entry => _,
+            in("a2") 0usize,
+            in("a6") 0usize,
+            in("a7") SBI_EXT_HSM,
+        );
+    }
+    error == 0
+}

crates/adapter-riscv/src/trap/mod.rs

@@ -60,7 +60,14 @@ _from_user:
 
     mv a0, sp
     call trap_handler_rust
+    j _restore_frame
 
+# trap_return(frame): resume whatever `frame` describes, as if returning
+# from a trap that saved it. Used to enter userspace for the first time.
+.globl trap_return
+trap_return:
+    mv sp, a0
+_restore_frame:
     ld t0, 256(sp)
     csrw sepc, t0
     ld t0, 264(sp)
@@ -121,6 +128,7 @@ _sret_smode:
 );
 
 const CAUSE_INTERRUPT_BIT: usize = 1 << 63;
+const CAUSE_S_SOFTWARE: usize = 1;
 const CAUSE_S_TIMER: usize = 5;
 const CAUSE_S_EXTERNAL: usize = 9;
 const CAUSE_ECALL_U: usize = 8;
@@ -177,88 +185,67 @@ fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFra
 }
 
 fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
-    let kern = unsafe { &mut *crate::kernel_ptr() };
+    if let Some(result) = kern().take_pending_result(tid) {
-    if let Some(result) = kern.take_pending_result(tid) {
         write_result(frame, &result);
     }
 }
 
+unsafe extern "C" {
+    fn trap_return(frame: *mut TrapFrame) -> !;
+}
+
+fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
+    // SAFETY: only called with the kernel lock held.
+    unsafe { &mut *crate::kernel_ptr() }
+}
+
+fn this_core() -> types::id::CoreId {
+    types::id::CoreId::from_raw(crate::smp::hart_id() as u8)
+}
+
+/// Saves `frame` into `from` (if it still exists) and loads `to`.
 fn switch_to_thread(
     frame: &mut TrapFrame,
-    current_tid: types::id::ThreadId,
+    from: Option<types::id::ThreadId>,
-    next_tid: types::id::ThreadId,
+    to: types::id::ThreadId,
 ) {
-    let kern = unsafe { &mut *crate::kernel_ptr() };
+    let kern = kern();
-
+    if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
-    if let Some(ctx) = kern.thread_context_mut(current_tid) {
         save_frame_to_context(frame, ctx);
     }
-
+    if let Some(ctx) = kern.thread_context(to) {
-    let next_pid = kern.thread_process(next_tid);
-    let current_pid = kern.thread_process(current_tid);
-
-    if let Some(ctx) = kern.thread_context(next_tid) {
         load_context_to_frame(ctx, frame);
     }
-
+    apply_pending_result(frame, to);
-    apply_pending_result(frame, next_tid);
-
-    if next_pid != current_pid
-        && let Some(pid) = next_pid
-    {
-        let pt_addr = kern.platform_mut().page_table_for(pid);
-        if pt_addr != 0 {
-            let pt = crate::mmu::page_table::table_at(pt_addr);
-            crate::mmu::activate(pt);
-        }
-    }
 }
 
-fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) {
+/// Saves `from` and runs the next ready thread, idling until there is one.
-    let kern = unsafe { &mut *crate::kernel_ptr() };
+fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
-
+    if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
-    if let Some(ctx) = kern.thread_context_mut(current_tid) {
         save_frame_to_context(frame, ctx);
     }
-
+    let next = next_runnable();
-    let next_tid = next_runnable();
+    let _ = kern().set_thread_running(next, this_core());
-    let kern = unsafe { &mut *crate::kernel_ptr() };
+    switch_to_thread(frame, None, next);
-    let next_pid = kern.thread_process(next_tid);
-    let current_pid = kern.thread_process(current_tid);
-
-    let core = types::id::CoreId::from_raw(0);
-    let _ = kern.set_thread_running(next_tid, core);
-
-    if let Some(ctx) = kern.thread_context(next_tid) {
-        load_context_to_frame(ctx, frame);
-    }
-
-    apply_pending_result(frame, next_tid);
-
-    if next_pid != current_pid
-        && let Some(pid) = next_pid
-    {
-        let pt_addr = kern.platform_mut().page_table_for(pid);
-        if pt_addr != 0 {
-            let pt = crate::mmu::page_table::table_at(pt_addr);
-            crate::mmu::activate(pt);
-        }
-    }
 }
 
-/// The next thread to run, idling until an interrupt makes one ready.
+/// The next thread to run on this hart, idling until one is ready.
 ///
-/// We are inside a trap, so SIE is clear and interrupts are not taken.
+/// Idling gives up the kernel lock: an idle hart holds nothing, so other
-/// `wfi` still wakes once one is pending, and it is then serviced here by
+/// harts can make work for it (and kick it) meanwhile. It also leaves any
-/// polling `sip`, which is how a driver blocked on its IRQ notification
+/// user address space first, so nobody needs to shoot it down. SIE stays
-/// gets woken when nothing else can run.
+/// clear; `wfi` still wakes on a pending interrupt, serviced by polling.
 fn next_runnable() -> types::id::ThreadId {
     loop {
-        let kern = unsafe { &mut *crate::kernel_ptr() };
+        if let Some(tid) = kern().pick_next_thread() {
-        if let Some(tid) = kern.pick_next_thread() {
             return tid;
         }
+        crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
+        crate::smp::unlock();
         unsafe { core::arch::asm!("wfi") };
+        crate::smp::service_ipi();
+        crate::smp::lock();
+        let _ = crate::smp::take_reschedule();
         let sip: usize;
         unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
         if sip & (1 << CAUSE_S_TIMER) != 0 {
@@ -271,49 +258,123 @@ fn next_runnable() -> types::id::ThreadId {
     }
 }
 
-/// Claims every pending PLIC interrupt and hands each to the kernel.
+/// Claims every pending PLIC interrupt and hands each to the kernel. Only
+/// the boot hart receives external interrupts.
 fn dispatch_external() {
     loop {
         let irq_num = crate::hw::plic::Plic::claim();
         if irq_num == 0 {
             return;
         }
-        let kern = unsafe { &mut *crate::kernel_ptr() };
+        kern().handle_interrupt(types::id::IrqId::from_raw(irq_num as u16));
-        kern.handle_interrupt(types::id::IrqId::from_raw(irq_num as u16));
         crate::hw::plic::Plic::complete(irq_num);
     }
 }
 
+/// Last step before leaving the kernel to userspace: act on a reschedule
+/// request, make sure this hart has a thread (its own may have been taken
+/// away by another hart), and translate through that thread's address
+/// space.
+fn finish(frame: &mut TrapFrame) {
+    let kern = kern();
+    let current = kern.current_thread();
+    if crate::smp::take_reschedule()
+        && let Some(cur) = current
+        && let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
+    {
+        switch_to_thread(frame, Some(cur), next);
+    }
+    if kern.current_thread().is_none() {
+        switch_to_next(frame, None);
+    }
+    let root = kern
+        .current_thread()
+        .and_then(|t| kern.thread_process(t))
+        .map(|pid| kern.platform_mut().page_table_for(pid))
+        .unwrap_or(0);
+    if root != 0 && crate::mmu::active_root() != root {
+        crate::mmu::activate(crate::mmu::page_table::table_at(root));
+    }
+}
+
+/// Leaves the kernel for userspace for the first time on this hart, with
+/// the kernel lock held: runs the thread the kernel already made current
+/// here (init, on the boot hart) or the next one homed here.
+pub fn enter_first() -> ! {
+    let mut frame = TrapFrame {
+        regs: [0; 32],
+        sepc: 0,
+        sstatus: 0,
+    };
+    match kern().current_thread() {
+        Some(tid) => switch_to_thread(&mut frame, None, tid),
+        None => switch_to_next(&mut frame, None),
+    }
+    finish(&mut frame);
+    crate::smp::unlock();
+    // SAFETY: `frame` holds a user context; sscratch is set from it on
+    // the way out, so later traps land at the top of this frame.
+    unsafe { trap_return(&mut frame) }
+}
+
 #[unsafe(no_mangle)]
 extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
     let scause: usize;
     unsafe { core::arch::asm!("csrr {0}, scause", out(reg) scause) };
 
+    if frame.sstatus & SSTATUS_SPP != 0 {
+        // Checked before taking the lock: this hart may already hold it.
+        kernel_fault(frame, scause);
+    }
+    crate::smp::service_ipi();
+    crate::smp::lock();
     if scause & CAUSE_INTERRUPT_BIT != 0 {
         handle_interrupt(frame, scause & !CAUSE_INTERRUPT_BIT);
     } else {
         handle_exception(frame, scause);
     }
+    finish(frame);
+    crate::smp::unlock();
 }
 
 fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
     match cause {
         CAUSE_S_TIMER => {
             crate::hw::timer::handle_interrupt();
-            let kern = unsafe { &mut *crate::kernel_ptr() };
+            let kern = kern();
-            let core = types::id::CoreId::from_raw(0);
             if let Some(current_tid) = kern.current_thread() {
-                let decision = kern.handle_tick(core);
+                let decision = kern.handle_tick(this_core());
                 if let domain::scheduler::SchedulerDecision::Preempt(next) = decision {
-                    switch_to_thread(frame, current_tid, next);
+                    switch_to_thread(frame, Some(current_tid), next);
                 }
             }
         }
         CAUSE_S_EXTERNAL => dispatch_external(),
+        // Serviced on entry; any reschedule happens in `finish`.
+        CAUSE_S_SOFTWARE => {}
         _ => {}
     }
 }
 
+/// The kernel itself faulted (a W^X violation, a stack guard, a bad
+/// pointer). Its state can no longer be trusted, and no user thread is to
+/// blame.
+fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! {
+    let uart = crate::hw::uart::Uart::new();
+    uart.write_str("KERNEL FAULT: scause=");
+    uart.write_hex(scause);
+    let stval: usize;
+    unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
+    uart.write_str(" stval=");
+    uart.write_hex(stval);
+    uart.write_str(" sepc=");
+    uart.write_hex(frame.sepc);
+    uart.write_str(" hart=");
+    uart.write_dec(crate::smp::hart_id());
+    uart.put_char(b'\n');
+    crate::halt(&uart, "fault in kernel mode");
+}
+
 fn handle_exception(frame: &mut TrapFrame, scause: usize) {
     match scause {
         CAUSE_ECALL_U => {
@@ -324,48 +385,24 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
                 return;
             };
             {
-                let kern = unsafe { &mut *crate::kernel_ptr() };
+                let kern = kern();
                 let current_tid = kern.current_thread();
                 let outcome = kern.handle_syscall(syscall);
 
                 match outcome {
                     kernel::SyscallOutcome::Complete(result) => write_result(frame, &result),
-                    kernel::SyscallOutcome::BlockCaller(_) => {
+                    kernel::SyscallOutcome::BlockCaller(_) => switch_to_next(frame, current_tid),
-                        if let Some(tid) = current_tid {
-                            switch_to_next(frame, tid);
-                        }
-                    }
                     kernel::SyscallOutcome::SwitchTo(next) => {
-                        if let Some(tid) = current_tid {
+                        switch_to_thread(frame, current_tid, next)
-                            switch_to_thread(frame, tid, next);
-                        }
                     }
                     kernel::SyscallOutcome::TerminateCaller => {
                         let uart = crate::hw::uart::Uart::new();
                         uart.write_str("k-os: process exited\n");
-                        if let Some(tid) = current_tid {
+                        switch_to_next(frame, current_tid);
-                            switch_to_next(frame, tid);
                     }
                 }
             }
         }
-        }
-        _ if frame.sstatus & SSTATUS_SPP != 0 => {
-            // The kernel itself faulted (a W^X violation, the stack guard,
-            // a bad pointer). Its state can no longer be trusted, and no
-            // user thread is to blame.
-            let uart = crate::hw::uart::Uart::new();
-            uart.write_str("KERNEL FAULT: scause=");
-            uart.write_hex(scause);
-            let stval: usize;
-            unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
-            uart.write_str(" stval=");
-            uart.write_hex(stval);
-            uart.write_str(" sepc=");
-            uart.write_hex(frame.sepc);
-            uart.put_char(b'\n');
-            crate::halt(&uart, "fault in kernel mode");
-        }
         _ => {
             let uart = crate::hw::uart::Uart::new();
             uart.write_str("FAULT: scause=");
@@ -378,7 +415,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
             uart.write_hex(frame.sepc);
             uart.put_char(b'\n');
 
-            let kern = unsafe { &mut *crate::kernel_ptr() };
+            let kern = kern();
             if let Some(current_tid) = kern.current_thread() {
                 let fault = match scause {
                     2 => domain::fault::Fault::IllegalInstruction,
@@ -388,12 +425,10 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
                     _ => domain::fault::Fault::InvalidSyscall,
                 };
                 kern.handle_fault(current_tid, fault, frame.sepc);
-                switch_to_next(frame, current_tid);
+                switch_to_next(frame, Some(current_tid));
-            } else {
-                loop {
-                    unsafe { core::arch::asm!("wfi") };
-                }
             }
+            // With no current thread (another hart just destroyed it),
+            // `finish` moves on to the next one.
         }
     }
 }

crates/domain/src/scheduler/multi_core.rs

@@ -47,6 +47,23 @@ impl MultiCoreScheduler {
         self.placements.get(thread.raw() as usize)?.as_ref()
     }
 
+    pub fn num_cores(&self) -> u8 {
+        self.num_cores
+    }
+
+    /// The highest priority waiting in `core`'s run queue.
+    pub fn highest_waiting(&self, core: CoreId) -> Option<Priority> {
+        self.core(core)?.highest_waiting_priority()
+    }
+
+    /// The core whose run queue holds `thread`, if any.
+    pub fn queued_on(&self, thread: ThreadId) -> Option<CoreId> {
+        self.active()
+            .iter()
+            .find(|c| c.is_queued(thread))
+            .map(|c| c.core())
+    }
+
     /// Whether `thread` waits in some core's run queue.
     pub fn is_queued(&self, thread: ThreadId) -> bool {
         self.active().iter().any(|c| c.is_queued(thread))

crates/domain/src/thread/mod.rs

@@ -306,6 +306,14 @@ impl ThreadTable {
         Ok(id)
     }
 
+    /// Every live thread with its id.
+    pub fn iter(&self) -> impl Iterator<Item = (ThreadId, &Thread)> {
+        self.threads
+            .iter()
+            .enumerate()
+            .filter_map(|(i, t)| Some((ThreadId::from_raw(i as u16), t.as_ref()?)))
+    }
+
     pub fn get(&self, id: ThreadId) -> Result<&Thread, KernelError> {
         let idx = id.raw() as usize;
         self.threads

crates/kernel/src/lib.rs

@@ -21,6 +21,7 @@ pub mod interrupt;
 mod memory;
 #[cfg(kani)]
 pub mod proofs;
+mod smp;
 pub mod syscall;
 
 pub use board::BoardDevice;
@@ -65,6 +66,8 @@ pub struct Kernel<P: Platform> {
     notifications: NotificationTable,
     ipc: [syscall::IpcState; MAX_THREADS],
     pending_results: [Option<SyscallResult>; MAX_THREADS],
+    /// The core each thread runs on (see smp.rs).
+    homes: [types::id::CoreId; MAX_THREADS],
 }
 
 impl<P: Platform> Kernel<P> {
@@ -83,6 +86,7 @@ impl<P: Platform> Kernel<P> {
             notifications: NotificationTable::new(),
             ipc: [syscall::IpcState::default(); MAX_THREADS],
             pending_results: [const { None }; MAX_THREADS],
+            homes: [types::id::CoreId::from_raw(0); MAX_THREADS],
         }
     }
 
@@ -115,6 +119,7 @@ impl<P: Platform> Kernel<P> {
             addr_of_mut!((*slot).notifications).write(NotificationTable::new());
             addr_of_mut!((*slot).ipc).write([syscall::IpcState::default(); MAX_THREADS]);
             addr_of_mut!((*slot).pending_results).write([const { None }; MAX_THREADS]);
+            addr_of_mut!((*slot).homes).write([types::id::CoreId::from_raw(0); MAX_THREADS]);
         }
     }
 
@@ -142,6 +147,7 @@ impl<P: Platform> Kernel<P> {
             .insert(cap)?;
 
         let core = self.platform.core_id();
+        self.set_home(tid, core);
         self.scheduler.add(tid, priority, core);
         self.scheduler.next(core);
         self.threads.get_mut(tid)?.set_running(core)?;
@@ -211,14 +217,13 @@ impl<P: Platform> Kernel<P> {
     }
 
     pub fn spawn_thread_internal(&mut self, pid: ProcessId) -> Option<ThreadId> {
-        use domain::scheduler::Scheduler;
         let tid = self
             .threads
             .spawn(pid, Priority::DEFAULT, TimeSlice::DEFAULT, None)
             .ok()?;
         let _ = self.processes.get_mut(pid).map(|p| p.add_thread(tid));
-        let core = self.platform.core_id();
+        self.place(tid);
-        self.scheduler.add(tid, Priority::DEFAULT, core);
+        self.make_ready(tid, Priority::DEFAULT);
         Some(tid)
     }
 
@@ -240,31 +245,6 @@ impl<P: Platform> Kernel<P> {
         self.grant_cap(to, resource, permissions)
     }
 
-    pub fn switch_to_thread(&mut self, tid: ThreadId) {
-        use domain::scheduler::Scheduler;
-        let core = self.platform.core_id();
-
-        if let Some(current) = self.scheduler.current(core) {
-            if current == tid {
-                return;
-            }
-            if let Ok(t) = self.threads.get_mut(current)
-                && matches!(t.state(), ThreadState::Running(_))
-            {
-                let priority = t.priority();
-                let _ = t.preempt();
-                self.scheduler.add(current, priority, core);
-            }
-        }
-
-        if let Ok(t) = self.threads.get(tid)
-            && matches!(t.state(), ThreadState::Ready)
-            && self.scheduler.run_now(core, tid)
-        {
-            let _ = self.threads.get_mut(tid).map(|t| t.set_running(core));
-        }
-    }
-
     fn caller(&self) -> Result<(ThreadId, ProcessId), KernelError> {
         let core = self.platform.core_id();
         let tid = self
@@ -311,6 +291,7 @@ impl<P: Platform> Kernel<P> {
             if let Some(caller) = ipc.reply_to {
                 self.release_caller(caller);
             }
+            self.evict(*tid);
             self.scheduler.remove(*tid);
             self.set_pending_result(*tid, None);
             let _ = self.threads.destroy(*tid);

crates/kernel/src/smp.rs

@@ -0,0 +1,119 @@
+//! Cores. Every thread has a home core, chosen when it is created, and
+//! only ever runs there: it waits in that core's run queue and is never
+//! migrated. Making a thread ready from another core sends that core an
+//! IPI when it has to look at its queue again (it is idle, or the thread
+//! outranks what it is running), and so does taking away the thread a
+//! core is running.
+
+use domain::{
+    scheduler::{Scheduler, SchedulerDecision},
+    thread::ThreadState,
+};
+use platform::Platform;
+use types::{
+    bounded::Priority,
+    id::{CoreId, ThreadId},
+};
+
+use crate::Kernel;
+
+impl<P: Platform> Kernel<P> {
+    /// The core `tid` runs on.
+    pub fn home(&self, tid: ThreadId) -> CoreId {
+        self.homes
+            .get(tid.raw() as usize)
+            .copied()
+            .unwrap_or(CoreId::from_raw(0))
+    }
+
+    /// Homes a new thread on `core`.
+    pub(crate) fn set_home(&mut self, tid: ThreadId, core: CoreId) {
+        if let Some(h) = self.homes.get_mut(tid.raw() as usize) {
+            *h = core;
+        }
+    }
+
+    /// Homes a new thread on the core with the fewest live threads.
+    pub(crate) fn place(&mut self, tid: ThreadId) {
+        let cores = self.scheduler.num_cores().max(1);
+        let load = |core: CoreId| {
+            self.threads
+                .iter()
+                .filter(|(t, _)| *t != tid && self.home(*t) == core)
+                .count()
+        };
+        let best = (0..cores)
+            .map(CoreId::from_raw)
+            .min_by_key(|c| load(*c))
+            .unwrap_or(CoreId::from_raw(0));
+        self.set_home(tid, best);
+    }
+
+    /// Queues a ready thread on its home core, and kicks that core if it
+    /// is another one that should look at its queue now.
+    pub(crate) fn make_ready(&mut self, tid: ThreadId, priority: Priority) {
+        let home = self.home(tid);
+        self.scheduler.add(tid, priority, home);
+        if home == self.platform.core_id() {
+            return;
+        }
+        let outranks = match self.scheduler.current(home) {
+            None => true,
+            Some(running) => self
+                .threads
+                .get(running)
+                .is_ok_and(|t| priority > t.priority()),
+        };
+        if outranks {
+            self.platform.send_ipi(home);
+        }
+    }
+
+    /// Before `tid` is destroyed: if another core is running it, that core
+    /// must stop and pick something else.
+    pub(crate) fn evict(&mut self, tid: ThreadId) {
+        if let Ok(ThreadState::Running(core)) = self.threads.get(tid).map(|t| *t.state())
+            && core != self.platform.core_id()
+        {
+            self.platform.send_ipi(core);
+        }
+    }
+
+    /// What `core` should do after an IPI: keep running its thread, or
+    /// switch to a higher-priority one now waiting. When it has no thread
+    /// (it was idle, or its thread was destroyed) the caller picks the
+    /// next one itself.
+    pub fn reschedule(&mut self, core: CoreId) -> SchedulerDecision {
+        let Some(current) = self.scheduler.current(core) else {
+            return SchedulerDecision::Continue;
+        };
+        let Ok(thread) = self.threads.get(current) else {
+            return SchedulerDecision::Continue;
+        };
+        let priority = thread.priority();
+        if self
+            .scheduler
+            .highest_waiting(core)
+            .is_none_or(|p| p <= priority)
+        {
+            return SchedulerDecision::Continue;
+        }
+        let Some(next) = self.scheduler.next(core) else {
+            return SchedulerDecision::Continue;
+        };
+        if self
+            .threads
+            .get_mut(next)
+            .and_then(|t| t.set_running(core))
+            .is_err()
+        {
+            return SchedulerDecision::Continue;
+        }
+        if let Ok(t) = self.threads.get_mut(current)
+            && t.preempt().is_ok()
+        {
+            self.scheduler.add(current, priority, core);
+        }
+        SchedulerDecision::Preempt(next)
+    }
+}

crates/kernel/src/syscall/ipc.rs

@@ -102,8 +102,7 @@ impl<P: Platform> Kernel<P> {
         let priority = thread.priority();
         if thread.unblock().is_ok() {
             self.set_pending_result(tid, Some(result));
-            let core = self.platform.core_id();
+            self.make_ready(tid, priority);
-            self.scheduler.add(tid, priority, core);
         }
     }
 
@@ -234,6 +233,8 @@ impl<P: Platform> Kernel<P> {
             }
             self.block_caller(tid, BlockReason::AwaitingReply);
             // Run the server straight away instead of waiting for its turn.
+            // Only works if it lives on this core (run_now finds it in this
+            // core's queue only); otherwise make_ready kicked its core.
             let core = self.platform.core_id();
             if self.scheduler.run_now(core, receiver)
                 && self

crates/kernel/src/syscall/lifecycle.rs

@@ -10,7 +10,6 @@ impl<P: Platform> Kernel<P> {
         entry: types::address::EntryPoint,
         stack: types::address::StackPointer,
     ) -> SyscallOutcome {
-        use domain::scheduler::Scheduler;
         use domain::thread::SavedContext;
 
         let priority = types::bounded::Priority::DEFAULT;
@@ -30,8 +29,8 @@ impl<P: Platform> Kernel<P> {
             return SyscallOutcome::Complete(SyscallResult::Error(e));
         }
 
-        let core = self.platform.core_id();
+        self.place(tid);
-        self.scheduler.add(tid, priority, core);
+        self.make_ready(tid, priority);
         SyscallOutcome::Complete(SyscallResult::Value(tid.raw() as usize))
     }
 
@@ -89,7 +88,6 @@ impl<P: Platform> Kernel<P> {
         entry: types::address::EntryPoint,
         stack: types::address::StackPointer,
     ) -> SyscallOutcome {
-        use domain::scheduler::Scheduler;
         use domain::thread::SavedContext;
 
         let target_pid = match self.check_cap(
@@ -132,8 +130,8 @@ impl<P: Platform> Kernel<P> {
             return SyscallOutcome::Complete(SyscallResult::Error(e));
         }
 
-        let core = self.platform.core_id();
+        self.place(tid);
-        self.scheduler.add(tid, priority, core);
+        self.make_ready(tid, priority);
         SyscallOutcome::Complete(SyscallResult::Value(tid.raw() as usize))
     }
 

crates/kernel/src/tests/helpers.rs

@@ -37,6 +37,7 @@ impl Kernel<MockPlatform> {
             .expect("spawn");
         let _ = self.processes.get_mut(pid).map(|p| p.add_thread(tid));
         let core = self.platform.core_id();
+        self.set_home(tid, core);
         self.scheduler.add(tid, priority, core);
         tid
     }
@@ -54,9 +55,21 @@ impl Kernel<MockPlatform> {
 
     pub fn switch_to_thread_for_test(&mut self, tid: ThreadId) {
         let core = self.platform.core_id();
+        assert_eq!(
+            self.home(tid),
+            core,
+            "a thread only ever runs on its home core"
+        );
 
+        // Only a thread ready in this core's queue can be switched to;
+        // otherwise leave the core as it is.
+        let switchable = self.scheduler.queued_on(tid) == Some(core)
+            && self
+                .threads
+                .get(tid)
+                .is_ok_and(|t| matches!(t.state(), ThreadState::Ready));
         if let Some(current) = self.scheduler.current(core) {
-            if current == tid {
+            if current == tid || !switchable {
                 return;
             }
             if let Ok(t) = self.threads.get_mut(current)

crates/kernel/src/tests/mock_platform.rs

@@ -37,6 +37,8 @@ pub struct MockPlatform {
     mappings: [Option<PageMapping>; MAX_MAPPINGS],
     tables: [Option<LeafTable>; MAX_TABLES],
     zeroed: [Option<usize>; MAX_ZEROED],
+    /// Cores an IPI was sent to and that have not taken it yet.
+    ipis: [bool; 8],
 }
 
 impl MockPlatform {
@@ -49,9 +51,27 @@ impl MockPlatform {
             mappings: [None; MAX_MAPPINGS],
             tables: [None; MAX_TABLES],
             zeroed: [None; MAX_ZEROED],
+            ipis: [false; 8],
         }
     }
 
+    /// Runs the kernel as if on `core` from now on.
+    pub fn set_core(&mut self, core: CoreId) {
+        self.core = core;
+    }
+
+    pub fn ipi_pending(&self, core: CoreId) -> bool {
+        self.ipis.get(core.raw() as usize).copied().unwrap_or(false)
+    }
+
+    /// Delivers `core`'s pending IPI: true if there was one.
+    pub fn take_ipi(&mut self, core: CoreId) -> bool {
+        self.ipis
+            .get_mut(core.raw() as usize)
+            .map(core::mem::take)
+            .unwrap_or(false)
+    }
+
     /// Physical address `va` is mapped to in `process`, as the "hardware" sees it.
     pub fn mapping(&self, process: ProcessId, va: usize) -> Option<usize> {
         self.mappings
@@ -207,7 +227,12 @@ impl Platform for MockPlatform {
     }
     fn mask_irq(&mut self, _irq: IrqId) {}
     fn unmask_irq(&mut self, _irq: IrqId) {}
-    fn send_ipi(&mut self, _target: CoreId) {}
+    fn send_ipi(&mut self, target: CoreId) {
+        assert_ne!(target, self.core, "a core never IPIs itself");
+        if let Some(p) = self.ipis.get_mut(target.raw() as usize) {
+            *p = true;
+        }
+    }
     fn core_id(&self) -> CoreId {
         self.core
     }

crates/kernel/src/tests/mod.rs

@@ -15,6 +15,7 @@ mod lifetimes;
 mod memory;
 mod revocation;
 mod simple;
+mod smp;
 mod storm;
 mod tick;
 

crates/kernel/src/tests/smp.rs

@@ -0,0 +1,178 @@
+extern crate std;
+
+use super::*;
+use crate::{SyscallOutcome, SyscallResult};
+use domain::{
+    message::Message,
+    resource::Resource,
+    scheduler::SchedulerDecision,
+    syscall::Syscall,
+    thread::{BlockReason, ThreadState},
+};
+use std::vec::Vec;
+use types::{
+    address::PhysicalAddress,
+    id::{CoreId, ThreadId},
+    permission::{Permission, PermissionSet},
+};
+
+fn core(c: u8) -> CoreId {
+    CoreId::from_raw(c)
+}
+
+/// A booted kernel on `cores` cores, init running on core 0.
+fn smp(cores: u8) -> (Kernel<MockPlatform>, ProcessId) {
+    let mut k = Kernel::new(
+        MockPlatform::new(cores),
+        PhysicalAddress::from_raw(0x8400_0000),
+        1024,
+    );
+    let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap();
+    (k, init)
+}
+
+/// Acts as `c` from now on, running the next thread there if it is idle.
+fn on(k: &mut Kernel<MockPlatform>, c: CoreId) {
+    k.platform_mut().set_core(c);
+    if k.current_thread().is_none()
+        && let Some(next) = k.pick_next_thread()
+    {
+        k.set_thread_running(next, c).unwrap();
+    }
+}
+
+fn state(k: &Kernel<MockPlatform>, t: ThreadId) -> Option<ThreadState> {
+    k.thread_state_for_test(t)
+}
+
+#[test]
+fn new_threads_are_spread_over_the_cores() {
+    let (mut k, init) = smp(4);
+    let homes: Vec<CoreId> = (0..3)
+        .map(|_| {
+            let t = k.spawn_thread_internal(init).unwrap();
+            k.home(t)
+        })
+        .collect();
+    assert_eq!(
+        homes,
+        [core(1), core(2), core(3)],
+        "init already has core 0"
+    );
+}
+
+#[test]
+fn a_thread_made_ready_for_an_idle_core_kicks_it() {
+    let (mut k, init) = smp(2);
+    let t = k.spawn_thread_internal(init).unwrap();
+    assert_eq!(k.home(t), core(1));
+    assert!(k.platform_mut().ipi_pending(core(1)));
+    assert!(!k.platform_mut().ipi_pending(core(0)));
+}
+
+#[test]
+fn a_thread_made_ready_on_this_core_sends_no_ipi() {
+    let (mut k, init) = smp(1);
+    k.spawn_thread_internal(init).unwrap();
+    assert!(!k.platform_mut().ipi_pending(core(0)));
+}
+
+#[test]
+fn a_busy_core_is_not_kicked_for_an_equal_priority_thread() {
+    let (mut k, init) = smp(2);
+    let a = k.spawn_thread_internal(init).unwrap();
+    on(&mut k, core(1));
+    assert_eq!(state(&k, a), Some(ThreadState::Running(core(1))));
+    k.platform_mut().take_ipi(core(1));
+    k.platform_mut().set_core(core(0));
+    // A second thread homed on core 1 while it is busy at the same priority.
+    let b = k.spawn_thread_internal(init).unwrap();
+    let c = k.spawn_thread_internal(init).unwrap();
+    let on_one = if k.home(b) == core(1) { b } else { c };
+    assert_eq!(k.home(on_one), core(1));
+    assert!(!k.platform_mut().ipi_pending(core(1)));
+}
+
+#[test]
+fn killing_a_thread_running_on_another_core_stops_that_core() {
+    let (mut k, init) = smp(2);
+    let child = k.create_process_internal().unwrap();
+    let t = k.spawn_thread_internal(child).unwrap();
+    on(&mut k, core(1));
+    k.platform_mut().take_ipi(core(1));
+    assert_eq!(state(&k, t), Some(ThreadState::Running(core(1))));
+
+    k.platform_mut().set_core(core(0));
+    let cap = k.grant_cap_for_test(
+        init,
+        Resource::Process(child),
+        PermissionSet::empty().with(Permission::Execute),
+    );
+    let out = k.handle_syscall(Syscall::Kill { process: cap });
+    assert!(matches!(out, SyscallOutcome::Complete(SyscallResult::Ok)));
+    assert!(k.platform_mut().ipi_pending(core(1)), "core 1 never told");
+    assert_eq!(k.scheduler.current(core(1)), None);
+    k.check_invariants().unwrap();
+}
+
+#[test]
+fn a_call_to_a_server_on_another_core_does_not_pull_it_over() {
+    let (mut k, init) = smp(2);
+    let ep = k.create_endpoint(init).unwrap();
+    let all = PermissionSet::empty()
+        .with(Permission::Read)
+        .with(Permission::Execute);
+    let slot = k.grant_cap_for_test(init, Resource::Endpoint(ep), all);
+    let server = k.spawn_thread_internal(init).unwrap();
+    assert_eq!(k.home(server), core(1));
+
+    on(&mut k, core(1));
+    k.platform_mut().take_ipi(core(1));
+    let out = k.handle_syscall(Syscall::Receive { endpoint: slot });
+    assert!(matches!(out, SyscallOutcome::BlockCaller(_)));
+    k.platform_mut().set_core(core(0));
+
+    let out = k.handle_syscall(Syscall::Call {
+        endpoint: slot,
+        message: Message::empty(),
+    });
+    assert!(
+        matches!(out, SyscallOutcome::BlockCaller(BlockReason::AwaitingReply)),
+        "no direct switch across cores"
+    );
+    assert_eq!(state(&k, server), Some(ThreadState::Ready));
+    assert!(k.platform_mut().ipi_pending(core(1)));
+    k.check_invariants().unwrap();
+}
+
+#[test]
+fn reschedule_switches_only_for_a_higher_priority_thread() {
+    let (mut k, init) = smp(2);
+    let a = k.spawn_thread_internal(init).unwrap();
+    on(&mut k, core(1));
+    assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
+
+    // An equal-priority one waits its turn (the tick handles that).
+    let peer = k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT);
+    assert_eq!(k.home(peer), core(1));
+    assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
+
+    // A higher-priority thread arrives on core 1's queue.
+    let p = Priority::new(Priority::DEFAULT.value() + 1).unwrap();
+    let urgent = k.spawn_thread_for_test(init, p, TimeSlice::DEFAULT);
+    assert_eq!(k.home(urgent), core(1));
+
+    assert!(matches!(
+        k.reschedule(core(1)),
+        SchedulerDecision::Preempt(t) if t == urgent
+    ));
+    assert_eq!(state(&k, urgent), Some(ThreadState::Running(core(1))));
+    assert_eq!(state(&k, a), Some(ThreadState::Ready));
+}
+
+#[test]
+fn reschedule_on_an_idle_core_leaves_picking_to_the_caller() {
+    let (mut k, _) = smp(2);
+    k.platform_mut().set_core(core(1));
+    assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
+}

crates/kernel/src/tests/storm.rs

@@ -208,6 +208,20 @@ impl Kernel<MockPlatform> {
             if self.scheduler.is_queued(t) != matches!(state, ThreadState::Ready) {
                 return Err(format!("{t:?} in {state:?} has wrong run-queue membership"));
             }
+            // ---- cores: a thread runs and waits only on its home core ----
+            let home = self.home(t);
+            match state {
+                ThreadState::Running(c) if c != home => {
+                    return Err(format!("{t:?} runs on {c:?}, its home is {home:?}"));
+                }
+                ThreadState::Running(c) if self.scheduler.current(c) != Some(t) => {
+                    return Err(format!("{t:?} Running on {c:?} but not its current thread"));
+                }
+                ThreadState::Ready if self.scheduler.queued_on(t) != Some(home) => {
+                    return Err(format!("{t:?} queued away from its home {home:?}"));
+                }
+                _ => {}
+            }
             if let Some(r) = ipc.reply_to
                 && self.threads.generation(r.tid) == Some(r.generation)
                 && self.threads.get(r.tid).ok().map(|x| *x.state())
@@ -216,6 +230,22 @@ impl Kernel<MockPlatform> {
                 return Err(format!("{t:?} holds a reply for a thread not waiting"));
             }
         }
+        for c in 0..self.scheduler.num_cores() {
+            let core = types::id::CoreId::from_raw(c);
+            let current = self.scheduler.current(core);
+            if let Some(t) = current
+                && self.threads.get(t).ok().map(|x| *x.state()) != Some(ThreadState::Running(core))
+            {
+                return Err(format!("{core:?}'s current {t:?} is not running there"));
+            }
+            // No lost wakeups: an idle core with work waiting has been told.
+            if current.is_none()
+                && self.scheduler.highest_waiting(core).is_some()
+                && !self.platform.ipi_pending(core)
+            {
+                return Err(format!("{core:?} idles with ready threads and no IPI"));
+            }
+        }
         Ok(())
     }
 }
@@ -282,6 +312,10 @@ fn new_round(k: &mut Kernel<MockPlatform>, init: ProcessId, init_thread: ThreadI
         .create_pool_region(hub, types::bounded::PageCount::new(2).unwrap())
         .ok();
     let mut members = std::vec![init];
+    // CreateProcess below runs as init, so on init's core.
+    let init_core = k.home(init_thread);
+    k.platform_mut().set_core(init_core);
+    resume(k, init_core);
     for _ in 0..2 {
         k.switch_to_thread_for_test(init_thread);
         if let crate::SyscallOutcome::Complete(crate::SyscallResult::Slot(cap)) =
@@ -292,7 +326,7 @@ fn new_round(k: &mut Kernel<MockPlatform>, init: ProcessId, init_thread: ThreadI
         {
             let pid = k.process_of_cap_for_test(cap);
             for _ in 0..2 {
-                k.spawn_thread_for_test(pid, Priority::DEFAULT, TimeSlice::DEFAULT);
+                k.spawn_thread_internal(pid);
             }
             members.push(pid);
         } else if std::env::var("STORM_DEBUG").is_ok() {
@@ -326,11 +360,40 @@ fn region_slots(k: &Kernel<MockPlatform>, pid: Option<ProcessId>) -> std::vec::V
         .collect()
 }
 
+/// What the adapter does whenever a core is left without a thread: run the
+/// next one in its queue, if any.
+fn resume(k: &mut Kernel<MockPlatform>, core: types::id::CoreId) {
+    if k.current_thread().is_none()
+        && let Some(next) = k.pick_next_thread()
+    {
+        k.set_thread_running(next, core).unwrap();
+    }
+}
+
 #[test]
 fn the_kernel_survives_a_syscall_storm() {
+    storm(1);
+}
+
+/// The same storm with threads homed on four cores. Each step runs on one
+/// core, as the thread that core is running; the kernel is still entered
+/// one call at a time (stage 1 takes a single lock), but threads on other
+/// cores are live meanwhile and IPIs arrive late.
+#[test]
+fn the_kernel_survives_a_syscall_storm_on_four_cores() {
+    storm(4);
+}
+
+fn storm(cores: u8) {
+    use types::id::CoreId;
     let mut total = [0usize; 36 + 1];
     for seed in [17u64, 0xABCD_EF01, 4242, 0x1357_9BDF, 8] {
-        let (mut k, init) = kernel_with_process();
+        let mut k = Kernel::new(
+            MockPlatform::new(cores),
+            types::address::PhysicalAddress::from_raw(0x8400_0000),
+            1024,
+        );
+        let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap();
         let init_thread = k.current_thread().unwrap();
         k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT);
         let mut rng = Rng(seed);
@@ -360,13 +423,35 @@ fn the_kernel_survives_a_syscall_storm() {
                 // init may be blocked on objects it made itself.
                 k.destroy_endpoints_owned_by(init);
                 k.destroy_notifications_owned_by(init);
+                let here = k.platform_mut().core_id();
+                resume(&mut k, here);
                 k.check_invariants()
                     .unwrap_or_else(|e| panic!("seed {seed:#x} step {step} after teardown: {e}"));
                 new_round(&mut k, init, init_thread);
                 rounds += 1;
                 continue;
             }
-            let tid = runnable[rng.below(runnable.len())];
+            // Pick a core and act as the thread it runs, after it takes any
+            // IPI that has reached it. Now and then switch it to another
+            // ready thread of its own, as a tick or a yield would.
+            let core = CoreId::from_raw(rng.below(cores as usize) as u8);
+            k.platform_mut().set_core(core);
+            if k.platform_mut().take_ipi(core) {
+                let _ = k.reschedule(core);
+            }
+            resume(&mut k, core);
+            let homed: std::vec::Vec<ThreadId> = runnable
+                .iter()
+                .copied()
+                .filter(|t| k.home(*t) == core)
+                .collect();
+            if homed.is_empty() {
+                continue;
+            }
+            let tid = match k.current_thread() {
+                Some(t) if rng.below(3) != 0 => t,
+                _ => homed[rng.below(homed.len())],
+            };
             let is_init = k.thread_process(tid) == Some(init);
             let nr = rng.below(36 + 1);
             // init exiting ends the test. Halt stays in: nobody here holds
@@ -399,6 +484,7 @@ fn the_kernel_survives_a_syscall_storm() {
             k.switch_to_thread_for_test(tid);
             k.take_pending_result(tid);
             let out = k.handle_syscall(syscall);
+            resume(&mut k, core);
             executed += 1;
             if !matches!(
                 out,

docs/ADAPTERS.md

@@ -21,9 +21,10 @@ This generates `crates/adapter-<arch>/` with the standard structure. Then:
    the final kernel table may be mapped in the lower half.
    Keep text, read-only data and writable data on separate pages and
    export their page-aligned boundaries (`_text_start`, `_text_end`,
-   `_rodata_end`, `_stack_guard`, `_stack_bottom`): the kernel maps each
+   `_rodata_end`, `_stacks_start`, `_stacks_end`): the kernel maps each
-   range with only the access it needs (W^X) and leaves a guard page under
+   range with only the access it needs (W^X). The stacks area holds one
-   the stack.
+   slot per hart, a guard page (left unmapped) then that hart's kernel
+   stack; the boot stub picks the slot from the hart id.
 5. Implement the TODOs in `hw/`, `mmu/`, `trap/`, and `platform.rs`
 
 ## Structure