refactor(adapter): split trap/page_table/smp/entry by concern; trim comments

crates/adapter-riscv/src/smp/hints.rs

@@ -0,0 +1,42 @@
+//! State published under the kernel lock for lock-free fast paths to read.
+
+use core::sync::atomic::{AtomicUsize, Ordering};
+
+use super::{MAX_HARTS, hart_id};
+
+/// Per core: highest waiting priority + 1, 0 if its queue is empty.
+static WAITING: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+/// Per hart: the running thread's process id + 1 and priority + 1.
+static RUNNING_PID: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+static RUNNING_PRIORITY: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+
+pub fn publish_waiting(core: usize, highest: Option<u8>) {
+    if let Some(w) = WAITING.get(core) {
+        w.store(highest.map_or(0, |p| p as usize + 1), Ordering::Release);
+    }
+}
+
+pub fn publish_running(pid: Option<u16>, priority: Option<u8>) {
+    let me = hart_id();
+    if let (Some(p), Some(q)) = (RUNNING_PID.get(me), RUNNING_PRIORITY.get(me)) {
+        p.store(pid.map_or(0, |x| x as usize + 1), Ordering::Release);
+        q.store(priority.map_or(0, |x| x as usize + 1), Ordering::Release);
+    }
+}
+
+/// The process the thread running on this hart belongs to.
+pub fn running_pid() -> Option<usize> {
+    let v = RUNNING_PID.get(hart_id())?.load(Ordering::Acquire);
+    v.checked_sub(1)
+}
+
+/// Whether a Yield by the running thread would change nothing: no thread
+/// waits on this core at its priority or above.
+pub fn yield_is_noop() -> bool {
+    let me = hart_id();
+    let (Some(w), Some(p)) = (WAITING.get(me), RUNNING_PRIORITY.get(me)) else {
+        return false;
+    };
+    let mine = p.load(Ordering::Acquire);
+    mine != 0 && w.load(Ordering::Acquire) < mine
+}

crates/adapter-riscv/src/smp/ipi.rs

@@ -0,0 +1,96 @@
+//! Reschedule and leave-root IPIs, serviced without the kernel lock.
+
+use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use super::sbi::sbi_send_ipi;
+use super::{MAX_HARTS, hart_id, kernel_root, num_harts};
+use crate::mmu;
+
+const SIP_SSIP: usize = 1 << 1;
+
+/// The root table each hart translates with.
+static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS];
+static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
+
+/// Called by `mmu::activate`.
+pub fn note_active_root(root: usize) {
+    if let Some(r) = ACTIVE_ROOT.get(hart_id()) {
+        r.store(root, Ordering::SeqCst);
+    }
+}
+
+/// Whether this hart has an IPI or a reschedule request it has not acted
+/// on; fast paths must then take the slow path.
+pub fn has_pending_request() -> bool {
+    let sip: usize;
+    // SAFETY: reading sip has no side effects.
+    unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
+    sip & SIP_SSIP != 0
+        || RESCHEDULE
+            .get(hart_id())
+            .is_some_and(|r| r.load(Ordering::SeqCst))
+}
+
+// ---- IPIs ----
+
+/// The lock-free half of an IPI: acknowledges it and leaves the root being
+/// changed, if asked. Safe to call at any time.
+pub fn service_ipi() {
+    let sip: usize;
+    // SAFETY: reading sip has no side effects.
+    unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
+    if sip & SIP_SSIP == 0 {
+        return;
+    }
+    // Clear first: a request posted after this raises SSIP again.
+    // SAFETY: clearing our own pending software interrupt.
+    unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) };
+    let me = hart_id();
+    let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else {
+        return;
+    };
+    let root = leave.swap(0, Ordering::SeqCst);
+    if root != 0 {
+        if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) {
+            mmu::activate(mmu::page_table::table_at(kernel_root()));
+        }
+        // SAFETY: flushing translations has no other effect.
+        unsafe { core::arch::asm!("sfence.vma zero, zero") };
+        left.fetch_add(1, Ordering::SeqCst);
+    }
+}
+
+/// Asks `hart` to look at its run queue again, on its way out of the
+/// kernel.
+pub fn kick(hart: usize) {
+    if let Some(r) = RESCHEDULE.get(hart) {
+        r.store(true, Ordering::SeqCst);
+        sbi_send_ipi(hart);
+    }
+}
+
+/// Consumes this hart's reschedule request.
+pub fn take_reschedule() -> bool {
+    RESCHEDULE
+        .get(hart_id())
+        .is_some_and(|r| r.swap(false, Ordering::SeqCst))
+}
+
+/// Makes every other hart leave `root` and flush, waiting for each.
+/// Kernel lock held, so no hart can start using `root` meanwhile.
+pub fn shoot_down(root: usize) {
+    let me = hart_id();
+    for hart in 0..num_harts() {
+        if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root {
+            continue;
+        }
+        let seen = LEFT[hart].load(Ordering::SeqCst);
+        LEAVE_ROOT[hart].store(root, Ordering::SeqCst);
+        sbi_send_ipi(hart);
+        while LEFT[hart].load(Ordering::SeqCst) == seen {
+            core::hint::spin_loop();
+        }
+    }
+}

crates/adapter-riscv/src/smp/lock.rs

@@ -0,0 +1,22 @@
+//! The kernel lock: a ticket lock that services IPIs while waiting.
+
+use core::sync::atomic::{AtomicU32, Ordering};
+
+use super::service_ipi;
+
+static NEXT_TICKET: AtomicU32 = AtomicU32::new(0);
+static SERVING: AtomicU32 = AtomicU32::new(0);
+
+/// Takes the kernel lock, first come first served. Leave-root requests
+/// are serviced while waiting.
+pub fn lock() {
+    let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
+    while SERVING.load(Ordering::Acquire) != ticket {
+        service_ipi();
+        core::hint::spin_loop();
+    }
+}
+
+pub fn unlock() {
+    SERVING.fetch_add(1, Ordering::Release);
+}

crates/adapter-riscv/src/smp/mod.rs

@@ -0,0 +1,61 @@
+//! Harts: per-hart kernel stacks (the hart id doubles as the kernel `CoreId`),
+//! the kernel lock, and IPIs. The kernel runs under one lock; userspace runs in parallel.
+
+mod hints;
+mod ipi;
+mod lock;
+mod sbi;
+
+pub use hints::{publish_running, publish_waiting, running_pid, yield_is_noop};
+pub use ipi::{
+    has_pending_request, kick, note_active_root, service_ipi, shoot_down, take_reschedule,
+};
+pub use lock::{lock, unlock};
+pub use sbi::{probe_harts, sbi_hart_start};
+
+use core::sync::atomic::{AtomicUsize, Ordering};
+
+/// Must match linker.ld.
+pub const MAX_HARTS: usize = 8;
+/// A guard page and a 512 KiB stack. Must match linker.ld.
+pub const STACK_SLOT: usize = 4096 + 0x8_0000;
+
+/// The hart running this code, from which kernel stack it is on.
+pub fn hart_id() -> usize {
+    let sp: usize;
+    // SAFETY: reading sp has no side effects.
+    unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) };
+    (sp - stacks_start()) / STACK_SLOT
+}
+
+pub fn stacks_start() -> usize {
+    unsafe extern "C" {
+        static _stacks_start: u8;
+    }
+    // SAFETY: only the symbol's address is taken.
+    unsafe { &_stacks_start as *const u8 as usize }
+}
+
+static BOOT_HART: AtomicUsize = AtomicUsize::new(0);
+static NUM_HARTS: AtomicUsize = AtomicUsize::new(1);
+/// Physical address of the kernel's own root table.
+static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0);
+
+pub fn boot_hart() -> usize {
+    BOOT_HART.load(Ordering::Relaxed)
+}
+
+pub fn num_harts() -> usize {
+    NUM_HARTS.load(Ordering::Relaxed)
+}
+
+pub fn kernel_root() -> usize {
+    KERNEL_ROOT.load(Ordering::Relaxed)
+}
+
+/// Records the boot hart, the hart count and the kernel root table.
+pub fn init(boot: usize, harts: usize, kernel_root: usize) {
+    BOOT_HART.store(boot, Ordering::Relaxed);
+    NUM_HARTS.store(harts, Ordering::Relaxed);
+    KERNEL_ROOT.store(kernel_root, Ordering::Relaxed);
+}

crates/adapter-riscv/src/smp/sbi.rs

@@ -0,0 +1,60 @@
+//! SBI calls: IPIs and hart state management.
+
+use super::MAX_HARTS;
+
+const SBI_EXT_IPI: usize = 0x735049;
+const SBI_EXT_HSM: usize = 0x48534D;
+
+pub(super) fn sbi_send_ipi(hart: usize) {
+    // SAFETY: an SBI call; a0 and a1 come back clobbered.
+    unsafe {
+        core::arch::asm!(
+            "ecall",
+            inlateout("a0") 1usize => _,
+            inlateout("a1") hart => _,
+            in("a6") 0usize,
+            in("a7") SBI_EXT_IPI,
+        );
+    }
+}
+
+/// Whether `hart` exists, per SBI HSM `hart_get_status`.
+fn sbi_hart_exists(hart: usize) -> bool {
+    let error: isize;
+    // SAFETY: an SBI call; a0 and a1 come back clobbered.
+    unsafe {
+        core::arch::asm!(
+            "ecall",
+            inlateout("a0") hart => error,
+            lateout("a1") _,
+            in("a6") 2usize,
+            in("a7") SBI_EXT_HSM,
+        );
+    }
+    error == 0
+}
+
+/// The number of harts, counting up from hart 0 until one is missing.
+pub fn probe_harts() -> usize {
+    (0..MAX_HARTS)
+        .take_while(|h| sbi_hart_exists(*h))
+        .count()
+        .max(1)
+}
+
+/// Starts `hart` at the physical address `entry` with the MMU off.
+pub fn sbi_hart_start(hart: usize, entry: usize) -> bool {
+    let error: isize;
+    // SAFETY: an SBI call; a0 and a1 come back clobbered.
+    unsafe {
+        core::arch::asm!(
+            "ecall",
+            inlateout("a0") hart => error,
+            inlateout("a1") entry => _,
+            in("a2") 0usize,
+            in("a6") 0usize,
+            in("a7") SBI_EXT_HSM,
+        );
+    }
+    error == 0
+}