feat: SMP stage 1 (4 harts, per-hart guarded stacks, home-core affinity, IPIs, TLB shootdown, one temporary kernel lock), 4-core storm

crates/adapter-riscv/src/main.rs

@@ -9,6 +9,7 @@ mod hw;
 mod kernel_cell;
 mod mmu;
 mod platform;
+mod smp;
 mod trap;
 
 use core::arch::global_asm;
@@ -18,8 +19,8 @@ use kernel_cell::KernelCell;
 static KERNEL: KernelCell<kernel::Kernel<platform::RiscVPlatform>> = KernelCell::uninit();
 
 fn kern_mut() -> &'static mut kernel::Kernel<platform::RiscVPlatform> {
-    // SAFETY: single core; boot code runs before any trap can touch the
-    // kernel.
+    // SAFETY: boot runs with the kernel lock held and before any other
+    // hart has started.
     unsafe { &mut *kernel_ptr() }
 }
 
@@ -27,28 +28,55 @@ pub fn kernel_ptr() -> *mut kernel::Kernel<platform::RiscVPlatform> {
     KERNEL.as_ptr()
 }
 
-// Firmware enters `_start` at the physical load address with the MMU off,
-// but the kernel is linked in the upper half. Using only PC-relative code,
-// the stub turns on `boot_page_table`, which maps RAM twice (at its
-// physical address, just so the next instruction still exists, and in the
-// upper half) plus the MMIO window high, then jumps to its own high address.
-// kmain replaces this table with the real one, which maps nothing low.
+// Firmware enters `_start` (and SBI starts other harts at
+// `_secondary_start`) at the physical load address with the MMU off and
+// the hart id in a0, but the kernel is linked in the upper half. Using
+// only PC-relative code, the stub turns on `boot_page_table`, which maps
+// RAM twice (at its physical address, just so the next instruction still
+// exists, and in the upper half) plus the MMIO window high, then jumps to
+// its own high address and takes the hart's kernel stack. The kernel then
+// switches every hart to the real table, which maps nothing low.
 global_asm!(
     r#"
 .section .text.entry
 .globl _start
 _start:
+    lla t2, start_high_address
+    j 1f
+.globl _secondary_start
+_secondary_start:
+    lla t2, secondary_high_address
+1:
+    li t0, 8                # smp::MAX_HARTS
+    bgeu a0, t0, spin
+    mv s0, a0
     lla t0, boot_page_table
     srli t0, t0, 12
     li t1, 8 << 60
     or t0, t0, t1
     csrw satp, t0
     sfence.vma
-    ld t0, start_high_address
+    ld t0, 0(t2)
     jr t0
+
+# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT
 start_high:
-    lla sp, _stack_top
+    call hart_stack
+    mv a0, s0
     call kmain
+    j spin
+secondary_high:
+    call hart_stack
+    mv a0, s0
+    call kmain_secondary
+    j spin
+hart_stack:
+    lla sp, _stacks_start
+    addi t0, s0, 1
+    li t1, 0x81000
+    mul t0, t0, t1
+    add sp, sp, t0
+    ret
 spin:
     wfi
     j spin
@@ -57,6 +85,8 @@ spin:
 .balign 8
 start_high_address:
     .quad start_high
+secondary_high_address:
+    .quad secondary_high
 .popsection
 
 # Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with
@@ -79,15 +109,21 @@ boot_page_table:
 const SHARED_BUF_VA: usize = 0x30000;
 
 #[unsafe(no_mangle)]
-extern "C" fn kmain() -> ! {
+extern "C" fn kmain(hart: usize) -> ! {
     let uart = Uart::new();
     uart.write_str("k-os: booted on RISC-V\n");
 
-    boot::hardware::init(&uart);
+    boot::hardware::init(&uart, hart);
+    // Held until the boot hart first enters userspace; harts started
+    // below wait for it. Not before init: zeroing .bss resets the lock.
+    smp::lock();
     let mut blk =
         boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device"));
 
-    let (init, _) = boot::loader::load_init(&uart, &mut blk);
+    let (init, init_thread) = boot::loader::load_init(&uart, &mut blk);
+    if let Some(ctx) = kern_mut().thread_context_mut(init_thread) {
+        ctx.registers_mut()[10] = SHARED_BUF_VA;
+    }
     let storage = boot::loader::load_server(&uart, &mut blk, b"storage.elf", "storage");
     let display = boot::loader::load_server(&uart, &mut blk, b"display.elf", "display");
     let input = boot::loader::load_server(&uart, &mut blk, b"input.elf", "input");
@@ -106,8 +142,35 @@ extern "C" fn kmain() -> ! {
         .grant_boot_authority(init.pid, &board)
         .unwrap_or_else(|_| halt(&uart, "boot authority failed"));
 
+    start_other_harts(&uart, hart);
     uart.write_str("k-os: entering userspace\n");
-    boot::loader::enter_userspace(init.pid, init.entry, SHARED_BUF_VA);
+    trap::enter_first();
+}
+
+fn start_other_harts(uart: &Uart, boot: usize) {
+    unsafe extern "C" {
+        fn _secondary_start();
+    }
+    // Harts start with the MMU off, so at the entry's physical address.
+    let entry = mmu::virt_to_phys(_secondary_start as *const () as usize);
+    for hart in (0..smp::num_harts()).filter(|h| *h != boot) {
+        if !smp::sbi_hart_start(hart, entry) {
+            halt(uart, "could not start a hart");
+        }
+    }
+    uart.write_str("k-os: ");
+    uart.write_dec(smp::num_harts());
+    uart.write_str(" harts\n");
+}
+
+/// Where every hart but the boot hart starts, on its own stack.
+#[unsafe(no_mangle)]
+extern "C" fn kmain_secondary(_hart: usize) -> ! {
+    trap::init();
+    hw::timer::init();
+    mmu::activate(mmu::page_table::table_at(smp::kernel_root()));
+    smp::lock();
+    trap::enter_first();
 }
 
 pub fn halt(uart: &Uart, msg: &str) -> ! {