GKaszewski/k-os
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: SMP stage 1 (4 harts, per-hart guarded stacks, home-core affinity, IPIs, TLB shootdown, one temporary kernel lock), 4-core storm

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-27 00:56:15 +01:00
parent b87a62475f
commit 2e665fd66e
22 changed files with 995 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
crates/adapter-riscv/linker.ld
View File

@@ -46,15 +46,13 @@ SECTIONS {
_bss_end = .;
}
/* Left unmapped, so overflowing the kernel stack faults instead of
silently overwriting .bss. */
/* One kernel stack per hart (smp::MAX_HARTS, smp::STACK_SLOT): each
slot is an unmapped guard page, so overflowing a stack faults
instead of silently overwriting what lies below, then the stack. */
. = ALIGN(4096);
_stack_guard = .;
. += 4096;
_stack_bottom = .;
. += 0x80000;
_stack_top = .;
_stacks_start = .;
. += 8 * (4096 + 0x80000);
_stacks_end = .;
_heap_start = .;
}

14
crates/adapter-riscv/src/boot/hardware.rs
View File

@@ -8,12 +8,21 @@ unsafe extern "C" {
static _heap_start: u8;
}
pub fn init(uart: &Uart) -> &'static mut mmu::PageTable {
pub fn init(uart: &Uart, boot_hart: usize) -> &'static mut mmu::PageTable {
zero_bss();
let heap_start = unsafe { &_heap_start as *const u8 as usize };
mmu::init_allocator(heap_start);
let kernel_pt = mmu::init_kernel_page_table();
let harts = crate::smp::probe_harts();
if boot_hart >= harts {
crate::halt(uart, "boot hart outside the hart range");
}
crate::smp::init(
boot_hart,
harts,
mmu::virt_to_phys(kernel_pt as *const mmu::PageTable as usize),
);
mmu::activate(kernel_pt);
uart.write_str("k-os: MMU enabled (Sv39)\n");
match mmu::check_wx(kernel_pt) {
@@ -46,7 +55,8 @@ pub fn init(uart: &Uart) -> &'static mut mmu::PageTable {
if board::USER_POOL_PAGES > domain::memory::frame::MAX_FRAMES {
crate::halt(uart, "user pool larger than the frame table");
}
let riscv_platform = platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize);
let riscv_platform =
platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize, harts as u8);
// SAFETY: `init_in_place` writes every field of the kernel.
unsafe {
KERNEL.init_with(|slot| {

33
crates/adapter-riscv/src/boot/loader.rs
View File

@@ -13,7 +13,6 @@ const MAX_FILE_SIZE: usize = 64 * 1024;
pub struct LoadedProcess {
pub pid: ProcessId,
pub entry: usize,
}
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
@@ -48,7 +47,7 @@ pub fn load_init(uart: &Uart, blk: &mut VirtioBlk) -> (LoadedProcess, ThreadId)
setup_thread(tid, entry);
uart.write_str("k-os: init loaded\n");
(LoadedProcess { pid, entry }, tid)
(LoadedProcess { pid }, tid)
}
pub fn load_server(
@@ -69,35 +68,7 @@ pub fn load_server(
uart.write_str(label);
uart.write_str(" loaded\n");
LoadedProcess { pid, entry }
}
pub fn enter_userspace(pid: ProcessId, entry: usize, shared_buf_va: usize) -> ! {
let pt_addr = kern().platform_mut().page_table_for(pid);
if pt_addr != 0 {
mmu::activate(mmu::page_table::table_at(pt_addr));
}
let kernel_sp: usize;
unsafe { core::arch::asm!("mv {0}, sp", out(reg) kernel_sp) };
crate::trap::set_kernel_stack(kernel_sp);
unsafe {
core::arch::asm!(
"csrc sstatus, {spp}",
"csrs sstatus, {spie}",
"csrw sepc, {entry}",
"mv sp, {stack}",
"mv a0, {arg}",
"sret",
spp = in(reg) 1usize << 8,
spie = in(reg) 1usize << 5,
entry = in(reg) entry,
stack = in(reg) USER_VA_BASE,
arg = in(reg) shared_buf_va,
options(noreturn),
);
}
LoadedProcess { pid }
}
fn setup_thread(tid: ThreadId, entry: usize) {

31
crates/adapter-riscv/src/hw/plic.rs
View File

@@ -2,21 +2,36 @@ use crate::board;
const PLIC: usize = crate::mmu::phys_to_virt(board::PLIC_BASE);
const PLIC_PRIORITY: usize = PLIC;
const PLIC_S_ENABLE: usize = PLIC + 0x2080;
const PLIC_S_THRESHOLD: usize = PLIC + 0x201000;
const PLIC_S_CLAIM: usize = PLIC + 0x201004;
// External interrupts go to the boot hart only: its supervisor context is
// 2 * hart + 1 on QEMU virt (even contexts are machine mode).
fn context() -> usize {
2 * crate::smp::boot_hart() + 1
}
fn enable_base() -> usize {
PLIC + 0x2000 + context() * 0x80
}
fn threshold() -> usize {
PLIC + 0x20_0000 + context() * 0x1000
}
fn claim_reg() -> usize {
threshold() + 4
}
pub struct Plic;
impl Plic {
pub fn init() {
unsafe { core::ptr::write_volatile(PLIC_S_THRESHOLD as *mut u32, 0) };
unsafe { core::ptr::write_volatile(threshold() as *mut u32, 0) };
}
pub fn enable(irq: u32) {
let word = (irq / 32) as usize;
let bit = irq % 32;
let addr = PLIC_S_ENABLE + word * 4;
let addr = enable_base() + word * 4;
unsafe {
let current = core::ptr::read_volatile(addr as *const u32);
core::ptr::write_volatile(addr as *mut u32, current | (1 << bit));
@@ -28,7 +43,7 @@ impl Plic {
pub fn disable(irq: u32) {
let word = (irq / 32) as usize;
let bit = irq % 32;
let addr = PLIC_S_ENABLE + word * 4;
let addr = enable_base() + word * 4;
unsafe {
let current = core::ptr::read_volatile(addr as *const u32);
core::ptr::write_volatile(addr as *mut u32, current & !(1 << bit));
@@ -36,10 +51,10 @@ impl Plic {
}
pub fn claim() -> u32 {
unsafe { core::ptr::read_volatile(PLIC_S_CLAIM as *const u32) }
unsafe { core::ptr::read_volatile(claim_reg() as *const u32) }
}
pub fn complete(irq: u32) {
unsafe { core::ptr::write_volatile(PLIC_S_CLAIM as *mut u32, irq) };
unsafe { core::ptr::write_volatile(claim_reg() as *mut u32, irq) };
}
}

91
crates/adapter-riscv/src/main.rs
View File

@@ -9,6 +9,7 @@ mod hw;
mod kernel_cell;
mod mmu;
mod platform;
mod smp;
mod trap;
use core::arch::global_asm;
@@ -18,8 +19,8 @@ use kernel_cell::KernelCell;
static KERNEL: KernelCell<kernel::Kernel<platform::RiscVPlatform>> = KernelCell::uninit();
fn kern_mut() -> &'static mut kernel::Kernel<platform::RiscVPlatform> {
// SAFETY: single core; boot code runs before any trap can touch the
// kernel.
// SAFETY: boot runs with the kernel lock held and before any other
// hart has started.
unsafe { &mut *kernel_ptr() }
}
@@ -27,28 +28,55 @@ pub fn kernel_ptr() -> *mut kernel::Kernel<platform::RiscVPlatform> {
KERNEL.as_ptr()
}
// Firmware enters `_start` at the physical load address with the MMU off,
// but the kernel is linked in the upper half. Using only PC-relative code,
// the stub turns on `boot_page_table`, which maps RAM twice (at its
// physical address, just so the next instruction still exists, and in the
// upper half) plus the MMIO window high, then jumps to its own high address.
// kmain replaces this table with the real one, which maps nothing low.
// Firmware enters `_start` (and SBI starts other harts at
// `_secondary_start`) at the physical load address with the MMU off and
// the hart id in a0, but the kernel is linked in the upper half. Using
// only PC-relative code, the stub turns on `boot_page_table`, which maps
// RAM twice (at its physical address, just so the next instruction still
// exists, and in the upper half) plus the MMIO window high, then jumps to
// its own high address and takes the hart's kernel stack. The kernel then
// switches every hart to the real table, which maps nothing low.
global_asm!(
r#"
.section .text.entry
.globl _start
_start:
lla t2, start_high_address
j 1f
.globl _secondary_start
_secondary_start:
lla t2, secondary_high_address
1:
li t0, 8 # smp::MAX_HARTS
bgeu a0, t0, spin
mv s0, a0
lla t0, boot_page_table
srli t0, t0, 12
li t1, 8 << 60
or t0, t0, t1
csrw satp, t0
sfence.vma
ld t0, start_high_address
ld t0, 0(t2)
jr t0
# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT
start_high:
lla sp, _stack_top
call hart_stack
mv a0, s0
call kmain
j spin
secondary_high:
call hart_stack
mv a0, s0
call kmain_secondary
j spin
hart_stack:
lla sp, _stacks_start
addi t0, s0, 1
li t1, 0x81000
mul t0, t0, t1
add sp, sp, t0
ret
spin:
wfi
j spin
@@ -57,6 +85,8 @@ spin:
.balign 8
start_high_address:
.quad start_high
secondary_high_address:
.quad secondary_high
.popsection
# Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with
@@ -79,15 +109,21 @@ boot_page_table:
const SHARED_BUF_VA: usize = 0x30000;
#[unsafe(no_mangle)]
extern "C" fn kmain() -> ! {
extern "C" fn kmain(hart: usize) -> ! {
let uart = Uart::new();
uart.write_str("k-os: booted on RISC-V\n");
boot::hardware::init(&uart);
boot::hardware::init(&uart, hart);
// Held until the boot hart first enters userspace; harts started
// below wait for it. Not before init: zeroing .bss resets the lock.
smp::lock();
let mut blk =
boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device"));
let (init, _) = boot::loader::load_init(&uart, &mut blk);
let (init, init_thread) = boot::loader::load_init(&uart, &mut blk);
if let Some(ctx) = kern_mut().thread_context_mut(init_thread) {
ctx.registers_mut()[10] = SHARED_BUF_VA;
}
let storage = boot::loader::load_server(&uart, &mut blk, b"storage.elf", "storage");
let display = boot::loader::load_server(&uart, &mut blk, b"display.elf", "display");
let input = boot::loader::load_server(&uart, &mut blk, b"input.elf", "input");
@@ -106,8 +142,35 @@ extern "C" fn kmain() -> ! {
.grant_boot_authority(init.pid, &board)
.unwrap_or_else(|_| halt(&uart, "boot authority failed"));
start_other_harts(&uart, hart);
uart.write_str("k-os: entering userspace\n");
boot::loader::enter_userspace(init.pid, init.entry, SHARED_BUF_VA);
trap::enter_first();
}
fn start_other_harts(uart: &Uart, boot: usize) {
unsafe extern "C" {
fn _secondary_start();
}
// Harts start with the MMU off, so at the entry's physical address.
let entry = mmu::virt_to_phys(_secondary_start as *const () as usize);
for hart in (0..smp::num_harts()).filter(|h| *h != boot) {
if !smp::sbi_hart_start(hart, entry) {
halt(uart, "could not start a hart");
}
}
uart.write_str("k-os: ");
uart.write_dec(smp::num_harts());
uart.write_str(" harts\n");
}
/// Where every hart but the boot hart starts, on its own stack.
#[unsafe(no_mangle)]
extern "C" fn kmain_secondary(_hart: usize) -> ! {
trap::init();
hw::timer::init();
mmu::activate(mmu::page_table::table_at(smp::kernel_root()));
smp::lock();
trap::enter_first();
}
pub fn halt(uart: &Uart, msg: &str) -> ! {

44
crates/adapter-riscv/src/mmu/page_table.rs
View File

@@ -1,6 +1,7 @@
use super::allocator::alloc_page_table;
use super::{phys_to_virt, virt_to_phys};
use crate::board;
use crate::smp;
use platform::{FrameSource, MapError};
use types::address::{PhysicalAddress, VirtualAddress};
use types::mapping::PagePerms;
@@ -148,20 +149,31 @@ pub fn init_kernel_page_table() -> &'static mut PageTable {
// The kernel image, then the rest of RAM, each with only the access it
// needs: code is never writable, data never executable. RAM below the
// image belongs to the firmware and stays unmapped, as does the guard
// page under the kernel stack. Nothing is mapped in the lower half,
// which is userspace's alone.
// page under each hart's kernel stack. Nothing is mapped in the lower
// half, which is userspace's alone.
let layout = KernelLayout::from_linker();
let ram_end = phys_to_virt(board::RAM_BASE + board::RAM_SIZE);
let stacks = (0..smp::MAX_HARTS).map(|h| {
let slot = layout.stacks_start + h * smp::STACK_SLOT;
(
slot + PAGE_SIZE,
slot + smp::STACK_SLOT,
Sv39Flags::kernel_rw(),
)
});
for (start, end, flags) in [
(layout.text_start, layout.text_end, Sv39Flags::kernel_rx()),
(layout.text_end, layout.rodata_end, Sv39Flags::kernel_ro()),
(
layout.rodata_end,
layout.stack_guard,
layout.stacks_start,
Sv39Flags::kernel_rw(),
),
(layout.stack_bottom, ram_end, Sv39Flags::kernel_rw()),
] {
(layout.stacks_end, ram_end, Sv39Flags::kernel_rw()),
]
.into_iter()
.chain(stacks)
{
let (va, pa) = direct(virt_to_phys(start));
map(
root,
@@ -200,8 +212,8 @@ pub struct KernelLayout {
pub text_start: usize,
pub text_end: usize,
pub rodata_end: usize,
pub stack_guard: usize,
pub stack_bottom: usize,
pub stacks_start: usize,
pub stacks_end: usize,
}
impl KernelLayout {
@@ -210,8 +222,8 @@ impl KernelLayout {
static _text_start: u8;
static _text_end: u8;
static _rodata_end: u8;
static _stack_guard: u8;
static _stack_bottom: u8;
static _stacks_start: u8;
static _stacks_end: u8;
}
// SAFETY: only the symbols' addresses are taken, never their values.
unsafe {
@@ -219,8 +231,8 @@ impl KernelLayout {
text_start: &_text_start as *const u8 as usize,
text_end: &_text_end as *const u8 as usize,
rodata_end: &_rodata_end as *const u8 as usize,
stack_guard: &_stack_guard as *const u8 as usize,
stack_bottom: &_stack_bottom as *const u8 as usize,
stacks_start: &_stacks_start as *const u8 as usize,
stacks_end: &_stacks_end as *const u8 as usize,
}
}
}
@@ -240,12 +252,15 @@ pub enum WxViolation {
/// Walks every leaf of `root` and checks the kernel layout: nothing is
/// both writable and executable, only kernel text is executable, the
/// firmware and the stack guard are unmapped, and nothing at all is mapped
/// firmware and every stack guard are unmapped, and nothing at all is mapped
/// in the lower half. Returns the number of executable pages.
pub fn check_wx(root: &PageTable) -> Result<usize, WxViolation> {
let layout = KernelLayout::from_linker();
let text = layout.text_start..layout.text_end;
let guard = layout.stack_guard..layout.stack_bottom;
let is_guard = |va: usize| {
(layout.stacks_start..layout.stacks_end).contains(&va)
&& (va - layout.stacks_start).is_multiple_of(smp::STACK_SLOT)
};
let firmware = phys_to_virt(board::RAM_BASE)..layout.text_start;
let mut executable = 0;
for (va, bits) in leaves(root) {
@@ -261,7 +276,7 @@ pub fn check_wx(root: &PageTable) -> Result<usize, WxViolation> {
}
executable += 1;
}
if guard.contains(&va) || firmware.contains(&va) {
if is_guard(va) || firmware.contains(&va) {
return Err(WxViolation::MappedHole(va));
}
}
@@ -299,6 +314,7 @@ fn leaves(root: &PageTable) -> impl Iterator<Item = (usize, u64)> + '_ {
}
pub fn activate(root: &PageTable) {
smp::note_active_root(virt_to_phys(root as *const PageTable as usize));
let satp = (8u64 << 60) | table_ppn(root);
unsafe {
core::arch::asm!(

19
crates/adapter-riscv/src/platform.rs
View File

@@ -10,6 +10,7 @@ use types::{
use crate::board;
use crate::hw::{plic::Plic, timer};
use crate::mmu;
use crate::smp;
use crate::trap;
pub struct RiscVPlatform {
@@ -20,9 +21,9 @@ pub struct RiscVPlatform {
}
impl RiscVPlatform {
pub fn new(kernel_pt_addr: usize) -> Self {
pub fn new(kernel_pt_addr: usize, num_cores: u8) -> Self {
Self {
num_cores: 1,
num_cores,
kernel_pt: kernel_pt_addr,
page_tables: [0usize; MAX_PROCESSES],
}
@@ -63,8 +64,10 @@ impl Platform for RiscVPlatform {
let Some(root) = self.root(process) else {
return;
};
// The caller may be tearing down the space we are running on; get
// off it before the kernel frees its frames.
// No hart may still translate through it once the kernel frees its
// frames: other harts leave it (and wait for nothing but that), and
// the caller may be tearing down the space we are running on.
smp::shoot_down(root);
if mmu::active_root() == root {
// SAFETY: the kernel page table lives in the boot heap forever.
mmu::activate(unsafe { &*(self.kernel_pt as *const mmu::PageTable) });
@@ -95,6 +98,8 @@ impl Platform for RiscVPlatform {
fn unmap_page(&mut self, process: ProcessId, va: VirtualAddress) {
if let Some(root) = self.root(process) {
mmu::unmap_user_page(root, va.raw());
// Other harts running this process may still cache it.
smp::shoot_down(root);
}
}
@@ -144,10 +149,12 @@ impl Platform for RiscVPlatform {
Plic::enable(irq.raw() as u32);
}
fn send_ipi(&mut self, _target: CoreId) {}
fn send_ipi(&mut self, target: CoreId) {
smp::kick(target.raw() as usize);
}
fn core_id(&self) -> CoreId {
CoreId::from_raw(0)
CoreId::from_raw(smp::hart_id() as u8)
}
fn num_cores(&self) -> u8 {

220
crates/adapter-riscv/src/smp.rs Normal file
View File

@@ -0,0 +1,220 @@
//! Harts. Each hart has its own kernel stack, and its hart id doubles as
//! its kernel `CoreId`. The kernel is entered under one lock for now (stage
//! 1 of multi-core; the kernel itself stays a sequential program), so
//! userspace runs in parallel and the kernel does not.
//!
//! Harts talk through IPIs carrying two kinds of request:
//! - reschedule: the kernel made work for this hart, or took its thread
//! away; handled under the lock on the way out of the kernel.
//! - leave root R: a page table this hart may be using is changing or about
//! to be freed; the hart switches to the kernel's root and flushes its
//! TLB. This needs no lock, so it is serviced even while a hart spins
//! waiting for the lock or idles, and the sender (which holds the lock)
//! can wait for the acknowledgement without deadlocking.
use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
use crate::mmu;
/// Must match linker.ld.
pub const MAX_HARTS: usize = 8;
/// A guard page and a 512 KiB stack (boot builds big tables on it). Must
/// match linker.ld.
pub const STACK_SLOT: usize = 4096 + 0x8_0000;
const SIP_SSIP: usize = 1 << 1;
/// The hart running this code, from which kernel stack it is on.
pub fn hart_id() -> usize {
let sp: usize;
// SAFETY: reading sp has no side effects.
unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) };
(sp - stacks_start()) / STACK_SLOT
}
pub fn stacks_start() -> usize {
unsafe extern "C" {
static _stacks_start: u8;
}
// SAFETY: only the symbol's address is taken.
unsafe { &_stacks_start as *const u8 as usize }
}
static BOOT_HART: AtomicUsize = AtomicUsize::new(0);
static NUM_HARTS: AtomicUsize = AtomicUsize::new(1);
/// Physical address of the kernel's own root table.
static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0);
/// The root table each hart translates with.
static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS];
static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
pub fn boot_hart() -> usize {
BOOT_HART.load(Ordering::Relaxed)
}
pub fn num_harts() -> usize {
NUM_HARTS.load(Ordering::Relaxed)
}
pub fn kernel_root() -> usize {
KERNEL_ROOT.load(Ordering::Relaxed)
}
/// Records which hart booted and how many there are, and where the
/// kernel's root table is.
pub fn init(boot: usize, harts: usize, kernel_root: usize) {
BOOT_HART.store(boot, Ordering::Relaxed);
NUM_HARTS.store(harts, Ordering::Relaxed);
KERNEL_ROOT.store(kernel_root, Ordering::Relaxed);
}
/// Called by `mmu::activate`.
pub fn note_active_root(root: usize) {
if let Some(r) = ACTIVE_ROOT.get(hart_id()) {
r.store(root, Ordering::SeqCst);
}
}
// ---- the kernel lock ----
static NEXT_TICKET: AtomicU32 = AtomicU32::new(0);
static SERVING: AtomicU32 = AtomicU32::new(0);
/// Takes the kernel lock, first come first served. Leave-root requests
/// are serviced while waiting.
pub fn lock() {
let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
while SERVING.load(Ordering::Acquire) != ticket {
service_ipi();
core::hint::spin_loop();
}
}
pub fn unlock() {
SERVING.fetch_add(1, Ordering::Release);
}
// ---- IPIs ----
/// The lock-free half of an IPI: acknowledges it and, if asked, leaves the
/// root being changed. Safe to call at any time, any number of times.
pub fn service_ipi() {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & SIP_SSIP == 0 {
return;
}
// Clear first: a request posted after this raises SSIP again.
// SAFETY: clearing our own pending software interrupt.
unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) };
let me = hart_id();
let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else {
return;
};
let root = leave.swap(0, Ordering::SeqCst);
if root != 0 {
if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) {
mmu::activate(mmu::page_table::table_at(kernel_root()));
}
// SAFETY: flushing translations has no other effect.
unsafe { core::arch::asm!("sfence.vma zero, zero") };
left.fetch_add(1, Ordering::SeqCst);
}
}
/// Asks `hart` to look at its run queue again, on its way out of the
/// kernel.
pub fn kick(hart: usize) {
if let Some(r) = RESCHEDULE.get(hart) {
r.store(true, Ordering::SeqCst);
sbi_send_ipi(hart);
}
}
/// Consumes this hart's reschedule request.
pub fn take_reschedule() -> bool {
RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.swap(false, Ordering::SeqCst))
}
/// Makes every other hart that translates through `root` leave it and
/// flush, and waits until each has. Called with the kernel lock held, so
/// no hart can start using `root` meanwhile.
pub fn shoot_down(root: usize) {
let me = hart_id();
for hart in 0..num_harts() {
if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root {
continue;
}
let seen = LEFT[hart].load(Ordering::SeqCst);
LEAVE_ROOT[hart].store(root, Ordering::SeqCst);
sbi_send_ipi(hart);
while LEFT[hart].load(Ordering::SeqCst) == seen {
core::hint::spin_loop();
}
}
}
// ---- SBI ----
const SBI_EXT_IPI: usize = 0x735049;
const SBI_EXT_HSM: usize = 0x48534D;
fn sbi_send_ipi(hart: usize) {
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") 1usize => _,
inlateout("a1") hart => _,
in("a6") 0usize,
in("a7") SBI_EXT_IPI,
);
}
}
/// Whether `hart` exists, per SBI HSM `hart_get_status`.
fn sbi_hart_exists(hart: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
lateout("a1") _,
in("a6") 2usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}
/// The number of harts, counting up from hart 0 until one is missing.
pub fn probe_harts() -> usize {
(0..MAX_HARTS)
.take_while(|h| sbi_hart_exists(*h))
.count()
.max(1)
}
/// Starts `hart` at the physical address `entry` with the MMU off.
pub fn sbi_hart_start(hart: usize, entry: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
inlateout("a1") entry => _,
in("a2") 0usize,
in("a6") 0usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}

235
crates/adapter-riscv/src/trap/mod.rs
View File

@@ -60,7 +60,14 @@ _from_user:
mv a0, sp
call trap_handler_rust
j _restore_frame
# trap_return(frame): resume whatever `frame` describes, as if returning
# from a trap that saved it. Used to enter userspace for the first time.
.globl trap_return
trap_return:
mv sp, a0
_restore_frame:
ld t0, 256(sp)
csrw sepc, t0
ld t0, 264(sp)
@@ -121,6 +128,7 @@ _sret_smode:
);
const CAUSE_INTERRUPT_BIT: usize = 1 << 63;
const CAUSE_S_SOFTWARE: usize = 1;
const CAUSE_S_TIMER: usize = 5;
const CAUSE_S_EXTERNAL: usize = 9;
const CAUSE_ECALL_U: usize = 8;
@@ -177,88 +185,67 @@ fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFra
}
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(result) = kern.take_pending_result(tid) {
if let Some(result) = kern().take_pending_result(tid) {
write_result(frame, &result);
}
}
unsafe extern "C" {
fn trap_return(frame: *mut TrapFrame) -> !;
}
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
// SAFETY: only called with the kernel lock held.
unsafe { &mut *crate::kernel_ptr() }
}
fn this_core() -> types::id::CoreId {
types::id::CoreId::from_raw(crate::smp::hart_id() as u8)
}
/// Saves `frame` into `from` (if it still exists) and loads `to`.
fn switch_to_thread(
frame: &mut TrapFrame,
current_tid: types::id::ThreadId,
next_tid: types::id::ThreadId,
from: Option<types::id::ThreadId>,
to: types::id::ThreadId,
) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(ctx) = kern.thread_context_mut(current_tid) {
let kern = kern();
if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next_pid = kern.thread_process(next_tid);
let current_pid = kern.thread_process(current_tid);
if let Some(ctx) = kern.thread_context(next_tid) {
if let Some(ctx) = kern.thread_context(to) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, next_tid);
if next_pid != current_pid
&& let Some(pid) = next_pid
{
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = crate::mmu::page_table::table_at(pt_addr);
crate::mmu::activate(pt);
}
}
apply_pending_result(frame, to);
}
fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(ctx) = kern.thread_context_mut(current_tid) {
/// Saves `from` and runs the next ready thread, idling until there is one.
fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next_tid = next_runnable();
let kern = unsafe { &mut *crate::kernel_ptr() };
let next_pid = kern.thread_process(next_tid);
let current_pid = kern.thread_process(current_tid);
let core = types::id::CoreId::from_raw(0);
let _ = kern.set_thread_running(next_tid, core);
if let Some(ctx) = kern.thread_context(next_tid) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, next_tid);
if next_pid != current_pid
&& let Some(pid) = next_pid
{
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = crate::mmu::page_table::table_at(pt_addr);
crate::mmu::activate(pt);
}
}
let next = next_runnable();
let _ = kern().set_thread_running(next, this_core());
switch_to_thread(frame, None, next);
}
/// The next thread to run, idling until an interrupt makes one ready.
/// The next thread to run on this hart, idling until one is ready.
///
/// We are inside a trap, so SIE is clear and interrupts are not taken.
/// `wfi` still wakes once one is pending, and it is then serviced here by
/// polling `sip`, which is how a driver blocked on its IRQ notification
/// gets woken when nothing else can run.
/// Idling gives up the kernel lock: an idle hart holds nothing, so other
/// harts can make work for it (and kick it) meanwhile. It also leaves any
/// user address space first, so nobody needs to shoot it down. SIE stays
/// clear; `wfi` still wakes on a pending interrupt, serviced by polling.
fn next_runnable() -> types::id::ThreadId {
loop {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(tid) = kern.pick_next_thread() {
if let Some(tid) = kern().pick_next_thread() {
return tid;
}
crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
crate::smp::unlock();
unsafe { core::arch::asm!("wfi") };
crate::smp::service_ipi();
crate::smp::lock();
let _ = crate::smp::take_reschedule();
let sip: usize;
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & (1 << CAUSE_S_TIMER) != 0 {
@@ -271,49 +258,123 @@ fn next_runnable() -> types::id::ThreadId {
}
}
/// Claims every pending PLIC interrupt and hands each to the kernel.
/// Claims every pending PLIC interrupt and hands each to the kernel. Only
/// the boot hart receives external interrupts.
fn dispatch_external() {
loop {
let irq_num = crate::hw::plic::Plic::claim();
if irq_num == 0 {
return;
}
let kern = unsafe { &mut *crate::kernel_ptr() };
kern.handle_interrupt(types::id::IrqId::from_raw(irq_num as u16));
kern().handle_interrupt(types::id::IrqId::from_raw(irq_num as u16));
crate::hw::plic::Plic::complete(irq_num);
}
}
/// Last step before leaving the kernel to userspace: act on a reschedule
/// request, make sure this hart has a thread (its own may have been taken
/// away by another hart), and translate through that thread's address
/// space.
fn finish(frame: &mut TrapFrame) {
let kern = kern();
let current = kern.current_thread();
if crate::smp::take_reschedule()
&& let Some(cur) = current
&& let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
{
switch_to_thread(frame, Some(cur), next);
}
if kern.current_thread().is_none() {
switch_to_next(frame, None);
}
let root = kern
.current_thread()
.and_then(|t| kern.thread_process(t))
.map(|pid| kern.platform_mut().page_table_for(pid))
.unwrap_or(0);
if root != 0 && crate::mmu::active_root() != root {
crate::mmu::activate(crate::mmu::page_table::table_at(root));
}
}
/// Leaves the kernel for userspace for the first time on this hart, with
/// the kernel lock held: runs the thread the kernel already made current
/// here (init, on the boot hart) or the next one homed here.
pub fn enter_first() -> ! {
let mut frame = TrapFrame {
regs: [0; 32],
sepc: 0,
sstatus: 0,
};
match kern().current_thread() {
Some(tid) => switch_to_thread(&mut frame, None, tid),
None => switch_to_next(&mut frame, None),
}
finish(&mut frame);
crate::smp::unlock();
// SAFETY: `frame` holds a user context; sscratch is set from it on
// the way out, so later traps land at the top of this frame.
unsafe { trap_return(&mut frame) }
}
#[unsafe(no_mangle)]
extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
let scause: usize;
unsafe { core::arch::asm!("csrr {0}, scause", out(reg) scause) };
if frame.sstatus & SSTATUS_SPP != 0 {
// Checked before taking the lock: this hart may already hold it.
kernel_fault(frame, scause);
}
crate::smp::service_ipi();
crate::smp::lock();
if scause & CAUSE_INTERRUPT_BIT != 0 {
handle_interrupt(frame, scause & !CAUSE_INTERRUPT_BIT);
} else {
handle_exception(frame, scause);
}
finish(frame);
crate::smp::unlock();
}
fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
match cause {
CAUSE_S_TIMER => {
crate::hw::timer::handle_interrupt();
let kern = unsafe { &mut *crate::kernel_ptr() };
let core = types::id::CoreId::from_raw(0);
let kern = kern();
if let Some(current_tid) = kern.current_thread() {
let decision = kern.handle_tick(core);
let decision = kern.handle_tick(this_core());
if let domain::scheduler::SchedulerDecision::Preempt(next) = decision {
switch_to_thread(frame, current_tid, next);
switch_to_thread(frame, Some(current_tid), next);
}
}
}
CAUSE_S_EXTERNAL => dispatch_external(),
// Serviced on entry; any reschedule happens in `finish`.
CAUSE_S_SOFTWARE => {}
_ => {}
}
}
/// The kernel itself faulted (a W^X violation, a stack guard, a bad
/// pointer). Its state can no longer be trusted, and no user thread is to
/// blame.
fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! {
let uart = crate::hw::uart::Uart::new();
uart.write_str("KERNEL FAULT: scause=");
uart.write_hex(scause);
let stval: usize;
unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
uart.write_str(" stval=");
uart.write_hex(stval);
uart.write_str(" sepc=");
uart.write_hex(frame.sepc);
uart.write_str(" hart=");
uart.write_dec(crate::smp::hart_id());
uart.put_char(b'\n');
crate::halt(&uart, "fault in kernel mode");
}
fn handle_exception(frame: &mut TrapFrame, scause: usize) {
match scause {
CAUSE_ECALL_U => {
@@ -324,48 +385,24 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
return;
};
{
let kern = unsafe { &mut *crate::kernel_ptr() };
let kern = kern();
let current_tid = kern.current_thread();
let outcome = kern.handle_syscall(syscall);
match outcome {
kernel::SyscallOutcome::Complete(result) => write_result(frame, &result),
kernel::SyscallOutcome::BlockCaller(_) => {
if let Some(tid) = current_tid {
switch_to_next(frame, tid);
}
}
kernel::SyscallOutcome::BlockCaller(_) => switch_to_next(frame, current_tid),
kernel::SyscallOutcome::SwitchTo(next) => {
if let Some(tid) = current_tid {
switch_to_thread(frame, tid, next);
}
switch_to_thread(frame, current_tid, next)
}
kernel::SyscallOutcome::TerminateCaller => {
let uart = crate::hw::uart::Uart::new();
uart.write_str("k-os: process exited\n");
if let Some(tid) = current_tid {
switch_to_next(frame, tid);
}
switch_to_next(frame, current_tid);
}
}
}
}
_ if frame.sstatus & SSTATUS_SPP != 0 => {
// The kernel itself faulted (a W^X violation, the stack guard,
// a bad pointer). Its state can no longer be trusted, and no
// user thread is to blame.
let uart = crate::hw::uart::Uart::new();
uart.write_str("KERNEL FAULT: scause=");
uart.write_hex(scause);
let stval: usize;
unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
uart.write_str(" stval=");
uart.write_hex(stval);
uart.write_str(" sepc=");
uart.write_hex(frame.sepc);
uart.put_char(b'\n');
crate::halt(&uart, "fault in kernel mode");
}
_ => {
let uart = crate::hw::uart::Uart::new();
uart.write_str("FAULT: scause=");
@@ -378,7 +415,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
uart.write_hex(frame.sepc);
uart.put_char(b'\n');
let kern = unsafe { &mut *crate::kernel_ptr() };
let kern = kern();
if let Some(current_tid) = kern.current_thread() {
let fault = match scause {
2 => domain::fault::Fault::IllegalInstruction,
@@ -388,12 +425,10 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
_ => domain::fault::Fault::InvalidSyscall,
};
kern.handle_fault(current_tid, fault, frame.sepc);
switch_to_next(frame, current_tid);
} else {
loop {
unsafe { core::arch::asm!("wfi") };
}
switch_to_next(frame, Some(current_tid));
}
// With no current thread (another hart just destroyed it),
// `finish` moves on to the next one.
}
}
}

17
crates/domain/src/scheduler/multi_core.rs
View File

@@ -47,6 +47,23 @@ impl MultiCoreScheduler {
self.placements.get(thread.raw() as usize)?.as_ref()
}
pub fn num_cores(&self) -> u8 {
self.num_cores
}
/// The highest priority waiting in `core`'s run queue.
pub fn highest_waiting(&self, core: CoreId) -> Option<Priority> {
self.core(core)?.highest_waiting_priority()
}
/// The core whose run queue holds `thread`, if any.
pub fn queued_on(&self, thread: ThreadId) -> Option<CoreId> {
self.active()
.iter()
.find(|c| c.is_queued(thread))
.map(|c| c.core())
}
/// Whether `thread` waits in some core's run queue.
pub fn is_queued(&self, thread: ThreadId) -> bool {
self.active().iter().any(|c| c.is_queued(thread))

8
crates/domain/src/thread/mod.rs
View File

@@ -306,6 +306,14 @@ impl ThreadTable {
Ok(id)
}
/// Every live thread with its id.
pub fn iter(&self) -> impl Iterator<Item = (ThreadId, &Thread)> {
self.threads
.iter()
.enumerate()
.filter_map(|(i, t)| Some((ThreadId::from_raw(i as u16), t.as_ref()?)))
}
pub fn get(&self, id: ThreadId) -> Result<&Thread, KernelError> {
let idx = id.raw() as usize;
self.threads

37
crates/kernel/src/lib.rs
View File

@@ -21,6 +21,7 @@ pub mod interrupt;
mod memory;
#[cfg(kani)]
pub mod proofs;
mod smp;
pub mod syscall;
pub use board::BoardDevice;
@@ -65,6 +66,8 @@ pub struct Kernel<P: Platform> {
notifications: NotificationTable,
ipc: [syscall::IpcState; MAX_THREADS],
pending_results: [Option<SyscallResult>; MAX_THREADS],
/// The core each thread runs on (see smp.rs).
homes: [types::id::CoreId; MAX_THREADS],
}
impl<P: Platform> Kernel<P> {
@@ -83,6 +86,7 @@ impl<P: Platform> Kernel<P> {
notifications: NotificationTable::new(),
ipc: [syscall::IpcState::default(); MAX_THREADS],
pending_results: [const { None }; MAX_THREADS],
homes: [types::id::CoreId::from_raw(0); MAX_THREADS],
}
}
@@ -115,6 +119,7 @@ impl<P: Platform> Kernel<P> {
addr_of_mut!((*slot).notifications).write(NotificationTable::new());
addr_of_mut!((*slot).ipc).write([syscall::IpcState::default(); MAX_THREADS]);
addr_of_mut!((*slot).pending_results).write([const { None }; MAX_THREADS]);
addr_of_mut!((*slot).homes).write([types::id::CoreId::from_raw(0); MAX_THREADS]);
}
}
@@ -142,6 +147,7 @@ impl<P: Platform> Kernel<P> {
.insert(cap)?;
let core = self.platform.core_id();
self.set_home(tid, core);
self.scheduler.add(tid, priority, core);
self.scheduler.next(core);
self.threads.get_mut(tid)?.set_running(core)?;
@@ -211,14 +217,13 @@ impl<P: Platform> Kernel<P> {
}
pub fn spawn_thread_internal(&mut self, pid: ProcessId) -> Option<ThreadId> {
use domain::scheduler::Scheduler;
let tid = self
.threads
.spawn(pid, Priority::DEFAULT, TimeSlice::DEFAULT, None)
.ok()?;
let _ = self.processes.get_mut(pid).map(|p| p.add_thread(tid));
let core = self.platform.core_id();
self.scheduler.add(tid, Priority::DEFAULT, core);
self.place(tid);
self.make_ready(tid, Priority::DEFAULT);
Some(tid)
}
@@ -240,31 +245,6 @@ impl<P: Platform> Kernel<P> {
self.grant_cap(to, resource, permissions)
}
pub fn switch_to_thread(&mut self, tid: ThreadId) {
use domain::scheduler::Scheduler;
let core = self.platform.core_id();
if let Some(current) = self.scheduler.current(core) {
if current == tid {
return;
}
if let Ok(t) = self.threads.get_mut(current)
&& matches!(t.state(), ThreadState::Running(_))
{
let priority = t.priority();
let _ = t.preempt();
self.scheduler.add(current, priority, core);
}
}
if let Ok(t) = self.threads.get(tid)
&& matches!(t.state(), ThreadState::Ready)
&& self.scheduler.run_now(core, tid)
{
let _ = self.threads.get_mut(tid).map(|t| t.set_running(core));
}
}
fn caller(&self) -> Result<(ThreadId, ProcessId), KernelError> {
let core = self.platform.core_id();
let tid = self
@@ -311,6 +291,7 @@ impl<P: Platform> Kernel<P> {
if let Some(caller) = ipc.reply_to {
self.release_caller(caller);
}
self.evict(*tid);
self.scheduler.remove(*tid);
self.set_pending_result(*tid, None);
let _ = self.threads.destroy(*tid);

119
crates/kernel/src/smp.rs Normal file
View File

@@ -0,0 +1,119 @@
//! Cores. Every thread has a home core, chosen when it is created, and
//! only ever runs there: it waits in that core's run queue and is never
//! migrated. Making a thread ready from another core sends that core an
//! IPI when it has to look at its queue again (it is idle, or the thread
//! outranks what it is running), and so does taking away the thread a
//! core is running.
use domain::{
scheduler::{Scheduler, SchedulerDecision},
thread::ThreadState,
};
use platform::Platform;
use types::{
bounded::Priority,
id::{CoreId, ThreadId},
};
use crate::Kernel;
impl<P: Platform> Kernel<P> {
/// The core `tid` runs on.
pub fn home(&self, tid: ThreadId) -> CoreId {
self.homes
.get(tid.raw() as usize)
.copied()
.unwrap_or(CoreId::from_raw(0))
}
/// Homes a new thread on `core`.
pub(crate) fn set_home(&mut self, tid: ThreadId, core: CoreId) {
if let Some(h) = self.homes.get_mut(tid.raw() as usize) {
*h = core;
}
}
/// Homes a new thread on the core with the fewest live threads.
pub(crate) fn place(&mut self, tid: ThreadId) {
let cores = self.scheduler.num_cores().max(1);
let load = |core: CoreId| {
self.threads
.iter()
.filter(|(t, _)| *t != tid && self.home(*t) == core)
.count()
};
let best = (0..cores)
.map(CoreId::from_raw)
.min_by_key(|c| load(*c))
.unwrap_or(CoreId::from_raw(0));
self.set_home(tid, best);
}
/// Queues a ready thread on its home core, and kicks that core if it
/// is another one that should look at its queue now.
pub(crate) fn make_ready(&mut self, tid: ThreadId, priority: Priority) {
let home = self.home(tid);
self.scheduler.add(tid, priority, home);
if home == self.platform.core_id() {
return;
}
let outranks = match self.scheduler.current(home) {
None => true,
Some(running) => self
.threads
.get(running)
.is_ok_and(|t| priority > t.priority()),
};
if outranks {
self.platform.send_ipi(home);
}
}
/// Before `tid` is destroyed: if another core is running it, that core
/// must stop and pick something else.
pub(crate) fn evict(&mut self, tid: ThreadId) {
if let Ok(ThreadState::Running(core)) = self.threads.get(tid).map(|t| *t.state())
&& core != self.platform.core_id()
{
self.platform.send_ipi(core);
}
}
/// What `core` should do after an IPI: keep running its thread, or
/// switch to a higher-priority one now waiting. When it has no thread
/// (it was idle, or its thread was destroyed) the caller picks the
/// next one itself.
pub fn reschedule(&mut self, core: CoreId) -> SchedulerDecision {
let Some(current) = self.scheduler.current(core) else {
return SchedulerDecision::Continue;
};
let Ok(thread) = self.threads.get(current) else {
return SchedulerDecision::Continue;
};
let priority = thread.priority();
if self
.scheduler
.highest_waiting(core)
.is_none_or(|p| p <= priority)
{
return SchedulerDecision::Continue;
}
let Some(next) = self.scheduler.next(core) else {
return SchedulerDecision::Continue;
};
if self
.threads
.get_mut(next)
.and_then(|t| t.set_running(core))
.is_err()
{
return SchedulerDecision::Continue;
}
if let Ok(t) = self.threads.get_mut(current)
&& t.preempt().is_ok()
{
self.scheduler.add(current, priority, core);
}
SchedulerDecision::Preempt(next)
}
}

5
crates/kernel/src/syscall/ipc.rs
View File

@@ -102,8 +102,7 @@ impl<P: Platform> Kernel<P> {
let priority = thread.priority();
if thread.unblock().is_ok() {
self.set_pending_result(tid, Some(result));
let core = self.platform.core_id();
self.scheduler.add(tid, priority, core);
self.make_ready(tid, priority);
}
}
@@ -234,6 +233,8 @@ impl<P: Platform> Kernel<P> {
}
self.block_caller(tid, BlockReason::AwaitingReply);
// Run the server straight away instead of waiting for its turn.
// Only works if it lives on this core (run_now finds it in this
// core's queue only); otherwise make_ready kicked its core.
let core = self.platform.core_id();
if self.scheduler.run_now(core, receiver)
&& self

10
crates/kernel/src/syscall/lifecycle.rs
View File

@@ -10,7 +10,6 @@ impl<P: Platform> Kernel<P> {
entry: types::address::EntryPoint,
stack: types::address::StackPointer,
) -> SyscallOutcome {
use domain::scheduler::Scheduler;
use domain::thread::SavedContext;
let priority = types::bounded::Priority::DEFAULT;
@@ -30,8 +29,8 @@ impl<P: Platform> Kernel<P> {
return SyscallOutcome::Complete(SyscallResult::Error(e));
}
let core = self.platform.core_id();
self.scheduler.add(tid, priority, core);
self.place(tid);
self.make_ready(tid, priority);
SyscallOutcome::Complete(SyscallResult::Value(tid.raw() as usize))
}
@@ -89,7 +88,6 @@ impl<P: Platform> Kernel<P> {
entry: types::address::EntryPoint,
stack: types::address::StackPointer,
) -> SyscallOutcome {
use domain::scheduler::Scheduler;
use domain::thread::SavedContext;
let target_pid = match self.check_cap(
@@ -132,8 +130,8 @@ impl<P: Platform> Kernel<P> {
return SyscallOutcome::Complete(SyscallResult::Error(e));
}
let core = self.platform.core_id();
self.scheduler.add(tid, priority, core);
self.place(tid);
self.make_ready(tid, priority);
SyscallOutcome::Complete(SyscallResult::Value(tid.raw() as usize))
}

15
crates/kernel/src/tests/helpers.rs
View File

@@ -37,6 +37,7 @@ impl Kernel<MockPlatform> {
.expect("spawn");
let _ = self.processes.get_mut(pid).map(|p| p.add_thread(tid));
let core = self.platform.core_id();
self.set_home(tid, core);
self.scheduler.add(tid, priority, core);
tid
}
@@ -54,9 +55,21 @@ impl Kernel<MockPlatform> {
pub fn switch_to_thread_for_test(&mut self, tid: ThreadId) {
let core = self.platform.core_id();
assert_eq!(
self.home(tid),
core,
"a thread only ever runs on its home core"
);
// Only a thread ready in this core's queue can be switched to;
// otherwise leave the core as it is.
let switchable = self.scheduler.queued_on(tid) == Some(core)
&& self
.threads
.get(tid)
.is_ok_and(|t| matches!(t.state(), ThreadState::Ready));
if let Some(current) = self.scheduler.current(core) {
if current == tid {
if current == tid || !switchable {
return;
}
if let Ok(t) = self.threads.get_mut(current)

27
crates/kernel/src/tests/mock_platform.rs
View File

@@ -37,6 +37,8 @@ pub struct MockPlatform {
mappings: [Option<PageMapping>; MAX_MAPPINGS],
tables: [Option<LeafTable>; MAX_TABLES],
zeroed: [Option<usize>; MAX_ZEROED],
/// Cores an IPI was sent to and that have not taken it yet.
ipis: [bool; 8],
}
impl MockPlatform {
@@ -49,9 +51,27 @@ impl MockPlatform {
mappings: [None; MAX_MAPPINGS],
tables: [None; MAX_TABLES],
zeroed: [None; MAX_ZEROED],
ipis: [false; 8],
}
}
/// Runs the kernel as if on `core` from now on.
pub fn set_core(&mut self, core: CoreId) {
self.core = core;
}
pub fn ipi_pending(&self, core: CoreId) -> bool {
self.ipis.get(core.raw() as usize).copied().unwrap_or(false)
}
/// Delivers `core`'s pending IPI: true if there was one.
pub fn take_ipi(&mut self, core: CoreId) -> bool {
self.ipis
.get_mut(core.raw() as usize)
.map(core::mem::take)
.unwrap_or(false)
}
/// Physical address `va` is mapped to in `process`, as the "hardware" sees it.
pub fn mapping(&self, process: ProcessId, va: usize) -> Option<usize> {
self.mappings
@@ -207,7 +227,12 @@ impl Platform for MockPlatform {
}
fn mask_irq(&mut self, _irq: IrqId) {}
fn unmask_irq(&mut self, _irq: IrqId) {}
fn send_ipi(&mut self, _target: CoreId) {}
fn send_ipi(&mut self, target: CoreId) {
assert_ne!(target, self.core, "a core never IPIs itself");
if let Some(p) = self.ipis.get_mut(target.raw() as usize) {
*p = true;
}
}
fn core_id(&self) -> CoreId {
self.core
}

1
crates/kernel/src/tests/mod.rs
View File

@@ -15,6 +15,7 @@ mod lifetimes;
mod memory;
mod revocation;
mod simple;
mod smp;
mod storm;
mod tick;

178
crates/kernel/src/tests/smp.rs Normal file
View File

@@ -0,0 +1,178 @@
extern crate std;
use super::*;
use crate::{SyscallOutcome, SyscallResult};
use domain::{
message::Message,
resource::Resource,
scheduler::SchedulerDecision,
syscall::Syscall,
thread::{BlockReason, ThreadState},
};
use std::vec::Vec;
use types::{
address::PhysicalAddress,
id::{CoreId, ThreadId},
permission::{Permission, PermissionSet},
};
fn core(c: u8) -> CoreId {
CoreId::from_raw(c)
}
/// A booted kernel on `cores` cores, init running on core 0.
fn smp(cores: u8) -> (Kernel<MockPlatform>, ProcessId) {
let mut k = Kernel::new(
MockPlatform::new(cores),
PhysicalAddress::from_raw(0x8400_0000),
1024,
);
let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap();
(k, init)
}
/// Acts as `c` from now on, running the next thread there if it is idle.
fn on(k: &mut Kernel<MockPlatform>, c: CoreId) {
k.platform_mut().set_core(c);
if k.current_thread().is_none()
&& let Some(next) = k.pick_next_thread()
{
k.set_thread_running(next, c).unwrap();
}
}
fn state(k: &Kernel<MockPlatform>, t: ThreadId) -> Option<ThreadState> {
k.thread_state_for_test(t)
}
#[test]
fn new_threads_are_spread_over_the_cores() {
let (mut k, init) = smp(4);
let homes: Vec<CoreId> = (0..3)
.map(|_| {
let t = k.spawn_thread_internal(init).unwrap();
k.home(t)
})
.collect();
assert_eq!(
homes,
[core(1), core(2), core(3)],
"init already has core 0"
);
}
#[test]
fn a_thread_made_ready_for_an_idle_core_kicks_it() {
let (mut k, init) = smp(2);
let t = k.spawn_thread_internal(init).unwrap();
assert_eq!(k.home(t), core(1));
assert!(k.platform_mut().ipi_pending(core(1)));
assert!(!k.platform_mut().ipi_pending(core(0)));
}
#[test]
fn a_thread_made_ready_on_this_core_sends_no_ipi() {
let (mut k, init) = smp(1);
k.spawn_thread_internal(init).unwrap();
assert!(!k.platform_mut().ipi_pending(core(0)));
}
#[test]
fn a_busy_core_is_not_kicked_for_an_equal_priority_thread() {
let (mut k, init) = smp(2);
let a = k.spawn_thread_internal(init).unwrap();
on(&mut k, core(1));
assert_eq!(state(&k, a), Some(ThreadState::Running(core(1))));
k.platform_mut().take_ipi(core(1));
k.platform_mut().set_core(core(0));
// A second thread homed on core 1 while it is busy at the same priority.
let b = k.spawn_thread_internal(init).unwrap();
let c = k.spawn_thread_internal(init).unwrap();
let on_one = if k.home(b) == core(1) { b } else { c };
assert_eq!(k.home(on_one), core(1));
assert!(!k.platform_mut().ipi_pending(core(1)));
}
#[test]
fn killing_a_thread_running_on_another_core_stops_that_core() {
let (mut k, init) = smp(2);
let child = k.create_process_internal().unwrap();
let t = k.spawn_thread_internal(child).unwrap();
on(&mut k, core(1));
k.platform_mut().take_ipi(core(1));
assert_eq!(state(&k, t), Some(ThreadState::Running(core(1))));
k.platform_mut().set_core(core(0));
let cap = k.grant_cap_for_test(
init,
Resource::Process(child),
PermissionSet::empty().with(Permission::Execute),
);
let out = k.handle_syscall(Syscall::Kill { process: cap });
assert!(matches!(out, SyscallOutcome::Complete(SyscallResult::Ok)));
assert!(k.platform_mut().ipi_pending(core(1)), "core 1 never told");
assert_eq!(k.scheduler.current(core(1)), None);
k.check_invariants().unwrap();
}
#[test]
fn a_call_to_a_server_on_another_core_does_not_pull_it_over() {
let (mut k, init) = smp(2);
let ep = k.create_endpoint(init).unwrap();
let all = PermissionSet::empty()
.with(Permission::Read)
.with(Permission::Execute);
let slot = k.grant_cap_for_test(init, Resource::Endpoint(ep), all);
let server = k.spawn_thread_internal(init).unwrap();
assert_eq!(k.home(server), core(1));
on(&mut k, core(1));
k.platform_mut().take_ipi(core(1));
let out = k.handle_syscall(Syscall::Receive { endpoint: slot });
assert!(matches!(out, SyscallOutcome::BlockCaller(_)));
k.platform_mut().set_core(core(0));
let out = k.handle_syscall(Syscall::Call {
endpoint: slot,
message: Message::empty(),
});
assert!(
matches!(out, SyscallOutcome::BlockCaller(BlockReason::AwaitingReply)),
"no direct switch across cores"
);
assert_eq!(state(&k, server), Some(ThreadState::Ready));
assert!(k.platform_mut().ipi_pending(core(1)));
k.check_invariants().unwrap();
}
#[test]
fn reschedule_switches_only_for_a_higher_priority_thread() {
let (mut k, init) = smp(2);
let a = k.spawn_thread_internal(init).unwrap();
on(&mut k, core(1));
assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
// An equal-priority one waits its turn (the tick handles that).
let peer = k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT);
assert_eq!(k.home(peer), core(1));
assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
// A higher-priority thread arrives on core 1's queue.
let p = Priority::new(Priority::DEFAULT.value() + 1).unwrap();
let urgent = k.spawn_thread_for_test(init, p, TimeSlice::DEFAULT);
assert_eq!(k.home(urgent), core(1));
assert!(matches!(
k.reschedule(core(1)),
SchedulerDecision::Preempt(t) if t == urgent
));
assert_eq!(state(&k, urgent), Some(ThreadState::Running(core(1))));
assert_eq!(state(&k, a), Some(ThreadState::Ready));
}
#[test]
fn reschedule_on_an_idle_core_leaves_picking_to_the_caller() {
let (mut k, _) = smp(2);
k.platform_mut().set_core(core(1));
assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
}

92
crates/kernel/src/tests/storm.rs
View File

@@ -208,6 +208,20 @@ impl Kernel<MockPlatform> {
if self.scheduler.is_queued(t) != matches!(state, ThreadState::Ready) {
return Err(format!("{t:?} in {state:?} has wrong run-queue membership"));
}
// ---- cores: a thread runs and waits only on its home core ----
let home = self.home(t);
match state {
ThreadState::Running(c) if c != home => {
return Err(format!("{t:?} runs on {c:?}, its home is {home:?}"));
}
ThreadState::Running(c) if self.scheduler.current(c) != Some(t) => {
return Err(format!("{t:?} Running on {c:?} but not its current thread"));
}
ThreadState::Ready if self.scheduler.queued_on(t) != Some(home) => {
return Err(format!("{t:?} queued away from its home {home:?}"));
}
_ => {}
}
if let Some(r) = ipc.reply_to
&& self.threads.generation(r.tid) == Some(r.generation)
&& self.threads.get(r.tid).ok().map(|x| *x.state())
@@ -216,6 +230,22 @@ impl Kernel<MockPlatform> {
return Err(format!("{t:?} holds a reply for a thread not waiting"));
}
}
for c in 0..self.scheduler.num_cores() {
let core = types::id::CoreId::from_raw(c);
let current = self.scheduler.current(core);
if let Some(t) = current
&& self.threads.get(t).ok().map(|x| *x.state()) != Some(ThreadState::Running(core))
{
return Err(format!("{core:?}'s current {t:?} is not running there"));
}
// No lost wakeups: an idle core with work waiting has been told.
if current.is_none()
&& self.scheduler.highest_waiting(core).is_some()
&& !self.platform.ipi_pending(core)
{
return Err(format!("{core:?} idles with ready threads and no IPI"));
}
}
Ok(())
}
}
@@ -282,6 +312,10 @@ fn new_round(k: &mut Kernel<MockPlatform>, init: ProcessId, init_thread: ThreadI
.create_pool_region(hub, types::bounded::PageCount::new(2).unwrap())
.ok();
let mut members = std::vec![init];
// CreateProcess below runs as init, so on init's core.
let init_core = k.home(init_thread);
k.platform_mut().set_core(init_core);
resume(k, init_core);
for _ in 0..2 {
k.switch_to_thread_for_test(init_thread);
if let crate::SyscallOutcome::Complete(crate::SyscallResult::Slot(cap)) =
@@ -292,7 +326,7 @@ fn new_round(k: &mut Kernel<MockPlatform>, init: ProcessId, init_thread: ThreadI
{
let pid = k.process_of_cap_for_test(cap);
for _ in 0..2 {
k.spawn_thread_for_test(pid, Priority::DEFAULT, TimeSlice::DEFAULT);
k.spawn_thread_internal(pid);
}
members.push(pid);
} else if std::env::var("STORM_DEBUG").is_ok() {
@@ -326,11 +360,40 @@ fn region_slots(k: &Kernel<MockPlatform>, pid: Option<ProcessId>) -> std::vec::V
.collect()
}
/// What the adapter does whenever a core is left without a thread: run the
/// next one in its queue, if any.
fn resume(k: &mut Kernel<MockPlatform>, core: types::id::CoreId) {
if k.current_thread().is_none()
&& let Some(next) = k.pick_next_thread()
{
k.set_thread_running(next, core).unwrap();
}
}
#[test]
fn the_kernel_survives_a_syscall_storm() {
storm(1);
}
/// The same storm with threads homed on four cores. Each step runs on one
/// core, as the thread that core is running; the kernel is still entered
/// one call at a time (stage 1 takes a single lock), but threads on other
/// cores are live meanwhile and IPIs arrive late.
#[test]
fn the_kernel_survives_a_syscall_storm_on_four_cores() {
storm(4);
}
fn storm(cores: u8) {
use types::id::CoreId;
let mut total = [0usize; 36 + 1];
for seed in [17u64, 0xABCD_EF01, 4242, 0x1357_9BDF, 8] {
let (mut k, init) = kernel_with_process();
let mut k = Kernel::new(
MockPlatform::new(cores),
types::address::PhysicalAddress::from_raw(0x8400_0000),
1024,
);
let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap();
let init_thread = k.current_thread().unwrap();
k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT);
let mut rng = Rng(seed);
@@ -360,13 +423,35 @@ fn the_kernel_survives_a_syscall_storm() {
// init may be blocked on objects it made itself.
k.destroy_endpoints_owned_by(init);
k.destroy_notifications_owned_by(init);
let here = k.platform_mut().core_id();
resume(&mut k, here);
k.check_invariants()
.unwrap_or_else(|e| panic!("seed {seed:#x} step {step} after teardown: {e}"));
new_round(&mut k, init, init_thread);
rounds += 1;
continue;
}
let tid = runnable[rng.below(runnable.len())];
// Pick a core and act as the thread it runs, after it takes any
// IPI that has reached it. Now and then switch it to another
// ready thread of its own, as a tick or a yield would.
let core = CoreId::from_raw(rng.below(cores as usize) as u8);
k.platform_mut().set_core(core);
if k.platform_mut().take_ipi(core) {
let _ = k.reschedule(core);
}
resume(&mut k, core);
let homed: std::vec::Vec<ThreadId> = runnable
.iter()
.copied()
.filter(|t| k.home(*t) == core)
.collect();
if homed.is_empty() {
continue;
}
let tid = match k.current_thread() {
Some(t) if rng.below(3) != 0 => t,
_ => homed[rng.below(homed.len())],
};
let is_init = k.thread_process(tid) == Some(init);
let nr = rng.below(36 + 1);
// init exiting ends the test. Halt stays in: nobody here holds
@@ -399,6 +484,7 @@ fn the_kernel_survives_a_syscall_storm() {
k.switch_to_thread_for_test(tid);
k.take_pending_result(tid);
let out = k.handle_syscall(syscall);
resume(&mut k, core);
executed += 1;
if !matches!(
out,