feat: SMP stage 1 (4 harts, per-hart guarded stacks, home-core affinity, IPIs, TLB shootdown, one temporary kernel lock), 4-core storm

This commit is contained in:
2026-10-27 00:56:15 +01:00
parent b87a62475f
commit 2e665fd66e
22 changed files with 995 additions and 230 deletions

View File

@@ -21,6 +21,7 @@ pub mod interrupt;
mod memory;
#[cfg(kani)]
pub mod proofs;
mod smp;
pub mod syscall;
pub use board::BoardDevice;
@@ -65,6 +66,8 @@ pub struct Kernel<P: Platform> {
notifications: NotificationTable,
ipc: [syscall::IpcState; MAX_THREADS],
pending_results: [Option<SyscallResult>; MAX_THREADS],
/// The core each thread runs on (see smp.rs).
homes: [types::id::CoreId; MAX_THREADS],
}
impl<P: Platform> Kernel<P> {
@@ -83,6 +86,7 @@ impl<P: Platform> Kernel<P> {
notifications: NotificationTable::new(),
ipc: [syscall::IpcState::default(); MAX_THREADS],
pending_results: [const { None }; MAX_THREADS],
homes: [types::id::CoreId::from_raw(0); MAX_THREADS],
}
}
@@ -115,6 +119,7 @@ impl<P: Platform> Kernel<P> {
addr_of_mut!((*slot).notifications).write(NotificationTable::new());
addr_of_mut!((*slot).ipc).write([syscall::IpcState::default(); MAX_THREADS]);
addr_of_mut!((*slot).pending_results).write([const { None }; MAX_THREADS]);
addr_of_mut!((*slot).homes).write([types::id::CoreId::from_raw(0); MAX_THREADS]);
}
}
@@ -142,6 +147,7 @@ impl<P: Platform> Kernel<P> {
.insert(cap)?;
let core = self.platform.core_id();
self.set_home(tid, core);
self.scheduler.add(tid, priority, core);
self.scheduler.next(core);
self.threads.get_mut(tid)?.set_running(core)?;
@@ -211,14 +217,13 @@ impl<P: Platform> Kernel<P> {
}
pub fn spawn_thread_internal(&mut self, pid: ProcessId) -> Option<ThreadId> {
use domain::scheduler::Scheduler;
let tid = self
.threads
.spawn(pid, Priority::DEFAULT, TimeSlice::DEFAULT, None)
.ok()?;
let _ = self.processes.get_mut(pid).map(|p| p.add_thread(tid));
let core = self.platform.core_id();
self.scheduler.add(tid, Priority::DEFAULT, core);
self.place(tid);
self.make_ready(tid, Priority::DEFAULT);
Some(tid)
}
@@ -240,31 +245,6 @@ impl<P: Platform> Kernel<P> {
self.grant_cap(to, resource, permissions)
}
pub fn switch_to_thread(&mut self, tid: ThreadId) {
use domain::scheduler::Scheduler;
let core = self.platform.core_id();
if let Some(current) = self.scheduler.current(core) {
if current == tid {
return;
}
if let Ok(t) = self.threads.get_mut(current)
&& matches!(t.state(), ThreadState::Running(_))
{
let priority = t.priority();
let _ = t.preempt();
self.scheduler.add(current, priority, core);
}
}
if let Ok(t) = self.threads.get(tid)
&& matches!(t.state(), ThreadState::Ready)
&& self.scheduler.run_now(core, tid)
{
let _ = self.threads.get_mut(tid).map(|t| t.set_running(core));
}
}
fn caller(&self) -> Result<(ThreadId, ProcessId), KernelError> {
let core = self.platform.core_id();
let tid = self
@@ -311,6 +291,7 @@ impl<P: Platform> Kernel<P> {
if let Some(caller) = ipc.reply_to {
self.release_caller(caller);
}
self.evict(*tid);
self.scheduler.remove(*tid);
self.set_pending_result(*tid, None);
let _ = self.threads.destroy(*tid);

119
crates/kernel/src/smp.rs Normal file
View File

@@ -0,0 +1,119 @@
//! Cores. Every thread has a home core, chosen when it is created, and
//! only ever runs there: it waits in that core's run queue and is never
//! migrated. Making a thread ready from another core sends that core an
//! IPI when it has to look at its queue again (it is idle, or the thread
//! outranks what it is running), and so does taking away the thread a
//! core is running.
use domain::{
scheduler::{Scheduler, SchedulerDecision},
thread::ThreadState,
};
use platform::Platform;
use types::{
bounded::Priority,
id::{CoreId, ThreadId},
};
use crate::Kernel;
impl<P: Platform> Kernel<P> {
/// The core `tid` runs on.
pub fn home(&self, tid: ThreadId) -> CoreId {
self.homes
.get(tid.raw() as usize)
.copied()
.unwrap_or(CoreId::from_raw(0))
}
/// Homes a new thread on `core`.
pub(crate) fn set_home(&mut self, tid: ThreadId, core: CoreId) {
if let Some(h) = self.homes.get_mut(tid.raw() as usize) {
*h = core;
}
}
/// Homes a new thread on the core with the fewest live threads.
pub(crate) fn place(&mut self, tid: ThreadId) {
let cores = self.scheduler.num_cores().max(1);
let load = |core: CoreId| {
self.threads
.iter()
.filter(|(t, _)| *t != tid && self.home(*t) == core)
.count()
};
let best = (0..cores)
.map(CoreId::from_raw)
.min_by_key(|c| load(*c))
.unwrap_or(CoreId::from_raw(0));
self.set_home(tid, best);
}
/// Queues a ready thread on its home core, and kicks that core if it
/// is another one that should look at its queue now.
pub(crate) fn make_ready(&mut self, tid: ThreadId, priority: Priority) {
let home = self.home(tid);
self.scheduler.add(tid, priority, home);
if home == self.platform.core_id() {
return;
}
let outranks = match self.scheduler.current(home) {
None => true,
Some(running) => self
.threads
.get(running)
.is_ok_and(|t| priority > t.priority()),
};
if outranks {
self.platform.send_ipi(home);
}
}
/// Before `tid` is destroyed: if another core is running it, that core
/// must stop and pick something else.
pub(crate) fn evict(&mut self, tid: ThreadId) {
if let Ok(ThreadState::Running(core)) = self.threads.get(tid).map(|t| *t.state())
&& core != self.platform.core_id()
{
self.platform.send_ipi(core);
}
}
/// What `core` should do after an IPI: keep running its thread, or
/// switch to a higher-priority one now waiting. When it has no thread
/// (it was idle, or its thread was destroyed) the caller picks the
/// next one itself.
pub fn reschedule(&mut self, core: CoreId) -> SchedulerDecision {
let Some(current) = self.scheduler.current(core) else {
return SchedulerDecision::Continue;
};
let Ok(thread) = self.threads.get(current) else {
return SchedulerDecision::Continue;
};
let priority = thread.priority();
if self
.scheduler
.highest_waiting(core)
.is_none_or(|p| p <= priority)
{
return SchedulerDecision::Continue;
}
let Some(next) = self.scheduler.next(core) else {
return SchedulerDecision::Continue;
};
if self
.threads
.get_mut(next)
.and_then(|t| t.set_running(core))
.is_err()
{
return SchedulerDecision::Continue;
}
if let Ok(t) = self.threads.get_mut(current)
&& t.preempt().is_ok()
{
self.scheduler.add(current, priority, core);
}
SchedulerDecision::Preempt(next)
}
}

View File

@@ -102,8 +102,7 @@ impl<P: Platform> Kernel<P> {
let priority = thread.priority();
if thread.unblock().is_ok() {
self.set_pending_result(tid, Some(result));
let core = self.platform.core_id();
self.scheduler.add(tid, priority, core);
self.make_ready(tid, priority);
}
}
@@ -234,6 +233,8 @@ impl<P: Platform> Kernel<P> {
}
self.block_caller(tid, BlockReason::AwaitingReply);
// Run the server straight away instead of waiting for its turn.
// Only works if it lives on this core (run_now finds it in this
// core's queue only); otherwise make_ready kicked its core.
let core = self.platform.core_id();
if self.scheduler.run_now(core, receiver)
&& self

View File

@@ -10,7 +10,6 @@ impl<P: Platform> Kernel<P> {
entry: types::address::EntryPoint,
stack: types::address::StackPointer,
) -> SyscallOutcome {
use domain::scheduler::Scheduler;
use domain::thread::SavedContext;
let priority = types::bounded::Priority::DEFAULT;
@@ -30,8 +29,8 @@ impl<P: Platform> Kernel<P> {
return SyscallOutcome::Complete(SyscallResult::Error(e));
}
let core = self.platform.core_id();
self.scheduler.add(tid, priority, core);
self.place(tid);
self.make_ready(tid, priority);
SyscallOutcome::Complete(SyscallResult::Value(tid.raw() as usize))
}
@@ -89,7 +88,6 @@ impl<P: Platform> Kernel<P> {
entry: types::address::EntryPoint,
stack: types::address::StackPointer,
) -> SyscallOutcome {
use domain::scheduler::Scheduler;
use domain::thread::SavedContext;
let target_pid = match self.check_cap(
@@ -132,8 +130,8 @@ impl<P: Platform> Kernel<P> {
return SyscallOutcome::Complete(SyscallResult::Error(e));
}
let core = self.platform.core_id();
self.scheduler.add(tid, priority, core);
self.place(tid);
self.make_ready(tid, priority);
SyscallOutcome::Complete(SyscallResult::Value(tid.raw() as usize))
}

View File

@@ -37,6 +37,7 @@ impl Kernel<MockPlatform> {
.expect("spawn");
let _ = self.processes.get_mut(pid).map(|p| p.add_thread(tid));
let core = self.platform.core_id();
self.set_home(tid, core);
self.scheduler.add(tid, priority, core);
tid
}
@@ -54,9 +55,21 @@ impl Kernel<MockPlatform> {
pub fn switch_to_thread_for_test(&mut self, tid: ThreadId) {
let core = self.platform.core_id();
assert_eq!(
self.home(tid),
core,
"a thread only ever runs on its home core"
);
// Only a thread ready in this core's queue can be switched to;
// otherwise leave the core as it is.
let switchable = self.scheduler.queued_on(tid) == Some(core)
&& self
.threads
.get(tid)
.is_ok_and(|t| matches!(t.state(), ThreadState::Ready));
if let Some(current) = self.scheduler.current(core) {
if current == tid {
if current == tid || !switchable {
return;
}
if let Ok(t) = self.threads.get_mut(current)

View File

@@ -37,6 +37,8 @@ pub struct MockPlatform {
mappings: [Option<PageMapping>; MAX_MAPPINGS],
tables: [Option<LeafTable>; MAX_TABLES],
zeroed: [Option<usize>; MAX_ZEROED],
/// Cores an IPI was sent to and that have not taken it yet.
ipis: [bool; 8],
}
impl MockPlatform {
@@ -49,9 +51,27 @@ impl MockPlatform {
mappings: [None; MAX_MAPPINGS],
tables: [None; MAX_TABLES],
zeroed: [None; MAX_ZEROED],
ipis: [false; 8],
}
}
/// Runs the kernel as if on `core` from now on.
pub fn set_core(&mut self, core: CoreId) {
self.core = core;
}
pub fn ipi_pending(&self, core: CoreId) -> bool {
self.ipis.get(core.raw() as usize).copied().unwrap_or(false)
}
/// Delivers `core`'s pending IPI: true if there was one.
pub fn take_ipi(&mut self, core: CoreId) -> bool {
self.ipis
.get_mut(core.raw() as usize)
.map(core::mem::take)
.unwrap_or(false)
}
/// Physical address `va` is mapped to in `process`, as the "hardware" sees it.
pub fn mapping(&self, process: ProcessId, va: usize) -> Option<usize> {
self.mappings
@@ -207,7 +227,12 @@ impl Platform for MockPlatform {
}
fn mask_irq(&mut self, _irq: IrqId) {}
fn unmask_irq(&mut self, _irq: IrqId) {}
fn send_ipi(&mut self, _target: CoreId) {}
fn send_ipi(&mut self, target: CoreId) {
assert_ne!(target, self.core, "a core never IPIs itself");
if let Some(p) = self.ipis.get_mut(target.raw() as usize) {
*p = true;
}
}
fn core_id(&self) -> CoreId {
self.core
}

View File

@@ -15,6 +15,7 @@ mod lifetimes;
mod memory;
mod revocation;
mod simple;
mod smp;
mod storm;
mod tick;

View File

@@ -0,0 +1,178 @@
extern crate std;
use super::*;
use crate::{SyscallOutcome, SyscallResult};
use domain::{
message::Message,
resource::Resource,
scheduler::SchedulerDecision,
syscall::Syscall,
thread::{BlockReason, ThreadState},
};
use std::vec::Vec;
use types::{
address::PhysicalAddress,
id::{CoreId, ThreadId},
permission::{Permission, PermissionSet},
};
fn core(c: u8) -> CoreId {
CoreId::from_raw(c)
}
/// A booted kernel on `cores` cores, init running on core 0.
fn smp(cores: u8) -> (Kernel<MockPlatform>, ProcessId) {
let mut k = Kernel::new(
MockPlatform::new(cores),
PhysicalAddress::from_raw(0x8400_0000),
1024,
);
let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap();
(k, init)
}
/// Acts as `c` from now on, running the next thread there if it is idle.
fn on(k: &mut Kernel<MockPlatform>, c: CoreId) {
k.platform_mut().set_core(c);
if k.current_thread().is_none()
&& let Some(next) = k.pick_next_thread()
{
k.set_thread_running(next, c).unwrap();
}
}
fn state(k: &Kernel<MockPlatform>, t: ThreadId) -> Option<ThreadState> {
k.thread_state_for_test(t)
}
#[test]
fn new_threads_are_spread_over_the_cores() {
let (mut k, init) = smp(4);
let homes: Vec<CoreId> = (0..3)
.map(|_| {
let t = k.spawn_thread_internal(init).unwrap();
k.home(t)
})
.collect();
assert_eq!(
homes,
[core(1), core(2), core(3)],
"init already has core 0"
);
}
#[test]
fn a_thread_made_ready_for_an_idle_core_kicks_it() {
let (mut k, init) = smp(2);
let t = k.spawn_thread_internal(init).unwrap();
assert_eq!(k.home(t), core(1));
assert!(k.platform_mut().ipi_pending(core(1)));
assert!(!k.platform_mut().ipi_pending(core(0)));
}
#[test]
fn a_thread_made_ready_on_this_core_sends_no_ipi() {
let (mut k, init) = smp(1);
k.spawn_thread_internal(init).unwrap();
assert!(!k.platform_mut().ipi_pending(core(0)));
}
#[test]
fn a_busy_core_is_not_kicked_for_an_equal_priority_thread() {
let (mut k, init) = smp(2);
let a = k.spawn_thread_internal(init).unwrap();
on(&mut k, core(1));
assert_eq!(state(&k, a), Some(ThreadState::Running(core(1))));
k.platform_mut().take_ipi(core(1));
k.platform_mut().set_core(core(0));
// A second thread homed on core 1 while it is busy at the same priority.
let b = k.spawn_thread_internal(init).unwrap();
let c = k.spawn_thread_internal(init).unwrap();
let on_one = if k.home(b) == core(1) { b } else { c };
assert_eq!(k.home(on_one), core(1));
assert!(!k.platform_mut().ipi_pending(core(1)));
}
#[test]
fn killing_a_thread_running_on_another_core_stops_that_core() {
let (mut k, init) = smp(2);
let child = k.create_process_internal().unwrap();
let t = k.spawn_thread_internal(child).unwrap();
on(&mut k, core(1));
k.platform_mut().take_ipi(core(1));
assert_eq!(state(&k, t), Some(ThreadState::Running(core(1))));
k.platform_mut().set_core(core(0));
let cap = k.grant_cap_for_test(
init,
Resource::Process(child),
PermissionSet::empty().with(Permission::Execute),
);
let out = k.handle_syscall(Syscall::Kill { process: cap });
assert!(matches!(out, SyscallOutcome::Complete(SyscallResult::Ok)));
assert!(k.platform_mut().ipi_pending(core(1)), "core 1 never told");
assert_eq!(k.scheduler.current(core(1)), None);
k.check_invariants().unwrap();
}
#[test]
fn a_call_to_a_server_on_another_core_does_not_pull_it_over() {
let (mut k, init) = smp(2);
let ep = k.create_endpoint(init).unwrap();
let all = PermissionSet::empty()
.with(Permission::Read)
.with(Permission::Execute);
let slot = k.grant_cap_for_test(init, Resource::Endpoint(ep), all);
let server = k.spawn_thread_internal(init).unwrap();
assert_eq!(k.home(server), core(1));
on(&mut k, core(1));
k.platform_mut().take_ipi(core(1));
let out = k.handle_syscall(Syscall::Receive { endpoint: slot });
assert!(matches!(out, SyscallOutcome::BlockCaller(_)));
k.platform_mut().set_core(core(0));
let out = k.handle_syscall(Syscall::Call {
endpoint: slot,
message: Message::empty(),
});
assert!(
matches!(out, SyscallOutcome::BlockCaller(BlockReason::AwaitingReply)),
"no direct switch across cores"
);
assert_eq!(state(&k, server), Some(ThreadState::Ready));
assert!(k.platform_mut().ipi_pending(core(1)));
k.check_invariants().unwrap();
}
#[test]
fn reschedule_switches_only_for_a_higher_priority_thread() {
let (mut k, init) = smp(2);
let a = k.spawn_thread_internal(init).unwrap();
on(&mut k, core(1));
assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
// An equal-priority one waits its turn (the tick handles that).
let peer = k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT);
assert_eq!(k.home(peer), core(1));
assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
// A higher-priority thread arrives on core 1's queue.
let p = Priority::new(Priority::DEFAULT.value() + 1).unwrap();
let urgent = k.spawn_thread_for_test(init, p, TimeSlice::DEFAULT);
assert_eq!(k.home(urgent), core(1));
assert!(matches!(
k.reschedule(core(1)),
SchedulerDecision::Preempt(t) if t == urgent
));
assert_eq!(state(&k, urgent), Some(ThreadState::Running(core(1))));
assert_eq!(state(&k, a), Some(ThreadState::Ready));
}
#[test]
fn reschedule_on_an_idle_core_leaves_picking_to_the_caller() {
let (mut k, _) = smp(2);
k.platform_mut().set_core(core(1));
assert!(matches!(k.reschedule(core(1)), SchedulerDecision::Continue));
}

View File

@@ -208,6 +208,20 @@ impl Kernel<MockPlatform> {
if self.scheduler.is_queued(t) != matches!(state, ThreadState::Ready) {
return Err(format!("{t:?} in {state:?} has wrong run-queue membership"));
}
// ---- cores: a thread runs and waits only on its home core ----
let home = self.home(t);
match state {
ThreadState::Running(c) if c != home => {
return Err(format!("{t:?} runs on {c:?}, its home is {home:?}"));
}
ThreadState::Running(c) if self.scheduler.current(c) != Some(t) => {
return Err(format!("{t:?} Running on {c:?} but not its current thread"));
}
ThreadState::Ready if self.scheduler.queued_on(t) != Some(home) => {
return Err(format!("{t:?} queued away from its home {home:?}"));
}
_ => {}
}
if let Some(r) = ipc.reply_to
&& self.threads.generation(r.tid) == Some(r.generation)
&& self.threads.get(r.tid).ok().map(|x| *x.state())
@@ -216,6 +230,22 @@ impl Kernel<MockPlatform> {
return Err(format!("{t:?} holds a reply for a thread not waiting"));
}
}
for c in 0..self.scheduler.num_cores() {
let core = types::id::CoreId::from_raw(c);
let current = self.scheduler.current(core);
if let Some(t) = current
&& self.threads.get(t).ok().map(|x| *x.state()) != Some(ThreadState::Running(core))
{
return Err(format!("{core:?}'s current {t:?} is not running there"));
}
// No lost wakeups: an idle core with work waiting has been told.
if current.is_none()
&& self.scheduler.highest_waiting(core).is_some()
&& !self.platform.ipi_pending(core)
{
return Err(format!("{core:?} idles with ready threads and no IPI"));
}
}
Ok(())
}
}
@@ -282,6 +312,10 @@ fn new_round(k: &mut Kernel<MockPlatform>, init: ProcessId, init_thread: ThreadI
.create_pool_region(hub, types::bounded::PageCount::new(2).unwrap())
.ok();
let mut members = std::vec![init];
// CreateProcess below runs as init, so on init's core.
let init_core = k.home(init_thread);
k.platform_mut().set_core(init_core);
resume(k, init_core);
for _ in 0..2 {
k.switch_to_thread_for_test(init_thread);
if let crate::SyscallOutcome::Complete(crate::SyscallResult::Slot(cap)) =
@@ -292,7 +326,7 @@ fn new_round(k: &mut Kernel<MockPlatform>, init: ProcessId, init_thread: ThreadI
{
let pid = k.process_of_cap_for_test(cap);
for _ in 0..2 {
k.spawn_thread_for_test(pid, Priority::DEFAULT, TimeSlice::DEFAULT);
k.spawn_thread_internal(pid);
}
members.push(pid);
} else if std::env::var("STORM_DEBUG").is_ok() {
@@ -326,11 +360,40 @@ fn region_slots(k: &Kernel<MockPlatform>, pid: Option<ProcessId>) -> std::vec::V
.collect()
}
/// What the adapter does whenever a core is left without a thread: run the
/// next one in its queue, if any.
fn resume(k: &mut Kernel<MockPlatform>, core: types::id::CoreId) {
if k.current_thread().is_none()
&& let Some(next) = k.pick_next_thread()
{
k.set_thread_running(next, core).unwrap();
}
}
#[test]
fn the_kernel_survives_a_syscall_storm() {
storm(1);
}
/// The same storm with threads homed on four cores. Each step runs on one
/// core, as the thread that core is running; the kernel is still entered
/// one call at a time (stage 1 takes a single lock), but threads on other
/// cores are live meanwhile and IPIs arrive late.
#[test]
fn the_kernel_survives_a_syscall_storm_on_four_cores() {
storm(4);
}
fn storm(cores: u8) {
use types::id::CoreId;
let mut total = [0usize; 36 + 1];
for seed in [17u64, 0xABCD_EF01, 4242, 0x1357_9BDF, 8] {
let (mut k, init) = kernel_with_process();
let mut k = Kernel::new(
MockPlatform::new(cores),
types::address::PhysicalAddress::from_raw(0x8400_0000),
1024,
);
let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap();
let init_thread = k.current_thread().unwrap();
k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT);
let mut rng = Rng(seed);
@@ -360,13 +423,35 @@ fn the_kernel_survives_a_syscall_storm() {
// init may be blocked on objects it made itself.
k.destroy_endpoints_owned_by(init);
k.destroy_notifications_owned_by(init);
let here = k.platform_mut().core_id();
resume(&mut k, here);
k.check_invariants()
.unwrap_or_else(|e| panic!("seed {seed:#x} step {step} after teardown: {e}"));
new_round(&mut k, init, init_thread);
rounds += 1;
continue;
}
let tid = runnable[rng.below(runnable.len())];
// Pick a core and act as the thread it runs, after it takes any
// IPI that has reached it. Now and then switch it to another
// ready thread of its own, as a tick or a yield would.
let core = CoreId::from_raw(rng.below(cores as usize) as u8);
k.platform_mut().set_core(core);
if k.platform_mut().take_ipi(core) {
let _ = k.reschedule(core);
}
resume(&mut k, core);
let homed: std::vec::Vec<ThreadId> = runnable
.iter()
.copied()
.filter(|t| k.home(*t) == core)
.collect();
if homed.is_empty() {
continue;
}
let tid = match k.current_thread() {
Some(t) if rng.below(3) != 0 => t,
_ => homed[rng.below(homed.len())],
};
let is_init = k.thread_process(tid) == Some(init);
let nr = rng.below(36 + 1);
// init exiting ends the test. Halt stays in: nobody here holds
@@ -399,6 +484,7 @@ fn the_kernel_survives_a_syscall_storm() {
k.switch_to_thread_for_test(tid);
k.take_pending_result(tid);
let out = k.handle_syscall(syscall);
resume(&mut k, core);
executed += 1;
if !matches!(
out,