GKaszewski/k-os
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: SMP stage 1 (4 harts, per-hart guarded stacks, home-core affinity, IPIs, TLB shootdown, one temporary kernel lock), 4-core storm

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-27 00:56:15 +01:00
parent b87a62475f
commit 2e665fd66e
22 changed files with 995 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
crates/adapter-riscv/src/trap/mod.rs
View File

@@ -60,7 +60,14 @@ _from_user:
mv a0, sp
call trap_handler_rust
j _restore_frame
# trap_return(frame): resume whatever `frame` describes, as if returning
# from a trap that saved it. Used to enter userspace for the first time.
.globl trap_return
trap_return:
mv sp, a0
_restore_frame:
ld t0, 256(sp)
csrw sepc, t0
ld t0, 264(sp)
@@ -121,6 +128,7 @@ _sret_smode:
);
const CAUSE_INTERRUPT_BIT: usize = 1 << 63;
const CAUSE_S_SOFTWARE: usize = 1;
const CAUSE_S_TIMER: usize = 5;
const CAUSE_S_EXTERNAL: usize = 9;
const CAUSE_ECALL_U: usize = 8;
@@ -177,88 +185,67 @@ fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFra
}
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(result) = kern.take_pending_result(tid) {
if let Some(result) = kern().take_pending_result(tid) {
write_result(frame, &result);
}
}
unsafe extern "C" {
fn trap_return(frame: *mut TrapFrame) -> !;
}
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
// SAFETY: only called with the kernel lock held.
unsafe { &mut *crate::kernel_ptr() }
}
fn this_core() -> types::id::CoreId {
types::id::CoreId::from_raw(crate::smp::hart_id() as u8)
}
/// Saves `frame` into `from` (if it still exists) and loads `to`.
fn switch_to_thread(
frame: &mut TrapFrame,
current_tid: types::id::ThreadId,
next_tid: types::id::ThreadId,
from: Option<types::id::ThreadId>,
to: types::id::ThreadId,
) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(ctx) = kern.thread_context_mut(current_tid) {
let kern = kern();
if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next_pid = kern.thread_process(next_tid);
let current_pid = kern.thread_process(current_tid);
if let Some(ctx) = kern.thread_context(next_tid) {
if let Some(ctx) = kern.thread_context(to) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, next_tid);
if next_pid != current_pid
&& let Some(pid) = next_pid
{
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = crate::mmu::page_table::table_at(pt_addr);
crate::mmu::activate(pt);
}
}
apply_pending_result(frame, to);
}
fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(ctx) = kern.thread_context_mut(current_tid) {
/// Saves `from` and runs the next ready thread, idling until there is one.
fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next_tid = next_runnable();
let kern = unsafe { &mut *crate::kernel_ptr() };
let next_pid = kern.thread_process(next_tid);
let current_pid = kern.thread_process(current_tid);
let core = types::id::CoreId::from_raw(0);
let _ = kern.set_thread_running(next_tid, core);
if let Some(ctx) = kern.thread_context(next_tid) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, next_tid);
if next_pid != current_pid
&& let Some(pid) = next_pid
{
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = crate::mmu::page_table::table_at(pt_addr);
crate::mmu::activate(pt);
}
}
let next = next_runnable();
let _ = kern().set_thread_running(next, this_core());
switch_to_thread(frame, None, next);
}
/// The next thread to run, idling until an interrupt makes one ready.
/// The next thread to run on this hart, idling until one is ready.
///
/// We are inside a trap, so SIE is clear and interrupts are not taken.
/// `wfi` still wakes once one is pending, and it is then serviced here by
/// polling `sip`, which is how a driver blocked on its IRQ notification
/// gets woken when nothing else can run.
/// Idling gives up the kernel lock: an idle hart holds nothing, so other
/// harts can make work for it (and kick it) meanwhile. It also leaves any
/// user address space first, so nobody needs to shoot it down. SIE stays
/// clear; `wfi` still wakes on a pending interrupt, serviced by polling.
fn next_runnable() -> types::id::ThreadId {
loop {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(tid) = kern.pick_next_thread() {
if let Some(tid) = kern().pick_next_thread() {
return tid;
}
crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
crate::smp::unlock();
unsafe { core::arch::asm!("wfi") };
crate::smp::service_ipi();
crate::smp::lock();
let _ = crate::smp::take_reschedule();
let sip: usize;
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & (1 << CAUSE_S_TIMER) != 0 {
@@ -271,49 +258,123 @@ fn next_runnable() -> types::id::ThreadId {
}
}
/// Claims every pending PLIC interrupt and hands each to the kernel.
/// Claims every pending PLIC interrupt and hands each to the kernel. Only
/// the boot hart receives external interrupts.
fn dispatch_external() {
loop {
let irq_num = crate::hw::plic::Plic::claim();
if irq_num == 0 {
return;
}
let kern = unsafe { &mut *crate::kernel_ptr() };
kern.handle_interrupt(types::id::IrqId::from_raw(irq_num as u16));
kern().handle_interrupt(types::id::IrqId::from_raw(irq_num as u16));
crate::hw::plic::Plic::complete(irq_num);
}
}
/// Last step before leaving the kernel to userspace: act on a reschedule
/// request, make sure this hart has a thread (its own may have been taken
/// away by another hart), and translate through that thread's address
/// space.
fn finish(frame: &mut TrapFrame) {
let kern = kern();
let current = kern.current_thread();
if crate::smp::take_reschedule()
&& let Some(cur) = current
&& let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
{
switch_to_thread(frame, Some(cur), next);
}
if kern.current_thread().is_none() {
switch_to_next(frame, None);
}
let root = kern
.current_thread()
.and_then(|t| kern.thread_process(t))
.map(|pid| kern.platform_mut().page_table_for(pid))
.unwrap_or(0);
if root != 0 && crate::mmu::active_root() != root {
crate::mmu::activate(crate::mmu::page_table::table_at(root));
}
}
/// Leaves the kernel for userspace for the first time on this hart, with
/// the kernel lock held: runs the thread the kernel already made current
/// here (init, on the boot hart) or the next one homed here.
pub fn enter_first() -> ! {
let mut frame = TrapFrame {
regs: [0; 32],
sepc: 0,
sstatus: 0,
};
match kern().current_thread() {
Some(tid) => switch_to_thread(&mut frame, None, tid),
None => switch_to_next(&mut frame, None),
}
finish(&mut frame);
crate::smp::unlock();
// SAFETY: `frame` holds a user context; sscratch is set from it on
// the way out, so later traps land at the top of this frame.
unsafe { trap_return(&mut frame) }
}
#[unsafe(no_mangle)]
extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
let scause: usize;
unsafe { core::arch::asm!("csrr {0}, scause", out(reg) scause) };
if frame.sstatus & SSTATUS_SPP != 0 {
// Checked before taking the lock: this hart may already hold it.
kernel_fault(frame, scause);
}
crate::smp::service_ipi();
crate::smp::lock();
if scause & CAUSE_INTERRUPT_BIT != 0 {
handle_interrupt(frame, scause & !CAUSE_INTERRUPT_BIT);
} else {
handle_exception(frame, scause);
}
finish(frame);
crate::smp::unlock();
}
fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
match cause {
CAUSE_S_TIMER => {
crate::hw::timer::handle_interrupt();
let kern = unsafe { &mut *crate::kernel_ptr() };
let core = types::id::CoreId::from_raw(0);
let kern = kern();
if let Some(current_tid) = kern.current_thread() {
let decision = kern.handle_tick(core);
let decision = kern.handle_tick(this_core());
if let domain::scheduler::SchedulerDecision::Preempt(next) = decision {
switch_to_thread(frame, current_tid, next);
switch_to_thread(frame, Some(current_tid), next);
}
}
}
CAUSE_S_EXTERNAL => dispatch_external(),
// Serviced on entry; any reschedule happens in `finish`.
CAUSE_S_SOFTWARE => {}
_ => {}
}
}
/// The kernel itself faulted (a W^X violation, a stack guard, a bad
/// pointer). Its state can no longer be trusted, and no user thread is to
/// blame.
fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! {
let uart = crate::hw::uart::Uart::new();
uart.write_str("KERNEL FAULT: scause=");
uart.write_hex(scause);
let stval: usize;
unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
uart.write_str(" stval=");
uart.write_hex(stval);
uart.write_str(" sepc=");
uart.write_hex(frame.sepc);
uart.write_str(" hart=");
uart.write_dec(crate::smp::hart_id());
uart.put_char(b'\n');
crate::halt(&uart, "fault in kernel mode");
}
fn handle_exception(frame: &mut TrapFrame, scause: usize) {
match scause {
CAUSE_ECALL_U => {
@@ -324,48 +385,24 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
return;
};
{
let kern = unsafe { &mut *crate::kernel_ptr() };
let kern = kern();
let current_tid = kern.current_thread();
let outcome = kern.handle_syscall(syscall);
match outcome {
kernel::SyscallOutcome::Complete(result) => write_result(frame, &result),
kernel::SyscallOutcome::BlockCaller(_) => {
if let Some(tid) = current_tid {
switch_to_next(frame, tid);
}
}
kernel::SyscallOutcome::BlockCaller(_) => switch_to_next(frame, current_tid),
kernel::SyscallOutcome::SwitchTo(next) => {
if let Some(tid) = current_tid {
switch_to_thread(frame, tid, next);
}
switch_to_thread(frame, current_tid, next)
}
kernel::SyscallOutcome::TerminateCaller => {
let uart = crate::hw::uart::Uart::new();
uart.write_str("k-os: process exited\n");
if let Some(tid) = current_tid {
switch_to_next(frame, tid);
}
switch_to_next(frame, current_tid);
}
}
}
}
_ if frame.sstatus & SSTATUS_SPP != 0 => {
// The kernel itself faulted (a W^X violation, the stack guard,
// a bad pointer). Its state can no longer be trusted, and no
// user thread is to blame.
let uart = crate::hw::uart::Uart::new();
uart.write_str("KERNEL FAULT: scause=");
uart.write_hex(scause);
let stval: usize;
unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
uart.write_str(" stval=");
uart.write_hex(stval);
uart.write_str(" sepc=");
uart.write_hex(frame.sepc);
uart.put_char(b'\n');
crate::halt(&uart, "fault in kernel mode");
}
_ => {
let uart = crate::hw::uart::Uart::new();
uart.write_str("FAULT: scause=");
@@ -378,7 +415,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
uart.write_hex(frame.sepc);
uart.put_char(b'\n');
let kern = unsafe { &mut *crate::kernel_ptr() };
let kern = kern();
if let Some(current_tid) = kern.current_thread() {
let fault = match scause {
2 => domain::fault::Fault::IllegalInstruction,
@@ -388,12 +425,10 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
_ => domain::fault::Fault::InvalidSyscall,
};
kern.handle_fault(current_tid, fault, frame.sepc);
switch_to_next(frame, current_tid);
} else {
loop {
unsafe { core::arch::asm!("wfi") };
}
switch_to_next(frame, Some(current_tid));
}
// With no current thread (another hart just destroyed it),
// `finish` moves on to the next one.
}
}
}