GKaszewski/k-os
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: SMP stage 1 (4 harts, per-hart guarded stacks, home-core affinity, IPIs, TLB shootdown, one temporary kernel lock), 4-core storm

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-27 00:56:15 +01:00
parent b87a62475f
commit 2e665fd66e
22 changed files with 995 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
crates/adapter-riscv/linker.ld
View File

@@ -46,15 +46,13 @@ SECTIONS {
_bss_end = .;
}
/* Left unmapped, so overflowing the kernel stack faults instead of
silently overwriting .bss. */
/* One kernel stack per hart (smp::MAX_HARTS, smp::STACK_SLOT): each
slot is an unmapped guard page, so overflowing a stack faults
instead of silently overwriting what lies below, then the stack. */
. = ALIGN(4096);
_stack_guard = .;
. += 4096;
_stack_bottom = .;
. += 0x80000;
_stack_top = .;
_stacks_start = .;
. += 8 * (4096 + 0x80000);
_stacks_end = .;
_heap_start = .;
}

14
crates/adapter-riscv/src/boot/hardware.rs
View File

@@ -8,12 +8,21 @@ unsafe extern "C" {
static _heap_start: u8;
}
pub fn init(uart: &Uart) -> &'static mut mmu::PageTable {
pub fn init(uart: &Uart, boot_hart: usize) -> &'static mut mmu::PageTable {
zero_bss();
let heap_start = unsafe { &_heap_start as *const u8 as usize };
mmu::init_allocator(heap_start);
let kernel_pt = mmu::init_kernel_page_table();
let harts = crate::smp::probe_harts();
if boot_hart >= harts {
crate::halt(uart, "boot hart outside the hart range");
}
crate::smp::init(
boot_hart,
harts,
mmu::virt_to_phys(kernel_pt as *const mmu::PageTable as usize),
);
mmu::activate(kernel_pt);
uart.write_str("k-os: MMU enabled (Sv39)\n");
match mmu::check_wx(kernel_pt) {
@@ -46,7 +55,8 @@ pub fn init(uart: &Uart) -> &'static mut mmu::PageTable {
if board::USER_POOL_PAGES > domain::memory::frame::MAX_FRAMES {
crate::halt(uart, "user pool larger than the frame table");
}
let riscv_platform = platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize);
let riscv_platform =
platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize, harts as u8);
// SAFETY: `init_in_place` writes every field of the kernel.
unsafe {
KERNEL.init_with(|slot| {

33
crates/adapter-riscv/src/boot/loader.rs
View File

@@ -13,7 +13,6 @@ const MAX_FILE_SIZE: usize = 64 * 1024;
pub struct LoadedProcess {
pub pid: ProcessId,
pub entry: usize,
}
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
@@ -48,7 +47,7 @@ pub fn load_init(uart: &Uart, blk: &mut VirtioBlk) -> (LoadedProcess, ThreadId)
setup_thread(tid, entry);
uart.write_str("k-os: init loaded\n");
(LoadedProcess { pid, entry }, tid)
(LoadedProcess { pid }, tid)
}
pub fn load_server(
@@ -69,35 +68,7 @@ pub fn load_server(
uart.write_str(label);
uart.write_str(" loaded\n");
LoadedProcess { pid, entry }
}
pub fn enter_userspace(pid: ProcessId, entry: usize, shared_buf_va: usize) -> ! {
let pt_addr = kern().platform_mut().page_table_for(pid);
if pt_addr != 0 {
mmu::activate(mmu::page_table::table_at(pt_addr));
}
let kernel_sp: usize;
unsafe { core::arch::asm!("mv {0}, sp", out(reg) kernel_sp) };
crate::trap::set_kernel_stack(kernel_sp);
unsafe {
core::arch::asm!(
"csrc sstatus, {spp}",
"csrs sstatus, {spie}",
"csrw sepc, {entry}",
"mv sp, {stack}",
"mv a0, {arg}",
"sret",
spp = in(reg) 1usize << 8,
spie = in(reg) 1usize << 5,
entry = in(reg) entry,
stack = in(reg) USER_VA_BASE,
arg = in(reg) shared_buf_va,
options(noreturn),
);
}
LoadedProcess { pid }
}
fn setup_thread(tid: ThreadId, entry: usize) {

31
crates/adapter-riscv/src/hw/plic.rs
View File

@@ -2,21 +2,36 @@ use crate::board;
const PLIC: usize = crate::mmu::phys_to_virt(board::PLIC_BASE);
const PLIC_PRIORITY: usize = PLIC;
const PLIC_S_ENABLE: usize = PLIC + 0x2080;
const PLIC_S_THRESHOLD: usize = PLIC + 0x201000;
const PLIC_S_CLAIM: usize = PLIC + 0x201004;
// External interrupts go to the boot hart only: its supervisor context is
// 2 * hart + 1 on QEMU virt (even contexts are machine mode).
fn context() -> usize {
2 * crate::smp::boot_hart() + 1
}
fn enable_base() -> usize {
PLIC + 0x2000 + context() * 0x80
}
fn threshold() -> usize {
PLIC + 0x20_0000 + context() * 0x1000
}
fn claim_reg() -> usize {
threshold() + 4
}
pub struct Plic;
impl Plic {
pub fn init() {
unsafe { core::ptr::write_volatile(PLIC_S_THRESHOLD as *mut u32, 0) };
unsafe { core::ptr::write_volatile(threshold() as *mut u32, 0) };
}
pub fn enable(irq: u32) {
let word = (irq / 32) as usize;
let bit = irq % 32;
let addr = PLIC_S_ENABLE + word * 4;
let addr = enable_base() + word * 4;
unsafe {
let current = core::ptr::read_volatile(addr as *const u32);
core::ptr::write_volatile(addr as *mut u32, current | (1 << bit));
@@ -28,7 +43,7 @@ impl Plic {
pub fn disable(irq: u32) {
let word = (irq / 32) as usize;
let bit = irq % 32;
let addr = PLIC_S_ENABLE + word * 4;
let addr = enable_base() + word * 4;
unsafe {
let current = core::ptr::read_volatile(addr as *const u32);
core::ptr::write_volatile(addr as *mut u32, current & !(1 << bit));
@@ -36,10 +51,10 @@ impl Plic {
}
pub fn claim() -> u32 {
unsafe { core::ptr::read_volatile(PLIC_S_CLAIM as *const u32) }
unsafe { core::ptr::read_volatile(claim_reg() as *const u32) }
}
pub fn complete(irq: u32) {
unsafe { core::ptr::write_volatile(PLIC_S_CLAIM as *mut u32, irq) };
unsafe { core::ptr::write_volatile(claim_reg() as *mut u32, irq) };
}
}

91
crates/adapter-riscv/src/main.rs
View File

@@ -9,6 +9,7 @@ mod hw;
mod kernel_cell;
mod mmu;
mod platform;
mod smp;
mod trap;
use core::arch::global_asm;
@@ -18,8 +19,8 @@ use kernel_cell::KernelCell;
static KERNEL: KernelCell<kernel::Kernel<platform::RiscVPlatform>> = KernelCell::uninit();
fn kern_mut() -> &'static mut kernel::Kernel<platform::RiscVPlatform> {
// SAFETY: single core; boot code runs before any trap can touch the
// kernel.
// SAFETY: boot runs with the kernel lock held and before any other
// hart has started.
unsafe { &mut *kernel_ptr() }
}
@@ -27,28 +28,55 @@ pub fn kernel_ptr() -> *mut kernel::Kernel<platform::RiscVPlatform> {
KERNEL.as_ptr()
}
// Firmware enters `_start` at the physical load address with the MMU off,
// but the kernel is linked in the upper half. Using only PC-relative code,
// the stub turns on `boot_page_table`, which maps RAM twice (at its
// physical address, just so the next instruction still exists, and in the
// upper half) plus the MMIO window high, then jumps to its own high address.
// kmain replaces this table with the real one, which maps nothing low.
// Firmware enters `_start` (and SBI starts other harts at
// `_secondary_start`) at the physical load address with the MMU off and
// the hart id in a0, but the kernel is linked in the upper half. Using
// only PC-relative code, the stub turns on `boot_page_table`, which maps
// RAM twice (at its physical address, just so the next instruction still
// exists, and in the upper half) plus the MMIO window high, then jumps to
// its own high address and takes the hart's kernel stack. The kernel then
// switches every hart to the real table, which maps nothing low.
global_asm!(
r#"
.section .text.entry
.globl _start
_start:
lla t2, start_high_address
j 1f
.globl _secondary_start
_secondary_start:
lla t2, secondary_high_address
1:
li t0, 8 # smp::MAX_HARTS
bgeu a0, t0, spin
mv s0, a0
lla t0, boot_page_table
srli t0, t0, 12
li t1, 8 << 60
or t0, t0, t1
csrw satp, t0
sfence.vma
ld t0, start_high_address
ld t0, 0(t2)
jr t0
# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT
start_high:
lla sp, _stack_top
call hart_stack
mv a0, s0
call kmain
j spin
secondary_high:
call hart_stack
mv a0, s0
call kmain_secondary
j spin
hart_stack:
lla sp, _stacks_start
addi t0, s0, 1
li t1, 0x81000
mul t0, t0, t1
add sp, sp, t0
ret
spin:
wfi
j spin
@@ -57,6 +85,8 @@ spin:
.balign 8
start_high_address:
.quad start_high
secondary_high_address:
.quad secondary_high
.popsection
# Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with
@@ -79,15 +109,21 @@ boot_page_table:
const SHARED_BUF_VA: usize = 0x30000;
#[unsafe(no_mangle)]
extern "C" fn kmain() -> ! {
extern "C" fn kmain(hart: usize) -> ! {
let uart = Uart::new();
uart.write_str("k-os: booted on RISC-V\n");
boot::hardware::init(&uart);
boot::hardware::init(&uart, hart);
// Held until the boot hart first enters userspace; harts started
// below wait for it. Not before init: zeroing .bss resets the lock.
smp::lock();
let mut blk =
boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device"));
let (init, _) = boot::loader::load_init(&uart, &mut blk);
let (init, init_thread) = boot::loader::load_init(&uart, &mut blk);
if let Some(ctx) = kern_mut().thread_context_mut(init_thread) {
ctx.registers_mut()[10] = SHARED_BUF_VA;
}
let storage = boot::loader::load_server(&uart, &mut blk, b"storage.elf", "storage");
let display = boot::loader::load_server(&uart, &mut blk, b"display.elf", "display");
let input = boot::loader::load_server(&uart, &mut blk, b"input.elf", "input");
@@ -106,8 +142,35 @@ extern "C" fn kmain() -> ! {
.grant_boot_authority(init.pid, &board)
.unwrap_or_else(|_| halt(&uart, "boot authority failed"));
start_other_harts(&uart, hart);
uart.write_str("k-os: entering userspace\n");
boot::loader::enter_userspace(init.pid, init.entry, SHARED_BUF_VA);
trap::enter_first();
}
fn start_other_harts(uart: &Uart, boot: usize) {
unsafe extern "C" {
fn _secondary_start();
}
// Harts start with the MMU off, so at the entry's physical address.
let entry = mmu::virt_to_phys(_secondary_start as *const () as usize);
for hart in (0..smp::num_harts()).filter(|h| *h != boot) {
if !smp::sbi_hart_start(hart, entry) {
halt(uart, "could not start a hart");
}
}
uart.write_str("k-os: ");
uart.write_dec(smp::num_harts());
uart.write_str(" harts\n");
}
/// Where every hart but the boot hart starts, on its own stack.
#[unsafe(no_mangle)]
extern "C" fn kmain_secondary(_hart: usize) -> ! {
trap::init();
hw::timer::init();
mmu::activate(mmu::page_table::table_at(smp::kernel_root()));
smp::lock();
trap::enter_first();
}
pub fn halt(uart: &Uart, msg: &str) -> ! {

44
crates/adapter-riscv/src/mmu/page_table.rs
View File

@@ -1,6 +1,7 @@
use super::allocator::alloc_page_table;
use super::{phys_to_virt, virt_to_phys};
use crate::board;
use crate::smp;
use platform::{FrameSource, MapError};
use types::address::{PhysicalAddress, VirtualAddress};
use types::mapping::PagePerms;
@@ -148,20 +149,31 @@ pub fn init_kernel_page_table() -> &'static mut PageTable {
// The kernel image, then the rest of RAM, each with only the access it
// needs: code is never writable, data never executable. RAM below the
// image belongs to the firmware and stays unmapped, as does the guard
// page under the kernel stack. Nothing is mapped in the lower half,
// which is userspace's alone.
// page under each hart's kernel stack. Nothing is mapped in the lower
// half, which is userspace's alone.
let layout = KernelLayout::from_linker();
let ram_end = phys_to_virt(board::RAM_BASE + board::RAM_SIZE);
let stacks = (0..smp::MAX_HARTS).map(|h| {
let slot = layout.stacks_start + h * smp::STACK_SLOT;
(
slot + PAGE_SIZE,
slot + smp::STACK_SLOT,
Sv39Flags::kernel_rw(),
)
});
for (start, end, flags) in [
(layout.text_start, layout.text_end, Sv39Flags::kernel_rx()),
(layout.text_end, layout.rodata_end, Sv39Flags::kernel_ro()),
(
layout.rodata_end,
layout.stack_guard,
layout.stacks_start,
Sv39Flags::kernel_rw(),
),
(layout.stack_bottom, ram_end, Sv39Flags::kernel_rw()),
] {
(layout.stacks_end, ram_end, Sv39Flags::kernel_rw()),
]
.into_iter()
.chain(stacks)
{
let (va, pa) = direct(virt_to_phys(start));
map(
root,
@@ -200,8 +212,8 @@ pub struct KernelLayout {
pub text_start: usize,
pub text_end: usize,
pub rodata_end: usize,
pub stack_guard: usize,
pub stack_bottom: usize,
pub stacks_start: usize,
pub stacks_end: usize,
}
impl KernelLayout {
@@ -210,8 +222,8 @@ impl KernelLayout {
static _text_start: u8;
static _text_end: u8;
static _rodata_end: u8;
static _stack_guard: u8;
static _stack_bottom: u8;
static _stacks_start: u8;
static _stacks_end: u8;
}
// SAFETY: only the symbols' addresses are taken, never their values.
unsafe {
@@ -219,8 +231,8 @@ impl KernelLayout {
text_start: &_text_start as *const u8 as usize,
text_end: &_text_end as *const u8 as usize,
rodata_end: &_rodata_end as *const u8 as usize,
stack_guard: &_stack_guard as *const u8 as usize,
stack_bottom: &_stack_bottom as *const u8 as usize,
stacks_start: &_stacks_start as *const u8 as usize,
stacks_end: &_stacks_end as *const u8 as usize,
}
}
}
@@ -240,12 +252,15 @@ pub enum WxViolation {
/// Walks every leaf of `root` and checks the kernel layout: nothing is
/// both writable and executable, only kernel text is executable, the
/// firmware and the stack guard are unmapped, and nothing at all is mapped
/// firmware and every stack guard are unmapped, and nothing at all is mapped
/// in the lower half. Returns the number of executable pages.
pub fn check_wx(root: &PageTable) -> Result<usize, WxViolation> {
let layout = KernelLayout::from_linker();
let text = layout.text_start..layout.text_end;
let guard = layout.stack_guard..layout.stack_bottom;
let is_guard = |va: usize| {
(layout.stacks_start..layout.stacks_end).contains(&va)
&& (va - layout.stacks_start).is_multiple_of(smp::STACK_SLOT)
};
let firmware = phys_to_virt(board::RAM_BASE)..layout.text_start;
let mut executable = 0;
for (va, bits) in leaves(root) {
@@ -261,7 +276,7 @@ pub fn check_wx(root: &PageTable) -> Result<usize, WxViolation> {
}
executable += 1;
}
if guard.contains(&va) || firmware.contains(&va) {
if is_guard(va) || firmware.contains(&va) {
return Err(WxViolation::MappedHole(va));
}
}
@@ -299,6 +314,7 @@ fn leaves(root: &PageTable) -> impl Iterator<Item = (usize, u64)> + '_ {
}
pub fn activate(root: &PageTable) {
smp::note_active_root(virt_to_phys(root as *const PageTable as usize));
let satp = (8u64 << 60) | table_ppn(root);
unsafe {
core::arch::asm!(

19
crates/adapter-riscv/src/platform.rs
View File

@@ -10,6 +10,7 @@ use types::{
use crate::board;
use crate::hw::{plic::Plic, timer};
use crate::mmu;
use crate::smp;
use crate::trap;
pub struct RiscVPlatform {
@@ -20,9 +21,9 @@ pub struct RiscVPlatform {
}
impl RiscVPlatform {
pub fn new(kernel_pt_addr: usize) -> Self {
pub fn new(kernel_pt_addr: usize, num_cores: u8) -> Self {
Self {
num_cores: 1,
num_cores,
kernel_pt: kernel_pt_addr,
page_tables: [0usize; MAX_PROCESSES],
}
@@ -63,8 +64,10 @@ impl Platform for RiscVPlatform {
let Some(root) = self.root(process) else {
return;
};
// The caller may be tearing down the space we are running on; get
// off it before the kernel frees its frames.
// No hart may still translate through it once the kernel frees its
// frames: other harts leave it (and wait for nothing but that), and
// the caller may be tearing down the space we are running on.
smp::shoot_down(root);
if mmu::active_root() == root {
// SAFETY: the kernel page table lives in the boot heap forever.
mmu::activate(unsafe { &*(self.kernel_pt as *const mmu::PageTable) });
@@ -95,6 +98,8 @@ impl Platform for RiscVPlatform {
fn unmap_page(&mut self, process: ProcessId, va: VirtualAddress) {
if let Some(root) = self.root(process) {
mmu::unmap_user_page(root, va.raw());
// Other harts running this process may still cache it.
smp::shoot_down(root);
}
}
@@ -144,10 +149,12 @@ impl Platform for RiscVPlatform {
Plic::enable(irq.raw() as u32);
}
fn send_ipi(&mut self, _target: CoreId) {}
fn send_ipi(&mut self, target: CoreId) {
smp::kick(target.raw() as usize);
}
fn core_id(&self) -> CoreId {
CoreId::from_raw(0)
CoreId::from_raw(smp::hart_id() as u8)
}
fn num_cores(&self) -> u8 {

220
crates/adapter-riscv/src/smp.rs Normal file
View File

@@ -0,0 +1,220 @@
//! Harts. Each hart has its own kernel stack, and its hart id doubles as
//! its kernel `CoreId`. The kernel is entered under one lock for now (stage
//! 1 of multi-core; the kernel itself stays a sequential program), so
//! userspace runs in parallel and the kernel does not.
//!
//! Harts talk through IPIs carrying two kinds of request:
//! - reschedule: the kernel made work for this hart, or took its thread
//! away; handled under the lock on the way out of the kernel.
//! - leave root R: a page table this hart may be using is changing or about
//! to be freed; the hart switches to the kernel's root and flushes its
//! TLB. This needs no lock, so it is serviced even while a hart spins
//! waiting for the lock or idles, and the sender (which holds the lock)
//! can wait for the acknowledgement without deadlocking.
use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
use crate::mmu;
/// Must match linker.ld.
pub const MAX_HARTS: usize = 8;
/// A guard page and a 512 KiB stack (boot builds big tables on it). Must
/// match linker.ld.
pub const STACK_SLOT: usize = 4096 + 0x8_0000;
const SIP_SSIP: usize = 1 << 1;
/// The hart running this code, from which kernel stack it is on.
pub fn hart_id() -> usize {
let sp: usize;
// SAFETY: reading sp has no side effects.
unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) };
(sp - stacks_start()) / STACK_SLOT
}
pub fn stacks_start() -> usize {
unsafe extern "C" {
static _stacks_start: u8;
}
// SAFETY: only the symbol's address is taken.
unsafe { &_stacks_start as *const u8 as usize }
}
static BOOT_HART: AtomicUsize = AtomicUsize::new(0);
static NUM_HARTS: AtomicUsize = AtomicUsize::new(1);
/// Physical address of the kernel's own root table.
static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0);
/// The root table each hart translates with.
static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS];
static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
pub fn boot_hart() -> usize {
BOOT_HART.load(Ordering::Relaxed)
}
pub fn num_harts() -> usize {
NUM_HARTS.load(Ordering::Relaxed)
}
pub fn kernel_root() -> usize {
KERNEL_ROOT.load(Ordering::Relaxed)
}
/// Records which hart booted and how many there are, and where the
/// kernel's root table is.
pub fn init(boot: usize, harts: usize, kernel_root: usize) {
BOOT_HART.store(boot, Ordering::Relaxed);
NUM_HARTS.store(harts, Ordering::Relaxed);
KERNEL_ROOT.store(kernel_root, Ordering::Relaxed);
}
/// Called by `mmu::activate`.
pub fn note_active_root(root: usize) {
if let Some(r) = ACTIVE_ROOT.get(hart_id()) {
r.store(root, Ordering::SeqCst);
}
}
// ---- the kernel lock ----
static NEXT_TICKET: AtomicU32 = AtomicU32::new(0);
static SERVING: AtomicU32 = AtomicU32::new(0);
/// Takes the kernel lock, first come first served. Leave-root requests
/// are serviced while waiting.
pub fn lock() {
let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
while SERVING.load(Ordering::Acquire) != ticket {
service_ipi();
core::hint::spin_loop();
}
}
pub fn unlock() {
SERVING.fetch_add(1, Ordering::Release);
}
// ---- IPIs ----
/// The lock-free half of an IPI: acknowledges it and, if asked, leaves the
/// root being changed. Safe to call at any time, any number of times.
pub fn service_ipi() {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & SIP_SSIP == 0 {
return;
}
// Clear first: a request posted after this raises SSIP again.
// SAFETY: clearing our own pending software interrupt.
unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) };
let me = hart_id();
let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else {
return;
};
let root = leave.swap(0, Ordering::SeqCst);
if root != 0 {
if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) {
mmu::activate(mmu::page_table::table_at(kernel_root()));
}
// SAFETY: flushing translations has no other effect.
unsafe { core::arch::asm!("sfence.vma zero, zero") };
left.fetch_add(1, Ordering::SeqCst);
}
}
/// Asks `hart` to look at its run queue again, on its way out of the
/// kernel.
pub fn kick(hart: usize) {
if let Some(r) = RESCHEDULE.get(hart) {
r.store(true, Ordering::SeqCst);
sbi_send_ipi(hart);
}
}
/// Consumes this hart's reschedule request.
pub fn take_reschedule() -> bool {
RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.swap(false, Ordering::SeqCst))
}
/// Makes every other hart that translates through `root` leave it and
/// flush, and waits until each has. Called with the kernel lock held, so
/// no hart can start using `root` meanwhile.
pub fn shoot_down(root: usize) {
let me = hart_id();
for hart in 0..num_harts() {
if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root {
continue;
}
let seen = LEFT[hart].load(Ordering::SeqCst);
LEAVE_ROOT[hart].store(root, Ordering::SeqCst);
sbi_send_ipi(hart);
while LEFT[hart].load(Ordering::SeqCst) == seen {
core::hint::spin_loop();
}
}
}
// ---- SBI ----
const SBI_EXT_IPI: usize = 0x735049;
const SBI_EXT_HSM: usize = 0x48534D;
fn sbi_send_ipi(hart: usize) {
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") 1usize => _,
inlateout("a1") hart => _,
in("a6") 0usize,
in("a7") SBI_EXT_IPI,
);
}
}
/// Whether `hart` exists, per SBI HSM `hart_get_status`.
fn sbi_hart_exists(hart: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
lateout("a1") _,
in("a6") 2usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}
/// The number of harts, counting up from hart 0 until one is missing.
pub fn probe_harts() -> usize {
(0..MAX_HARTS)
.take_while(|h| sbi_hart_exists(*h))
.count()
.max(1)
}
/// Starts `hart` at the physical address `entry` with the MMU off.
pub fn sbi_hart_start(hart: usize, entry: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
inlateout("a1") entry => _,
in("a2") 0usize,
in("a6") 0usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}

235
crates/adapter-riscv/src/trap/mod.rs
View File

@@ -60,7 +60,14 @@ _from_user:
mv a0, sp
call trap_handler_rust
j _restore_frame
# trap_return(frame): resume whatever `frame` describes, as if returning
# from a trap that saved it. Used to enter userspace for the first time.
.globl trap_return
trap_return:
mv sp, a0
_restore_frame:
ld t0, 256(sp)
csrw sepc, t0
ld t0, 264(sp)
@@ -121,6 +128,7 @@ _sret_smode:
);
const CAUSE_INTERRUPT_BIT: usize = 1 << 63;
const CAUSE_S_SOFTWARE: usize = 1;
const CAUSE_S_TIMER: usize = 5;
const CAUSE_S_EXTERNAL: usize = 9;
const CAUSE_ECALL_U: usize = 8;
@@ -177,88 +185,67 @@ fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFra
}
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(result) = kern.take_pending_result(tid) {
if let Some(result) = kern().take_pending_result(tid) {
write_result(frame, &result);
}
}
unsafe extern "C" {
fn trap_return(frame: *mut TrapFrame) -> !;
}
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
// SAFETY: only called with the kernel lock held.
unsafe { &mut *crate::kernel_ptr() }
}
fn this_core() -> types::id::CoreId {
types::id::CoreId::from_raw(crate::smp::hart_id() as u8)
}
/// Saves `frame` into `from` (if it still exists) and loads `to`.
fn switch_to_thread(
frame: &mut TrapFrame,
current_tid: types::id::ThreadId,
next_tid: types::id::ThreadId,
from: Option<types::id::ThreadId>,
to: types::id::ThreadId,
) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(ctx) = kern.thread_context_mut(current_tid) {
let kern = kern();
if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next_pid = kern.thread_process(next_tid);
let current_pid = kern.thread_process(current_tid);
if let Some(ctx) = kern.thread_context(next_tid) {
if let Some(ctx) = kern.thread_context(to) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, next_tid);
if next_pid != current_pid
&& let Some(pid) = next_pid
{
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = crate::mmu::page_table::table_at(pt_addr);
crate::mmu::activate(pt);
}
}
apply_pending_result(frame, to);
}
fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(ctx) = kern.thread_context_mut(current_tid) {
/// Saves `from` and runs the next ready thread, idling until there is one.
fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next_tid = next_runnable();
let kern = unsafe { &mut *crate::kernel_ptr() };
let next_pid = kern.thread_process(next_tid);
let current_pid = kern.thread_process(current_tid);
let core = types::id::CoreId::from_raw(0);
let _ = kern.set_thread_running(next_tid, core);
if let Some(ctx) = kern.thread_context(next_tid) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, next_tid);
if next_pid != current_pid
&& let Some(pid) = next_pid
{
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = crate::mmu::page_table::table_at(pt_addr);
crate::mmu::activate(pt);
}
}
let next = next_runnable();
let _ = kern().set_thread_running(next, this_core());
switch_to_thread(frame, None, next);
}
/// The next thread to run, idling until an interrupt makes one ready.
/// The next thread to run on this hart, idling until one is ready.
///
/// We are inside a trap, so SIE is clear and interrupts are not taken.
/// `wfi` still wakes once one is pending, and it is then serviced here by
/// polling `sip`, which is how a driver blocked on its IRQ notification
/// gets woken when nothing else can run.
/// Idling gives up the kernel lock: an idle hart holds nothing, so other
/// harts can make work for it (and kick it) meanwhile. It also leaves any
/// user address space first, so nobody needs to shoot it down. SIE stays
/// clear; `wfi` still wakes on a pending interrupt, serviced by polling.
fn next_runnable() -> types::id::ThreadId {
loop {
let kern = unsafe { &mut *crate::kernel_ptr() };
if let Some(tid) = kern.pick_next_thread() {
if let Some(tid) = kern().pick_next_thread() {
return tid;
}
crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
crate::smp::unlock();
unsafe { core::arch::asm!("wfi") };
crate::smp::service_ipi();
crate::smp::lock();
let _ = crate::smp::take_reschedule();
let sip: usize;
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & (1 << CAUSE_S_TIMER) != 0 {
@@ -271,49 +258,123 @@ fn next_runnable() -> types::id::ThreadId {
}
}
/// Claims every pending PLIC interrupt and hands each to the kernel.
/// Claims every pending PLIC interrupt and hands each to the kernel. Only
/// the boot hart receives external interrupts.
fn dispatch_external() {
loop {
let irq_num = crate::hw::plic::Plic::claim();
if irq_num == 0 {
return;
}
let kern = unsafe { &mut *crate::kernel_ptr() };
kern.handle_interrupt(types::id::IrqId::from_raw(irq_num as u16));
kern().handle_interrupt(types::id::IrqId::from_raw(irq_num as u16));
crate::hw::plic::Plic::complete(irq_num);
}
}
/// Last step before leaving the kernel to userspace: act on a reschedule
/// request, make sure this hart has a thread (its own may have been taken
/// away by another hart), and translate through that thread's address
/// space.
fn finish(frame: &mut TrapFrame) {
let kern = kern();
let current = kern.current_thread();
if crate::smp::take_reschedule()
&& let Some(cur) = current
&& let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
{
switch_to_thread(frame, Some(cur), next);
}
if kern.current_thread().is_none() {
switch_to_next(frame, None);
}
let root = kern
.current_thread()
.and_then(|t| kern.thread_process(t))
.map(|pid| kern.platform_mut().page_table_for(pid))
.unwrap_or(0);
if root != 0 && crate::mmu::active_root() != root {
crate::mmu::activate(crate::mmu::page_table::table_at(root));
}
}
/// Leaves the kernel for userspace for the first time on this hart, with
/// the kernel lock held: runs the thread the kernel already made current
/// here (init, on the boot hart) or the next one homed here.
pub fn enter_first() -> ! {
let mut frame = TrapFrame {
regs: [0; 32],
sepc: 0,
sstatus: 0,
};
match kern().current_thread() {
Some(tid) => switch_to_thread(&mut frame, None, tid),
None => switch_to_next(&mut frame, None),
}
finish(&mut frame);
crate::smp::unlock();
// SAFETY: `frame` holds a user context; sscratch is set from it on
// the way out, so later traps land at the top of this frame.
unsafe { trap_return(&mut frame) }
}
#[unsafe(no_mangle)]
extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
let scause: usize;
unsafe { core::arch::asm!("csrr {0}, scause", out(reg) scause) };
if frame.sstatus & SSTATUS_SPP != 0 {
// Checked before taking the lock: this hart may already hold it.
kernel_fault(frame, scause);
}
crate::smp::service_ipi();
crate::smp::lock();
if scause & CAUSE_INTERRUPT_BIT != 0 {
handle_interrupt(frame, scause & !CAUSE_INTERRUPT_BIT);
} else {
handle_exception(frame, scause);
}
finish(frame);
crate::smp::unlock();
}
fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
match cause {
CAUSE_S_TIMER => {
crate::hw::timer::handle_interrupt();
let kern = unsafe { &mut *crate::kernel_ptr() };
let core = types::id::CoreId::from_raw(0);
let kern = kern();
if let Some(current_tid) = kern.current_thread() {
let decision = kern.handle_tick(core);
let decision = kern.handle_tick(this_core());
if let domain::scheduler::SchedulerDecision::Preempt(next) = decision {
switch_to_thread(frame, current_tid, next);
switch_to_thread(frame, Some(current_tid), next);
}
}
}
CAUSE_S_EXTERNAL => dispatch_external(),
// Serviced on entry; any reschedule happens in `finish`.
CAUSE_S_SOFTWARE => {}
_ => {}
}
}
/// The kernel itself faulted (a W^X violation, a stack guard, a bad
/// pointer). Its state can no longer be trusted, and no user thread is to
/// blame.
fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! {
let uart = crate::hw::uart::Uart::new();
uart.write_str("KERNEL FAULT: scause=");
uart.write_hex(scause);
let stval: usize;
unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
uart.write_str(" stval=");
uart.write_hex(stval);
uart.write_str(" sepc=");
uart.write_hex(frame.sepc);
uart.write_str(" hart=");
uart.write_dec(crate::smp::hart_id());
uart.put_char(b'\n');
crate::halt(&uart, "fault in kernel mode");
}
fn handle_exception(frame: &mut TrapFrame, scause: usize) {
match scause {
CAUSE_ECALL_U => {
@@ -324,48 +385,24 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
return;
};
{
let kern = unsafe { &mut *crate::kernel_ptr() };
let kern = kern();
let current_tid = kern.current_thread();
let outcome = kern.handle_syscall(syscall);
match outcome {
kernel::SyscallOutcome::Complete(result) => write_result(frame, &result),
kernel::SyscallOutcome::BlockCaller(_) => {
if let Some(tid) = current_tid {
switch_to_next(frame, tid);
}
}
kernel::SyscallOutcome::BlockCaller(_) => switch_to_next(frame, current_tid),
kernel::SyscallOutcome::SwitchTo(next) => {
if let Some(tid) = current_tid {
switch_to_thread(frame, tid, next);
}
switch_to_thread(frame, current_tid, next)
}
kernel::SyscallOutcome::TerminateCaller => {
let uart = crate::hw::uart::Uart::new();
uart.write_str("k-os: process exited\n");
if let Some(tid) = current_tid {
switch_to_next(frame, tid);
}
switch_to_next(frame, current_tid);
}
}
}
}
_ if frame.sstatus & SSTATUS_SPP != 0 => {
// The kernel itself faulted (a W^X violation, the stack guard,
// a bad pointer). Its state can no longer be trusted, and no
// user thread is to blame.
let uart = crate::hw::uart::Uart::new();
uart.write_str("KERNEL FAULT: scause=");
uart.write_hex(scause);
let stval: usize;
unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
uart.write_str(" stval=");
uart.write_hex(stval);
uart.write_str(" sepc=");
uart.write_hex(frame.sepc);
uart.put_char(b'\n');
crate::halt(&uart, "fault in kernel mode");
}
_ => {
let uart = crate::hw::uart::Uart::new();
uart.write_str("FAULT: scause=");
@@ -378,7 +415,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
uart.write_hex(frame.sepc);
uart.put_char(b'\n');
let kern = unsafe { &mut *crate::kernel_ptr() };
let kern = kern();
if let Some(current_tid) = kern.current_thread() {
let fault = match scause {
2 => domain::fault::Fault::IllegalInstruction,
@@ -388,12 +425,10 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
_ => domain::fault::Fault::InvalidSyscall,
};
kern.handle_fault(current_tid, fault, frame.sepc);
switch_to_next(frame, current_tid);
} else {
loop {
unsafe { core::arch::asm!("wfi") };
}
switch_to_next(frame, Some(current_tid));
}
// With no current thread (another hart just destroyed it),
// `finish` moves on to the next one.
}
}
}