GKaszewski/k-os
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(adapter): split trap/page_table/smp/entry by concern; trim comments

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-27 13:21:53 +01:00
parent 95dbe7b9d5
commit 99273db6d3
24 changed files with 912 additions and 928 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/adapter-riscv/src/boot_fs.rs
View File

@@ -67,7 +67,6 @@ pub fn read_file_from_disk(
) -> Option<usize> {
let mut sector_buf = [0u8; SECTOR_SIZE];
// Read boot sector
if !blk.read_sector(0, &mut sector_buf) {
return None;
}
@@ -75,7 +74,6 @@ pub fn read_file_from_disk(
let fat_name = to_fat_name(filename);
// Search root directory
let mut first_cluster: u16 = 0;
let mut file_size: u32 = 0;
let mut found = false;
@@ -132,7 +130,6 @@ pub fn read_file_from_disk(
read += chunk;
}
// Read next cluster from FAT
let fat_offset = (cluster as u32) * 2;
let fat_sector = fs.reserved_sectors as u32 + fat_offset / SECTOR_SIZE as u32;
let fat_byte = (fat_offset % SECTOR_SIZE as u32) as usize;

6
crates/adapter-riscv/src/devices/mod.rs
View File

@@ -1,8 +1,4 @@
// What boot still sets up for userspace, all through the kernel's mapping
// API: the display's page quota and the BootInfo page that tells init
// where its device capabilities are. Drivers set up their own devices
// with the capabilities init hands them; processes share memory through
// regions they create.
// Boot-time userspace setup: the display's page quota and init's BootInfo page.
use crate::hw::uart::Uart;
use types::address::VirtualAddress;

73
crates/adapter-riscv/src/entry.rs Normal file
View File

@@ -0,0 +1,73 @@
use core::arch::global_asm;
// Entered with the MMU off at the physical address; the kernel is linked high.
// PC-relative code only: enable `boot_page_table` (RAM mapped low for the next
// instruction and high, MMIO high), then jump high and take the hart's stack.
global_asm!(
r#"
.section .text.entry
.globl _start
_start:
lla t2, start_high_address
j 1f
.globl _secondary_start
_secondary_start:
lla t2, secondary_high_address
1:
li t0, 8 # smp::MAX_HARTS
bgeu a0, t0, spin
mv s0, a0
lla t0, boot_page_table
srli t0, t0, 12
li t1, 8 << 60
or t0, t0, t1
csrw satp, t0
sfence.vma
ld t0, 0(t2)
jr t0
# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT
start_high:
call hart_stack
mv a0, s0
call kmain
j spin
secondary_high:
call hart_stack
mv a0, s0
call kmain_secondary
j spin
hart_stack:
lla sp, _stacks_start
addi t0, s0, 1
li t1, 0x81000
mul t0, t0, t1
add sp, sp, t0
ret
spin:
wfi
j spin
.pushsection .rodata
.balign 8
start_high_address:
.quad start_high
secondary_high_address:
.quad secondary_high
.popsection
# Sv39 root of 1 GiB leaves; flags V|R|W|A|D = 0xc7, plus X = 0xcf.
.pushsection .data
.balign 4096
boot_page_table:
.quad 0
.quad 0
.quad 0x200000cf # [2] 0x8000_0000 -> RAM, for the jump only
.fill 253, 8, 0
.quad 0x000000c7 # [256] PHYS_OFFSET + 0 -> MMIO
.quad 0
.quad 0x200000cf # [258] PHYS_OFFSET + 0x8000_0000 -> RAM
.fill 253, 8, 0
.popsection
"#
);

9
crates/adapter-riscv/src/kernel_cell.rs
View File

@@ -7,7 +7,7 @@ pub struct KernelCell<T> {
inner: UnsafeCell<MaybeUninit<T>>,
}
// Single-core, interrupts disabled during kernel access.
// Access is serialized by the kernel lock.
unsafe impl<T> Sync for KernelCell<T> {}
impl<T> KernelCell<T> {
@@ -18,8 +18,7 @@ impl<T> KernelCell<T> {
}
}
/// Initializes the cell by letting `build` write the value straight
/// into its storage, so a large `T` never lives on the stack.
/// Initializes in place, so a large `T` never lives on the stack.
///
/// # Safety
/// `build` must fully initialize the `T` behind the pointer.
@@ -28,9 +27,7 @@ impl<T> KernelCell<T> {
self.initialized.store(true, Ordering::Release);
}
/// Returns a raw pointer to the kernel. Callers must ensure
/// no aliasing `&mut` references exist. Safe on single-core
/// with interrupts disabled during kernel calls.
/// Raw pointer to the kernel; callers must avoid aliasing `&mut`.
pub fn as_ptr(&self) -> *mut T {
debug_assert!(self.initialized.load(Ordering::Acquire));
unsafe { (*self.inner.get()).as_mut_ptr() }

85
crates/adapter-riscv/src/main.rs
View File

@@ -5,6 +5,7 @@ mod board;
mod boot;
mod boot_fs;
mod devices;
mod entry;
mod hw;
mod kernel_cell;
mod mmu;
@@ -12,7 +13,6 @@ mod platform;
mod smp;
mod trap;
use core::arch::global_asm;
use hw::uart::Uart;
use kernel_cell::KernelCell;
@@ -28,92 +28,13 @@ pub fn kernel_ptr() -> *mut kernel::Kernel<platform::RiscVPlatform> {
KERNEL.as_ptr()
}
// Firmware enters `_start` (and SBI starts other harts at
// `_secondary_start`) at the physical load address with the MMU off and
// the hart id in a0, but the kernel is linked in the upper half. Using
// only PC-relative code, the stub turns on `boot_page_table`, which maps
// RAM twice (at its physical address, just so the next instruction still
// exists, and in the upper half) plus the MMIO window high, then jumps to
// its own high address and takes the hart's kernel stack. The kernel then
// switches every hart to the real table, which maps nothing low.
global_asm!(
r#"
.section .text.entry
.globl _start
_start:
lla t2, start_high_address
j 1f
.globl _secondary_start
_secondary_start:
lla t2, secondary_high_address
1:
li t0, 8 # smp::MAX_HARTS
bgeu a0, t0, spin
mv s0, a0
lla t0, boot_page_table
srli t0, t0, 12
li t1, 8 << 60
or t0, t0, t1
csrw satp, t0
sfence.vma
ld t0, 0(t2)
jr t0
# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT
start_high:
call hart_stack
mv a0, s0
call kmain
j spin
secondary_high:
call hart_stack
mv a0, s0
call kmain_secondary
j spin
hart_stack:
lla sp, _stacks_start
addi t0, s0, 1
li t1, 0x81000
mul t0, t0, t1
add sp, sp, t0
ret
spin:
wfi
j spin
.pushsection .rodata
.balign 8
start_high_address:
.quad start_high
secondary_high_address:
.quad secondary_high
.popsection
# Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with
# flags V|R|W|A|D = 0xc7, plus X = 0xcf.
.pushsection .data
.balign 4096
boot_page_table:
.quad 0
.quad 0
.quad 0x200000cf # [2] 0x8000_0000 -> RAM, for the jump only
.fill 253, 8, 0
.quad 0x000000c7 # [256] PHYS_OFFSET + 0 -> MMIO
.quad 0
.quad 0x200000cf # [258] PHYS_OFFSET + 0x8000_0000 -> RAM
.fill 253, 8, 0
.popsection
"#
);
#[unsafe(no_mangle)]
extern "C" fn kmain(hart: usize) -> ! {
let uart = Uart::new();
uart.write_str("k-os: booted on RISC-V\n");
boot::hardware::init(&uart, hart);
// Held until the boot hart first enters userspace; harts started
// below wait for it. Not before init: zeroing .bss resets the lock.
// Not before init: zeroing .bss resets the lock.
smp::lock();
let mut blk =
boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device"));
@@ -129,7 +50,7 @@ extern "C" fn kmain(hart: usize) -> ! {
.unwrap_or_else(|_| halt(&uart, "board registration failed"));
devices::display_quota(&uart, display.pid);
boot::caps::grant_all(&uart, init.pid, storage.pid, display.pid, input.pid);
// After grant_all, so the slots init already relies on stay put.
// After grant_all: init already relies on those slots.
let mut placed = [None; board::BOARD_DEVICES];
let system_control = kern_mut()
.grant_boot_authority(init.pid, &board, &mut placed)

4
crates/adapter-riscv/src/mmu/allocator.rs
View File

@@ -1,9 +1,7 @@
use super::page_table::{PAGE_SIZE, PageTable};
use crate::board;
// Boot-time bump allocator for kernel-only memory: the kernel page table
// and boot scratch buffers. User memory and user page tables come from the
// kernel's frame table instead. It must never reach the user pool.
// Boot bump allocator for kernel-only memory; must never reach the user pool.
static mut NEXT_FREE: usize = 0;
pub fn init_allocator(start: usize) {

9
crates/adapter-riscv/src/mmu/mod.rs
View File

@@ -7,10 +7,8 @@ pub use page_table::{
create_user_root, init_kernel_page_table, map_user_page, translate_user, unmap_user_page,
};
/// The kernel lives in the upper half of the address space: all of RAM and
/// the MMIO windows appear at their physical address plus this offset (the
/// start of Sv39's upper half), and the kernel image is linked there too,
/// so one offset converts any kernel address. Must match linker.ld.
/// Direct-map offset: RAM, MMIO and the kernel image sit at physical + this
/// (start of Sv39's upper half). Must match linker.ld.
pub const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000;
/// Where the kernel sees physical address `pa`.
@@ -18,8 +16,7 @@ pub const fn phys_to_virt(pa: usize) -> usize {
pa + PHYS_OFFSET
}
/// The physical address behind kernel address `va` (a static, the boot
/// heap, a page table). Only valid for addresses in the direct map.
/// The physical address behind a direct-map kernel address `va`.
pub const fn virt_to_phys(va: usize) -> usize {
va - PHYS_OFFSET
}

244
crates/adapter-riscv/src/mmu/page_table.rs → crates/adapter-riscv/src/mmu/page_table/kernel.rs
View File

@@ -1,139 +1,12 @@
use super::allocator::alloc_page_table;
use super::{phys_to_virt, virt_to_phys};
use super::pte::{
Mapping, PAGE_SIZE, PTE_W, PTE_X, PageTable, PageTableEntry, Sv39Flags, map, table_at,
table_ppn,
};
use crate::board;
use crate::mmu::allocator::alloc_page_table;
use crate::mmu::{phys_to_virt, virt_to_phys};
use crate::smp;
use platform::{FrameSource, MapError};
use types::address::{PhysicalAddress, VirtualAddress};
use types::mapping::PagePerms;
pub use types::mapping::Mapping;
pub const PAGE_SIZE: usize = 4096;
const PTE_V: u64 = 1 << 0;
const PTE_R: u64 = 1 << 1;
const PTE_W: u64 = 1 << 2;
const PTE_X: u64 = 1 << 3;
const PTE_U: u64 = 1 << 4;
const PTE_G: u64 = 1 << 5;
const PTE_A: u64 = 1 << 6;
const PTE_D: u64 = 1 << 7;
#[derive(Clone, Copy)]
pub struct Sv39Flags {
bits: u64,
}
impl Sv39Flags {
pub fn kernel_rx() -> Self {
Self {
bits: PTE_R | PTE_X | PTE_G,
}
}
pub fn kernel_ro() -> Self {
Self {
bits: PTE_R | PTE_G,
}
}
pub fn kernel_rw() -> Self {
Self {
bits: PTE_R | PTE_W | PTE_G,
}
}
pub fn user(perms: PagePerms) -> Self {
let bits = match perms {
PagePerms::ReadOnly => PTE_R | PTE_U,
PagePerms::ReadWrite => PTE_R | PTE_W | PTE_U,
PagePerms::ReadExecute => PTE_R | PTE_X | PTE_U,
};
Self { bits }
}
fn to_pte_flags(self) -> u64 {
PTE_V | PTE_A | PTE_D | self.bits
}
}
#[derive(Clone, Copy)]
#[repr(transparent)]
pub struct PageTableEntry(u64);
impl PageTableEntry {
pub fn empty() -> Self {
Self(0)
}
fn new_leaf(ppn: u64, flags: Sv39Flags) -> Self {
Self(flags.to_pte_flags() | (ppn << 10))
}
fn new_branch(ppn: u64) -> Self {
Self(PTE_V | (ppn << 10))
}
pub fn is_valid(&self) -> bool {
self.0 & PTE_V != 0
}
pub fn ppn(&self) -> u64 {
(self.0 >> 10) & 0xFFF_FFFF_FFFF
}
}
#[repr(C, align(4096))]
pub struct PageTable {
pub entries: [PageTableEntry; 512],
}
impl PageTable {
pub fn zero(&mut self) {
for entry in &mut self.entries {
*entry = PageTableEntry::empty();
}
}
}
fn vpn(va: usize, level: usize) -> usize {
(va >> (12 + 9 * level)) & 0x1FF
}
fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) {
let vpn2 = vpn(va, 2);
let vpn1 = vpn(va, 1);
let vpn0 = vpn(va, 0);
if !root.entries[vpn2].is_valid() {
root.entries[vpn2] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
}
let l1 = table_at((root.entries[vpn2].ppn() << 12) as usize);
if !l1.entries[vpn1].is_valid() {
l1.entries[vpn1] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
}
let l0 = table_at((l1.entries[vpn1].ppn() << 12) as usize);
let ppn = (pa as u64) >> 12;
l0.entries[vpn0] = PageTableEntry::new_leaf(ppn, flags);
}
/// The physical page number of a page table the kernel holds a reference to.
fn table_ppn(table: &PageTable) -> u64 {
(virt_to_phys(table as *const PageTable as usize) as u64) >> 12
}
pub fn map(root: &mut PageTable, mapping: Mapping<Sv39Flags>) {
for i in 0..mapping.pages {
let offset = i * PAGE_SIZE;
map_page(
root,
mapping.va.raw() + offset,
mapping.pa.raw() + offset,
mapping.flags,
);
}
}
pub fn init_kernel_page_table() -> &'static mut PageTable {
let root = alloc_page_table();
@@ -325,111 +198,6 @@ pub fn activate(root: &PageTable) {
}
}
// ---- user address spaces ----
//
// User mappings live in [USER_VA_START, USER_VA_END), which is root slot 0,
// the L1 slots below USER_VA_END. Every address space gets its own root and its
// own L1 for slot 0, and the kernel maps nothing in the lower half, so all
// page tables below the user window are private to one process. The
// kernel's upper-half entries are shared by copying them.
pub const USER_VA_START: usize = 0x1000;
pub const USER_VA_END: usize = 0x0200_0000;
/// The page table at physical address `pa`, through the direct map.
pub fn table_at(pa: usize) -> &'static mut PageTable {
// SAFETY: every page table lives in RAM, which the kernel maps at
// `phys_to_virt`.
unsafe { &mut *(phys_to_virt(pa) as *mut PageTable) }
}
fn fresh_table(frames: &mut dyn FrameSource) -> Result<usize, MapError> {
let pa = frames
.alloc_table_frame()
.ok_or(MapError::OutOfFrames)?
.raw();
table_at(pa).zero();
Ok(pa)
}
fn in_user_window(va: usize) -> bool {
va.is_multiple_of(PAGE_SIZE) && (USER_VA_START..USER_VA_END).contains(&va)
}
/// Builds a root table sharing the kernel's mappings, with a private L1 for
/// the user window. Returns the root's physical address.
pub fn create_user_root(
kernel_root: &PageTable,
frames: &mut dyn FrameSource,
) -> Result<usize, MapError> {
let root_pa = fresh_table(frames)?;
let l1_pa = fresh_table(frames)?;
let root = table_at(root_pa);
for (mine, kernels) in root.entries.iter_mut().zip(&kernel_root.entries).skip(256) {
*mine = *kernels;
}
root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12);
Ok(root_pa)
}
fn user_leaf_table(
root_pa: usize,
va: usize,
frames: Option<&mut dyn FrameSource>,
) -> Result<&'static mut PageTable, MapError> {
let root = table_at(root_pa);
let l1 = table_at((root.entries[0].ppn() << 12) as usize);
let slot = &mut l1.entries[vpn(va, 1)];
if !slot.is_valid() {
let frames = frames.ok_or(MapError::NoAddressSpace)?;
let l0_pa = fresh_table(frames)?;
*slot = PageTableEntry::new_branch((l0_pa as u64) >> 12);
}
Ok(table_at((slot.ppn() << 12) as usize))
}
pub fn map_user_page(
root_pa: usize,
va: usize,
pa: usize,
flags: Sv39Flags,
frames: &mut dyn FrameSource,
) -> Result<(), MapError> {
if !in_user_window(va) || !pa.is_multiple_of(PAGE_SIZE) {
return Err(MapError::InvalidAddress);
}
let l0 = user_leaf_table(root_pa, va, Some(frames))?;
l0.entries[vpn(va, 0)] = PageTableEntry::new_leaf((pa as u64) >> 12, flags);
flush_tlb(va);
Ok(())
}
/// The physical page `va` maps to in the address space at `root_pa`.
pub fn translate_user(root_pa: usize, va: usize) -> Option<usize> {
if !in_user_window(va) {
return None;
}
let l0 = user_leaf_table(root_pa, va, None).ok()?;
let entry = l0.entries[vpn(va, 0)];
entry.is_valid().then(|| (entry.ppn() << 12) as usize)
}
pub fn unmap_user_page(root_pa: usize, va: usize) {
if !in_user_window(va) {
return;
}
if let Ok(l0) = user_leaf_table(root_pa, va, None) {
l0.entries[vpn(va, 0)] = PageTableEntry::empty();
flush_tlb(va);
}
}
fn flush_tlb(va: usize) {
// SAFETY: sfence.vma only invalidates translation caches.
unsafe { core::arch::asm!("sfence.vma {0}, zero", in(reg) va) };
}
/// Physical address of the root table the hart is translating with.
pub fn active_root() -> usize {
let satp: usize;

11
crates/adapter-riscv/src/mmu/page_table/mod.rs Normal file
View File

@@ -0,0 +1,11 @@
//! Sv39 page tables: entries and mapping, the kernel table, user address spaces.
mod kernel;
mod pte;
mod user;
pub use kernel::{WxViolation, activate, active_root, check_wx, init_kernel_page_table};
pub use pte::{PAGE_SIZE, PageTable, Sv39Flags, table_at};
pub use user::{
USER_VA_END, USER_VA_START, create_user_root, map_user_page, translate_user, unmap_user_page,
};

139
crates/adapter-riscv/src/mmu/page_table/pte.rs Normal file
View File

@@ -0,0 +1,139 @@
use crate::mmu::allocator::alloc_page_table;
use crate::mmu::{phys_to_virt, virt_to_phys};
use types::mapping::PagePerms;
pub use types::mapping::Mapping;
pub const PAGE_SIZE: usize = 4096;
const PTE_V: u64 = 1 << 0;
const PTE_R: u64 = 1 << 1;
pub(super) const PTE_W: u64 = 1 << 2;
pub(super) const PTE_X: u64 = 1 << 3;
const PTE_U: u64 = 1 << 4;
const PTE_G: u64 = 1 << 5;
const PTE_A: u64 = 1 << 6;
const PTE_D: u64 = 1 << 7;
#[derive(Clone, Copy)]
pub struct Sv39Flags {
bits: u64,
}
impl Sv39Flags {
pub fn kernel_rx() -> Self {
Self {
bits: PTE_R | PTE_X | PTE_G,
}
}
pub fn kernel_ro() -> Self {
Self {
bits: PTE_R | PTE_G,
}
}
pub fn kernel_rw() -> Self {
Self {
bits: PTE_R | PTE_W | PTE_G,
}
}
pub fn user(perms: PagePerms) -> Self {
let bits = match perms {
PagePerms::ReadOnly => PTE_R | PTE_U,
PagePerms::ReadWrite => PTE_R | PTE_W | PTE_U,
PagePerms::ReadExecute => PTE_R | PTE_X | PTE_U,
};
Self { bits }
}
fn to_pte_flags(self) -> u64 {
PTE_V | PTE_A | PTE_D | self.bits
}
}
#[derive(Clone, Copy)]
#[repr(transparent)]
pub struct PageTableEntry(pub(super) u64);
impl PageTableEntry {
pub fn empty() -> Self {
Self(0)
}
pub(super) fn new_leaf(ppn: u64, flags: Sv39Flags) -> Self {
Self(flags.to_pte_flags() | (ppn << 10))
}
pub(super) fn new_branch(ppn: u64) -> Self {
Self(PTE_V | (ppn << 10))
}
pub fn is_valid(&self) -> bool {
self.0 & PTE_V != 0
}
pub fn ppn(&self) -> u64 {
(self.0 >> 10) & 0xFFF_FFFF_FFFF
}
}
#[repr(C, align(4096))]
pub struct PageTable {
pub entries: [PageTableEntry; 512],
}
impl PageTable {
pub fn zero(&mut self) {
for entry in &mut self.entries {
*entry = PageTableEntry::empty();
}
}
}
pub(super) fn vpn(va: usize, level: usize) -> usize {
(va >> (12 + 9 * level)) & 0x1FF
}
fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) {
let vpn2 = vpn(va, 2);
let vpn1 = vpn(va, 1);
let vpn0 = vpn(va, 0);
if !root.entries[vpn2].is_valid() {
root.entries[vpn2] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
}
let l1 = table_at((root.entries[vpn2].ppn() << 12) as usize);
if !l1.entries[vpn1].is_valid() {
l1.entries[vpn1] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
}
let l0 = table_at((l1.entries[vpn1].ppn() << 12) as usize);
let ppn = (pa as u64) >> 12;
l0.entries[vpn0] = PageTableEntry::new_leaf(ppn, flags);
}
/// The physical page number of a page table the kernel holds a reference to.
pub(super) fn table_ppn(table: &PageTable) -> u64 {
(virt_to_phys(table as *const PageTable as usize) as u64) >> 12
}
pub fn map(root: &mut PageTable, mapping: Mapping<Sv39Flags>) {
for i in 0..mapping.pages {
let offset = i * PAGE_SIZE;
map_page(
root,
mapping.va.raw() + offset,
mapping.pa.raw() + offset,
mapping.flags,
);
}
}
/// The page table at physical address `pa`, through the direct map.
pub fn table_at(pa: usize) -> &'static mut PageTable {
// SAFETY: every page table lives in RAM, which the kernel maps at
// `phys_to_virt`.
unsafe { &mut *(phys_to_virt(pa) as *mut PageTable) }
}

93
crates/adapter-riscv/src/mmu/page_table/user.rs Normal file
View File

@@ -0,0 +1,93 @@
use super::pte::{PAGE_SIZE, PageTable, PageTableEntry, Sv39Flags, table_at, vpn};
use platform::{FrameSource, MapError};
/// User mappings live in root slot 0, in a private L1 per address space.
pub const USER_VA_START: usize = 0x1000;
pub const USER_VA_END: usize = 0x0200_0000;
fn fresh_table(frames: &mut dyn FrameSource) -> Result<usize, MapError> {
let pa = frames
.alloc_table_frame()
.ok_or(MapError::OutOfFrames)?
.raw();
table_at(pa).zero();
Ok(pa)
}
fn in_user_window(va: usize) -> bool {
va.is_multiple_of(PAGE_SIZE) && (USER_VA_START..USER_VA_END).contains(&va)
}
/// Builds a root table sharing the kernel's mappings, with a private L1 for
/// the user window. Returns the root's physical address.
pub fn create_user_root(
kernel_root: &PageTable,
frames: &mut dyn FrameSource,
) -> Result<usize, MapError> {
let root_pa = fresh_table(frames)?;
let l1_pa = fresh_table(frames)?;
let root = table_at(root_pa);
for (mine, kernels) in root.entries.iter_mut().zip(&kernel_root.entries).skip(256) {
*mine = *kernels;
}
root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12);
Ok(root_pa)
}
fn user_leaf_table(
root_pa: usize,
va: usize,
frames: Option<&mut dyn FrameSource>,
) -> Result<&'static mut PageTable, MapError> {
let root = table_at(root_pa);
let l1 = table_at((root.entries[0].ppn() << 12) as usize);
let slot = &mut l1.entries[vpn(va, 1)];
if !slot.is_valid() {
let frames = frames.ok_or(MapError::NoAddressSpace)?;
let l0_pa = fresh_table(frames)?;
*slot = PageTableEntry::new_branch((l0_pa as u64) >> 12);
}
Ok(table_at((slot.ppn() << 12) as usize))
}
pub fn map_user_page(
root_pa: usize,
va: usize,
pa: usize,
flags: Sv39Flags,
frames: &mut dyn FrameSource,
) -> Result<(), MapError> {
if !in_user_window(va) || !pa.is_multiple_of(PAGE_SIZE) {
return Err(MapError::InvalidAddress);
}
let l0 = user_leaf_table(root_pa, va, Some(frames))?;
l0.entries[vpn(va, 0)] = PageTableEntry::new_leaf((pa as u64) >> 12, flags);
flush_tlb(va);
Ok(())
}
/// The physical page `va` maps to in the address space at `root_pa`.
pub fn translate_user(root_pa: usize, va: usize) -> Option<usize> {
if !in_user_window(va) {
return None;
}
let l0 = user_leaf_table(root_pa, va, None).ok()?;
let entry = l0.entries[vpn(va, 0)];
entry.is_valid().then(|| (entry.ppn() << 12) as usize)
}
pub fn unmap_user_page(root_pa: usize, va: usize) {
if !in_user_window(va) {
return;
}
if let Ok(l0) = user_leaf_table(root_pa, va, None) {
l0.entries[vpn(va, 0)] = PageTableEntry::empty();
flush_tlb(va);
}
}
fn flush_tlb(va: usize) {
// SAFETY: sfence.vma only invalidates translation caches.
unsafe { core::arch::asm!("sfence.vma {0}, zero", in(reg) va) };
}

5
crates/adapter-riscv/src/platform.rs
View File

@@ -64,9 +64,8 @@ impl Platform for RiscVPlatform {
let Some(root) = self.root(process) else {
return;
};
// No hart may still translate through it once the kernel frees its
// frames: other harts leave it (and wait for nothing but that), and
// the caller may be tearing down the space we are running on.
// No hart may translate through it once its frames are freed; that
// includes this one, which may be running on it.
smp::shoot_down(root);
if mmu::active_root() == root {
// SAFETY: the kernel page table lives in the boot heap forever.

275
crates/adapter-riscv/src/smp.rs
View File

@@ -1,275 +0,0 @@
//! Harts. Each hart has its own kernel stack, and its hart id doubles as
//! its kernel `CoreId`. The kernel is entered under one lock for now (stage
//! 1 of multi-core; the kernel itself stays a sequential program), so
//! userspace runs in parallel and the kernel does not.
//!
//! Harts talk through IPIs carrying two kinds of request:
//! - reschedule: the kernel made work for this hart, or took its thread
//! away; handled under the lock on the way out of the kernel.
//! - leave root R: a page table this hart may be using is changing or about
//! to be freed; the hart switches to the kernel's root and flushes its
//! TLB. This needs no lock, so it is serviced even while a hart spins
//! waiting for the lock or idles, and the sender (which holds the lock)
//! can wait for the acknowledgement without deadlocking.
use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
use crate::mmu;
/// Must match linker.ld.
pub const MAX_HARTS: usize = 8;
/// A guard page and a 512 KiB stack (boot builds big tables on it). Must
/// match linker.ld.
pub const STACK_SLOT: usize = 4096 + 0x8_0000;
const SIP_SSIP: usize = 1 << 1;
/// The hart running this code, from which kernel stack it is on.
pub fn hart_id() -> usize {
let sp: usize;
// SAFETY: reading sp has no side effects.
unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) };
(sp - stacks_start()) / STACK_SLOT
}
pub fn stacks_start() -> usize {
unsafe extern "C" {
static _stacks_start: u8;
}
// SAFETY: only the symbol's address is taken.
unsafe { &_stacks_start as *const u8 as usize }
}
static BOOT_HART: AtomicUsize = AtomicUsize::new(0);
static NUM_HARTS: AtomicUsize = AtomicUsize::new(1);
/// Physical address of the kernel's own root table.
static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0);
/// The root table each hart translates with.
static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS];
static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
pub fn boot_hart() -> usize {
BOOT_HART.load(Ordering::Relaxed)
}
pub fn num_harts() -> usize {
NUM_HARTS.load(Ordering::Relaxed)
}
pub fn kernel_root() -> usize {
KERNEL_ROOT.load(Ordering::Relaxed)
}
/// Records which hart booted and how many there are, and where the
/// kernel's root table is.
pub fn init(boot: usize, harts: usize, kernel_root: usize) {
BOOT_HART.store(boot, Ordering::Relaxed);
NUM_HARTS.store(harts, Ordering::Relaxed);
KERNEL_ROOT.store(kernel_root, Ordering::Relaxed);
}
/// Called by `mmu::activate`.
pub fn note_active_root(root: usize) {
if let Some(r) = ACTIVE_ROOT.get(hart_id()) {
r.store(root, Ordering::SeqCst);
}
}
// ---- the kernel lock ----
static NEXT_TICKET: AtomicU32 = AtomicU32::new(0);
static SERVING: AtomicU32 = AtomicU32::new(0);
/// Takes the kernel lock, first come first served. Leave-root requests
/// are serviced while waiting.
pub fn lock() {
let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
while SERVING.load(Ordering::Acquire) != ticket {
service_ipi();
core::hint::spin_loop();
}
}
pub fn unlock() {
SERVING.fetch_add(1, Ordering::Release);
}
// ---- what lock-free fast paths may know ----
//
// Published under the kernel lock just before it is released, read without
// it. A fast path that reads them while another hart holds the lock simply
// takes effect before that hart's operation.
/// Per core: highest waiting priority + 1, 0 if its queue is empty.
static WAITING: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
/// Per hart: the running thread's process id + 1 and priority + 1.
static RUNNING_PID: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RUNNING_PRIORITY: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
pub fn publish_waiting(core: usize, highest: Option<u8>) {
if let Some(w) = WAITING.get(core) {
w.store(highest.map_or(0, |p| p as usize + 1), Ordering::Release);
}
}
pub fn publish_running(pid: Option<u16>, priority: Option<u8>) {
let me = hart_id();
if let (Some(p), Some(q)) = (RUNNING_PID.get(me), RUNNING_PRIORITY.get(me)) {
p.store(pid.map_or(0, |x| x as usize + 1), Ordering::Release);
q.store(priority.map_or(0, |x| x as usize + 1), Ordering::Release);
}
}
/// The process the thread running on this hart belongs to.
pub fn running_pid() -> Option<usize> {
let v = RUNNING_PID.get(hart_id())?.load(Ordering::Acquire);
v.checked_sub(1)
}
/// Whether a Yield by the running thread would change nothing: no thread
/// waits on this core at its priority or above.
pub fn yield_is_noop() -> bool {
let me = hart_id();
let (Some(w), Some(p)) = (WAITING.get(me), RUNNING_PRIORITY.get(me)) else {
return false;
};
let mine = p.load(Ordering::Acquire);
mine != 0 && w.load(Ordering::Acquire) < mine
}
/// Whether this hart has an IPI or a reschedule request it has not acted
/// on; fast paths must then take the slow path.
pub fn has_pending_request() -> bool {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
sip & SIP_SSIP != 0
|| RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.load(Ordering::SeqCst))
}
// ---- IPIs ----
/// The lock-free half of an IPI: acknowledges it and, if asked, leaves the
/// root being changed. Safe to call at any time, any number of times.
pub fn service_ipi() {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & SIP_SSIP == 0 {
return;
}
// Clear first: a request posted after this raises SSIP again.
// SAFETY: clearing our own pending software interrupt.
unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) };
let me = hart_id();
let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else {
return;
};
let root = leave.swap(0, Ordering::SeqCst);
if root != 0 {
if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) {
mmu::activate(mmu::page_table::table_at(kernel_root()));
}
// SAFETY: flushing translations has no other effect.
unsafe { core::arch::asm!("sfence.vma zero, zero") };
left.fetch_add(1, Ordering::SeqCst);
}
}
/// Asks `hart` to look at its run queue again, on its way out of the
/// kernel.
pub fn kick(hart: usize) {
if let Some(r) = RESCHEDULE.get(hart) {
r.store(true, Ordering::SeqCst);
sbi_send_ipi(hart);
}
}
/// Consumes this hart's reschedule request.
pub fn take_reschedule() -> bool {
RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.swap(false, Ordering::SeqCst))
}
/// Makes every other hart that translates through `root` leave it and
/// flush, and waits until each has. Called with the kernel lock held, so
/// no hart can start using `root` meanwhile.
pub fn shoot_down(root: usize) {
let me = hart_id();
for hart in 0..num_harts() {
if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root {
continue;
}
let seen = LEFT[hart].load(Ordering::SeqCst);
LEAVE_ROOT[hart].store(root, Ordering::SeqCst);
sbi_send_ipi(hart);
while LEFT[hart].load(Ordering::SeqCst) == seen {
core::hint::spin_loop();
}
}
}
// ---- SBI ----
const SBI_EXT_IPI: usize = 0x735049;
const SBI_EXT_HSM: usize = 0x48534D;
fn sbi_send_ipi(hart: usize) {
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") 1usize => _,
inlateout("a1") hart => _,
in("a6") 0usize,
in("a7") SBI_EXT_IPI,
);
}
}
/// Whether `hart` exists, per SBI HSM `hart_get_status`.
fn sbi_hart_exists(hart: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
lateout("a1") _,
in("a6") 2usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}
/// The number of harts, counting up from hart 0 until one is missing.
pub fn probe_harts() -> usize {
(0..MAX_HARTS)
.take_while(|h| sbi_hart_exists(*h))
.count()
.max(1)
}
/// Starts `hart` at the physical address `entry` with the MMU off.
pub fn sbi_hart_start(hart: usize, entry: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
inlateout("a1") entry => _,
in("a2") 0usize,
in("a6") 0usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}

42
crates/adapter-riscv/src/smp/hints.rs Normal file
View File

@@ -0,0 +1,42 @@
//! State published under the kernel lock for lock-free fast paths to read.
use core::sync::atomic::{AtomicUsize, Ordering};
use super::{MAX_HARTS, hart_id};
/// Per core: highest waiting priority + 1, 0 if its queue is empty.
static WAITING: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
/// Per hart: the running thread's process id + 1 and priority + 1.
static RUNNING_PID: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RUNNING_PRIORITY: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
pub fn publish_waiting(core: usize, highest: Option<u8>) {
if let Some(w) = WAITING.get(core) {
w.store(highest.map_or(0, |p| p as usize + 1), Ordering::Release);
}
}
pub fn publish_running(pid: Option<u16>, priority: Option<u8>) {
let me = hart_id();
if let (Some(p), Some(q)) = (RUNNING_PID.get(me), RUNNING_PRIORITY.get(me)) {
p.store(pid.map_or(0, |x| x as usize + 1), Ordering::Release);
q.store(priority.map_or(0, |x| x as usize + 1), Ordering::Release);
}
}
/// The process the thread running on this hart belongs to.
pub fn running_pid() -> Option<usize> {
let v = RUNNING_PID.get(hart_id())?.load(Ordering::Acquire);
v.checked_sub(1)
}
/// Whether a Yield by the running thread would change nothing: no thread
/// waits on this core at its priority or above.
pub fn yield_is_noop() -> bool {
let me = hart_id();
let (Some(w), Some(p)) = (WAITING.get(me), RUNNING_PRIORITY.get(me)) else {
return false;
};
let mine = p.load(Ordering::Acquire);
mine != 0 && w.load(Ordering::Acquire) < mine
}

96
crates/adapter-riscv/src/smp/ipi.rs Normal file
View File

@@ -0,0 +1,96 @@
//! Reschedule and leave-root IPIs, serviced without the kernel lock.
use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use super::sbi::sbi_send_ipi;
use super::{MAX_HARTS, hart_id, kernel_root, num_harts};
use crate::mmu;
const SIP_SSIP: usize = 1 << 1;
/// The root table each hart translates with.
static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS];
static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
/// Called by `mmu::activate`.
pub fn note_active_root(root: usize) {
if let Some(r) = ACTIVE_ROOT.get(hart_id()) {
r.store(root, Ordering::SeqCst);
}
}
/// Whether this hart has an IPI or a reschedule request it has not acted
/// on; fast paths must then take the slow path.
pub fn has_pending_request() -> bool {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
sip & SIP_SSIP != 0
|| RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.load(Ordering::SeqCst))
}
// ---- IPIs ----
/// The lock-free half of an IPI: acknowledges it and leaves the root being
/// changed, if asked. Safe to call at any time.
pub fn service_ipi() {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & SIP_SSIP == 0 {
return;
}
// Clear first: a request posted after this raises SSIP again.
// SAFETY: clearing our own pending software interrupt.
unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) };
let me = hart_id();
let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else {
return;
};
let root = leave.swap(0, Ordering::SeqCst);
if root != 0 {
if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) {
mmu::activate(mmu::page_table::table_at(kernel_root()));
}
// SAFETY: flushing translations has no other effect.
unsafe { core::arch::asm!("sfence.vma zero, zero") };
left.fetch_add(1, Ordering::SeqCst);
}
}
/// Asks `hart` to look at its run queue again, on its way out of the
/// kernel.
pub fn kick(hart: usize) {
if let Some(r) = RESCHEDULE.get(hart) {
r.store(true, Ordering::SeqCst);
sbi_send_ipi(hart);
}
}
/// Consumes this hart's reschedule request.
pub fn take_reschedule() -> bool {
RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.swap(false, Ordering::SeqCst))
}
/// Makes every other hart leave `root` and flush, waiting for each.
/// Kernel lock held, so no hart can start using `root` meanwhile.
pub fn shoot_down(root: usize) {
let me = hart_id();
for hart in 0..num_harts() {
if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root {
continue;
}
let seen = LEFT[hart].load(Ordering::SeqCst);
LEAVE_ROOT[hart].store(root, Ordering::SeqCst);
sbi_send_ipi(hart);
while LEFT[hart].load(Ordering::SeqCst) == seen {
core::hint::spin_loop();
}
}
}

22
crates/adapter-riscv/src/smp/lock.rs Normal file
View File

@@ -0,0 +1,22 @@
//! The kernel lock: a ticket lock that services IPIs while waiting.
use core::sync::atomic::{AtomicU32, Ordering};
use super::service_ipi;
static NEXT_TICKET: AtomicU32 = AtomicU32::new(0);
static SERVING: AtomicU32 = AtomicU32::new(0);
/// Takes the kernel lock, first come first served. Leave-root requests
/// are serviced while waiting.
pub fn lock() {
let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
while SERVING.load(Ordering::Acquire) != ticket {
service_ipi();
core::hint::spin_loop();
}
}
pub fn unlock() {
SERVING.fetch_add(1, Ordering::Release);
}

61
crates/adapter-riscv/src/smp/mod.rs Normal file
View File

@@ -0,0 +1,61 @@
//! Harts: per-hart kernel stacks (the hart id doubles as the kernel `CoreId`),
//! the kernel lock, and IPIs. The kernel runs under one lock; userspace runs in parallel.
mod hints;
mod ipi;
mod lock;
mod sbi;
pub use hints::{publish_running, publish_waiting, running_pid, yield_is_noop};
pub use ipi::{
has_pending_request, kick, note_active_root, service_ipi, shoot_down, take_reschedule,
};
pub use lock::{lock, unlock};
pub use sbi::{probe_harts, sbi_hart_start};
use core::sync::atomic::{AtomicUsize, Ordering};
/// Must match linker.ld.
pub const MAX_HARTS: usize = 8;
/// A guard page and a 512 KiB stack. Must match linker.ld.
pub const STACK_SLOT: usize = 4096 + 0x8_0000;
/// The hart running this code, from which kernel stack it is on.
pub fn hart_id() -> usize {
let sp: usize;
// SAFETY: reading sp has no side effects.
unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) };
(sp - stacks_start()) / STACK_SLOT
}
pub fn stacks_start() -> usize {
unsafe extern "C" {
static _stacks_start: u8;
}
// SAFETY: only the symbol's address is taken.
unsafe { &_stacks_start as *const u8 as usize }
}
static BOOT_HART: AtomicUsize = AtomicUsize::new(0);
static NUM_HARTS: AtomicUsize = AtomicUsize::new(1);
/// Physical address of the kernel's own root table.
static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0);
pub fn boot_hart() -> usize {
BOOT_HART.load(Ordering::Relaxed)
}
pub fn num_harts() -> usize {
NUM_HARTS.load(Ordering::Relaxed)
}
pub fn kernel_root() -> usize {
KERNEL_ROOT.load(Ordering::Relaxed)
}
/// Records the boot hart, the hart count and the kernel root table.
pub fn init(boot: usize, harts: usize, kernel_root: usize) {
BOOT_HART.store(boot, Ordering::Relaxed);
NUM_HARTS.store(harts, Ordering::Relaxed);
KERNEL_ROOT.store(kernel_root, Ordering::Relaxed);
}

60
crates/adapter-riscv/src/smp/sbi.rs Normal file
View File

@@ -0,0 +1,60 @@
//! SBI calls: IPIs and hart state management.
use super::MAX_HARTS;
const SBI_EXT_IPI: usize = 0x735049;
const SBI_EXT_HSM: usize = 0x48534D;
pub(super) fn sbi_send_ipi(hart: usize) {
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") 1usize => _,
inlateout("a1") hart => _,
in("a6") 0usize,
in("a7") SBI_EXT_IPI,
);
}
}
/// Whether `hart` exists, per SBI HSM `hart_get_status`.
fn sbi_hart_exists(hart: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
lateout("a1") _,
in("a6") 2usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}
/// The number of harts, counting up from hart 0 until one is missing.
pub fn probe_harts() -> usize {
(0..MAX_HARTS)
.take_while(|h| sbi_hart_exists(*h))
.count()
.max(1)
}
/// Starts `hart` at the physical address `entry` with the MMU off.
pub fn sbi_hart_start(hart: usize, entry: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
inlateout("a1") entry => _,
in("a2") 0usize,
in("a6") 0usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}

26
crates/adapter-riscv/src/trap/fast.rs Normal file
View File

@@ -0,0 +1,26 @@
use super::{TrapFrame, write_result};
/// Syscalls answered without the kernel lock because they touch nothing it
/// guards. A pending IPI or reschedule request forces the slow path.
pub(super) fn fast_syscall(frame: &mut TrapFrame) -> bool {
use types::syscall_abi::{SYS_DEBUG_PUTCHAR, SYS_GET_PROCESS_ID, SYS_GET_TIME, SYS_YIELD};
if crate::smp::has_pending_request() {
return false;
}
let result = match frame.regs[17] {
SYS_GET_TIME => kernel::SyscallResult::Value(crate::hw::timer::read_time() as usize),
SYS_GET_PROCESS_ID => match crate::smp::running_pid() {
Some(pid) => kernel::SyscallResult::Value(pid),
None => return false,
},
SYS_DEBUG_PUTCHAR => {
crate::hw::uart::Uart::new().put_char(frame.regs[10] as u8);
kernel::SyscallResult::Ok
}
SYS_YIELD if crate::smp::yield_is_noop() => kernel::SyscallResult::Ok,
_ => return false,
};
frame.sepc += 4;
write_result(frame, &result);
true
}

315
crates/adapter-riscv/src/trap/mod.rs
View File

@@ -1,4 +1,10 @@
use core::arch::global_asm;
mod fast;
mod sched;
mod vector;
use fast::fast_syscall;
pub use sched::enter_first;
use sched::{finish, release, switch_to_next, switch_to_thread};
#[repr(C)]
pub struct TrapFrame {
@@ -7,126 +13,6 @@ pub struct TrapFrame {
pub sstatus: usize,
}
global_asm!(
r#"
.section .text
.align 4
.globl trap_vector
trap_vector:
csrrw sp, sscratch, sp
bnez sp, _from_user
csrrw sp, sscratch, sp
_from_user:
addi sp, sp, -272
sd x1, 8(sp)
sd x3, 24(sp)
sd x4, 32(sp)
sd x5, 40(sp)
sd x6, 48(sp)
sd x7, 56(sp)
sd x8, 64(sp)
sd x9, 72(sp)
sd x10, 80(sp)
sd x11, 88(sp)
sd x12, 96(sp)
sd x13, 104(sp)
sd x14, 112(sp)
sd x15, 120(sp)
sd x16, 128(sp)
sd x17, 136(sp)
sd x18, 144(sp)
sd x19, 152(sp)
sd x20, 160(sp)
sd x21, 168(sp)
sd x22, 176(sp)
sd x23, 184(sp)
sd x24, 192(sp)
sd x25, 200(sp)
sd x26, 208(sp)
sd x27, 216(sp)
sd x28, 224(sp)
sd x29, 232(sp)
sd x30, 240(sp)
sd x31, 248(sp)
csrr t0, sscratch
sd t0, 16(sp)
csrr t0, sepc
sd t0, 256(sp)
csrr t0, sstatus
sd t0, 264(sp)
mv a0, sp
call trap_handler_rust
j _restore_frame
# trap_return(frame): resume whatever `frame` describes, as if returning
# from a trap that saved it. Used to enter userspace for the first time.
.globl trap_return
trap_return:
mv sp, a0
_restore_frame:
ld t0, 256(sp)
csrw sepc, t0
ld t0, 264(sp)
csrw sstatus, t0
andi t1, t0, 0x100
bnez t1, _restore_smode
ld t0, 16(sp)
csrw sscratch, t0
_restore_smode:
ld x1, 8(sp)
ld x3, 24(sp)
ld x4, 32(sp)
ld x6, 48(sp)
ld x7, 56(sp)
ld x8, 64(sp)
ld x9, 72(sp)
ld x10, 80(sp)
ld x11, 88(sp)
ld x12, 96(sp)
ld x13, 104(sp)
ld x14, 112(sp)
ld x15, 120(sp)
ld x16, 128(sp)
ld x17, 136(sp)
ld x18, 144(sp)
ld x19, 152(sp)
ld x20, 160(sp)
ld x21, 168(sp)
ld x22, 176(sp)
ld x23, 184(sp)
ld x24, 192(sp)
ld x25, 200(sp)
ld x26, 208(sp)
ld x27, 216(sp)
ld x28, 224(sp)
ld x29, 232(sp)
ld x30, 240(sp)
ld x31, 248(sp)
# Every GPR except t0 (x5) is restored. Decide where we return to while
# t0 is still free, and restore it last on each path; using t0 after
# its restore would corrupt the interrupted program's t0.
csrr t0, sstatus
andi t0, t0, 0x100
bnez t0, _sret_smode
ld x5, 40(sp)
addi sp, sp, 272
csrrw sp, sscratch, sp
sret
_sret_smode:
ld x5, 40(sp)
addi sp, sp, 272
sret
"#
);
const CAUSE_INTERRUPT_BIT: usize = 1 << 63;
const CAUSE_S_SOFTWARE: usize = 1;
const CAUSE_S_TIMER: usize = 5;
@@ -143,7 +29,7 @@ pub fn init() {
let addr = trap_vector as *const () as usize;
core::arch::asm!("csrw stvec, {0}", in(reg) addr);
core::arch::asm!("csrw sscratch, zero");
// SIE stays clear — interrupts are enabled only when entering userspace via sret
// SIE stays clear; interrupts are enabled only by sret into userspace.
let sie_bits = (1usize << 5) | (1usize << 9) | (1usize << 1);
core::arch::asm!("csrs sie, {0}", in(reg) sie_bits);
}
@@ -169,31 +55,6 @@ fn write_result(frame: &mut TrapFrame, result: &kernel::SyscallResult) {
set_results(frame, kernel::abi::encode(result));
}
fn save_frame_to_context(frame: &TrapFrame, ctx: &mut domain::thread::SavedContext) {
ctx.registers_mut().copy_from_slice(&frame.regs);
ctx.set_pc(frame.sepc);
ctx.set_status(frame.sstatus);
}
fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFrame) {
frame.regs.copy_from_slice(ctx.registers());
frame.sepc = ctx.program_counter();
frame.sstatus = ctx.status();
// ensure returning to U-mode
frame.sstatus &= !(1 << 8); // clear SPP
frame.sstatus |= 1 << 5; // set SPIE
}
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
if let Some(result) = kern().take_pending_result(tid) {
write_result(frame, &result);
}
}
unsafe extern "C" {
fn trap_return(frame: *mut TrapFrame) -> !;
}
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
// SAFETY: only called with the kernel lock held.
unsafe { &mut *crate::kernel_ptr() }
@@ -203,63 +64,7 @@ fn this_core() -> types::id::CoreId {
types::id::CoreId::from_raw(crate::smp::hart_id() as u8)
}
/// Saves `frame` into `from` (if it still exists) and loads `to`.
fn switch_to_thread(
frame: &mut TrapFrame,
from: Option<types::id::ThreadId>,
to: types::id::ThreadId,
) {
let kern = kern();
if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
if let Some(ctx) = kern.thread_context(to) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, to);
}
/// Saves `from` and runs the next ready thread, idling until there is one.
fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next = next_runnable();
let _ = kern().set_thread_running(next, this_core());
switch_to_thread(frame, None, next);
}
/// The next thread to run on this hart, idling until one is ready.
///
/// Idling gives up the kernel lock: an idle hart holds nothing, so other
/// harts can make work for it (and kick it) meanwhile. It also leaves any
/// user address space first, so nobody needs to shoot it down. SIE stays
/// clear; `wfi` still wakes on a pending interrupt, serviced by polling.
fn next_runnable() -> types::id::ThreadId {
loop {
if let Some(tid) = kern().pick_next_thread() {
return tid;
}
crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
release();
unsafe { core::arch::asm!("wfi") };
crate::smp::service_ipi();
crate::smp::lock();
let _ = crate::smp::take_reschedule();
let sip: usize;
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & (1 << CAUSE_S_TIMER) != 0 {
// No thread is running, so there is no time slice to charge.
crate::hw::timer::handle_interrupt();
}
if sip & (1 << CAUSE_S_EXTERNAL) != 0 {
dispatch_external();
}
}
}
/// Claims every pending PLIC interrupt and hands each to the kernel. Only
/// the boot hart receives external interrupts.
/// Claims every pending PLIC interrupt (boot hart only) and hands it to the kernel.
fn dispatch_external() {
loop {
let irq_num = crate::hw::plic::Plic::claim();
@@ -271,100 +76,6 @@ fn dispatch_external() {
}
}
/// Releases the kernel lock, first publishing what the lock-free fast
/// paths may rely on.
fn release() {
let kern = kern();
for core in 0..crate::smp::num_harts() {
let id = types::id::CoreId::from_raw(core as u8);
crate::smp::publish_waiting(core, kern.highest_waiting(id).map(|p| p.value()));
}
let running = kern.current_thread();
crate::smp::publish_running(
running
.and_then(|t| kern.thread_process(t))
.map(|p| p.raw()),
running
.and_then(|t| kern.thread_priority(t))
.map(|p| p.value()),
);
crate::smp::unlock();
}
/// Syscalls answered without the kernel lock, as the kernel would answer
/// them: they read nothing the lock guards (the time, the UART, the
/// running thread's fixed process id) or, for a Yield that has nobody to
/// yield to, change nothing. Any pending IPI or reschedule request sends
/// the call down the slow path, so no request is ever skipped.
fn fast_syscall(frame: &mut TrapFrame) -> bool {
use types::syscall_abi::{SYS_DEBUG_PUTCHAR, SYS_GET_PROCESS_ID, SYS_GET_TIME, SYS_YIELD};
if crate::smp::has_pending_request() {
return false;
}
let result = match frame.regs[17] {
SYS_GET_TIME => kernel::SyscallResult::Value(crate::hw::timer::read_time() as usize),
SYS_GET_PROCESS_ID => match crate::smp::running_pid() {
Some(pid) => kernel::SyscallResult::Value(pid),
None => return false,
},
SYS_DEBUG_PUTCHAR => {
crate::hw::uart::Uart::new().put_char(frame.regs[10] as u8);
kernel::SyscallResult::Ok
}
SYS_YIELD if crate::smp::yield_is_noop() => kernel::SyscallResult::Ok,
_ => return false,
};
frame.sepc += 4;
write_result(frame, &result);
true
}
/// Last step before leaving the kernel to userspace: act on a reschedule
/// request, make sure this hart has a thread (its own may have been taken
/// away by another hart), and translate through that thread's address
/// space.
fn finish(frame: &mut TrapFrame) {
let kern = kern();
let current = kern.current_thread();
if crate::smp::take_reschedule()
&& let Some(cur) = current
&& let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
{
switch_to_thread(frame, Some(cur), next);
}
if kern.current_thread().is_none() {
switch_to_next(frame, None);
}
let root = kern
.current_thread()
.and_then(|t| kern.thread_process(t))
.map(|pid| kern.platform_mut().page_table_for(pid))
.unwrap_or(0);
if root != 0 && crate::mmu::active_root() != root {
crate::mmu::activate(crate::mmu::page_table::table_at(root));
}
}
/// Leaves the kernel for userspace for the first time on this hart, with
/// the kernel lock held: runs the thread the kernel already made current
/// here (init, on the boot hart) or the next one homed here.
pub fn enter_first() -> ! {
let mut frame = TrapFrame {
regs: [0; 32],
sepc: 0,
sstatus: 0,
};
match kern().current_thread() {
Some(tid) => switch_to_thread(&mut frame, None, tid),
None => switch_to_next(&mut frame, None),
}
finish(&mut frame);
release();
// SAFETY: `frame` holds a user context; sscratch is set from it on
// the way out, so later traps land at the top of this frame.
unsafe { trap_return(&mut frame) }
}
#[unsafe(no_mangle)]
extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
let scause: usize;
@@ -401,15 +112,12 @@ fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
}
}
CAUSE_S_EXTERNAL => dispatch_external(),
// Serviced on entry; any reschedule happens in `finish`.
CAUSE_S_SOFTWARE => {}
_ => {}
}
}
/// The kernel itself faulted (a W^X violation, a stack guard, a bad
/// pointer). Its state can no longer be trusted, and no user thread is to
/// blame.
/// The kernel itself faulted; its state is untrusted, so halt.
fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! {
let uart = crate::hw::uart::Uart::new();
uart.write_str("KERNEL FAULT: scause=");
@@ -478,8 +186,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
kern.handle_fault(current_tid, fault, frame.sepc);
switch_to_next(frame, Some(current_tid));
}
// With no current thread (another hart just destroyed it),
// `finish` moves on to the next one.
// No current thread (destroyed by another hart): `finish` picks one.
}
}
}

139
crates/adapter-riscv/src/trap/sched.rs Normal file
View File

@@ -0,0 +1,139 @@
use super::{
CAUSE_S_EXTERNAL, CAUSE_S_TIMER, TrapFrame, dispatch_external, kern, this_core, write_result,
};
fn save_frame_to_context(frame: &TrapFrame, ctx: &mut domain::thread::SavedContext) {
ctx.registers_mut().copy_from_slice(&frame.regs);
ctx.set_pc(frame.sepc);
ctx.set_status(frame.sstatus);
}
fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFrame) {
frame.regs.copy_from_slice(ctx.registers());
frame.sepc = ctx.program_counter();
frame.sstatus = ctx.status();
frame.sstatus &= !(1 << 8); // SPP: return to U-mode
frame.sstatus |= 1 << 5; // SPIE
}
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
if let Some(result) = kern().take_pending_result(tid) {
write_result(frame, &result);
}
}
unsafe extern "C" {
fn trap_return(frame: *mut TrapFrame) -> !;
}
/// Saves `frame` into `from` (if it still exists) and loads `to`.
pub(super) fn switch_to_thread(
frame: &mut TrapFrame,
from: Option<types::id::ThreadId>,
to: types::id::ThreadId,
) {
let kern = kern();
if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
if let Some(ctx) = kern.thread_context(to) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, to);
}
/// Saves `from` and runs the next ready thread, idling until there is one.
pub(super) fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next = next_runnable();
let _ = kern().set_thread_running(next, this_core());
switch_to_thread(frame, None, next);
}
/// The next thread to run on this hart, idling (lock released, kernel
/// address space active so no shootdown is needed) until one is ready.
fn next_runnable() -> types::id::ThreadId {
loop {
if let Some(tid) = kern().pick_next_thread() {
return tid;
}
crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
release();
unsafe { core::arch::asm!("wfi") };
crate::smp::service_ipi();
crate::smp::lock();
let _ = crate::smp::take_reschedule();
let sip: usize;
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & (1 << CAUSE_S_TIMER) != 0 {
// No thread is running, so there is no time slice to charge.
crate::hw::timer::handle_interrupt();
}
if sip & (1 << CAUSE_S_EXTERNAL) != 0 {
dispatch_external();
}
}
}
/// Releases the kernel lock after publishing state the lock-free fast paths read.
pub(super) fn release() {
let kern = kern();
for core in 0..crate::smp::num_harts() {
let id = types::id::CoreId::from_raw(core as u8);
crate::smp::publish_waiting(core, kern.highest_waiting(id).map(|p| p.value()));
}
let running = kern.current_thread();
crate::smp::publish_running(
running
.and_then(|t| kern.thread_process(t))
.map(|p| p.raw()),
running
.and_then(|t| kern.thread_priority(t))
.map(|p| p.value()),
);
crate::smp::unlock();
}
/// Last step before returning to userspace: apply a reschedule request,
/// ensure this hart has a thread, and activate its address space.
pub(super) fn finish(frame: &mut TrapFrame) {
let kern = kern();
let current = kern.current_thread();
if crate::smp::take_reschedule()
&& let Some(cur) = current
&& let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
{
switch_to_thread(frame, Some(cur), next);
}
if kern.current_thread().is_none() {
switch_to_next(frame, None);
}
let root = kern
.current_thread()
.and_then(|t| kern.thread_process(t))
.map(|pid| kern.platform_mut().page_table_for(pid))
.unwrap_or(0);
if root != 0 && crate::mmu::active_root() != root {
crate::mmu::activate(crate::mmu::page_table::table_at(root));
}
}
/// Enters userspace for the first time on this hart (kernel lock held).
pub fn enter_first() -> ! {
let mut frame = TrapFrame {
regs: [0; 32],
sepc: 0,
sstatus: 0,
};
match kern().current_thread() {
Some(tid) => switch_to_thread(&mut frame, None, tid),
None => switch_to_next(&mut frame, None),
}
finish(&mut frame);
release();
// SAFETY: `frame` holds a user context; sscratch is set from it on
// the way out, so later traps land at the top of this frame.
unsafe { trap_return(&mut frame) }
}

118
crates/adapter-riscv/src/trap/vector.rs Normal file
View File

@@ -0,0 +1,118 @@
use core::arch::global_asm;
global_asm!(
r#"
.section .text
.align 4
.globl trap_vector
trap_vector:
csrrw sp, sscratch, sp
bnez sp, _from_user
csrrw sp, sscratch, sp
_from_user:
addi sp, sp, -272
sd x1, 8(sp)
sd x3, 24(sp)
sd x4, 32(sp)
sd x5, 40(sp)
sd x6, 48(sp)
sd x7, 56(sp)
sd x8, 64(sp)
sd x9, 72(sp)
sd x10, 80(sp)
sd x11, 88(sp)
sd x12, 96(sp)
sd x13, 104(sp)
sd x14, 112(sp)
sd x15, 120(sp)
sd x16, 128(sp)
sd x17, 136(sp)
sd x18, 144(sp)
sd x19, 152(sp)
sd x20, 160(sp)
sd x21, 168(sp)
sd x22, 176(sp)
sd x23, 184(sp)
sd x24, 192(sp)
sd x25, 200(sp)
sd x26, 208(sp)
sd x27, 216(sp)
sd x28, 224(sp)
sd x29, 232(sp)
sd x30, 240(sp)
sd x31, 248(sp)
csrr t0, sscratch
sd t0, 16(sp)
csrr t0, sepc
sd t0, 256(sp)
csrr t0, sstatus
sd t0, 264(sp)
mv a0, sp
call trap_handler_rust
j _restore_frame
# trap_return(frame): resume `frame` as if returning from the trap that saved it.
.globl trap_return
trap_return:
mv sp, a0
_restore_frame:
ld t0, 256(sp)
csrw sepc, t0
ld t0, 264(sp)
csrw sstatus, t0
andi t1, t0, 0x100
bnez t1, _restore_smode
ld t0, 16(sp)
csrw sscratch, t0
_restore_smode:
ld x1, 8(sp)
ld x3, 24(sp)
ld x4, 32(sp)
ld x6, 48(sp)
ld x7, 56(sp)
ld x8, 64(sp)
ld x9, 72(sp)
ld x10, 80(sp)
ld x11, 88(sp)
ld x12, 96(sp)
ld x13, 104(sp)
ld x14, 112(sp)
ld x15, 120(sp)
ld x16, 128(sp)
ld x17, 136(sp)
ld x18, 144(sp)
ld x19, 152(sp)
ld x20, 160(sp)
ld x21, 168(sp)
ld x22, 176(sp)
ld x23, 184(sp)
ld x24, 192(sp)
ld x25, 200(sp)
ld x26, 208(sp)
ld x27, 216(sp)
ld x28, 224(sp)
ld x29, 232(sp)
ld x30, 240(sp)
ld x31, 248(sp)
# t0 (x5) is restored last on each path; it is the only scratch register.
csrr t0, sstatus
andi t0, t0, 0x100
bnez t0, _sret_smode
ld x5, 40(sp)
addi sp, sp, 272
csrrw sp, sscratch, sp
sret
_sret_smode:
ld x5, 40(sp)
addi sp, sp, 272
sret
"#
);

2
crates/hw-virtio/src/gpu.rs
View File

@@ -75,7 +75,7 @@ pub fn resource_flush(mmio: &impl MmioAccess, queue: &VirtioQueue, dma: &GpuDmaA
fn write_cmd_header(cmd_va: VirtualAddress, cmd_type: u32) {
unsafe {
let p = cmd_va.raw() as *mut u8;
// Use volatile writes to zero the buffer — ensures device sees the zeros
// Volatile so the device sees the zeros.
for i in 0..128 {
core::ptr::write_volatile((p as *mut u32).add(i), 0);
}

3
crates/hw-virtio/src/mmio.rs
View File

@@ -89,8 +89,7 @@ pub const MMIO_FIRST: usize = 0x10001000;
pub const MMIO_STRIDE: usize = 0x1000;
pub const MMIO_SLOTS: usize = 8;
/// Probes the MMIO slots starting at `first`, the address the caller sees
/// slot 0 at (`MMIO_FIRST` itself, or wherever it is mapped).
/// Probes the MMIO slots, with slot 0 at `first` as the caller maps it.
pub fn scan_for_device(first: usize, device_id: u32) -> Option<DirectMmio> {
for slot in 0..MMIO_SLOTS {
if let Some(mmio) = probe_slot(first + slot * MMIO_STRIDE, device_id) {