refactor(adapter): split trap/page_table/smp/entry by concern; trim comments
This commit is contained in:
@@ -1,4 +1,10 @@
|
||||
use core::arch::global_asm;
|
||||
mod fast;
|
||||
mod sched;
|
||||
mod vector;
|
||||
|
||||
use fast::fast_syscall;
|
||||
pub use sched::enter_first;
|
||||
use sched::{finish, release, switch_to_next, switch_to_thread};
|
||||
|
||||
#[repr(C)]
|
||||
pub struct TrapFrame {
|
||||
@@ -7,126 +13,6 @@ pub struct TrapFrame {
|
||||
pub sstatus: usize,
|
||||
}
|
||||
|
||||
global_asm!(
|
||||
r#"
|
||||
.section .text
|
||||
.align 4
|
||||
.globl trap_vector
|
||||
trap_vector:
|
||||
csrrw sp, sscratch, sp
|
||||
bnez sp, _from_user
|
||||
csrrw sp, sscratch, sp
|
||||
_from_user:
|
||||
addi sp, sp, -272
|
||||
|
||||
sd x1, 8(sp)
|
||||
sd x3, 24(sp)
|
||||
sd x4, 32(sp)
|
||||
sd x5, 40(sp)
|
||||
sd x6, 48(sp)
|
||||
sd x7, 56(sp)
|
||||
sd x8, 64(sp)
|
||||
sd x9, 72(sp)
|
||||
sd x10, 80(sp)
|
||||
sd x11, 88(sp)
|
||||
sd x12, 96(sp)
|
||||
sd x13, 104(sp)
|
||||
sd x14, 112(sp)
|
||||
sd x15, 120(sp)
|
||||
sd x16, 128(sp)
|
||||
sd x17, 136(sp)
|
||||
sd x18, 144(sp)
|
||||
sd x19, 152(sp)
|
||||
sd x20, 160(sp)
|
||||
sd x21, 168(sp)
|
||||
sd x22, 176(sp)
|
||||
sd x23, 184(sp)
|
||||
sd x24, 192(sp)
|
||||
sd x25, 200(sp)
|
||||
sd x26, 208(sp)
|
||||
sd x27, 216(sp)
|
||||
sd x28, 224(sp)
|
||||
sd x29, 232(sp)
|
||||
sd x30, 240(sp)
|
||||
sd x31, 248(sp)
|
||||
|
||||
csrr t0, sscratch
|
||||
sd t0, 16(sp)
|
||||
|
||||
csrr t0, sepc
|
||||
sd t0, 256(sp)
|
||||
csrr t0, sstatus
|
||||
sd t0, 264(sp)
|
||||
|
||||
mv a0, sp
|
||||
call trap_handler_rust
|
||||
j _restore_frame
|
||||
|
||||
# trap_return(frame): resume whatever `frame` describes, as if returning
|
||||
# from a trap that saved it. Used to enter userspace for the first time.
|
||||
.globl trap_return
|
||||
trap_return:
|
||||
mv sp, a0
|
||||
_restore_frame:
|
||||
ld t0, 256(sp)
|
||||
csrw sepc, t0
|
||||
ld t0, 264(sp)
|
||||
csrw sstatus, t0
|
||||
|
||||
andi t1, t0, 0x100
|
||||
bnez t1, _restore_smode
|
||||
|
||||
ld t0, 16(sp)
|
||||
csrw sscratch, t0
|
||||
|
||||
_restore_smode:
|
||||
ld x1, 8(sp)
|
||||
ld x3, 24(sp)
|
||||
ld x4, 32(sp)
|
||||
ld x6, 48(sp)
|
||||
ld x7, 56(sp)
|
||||
ld x8, 64(sp)
|
||||
ld x9, 72(sp)
|
||||
ld x10, 80(sp)
|
||||
ld x11, 88(sp)
|
||||
ld x12, 96(sp)
|
||||
ld x13, 104(sp)
|
||||
ld x14, 112(sp)
|
||||
ld x15, 120(sp)
|
||||
ld x16, 128(sp)
|
||||
ld x17, 136(sp)
|
||||
ld x18, 144(sp)
|
||||
ld x19, 152(sp)
|
||||
ld x20, 160(sp)
|
||||
ld x21, 168(sp)
|
||||
ld x22, 176(sp)
|
||||
ld x23, 184(sp)
|
||||
ld x24, 192(sp)
|
||||
ld x25, 200(sp)
|
||||
ld x26, 208(sp)
|
||||
ld x27, 216(sp)
|
||||
ld x28, 224(sp)
|
||||
ld x29, 232(sp)
|
||||
ld x30, 240(sp)
|
||||
ld x31, 248(sp)
|
||||
|
||||
# Every GPR except t0 (x5) is restored. Decide where we return to while
|
||||
# t0 is still free, and restore it last on each path; using t0 after
|
||||
# its restore would corrupt the interrupted program's t0.
|
||||
csrr t0, sstatus
|
||||
andi t0, t0, 0x100
|
||||
bnez t0, _sret_smode
|
||||
ld x5, 40(sp)
|
||||
addi sp, sp, 272
|
||||
csrrw sp, sscratch, sp
|
||||
sret
|
||||
_sret_smode:
|
||||
ld x5, 40(sp)
|
||||
addi sp, sp, 272
|
||||
sret
|
||||
"#
|
||||
);
|
||||
|
||||
const CAUSE_INTERRUPT_BIT: usize = 1 << 63;
|
||||
const CAUSE_S_SOFTWARE: usize = 1;
|
||||
const CAUSE_S_TIMER: usize = 5;
|
||||
@@ -143,7 +29,7 @@ pub fn init() {
|
||||
let addr = trap_vector as *const () as usize;
|
||||
core::arch::asm!("csrw stvec, {0}", in(reg) addr);
|
||||
core::arch::asm!("csrw sscratch, zero");
|
||||
// SIE stays clear — interrupts are enabled only when entering userspace via sret
|
||||
// SIE stays clear; interrupts are enabled only by sret into userspace.
|
||||
let sie_bits = (1usize << 5) | (1usize << 9) | (1usize << 1);
|
||||
core::arch::asm!("csrs sie, {0}", in(reg) sie_bits);
|
||||
}
|
||||
@@ -169,31 +55,6 @@ fn write_result(frame: &mut TrapFrame, result: &kernel::SyscallResult) {
|
||||
set_results(frame, kernel::abi::encode(result));
|
||||
}
|
||||
|
||||
fn save_frame_to_context(frame: &TrapFrame, ctx: &mut domain::thread::SavedContext) {
|
||||
ctx.registers_mut().copy_from_slice(&frame.regs);
|
||||
ctx.set_pc(frame.sepc);
|
||||
ctx.set_status(frame.sstatus);
|
||||
}
|
||||
|
||||
fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFrame) {
|
||||
frame.regs.copy_from_slice(ctx.registers());
|
||||
frame.sepc = ctx.program_counter();
|
||||
frame.sstatus = ctx.status();
|
||||
// ensure returning to U-mode
|
||||
frame.sstatus &= !(1 << 8); // clear SPP
|
||||
frame.sstatus |= 1 << 5; // set SPIE
|
||||
}
|
||||
|
||||
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
|
||||
if let Some(result) = kern().take_pending_result(tid) {
|
||||
write_result(frame, &result);
|
||||
}
|
||||
}
|
||||
|
||||
unsafe extern "C" {
|
||||
fn trap_return(frame: *mut TrapFrame) -> !;
|
||||
}
|
||||
|
||||
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
|
||||
// SAFETY: only called with the kernel lock held.
|
||||
unsafe { &mut *crate::kernel_ptr() }
|
||||
@@ -203,63 +64,7 @@ fn this_core() -> types::id::CoreId {
|
||||
types::id::CoreId::from_raw(crate::smp::hart_id() as u8)
|
||||
}
|
||||
|
||||
/// Saves `frame` into `from` (if it still exists) and loads `to`.
|
||||
fn switch_to_thread(
|
||||
frame: &mut TrapFrame,
|
||||
from: Option<types::id::ThreadId>,
|
||||
to: types::id::ThreadId,
|
||||
) {
|
||||
let kern = kern();
|
||||
if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
|
||||
save_frame_to_context(frame, ctx);
|
||||
}
|
||||
if let Some(ctx) = kern.thread_context(to) {
|
||||
load_context_to_frame(ctx, frame);
|
||||
}
|
||||
apply_pending_result(frame, to);
|
||||
}
|
||||
|
||||
/// Saves `from` and runs the next ready thread, idling until there is one.
|
||||
fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
|
||||
if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
|
||||
save_frame_to_context(frame, ctx);
|
||||
}
|
||||
let next = next_runnable();
|
||||
let _ = kern().set_thread_running(next, this_core());
|
||||
switch_to_thread(frame, None, next);
|
||||
}
|
||||
|
||||
/// The next thread to run on this hart, idling until one is ready.
|
||||
///
|
||||
/// Idling gives up the kernel lock: an idle hart holds nothing, so other
|
||||
/// harts can make work for it (and kick it) meanwhile. It also leaves any
|
||||
/// user address space first, so nobody needs to shoot it down. SIE stays
|
||||
/// clear; `wfi` still wakes on a pending interrupt, serviced by polling.
|
||||
fn next_runnable() -> types::id::ThreadId {
|
||||
loop {
|
||||
if let Some(tid) = kern().pick_next_thread() {
|
||||
return tid;
|
||||
}
|
||||
crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
|
||||
release();
|
||||
unsafe { core::arch::asm!("wfi") };
|
||||
crate::smp::service_ipi();
|
||||
crate::smp::lock();
|
||||
let _ = crate::smp::take_reschedule();
|
||||
let sip: usize;
|
||||
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
|
||||
if sip & (1 << CAUSE_S_TIMER) != 0 {
|
||||
// No thread is running, so there is no time slice to charge.
|
||||
crate::hw::timer::handle_interrupt();
|
||||
}
|
||||
if sip & (1 << CAUSE_S_EXTERNAL) != 0 {
|
||||
dispatch_external();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Claims every pending PLIC interrupt and hands each to the kernel. Only
|
||||
/// the boot hart receives external interrupts.
|
||||
/// Claims every pending PLIC interrupt (boot hart only) and hands it to the kernel.
|
||||
fn dispatch_external() {
|
||||
loop {
|
||||
let irq_num = crate::hw::plic::Plic::claim();
|
||||
@@ -271,100 +76,6 @@ fn dispatch_external() {
|
||||
}
|
||||
}
|
||||
|
||||
/// Releases the kernel lock, first publishing what the lock-free fast
|
||||
/// paths may rely on.
|
||||
fn release() {
|
||||
let kern = kern();
|
||||
for core in 0..crate::smp::num_harts() {
|
||||
let id = types::id::CoreId::from_raw(core as u8);
|
||||
crate::smp::publish_waiting(core, kern.highest_waiting(id).map(|p| p.value()));
|
||||
}
|
||||
let running = kern.current_thread();
|
||||
crate::smp::publish_running(
|
||||
running
|
||||
.and_then(|t| kern.thread_process(t))
|
||||
.map(|p| p.raw()),
|
||||
running
|
||||
.and_then(|t| kern.thread_priority(t))
|
||||
.map(|p| p.value()),
|
||||
);
|
||||
crate::smp::unlock();
|
||||
}
|
||||
|
||||
/// Syscalls answered without the kernel lock, as the kernel would answer
|
||||
/// them: they read nothing the lock guards (the time, the UART, the
|
||||
/// running thread's fixed process id) or, for a Yield that has nobody to
|
||||
/// yield to, change nothing. Any pending IPI or reschedule request sends
|
||||
/// the call down the slow path, so no request is ever skipped.
|
||||
fn fast_syscall(frame: &mut TrapFrame) -> bool {
|
||||
use types::syscall_abi::{SYS_DEBUG_PUTCHAR, SYS_GET_PROCESS_ID, SYS_GET_TIME, SYS_YIELD};
|
||||
if crate::smp::has_pending_request() {
|
||||
return false;
|
||||
}
|
||||
let result = match frame.regs[17] {
|
||||
SYS_GET_TIME => kernel::SyscallResult::Value(crate::hw::timer::read_time() as usize),
|
||||
SYS_GET_PROCESS_ID => match crate::smp::running_pid() {
|
||||
Some(pid) => kernel::SyscallResult::Value(pid),
|
||||
None => return false,
|
||||
},
|
||||
SYS_DEBUG_PUTCHAR => {
|
||||
crate::hw::uart::Uart::new().put_char(frame.regs[10] as u8);
|
||||
kernel::SyscallResult::Ok
|
||||
}
|
||||
SYS_YIELD if crate::smp::yield_is_noop() => kernel::SyscallResult::Ok,
|
||||
_ => return false,
|
||||
};
|
||||
frame.sepc += 4;
|
||||
write_result(frame, &result);
|
||||
true
|
||||
}
|
||||
|
||||
/// Last step before leaving the kernel to userspace: act on a reschedule
|
||||
/// request, make sure this hart has a thread (its own may have been taken
|
||||
/// away by another hart), and translate through that thread's address
|
||||
/// space.
|
||||
fn finish(frame: &mut TrapFrame) {
|
||||
let kern = kern();
|
||||
let current = kern.current_thread();
|
||||
if crate::smp::take_reschedule()
|
||||
&& let Some(cur) = current
|
||||
&& let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
|
||||
{
|
||||
switch_to_thread(frame, Some(cur), next);
|
||||
}
|
||||
if kern.current_thread().is_none() {
|
||||
switch_to_next(frame, None);
|
||||
}
|
||||
let root = kern
|
||||
.current_thread()
|
||||
.and_then(|t| kern.thread_process(t))
|
||||
.map(|pid| kern.platform_mut().page_table_for(pid))
|
||||
.unwrap_or(0);
|
||||
if root != 0 && crate::mmu::active_root() != root {
|
||||
crate::mmu::activate(crate::mmu::page_table::table_at(root));
|
||||
}
|
||||
}
|
||||
|
||||
/// Leaves the kernel for userspace for the first time on this hart, with
|
||||
/// the kernel lock held: runs the thread the kernel already made current
|
||||
/// here (init, on the boot hart) or the next one homed here.
|
||||
pub fn enter_first() -> ! {
|
||||
let mut frame = TrapFrame {
|
||||
regs: [0; 32],
|
||||
sepc: 0,
|
||||
sstatus: 0,
|
||||
};
|
||||
match kern().current_thread() {
|
||||
Some(tid) => switch_to_thread(&mut frame, None, tid),
|
||||
None => switch_to_next(&mut frame, None),
|
||||
}
|
||||
finish(&mut frame);
|
||||
release();
|
||||
// SAFETY: `frame` holds a user context; sscratch is set from it on
|
||||
// the way out, so later traps land at the top of this frame.
|
||||
unsafe { trap_return(&mut frame) }
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
|
||||
let scause: usize;
|
||||
@@ -401,15 +112,12 @@ fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
|
||||
}
|
||||
}
|
||||
CAUSE_S_EXTERNAL => dispatch_external(),
|
||||
// Serviced on entry; any reschedule happens in `finish`.
|
||||
CAUSE_S_SOFTWARE => {}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// The kernel itself faulted (a W^X violation, a stack guard, a bad
|
||||
/// pointer). Its state can no longer be trusted, and no user thread is to
|
||||
/// blame.
|
||||
/// The kernel itself faulted; its state is untrusted, so halt.
|
||||
fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! {
|
||||
let uart = crate::hw::uart::Uart::new();
|
||||
uart.write_str("KERNEL FAULT: scause=");
|
||||
@@ -478,8 +186,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
|
||||
kern.handle_fault(current_tid, fault, frame.sepc);
|
||||
switch_to_next(frame, Some(current_tid));
|
||||
}
|
||||
// With no current thread (another hart just destroyed it),
|
||||
// `finish` moves on to the next one.
|
||||
// No current thread (destroyed by another hart): `finish` picks one.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user