509 lines
15 KiB
Rust
509 lines
15 KiB
Rust
use core::arch::global_asm;
|
|
|
|
#[repr(C)]
|
|
pub struct TrapFrame {
|
|
pub regs: [usize; 32],
|
|
pub sepc: usize,
|
|
pub sstatus: usize,
|
|
}
|
|
|
|
impl TrapFrame {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
regs: [0; 32],
|
|
sepc: 0,
|
|
sstatus: 0,
|
|
}
|
|
}
|
|
}
|
|
|
|
global_asm!(
|
|
r#"
|
|
.align 4
|
|
.globl trap_vector
|
|
trap_vector:
|
|
csrrw sp, sscratch, sp
|
|
bnez sp, _from_user
|
|
csrrw sp, sscratch, sp
|
|
_from_user:
|
|
addi sp, sp, -272
|
|
|
|
sd x1, 8(sp)
|
|
sd x3, 24(sp)
|
|
sd x4, 32(sp)
|
|
sd x5, 40(sp)
|
|
sd x6, 48(sp)
|
|
sd x7, 56(sp)
|
|
sd x8, 64(sp)
|
|
sd x9, 72(sp)
|
|
sd x10, 80(sp)
|
|
sd x11, 88(sp)
|
|
sd x12, 96(sp)
|
|
sd x13, 104(sp)
|
|
sd x14, 112(sp)
|
|
sd x15, 120(sp)
|
|
sd x16, 128(sp)
|
|
sd x17, 136(sp)
|
|
sd x18, 144(sp)
|
|
sd x19, 152(sp)
|
|
sd x20, 160(sp)
|
|
sd x21, 168(sp)
|
|
sd x22, 176(sp)
|
|
sd x23, 184(sp)
|
|
sd x24, 192(sp)
|
|
sd x25, 200(sp)
|
|
sd x26, 208(sp)
|
|
sd x27, 216(sp)
|
|
sd x28, 224(sp)
|
|
sd x29, 232(sp)
|
|
sd x30, 240(sp)
|
|
sd x31, 248(sp)
|
|
|
|
csrr t0, sscratch
|
|
sd t0, 16(sp)
|
|
|
|
csrr t0, sepc
|
|
sd t0, 256(sp)
|
|
csrr t0, sstatus
|
|
sd t0, 264(sp)
|
|
|
|
mv a0, sp
|
|
call trap_handler_rust
|
|
|
|
ld t0, 256(sp)
|
|
csrw sepc, t0
|
|
ld t0, 264(sp)
|
|
csrw sstatus, t0
|
|
|
|
andi t1, t0, 0x100
|
|
bnez t1, _restore_smode
|
|
|
|
ld t0, 16(sp)
|
|
csrw sscratch, t0
|
|
|
|
_restore_smode:
|
|
ld x1, 8(sp)
|
|
ld x3, 24(sp)
|
|
ld x4, 32(sp)
|
|
ld x5, 40(sp)
|
|
ld x6, 48(sp)
|
|
ld x7, 56(sp)
|
|
ld x8, 64(sp)
|
|
ld x9, 72(sp)
|
|
ld x10, 80(sp)
|
|
ld x11, 88(sp)
|
|
ld x12, 96(sp)
|
|
ld x13, 104(sp)
|
|
ld x14, 112(sp)
|
|
ld x15, 120(sp)
|
|
ld x16, 128(sp)
|
|
ld x17, 136(sp)
|
|
ld x18, 144(sp)
|
|
ld x19, 152(sp)
|
|
ld x20, 160(sp)
|
|
ld x21, 168(sp)
|
|
ld x22, 176(sp)
|
|
ld x23, 184(sp)
|
|
ld x24, 192(sp)
|
|
ld x25, 200(sp)
|
|
ld x26, 208(sp)
|
|
ld x27, 216(sp)
|
|
ld x28, 224(sp)
|
|
ld x29, 232(sp)
|
|
ld x30, 240(sp)
|
|
ld x31, 248(sp)
|
|
|
|
addi sp, sp, 272
|
|
|
|
csrr t0, sstatus
|
|
andi t0, t0, 0x100
|
|
bnez t0, _sret_smode
|
|
csrrw sp, sscratch, sp
|
|
_sret_smode:
|
|
sret
|
|
"#
|
|
);
|
|
|
|
const CAUSE_INTERRUPT_BIT: usize = 1 << 63;
|
|
const CAUSE_S_TIMER: usize = 5;
|
|
const CAUSE_S_EXTERNAL: usize = 9;
|
|
const CAUSE_ECALL_U: usize = 8;
|
|
|
|
pub fn init() {
|
|
unsafe extern "C" {
|
|
fn trap_vector();
|
|
}
|
|
unsafe {
|
|
let addr = trap_vector as usize;
|
|
core::arch::asm!("csrw stvec, {0}", in(reg) addr);
|
|
core::arch::asm!("csrw sscratch, zero");
|
|
// SIE stays clear — interrupts are enabled only when entering userspace via sret
|
|
let sie_bits = (1usize << 5) | (1usize << 9) | (1usize << 1);
|
|
core::arch::asm!("csrs sie, {0}", in(reg) sie_bits);
|
|
}
|
|
}
|
|
|
|
pub fn set_kernel_stack(kernel_sp: usize) {
|
|
unsafe {
|
|
core::arch::asm!("csrw sscratch, {0}", in(reg) kernel_sp);
|
|
}
|
|
}
|
|
|
|
fn decode_syscall(frame: &TrapFrame) -> Option<domain::syscall::Syscall> {
|
|
use domain::message::Message;
|
|
use domain::syscall::{Syscall, WaitSet};
|
|
use types::syscall_abi::*;
|
|
|
|
let nr = frame.regs[17]; // a7
|
|
let a0 = frame.regs[10];
|
|
let a1 = frame.regs[11];
|
|
let a2 = frame.regs[12];
|
|
let a3 = frame.regs[13];
|
|
let a4 = frame.regs[14];
|
|
|
|
match nr {
|
|
SYS_EXIT => Some(Syscall::Exit),
|
|
SYS_HALT => Some(Syscall::Halt),
|
|
SYS_EXIT_THREAD => Some(Syscall::ExitThread),
|
|
SYS_YIELD => Some(Syscall::Yield),
|
|
SYS_GET_TIME => Some(Syscall::GetTime),
|
|
SYS_GET_PROCESS_ID => Some(Syscall::GetProcessId),
|
|
SYS_SEND => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
let msg = Message::from_words([a1, a2, a3, a4]);
|
|
Some(Syscall::Send {
|
|
endpoint: slot,
|
|
message: msg,
|
|
})
|
|
}
|
|
SYS_RECEIVE => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
Some(Syscall::Receive { endpoint: slot })
|
|
}
|
|
SYS_DELEGATE => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
let to = types::bounded::CapSlot::new(a1 as u8)?;
|
|
let perms = types::permission::PermissionSet::from_bits(a2 as u8);
|
|
Some(Syscall::Delegate {
|
|
slot,
|
|
to_endpoint: to,
|
|
permissions: perms,
|
|
})
|
|
}
|
|
SYS_WAIT_FOR_INTERRUPT => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
Some(Syscall::WaitForInterrupt { irq: slot })
|
|
}
|
|
SYS_ACKNOWLEDGE_INTERRUPT => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
Some(Syscall::AcknowledgeInterrupt { irq: slot })
|
|
}
|
|
SYS_WAIT_ANY => {
|
|
let ep0 = if a0 != usize::MAX {
|
|
types::bounded::CapSlot::new(a0 as u8)
|
|
} else {
|
|
None
|
|
};
|
|
let ep1 = if a1 != usize::MAX {
|
|
types::bounded::CapSlot::new(a1 as u8)
|
|
} else {
|
|
None
|
|
};
|
|
let ep2 = if a2 != usize::MAX {
|
|
types::bounded::CapSlot::new(a2 as u8)
|
|
} else {
|
|
None
|
|
};
|
|
let ep3 = if a3 != usize::MAX {
|
|
types::bounded::CapSlot::new(a3 as u8)
|
|
} else {
|
|
None
|
|
};
|
|
let sources = WaitSet::new([ep0, ep1, ep2, ep3], [None, None, None, None]);
|
|
Some(Syscall::WaitAny { sources })
|
|
}
|
|
SYS_CREATE_PROCESS => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
Some(Syscall::CreateProcess {
|
|
create_cap: slot,
|
|
image: types::address::DataRef::empty(),
|
|
})
|
|
}
|
|
SYS_KILL => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
Some(Syscall::Kill { process: slot })
|
|
}
|
|
SYS_REVOKE => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
Some(Syscall::Revoke { slot })
|
|
}
|
|
SYS_SPAWN_THREAD => Some(Syscall::SpawnThread {
|
|
entry: types::address::EntryPoint::from_raw(a0),
|
|
stack: types::address::StackPointer::from_raw(a1),
|
|
}),
|
|
SYS_REQUEST_PAGES => {
|
|
let count = types::bounded::PageCount::new(a0 as u16)?;
|
|
Some(Syscall::RequestPages { count })
|
|
}
|
|
SYS_RELEASE_PAGES => {
|
|
let count = types::bounded::PageCount::new(a1 as u16)?;
|
|
Some(Syscall::ReleasePages {
|
|
addr: types::address::VirtualAddress::from_raw(a0),
|
|
count,
|
|
})
|
|
}
|
|
SYS_SPAWN_THREAD_IN => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
Some(Syscall::SpawnThreadIn {
|
|
process: slot,
|
|
entry: types::address::EntryPoint::from_raw(a1),
|
|
stack: types::address::StackPointer::from_raw(a2),
|
|
})
|
|
}
|
|
SYS_CREATE_ENDPOINT => Some(Syscall::CreateEndpoint),
|
|
SYS_DEBUG_PUTCHAR => Some(Syscall::DebugPutchar { byte: a0 as u8 }),
|
|
SYS_MAP_PAGE => {
|
|
let slot = types::bounded::CapSlot::new(a0 as u8)?;
|
|
Some(Syscall::MapPage {
|
|
process: slot,
|
|
src_va: types::address::VirtualAddress::from_raw(a1),
|
|
dst_va: types::address::VirtualAddress::from_raw(a2),
|
|
executable: a3 != 0,
|
|
})
|
|
}
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
fn write_result(frame: &mut TrapFrame, outcome: &kernel::SyscallOutcome) {
|
|
use kernel::{SyscallOutcome, SyscallResult};
|
|
|
|
match outcome {
|
|
SyscallOutcome::Complete(result) => match result {
|
|
SyscallResult::Ok => frame.regs[10] = 0,
|
|
SyscallResult::Value(v) => frame.regs[10] = *v,
|
|
SyscallResult::Slot(s) => frame.regs[10] = s.value() as usize,
|
|
SyscallResult::Message(m) => {
|
|
frame.regs[10] = m.word(0);
|
|
frame.regs[11] = m.word(1);
|
|
frame.regs[12] = m.word(2);
|
|
frame.regs[13] = m.word(3);
|
|
}
|
|
SyscallResult::Error(_) => frame.regs[10] = usize::MAX,
|
|
SyscallResult::WakeReason(_) => frame.regs[10] = 0,
|
|
},
|
|
SyscallOutcome::BlockCaller(_) => {}
|
|
SyscallOutcome::SwitchTo(_) => frame.regs[10] = 0,
|
|
SyscallOutcome::TerminateCaller => {}
|
|
}
|
|
}
|
|
|
|
fn save_frame_to_context(frame: &TrapFrame, ctx: &mut domain::thread::SavedContext) {
|
|
ctx.registers_mut().copy_from_slice(&frame.regs);
|
|
ctx.set_pc(frame.sepc);
|
|
ctx.set_status(frame.sstatus);
|
|
}
|
|
|
|
fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFrame) {
|
|
frame.regs.copy_from_slice(ctx.registers());
|
|
frame.sepc = ctx.program_counter();
|
|
frame.sstatus = ctx.status();
|
|
// ensure returning to U-mode
|
|
frame.sstatus &= !(1 << 8); // clear SPP
|
|
frame.sstatus |= 1 << 5; // set SPIE
|
|
}
|
|
|
|
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
|
|
let kern = unsafe { &mut *crate::kernel_ptr() };
|
|
if let Some(result) = kern.take_pending_result(tid) {
|
|
write_result_direct(frame, &result);
|
|
}
|
|
}
|
|
|
|
fn write_result_direct(frame: &mut TrapFrame, result: &kernel::SyscallResult) {
|
|
use kernel::SyscallResult;
|
|
match result {
|
|
SyscallResult::Ok => frame.regs[10] = 0,
|
|
SyscallResult::Value(v) => frame.regs[10] = *v,
|
|
SyscallResult::Slot(s) => frame.regs[10] = s.value() as usize,
|
|
SyscallResult::Message(m) => {
|
|
frame.regs[10] = m.word(0);
|
|
frame.regs[11] = m.word(1);
|
|
frame.regs[12] = m.word(2);
|
|
frame.regs[13] = m.word(3);
|
|
}
|
|
SyscallResult::Error(_) => frame.regs[10] = usize::MAX,
|
|
SyscallResult::WakeReason(_) => frame.regs[10] = 0,
|
|
}
|
|
}
|
|
|
|
fn switch_to_thread(
|
|
frame: &mut TrapFrame,
|
|
current_tid: types::id::ThreadId,
|
|
next_tid: types::id::ThreadId,
|
|
) {
|
|
let kern = unsafe { &mut *crate::kernel_ptr() };
|
|
|
|
if let Some(ctx) = kern.thread_context_mut(current_tid) {
|
|
save_frame_to_context(frame, ctx);
|
|
}
|
|
|
|
let next_pid = kern.thread_process(next_tid);
|
|
let current_pid = kern.thread_process(current_tid);
|
|
|
|
if let Some(ctx) = kern.thread_context(next_tid) {
|
|
load_context_to_frame(ctx, frame);
|
|
}
|
|
|
|
apply_pending_result(frame, next_tid);
|
|
|
|
if next_pid != current_pid {
|
|
if let Some(pid) = next_pid {
|
|
let pt_addr = kern.platform_mut().page_table_for(pid);
|
|
if pt_addr != 0 {
|
|
let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) };
|
|
crate::mmu::activate(pt);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) {
|
|
let kern = unsafe { &mut *crate::kernel_ptr() };
|
|
|
|
if let Some(ctx) = kern.thread_context_mut(current_tid) {
|
|
save_frame_to_context(frame, ctx);
|
|
}
|
|
|
|
if let Some(next_tid) = kern.pick_next_thread() {
|
|
let next_pid = kern.thread_process(next_tid);
|
|
let current_pid = kern.thread_process(current_tid);
|
|
|
|
let core = types::id::CoreId::from_raw(0);
|
|
let _ = kern.set_thread_running(next_tid, core);
|
|
|
|
if let Some(ctx) = kern.thread_context(next_tid) {
|
|
load_context_to_frame(ctx, frame);
|
|
}
|
|
|
|
apply_pending_result(frame, next_tid);
|
|
|
|
if next_pid != current_pid {
|
|
if let Some(pid) = next_pid {
|
|
let pt_addr = kern.platform_mut().page_table_for(pid);
|
|
if pt_addr != 0 {
|
|
let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) };
|
|
crate::mmu::activate(pt);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
loop {
|
|
unsafe { core::arch::asm!("wfi") };
|
|
}
|
|
}
|
|
}
|
|
|
|
#[unsafe(no_mangle)]
|
|
extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
|
|
let scause: usize;
|
|
unsafe { core::arch::asm!("csrr {0}, scause", out(reg) scause) };
|
|
|
|
if scause & CAUSE_INTERRUPT_BIT != 0 {
|
|
handle_interrupt(frame, scause & !CAUSE_INTERRUPT_BIT);
|
|
} else {
|
|
handle_exception(frame, scause);
|
|
}
|
|
}
|
|
|
|
fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
|
|
match cause {
|
|
CAUSE_S_TIMER => {
|
|
crate::hw::timer::handle_interrupt();
|
|
let kern = unsafe { &mut *crate::kernel_ptr() };
|
|
let core = types::id::CoreId::from_raw(0);
|
|
if let Some(current_tid) = kern.current_thread() {
|
|
let decision = kern.handle_tick(core);
|
|
if let domain::scheduler::SchedulerDecision::Preempt(next) = decision {
|
|
switch_to_thread(frame, current_tid, next);
|
|
}
|
|
}
|
|
}
|
|
CAUSE_S_EXTERNAL => {
|
|
let irq_num = crate::hw::plic::Plic::claim();
|
|
if irq_num != 0 {
|
|
let kern = unsafe { &mut *crate::kernel_ptr() };
|
|
let irq = types::id::IrqId::from_raw(irq_num as u16);
|
|
kern.handle_interrupt(irq);
|
|
crate::hw::plic::Plic::complete(irq_num);
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
fn handle_exception(frame: &mut TrapFrame, scause: usize) {
|
|
match scause {
|
|
CAUSE_ECALL_U => {
|
|
frame.sepc += 4;
|
|
|
|
if let Some(syscall) = decode_syscall(frame) {
|
|
let kern = unsafe { &mut *crate::kernel_ptr() };
|
|
let current_tid = kern.current_thread();
|
|
let outcome = kern.handle_syscall(syscall);
|
|
write_result(frame, &outcome);
|
|
|
|
match outcome {
|
|
kernel::SyscallOutcome::Complete(_) => {}
|
|
kernel::SyscallOutcome::BlockCaller(_) => {
|
|
if let Some(tid) = current_tid {
|
|
switch_to_next(frame, tid);
|
|
}
|
|
}
|
|
kernel::SyscallOutcome::SwitchTo(next) => {
|
|
if let Some(tid) = current_tid {
|
|
switch_to_thread(frame, tid, next);
|
|
}
|
|
}
|
|
kernel::SyscallOutcome::TerminateCaller => {
|
|
let uart = crate::hw::uart::Uart::new();
|
|
uart.write_str("k-os: process exited\n");
|
|
if let Some(tid) = current_tid {
|
|
switch_to_next(frame, tid);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
_ => {
|
|
let uart = crate::hw::uart::Uart::new();
|
|
uart.write_str("FAULT: scause=");
|
|
uart.write_hex(scause);
|
|
let stval: usize;
|
|
unsafe { core::arch::asm!("csrr {0}, stval", out(reg) stval) };
|
|
uart.write_str(" stval=");
|
|
uart.write_hex(stval);
|
|
uart.write_str(" sepc=");
|
|
uart.write_hex(frame.sepc);
|
|
uart.put_char(b'\n');
|
|
|
|
let kern = unsafe { &mut *crate::kernel_ptr() };
|
|
if let Some(current_tid) = kern.current_thread() {
|
|
let fault = match scause {
|
|
2 => domain::fault::Fault::IllegalInstruction,
|
|
5 | 7 | 13 | 15 => domain::fault::Fault::MemoryFault {
|
|
address: types::address::VirtualAddress::from_raw(stval),
|
|
},
|
|
_ => domain::fault::Fault::InvalidSyscall,
|
|
};
|
|
kern.handle_fault(current_tid, fault, frame.sepc);
|
|
switch_to_next(frame, current_tid);
|
|
} else {
|
|
loop {
|
|
unsafe { core::arch::asm!("wfi") };
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|