diff --git a/crates/adapter-riscv/src/smp.rs b/crates/adapter-riscv/src/smp.rs index 61b77f8..068a1c9 100644 --- a/crates/adapter-riscv/src/smp.rs +++ b/crates/adapter-riscv/src/smp.rs @@ -97,6 +97,61 @@ pub fn unlock() { SERVING.fetch_add(1, Ordering::Release); } +// ---- what lock-free fast paths may know ---- +// +// Published under the kernel lock just before it is released, read without +// it. A fast path that reads them while another hart holds the lock simply +// takes effect before that hart's operation. + +/// Per core: highest waiting priority + 1, 0 if its queue is empty. +static WAITING: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; +/// Per hart: the running thread's process id + 1 and priority + 1. +static RUNNING_PID: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; +static RUNNING_PRIORITY: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS]; + +pub fn publish_waiting(core: usize, highest: Option) { + if let Some(w) = WAITING.get(core) { + w.store(highest.map_or(0, |p| p as usize + 1), Ordering::Release); + } +} + +pub fn publish_running(pid: Option, priority: Option) { + let me = hart_id(); + if let (Some(p), Some(q)) = (RUNNING_PID.get(me), RUNNING_PRIORITY.get(me)) { + p.store(pid.map_or(0, |x| x as usize + 1), Ordering::Release); + q.store(priority.map_or(0, |x| x as usize + 1), Ordering::Release); + } +} + +/// The process the thread running on this hart belongs to. +pub fn running_pid() -> Option { + let v = RUNNING_PID.get(hart_id())?.load(Ordering::Acquire); + v.checked_sub(1) +} + +/// Whether a Yield by the running thread would change nothing: no thread +/// waits on this core at its priority or above. +pub fn yield_is_noop() -> bool { + let me = hart_id(); + let (Some(w), Some(p)) = (WAITING.get(me), RUNNING_PRIORITY.get(me)) else { + return false; + }; + let mine = p.load(Ordering::Acquire); + mine != 0 && w.load(Ordering::Acquire) < mine +} + +/// Whether this hart has an IPI or a reschedule request it has not acted +/// on; fast paths must then take the slow path. +pub fn has_pending_request() -> bool { + let sip: usize; + // SAFETY: reading sip has no side effects. + unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) }; + sip & SIP_SSIP != 0 + || RESCHEDULE + .get(hart_id()) + .is_some_and(|r| r.load(Ordering::SeqCst)) +} + // ---- IPIs ---- /// The lock-free half of an IPI: acknowledges it and, if asked, leaves the diff --git a/crates/adapter-riscv/src/trap/mod.rs b/crates/adapter-riscv/src/trap/mod.rs index 0b679e0..5753fae 100644 --- a/crates/adapter-riscv/src/trap/mod.rs +++ b/crates/adapter-riscv/src/trap/mod.rs @@ -241,7 +241,7 @@ fn next_runnable() -> types::id::ThreadId { return tid; } crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root())); - crate::smp::unlock(); + release(); unsafe { core::arch::asm!("wfi") }; crate::smp::service_ipi(); crate::smp::lock(); @@ -271,6 +271,54 @@ fn dispatch_external() { } } +/// Releases the kernel lock, first publishing what the lock-free fast +/// paths may rely on. +fn release() { + let kern = kern(); + for core in 0..crate::smp::num_harts() { + let id = types::id::CoreId::from_raw(core as u8); + crate::smp::publish_waiting(core, kern.highest_waiting(id).map(|p| p.value())); + } + let running = kern.current_thread(); + crate::smp::publish_running( + running + .and_then(|t| kern.thread_process(t)) + .map(|p| p.raw()), + running + .and_then(|t| kern.thread_priority(t)) + .map(|p| p.value()), + ); + crate::smp::unlock(); +} + +/// Syscalls answered without the kernel lock, as the kernel would answer +/// them: they read nothing the lock guards (the time, the UART, the +/// running thread's fixed process id) or, for a Yield that has nobody to +/// yield to, change nothing. Any pending IPI or reschedule request sends +/// the call down the slow path, so no request is ever skipped. +fn fast_syscall(frame: &mut TrapFrame) -> bool { + use types::syscall_abi::{SYS_DEBUG_PUTCHAR, SYS_GET_PROCESS_ID, SYS_GET_TIME, SYS_YIELD}; + if crate::smp::has_pending_request() { + return false; + } + let result = match frame.regs[17] { + SYS_GET_TIME => kernel::SyscallResult::Value(crate::hw::timer::read_time() as usize), + SYS_GET_PROCESS_ID => match crate::smp::running_pid() { + Some(pid) => kernel::SyscallResult::Value(pid), + None => return false, + }, + SYS_DEBUG_PUTCHAR => { + crate::hw::uart::Uart::new().put_char(frame.regs[10] as u8); + kernel::SyscallResult::Ok + } + SYS_YIELD if crate::smp::yield_is_noop() => kernel::SyscallResult::Ok, + _ => return false, + }; + frame.sepc += 4; + write_result(frame, &result); + true +} + /// Last step before leaving the kernel to userspace: act on a reschedule /// request, make sure this hart has a thread (its own may have been taken /// away by another hart), and translate through that thread's address @@ -311,7 +359,7 @@ pub fn enter_first() -> ! { None => switch_to_next(&mut frame, None), } finish(&mut frame); - crate::smp::unlock(); + release(); // SAFETY: `frame` holds a user context; sscratch is set from it on // the way out, so later traps land at the top of this frame. unsafe { trap_return(&mut frame) } @@ -326,6 +374,9 @@ extern "C" fn trap_handler_rust(frame: &mut TrapFrame) { // Checked before taking the lock: this hart may already hold it. kernel_fault(frame, scause); } + if scause == CAUSE_ECALL_U && fast_syscall(frame) { + return; + } crate::smp::service_ipi(); crate::smp::lock(); if scause & CAUSE_INTERRUPT_BIT != 0 { @@ -334,7 +385,7 @@ extern "C" fn trap_handler_rust(frame: &mut TrapFrame) { handle_exception(frame, scause); } finish(frame); - crate::smp::unlock(); + release(); } fn handle_interrupt(frame: &mut TrapFrame, cause: usize) { diff --git a/crates/kernel/src/smp.rs b/crates/kernel/src/smp.rs index f454ba5..fe5d8db 100644 --- a/crates/kernel/src/smp.rs +++ b/crates/kernel/src/smp.rs @@ -79,6 +79,15 @@ impl Kernel

{ } } + /// The highest priority waiting in `core`'s run queue. + pub fn highest_waiting(&self, core: CoreId) -> Option { + self.scheduler.highest_waiting(core) + } + + pub fn thread_priority(&self, tid: ThreadId) -> Option { + self.threads.get(tid).ok().map(|t| t.priority()) + } + /// What `core` should do after an IPI: keep running its thread, or /// switch to a higher-priority one now waiting. When it has no thread /// (it was idle, or its thread was destroyed) the caller picks the diff --git a/crates/kernel/src/tests/storm.rs b/crates/kernel/src/tests/storm.rs index 991aae4..c51934a 100644 --- a/crates/kernel/src/tests/storm.rs +++ b/crates/kernel/src/tests/storm.rs @@ -360,6 +360,22 @@ fn region_slots(k: &Kernel, pid: Option) -> std::vec::V .collect() } +/// Every thread's state and queue, and every core's current thread. +fn scheduling_state(k: &Kernel) -> std::vec::Vec { + let threads = k.live_threads().map(|t| { + format!( + "{t:?} {:?} {:?}", + k.thread_state_for_test(t), + k.scheduler.queued_on(t) + ) + }); + let cores = (0..k.scheduler.num_cores()).map(|c| { + let c = types::id::CoreId::from_raw(c); + format!("{c:?} {:?}", k.scheduler.current(c)) + }); + threads.chain(cores).collect() +} + /// What the adapter does whenever a core is left without a thread: run the /// next one in its queue, if any. fn resume(k: &mut Kernel, core: types::id::CoreId) { @@ -483,7 +499,27 @@ fn storm(cores: u8) { }; k.switch_to_thread_for_test(tid); k.take_pending_result(tid); + // The adapter answers this Yield without the kernel lock; check + // that the kernel would indeed have changed nothing. + let yield_is_noop = nr == SYS_YIELD + && k.current_thread() == Some(tid) + && k.highest_waiting(core) + .is_none_or(|w| Some(w) < k.thread_priority(tid)); + let before = yield_is_noop.then(|| scheduling_state(&k)); let out = k.handle_syscall(syscall); + if let Some(before) = before { + assert!( + matches!( + out, + crate::SyscallOutcome::Complete(crate::SyscallResult::Ok) + ), + "seed {seed:#x} step {step}: fast-path Yield did not return Ok" + ); + assert!( + before == scheduling_state(&k), + "seed {seed:#x} step {step}: fast-path Yield changed scheduling" + ); + } resume(&mut k, core); executed += 1; if !matches!(