//! Global kernel invariants, and a storm of random syscalls with random //! (mostly garbage) arguments decoded through the real ABI. The kernel must //! never panic and every invariant must hold after every single call. extern crate std; use std::format; use std::string::String; use super::*; use crate::syscall::ThreadRef; use domain::memory::{RegionBacking, frame::FrameUse}; use domain::thread::{BlockReason, MAX_THREADS, ThreadState}; use platform::Platform; use types::address::DataRef; use types::bounded::CapSlot; use types::id::ThreadId; use types::mapping::PagePerms; use types::syscall_abi::*; impl Kernel { fn live_threads(&self) -> impl Iterator + '_ { (0..MAX_THREADS as u16) .map(ThreadId::from_raw) .filter(|t| self.threads.get(*t).is_ok()) } /// Every invariant the kernel promises, over every live object. pub fn check_invariants(&self) -> Result<(), String> { // ---- memory ---- let mut charged = 0; for p in 0..domain::process::MAX_PROCESSES as u16 { let pid = ProcessId::from_raw(p); let usage = self.page_usage(pid); charged += usage; let owned = self .frames .iter() .filter(|(_, u)| { matches!(u, FrameUse::User { owner, .. } | FrameUse::PageTable { owner } if *owner == pid) }) .count(); let regions: usize = self .regions .iter() .filter(|(_, r)| r.owner() == pid && r.backing() == RegionBacking::Pool) .map(|(_, r)| r.pages().value() as usize) .sum(); if usage != owned + regions { return Err(format!( "quota of {pid:?} is {usage}, owns {owned}+{regions}" )); } if !self.process_exists(pid) && (usage != 0 || self.platform.mapping_count(pid) != 0) { return Err(format!("dead {pid:?} still holds memory")); } } if charged + self.free_page_count() != self.frames.capacity() { return Err(String::from("frames leaked or double counted")); } let window = MockPlatform::USER_VA_START..MockPlatform::USER_VA_END; for (pid, va, pa) in self.platform.mappings() { if !window.contains(&va) { return Err(format!("{pid:?} mapped outside the window at {va:#x}")); } let v = types::address::VirtualAddress::from_raw(va); let user = self.frames.find_user(pid, v).map(|(p, _)| p.raw()); let region = self.regions.covering(pid, v).and_then(|id| { let r = self.regions.get(id)?; let m = r.mappings().find(|m| m.process == pid)?; Some(r.base().raw() + (va - m.va.raw())) }); if user != Some(pa) && region != Some(pa) { return Err(format!("{pid:?} has unrecorded mapping {va:#x}")); } } for (pa, usage) in self.frames.iter() { if let FrameUse::User { owner, page, .. } = usage && self.platform.mapping(*owner, *page as usize * 4096) != Some(pa.raw()) { return Err(String::from("recorded user page is not mapped")); } } // ---- capabilities ---- for p in 0..domain::process::MAX_PROCESSES as u16 { let pid = ProcessId::from_raw(p); let Ok(process) = self.processes.get(pid) else { continue; }; for (slot, cap) in process.capabilities().iter() { if let Some(parent) = cap.parent() && !self.resolves(parent) { return Err(format!("{pid:?}/{slot:?} has a dangling parent")); } } } for (_, region) in self.regions.iter() { let has = |want| region.mappings().any(|m| m.perms == want); if has(PagePerms::ReadWrite) && has(PagePerms::ReadExecute) { return Err(String::from("region mapped both writable and executable")); } for m in region.mappings() { if let Some(via) = m.via && (!self.resolves(via) || via.process != m.process) { return Err(String::from("mapping outlived its capability")); } } } // ---- queues hold only live threads blocked on that very object ---- let blocked_as = |t: ThreadId, want: BlockReason| { self.threads.get(t).ok().map(|x| *x.state()) == Some(ThreadState::Blocked(want)) }; for ep in self.endpoints.iter() { let id = *ep.id(); for t in ep.senders() { if !blocked_as(t, BlockReason::Sending(id)) && !blocked_as(t, BlockReason::Calling(id)) { return Err(format!( "{t:?} queued as a sender on {id:?} but not blocked there" )); } } for t in ep.receivers() { if !blocked_as(t, BlockReason::Receiving(id)) { return Err(format!( "{t:?} queued as a receiver on {id:?} but not blocked there" )); } } } for n in self.notifications.iter() { for t in n.waiters() { if !blocked_as(t, BlockReason::WaitingNotification(n.id())) { return Err(format!( "{t:?} waits on {:?} but is not blocked there", n.id() )); } } if let Some(b) = n.bound() && self.ipc.get(b.raw() as usize).and_then(|s| s.bound) != Some(n.id()) { return Err(format!( "{:?} bound to {b:?} which does not know it", n.id() )); } } // ---- threads, IPC, scheduling ---- for t in self.live_threads() { let Ok(thread) = self.threads.get(t) else { continue; }; let state = *thread.state(); if !self.process_exists(thread.process()) { return Err(format!("{t:?} outlived its process")); } let ipc = self.ipc[t.raw() as usize]; let in_endpoint = |e: types::id::EndpointId| { self.endpoints .get(e) .map(|ep| (ep.is_sender(t), ep.is_receiver(t))) }; match state { ThreadState::Blocked(BlockReason::Sending(e) | BlockReason::Calling(e)) => { if in_endpoint(e) != Some((true, false)) || ipc.outgoing.is_none() { return Err(format!("{t:?} blocked sending but not queued")); } } ThreadState::Blocked(BlockReason::Receiving(e)) => { if in_endpoint(e) != Some((false, true)) { return Err(format!("{t:?} blocked receiving but not queued")); } } ThreadState::Blocked(BlockReason::WaitingNotification(n)) => { if !self.notifications.get(n).is_some_and(|x| x.is_waiter(t)) { return Err(format!("{t:?} waits on a notification it is not queued on")); } } ThreadState::Blocked(BlockReason::AwaitingReply) => { let me = ThreadRef { tid: t, generation: self .threads .generation(t) .unwrap_or(types::generation::Generation::initial()), }; let holders = self .live_threads() .filter(|s| self.ipc[s.raw() as usize].reply_to == Some(me)) .count(); if holders != 1 { return Err(format!("{t:?} awaits a reply held by {holders} threads")); } } _ => { if ipc.outgoing.is_some() { return Err(format!("{t:?} runnable with a queued message")); } } } if self.scheduler.is_queued(t) != matches!(state, ThreadState::Ready) { return Err(format!("{t:?} in {state:?} has wrong run-queue membership")); } // ---- cores: a thread runs and waits only on its home core ---- let home = self.home(t); match state { ThreadState::Running(c) if c != home => { return Err(format!("{t:?} runs on {c:?}, its home is {home:?}")); } ThreadState::Running(c) if self.scheduler.current(c) != Some(t) => { return Err(format!("{t:?} Running on {c:?} but not its current thread")); } ThreadState::Ready if self.scheduler.queued_on(t) != Some(home) => { return Err(format!("{t:?} queued away from its home {home:?}")); } _ => {} } if let Some(r) = ipc.reply_to && self.threads.generation(r.tid) == Some(r.generation) && self.threads.get(r.tid).ok().map(|x| *x.state()) != Some(ThreadState::Blocked(BlockReason::AwaitingReply)) { return Err(format!("{t:?} holds a reply for a thread not waiting")); } } for c in 0..self.scheduler.num_cores() { let core = types::id::CoreId::from_raw(c); let current = self.scheduler.current(core); if let Some(t) = current && self.threads.get(t).ok().map(|x| *x.state()) != Some(ThreadState::Running(core)) { return Err(format!("{core:?}'s current {t:?} is not running there")); } // No lost wakeups: an idle core with work waiting has been told. if current.is_none() && self.scheduler.highest_waiting(core).is_some() && !self.platform.ipi_pending(core) { return Err(format!("{core:?} idles with ready threads and no IPI")); } } Ok(()) } } struct Rng(u64); impl Rng { fn next(&mut self) -> u64 { self.0 ^= self.0 << 13; self.0 ^= self.0 >> 7; self.0 ^= self.0 << 17; self.0 } fn below(&mut self, n: usize) -> usize { (self.next() % n as u64) as usize } /// An argument that is usually meaningful and sometimes garbage. fn arg(&mut self) -> usize { match self.below(9) { 0..=2 => self.below(8), // small slots and counts 8 => self.below(3), // the first few slots 3 => NONE, 4 => 0x40000 + self.below(64) * 4096, // heap-ish addresses 5 => 0x10000 + self.below(32) * 4096, 6 => self.below(70), // slots just past the end _ => self.next() as usize, // anything at all } } } /// Sets up a fresh round: a hub process owning the shared endpoints, /// notification and region; two children with two threads each; and caps /// to all shared objects for init and the children. fn new_round(k: &mut Kernel, init: ProcessId, init_thread: ThreadId) { use domain::resource::Resource; let all = types::permission::PermissionSet::empty() .with(types::permission::Permission::Read) .with(types::permission::Permission::Write) .with(types::permission::Permission::Execute) .with(types::permission::Permission::Delegate); // init's caps to anything that died last round would fill its table. for slot in 1..=CapSlot::MAX { let slot = CapSlot::new(slot).unwrap(); if k.is_dead_cap_for_test(init, slot) { let _ = k.delete_cap(init, slot); } } let Some(hub) = k.create_process_internal() else { if std::env::var("STORM_DEBUG").is_ok() { std::eprintln!("no hub: free pages {}", k.free_page_count()); } return; }; let (Ok(e0), Ok(e1), Ok(n)) = ( k.create_endpoint(hub), k.create_endpoint(hub), k.create_notification(hub), ) else { return; }; let region = k .create_pool_region(hub, types::bounded::PageCount::new(2).unwrap()) .ok(); let mut members = std::vec![init]; // CreateProcess below runs as init, so on init's core. let init_core = k.home(init_thread); k.platform_mut().set_core(init_core); resume(k, init_core); for _ in 0..2 { k.switch_to_thread_for_test(init_thread); if let crate::SyscallOutcome::Complete(crate::SyscallResult::Slot(cap)) = k.handle_syscall(domain::syscall::Syscall::CreateProcess { create_cap: CapSlot::new(0).unwrap(), image: DataRef::empty(), }) { let pid = k.process_of_cap_for_test(cap); for _ in 0..2 { k.spawn_thread_internal(pid); } members.push(pid); } else if std::env::var("STORM_DEBUG").is_ok() { std::eprintln!( "create child failed; init thread state {:?}", k.thread_state_for_test(init_thread) ); } } for pid in members { for ep in [e0, e1] { let _ = k.grant_capability(pid, Resource::Endpoint(ep), all); } let _ = k.grant_capability(pid, Resource::Notification(n), all); if let Some(r) = region { let _ = k.grant_capability(pid, Resource::Memory(r), all); } } } /// The slots of every region capability `pid` holds. fn region_slots(k: &Kernel, pid: Option) -> std::vec::Vec { let Some(process) = pid.and_then(|p| k.processes.get(p).ok()) else { return std::vec::Vec::new(); }; process .capabilities() .iter() .filter(|(_, c)| matches!(c.resource(), domain::resource::Resource::Memory(_))) .map(|(s, _)| s) .collect() } /// What the adapter does whenever a core is left without a thread: run the /// next one in its queue, if any. fn resume(k: &mut Kernel, core: types::id::CoreId) { if k.current_thread().is_none() && let Some(next) = k.pick_next_thread() { k.set_thread_running(next, core).unwrap(); } } #[test] fn the_kernel_survives_a_syscall_storm() { storm(1); } /// The same storm with threads homed on four cores. Each step runs on one /// core, as the thread that core is running; the kernel is still entered /// one call at a time (stage 1 takes a single lock), but threads on other /// cores are live meanwhile and IPIs arrive late. #[test] fn the_kernel_survives_a_syscall_storm_on_four_cores() { storm(4); } fn storm(cores: u8) { use types::id::CoreId; let mut total = [0usize; 36 + 1]; for seed in [17u64, 0xABCD_EF01, 4242, 0x1357_9BDF, 8] { let mut k = Kernel::new( MockPlatform::new(cores), types::address::PhysicalAddress::from_raw(0x8400_0000), 1024, ); let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap(); let init_thread = k.current_thread().unwrap(); k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT); let mut rng = Rng(seed); let mut executed = 0; let mut rounds = 0; let mut succeeded = [0usize; 36 + 1]; new_round(&mut k, init, init_thread); let mut step = 0; while executed < 4000 && step < 200_000 { step += 1; let runnable: std::vec::Vec = k .live_threads() .filter(|t| { matches!( k.thread_state_for_test(*t), Some(ThreadState::Ready | ThreadState::Running(_)) ) }) .collect(); if runnable.is_empty() || rng.below(300) == 0 { // Kill everyone but init, hub included: destroying the shared // objects wakes whoever was blocked on them. for p in 1..domain::process::MAX_PROCESSES as u16 { k.destroy_process(ProcessId::from_raw(p)); } // init may be blocked on objects it made itself. k.destroy_endpoints_owned_by(init); k.destroy_notifications_owned_by(init); let here = k.platform_mut().core_id(); resume(&mut k, here); k.check_invariants() .unwrap_or_else(|e| panic!("seed {seed:#x} step {step} after teardown: {e}")); new_round(&mut k, init, init_thread); rounds += 1; continue; } // Pick a core and act as the thread it runs, after it takes any // IPI that has reached it. Now and then switch it to another // ready thread of its own, as a tick or a yield would. let core = CoreId::from_raw(rng.below(cores as usize) as u8); k.platform_mut().set_core(core); if k.platform_mut().take_ipi(core) { let _ = k.reschedule(core); } resume(&mut k, core); let homed: std::vec::Vec = runnable .iter() .copied() .filter(|t| k.home(*t) == core) .collect(); if homed.is_empty() { continue; } let tid = match k.current_thread() { Some(t) if rng.below(3) != 0 => t, _ => homed[rng.below(homed.len())], }; let is_init = k.thread_process(tid) == Some(init); let nr = rng.below(36 + 1); // init exiting ends the test. Halt stays in: nobody here holds // a SystemControl cap, so a halt that got through would panic // the mock platform and fail the test. if is_init && (nr == SYS_EXIT || nr == SYS_EXIT_THREAD) { continue; } let mut args: [usize; 6] = core::array::from_fn(|_| rng.arg()); // Random arguments almost never name a region cap, a free // address and valid perms at once; half the time, pick one of // the caller's region caps (often the hub's shared region) so // regions really are mapped into several processes. if nr == SYS_MAP_REGION && rng.below(2) == 0 { let regions = region_slots(&k, k.thread_process(tid)); if !regions.is_empty() { args[0] = regions[rng.below(regions.len())].value() as usize; args[1] = 0x10000 + rng.below(32) * 4096; // Read-write or read-execute: the pairs W^X forbids. args[2] = [PERMS_READ_WRITE, PERMS_READ_EXECUTE][rng.below(2)]; } } // init keeps the capability it creates processes with. if is_init && nr == SYS_DROP_CAP && args[0] == 0 { continue; } let Some(syscall) = crate::abi::decode(nr, &args) else { continue; }; k.switch_to_thread_for_test(tid); k.take_pending_result(tid); let out = k.handle_syscall(syscall); resume(&mut k, core); executed += 1; if !matches!( out, crate::SyscallOutcome::Complete(crate::SyscallResult::Error(_)) ) { succeeded[nr] += 1; } k.check_invariants() .unwrap_or_else(|e| panic!("seed {seed:#x} step {step} (syscall {nr}): {e}")); assert!( k.process_exists(init), "seed {seed:#x}: init died at step {step}" ); } if std::env::var("STORM_STATS").is_ok() { std::eprintln!( "seed {seed:#x}: executed {executed} rounds {rounds} ok per syscall {succeeded:?}" ); } assert!( executed >= 4000, "seed {seed:#x}: only {executed} syscalls ran" ); assert!( rounds < executed / 4, "seed {seed:#x}: rounds too short ({rounds} rounds)" ); for (t, n) in total.iter_mut().zip(succeeded) { *t += n; } } let exercised = [ SYS_SEND, SYS_RECEIVE, SYS_CALL, SYS_REPLY, SYS_REPLY_RECV, SYS_SIGNAL, SYS_WAIT, SYS_POLL, SYS_BIND_NOTIFICATION, SYS_MINT, SYS_DELEGATE, SYS_REVOKE, SYS_DROP_CAP, SYS_REQUEST_PAGES, SYS_RELEASE_PAGES, SYS_CREATE_REGION, SYS_MAP_REGION, SYS_REGION_ADDRESS, SYS_KILL, SYS_CREATE_PROCESS, SYS_SPAWN_THREAD, SYS_SPAWN_THREAD_IN, ]; for nr in exercised { assert!( total[nr] > 0, "syscall {nr} never succeeded in the whole storm" ); } }