feat(kernel): shared ABI decoder+proofs, no-panic gate, syscall storm, kani on shrunk kernel
This commit is contained in:
429
crates/kernel/src/tests/storm.rs
Normal file
429
crates/kernel/src/tests/storm.rs
Normal file
@@ -0,0 +1,429 @@
|
||||
//! Global kernel invariants, and a storm of random syscalls with random
|
||||
//! (mostly garbage) arguments decoded through the real ABI. The kernel must
|
||||
//! never panic and every invariant must hold after every single call.
|
||||
|
||||
extern crate std;
|
||||
|
||||
use std::format;
|
||||
use std::string::String;
|
||||
|
||||
use super::*;
|
||||
use crate::syscall::ThreadRef;
|
||||
use domain::memory::{RegionBacking, frame::FrameUse};
|
||||
use domain::thread::{BlockReason, MAX_THREADS, ThreadState};
|
||||
use platform::Platform;
|
||||
use types::address::DataRef;
|
||||
use types::bounded::CapSlot;
|
||||
use types::id::ThreadId;
|
||||
use types::syscall_abi::*;
|
||||
|
||||
impl Kernel<MockPlatform> {
|
||||
fn live_threads(&self) -> impl Iterator<Item = ThreadId> + '_ {
|
||||
(0..MAX_THREADS as u16)
|
||||
.map(ThreadId::from_raw)
|
||||
.filter(|t| self.threads.get(*t).is_ok())
|
||||
}
|
||||
|
||||
/// Every invariant the kernel promises, over every live object.
|
||||
pub fn check_invariants(&self) -> Result<(), String> {
|
||||
// ---- memory ----
|
||||
let mut charged = 0;
|
||||
for p in 0..domain::process::MAX_PROCESSES as u16 {
|
||||
let pid = ProcessId::from_raw(p);
|
||||
let usage = self.page_usage(pid);
|
||||
charged += usage;
|
||||
let owned = self
|
||||
.frames
|
||||
.iter()
|
||||
.filter(|(_, u)| {
|
||||
matches!(u, FrameUse::User { owner, .. } | FrameUse::PageTable { owner } if *owner == pid)
|
||||
})
|
||||
.count();
|
||||
let regions: usize = self
|
||||
.regions
|
||||
.iter()
|
||||
.filter(|(_, r)| r.owner() == pid && r.backing() == RegionBacking::Pool)
|
||||
.map(|(_, r)| r.pages().value() as usize)
|
||||
.sum();
|
||||
if usage != owned + regions {
|
||||
return Err(format!(
|
||||
"quota of {pid:?} is {usage}, owns {owned}+{regions}"
|
||||
));
|
||||
}
|
||||
if !self.process_exists(pid) && (usage != 0 || self.platform.mapping_count(pid) != 0) {
|
||||
return Err(format!("dead {pid:?} still holds memory"));
|
||||
}
|
||||
}
|
||||
if charged + self.free_page_count() != self.frames.capacity() {
|
||||
return Err(String::from("frames leaked or double counted"));
|
||||
}
|
||||
let window = MockPlatform::USER_VA_START..MockPlatform::USER_VA_END;
|
||||
for (pid, va, pa) in self.platform.mappings() {
|
||||
if !window.contains(&va) {
|
||||
return Err(format!("{pid:?} mapped outside the window at {va:#x}"));
|
||||
}
|
||||
let v = types::address::VirtualAddress::from_raw(va);
|
||||
let user = self.frames.find_user(pid, v).map(|(p, _)| p.raw());
|
||||
let region = self.regions.covering(pid, v).and_then(|id| {
|
||||
let r = self.regions.get(id)?;
|
||||
let m = r.mappings().find(|m| m.process == pid)?;
|
||||
Some(r.base().raw() + (va - m.va.raw()))
|
||||
});
|
||||
if user != Some(pa) && region != Some(pa) {
|
||||
return Err(format!("{pid:?} has unrecorded mapping {va:#x}"));
|
||||
}
|
||||
}
|
||||
for (pa, usage) in self.frames.iter() {
|
||||
if let FrameUse::User { owner, page, .. } = usage
|
||||
&& self.platform.mapping(*owner, *page as usize * 4096) != Some(pa.raw())
|
||||
{
|
||||
return Err(String::from("recorded user page is not mapped"));
|
||||
}
|
||||
}
|
||||
|
||||
// ---- capabilities ----
|
||||
for p in 0..domain::process::MAX_PROCESSES as u16 {
|
||||
let pid = ProcessId::from_raw(p);
|
||||
let Ok(process) = self.processes.get(pid) else {
|
||||
continue;
|
||||
};
|
||||
for (slot, cap) in process.capabilities().iter() {
|
||||
if let Some(parent) = cap.parent()
|
||||
&& !self.resolves(parent)
|
||||
{
|
||||
return Err(format!("{pid:?}/{slot:?} has a dangling parent"));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (_, region) in self.regions.iter() {
|
||||
for m in region.mappings() {
|
||||
if let Some(via) = m.via
|
||||
&& (!self.resolves(via) || via.process != m.process)
|
||||
{
|
||||
return Err(String::from("mapping outlived its capability"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- queues hold only live threads blocked on that very object ----
|
||||
let blocked_as = |t: ThreadId, want: BlockReason| {
|
||||
self.threads.get(t).ok().map(|x| *x.state()) == Some(ThreadState::Blocked(want))
|
||||
};
|
||||
for ep in self.endpoints.iter() {
|
||||
let id = *ep.id();
|
||||
for t in ep.senders() {
|
||||
if !blocked_as(t, BlockReason::Sending(id))
|
||||
&& !blocked_as(t, BlockReason::Calling(id))
|
||||
{
|
||||
return Err(format!(
|
||||
"{t:?} queued as a sender on {id:?} but not blocked there"
|
||||
));
|
||||
}
|
||||
}
|
||||
for t in ep.receivers() {
|
||||
if !blocked_as(t, BlockReason::Receiving(id)) {
|
||||
return Err(format!(
|
||||
"{t:?} queued as a receiver on {id:?} but not blocked there"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
for n in self.notifications.iter() {
|
||||
for t in n.waiters() {
|
||||
if !blocked_as(t, BlockReason::WaitingNotification(n.id())) {
|
||||
return Err(format!(
|
||||
"{t:?} waits on {:?} but is not blocked there",
|
||||
n.id()
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(b) = n.bound()
|
||||
&& self.ipc.get(b.raw() as usize).and_then(|s| s.bound) != Some(n.id())
|
||||
{
|
||||
return Err(format!(
|
||||
"{:?} bound to {b:?} which does not know it",
|
||||
n.id()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// ---- threads, IPC, scheduling ----
|
||||
for t in self.live_threads() {
|
||||
let Ok(thread) = self.threads.get(t) else {
|
||||
continue;
|
||||
};
|
||||
let state = *thread.state();
|
||||
if !self.process_exists(thread.process()) {
|
||||
return Err(format!("{t:?} outlived its process"));
|
||||
}
|
||||
let ipc = self.ipc[t.raw() as usize];
|
||||
let in_endpoint = |e: types::id::EndpointId| {
|
||||
self.endpoints
|
||||
.get(e)
|
||||
.map(|ep| (ep.is_sender(t), ep.is_receiver(t)))
|
||||
};
|
||||
match state {
|
||||
ThreadState::Blocked(BlockReason::Sending(e) | BlockReason::Calling(e)) => {
|
||||
if in_endpoint(e) != Some((true, false)) || ipc.outgoing.is_none() {
|
||||
return Err(format!("{t:?} blocked sending but not queued"));
|
||||
}
|
||||
}
|
||||
ThreadState::Blocked(BlockReason::Receiving(e)) => {
|
||||
if in_endpoint(e) != Some((false, true)) {
|
||||
return Err(format!("{t:?} blocked receiving but not queued"));
|
||||
}
|
||||
}
|
||||
ThreadState::Blocked(BlockReason::WaitingNotification(n)) => {
|
||||
if !self.notifications.get(n).is_some_and(|x| x.is_waiter(t)) {
|
||||
return Err(format!("{t:?} waits on a notification it is not queued on"));
|
||||
}
|
||||
}
|
||||
ThreadState::Blocked(BlockReason::AwaitingReply) => {
|
||||
let me = ThreadRef {
|
||||
tid: t,
|
||||
generation: self
|
||||
.threads
|
||||
.generation(t)
|
||||
.unwrap_or(types::generation::Generation::initial()),
|
||||
};
|
||||
let holders = self
|
||||
.live_threads()
|
||||
.filter(|s| self.ipc[s.raw() as usize].reply_to == Some(me))
|
||||
.count();
|
||||
if holders != 1 {
|
||||
return Err(format!("{t:?} awaits a reply held by {holders} threads"));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if ipc.outgoing.is_some() {
|
||||
return Err(format!("{t:?} runnable with a queued message"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if self.scheduler.is_queued(t) != matches!(state, ThreadState::Ready) {
|
||||
return Err(format!("{t:?} in {state:?} has wrong run-queue membership"));
|
||||
}
|
||||
if let Some(r) = ipc.reply_to
|
||||
&& self.threads.generation(r.tid) == Some(r.generation)
|
||||
&& self.threads.get(r.tid).ok().map(|x| *x.state())
|
||||
!= Some(ThreadState::Blocked(BlockReason::AwaitingReply))
|
||||
{
|
||||
return Err(format!("{t:?} holds a reply for a thread not waiting"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct Rng(u64);
|
||||
|
||||
impl Rng {
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 ^= self.0 << 13;
|
||||
self.0 ^= self.0 >> 7;
|
||||
self.0 ^= self.0 << 17;
|
||||
self.0
|
||||
}
|
||||
|
||||
fn below(&mut self, n: usize) -> usize {
|
||||
(self.next() % n as u64) as usize
|
||||
}
|
||||
|
||||
/// An argument that is usually meaningful and sometimes garbage.
|
||||
fn arg(&mut self) -> usize {
|
||||
match self.below(9) {
|
||||
0..=2 => self.below(8), // small slots and counts
|
||||
8 => self.below(3), // the first few slots
|
||||
3 => NONE,
|
||||
4 => 0x40000 + self.below(64) * 4096, // heap-ish addresses
|
||||
5 => 0x10000 + self.below(32) * 4096,
|
||||
6 => self.below(70), // slots just past the end
|
||||
_ => self.next() as usize, // anything at all
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets up a fresh round: a hub process owning the shared endpoints,
|
||||
/// notification and region; two children with two threads each; and caps
|
||||
/// to all shared objects for init and the children.
|
||||
fn new_round(k: &mut Kernel<MockPlatform>, init: ProcessId, init_thread: ThreadId) {
|
||||
use domain::resource::Resource;
|
||||
let all = types::permission::PermissionSet::empty()
|
||||
.with(types::permission::Permission::Read)
|
||||
.with(types::permission::Permission::Write)
|
||||
.with(types::permission::Permission::Execute)
|
||||
.with(types::permission::Permission::Delegate);
|
||||
// init's caps to anything that died last round would fill its table.
|
||||
for slot in 1..=CapSlot::MAX {
|
||||
let slot = CapSlot::new(slot).unwrap();
|
||||
if k.is_dead_cap_for_test(init, slot) {
|
||||
let _ = k.delete_cap(init, slot);
|
||||
}
|
||||
}
|
||||
let Some(hub) = k.create_process_internal() else {
|
||||
if std::env::var("STORM_DEBUG").is_ok() {
|
||||
std::eprintln!("no hub: free pages {}", k.free_page_count());
|
||||
}
|
||||
return;
|
||||
};
|
||||
let (Ok(e0), Ok(e1), Ok(n)) = (
|
||||
k.create_endpoint(hub),
|
||||
k.create_endpoint(hub),
|
||||
k.create_notification(hub),
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
let region = k
|
||||
.create_pool_region(hub, types::bounded::PageCount::new(2).unwrap())
|
||||
.ok();
|
||||
let mut members = std::vec![init];
|
||||
for _ in 0..2 {
|
||||
k.switch_to_thread_for_test(init_thread);
|
||||
if let crate::SyscallOutcome::Complete(crate::SyscallResult::Slot(cap)) =
|
||||
k.handle_syscall(domain::syscall::Syscall::CreateProcess {
|
||||
create_cap: CapSlot::new(0).unwrap(),
|
||||
image: DataRef::empty(),
|
||||
})
|
||||
{
|
||||
let pid = k.process_of_cap_for_test(cap);
|
||||
for _ in 0..2 {
|
||||
k.spawn_thread_for_test(pid, Priority::DEFAULT, TimeSlice::DEFAULT);
|
||||
}
|
||||
members.push(pid);
|
||||
} else if std::env::var("STORM_DEBUG").is_ok() {
|
||||
std::eprintln!(
|
||||
"create child failed; init thread state {:?}",
|
||||
k.thread_state_for_test(init_thread)
|
||||
);
|
||||
}
|
||||
}
|
||||
for pid in members {
|
||||
for ep in [e0, e1] {
|
||||
let _ = k.grant_capability(pid, Resource::Endpoint(ep), all);
|
||||
}
|
||||
let _ = k.grant_capability(pid, Resource::Notification(n), all);
|
||||
if let Some(r) = region {
|
||||
let _ = k.grant_capability(pid, Resource::Memory(r), all);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_kernel_survives_a_syscall_storm() {
|
||||
let mut total = [0usize; 36];
|
||||
for seed in [17u64, 0xABCD_EF01, 4242, 0x1357_9BDF, 8] {
|
||||
let (mut k, init) = kernel_with_process();
|
||||
let init_thread = k.current_thread().unwrap();
|
||||
k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT);
|
||||
let mut rng = Rng(seed);
|
||||
let mut executed = 0;
|
||||
let mut rounds = 0;
|
||||
let mut succeeded = [0usize; 36];
|
||||
new_round(&mut k, init, init_thread);
|
||||
|
||||
let mut step = 0;
|
||||
while executed < 4000 && step < 200_000 {
|
||||
step += 1;
|
||||
let runnable: std::vec::Vec<ThreadId> = k
|
||||
.live_threads()
|
||||
.filter(|t| {
|
||||
matches!(
|
||||
k.thread_state_for_test(*t),
|
||||
Some(ThreadState::Ready | ThreadState::Running(_))
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
if runnable.is_empty() || rng.below(300) == 0 {
|
||||
// Kill everyone but init, hub included: destroying the shared
|
||||
// objects wakes whoever was blocked on them.
|
||||
for p in 1..domain::process::MAX_PROCESSES as u16 {
|
||||
k.destroy_process(ProcessId::from_raw(p));
|
||||
}
|
||||
// init may be blocked on objects it made itself.
|
||||
k.destroy_endpoints_owned_by(init);
|
||||
k.destroy_notifications_owned_by(init);
|
||||
k.check_invariants()
|
||||
.unwrap_or_else(|e| panic!("seed {seed:#x} step {step} after teardown: {e}"));
|
||||
new_round(&mut k, init, init_thread);
|
||||
rounds += 1;
|
||||
continue;
|
||||
}
|
||||
let tid = runnable[rng.below(runnable.len())];
|
||||
let is_init = k.thread_process(tid) == Some(init);
|
||||
let nr = rng.below(36);
|
||||
// Halting stops the machine and init exiting ends the test.
|
||||
if nr == SYS_HALT || (is_init && (nr == SYS_EXIT || nr == SYS_EXIT_THREAD)) {
|
||||
continue;
|
||||
}
|
||||
let args: [usize; 6] = core::array::from_fn(|_| rng.arg());
|
||||
// init keeps the capability it creates processes with.
|
||||
if is_init && nr == SYS_DROP_CAP && args[0] == 0 {
|
||||
continue;
|
||||
}
|
||||
let Some(syscall) = crate::abi::decode(nr, &args) else {
|
||||
continue;
|
||||
};
|
||||
k.switch_to_thread_for_test(tid);
|
||||
k.take_pending_result(tid);
|
||||
let out = k.handle_syscall(syscall);
|
||||
executed += 1;
|
||||
if !matches!(
|
||||
out,
|
||||
crate::SyscallOutcome::Complete(crate::SyscallResult::Error(_))
|
||||
) {
|
||||
succeeded[nr] += 1;
|
||||
}
|
||||
k.check_invariants()
|
||||
.unwrap_or_else(|e| panic!("seed {seed:#x} step {step} (syscall {nr}): {e}"));
|
||||
assert!(
|
||||
k.process_exists(init),
|
||||
"seed {seed:#x}: init died at step {step}"
|
||||
);
|
||||
}
|
||||
if std::env::var("STORM_STATS").is_ok() {
|
||||
std::eprintln!(
|
||||
"seed {seed:#x}: executed {executed} rounds {rounds} ok per syscall {succeeded:?}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
executed >= 4000,
|
||||
"seed {seed:#x}: only {executed} syscalls ran"
|
||||
);
|
||||
assert!(
|
||||
rounds < executed / 4,
|
||||
"seed {seed:#x}: rounds too short ({rounds} rounds)"
|
||||
);
|
||||
for (t, n) in total.iter_mut().zip(succeeded) {
|
||||
*t += n;
|
||||
}
|
||||
}
|
||||
let exercised = [
|
||||
SYS_SEND,
|
||||
SYS_RECEIVE,
|
||||
SYS_CALL,
|
||||
SYS_REPLY,
|
||||
SYS_REPLY_RECV,
|
||||
SYS_SIGNAL,
|
||||
SYS_WAIT,
|
||||
SYS_POLL,
|
||||
SYS_BIND_NOTIFICATION,
|
||||
SYS_MINT,
|
||||
SYS_DELEGATE,
|
||||
SYS_REVOKE,
|
||||
SYS_DROP_CAP,
|
||||
SYS_REQUEST_PAGES,
|
||||
SYS_RELEASE_PAGES,
|
||||
SYS_CREATE_REGION,
|
||||
SYS_MAP_REGION,
|
||||
SYS_KILL,
|
||||
SYS_CREATE_PROCESS,
|
||||
SYS_SPAWN_THREAD,
|
||||
SYS_SPAWN_THREAD_IN,
|
||||
];
|
||||
for nr in exercised {
|
||||
assert!(
|
||||
total[nr] > 0,
|
||||
"syscall {nr} never succeeded in the whole storm"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user