diff --git a/crates/adapter-riscv/src/mmu/mod.rs b/crates/adapter-riscv/src/mmu/mod.rs index a2ba3bf..8c85eb4 100644 --- a/crates/adapter-riscv/src/mmu/mod.rs +++ b/crates/adapter-riscv/src/mmu/mod.rs @@ -4,7 +4,7 @@ pub mod page_table; pub use allocator::init_allocator; pub use page_table::{ PAGE_SIZE, PageTable, Sv39Flags, USER_VA_END, USER_VA_START, activate, active_root, check_wx, - create_user_root, init_kernel_page_table, map_user_page, unmap_user_page, + create_user_root, init_kernel_page_table, map_user_page, translate_user, unmap_user_page, }; /// The kernel lives in the upper half of the address space: all of RAM and diff --git a/crates/adapter-riscv/src/mmu/page_table.rs b/crates/adapter-riscv/src/mmu/page_table.rs index b023102..da4fd52 100644 --- a/crates/adapter-riscv/src/mmu/page_table.rs +++ b/crates/adapter-riscv/src/mmu/page_table.rs @@ -405,6 +405,16 @@ pub fn map_user_page( Ok(()) } +/// The physical page `va` maps to in the address space at `root_pa`. +pub fn translate_user(root_pa: usize, va: usize) -> Option { + if !in_user_window(va) { + return None; + } + let l0 = user_leaf_table(root_pa, va, None).ok()?; + let entry = l0.entries[vpn(va, 0)]; + entry.is_valid().then(|| (entry.ppn() << 12) as usize) +} + pub fn unmap_user_page(root_pa: usize, va: usize) { if !in_user_window(va) { return; diff --git a/crates/adapter-riscv/src/platform.rs b/crates/adapter-riscv/src/platform.rs index 7a268a8..976a47b 100644 --- a/crates/adapter-riscv/src/platform.rs +++ b/crates/adapter-riscv/src/platform.rs @@ -103,6 +103,11 @@ impl Platform for RiscVPlatform { } } + fn translate(&self, process: ProcessId, va: VirtualAddress) -> Option { + let root = self.root(process)?; + mmu::translate_user(root, va.raw()).map(PhysicalAddress::from_raw) + } + fn zero_frame(&mut self, pa: PhysicalAddress) { // SAFETY: pool frames are RAM, mapped at `phys_to_virt`, and the // kernel only zeroes frames it has just allocated. diff --git a/crates/domain/src/capability/mod.rs b/crates/domain/src/capability/mod.rs index 09bfcc5..b6c656b 100644 --- a/crates/domain/src/capability/mod.rs +++ b/crates/domain/src/capability/mod.rs @@ -5,11 +5,16 @@ use types::{ pub mod authority; pub mod derivation; +pub mod tree; #[cfg(test)] #[path = "derivation_tests.rs"] mod derivation_tests; +#[cfg(test)] +#[path = "tree_tests.rs"] +mod tree_tests; + use crate::{error::KernelError, generations::SlotGenerations, resource::Resource}; #[cfg(test)] diff --git a/crates/domain/src/capability/tree.rs b/crates/domain/src/capability/tree.rs new file mode 100644 index 0000000..952c5f5 --- /dev/null +++ b/crates/domain/src/capability/tree.rs @@ -0,0 +1,214 @@ +//! The capability derivation tree with explicit links, so deleting a +//! capability costs its number of children and revoking costs the size of +//! the subtree, instead of a scan of every capability in the system. +//! +//! Nodes are numbered `0..N` (the kernel flattens process and slot). Each +//! live node knows its parent, its first child and its siblings. The +//! fixpoint in `derivation::descendants` stays as the reference this tree +//! is checked against. + +/// Longest walk any operation may take: every node visited once, plus the +/// climb back up. Corrupted links can therefore never hang the kernel. +const fn budget(n: usize) -> usize { + 2 * n + 1 +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +struct Node { + live: bool, + parent: Option, + first_child: Option, + prev: Option, + next: Option, +} + +pub struct CapTree { + nodes: [Node; N], +} + +impl Default for CapTree { + fn default() -> Self { + Self::new() + } +} + +fn id(i: usize) -> Option { + u16::try_from(i).ok() +} + +impl CapTree { + pub fn new() -> Self { + Self { + nodes: [Node::default(); N], + } + } + + fn node(&self, i: usize) -> Option<&Node> { + self.nodes.get(i).filter(|n| n.live) + } + + fn node_mut(&mut self, i: usize) -> Option<&mut Node> { + self.nodes.get_mut(i).filter(|n| n.live) + } + + pub fn is_live(&self, i: usize) -> bool { + self.node(i).is_some() + } + + pub fn parent(&self, i: usize) -> Option { + self.node(i)?.parent.map(usize::from) + } + + /// Adds `i` as a child of `parent` (or as a root). Fails, changing + /// nothing, if `i` is already live or out of range, or `parent` is not + /// live. + pub fn attach(&mut self, i: usize, parent: Option) -> bool { + let (Some(me), Some(node)) = (id(i), self.nodes.get(i)) else { + return false; + }; + if node.live || parent.is_some_and(|p| p == i || !self.is_live(p)) { + return false; + } + let next = parent + .and_then(|p| self.node(p)) + .and_then(|p| p.first_child); + if let Some(n) = next.and_then(|n| self.node_mut(n.into())) { + n.prev = Some(me); + } + if let Some(p) = parent.and_then(|p| self.node_mut(p)) { + p.first_child = Some(me); + } + if let Some(node) = self.nodes.get_mut(i) { + *node = Node { + live: true, + parent: parent.and_then(id), + first_child: None, + prev: None, + next, + }; + } + true + } + + /// Unlinks `i` from its parent's child list, leaving its own children. + fn unlink(&mut self, i: usize) { + let Some(node) = self.node(i).copied() else { + return; + }; + match node.prev { + Some(p) => { + if let Some(p) = self.node_mut(p.into()) { + p.next = node.next; + } + } + None => { + if let Some(p) = node.parent.and_then(|p| self.node_mut(p.into())) { + p.first_child = node.next; + } + } + } + if let Some(n) = node.next.and_then(|n| self.node_mut(n.into())) { + n.prev = node.prev; + } + } + + /// Removes `i`; its children are adopted by its parent (or become + /// roots), so no descendant escapes a revocation further up. + pub fn detach(&mut self, i: usize) { + let Some(node) = self.node(i).copied() else { + return; + }; + self.unlink(i); + let mut child = node.first_child; + for _ in 0..budget(N) { + let Some(c) = child else { break }; + let next = self.node(c.into()).and_then(|n| n.next); + if let Some(n) = self.node_mut(c.into()) { + n.parent = None; + n.prev = None; + n.next = None; + } + // Re-attach under the grandparent; `attach` needs a dead node. + if let Some(n) = self.nodes.get_mut(usize::from(c)) { + n.live = false; + let kids = n.first_child; + if self.attach(c.into(), node.parent.map(usize::from)) + && let Some(n) = self.nodes.get_mut(usize::from(c)) + { + n.first_child = kids; + } + } + child = next; + } + if let Some(n) = self.nodes.get_mut(i) { + *n = Node::default(); + } + } + + /// The direct children of `i`, written into `out`; returns how many. + pub fn children(&self, i: usize, out: &mut [u16; N]) -> usize { + let mut count = 0; + let mut cur = self.node(i).and_then(|n| n.first_child); + for _ in 0..budget(N) { + let Some(c) = cur else { break }; + if let Some(slot) = out.get_mut(count) { + *slot = c; + count += 1; + } + cur = self.node(c.into()).and_then(|n| n.next); + } + count + } + + /// Every node strictly below `root`, written into `out` in pre-order; + /// returns how many. + pub fn descendants(&self, root: usize, out: &mut [u16; N]) -> usize { + let mut count = 0; + let Some(start) = self.node(root) else { + return 0; + }; + let mut cur = start.first_child; + for _ in 0..budget(N) { + let Some(c) = cur else { break }; + if let Some(slot) = out.get_mut(count) { + *slot = c; + count += 1; + } + cur = self.next_in_subtree(root, c.into()); + } + count + } + + /// The node after `cur` in a pre-order walk of `root`'s subtree. + fn next_in_subtree(&self, root: usize, cur: usize) -> Option { + if let Some(child) = self.node(cur)?.first_child { + return Some(child); + } + let mut at = cur; + for _ in 0..budget(N) { + if at == root { + return None; + } + let node = self.node(at)?; + if let Some(next) = node.next { + return Some(next); + } + at = node.parent?.into(); + } + None + } + + /// Removes every node strictly below `root`, keeping `root`. + pub fn remove_descendants(&mut self, root: usize) { + let mut doomed = [0u16; N]; + let n = self.descendants(root, &mut doomed); + for d in doomed.iter().take(n) { + if let Some(node) = self.nodes.get_mut(usize::from(*d)) { + *node = Node::default(); + } + } + if let Some(r) = self.node_mut(root) { + r.first_child = None; + } + } +} diff --git a/crates/domain/src/capability/tree_tests.rs b/crates/domain/src/capability/tree_tests.rs new file mode 100644 index 0000000..e48d652 --- /dev/null +++ b/crates/domain/src/capability/tree_tests.rs @@ -0,0 +1,120 @@ +use super::derivation::descendants; +use super::tree::CapTree; + +const N: usize = 12; + +/// The tree as a plain parent array, updated by the rules the tree must +/// follow; `descendants` from derivation.rs answers subtree questions. +struct Model { + live: [bool; N], + parents: [Option; N], +} + +impl Model { + fn detach(&mut self, i: usize) { + let grandparent = self.parents[i]; + for p in self.parents.iter_mut() { + if *p == Some(i as u16) { + *p = grandparent; + } + } + self.live[i] = false; + self.parents[i] = None; + } + + fn below(&self, root: usize) -> [bool; N] { + let mut d = descendants(&self.parents, root); + d[root] = false; + d + } +} + +struct Rng(u64); +impl Rng { + fn below(&mut self, n: usize) -> usize { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + (self.0 % n as u64) as usize + } +} + +fn check(t: &CapTree, m: &Model) { + for i in 0..N { + assert_eq!(t.is_live(i), m.live[i], "liveness of {i}"); + assert_eq!(t.parent(i), m.parents[i].map(usize::from), "parent of {i}"); + if !m.live[i] { + continue; + } + let mut out = [0u16; N]; + let n = t.descendants(i, &mut out); + let mut got = [false; N]; + for d in &out[..n] { + assert!(!got[*d as usize], "{d} listed twice under {i}"); + got[*d as usize] = true; + } + assert_eq!(got, m.below(i), "subtree of {i}"); + } +} + +#[test] +fn matches_the_parent_array_model_under_random_operations() { + for seed in 1..200u64 { + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + let mut t = CapTree::::new(); + let mut m = Model { + live: [false; N], + parents: [None; N], + }; + for _ in 0..300 { + let i = rng.below(N); + match rng.below(4) { + 0 | 1 => { + let parent = match rng.below(3) { + 0 => None, + _ => Some(rng.below(N)), + }; + let ok = !m.live[i] && parent.is_none_or(|p| p != i && m.live[p]); + assert_eq!(t.attach(i, parent), ok, "attach {i} under {parent:?}"); + if ok { + m.live[i] = true; + m.parents[i] = parent.map(|p| p as u16); + } + } + 2 => { + t.detach(i); + if m.live[i] { + m.detach(i); + } + } + _ => { + t.remove_descendants(i); + if m.live[i] { + let below = m.below(i); + for (j, b) in below.iter().enumerate() { + if *b { + m.live[j] = false; + m.parents[j] = None; + } + } + } + } + } + check(&t, &m); + } + } +} + +#[test] +fn detaching_a_middle_node_hands_its_children_up() { + let mut t = CapTree::<4>::new(); + assert!(t.attach(0, None)); + assert!(t.attach(1, Some(0))); + assert!(t.attach(2, Some(1))); + assert!(t.attach(3, Some(1))); + t.detach(1); + assert_eq!(t.parent(2), Some(0)); + assert_eq!(t.parent(3), Some(0)); + let mut out = [0u16; 4]; + assert_eq!(t.descendants(0, &mut out), 2); +} diff --git a/crates/domain/src/endpoint/mod.rs b/crates/domain/src/endpoint/mod.rs index 3dedec2..2170e43 100644 --- a/crates/domain/src/endpoint/mod.rs +++ b/crates/domain/src/endpoint/mod.rs @@ -154,13 +154,6 @@ impl EndpointTable { self.generations.advance(id.raw() as usize); Some(ep) } - - pub fn remove_thread_from_all(&mut self, thread: ThreadId) { - for ep in self.endpoints.iter_mut().flatten() { - ep.remove_sender(thread); - ep.remove_receiver(thread); - } - } } impl Default for EndpointTable { diff --git a/crates/domain/src/memory/frame/mod.rs b/crates/domain/src/memory/frame/mod.rs index 571261f..77f44e5 100644 --- a/crates/domain/src/memory/frame/mod.rs +++ b/crates/domain/src/memory/frame/mod.rs @@ -50,6 +50,8 @@ pub struct FrameTable { total: usize, frames: [Option; N], free: usize, + /// Where the next allocation starts looking: just past the last one. + cursor: usize, } impl FrameTable { @@ -60,6 +62,7 @@ impl FrameTable { total, frames: [None; N], free: total, + cursor: 0, } } @@ -101,14 +104,21 @@ impl FrameTable { self.frames.get(self.index_of(pa)?)?.as_ref() } + /// Next-fit: searches from just past the previous allocation and wraps + /// once, so a run of allocations costs O(1) each instead of rescanning + /// the allocated prefix every time. pub fn alloc(&mut self, usage: FrameUse) -> Option { - let (idx, slot) = self - .live_mut() - .iter_mut() - .enumerate() - .find(|(_, f)| f.is_none())?; - *slot = Some(usage); + if self.free == 0 { + return None; + } + let total = self.total; + let start = self.cursor.min(total); + let idx = (start..total) + .chain(0..start) + .find(|&i| self.frames.get(i).is_some_and(Option::is_none))?; + *self.frames.get_mut(idx)? = Some(usage); self.free -= 1; + self.cursor = idx + 1; Some(self.address_of(idx)) } @@ -168,6 +178,29 @@ impl FrameTable { } /// The frame mapped at `va` in `owner`, if `owner` holds it as a user page. + /// The perms of the frame at `pa` if it is `owner`'s user page at + /// `va`; `None` for any other frame. O(1): the caller supplies the + /// frame, this only confirms it. + pub fn user_at( + &self, + pa: PhysicalAddress, + owner: ProcessId, + va: VirtualAddress, + ) -> Option { + let page = u32::try_from(va.raw() / PAGE_SIZE).ok()?; + if !va.raw().is_multiple_of(PAGE_SIZE) { + return None; + } + match self.get(pa)? { + FrameUse::User { + owner: o, + page: p, + perms, + } if *o == owner && *p == page => Some(*perms), + _ => None, + } + } + pub fn find_user( &self, owner: ProcessId, diff --git a/crates/domain/src/memory/frame/proofs.rs b/crates/domain/src/memory/frame/proofs.rs index 5fb0bb9..c48cbaf 100644 --- a/crates/domain/src/memory/frame/proofs.rs +++ b/crates/domain/src/memory/frame/proofs.rs @@ -47,6 +47,8 @@ fn any_table() -> FrameTable { total: N, frames, free, + // Next-fit may resume anywhere, so the proofs start it anywhere. + cursor: kani::any::() % (N + 1), } } diff --git a/crates/domain/src/notification/mod.rs b/crates/domain/src/notification/mod.rs index 6dee2be..4c9106d 100644 --- a/crates/domain/src/notification/mod.rs +++ b/crates/domain/src/notification/mod.rs @@ -116,6 +116,11 @@ impl Notification { self.unbind(thread); } + /// Takes `thread` out of the waiters, leaving any binding. + pub fn remove_waiter(&mut self, thread: ThreadId) { + self.waiters.remove_thread(thread); + } + /// Drains every waiter, e.g. when the notification is destroyed. pub fn pop_waiter(&mut self) -> Option { self.waiters.pop() @@ -177,12 +182,6 @@ impl NotificationTable { .filter(move |n| n.owner == owner) .map(|n| n.id) } - - pub fn remove_thread_from_all(&mut self, thread: ThreadId) { - for n in self.slots.iter_mut().flatten() { - n.remove_thread(thread); - } - } } impl Default for NotificationTable { diff --git a/crates/kernel/src/captree.rs b/crates/kernel/src/captree.rs index fae69b3..f7e773c 100644 --- a/crates/kernel/src/captree.rs +++ b/crates/kernel/src/captree.rs @@ -2,9 +2,15 @@ //! Delegate, IPC transfer) records its parent; `Revoke` deletes a subtree //! and deleting a capability re-parents its children, so the tree can only //! shrink by explicit deletion, never by a middleman stepping out. +//! +//! Each capability records its parent (`Capability::parent`), and +//! `Kernel::tree` holds the same relation as explicit links, so deletion +//! and revocation cost the subtree, not a scan of every capability. Every +//! insertion and removal of a capability goes through this module, which +//! keeps the two in step. use domain::{ - capability::{CapRef, Capability, MAX_CAPS_PER_PROCESS, derivation::descendants}, + capability::{CapRef, Capability, MAX_CAPS_PER_PROCESS}, error::KernelError, process::MAX_PROCESSES, }; @@ -17,7 +23,7 @@ use types::{ use crate::Kernel; /// Every capability slot in the system, flattened. -const ALL_CAPS: usize = MAX_PROCESSES * MAX_CAPS_PER_PROCESS; +pub(crate) const ALL_CAPS: usize = MAX_PROCESSES * MAX_CAPS_PER_PROCESS; fn flat(pid: ProcessId, slot: CapSlot) -> usize { pid.raw() as usize * MAX_CAPS_PER_PROCESS + slot.value() as usize @@ -46,47 +52,55 @@ impl Kernel

{ self.cap_ref(r.process, r.slot) == Some(r) } - /// Stores `cap` in `to`'s table as a child of `parent`. + /// Stores `cap` in `to`'s table as a child of `parent` (a root if + /// `None`). pub(crate) fn insert_derived( &mut self, to: ProcessId, cap: Capability, parent: Option, ) -> Result { - self.processes + let parent = parent.filter(|r| self.resolves(*r)); + let slot = self + .processes .get_mut(to)? .capabilities_mut() - .insert(cap.with_parent(parent)) - } - - /// Calls `f` with every live capability's flat index and reference. - fn for_each_cap(&self, mut f: impl FnMut(usize, CapRef, &Capability)) { - for p in 0..MAX_PROCESSES { - let pid = ProcessId::from_raw(p as u16); - let Ok(process) = self.processes.get(pid) else { - continue; - }; - for (slot, cap) in process.capabilities().iter() { - if let Some(r) = self.cap_ref(pid, slot) { - f(flat(pid, slot), r, cap); - } - } + .insert(cap.with_parent(parent))?; + let linked = self + .tree + .attach(flat(to, slot), parent.map(|r| flat(r.process, r.slot))); + if !linked { + // A slot the tree still thinks is in use: refuse rather than + // let the two disagree. + let _ = self + .processes + .get_mut(to) + .map(|p| p.capabilities_mut().remove(slot)); + return Err(KernelError::NoFreeSlot); } + Ok(slot) } /// Removes every region mapping made through `via`. fn unmap_mappings_via(&mut self, via: CapRef) { - let mut doomed: [Option; domain::memory::MAX_REGIONS] = - [None; domain::memory::MAX_REGIONS]; - for (slot, (id, _)) in doomed.iter_mut().zip( - self.regions - .iter() - .filter(|(_, r)| r.mappings().any(|m| m.via == Some(via))), - ) { - *slot = Some(id); + self.unmap_mappings_where(|r| r == via); + } + + /// Removes every region mapping made through a capability `doomed` + /// picks, in one pass over the regions. + fn unmap_mappings_where(&mut self, doomed: impl Fn(CapRef) -> bool) { + const MAX: usize = domain::memory::MAX_REGIONS * domain::memory::MAX_REGION_MAPPINGS; + let mut found: [Option<(RegionId, ProcessId)>; MAX] = [None; MAX]; + let hits = self.regions.iter().flat_map(|(id, r)| { + r.mappings() + .filter(|m| m.via.is_some_and(&doomed)) + .map(move |m| (id, m.process)) + }); + for (slot, hit) in found.iter_mut().zip(hits) { + *slot = Some(hit); } - for id in doomed.into_iter().flatten() { - self.unmap_region_from(id, via.process); + for (id, pid) in found.into_iter().flatten() { + self.unmap_region_from(id, pid); } } @@ -99,22 +113,17 @@ impl Kernel

{ let grandparent = self.processes.get(pid)?.capabilities().get(slot)?.parent(); self.unmap_mappings_via(me); - let mut children = [false; ALL_CAPS]; - self.for_each_cap(|i, _, cap| { - if cap.parent() == Some(me) - && let Some(c) = children.get_mut(i) - { - *c = true; - } - }); - for (i, _) in children.iter().enumerate().filter(|(_, c)| **c) { - if let Some((cpid, cslot)) = unflat(i) + let mut children = [0u16; ALL_CAPS]; + let n = self.tree.children(flat(pid, slot), &mut children); + for c in children.iter().take(n) { + if let Some((cpid, cslot)) = unflat(usize::from(*c)) && let Ok(p) = self.processes.get_mut(cpid) && let Ok(child) = p.capabilities_mut().get_mut(cslot) { child.set_parent(grandparent); } } + self.tree.detach(flat(pid, slot)); self.processes.get_mut(pid)?.capabilities_mut().remove(slot) } @@ -128,31 +137,30 @@ impl Kernel

{ if self.cap_ref(pid, slot).is_none() { return Err(KernelError::InvalidSlot(slot)); } - // Snapshot the tree as parent indices. A parent that no longer - // resolves cannot exist (deletion re-parents), but is treated as - // a root rather than trusted. - let mut parents: [Option; ALL_CAPS] = [None; ALL_CAPS]; - self.for_each_cap(|i, _, cap| { - if let Some(p) = parents.get_mut(i) { - *p = cap - .parent() - .filter(|r| self.resolves(*r)) - .map(|r| flat(r.process, r.slot) as u16); - } - }); let root = flat(pid, slot); - let marked = descendants(&parents, root); - for (i, _) in marked.iter().enumerate().filter(|(i, m)| **m && *i != root) { - if let Some((dpid, dslot)) = unflat(i) - && let Some(r) = self.cap_ref(dpid, dslot) - { - self.unmap_mappings_via(r); + let mut doomed = [0u16; ALL_CAPS]; + let n = self.tree.descendants(root, &mut doomed); + let mut marked = [false; ALL_CAPS]; + for d in doomed.iter().take(n) { + if let Some(m) = marked.get_mut(usize::from(*d)) { + *m = true; + } + } + self.unmap_mappings_where(|r| { + marked + .get(flat(r.process, r.slot)) + .copied() + .unwrap_or(false) + }); + for d in doomed.iter().take(n) { + if let Some((dpid, dslot)) = unflat(usize::from(*d)) { let _ = self .processes .get_mut(dpid) .map(|p| p.capabilities_mut().remove(dslot)); } } + self.tree.remove_descendants(root); Ok(()) } diff --git a/crates/kernel/src/lib.rs b/crates/kernel/src/lib.rs index 049324b..acae0af 100644 --- a/crates/kernel/src/lib.rs +++ b/crates/kernel/src/lib.rs @@ -68,6 +68,10 @@ pub struct Kernel { pending_results: [Option; MAX_THREADS], /// The core each thread runs on (see smp.rs). homes: [types::id::CoreId; MAX_THREADS], + /// The capability derivation tree as links (see captree.rs). + tree: domain::capability::tree::CapTree<{ captree::ALL_CAPS }>, + /// Where each process's next heap search starts (see memory.rs). + heap_cursors: [usize; MAX_PROCESSES], } impl Kernel

{ @@ -87,6 +91,8 @@ impl Kernel

{ ipc: [syscall::IpcState::default(); MAX_THREADS], pending_results: [const { None }; MAX_THREADS], homes: [types::id::CoreId::from_raw(0); MAX_THREADS], + tree: domain::capability::tree::CapTree::new(), + heap_cursors: [0; MAX_PROCESSES], } } @@ -120,6 +126,8 @@ impl Kernel

{ addr_of_mut!((*slot).ipc).write([syscall::IpcState::default(); MAX_THREADS]); addr_of_mut!((*slot).pending_results).write([const { None }; MAX_THREADS]); addr_of_mut!((*slot).homes).write([types::id::CoreId::from_raw(0); MAX_THREADS]); + addr_of_mut!((*slot).tree).write(domain::capability::tree::CapTree::new()); + addr_of_mut!((*slot).heap_cursors).write([0; MAX_PROCESSES]); } } @@ -141,10 +149,7 @@ impl Kernel

{ PermissionSet::empty().with(Permission::Execute), Generation::initial(), ); - self.processes - .get_mut(pid)? - .capabilities_mut() - .insert(cap)?; + self.insert_derived(pid, cap, None)?; let core = self.platform.core_id(); self.set_home(tid, core); @@ -285,8 +290,14 @@ impl Kernel

{ pub(crate) fn cleanup_threads(&mut self, thread_ids: &[Option]) { use domain::scheduler::Scheduler; for tid in thread_ids.iter().flatten() { - self.endpoints.remove_thread_from_all(*tid); - self.notifications.remove_thread_from_all(*tid); + self.leave_queue(*tid); + if let Some(n) = self + .ipc(*tid) + .and_then(|s| s.bound) + .and_then(|n| self.notifications.get_mut(n)) + { + n.remove_thread(*tid); + } let ipc = self.ipc_mut(*tid).map(core::mem::take).unwrap_or_default(); if let Some(caller) = ipc.reply_to { self.release_caller(caller); @@ -298,6 +309,29 @@ impl Kernel

{ } } + /// Takes a blocked thread off the one queue its block reason names, so + /// nothing needs to scan every endpoint and notification. + pub(crate) fn leave_queue(&mut self, tid: ThreadId) { + use domain::thread::BlockReason::*; + let Ok(ThreadState::Blocked(reason)) = self.threads.get(tid).map(|t| *t.state()) else { + return; + }; + match reason { + Sending(ep) | Calling(ep) => { + self.endpoints.get_mut(ep).map(|e| e.remove_sender(tid)); + } + Receiving(ep) => { + self.endpoints.get_mut(ep).map(|e| e.remove_receiver(tid)); + } + WaitingNotification(n) => { + if let Some(n) = self.notifications.get_mut(n) { + n.remove_waiter(tid); + } + } + AwaitingReply => {} + } + } + /// Destroys every endpoint `owner` created, waking anyone blocked on /// them with an error instead of leaving them blocked forever. pub(crate) fn destroy_endpoints_owned_by(&mut self, owner: ProcessId) { @@ -387,6 +421,6 @@ impl Kernel

{ .resource_generation(&resource) .ok_or(KernelError::NoSuchObject)?; let cap = Capability::new(resource, permissions, generation); - self.processes.get_mut(to)?.capabilities_mut().insert(cap) + self.insert_derived(to, cap, None) } } diff --git a/crates/kernel/src/memory.rs b/crates/kernel/src/memory.rs index 5d0e8b1..80b9056 100644 --- a/crates/kernel/src/memory.rs +++ b/crates/kernel/src/memory.rs @@ -70,8 +70,19 @@ impl Kernel

{ && end <= P::USER_VA_END } + /// `pid`'s user page at `va`: the platform's page table says where it + /// points, the frame table confirms the frame really is that page. + fn user_frame( + &self, + pid: ProcessId, + va: VirtualAddress, + ) -> Option<(PhysicalAddress, PagePerms)> { + let pa = self.platform.translate(pid, va)?; + self.frames.user_at(pa, pid, va).map(|perms| (pa, perms)) + } + fn is_mapped(&self, pid: ProcessId, va: VirtualAddress) -> bool { - self.frames.find_user(pid, va).is_some() || self.regions.covering(pid, va).is_some() + self.user_frame(pid, va).is_some() || self.regions.covering(pid, va).is_some() } fn range_is_free(&self, pid: ProcessId, va: VirtualAddress, pages: usize) -> bool { @@ -173,7 +184,7 @@ impl Kernel

{ /// Removes one user page of `pid` and returns its frame to the pool. fn unmap_and_free(&mut self, pid: ProcessId, va: VirtualAddress) { - if let Some((pa, _)) = self.frames.find_user(pid, va) { + if let Some((pa, _)) = self.user_frame(pid, va) { self.platform.unmap_page(pid, va); self.frames.free(pa); if let Some(q) = self.quotas.get_mut(pid.raw() as usize) { @@ -185,7 +196,7 @@ impl Kernel

{ /// Frame backing the user page at `va` in `pid`. Lets boot code fill /// pages it has just mapped for a process. pub fn user_page_frame(&self, pid: ProcessId, va: VirtualAddress) -> Option { - self.frames.find_user(pid, va).map(|(pa, _)| pa) + self.user_frame(pid, va).map(|(pa, _)| pa) } /// Maps `pages` fresh zeroed pages at `va`, all or nothing. @@ -213,13 +224,35 @@ impl Kernel

{ Ok(()) } - /// First-fit search for `pages` free pages in the heap part of the window. + /// Next-fit search for `pages` free pages in the heap part of the + /// window: from just past `pid`'s previous request, then once more from + /// the bottom. Requests in a row cost O(pages) each instead of + /// rescanning everything allocated before. pub(crate) fn find_free_heap_range( &self, pid: ProcessId, pages: usize, ) -> Option { - let mut candidate = USER_HEAP_START.max(P::USER_VA_START); + let bottom = USER_HEAP_START.max(P::USER_VA_START); + let cursor = self + .heap_cursors + .get(pid.raw() as usize) + .copied() + .unwrap_or(0) + .max(bottom); + self.first_fit_from(pid, cursor, pages) + .or_else(|| self.first_fit_from(pid, bottom, pages)) + } + + /// Records where `pid`'s next heap search starts. + pub(crate) fn advance_heap_cursor(&mut self, pid: ProcessId, past: VirtualAddress) { + if let Some(c) = self.heap_cursors.get_mut(pid.raw() as usize) { + *c = past.raw(); + } + } + + fn first_fit_from(&self, pid: ProcessId, from: usize, pages: usize) -> Option { + let mut candidate = from; while Self::in_user_window(VirtualAddress::from_raw(candidate), pages) { let va = VirtualAddress::from_raw(candidate); match (0..pages).find(|&i| self.is_mapped(pid, page_at(va, i))) { @@ -241,7 +274,7 @@ impl Kernel

{ if !Self::in_user_window(va, pages) { return Err(KernelError::InvalidAddress); } - if !(0..pages).all(|i| self.frames.find_user(pid, page_at(va, i)).is_some()) { + if !(0..pages).all(|i| self.user_frame(pid, page_at(va, i)).is_some()) { return Err(KernelError::InvalidAddress); } for i in 0..pages { @@ -264,8 +297,7 @@ impl Kernel

{ return Err(KernelError::InvalidAddress); } let (pa, _) = self - .frames - .find_user(from, src) + .user_frame(from, src) .ok_or(KernelError::InvalidAddress)?; self.processes.get(to)?; if !self.range_is_free(to, dst, 1) { @@ -478,6 +510,9 @@ impl Kernel

{ if let Some(q) = self.quotas.get_mut(pid.raw() as usize) { *q = Quota::default(); } + if let Some(c) = self.heap_cursors.get_mut(pid.raw() as usize) { + *c = 0; + } } } diff --git a/crates/kernel/src/proofs/platform.rs b/crates/kernel/src/proofs/platform.rs index 71850f0..359550d 100644 --- a/crates/kernel/src/proofs/platform.rs +++ b/crates/kernel/src/proofs/platform.rs @@ -35,6 +35,9 @@ impl Platform for KaniPlatform { Ok(()) } fn unmap_page(&mut self, _process: ProcessId, _va: VirtualAddress) {} + fn translate(&self, _process: ProcessId, _va: VirtualAddress) -> Option { + None + } fn zero_frame(&mut self, _pa: PhysicalAddress) {} fn enable_user_mode(&mut self, _entry: EntryPoint, _stack: StackPointer) -> ! { kani::assume(false); diff --git a/crates/kernel/src/smp.rs b/crates/kernel/src/smp.rs index fe5d8db..19c9245 100644 --- a/crates/kernel/src/smp.rs +++ b/crates/kernel/src/smp.rs @@ -35,17 +35,17 @@ impl Kernel

{ /// Homes a new thread on the core with the fewest live threads. pub(crate) fn place(&mut self, tid: ThreadId) { - let cores = self.scheduler.num_cores().max(1); - let load = |core: CoreId| { - self.threads - .iter() - .filter(|(t, _)| *t != tid && self.home(*t) == core) - .count() - }; + let cores = usize::from(self.scheduler.num_cores().max(1)); + let mut load = [0usize; u8::MAX as usize + 1]; + for (t, _) in self.threads.iter().filter(|(t, _)| *t != tid) { + if let Some(n) = load.get_mut(usize::from(self.home(t).raw())) { + *n += 1; + } + } let best = (0..cores) - .map(CoreId::from_raw) - .min_by_key(|c| load(*c)) - .unwrap_or(CoreId::from_raw(0)); + .min_by_key(|c| load.get(*c).copied().unwrap_or(usize::MAX)) + .and_then(|c| u8::try_from(c).ok()) + .map_or(CoreId::from_raw(0), CoreId::from_raw); self.set_home(tid, best); } diff --git a/crates/kernel/src/syscall/ipc.rs b/crates/kernel/src/syscall/ipc.rs index a71344b..3c333f0 100644 --- a/crates/kernel/src/syscall/ipc.rs +++ b/crates/kernel/src/syscall/ipc.rs @@ -449,7 +449,7 @@ impl Kernel

{ self.wake(t, SyscallResult::Received(Delivery::signal(bits))); } SignalEffect::WakeBound(t, bits) => { - self.endpoints.remove_thread_from_all(t); + self.leave_queue(t); self.wake(t, SyscallResult::Received(Delivery::signal(bits))); } SignalEffect::Stored => {} diff --git a/crates/kernel/src/syscall/memory.rs b/crates/kernel/src/syscall/memory.rs index fdbe299..c7749c2 100644 --- a/crates/kernel/src/syscall/memory.rs +++ b/crates/kernel/src/syscall/memory.rs @@ -27,8 +27,14 @@ impl Kernel

{ self.find_free_heap_range(pid, pages) .ok_or(KernelError::OutOfMemory) .and_then(|va| { - self.map_fresh_range(pid, va, pages, PagePerms::ReadWrite) - .map(|()| SyscallResult::Value(va.raw())) + self.map_fresh_range(pid, va, pages, PagePerms::ReadWrite)?; + self.advance_heap_cursor( + pid, + VirtualAddress::from_raw( + va.raw() + pages * domain::memory::frame::PAGE_SIZE, + ), + ); + Ok(SyscallResult::Value(va.raw())) }), ) } diff --git a/crates/kernel/src/tests/bench.rs b/crates/kernel/src/tests/bench.rs new file mode 100644 index 0000000..5a433ec --- /dev/null +++ b/crates/kernel/src/tests/bench.rs @@ -0,0 +1,183 @@ +//! Worst-case timings of kernel paths that scan tables. Not run by default: +//! `cargo test -p kernel --lib bench -- --ignored --nocapture --test-threads 1`. +//! Numbers are host nanoseconds at the test profile's opt-level, good for +//! comparing before and after a change, not for absolute claims. + +extern crate std; + +use super::*; +use crate::{SyscallOutcome, SyscallResult}; +use domain::{capability::Capability, message::Message, resource::Resource, syscall::Syscall}; +use std::time::Instant; +use types::{ + bounded::{CapSlot, PageCount}, + generation::Generation, + permission::{Permission, PermissionSet}, +}; + +fn all() -> PermissionSet { + PermissionSet::empty() + .with(Permission::Read) + .with(Permission::Write) + .with(Permission::Execute) + .with(Permission::Delegate) +} + +fn report(name: &str, ops: usize, start: Instant) { + let ns = start.elapsed().as_nanos() as f64 / ops as f64; + std::eprintln!("BENCH {name}: {ns:.0} ns/op ({ops} ops)"); +} + +/// `n` processes besides init, each with a full capability table derived +/// from one cap in init, so deleting or revoking walks the whole space. +fn crowded( + n: usize, +) -> ( + Kernel, + ProcessId, + CapSlot, + std::vec::Vec, +) { + let (mut k, init) = kernel_with_process(); + let ep = k.create_endpoint(init).unwrap(); + let root = k.grant_cap_for_test(init, Resource::Endpoint(ep), all()); + let parent = k.cap_ref(init, root); + let mut pids = std::vec::Vec::new(); + for _ in 0..n { + let Some(pid) = k.create_process_internal() else { + break; + }; + while k + .insert_derived( + pid, + Capability::new(Resource::Endpoint(ep), all(), Generation::initial()), + parent, + ) + .is_ok() + {} + pids.push(pid); + } + (k, init, root, pids) +} + +#[test] +#[ignore] +fn bench_destroy_crowded_processes() { + let (mut k, _, _, pids) = crowded(60); + let start = Instant::now(); + for pid in &pids { + k.destroy_process(*pid); + } + report("destroy process with 64 derived caps", pids.len(), start); +} + +#[test] +#[ignore] +fn bench_revoke_whole_space() { + let mut total = 0; + let mut runs = 0; + for _ in 0..5 { + let (mut k, init, root, pids) = crowded(60); + let start = Instant::now(); + k.revoke_subtree(init, root).unwrap(); + total += start.elapsed().as_nanos(); + runs += 1; + assert!( + pids.iter() + .all(|p| k.cap_slots_for_test(*p).iter().all(|s| !s)) + ); + } + std::eprintln!( + "BENCH revoke ~3800 caps across 60 processes: {:.0} ns/op ({runs} ops)", + total as f64 / runs as f64 + ); +} + +#[test] +#[ignore] +fn bench_drop_cap_in_crowded_space() { + let (mut k, _, _, pids) = crowded(60); + let victim = pids[0]; + let start = Instant::now(); + let mut ops = 0; + for slot in 0..=CapSlot::MAX { + if k.delete_cap(victim, CapSlot::new(slot).unwrap()).is_ok() { + ops += 1; + } + } + report("delete one cap among ~3800", ops, start); +} + +#[test] +#[ignore] +fn bench_request_pages_until_full() { + let (mut k, _) = kernel_with_process(); + let start = Instant::now(); + let mut ops = 0; + while let SyscallOutcome::Complete(SyscallResult::Value(_)) = + k.handle_syscall(Syscall::RequestPages { + count: PageCount::new(1).unwrap(), + }) + { + ops += 1; + } + report( + "RequestPages(1) until the pool or quota runs out", + ops, + start, + ); +} + +#[test] +#[ignore] +fn bench_ipc_round_trip() { + let (mut k, init) = kernel_with_process(); + let client = k.current_thread().unwrap(); + let server = k.spawn_thread_for_test(init, Priority::DEFAULT, TimeSlice::DEFAULT); + let ep = k.create_endpoint(init).unwrap(); + let slot = k.grant_cap_for_test(init, Resource::Endpoint(ep), all()); + + k.switch_to_thread_for_test(server); + assert!(matches!( + k.handle_syscall(Syscall::Receive { endpoint: slot }), + SyscallOutcome::BlockCaller(_) + )); + let _ = k + .pick_next_thread() + .map(|t| k.set_thread_running(t, types::id::CoreId::from_raw(0))); + assert_eq!(k.current_thread(), Some(client)); + + let rounds = 20_000; + let start = Instant::now(); + for _ in 0..rounds { + let out = k.handle_syscall(Syscall::Call { + endpoint: slot, + message: Message::empty(), + }); + assert!(matches!(out, SyscallOutcome::SwitchTo(t) if t == server)); + let out = k.handle_syscall(Syscall::ReplyRecv { + endpoint: slot, + message: Message::empty(), + }); + assert!(matches!(out, SyscallOutcome::BlockCaller(_))); + let next = k.pick_next_thread().unwrap(); + k.set_thread_running(next, types::id::CoreId::from_raw(0)) + .unwrap(); + assert_eq!(next, client); + } + report("Call + ReplyRecv round trip", rounds, start); +} + +#[test] +#[ignore] +fn bench_create_spawn_kill() { + let (mut k, _) = kernel_with_process(); + let rounds = 2_000; + let start = Instant::now(); + for _ in 0..rounds { + let pid = k.create_process_internal().unwrap(); + k.spawn_thread_internal(pid).unwrap(); + k.destroy_process(pid); + } + report("create process + spawn thread + destroy", rounds, start); +} diff --git a/crates/kernel/src/tests/captree_model.rs b/crates/kernel/src/tests/captree_model.rs index bb9de30..dbd8517 100644 --- a/crates/kernel/src/tests/captree_model.rs +++ b/crates/kernel/src/tests/captree_model.rs @@ -93,12 +93,17 @@ impl Model { } } + /// Sorted, so a seed always means the same run (HashMap order is + /// random per process). fn held_by(&self, pid: ProcessId) -> Vec<(usize, CapSlot)> { - self.alive + let mut held: Vec<(usize, CapSlot)> = self + .alive .iter() .filter(|(_, (p, _))| *p == pid) .map(|(id, (_, s))| (*id, *s)) - .collect() + .collect(); + held.sort_by_key(|(id, _)| *id); + held } } @@ -302,7 +307,14 @@ fn newly_used(before: [bool; 64], after: [bool; 64]) -> CapSlot { #[test] fn derivation_tree_matches_the_model() { - for seed in [5u64, 0xBADC0DE, 31337, 0x5EED, 2024] { + let extra: u64 = std::env::var("MODEL_SEEDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0); + for seed in [5u64, 0xBADC0DE, 31337, 0x5EED, 2024] + .into_iter() + .chain(1000..1000 + extra) + { let mut world = World::new(); let mut rng = Rng(seed); let mut revokes = 0; @@ -314,6 +326,9 @@ fn derivation_tree_matches_the_model() { } world.check(); } + if seed >= 1000 { + continue; // extra seeds hunt for mismatches, not coverage + } assert!( revokes > 20, "seed {seed:#x}: too little deletion exercised" diff --git a/crates/kernel/src/tests/memory.rs b/crates/kernel/src/tests/memory.rs index 6c1a6b5..fed799f 100644 --- a/crates/kernel/src/tests/memory.rs +++ b/crates/kernel/src/tests/memory.rs @@ -1,8 +1,10 @@ use super::*; use crate::{SyscallOutcome, SyscallResult}; use domain::syscall::Syscall; +use platform::Platform; use types::address::VirtualAddress; use types::bounded::PageCount; +use types::mapping::PagePerms; #[test] fn request_pages_returns_user_va() { @@ -53,3 +55,78 @@ fn release_pages_returns_the_data_pages() { assert_eq!(k.free_page_count(), free_before + 4); assert_eq!(k.page_usage(pid), used_before - 4); } + +fn request(k: &mut Kernel, pages: u16) -> Option { + match k.handle_syscall(Syscall::RequestPages { + count: PageCount::new(pages).unwrap(), + }) { + SyscallOutcome::Complete(SyscallResult::Value(va)) => Some(va), + _ => None, + } +} + +#[test] +fn a_lying_page_table_cannot_hand_out_another_process_frame() { + // The kernel finds a page through the platform's page table but only + // believes the answer if its own frame table agrees. + let (mut k, a) = kernel_with_process(); + let b = k.create_process_internal().unwrap(); + k.map_fresh_range( + b, + VirtualAddress::from_raw(0x50000), + 1, + PagePerms::ReadWrite, + ) + .unwrap(); + let b_frame = k + .user_page_frame(b, VirtualAddress::from_raw(0x50000)) + .unwrap(); + let pages_before = k.page_usage(b); + + // A's page at 0x40000 is claimed to be B's frame. + k.platform_mut().lie_about(a, 0x40000, b_frame.raw()); + assert_eq!( + k.user_page_frame(a, VirtualAddress::from_raw(0x40000)), + None + ); + let out = k.handle_syscall(Syscall::ReleasePages { + addr: VirtualAddress::from_raw(0x40000), + count: PageCount::new(1).unwrap(), + }); + assert!(matches!( + out, + SyscallOutcome::Complete(SyscallResult::Error(_)) + )); + assert_eq!( + k.user_page_frame(b, VirtualAddress::from_raw(0x50000)), + Some(b_frame) + ); + assert_eq!(k.page_usage(b), pages_before); + + // Nor is B's frame A's under a lie about the very page B uses. + k.platform_mut().lie_about(a, 0x50000, b_frame.raw()); + assert_eq!( + k.user_page_frame(a, VirtualAddress::from_raw(0x50000)), + None + ); + // And a lie about the page number of B's own frame is caught too. + k.platform_mut().lie_about(b, 0x60000, b_frame.raw()); + assert_eq!( + k.user_page_frame(b, VirtualAddress::from_raw(0x60000)), + None + ); +} + +#[test] +fn the_heap_search_wraps_back_to_the_bottom() { + let (mut k, pid) = kernel_with_process(); + let first = request(&mut k, 1).unwrap(); + k.advance_heap_cursor( + pid, + VirtualAddress::from_raw(MockPlatform::USER_VA_END - 4096), + ); + let top = request(&mut k, 1).expect("the last page of the window"); + assert_eq!(top, MockPlatform::USER_VA_END - 4096); + let again = request(&mut k, 2).expect("wraps to the bottom"); + assert!(again < top && again != first); +} diff --git a/crates/kernel/src/tests/mock_platform.rs b/crates/kernel/src/tests/mock_platform.rs index 1b65399..69803ea 100644 --- a/crates/kernel/src/tests/mock_platform.rs +++ b/crates/kernel/src/tests/mock_platform.rs @@ -1,4 +1,7 @@ +extern crate std; + use platform::{FrameSource, MapError, Platform}; +use std::collections::{BTreeMap, BTreeSet}; use types::{ address::{EntryPoint, PhysicalAddress, StackPointer, VirtualAddress}, id::{CoreId, IrqId, ProcessId}, @@ -6,27 +9,10 @@ use types::{ time::{Ticks, TimerDuration}, }; -const MAX_MAPPINGS: usize = 2048; -const MAX_TABLES: usize = 256; const MAX_SPACES: usize = 64; -const MAX_ZEROED: usize = 2048; // Each 2 MiB of user VA needs one leaf table, like Sv39. const TABLE_SPAN: usize = 2 * 1024 * 1024; -#[derive(Clone, Copy)] -struct PageMapping { - process: ProcessId, - va: usize, - pa: usize, - perms: PagePerms, -} - -#[derive(Clone, Copy)] -struct LeafTable { - process: ProcessId, - span: usize, -} - /// Records what the "hardware" would hold so tests can check it against /// the kernel's own bookkeeping. pub struct MockPlatform { @@ -34,9 +20,15 @@ pub struct MockPlatform { core: CoreId, cores: u8, spaces: [bool; MAX_SPACES], - mappings: [Option; MAX_MAPPINGS], - tables: [Option; MAX_TABLES], - zeroed: [Option; MAX_ZEROED], + /// (process, va) -> (pa, perms). Ordered, so iteration is the same + /// on every run. + mappings: BTreeMap<(u16, usize), (usize, PagePerms)>, + /// (process, 2 MiB span) with a leaf table. + tables: BTreeSet<(u16, usize)>, + zeroed: BTreeSet, + /// Answers `translate` gives instead of the truth, to test that the + /// kernel never trusts them. + lies: BTreeMap<(u16, usize), usize>, /// Cores an IPI was sent to and that have not taken it yet. ipis: [bool; 8], } @@ -48,9 +40,10 @@ impl MockPlatform { core: CoreId::from_raw(0), cores: num_cores, spaces: [false; MAX_SPACES], - mappings: [None; MAX_MAPPINGS], - tables: [None; MAX_TABLES], - zeroed: [None; MAX_ZEROED], + mappings: BTreeMap::new(), + tables: BTreeSet::new(), + zeroed: BTreeSet::new(), + lies: BTreeMap::new(), ipis: [false; 8], } } @@ -74,34 +67,25 @@ impl MockPlatform { /// Physical address `va` is mapped to in `process`, as the "hardware" sees it. pub fn mapping(&self, process: ProcessId, va: usize) -> Option { - self.mappings - .iter() - .flatten() - .find(|m| m.process == process && m.va == va) - .map(|m| m.pa) + self.mappings.get(&(process.raw(), va)).map(|(pa, _)| *pa) } /// Every live mapping as (process, va, pa). pub fn mappings(&self) -> impl Iterator + '_ { self.mappings .iter() - .flatten() - .map(|m| (m.process, m.va, m.pa)) + .map(|((p, va), (pa, _))| (ProcessId::from_raw(*p), *va, *pa)) } pub fn perms(&self, process: ProcessId, va: usize) -> Option { self.mappings - .iter() - .flatten() - .find(|m| m.process == process && m.va == va) - .map(|m| m.perms) + .get(&(process.raw(), va)) + .map(|(_, perms)| *perms) } pub fn mapping_count(&self, process: ProcessId) -> usize { self.mappings - .iter() - .flatten() - .filter(|m| m.process == process) + .range((process.raw(), 0)..=(process.raw(), usize::MAX)) .count() } @@ -110,7 +94,12 @@ impl MockPlatform { } pub fn was_zeroed(&self, pa: usize) -> bool { - self.zeroed.contains(&Some(pa)) + self.zeroed.contains(&pa) + } + + /// From now on `translate(process, va)` answers `pa`. + pub fn lie_about(&mut self, process: ProcessId, va: usize, pa: usize) { + self.lies.insert((process.raw(), va), pa); } pub fn set_ticks(&mut self, ticks: Ticks) { @@ -135,16 +124,9 @@ impl Platform for MockPlatform { fn destroy_address_space(&mut self, process: ProcessId) { self.spaces[process.raw() as usize] = false; - for slot in &mut self.mappings { - if slot.is_some_and(|m| m.process == process) { - *slot = None; - } - } - for slot in &mut self.tables { - if slot.is_some_and(|t| t.process == process) { - *slot = None; - } - } + let p = process.raw(); + self.mappings.retain(|(owner, _), _| *owner != p); + self.tables.retain(|(owner, _)| *owner != p); } fn map_page( @@ -163,59 +145,28 @@ impl Platform for MockPlatform { return Err(MapError::InvalidAddress); } let span = va / TABLE_SPAN; - let have_table = self - .tables - .iter() - .flatten() - .any(|t| t.process == process && t.span == span); - if !have_table { + if !self.tables.contains(&(process.raw(), span)) { frames.alloc_table_frame().ok_or(MapError::OutOfFrames)?; - let slot = self - .tables - .iter_mut() - .find(|t| t.is_none()) - .expect("mock table slots"); - *slot = Some(LeafTable { process, span }); - } - let entry = PageMapping { - process, - va, - pa: pa.raw(), - perms, - }; - match self - .mappings - .iter_mut() - .find(|m| m.is_some_and(|m| m.process == process && m.va == va)) - { - Some(existing) => *existing = Some(entry), - None => { - let slot = self - .mappings - .iter_mut() - .find(|m| m.is_none()) - .expect("mock mapping slots"); - *slot = Some(entry); - } + self.tables.insert((process.raw(), span)); } + self.mappings.insert((process.raw(), va), (pa.raw(), perms)); Ok(()) } fn unmap_page(&mut self, process: ProcessId, va: VirtualAddress) { - for slot in &mut self.mappings { - if slot.is_some_and(|m| m.process == process && m.va == va.raw()) { - *slot = None; - } - } + self.mappings.remove(&(process.raw(), va.raw())); + } + + fn translate(&self, process: ProcessId, va: VirtualAddress) -> Option { + self.lies + .get(&(process.raw(), va.raw())) + .copied() + .or_else(|| self.mapping(process, va.raw())) + .map(PhysicalAddress::from_raw) } fn zero_frame(&mut self, pa: PhysicalAddress) { - if self.zeroed.contains(&Some(pa.raw())) { - return; - } - if let Some(slot) = self.zeroed.iter_mut().find(|z| z.is_none()) { - *slot = Some(pa.raw()); - } + self.zeroed.insert(pa.raw()); } fn enable_user_mode(&mut self, _entry: EntryPoint, _stack: StackPointer) -> ! { diff --git a/crates/kernel/src/tests/mod.rs b/crates/kernel/src/tests/mod.rs index 1c29cf2..39a26d1 100644 --- a/crates/kernel/src/tests/mod.rs +++ b/crates/kernel/src/tests/mod.rs @@ -2,6 +2,7 @@ mod helpers; mod mock_platform; mod abi; +mod bench; mod board; mod capability; mod captree_model; diff --git a/crates/kernel/src/tests/storm.rs b/crates/kernel/src/tests/storm.rs index c51934a..3baf54c 100644 --- a/crates/kernel/src/tests/storm.rs +++ b/crates/kernel/src/tests/storm.rs @@ -96,6 +96,31 @@ impl Kernel { } } } + // The tree's links say exactly what the capabilities' parents say. + for i in 0..crate::captree::ALL_CAPS { + let pid = ProcessId::from_raw((i / domain::capability::MAX_CAPS_PER_PROCESS) as u16); + let slot = CapSlot::new((i % domain::capability::MAX_CAPS_PER_PROCESS) as u8).unwrap(); + let cap = self + .processes + .get(pid) + .ok() + .and_then(|p| p.capabilities().get(slot).ok()); + if cap.is_some() != self.tree.is_live(i) { + return Err(format!( + "{pid:?}/{slot:?}: tree and table disagree on liveness" + )); + } + let recorded = cap.and_then(|c| c.parent()).map(|r| { + r.process.raw() as usize * domain::capability::MAX_CAPS_PER_PROCESS + + r.slot.value() as usize + }); + if cap.is_some() && recorded != self.tree.parent(i) { + return Err(format!( + "{pid:?}/{slot:?}: tree parent {:?}, cap parent {recorded:?}", + self.tree.parent(i) + )); + } + } for (_, region) in self.regions.iter() { let has = |want| region.mappings().any(|m| m.perms == want); if has(PagePerms::ReadWrite) && has(PagePerms::ReadExecute) { diff --git a/crates/platform/src/lib.rs b/crates/platform/src/lib.rs index 5c897d5..f325933 100644 --- a/crates/platform/src/lib.rs +++ b/crates/platform/src/lib.rs @@ -50,6 +50,10 @@ pub trait Platform { ) -> Result<(), MapError>; /// Removes the mapping at `va` and flushes it from the TLB. fn unmap_page(&mut self, process: ProcessId, va: VirtualAddress); + /// The frame `va` is mapped to in `process`, if any. Only a lookup: + /// the kernel confirms every answer against its own frame table, so a + /// wrong answer can make the kernel refuse but never trust a frame. + fn translate(&self, process: ProcessId, va: VirtualAddress) -> Option; /// Fills a frame with zeros before it is handed to a process. fn zero_frame(&mut self, pa: PhysicalAddress);