perf(kernel): linked cap tree, translate-based page lookup, next-fit frames/heap, targeted dequeue; host benches
This commit is contained in:
@@ -5,11 +5,16 @@ use types::{
|
||||
|
||||
pub mod authority;
|
||||
pub mod derivation;
|
||||
pub mod tree;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "derivation_tests.rs"]
|
||||
mod derivation_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "tree_tests.rs"]
|
||||
mod tree_tests;
|
||||
|
||||
use crate::{error::KernelError, generations::SlotGenerations, resource::Resource};
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
214
crates/domain/src/capability/tree.rs
Normal file
214
crates/domain/src/capability/tree.rs
Normal file
@@ -0,0 +1,214 @@
|
||||
//! The capability derivation tree with explicit links, so deleting a
|
||||
//! capability costs its number of children and revoking costs the size of
|
||||
//! the subtree, instead of a scan of every capability in the system.
|
||||
//!
|
||||
//! Nodes are numbered `0..N` (the kernel flattens process and slot). Each
|
||||
//! live node knows its parent, its first child and its siblings. The
|
||||
//! fixpoint in `derivation::descendants` stays as the reference this tree
|
||||
//! is checked against.
|
||||
|
||||
/// Longest walk any operation may take: every node visited once, plus the
|
||||
/// climb back up. Corrupted links can therefore never hang the kernel.
|
||||
const fn budget(n: usize) -> usize {
|
||||
2 * n + 1
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
struct Node {
|
||||
live: bool,
|
||||
parent: Option<u16>,
|
||||
first_child: Option<u16>,
|
||||
prev: Option<u16>,
|
||||
next: Option<u16>,
|
||||
}
|
||||
|
||||
pub struct CapTree<const N: usize> {
|
||||
nodes: [Node; N],
|
||||
}
|
||||
|
||||
impl<const N: usize> Default for CapTree<N> {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn id(i: usize) -> Option<u16> {
|
||||
u16::try_from(i).ok()
|
||||
}
|
||||
|
||||
impl<const N: usize> CapTree<N> {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
nodes: [Node::default(); N],
|
||||
}
|
||||
}
|
||||
|
||||
fn node(&self, i: usize) -> Option<&Node> {
|
||||
self.nodes.get(i).filter(|n| n.live)
|
||||
}
|
||||
|
||||
fn node_mut(&mut self, i: usize) -> Option<&mut Node> {
|
||||
self.nodes.get_mut(i).filter(|n| n.live)
|
||||
}
|
||||
|
||||
pub fn is_live(&self, i: usize) -> bool {
|
||||
self.node(i).is_some()
|
||||
}
|
||||
|
||||
pub fn parent(&self, i: usize) -> Option<usize> {
|
||||
self.node(i)?.parent.map(usize::from)
|
||||
}
|
||||
|
||||
/// Adds `i` as a child of `parent` (or as a root). Fails, changing
|
||||
/// nothing, if `i` is already live or out of range, or `parent` is not
|
||||
/// live.
|
||||
pub fn attach(&mut self, i: usize, parent: Option<usize>) -> bool {
|
||||
let (Some(me), Some(node)) = (id(i), self.nodes.get(i)) else {
|
||||
return false;
|
||||
};
|
||||
if node.live || parent.is_some_and(|p| p == i || !self.is_live(p)) {
|
||||
return false;
|
||||
}
|
||||
let next = parent
|
||||
.and_then(|p| self.node(p))
|
||||
.and_then(|p| p.first_child);
|
||||
if let Some(n) = next.and_then(|n| self.node_mut(n.into())) {
|
||||
n.prev = Some(me);
|
||||
}
|
||||
if let Some(p) = parent.and_then(|p| self.node_mut(p)) {
|
||||
p.first_child = Some(me);
|
||||
}
|
||||
if let Some(node) = self.nodes.get_mut(i) {
|
||||
*node = Node {
|
||||
live: true,
|
||||
parent: parent.and_then(id),
|
||||
first_child: None,
|
||||
prev: None,
|
||||
next,
|
||||
};
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Unlinks `i` from its parent's child list, leaving its own children.
|
||||
fn unlink(&mut self, i: usize) {
|
||||
let Some(node) = self.node(i).copied() else {
|
||||
return;
|
||||
};
|
||||
match node.prev {
|
||||
Some(p) => {
|
||||
if let Some(p) = self.node_mut(p.into()) {
|
||||
p.next = node.next;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
if let Some(p) = node.parent.and_then(|p| self.node_mut(p.into())) {
|
||||
p.first_child = node.next;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(n) = node.next.and_then(|n| self.node_mut(n.into())) {
|
||||
n.prev = node.prev;
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes `i`; its children are adopted by its parent (or become
|
||||
/// roots), so no descendant escapes a revocation further up.
|
||||
pub fn detach(&mut self, i: usize) {
|
||||
let Some(node) = self.node(i).copied() else {
|
||||
return;
|
||||
};
|
||||
self.unlink(i);
|
||||
let mut child = node.first_child;
|
||||
for _ in 0..budget(N) {
|
||||
let Some(c) = child else { break };
|
||||
let next = self.node(c.into()).and_then(|n| n.next);
|
||||
if let Some(n) = self.node_mut(c.into()) {
|
||||
n.parent = None;
|
||||
n.prev = None;
|
||||
n.next = None;
|
||||
}
|
||||
// Re-attach under the grandparent; `attach` needs a dead node.
|
||||
if let Some(n) = self.nodes.get_mut(usize::from(c)) {
|
||||
n.live = false;
|
||||
let kids = n.first_child;
|
||||
if self.attach(c.into(), node.parent.map(usize::from))
|
||||
&& let Some(n) = self.nodes.get_mut(usize::from(c))
|
||||
{
|
||||
n.first_child = kids;
|
||||
}
|
||||
}
|
||||
child = next;
|
||||
}
|
||||
if let Some(n) = self.nodes.get_mut(i) {
|
||||
*n = Node::default();
|
||||
}
|
||||
}
|
||||
|
||||
/// The direct children of `i`, written into `out`; returns how many.
|
||||
pub fn children(&self, i: usize, out: &mut [u16; N]) -> usize {
|
||||
let mut count = 0;
|
||||
let mut cur = self.node(i).and_then(|n| n.first_child);
|
||||
for _ in 0..budget(N) {
|
||||
let Some(c) = cur else { break };
|
||||
if let Some(slot) = out.get_mut(count) {
|
||||
*slot = c;
|
||||
count += 1;
|
||||
}
|
||||
cur = self.node(c.into()).and_then(|n| n.next);
|
||||
}
|
||||
count
|
||||
}
|
||||
|
||||
/// Every node strictly below `root`, written into `out` in pre-order;
|
||||
/// returns how many.
|
||||
pub fn descendants(&self, root: usize, out: &mut [u16; N]) -> usize {
|
||||
let mut count = 0;
|
||||
let Some(start) = self.node(root) else {
|
||||
return 0;
|
||||
};
|
||||
let mut cur = start.first_child;
|
||||
for _ in 0..budget(N) {
|
||||
let Some(c) = cur else { break };
|
||||
if let Some(slot) = out.get_mut(count) {
|
||||
*slot = c;
|
||||
count += 1;
|
||||
}
|
||||
cur = self.next_in_subtree(root, c.into());
|
||||
}
|
||||
count
|
||||
}
|
||||
|
||||
/// The node after `cur` in a pre-order walk of `root`'s subtree.
|
||||
fn next_in_subtree(&self, root: usize, cur: usize) -> Option<u16> {
|
||||
if let Some(child) = self.node(cur)?.first_child {
|
||||
return Some(child);
|
||||
}
|
||||
let mut at = cur;
|
||||
for _ in 0..budget(N) {
|
||||
if at == root {
|
||||
return None;
|
||||
}
|
||||
let node = self.node(at)?;
|
||||
if let Some(next) = node.next {
|
||||
return Some(next);
|
||||
}
|
||||
at = node.parent?.into();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Removes every node strictly below `root`, keeping `root`.
|
||||
pub fn remove_descendants(&mut self, root: usize) {
|
||||
let mut doomed = [0u16; N];
|
||||
let n = self.descendants(root, &mut doomed);
|
||||
for d in doomed.iter().take(n) {
|
||||
if let Some(node) = self.nodes.get_mut(usize::from(*d)) {
|
||||
*node = Node::default();
|
||||
}
|
||||
}
|
||||
if let Some(r) = self.node_mut(root) {
|
||||
r.first_child = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
120
crates/domain/src/capability/tree_tests.rs
Normal file
120
crates/domain/src/capability/tree_tests.rs
Normal file
@@ -0,0 +1,120 @@
|
||||
use super::derivation::descendants;
|
||||
use super::tree::CapTree;
|
||||
|
||||
const N: usize = 12;
|
||||
|
||||
/// The tree as a plain parent array, updated by the rules the tree must
|
||||
/// follow; `descendants` from derivation.rs answers subtree questions.
|
||||
struct Model {
|
||||
live: [bool; N],
|
||||
parents: [Option<u16>; N],
|
||||
}
|
||||
|
||||
impl Model {
|
||||
fn detach(&mut self, i: usize) {
|
||||
let grandparent = self.parents[i];
|
||||
for p in self.parents.iter_mut() {
|
||||
if *p == Some(i as u16) {
|
||||
*p = grandparent;
|
||||
}
|
||||
}
|
||||
self.live[i] = false;
|
||||
self.parents[i] = None;
|
||||
}
|
||||
|
||||
fn below(&self, root: usize) -> [bool; N] {
|
||||
let mut d = descendants(&self.parents, root);
|
||||
d[root] = false;
|
||||
d
|
||||
}
|
||||
}
|
||||
|
||||
struct Rng(u64);
|
||||
impl Rng {
|
||||
fn below(&mut self, n: usize) -> usize {
|
||||
self.0 ^= self.0 << 13;
|
||||
self.0 ^= self.0 >> 7;
|
||||
self.0 ^= self.0 << 17;
|
||||
(self.0 % n as u64) as usize
|
||||
}
|
||||
}
|
||||
|
||||
fn check(t: &CapTree<N>, m: &Model) {
|
||||
for i in 0..N {
|
||||
assert_eq!(t.is_live(i), m.live[i], "liveness of {i}");
|
||||
assert_eq!(t.parent(i), m.parents[i].map(usize::from), "parent of {i}");
|
||||
if !m.live[i] {
|
||||
continue;
|
||||
}
|
||||
let mut out = [0u16; N];
|
||||
let n = t.descendants(i, &mut out);
|
||||
let mut got = [false; N];
|
||||
for d in &out[..n] {
|
||||
assert!(!got[*d as usize], "{d} listed twice under {i}");
|
||||
got[*d as usize] = true;
|
||||
}
|
||||
assert_eq!(got, m.below(i), "subtree of {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_the_parent_array_model_under_random_operations() {
|
||||
for seed in 1..200u64 {
|
||||
let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15));
|
||||
let mut t = CapTree::<N>::new();
|
||||
let mut m = Model {
|
||||
live: [false; N],
|
||||
parents: [None; N],
|
||||
};
|
||||
for _ in 0..300 {
|
||||
let i = rng.below(N);
|
||||
match rng.below(4) {
|
||||
0 | 1 => {
|
||||
let parent = match rng.below(3) {
|
||||
0 => None,
|
||||
_ => Some(rng.below(N)),
|
||||
};
|
||||
let ok = !m.live[i] && parent.is_none_or(|p| p != i && m.live[p]);
|
||||
assert_eq!(t.attach(i, parent), ok, "attach {i} under {parent:?}");
|
||||
if ok {
|
||||
m.live[i] = true;
|
||||
m.parents[i] = parent.map(|p| p as u16);
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
t.detach(i);
|
||||
if m.live[i] {
|
||||
m.detach(i);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
t.remove_descendants(i);
|
||||
if m.live[i] {
|
||||
let below = m.below(i);
|
||||
for (j, b) in below.iter().enumerate() {
|
||||
if *b {
|
||||
m.live[j] = false;
|
||||
m.parents[j] = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
check(&t, &m);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detaching_a_middle_node_hands_its_children_up() {
|
||||
let mut t = CapTree::<4>::new();
|
||||
assert!(t.attach(0, None));
|
||||
assert!(t.attach(1, Some(0)));
|
||||
assert!(t.attach(2, Some(1)));
|
||||
assert!(t.attach(3, Some(1)));
|
||||
t.detach(1);
|
||||
assert_eq!(t.parent(2), Some(0));
|
||||
assert_eq!(t.parent(3), Some(0));
|
||||
let mut out = [0u16; 4];
|
||||
assert_eq!(t.descendants(0, &mut out), 2);
|
||||
}
|
||||
@@ -154,13 +154,6 @@ impl EndpointTable {
|
||||
self.generations.advance(id.raw() as usize);
|
||||
Some(ep)
|
||||
}
|
||||
|
||||
pub fn remove_thread_from_all(&mut self, thread: ThreadId) {
|
||||
for ep in self.endpoints.iter_mut().flatten() {
|
||||
ep.remove_sender(thread);
|
||||
ep.remove_receiver(thread);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for EndpointTable {
|
||||
|
||||
@@ -50,6 +50,8 @@ pub struct FrameTable<const N: usize = MAX_FRAMES> {
|
||||
total: usize,
|
||||
frames: [Option<FrameUse>; N],
|
||||
free: usize,
|
||||
/// Where the next allocation starts looking: just past the last one.
|
||||
cursor: usize,
|
||||
}
|
||||
|
||||
impl<const N: usize> FrameTable<N> {
|
||||
@@ -60,6 +62,7 @@ impl<const N: usize> FrameTable<N> {
|
||||
total,
|
||||
frames: [None; N],
|
||||
free: total,
|
||||
cursor: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,14 +104,21 @@ impl<const N: usize> FrameTable<N> {
|
||||
self.frames.get(self.index_of(pa)?)?.as_ref()
|
||||
}
|
||||
|
||||
/// Next-fit: searches from just past the previous allocation and wraps
|
||||
/// once, so a run of allocations costs O(1) each instead of rescanning
|
||||
/// the allocated prefix every time.
|
||||
pub fn alloc(&mut self, usage: FrameUse) -> Option<PhysicalAddress> {
|
||||
let (idx, slot) = self
|
||||
.live_mut()
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
.find(|(_, f)| f.is_none())?;
|
||||
*slot = Some(usage);
|
||||
if self.free == 0 {
|
||||
return None;
|
||||
}
|
||||
let total = self.total;
|
||||
let start = self.cursor.min(total);
|
||||
let idx = (start..total)
|
||||
.chain(0..start)
|
||||
.find(|&i| self.frames.get(i).is_some_and(Option::is_none))?;
|
||||
*self.frames.get_mut(idx)? = Some(usage);
|
||||
self.free -= 1;
|
||||
self.cursor = idx + 1;
|
||||
Some(self.address_of(idx))
|
||||
}
|
||||
|
||||
@@ -168,6 +178,29 @@ impl<const N: usize> FrameTable<N> {
|
||||
}
|
||||
|
||||
/// The frame mapped at `va` in `owner`, if `owner` holds it as a user page.
|
||||
/// The perms of the frame at `pa` if it is `owner`'s user page at
|
||||
/// `va`; `None` for any other frame. O(1): the caller supplies the
|
||||
/// frame, this only confirms it.
|
||||
pub fn user_at(
|
||||
&self,
|
||||
pa: PhysicalAddress,
|
||||
owner: ProcessId,
|
||||
va: VirtualAddress,
|
||||
) -> Option<PagePerms> {
|
||||
let page = u32::try_from(va.raw() / PAGE_SIZE).ok()?;
|
||||
if !va.raw().is_multiple_of(PAGE_SIZE) {
|
||||
return None;
|
||||
}
|
||||
match self.get(pa)? {
|
||||
FrameUse::User {
|
||||
owner: o,
|
||||
page: p,
|
||||
perms,
|
||||
} if *o == owner && *p == page => Some(*perms),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find_user(
|
||||
&self,
|
||||
owner: ProcessId,
|
||||
|
||||
@@ -47,6 +47,8 @@ fn any_table() -> FrameTable<N> {
|
||||
total: N,
|
||||
frames,
|
||||
free,
|
||||
// Next-fit may resume anywhere, so the proofs start it anywhere.
|
||||
cursor: kani::any::<usize>() % (N + 1),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -116,6 +116,11 @@ impl Notification {
|
||||
self.unbind(thread);
|
||||
}
|
||||
|
||||
/// Takes `thread` out of the waiters, leaving any binding.
|
||||
pub fn remove_waiter(&mut self, thread: ThreadId) {
|
||||
self.waiters.remove_thread(thread);
|
||||
}
|
||||
|
||||
/// Drains every waiter, e.g. when the notification is destroyed.
|
||||
pub fn pop_waiter(&mut self) -> Option<ThreadId> {
|
||||
self.waiters.pop()
|
||||
@@ -177,12 +182,6 @@ impl NotificationTable {
|
||||
.filter(move |n| n.owner == owner)
|
||||
.map(|n| n.id)
|
||||
}
|
||||
|
||||
pub fn remove_thread_from_all(&mut self, thread: ThreadId) {
|
||||
for n in self.slots.iter_mut().flatten() {
|
||||
n.remove_thread(thread);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for NotificationTable {
|
||||
|
||||
Reference in New Issue
Block a user