perf(kernel): linked cap tree, translate-based page lookup, next-fit frames/heap, targeted dequeue; host benches

crates/kernel/src/lib.rs

@@ -68,6 +68,10 @@ pub struct Kernel<P: Platform> {
     pending_results: [Option<SyscallResult>; MAX_THREADS],
     /// The core each thread runs on (see smp.rs).
     homes: [types::id::CoreId; MAX_THREADS],
+    /// The capability derivation tree as links (see captree.rs).
+    tree: domain::capability::tree::CapTree<{ captree::ALL_CAPS }>,
+    /// Where each process's next heap search starts (see memory.rs).
+    heap_cursors: [usize; MAX_PROCESSES],
 }
 
 impl<P: Platform> Kernel<P> {
@@ -87,6 +91,8 @@ impl<P: Platform> Kernel<P> {
             ipc: [syscall::IpcState::default(); MAX_THREADS],
             pending_results: [const { None }; MAX_THREADS],
             homes: [types::id::CoreId::from_raw(0); MAX_THREADS],
+            tree: domain::capability::tree::CapTree::new(),
+            heap_cursors: [0; MAX_PROCESSES],
         }
     }
 
@@ -120,6 +126,8 @@ impl<P: Platform> Kernel<P> {
             addr_of_mut!((*slot).ipc).write([syscall::IpcState::default(); MAX_THREADS]);
             addr_of_mut!((*slot).pending_results).write([const { None }; MAX_THREADS]);
             addr_of_mut!((*slot).homes).write([types::id::CoreId::from_raw(0); MAX_THREADS]);
+            addr_of_mut!((*slot).tree).write(domain::capability::tree::CapTree::new());
+            addr_of_mut!((*slot).heap_cursors).write([0; MAX_PROCESSES]);
         }
     }
 
@@ -141,10 +149,7 @@ impl<P: Platform> Kernel<P> {
             PermissionSet::empty().with(Permission::Execute),
             Generation::initial(),
         );
-        self.processes
-            .get_mut(pid)?
-            .capabilities_mut()
-            .insert(cap)?;
+        self.insert_derived(pid, cap, None)?;
 
         let core = self.platform.core_id();
         self.set_home(tid, core);
@@ -285,8 +290,14 @@ impl<P: Platform> Kernel<P> {
     pub(crate) fn cleanup_threads(&mut self, thread_ids: &[Option<ThreadId>]) {
         use domain::scheduler::Scheduler;
         for tid in thread_ids.iter().flatten() {
-            self.endpoints.remove_thread_from_all(*tid);
-            self.notifications.remove_thread_from_all(*tid);
+            self.leave_queue(*tid);
+            if let Some(n) = self
+                .ipc(*tid)
+                .and_then(|s| s.bound)
+                .and_then(|n| self.notifications.get_mut(n))
+            {
+                n.remove_thread(*tid);
+            }
             let ipc = self.ipc_mut(*tid).map(core::mem::take).unwrap_or_default();
             if let Some(caller) = ipc.reply_to {
                 self.release_caller(caller);
@@ -298,6 +309,29 @@ impl<P: Platform> Kernel<P> {
         }
     }
 
+    /// Takes a blocked thread off the one queue its block reason names, so
+    /// nothing needs to scan every endpoint and notification.
+    pub(crate) fn leave_queue(&mut self, tid: ThreadId) {
+        use domain::thread::BlockReason::*;
+        let Ok(ThreadState::Blocked(reason)) = self.threads.get(tid).map(|t| *t.state()) else {
+            return;
+        };
+        match reason {
+            Sending(ep) | Calling(ep) => {
+                self.endpoints.get_mut(ep).map(|e| e.remove_sender(tid));
+            }
+            Receiving(ep) => {
+                self.endpoints.get_mut(ep).map(|e| e.remove_receiver(tid));
+            }
+            WaitingNotification(n) => {
+                if let Some(n) = self.notifications.get_mut(n) {
+                    n.remove_waiter(tid);
+                }
+            }
+            AwaitingReply => {}
+        }
+    }
+
     /// Destroys every endpoint `owner` created, waking anyone blocked on
     /// them with an error instead of leaving them blocked forever.
     pub(crate) fn destroy_endpoints_owned_by(&mut self, owner: ProcessId) {
@@ -387,6 +421,6 @@ impl<P: Platform> Kernel<P> {
             .resource_generation(&resource)
             .ok_or(KernelError::NoSuchObject)?;
         let cap = Capability::new(resource, permissions, generation);
-        self.processes.get_mut(to)?.capabilities_mut().insert(cap)
+        self.insert_derived(to, cap, None)
     }
 }