feat(init): watchdog: pinger thread PINGs servers (framework answers, no server code), main kills after 5s unanswered -> crash path; not pinged: terminal, pm (handlers wait for user); give-up drains queued ping; spawntest hang check

crates/apps/spawntest/src/main.rs

@@ -200,6 +200,22 @@ fn restarted_after_crashes() -> Result<(), &'static str> {
     Ok(())
 }
 
+/// A server stuck in a handler is noticed by the watchdog, killed (which
+/// releases its caller) and restarted.
+fn restarted_after_hanging() -> Result<(), &'static str> {
+    let asked = syscall::get_time().raw();
+    // Comes back, with an error, once the hung server is killed.
+    let _ = syscall::request(FLAKY, [flaky::op::HANG, 0, 0, 0]);
+    let hung_ms = (syscall::get_time().raw() - asked) / 10_000;
+    if hung_ms < 4_500 {
+        return Err("the hung server was stopped before the watchdog's deadline");
+    }
+    if flaky()? != 1 {
+        return Err("the hung server was not restarted");
+    }
+    Ok(())
+}
+
 fn check() -> Result<(), &'static str> {
     let mut fs = Storage::attach(STORAGE, STORAGE_BUF_VA).map_err(|_| "cannot attach to storage")?;
     write_a_file(&mut fs)?;
@@ -232,6 +248,9 @@ fn check() -> Result<(), &'static str> {
 
     restarted_after_crashes()?;
     libkos::println!("spawntest: a crashed server is restarted, a crash loop backs off: ok");
+
+    restarted_after_hanging()?;
+    libkos::println!("spawntest: a hung server is noticed, stopped and restarted: ok");
     Ok(())
 }
 

crates/init/src/launch.rs

@@ -47,6 +47,10 @@ pub(crate) struct Launcher<'a> {
     console: Option<CapSlot>,
     /// Signals the death notification's timer bit.
     pub(crate) timer: CapSlot,
+    /// Signals that came in while init waited elsewhere, still to handle.
+    pub(crate) pending: usize,
+    /// When the watchdog's last round of pings started.
+    pub(crate) last_round: u64,
 }
 
 /// What became of starting a program.
@@ -88,6 +92,8 @@ impl<'a> Launcher<'a> {
             supervised: [None; MAX_SUPERVISED],
             console: libkos::start::console(),
             timer,
+            pending: 0,
+            last_round: 0,
         })
     }
 

crates/init/src/lib.rs

@@ -13,6 +13,7 @@ mod selftest;
 mod session;
 mod spawn;
 mod supervise;
+mod watchdog;
 
 use libkos::fs::Storage;
 use libkos::syscall;
@@ -65,6 +66,11 @@ pub fn run() -> ! {
     if !launcher.watch_storage() {
         crate::outln!("init: cannot watch storage");
     }
+    if !watchdog::start() {
+        crate::outln!("init: no watchdog thread; hangs go unnoticed");
+    }
+    // Starts the clock the watchdog runs on.
+    launcher.supervise(0);
 
     // The test image carries the SMP self-test; it and the tests run
     // instead of the desktop, and the last one halts the machine.

crates/init/src/manifest.rs

@@ -58,6 +58,9 @@ pub struct Program {
     /// Its clients keep state in it (a subscription, focus, a handle), so
     /// they are restarted after it is.
     pub restarts_clients: bool,
+    /// Pinged by the watchdog (see init's watchdog.rs): not if a handler of
+    /// its waits for the user, which a ping cannot tell from a hang.
+    pub pinged: bool,
 }
 
 /// Enough for a small program: its image, stack and a little heap.
@@ -85,6 +88,7 @@ const fn server(name: &'static str, file: &'static [u8], limits: Limits) -> Prog
         takes_ctrl_c: false,
         restart: true,
         restarts_clients: false,
+        pinged: true,
     }
 }
 
@@ -103,8 +107,8 @@ pub const SERVERS: &[Program] = &[
         ..server("display", b"bin/display", Limits { pages: 1280, regions: 4, ..SMALL })
     },
     // Text on the display, lines from the keyboard (it takes input focus).
-    // Its clients wait on lines and Ctrl+C.
+    // Its clients wait on lines and Ctrl+C; `READ` waits for the user.
-    Program { uses: &["display"], restarts_clients: true, ..server("terminal", b"bin/terminal", SMALL) },
+    Program { uses: &["display"], restarts_clients: true, pinged: false, ..server("terminal", b"bin/terminal", SMALL) },
     server("session", b"bin/session", SMALL),
 ];
 
@@ -133,6 +137,8 @@ pub const SESSION: &[Program] = &[Program {
     spawns: Some(Limits { pages: 512, endpoints: 8, notifications: 8, regions: 8 }),
     // The shell holds handles to the apps it ran, which die with it.
     restarts_clients: true,
+    // Installing waits for the user's consent.
+    pinged: false,
     ..server("permission", b"bin/permission", Limits { pages: 96, ..SMALL })
 }];
 

crates/init/src/supervise.rs

@@ -19,6 +19,7 @@ use types::time::Ticks;
 
 use crate::launch::{Launched, Launcher};
 use crate::manifest::Program;
+use crate::watchdog;
 
 /// Programs watched at once, at most.
 pub(crate) const MAX_SUPERVISED: usize = 12;
@@ -35,6 +36,8 @@ const TICKS_PER_MS: u64 = 10_000;
 const DELAYS_MS: [u64; 4] = [0, 1_000, 5_000, 30_000];
 /// A run this long was not part of a crash loop.
 const STABLE_MS: u64 = 60_000;
+/// How often init looks at the clock for the watchdog.
+const TICK_MS: u64 = 1_000;
 
 /// How many crashes in a row, and so how long before the next restart.
 #[derive(Clone, Copy, Default)]
@@ -76,6 +79,10 @@ pub(crate) struct Supervised {
     spawn: Option<Badge>,
     /// Until what this run held is let go of.
     holding: bool,
+    /// Its endpoint, if it serves one.
+    ep: Option<CapSlot>,
+    /// Killed for hanging: the watchdog's ping is queued on its endpoint.
+    hung: bool,
 }
 
 impl Supervised {
@@ -90,7 +97,7 @@ impl Supervised {
     }
 }
 
-fn now_ms() -> u64 {
+pub(crate) fn now_ms() -> u64 {
     syscall::get_time().raw() / TICKS_PER_MS
 }
 
@@ -105,6 +112,7 @@ impl Launcher<'_> {
         let index = self.supervised.iter().position(Option::is_none).ok_or("too many programs to watch")?;
         let started = self.launch(p)?;
         let note = started.note;
+        let ep = started.ep;
         if !self.watch(started.process, bit(index)) {
             let _ = syscall::kill(started.process);
             self.forget(started);
@@ -118,7 +126,10 @@ impl Launcher<'_> {
             grants: started.grants,
             spawn: started.spawn,
             holding: true,
+            ep,
+            hung: false,
         });
+        self.ping(index);
         Ok(note)
     }
 
@@ -161,8 +172,18 @@ impl Launcher<'_> {
         self.release(grants, spawn);
     }
 
+    /// Has the watchdog ping program `index`, if it is to be pinged.
+    fn ping(&self, index: usize) {
+        if let Some(s) = self.supervised[index]
+            && let Some(ep) = s.ep.filter(|_| s.program.pinged)
+        {
+            watchdog::watch(index, ep);
+        }
+    }
+
     /// Stops a running program on purpose; `restart` it once it is dead.
     fn stop(&mut self, index: usize, restart: bool) {
+        watchdog::unwatch(index);
         let Some(s) = self.supervised[index].as_mut() else { return };
         if let State::Running { process, .. } = s.state {
             let _ = syscall::kill(process);
@@ -187,19 +208,45 @@ impl Launcher<'_> {
 
     /// Death signals and the timer, from init's server loop.
     pub(crate) fn supervise(&mut self, bits: usize) {
-        if bits & STORAGE_BIT != 0 {
+        self.pending |= bits;
-            crate::outln!("init: storage died; nothing can go on without it");
+        while self.pending != 0 {
-            crate::idle();
+            let bits = core::mem::take(&mut self.pending);
-        }
+            if bits & STORAGE_BIT != 0 {
-        for index in 0..MAX_SUPERVISED {
+                crate::outln!("init: storage died; nothing can go on without it");
-            if bits & bit(index) != 0 {
+                crate::idle();
-                self.died(index);
+            }
+            for index in 0..MAX_SUPERVISED {
+                if bits & bit(index) != 0 {
+                    self.died(index);
+                }
             }
         }
+        self.watch_the_clock();
         self.restart_due();
     }
 
+    /// Kills a server whose ping has gone unanswered; starts ping rounds.
+    fn watch_the_clock(&mut self) {
+        let now = now_ms();
+        if let Some(index) = watchdog::hung(now)
+            && let Some(s) = self.supervised[index].as_mut()
+            && let State::Running { process, .. } = s.state
+            && !s.hung
+        {
+            crate::outln!("init: {} stopped answering; restarting it", s.program.name);
+            s.hung = true;
+            watchdog::unwatch(index);
+            // Its death comes back as a crash.
+            let _ = syscall::kill(process);
+        }
+        if now.saturating_sub(self.last_round) >= watchdog::ROUND_MS {
+            self.last_round = now;
+            watchdog::round();
+        }
+    }
+
     fn died(&mut self, index: usize) {
+        watchdog::unwatch(index);
         // Nothing left to let go of if init stopped it.
         self.let_go(index);
         let Some(s) = self.supervised[index] else { return };
@@ -219,15 +266,23 @@ impl Launcher<'_> {
         let mut backoff = s.backoff;
         let delay = restart.and_then(|ran| if crashed { backoff.after_crash(ran) } else { Some(0) });
         match (crashed, delay) {
-            (true, Some(_)) => crate::outln!("init: {} died, restarting it", s.program.name),
+            (true, Some(_)) if !s.hung => crate::outln!("init: {} died, restarting it", s.program.name),
             (true, None) if s.program.restart => crate::outln!("init: {} keeps dying; not restarting it again", s.program.name),
             _ => {}
         }
+        // Nobody will take the ping a hung server left queued: init does.
+        if delay.is_none()
+            && s.hung
+            && let Some(ep) = s.ep.filter(|_| watchdog::pinging(index))
+        {
+            self.pending |= watchdog::release_pinger(ep);
+        }
         self.supervised[index] = delay.map(|d| Supervised {
             state: State::Waiting { at: now_ms() + d },
             backoff,
             grants: [None; 2],
             spawn: None,
+            hung: false,
             ..s
         });
     }
@@ -250,8 +305,10 @@ impl Launcher<'_> {
                         grants: started.grants,
                         spawn: started.spawn,
                         holding: true,
+                        ep: started.ep,
                         ..s
                     });
+                    self.ping(index);
                     if s.program.restarts_clients {
                         self.restart_users_of(s.program.name);
                     }
@@ -268,13 +325,13 @@ impl Launcher<'_> {
                 }
             }
         }
+        // The clock ticks for the watchdog whatever else is due.
         let next = self.supervised.iter().flatten().filter_map(|s| match s.state {
             State::Waiting { at } => Some(at),
             _ => None,
-        }).min();
+        });
-        if let Some(at) = next {
+        let tick = now + TICK_MS;
-            self.wake_at(at);
+        self.wake_at(next.fold(tick, u64::min));
-        }
     }
 
     /// Whoever calls `name` kept state in the old one: restart them too.

crates/init/src/watchdog.rs

@@ -0,0 +1,108 @@
+//! Noticing a server that hangs (see the vision's Watchdog note). A thread
+//! of init's own calls each watched server with `server_framework::PING`
+//! in turn; the framework answers it without the server's code, so only a
+//! server stuck inside a handler (a deadlock, a loop) leaves it unanswered.
+//! Init's main thread, which must never wait on a server, only watches the
+//! clock: a ping unanswered for `HANG_MS` gets the server killed, and the
+//! death is handled like a crash (supervise.rs). The ping stays queued and
+//! the restarted server answers it.
+//!
+//! Servers whose handlers wait for the user (the terminal's `READ`, the
+//! permission manager's consent prompt) cannot be told from hung servers
+//! this way, so they are not pinged.
+
+use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
+use libkos::syscall;
+use types::bounded::CapSlot;
+
+use crate::supervise::MAX_SUPERVISED;
+
+/// Unanswered this long, a server is hung.
+pub(crate) const HANG_MS: u64 = 5_000;
+/// How often a round of pings starts.
+pub(crate) const ROUND_MS: u64 = 2_000;
+const NONE: usize = usize::MAX;
+
+/// Each supervised program's endpoint, if it is pinged.
+static TARGETS: [AtomicUsize; MAX_SUPERVISED] = [const { AtomicUsize::new(NONE) }; MAX_SUPERVISED];
+/// Which one the pinger is waiting on (`NONE`: none), and since when.
+static PINGING: AtomicUsize = AtomicUsize::new(NONE);
+static SINCE_MS: AtomicU64 = AtomicU64::new(0);
+/// The pinger's notification: a signal starts a round.
+static WAKE: AtomicUsize = AtomicUsize::new(NONE);
+
+fn slot(raw: usize) -> Option<CapSlot> {
+    u8::try_from(raw).ok().and_then(CapSlot::new)
+}
+
+/// Starts the pinger.
+pub(crate) fn start() -> bool {
+    let Ok(wake) = syscall::create_notification() else { return false };
+    WAKE.store(wake.value() as usize, Ordering::Release);
+    libkos::thread::spawn(&WAKE, pinger)
+}
+
+pub(crate) fn watch(index: usize, ep: CapSlot) {
+    TARGETS[index].store(ep.value() as usize, Ordering::Release);
+}
+
+pub(crate) fn unwatch(index: usize) {
+    TARGETS[index].store(NONE, Ordering::Release);
+}
+
+/// Has the pinger start a round (it finishes the last one first).
+pub(crate) fn round() {
+    if let Some(wake) = slot(WAKE.load(Ordering::Acquire)) {
+        let _ = syscall::signal(wake);
+    }
+}
+
+/// The program whose ping has gone unanswered too long, if any.
+pub(crate) fn hung(now_ms: u64) -> Option<usize> {
+    let index = PINGING.load(Ordering::Acquire);
+    let since = SINCE_MS.load(Ordering::Acquire);
+    (index != NONE && now_ms.saturating_sub(since) >= HANG_MS).then_some(index)
+}
+
+/// Whether the pinger waits on `index` right now.
+pub(crate) fn pinging(index: usize) -> bool {
+    PINGING.load(Ordering::Acquire) == index
+}
+
+fn pinger(_: &'static AtomicUsize) -> ! {
+    loop {
+        if let Some(wake) = slot(WAKE.load(Ordering::Acquire)) {
+            let _ = syscall::wait(wake);
+        }
+        for (index, target) in TARGETS.iter().enumerate() {
+            let Some(ep) = slot(target.load(Ordering::Acquire)) else { continue };
+            SINCE_MS.store(crate::supervise::now_ms(), Ordering::Release);
+            PINGING.store(index, Ordering::Release);
+            let _ = syscall::request(ep, [server_framework::PING, 0, 0, 0]);
+            PINGING.store(NONE, Ordering::Release);
+        }
+    }
+}
+
+/// Init gave up on a server it killed for hanging: the pinger's ping is
+/// still queued on its endpoint, and nothing will ever take it. Init takes
+/// it (failing anyone else queued there), so the pinger goes on. Signals
+/// that arrive meanwhile are returned, to be handled after.
+pub(crate) fn release_pinger(ep: CapSlot) -> usize {
+    use libkos::syscall::Received;
+    let mut signals = 0;
+    loop {
+        match syscall::receive_msg(ep) {
+            Ok(Received::Call { badge: 0, words, .. }) if words[0] == server_framework::PING => {
+                let _ = syscall::reply([server_framework::PING, 0, 0, 0], None);
+                return signals;
+            }
+            Ok(Received::Call { .. }) => {
+                let _ = syscall::reply([server_framework::MALFORMED, 0, 0, 0], None);
+            }
+            Ok(Received::Message { .. }) => {}
+            Ok(Received::Signal { bits }) => signals |= bits,
+            Err(_) => return signals,
+        }
+    }
+}

crates/server-framework/src/lib.rs

@@ -7,4 +7,4 @@ pub mod server;
 
 pub use codec::{Decode, Encode};
 pub use executor::block_on;
-pub use server::{MALFORMED, Server, serve};
+pub use server::{MALFORMED, PING, Server, serve};

crates/server-framework/src/server/mod.rs

@@ -9,6 +9,11 @@ use types::bounded::CapSlot;
 /// tag, so every client decodes it as an error.
 pub const MALFORMED: usize = usize::MAX;
 
+/// Init's watchdog asking whether the server still serves: answered with
+/// the same word by the loop itself, never seen by the server's code. No
+/// protocol uses it either.
+pub const PING: usize = usize::MAX - 1;
+
 pub trait Server {
     type Request: Decode;
     type Response: Encode;
@@ -37,8 +42,9 @@ pub trait Server {
 
     /// Serves `endpoint` until receiving on it fails (it was destroyed or
     /// the capability revoked), then returns why. Every `Call` gets a
-    /// reply, `[MALFORMED, ..]` if it does not decode; a one-way `Send` is
+    /// reply, `[MALFORMED, ..]` if it does not decode, `[PING, ..]` to a
-    /// handled without one; notification signals go to `signal`.
+    /// `PING`; a one-way `Send` is handled without one; notification
+    /// signals go to `signal`.
     fn run(&mut self, endpoint: CapSlot) -> libkos::syscall::Error {
         use libkos::syscall::{Received, receive_msg, reply_recv};
         let mut next = receive_msg(endpoint);
@@ -48,6 +54,7 @@ pub trait Server {
                 Err(e) => return e,
             };
             next = match message {
+                Received::Call { words, .. } if words[0] == PING => reply_recv(endpoint, [PING, 0, 0, 0], None),
                 Received::Call { badge, words, cap } => {
                     let (reply, give) = self.handle_call(badge, words, cap);
                     reply_recv(endpoint, reply, give)

docs/ipc.md

@@ -136,6 +136,15 @@ it holds the key and every grant. Init's main thread never waits on a
 server it restarts: its own output goes through a thread of its own
 (`init/src/output.rs`), which waits on the log server instead.
 
+Hangs are noticed by pinging (`init/src/watchdog.rs`): another thread of
+init's calls each server with `server_framework::PING` every 2 s, which
+the framework's loop answers without the server's code. A ping
+unanswered for 5 s means the server is stuck in a handler; init kills it,
+and the death is handled as a crash, so the restarted server answers the
+still-queued ping. The terminal and the permission manager are not
+pinged: their handlers wait for the user (`READ`, a consent prompt),
+which a ping cannot tell from a hang.
+
 A program's `uses` endpoints arrive as badged copies with `Execute` and
 `Delegate`: the badge (unique per copy) tells the server which client is
 calling, and `Delegate` lets the client send capabilities with its calls