From 3d69c100795cfb3b44d6078d0a5ffd3272835a5d Mon Sep 17 00:00:00 2001 From: Gabriel Kaszewski Date: Wed, 28 Oct 2026 01:51:39 +0100 Subject: [PATCH] feat(init): watchdog: pinger thread PINGs servers (framework answers, no server code), main kills after 5s unanswered -> crash path; not pinged: terminal, pm (handlers wait for user); give-up drains queued ping; spawntest hang check --- crates/apps/spawntest/src/main.rs | 19 ++++ crates/init/src/launch.rs | 6 ++ crates/init/src/lib.rs | 6 ++ crates/init/src/manifest.rs | 10 +- crates/init/src/supervise.rs | 83 ++++++++++++++--- crates/init/src/watchdog.rs | 108 ++++++++++++++++++++++ crates/server-framework/src/lib.rs | 2 +- crates/server-framework/src/server/mod.rs | 11 ++- docs/ipc.md | 9 ++ 9 files changed, 236 insertions(+), 18 deletions(-) create mode 100644 crates/init/src/watchdog.rs diff --git a/crates/apps/spawntest/src/main.rs b/crates/apps/spawntest/src/main.rs index 0d94d96..67ba779 100644 --- a/crates/apps/spawntest/src/main.rs +++ b/crates/apps/spawntest/src/main.rs @@ -200,6 +200,22 @@ fn restarted_after_crashes() -> Result<(), &'static str> { Ok(()) } +/// A server stuck in a handler is noticed by the watchdog, killed (which +/// releases its caller) and restarted. +fn restarted_after_hanging() -> Result<(), &'static str> { + let asked = syscall::get_time().raw(); + // Comes back, with an error, once the hung server is killed. + let _ = syscall::request(FLAKY, [flaky::op::HANG, 0, 0, 0]); + let hung_ms = (syscall::get_time().raw() - asked) / 10_000; + if hung_ms < 4_500 { + return Err("the hung server was stopped before the watchdog's deadline"); + } + if flaky()? != 1 { + return Err("the hung server was not restarted"); + } + Ok(()) +} + fn check() -> Result<(), &'static str> { let mut fs = Storage::attach(STORAGE, STORAGE_BUF_VA).map_err(|_| "cannot attach to storage")?; write_a_file(&mut fs)?; @@ -232,6 +248,9 @@ fn check() -> Result<(), &'static str> { restarted_after_crashes()?; libkos::println!("spawntest: a crashed server is restarted, a crash loop backs off: ok"); + + restarted_after_hanging()?; + libkos::println!("spawntest: a hung server is noticed, stopped and restarted: ok"); Ok(()) } diff --git a/crates/init/src/launch.rs b/crates/init/src/launch.rs index d2f91a0..7ff33bf 100644 --- a/crates/init/src/launch.rs +++ b/crates/init/src/launch.rs @@ -47,6 +47,10 @@ pub(crate) struct Launcher<'a> { console: Option, /// Signals the death notification's timer bit. pub(crate) timer: CapSlot, + /// Signals that came in while init waited elsewhere, still to handle. + pub(crate) pending: usize, + /// When the watchdog's last round of pings started. + pub(crate) last_round: u64, } /// What became of starting a program. @@ -88,6 +92,8 @@ impl<'a> Launcher<'a> { supervised: [None; MAX_SUPERVISED], console: libkos::start::console(), timer, + pending: 0, + last_round: 0, }) } diff --git a/crates/init/src/lib.rs b/crates/init/src/lib.rs index bb5a6e4..6335d4b 100644 --- a/crates/init/src/lib.rs +++ b/crates/init/src/lib.rs @@ -13,6 +13,7 @@ mod selftest; mod session; mod spawn; mod supervise; +mod watchdog; use libkos::fs::Storage; use libkos::syscall; @@ -65,6 +66,11 @@ pub fn run() -> ! { if !launcher.watch_storage() { crate::outln!("init: cannot watch storage"); } + if !watchdog::start() { + crate::outln!("init: no watchdog thread; hangs go unnoticed"); + } + // Starts the clock the watchdog runs on. + launcher.supervise(0); // The test image carries the SMP self-test; it and the tests run // instead of the desktop, and the last one halts the machine. diff --git a/crates/init/src/manifest.rs b/crates/init/src/manifest.rs index 3e9d8d4..fb70cf3 100644 --- a/crates/init/src/manifest.rs +++ b/crates/init/src/manifest.rs @@ -58,6 +58,9 @@ pub struct Program { /// Its clients keep state in it (a subscription, focus, a handle), so /// they are restarted after it is. pub restarts_clients: bool, + /// Pinged by the watchdog (see init's watchdog.rs): not if a handler of + /// its waits for the user, which a ping cannot tell from a hang. + pub pinged: bool, } /// Enough for a small program: its image, stack and a little heap. @@ -85,6 +88,7 @@ const fn server(name: &'static str, file: &'static [u8], limits: Limits) -> Prog takes_ctrl_c: false, restart: true, restarts_clients: false, + pinged: true, } } @@ -103,8 +107,8 @@ pub const SERVERS: &[Program] = &[ ..server("display", b"bin/display", Limits { pages: 1280, regions: 4, ..SMALL }) }, // Text on the display, lines from the keyboard (it takes input focus). - // Its clients wait on lines and Ctrl+C. - Program { uses: &["display"], restarts_clients: true, ..server("terminal", b"bin/terminal", SMALL) }, + // Its clients wait on lines and Ctrl+C; `READ` waits for the user. + Program { uses: &["display"], restarts_clients: true, pinged: false, ..server("terminal", b"bin/terminal", SMALL) }, server("session", b"bin/session", SMALL), ]; @@ -133,6 +137,8 @@ pub const SESSION: &[Program] = &[Program { spawns: Some(Limits { pages: 512, endpoints: 8, notifications: 8, regions: 8 }), // The shell holds handles to the apps it ran, which die with it. restarts_clients: true, + // Installing waits for the user's consent. + pinged: false, ..server("permission", b"bin/permission", Limits { pages: 96, ..SMALL }) }]; diff --git a/crates/init/src/supervise.rs b/crates/init/src/supervise.rs index 60d800d..da07cd7 100644 --- a/crates/init/src/supervise.rs +++ b/crates/init/src/supervise.rs @@ -19,6 +19,7 @@ use types::time::Ticks; use crate::launch::{Launched, Launcher}; use crate::manifest::Program; +use crate::watchdog; /// Programs watched at once, at most. pub(crate) const MAX_SUPERVISED: usize = 12; @@ -35,6 +36,8 @@ const TICKS_PER_MS: u64 = 10_000; const DELAYS_MS: [u64; 4] = [0, 1_000, 5_000, 30_000]; /// A run this long was not part of a crash loop. const STABLE_MS: u64 = 60_000; +/// How often init looks at the clock for the watchdog. +const TICK_MS: u64 = 1_000; /// How many crashes in a row, and so how long before the next restart. #[derive(Clone, Copy, Default)] @@ -76,6 +79,10 @@ pub(crate) struct Supervised { spawn: Option, /// Until what this run held is let go of. holding: bool, + /// Its endpoint, if it serves one. + ep: Option, + /// Killed for hanging: the watchdog's ping is queued on its endpoint. + hung: bool, } impl Supervised { @@ -90,7 +97,7 @@ impl Supervised { } } -fn now_ms() -> u64 { +pub(crate) fn now_ms() -> u64 { syscall::get_time().raw() / TICKS_PER_MS } @@ -105,6 +112,7 @@ impl Launcher<'_> { let index = self.supervised.iter().position(Option::is_none).ok_or("too many programs to watch")?; let started = self.launch(p)?; let note = started.note; + let ep = started.ep; if !self.watch(started.process, bit(index)) { let _ = syscall::kill(started.process); self.forget(started); @@ -118,7 +126,10 @@ impl Launcher<'_> { grants: started.grants, spawn: started.spawn, holding: true, + ep, + hung: false, }); + self.ping(index); Ok(note) } @@ -161,8 +172,18 @@ impl Launcher<'_> { self.release(grants, spawn); } + /// Has the watchdog ping program `index`, if it is to be pinged. + fn ping(&self, index: usize) { + if let Some(s) = self.supervised[index] + && let Some(ep) = s.ep.filter(|_| s.program.pinged) + { + watchdog::watch(index, ep); + } + } + /// Stops a running program on purpose; `restart` it once it is dead. fn stop(&mut self, index: usize, restart: bool) { + watchdog::unwatch(index); let Some(s) = self.supervised[index].as_mut() else { return }; if let State::Running { process, .. } = s.state { let _ = syscall::kill(process); @@ -187,19 +208,45 @@ impl Launcher<'_> { /// Death signals and the timer, from init's server loop. pub(crate) fn supervise(&mut self, bits: usize) { - if bits & STORAGE_BIT != 0 { - crate::outln!("init: storage died; nothing can go on without it"); - crate::idle(); - } - for index in 0..MAX_SUPERVISED { - if bits & bit(index) != 0 { - self.died(index); + self.pending |= bits; + while self.pending != 0 { + let bits = core::mem::take(&mut self.pending); + if bits & STORAGE_BIT != 0 { + crate::outln!("init: storage died; nothing can go on without it"); + crate::idle(); + } + for index in 0..MAX_SUPERVISED { + if bits & bit(index) != 0 { + self.died(index); + } } } + self.watch_the_clock(); self.restart_due(); } + /// Kills a server whose ping has gone unanswered; starts ping rounds. + fn watch_the_clock(&mut self) { + let now = now_ms(); + if let Some(index) = watchdog::hung(now) + && let Some(s) = self.supervised[index].as_mut() + && let State::Running { process, .. } = s.state + && !s.hung + { + crate::outln!("init: {} stopped answering; restarting it", s.program.name); + s.hung = true; + watchdog::unwatch(index); + // Its death comes back as a crash. + let _ = syscall::kill(process); + } + if now.saturating_sub(self.last_round) >= watchdog::ROUND_MS { + self.last_round = now; + watchdog::round(); + } + } + fn died(&mut self, index: usize) { + watchdog::unwatch(index); // Nothing left to let go of if init stopped it. self.let_go(index); let Some(s) = self.supervised[index] else { return }; @@ -219,15 +266,23 @@ impl Launcher<'_> { let mut backoff = s.backoff; let delay = restart.and_then(|ran| if crashed { backoff.after_crash(ran) } else { Some(0) }); match (crashed, delay) { - (true, Some(_)) => crate::outln!("init: {} died, restarting it", s.program.name), + (true, Some(_)) if !s.hung => crate::outln!("init: {} died, restarting it", s.program.name), (true, None) if s.program.restart => crate::outln!("init: {} keeps dying; not restarting it again", s.program.name), _ => {} } + // Nobody will take the ping a hung server left queued: init does. + if delay.is_none() + && s.hung + && let Some(ep) = s.ep.filter(|_| watchdog::pinging(index)) + { + self.pending |= watchdog::release_pinger(ep); + } self.supervised[index] = delay.map(|d| Supervised { state: State::Waiting { at: now_ms() + d }, backoff, grants: [None; 2], spawn: None, + hung: false, ..s }); } @@ -250,8 +305,10 @@ impl Launcher<'_> { grants: started.grants, spawn: started.spawn, holding: true, + ep: started.ep, ..s }); + self.ping(index); if s.program.restarts_clients { self.restart_users_of(s.program.name); } @@ -268,13 +325,13 @@ impl Launcher<'_> { } } } + // The clock ticks for the watchdog whatever else is due. let next = self.supervised.iter().flatten().filter_map(|s| match s.state { State::Waiting { at } => Some(at), _ => None, - }).min(); - if let Some(at) = next { - self.wake_at(at); - } + }); + let tick = now + TICK_MS; + self.wake_at(next.fold(tick, u64::min)); } /// Whoever calls `name` kept state in the old one: restart them too. diff --git a/crates/init/src/watchdog.rs b/crates/init/src/watchdog.rs new file mode 100644 index 0000000..fecc38d --- /dev/null +++ b/crates/init/src/watchdog.rs @@ -0,0 +1,108 @@ +//! Noticing a server that hangs (see the vision's Watchdog note). A thread +//! of init's own calls each watched server with `server_framework::PING` +//! in turn; the framework answers it without the server's code, so only a +//! server stuck inside a handler (a deadlock, a loop) leaves it unanswered. +//! Init's main thread, which must never wait on a server, only watches the +//! clock: a ping unanswered for `HANG_MS` gets the server killed, and the +//! death is handled like a crash (supervise.rs). The ping stays queued and +//! the restarted server answers it. +//! +//! Servers whose handlers wait for the user (the terminal's `READ`, the +//! permission manager's consent prompt) cannot be told from hung servers +//! this way, so they are not pinged. + +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use libkos::syscall; +use types::bounded::CapSlot; + +use crate::supervise::MAX_SUPERVISED; + +/// Unanswered this long, a server is hung. +pub(crate) const HANG_MS: u64 = 5_000; +/// How often a round of pings starts. +pub(crate) const ROUND_MS: u64 = 2_000; +const NONE: usize = usize::MAX; + +/// Each supervised program's endpoint, if it is pinged. +static TARGETS: [AtomicUsize; MAX_SUPERVISED] = [const { AtomicUsize::new(NONE) }; MAX_SUPERVISED]; +/// Which one the pinger is waiting on (`NONE`: none), and since when. +static PINGING: AtomicUsize = AtomicUsize::new(NONE); +static SINCE_MS: AtomicU64 = AtomicU64::new(0); +/// The pinger's notification: a signal starts a round. +static WAKE: AtomicUsize = AtomicUsize::new(NONE); + +fn slot(raw: usize) -> Option { + u8::try_from(raw).ok().and_then(CapSlot::new) +} + +/// Starts the pinger. +pub(crate) fn start() -> bool { + let Ok(wake) = syscall::create_notification() else { return false }; + WAKE.store(wake.value() as usize, Ordering::Release); + libkos::thread::spawn(&WAKE, pinger) +} + +pub(crate) fn watch(index: usize, ep: CapSlot) { + TARGETS[index].store(ep.value() as usize, Ordering::Release); +} + +pub(crate) fn unwatch(index: usize) { + TARGETS[index].store(NONE, Ordering::Release); +} + +/// Has the pinger start a round (it finishes the last one first). +pub(crate) fn round() { + if let Some(wake) = slot(WAKE.load(Ordering::Acquire)) { + let _ = syscall::signal(wake); + } +} + +/// The program whose ping has gone unanswered too long, if any. +pub(crate) fn hung(now_ms: u64) -> Option { + let index = PINGING.load(Ordering::Acquire); + let since = SINCE_MS.load(Ordering::Acquire); + (index != NONE && now_ms.saturating_sub(since) >= HANG_MS).then_some(index) +} + +/// Whether the pinger waits on `index` right now. +pub(crate) fn pinging(index: usize) -> bool { + PINGING.load(Ordering::Acquire) == index +} + +fn pinger(_: &'static AtomicUsize) -> ! { + loop { + if let Some(wake) = slot(WAKE.load(Ordering::Acquire)) { + let _ = syscall::wait(wake); + } + for (index, target) in TARGETS.iter().enumerate() { + let Some(ep) = slot(target.load(Ordering::Acquire)) else { continue }; + SINCE_MS.store(crate::supervise::now_ms(), Ordering::Release); + PINGING.store(index, Ordering::Release); + let _ = syscall::request(ep, [server_framework::PING, 0, 0, 0]); + PINGING.store(NONE, Ordering::Release); + } + } +} + +/// Init gave up on a server it killed for hanging: the pinger's ping is +/// still queued on its endpoint, and nothing will ever take it. Init takes +/// it (failing anyone else queued there), so the pinger goes on. Signals +/// that arrive meanwhile are returned, to be handled after. +pub(crate) fn release_pinger(ep: CapSlot) -> usize { + use libkos::syscall::Received; + let mut signals = 0; + loop { + match syscall::receive_msg(ep) { + Ok(Received::Call { badge: 0, words, .. }) if words[0] == server_framework::PING => { + let _ = syscall::reply([server_framework::PING, 0, 0, 0], None); + return signals; + } + Ok(Received::Call { .. }) => { + let _ = syscall::reply([server_framework::MALFORMED, 0, 0, 0], None); + } + Ok(Received::Message { .. }) => {} + Ok(Received::Signal { bits }) => signals |= bits, + Err(_) => return signals, + } + } +} diff --git a/crates/server-framework/src/lib.rs b/crates/server-framework/src/lib.rs index e322964..74c1431 100644 --- a/crates/server-framework/src/lib.rs +++ b/crates/server-framework/src/lib.rs @@ -7,4 +7,4 @@ pub mod server; pub use codec::{Decode, Encode}; pub use executor::block_on; -pub use server::{MALFORMED, Server, serve}; +pub use server::{MALFORMED, PING, Server, serve}; diff --git a/crates/server-framework/src/server/mod.rs b/crates/server-framework/src/server/mod.rs index 4cb7dfc..b0b8781 100644 --- a/crates/server-framework/src/server/mod.rs +++ b/crates/server-framework/src/server/mod.rs @@ -9,6 +9,11 @@ use types::bounded::CapSlot; /// tag, so every client decodes it as an error. pub const MALFORMED: usize = usize::MAX; +/// Init's watchdog asking whether the server still serves: answered with +/// the same word by the loop itself, never seen by the server's code. No +/// protocol uses it either. +pub const PING: usize = usize::MAX - 1; + pub trait Server { type Request: Decode; type Response: Encode; @@ -37,8 +42,9 @@ pub trait Server { /// Serves `endpoint` until receiving on it fails (it was destroyed or /// the capability revoked), then returns why. Every `Call` gets a - /// reply, `[MALFORMED, ..]` if it does not decode; a one-way `Send` is - /// handled without one; notification signals go to `signal`. + /// reply, `[MALFORMED, ..]` if it does not decode, `[PING, ..]` to a + /// `PING`; a one-way `Send` is handled without one; notification + /// signals go to `signal`. fn run(&mut self, endpoint: CapSlot) -> libkos::syscall::Error { use libkos::syscall::{Received, receive_msg, reply_recv}; let mut next = receive_msg(endpoint); @@ -48,6 +54,7 @@ pub trait Server { Err(e) => return e, }; next = match message { + Received::Call { words, .. } if words[0] == PING => reply_recv(endpoint, [PING, 0, 0, 0], None), Received::Call { badge, words, cap } => { let (reply, give) = self.handle_call(badge, words, cap); reply_recv(endpoint, reply, give) diff --git a/docs/ipc.md b/docs/ipc.md index aa062a5..38749e7 100644 --- a/docs/ipc.md +++ b/docs/ipc.md @@ -136,6 +136,15 @@ it holds the key and every grant. Init's main thread never waits on a server it restarts: its own output goes through a thread of its own (`init/src/output.rs`), which waits on the log server instead. +Hangs are noticed by pinging (`init/src/watchdog.rs`): another thread of +init's calls each server with `server_framework::PING` every 2 s, which +the framework's loop answers without the server's code. A ping +unanswered for 5 s means the server is stuck in a handler; init kills it, +and the death is handled as a crash, so the restarted server answers the +still-queued ping. The terminal and the permission manager are not +pinged: their handlers wait for the user (`READ`, a consent prompt), +which a ping cannot tell from a hang. + A program's `uses` endpoints arrive as badged copies with `Execute` and `Delegate`: the badge (unique per copy) tells the server which client is calling, and `Delegate` lets the client send capabilities with its calls