From 28f1ace00aa9215f5b0aa15c751a61773970b352 Mon Sep 17 00:00:00 2001 From: Gabriel Kaszewski Date: Wed, 28 Oct 2026 01:44:24 +0100 Subject: [PATCH] feat(init): supervise servers+session progs: restart on death, same endpoint, backoff 0/1/5/30s then give up, restarts_clients cascade; release grants/spawn at stop (stale-slot safe); storage death fatal; init output via printer thread (never blocks on log); terminal Ctrl+C via CONTROL-badged copy, latest wins; flaky test server + spawntest restart/backoff check --- Cargo.lock | 10 + Cargo.toml | 1 + Makefile | 2 +- crates/apps/flaky/Cargo.toml | 14 + crates/apps/flaky/src/lib.rs | 69 ++++ crates/apps/flaky/src/main.rs | 11 + crates/apps/shell/src/main.rs | 4 +- crates/apps/spawntest/Cargo.toml | 1 + crates/apps/spawntest/src/main.rs | 39 ++- crates/init/src/factory.rs | 2 +- crates/init/src/launch.rs | 72 ++-- crates/init/src/lib.rs | 37 +- crates/init/src/manifest.rs | 33 +- crates/init/src/output.rs | 102 ++++++ crates/init/src/selftest.rs | 10 +- crates/init/src/session.rs | 43 +-- crates/init/src/spawn.rs | 39 +-- crates/init/src/supervise.rs | 316 ++++++++++++++++++ crates/servers/terminal-server/src/server.rs | 15 +- .../terminal-server/src/tests/server.rs | 28 +- crates/types/src/terminal.rs | 10 +- docs/ipc.md | 21 +- scripts/make-disk-image.sh | 1 + 23 files changed, 736 insertions(+), 144 deletions(-) create mode 100644 crates/apps/flaky/Cargo.toml create mode 100644 crates/apps/flaky/src/lib.rs create mode 100644 crates/apps/flaky/src/main.rs create mode 100644 crates/init/src/output.rs create mode 100644 crates/init/src/supervise.rs diff --git a/Cargo.lock b/Cargo.lock index 041cad8..9efba8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -297,6 +297,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "flaky" +version = "0.1.0" +dependencies = [ + "libkos", + "server-framework", + "types", +] + [[package]] name = "gatekeeper" version = "0.1.0" @@ -573,6 +582,7 @@ dependencies = [ name = "spawntest" version = "0.1.0" dependencies = [ + "flaky", "libkos", "manifest", "types", diff --git a/Cargo.toml b/Cargo.toml index 3d979dc..734516f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/adapter-riscv", + "crates/apps/flaky", "crates/apps/hello", "crates/apps/shell", "crates/apps/cat", diff --git a/Makefile b/Makefile index 4b735bf..1c6661a 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ kani: kani-types kani-domain kani-kernel PROGRAMS = -p init -p echo -p sleep -p snake \ -p display-qemu -p input-qemu -p storage-qemu \ -p log-server -p permission-manager -p session-server -p smptest -p spawntest -p hello \ - -p terminal-server -p shell -p cat -p ls -p counter -p gatekeeper + -p terminal-server -p shell -p cat -p ls -p counter -p gatekeeper -p flaky # Programs first: the kernel image packs init and storage (see the # adapter's build.rs). diff --git a/crates/apps/flaky/Cargo.toml b/crates/apps/flaky/Cargo.toml new file mode 100644 index 0000000..c0a33e1 --- /dev/null +++ b/crates/apps/flaky/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "flaky" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "flaky" +test = false +bench = false + +[dependencies] +types = { workspace = true } +libkos = { workspace = true } +server-framework = { workspace = true } diff --git a/crates/apps/flaky/src/lib.rs b/crates/apps/flaky/src/lib.rs new file mode 100644 index 0000000..9b8b02d --- /dev/null +++ b/crates/apps/flaky/src/lib.rs @@ -0,0 +1,69 @@ +//! A server that dies or hangs when asked: on the test image only, so the +//! self-test can watch init restart it (see init's supervise.rs). + +#![no_std] +#![forbid(unsafe_code)] + +use libkos::syscall; +use server_framework::{Decode, Encode, MALFORMED, Server}; +use types::bounded::CapSlot; + +/// Requests (word 0); a reply carries how many requests this instance has +/// served, this one included: a restarted server starts at 1. +pub mod op { + pub const WHO: usize = 1; + /// Panics: the process dies. + pub const CRASH: usize = 2; + /// Blocks for good inside the handler. + pub const HANG: usize = 3; +} + +/// Everything goes through `handle_call`. +struct Never; + +impl Decode for Never { + fn decode(_: [usize; 4]) -> Option { + None + } +} + +impl Encode for Never { + fn encode(&self) -> [usize; 4] { + [MALFORMED, 0, 0, 0] + } +} + +struct Flaky { + served: usize, +} + +impl Server for Flaky { + type Request = Never; + type Response = Never; + + fn handle(&mut self, request: Never) -> Never { + request + } + + fn handle_call(&mut self, _badge: usize, words: [usize; 4], _cap: Option) -> ([usize; 4], Option) { + match words[0] { + op::WHO => {} + op::CRASH => panic!("asked to crash"), + op::HANG => { + // A notification nobody signals. + if let Ok(never) = syscall::create_notification() { + loop { + let _ = syscall::wait(never); + } + } + } + _ => return ([MALFORMED, 0, 0, 0], None), + } + self.served += 1; + ([self.served, 0, 0, 0], None) + } +} + +pub fn run() -> ! { + server_framework::serve(&mut Flaky { served: 0 }, CapSlot::const_new(0)) +} diff --git a/crates/apps/flaky/src/main.rs b/crates/apps/flaky/src/main.rs new file mode 100644 index 0000000..524071a --- /dev/null +++ b/crates/apps/flaky/src/main.rs @@ -0,0 +1,11 @@ +#![no_std] +#![forbid(unsafe_code)] +#![no_main] + +extern crate libkos; + +libkos::entry!(main); + +fn main() { + flaky::run(); +} diff --git a/crates/apps/shell/src/main.rs b/crates/apps/shell/src/main.rs index a9bf8ce..633fde2 100644 --- a/crates/apps/shell/src/main.rs +++ b/crates/apps/shell/src/main.rs @@ -25,6 +25,8 @@ const PERMISSION: CapSlot = CapSlot::const_new(1); /// The user's files, their `home`: the shell's `/`. const HOME: CapSlot = CapSlot::const_new(2); const SESSION: CapSlot = CapSlot::const_new(3); +/// The terminal again, the copy that takes Ctrl+C; apps get the plain one. +const CONTROL: CapSlot = CapSlot::const_new(4); /// Bits on the shell's notification: the program died (its death watch is /// unbadged), or Ctrl+C was typed (the terminal's copy). @@ -294,7 +296,7 @@ fn main() { println!("shell: no permission manager"); return; }; - if !libkos::terminal::on_interrupt(TERMINAL, done, types::badge::Badge::new(INTERRUPTED)) { + if !libkos::terminal::on_interrupt(CONTROL, done, types::badge::Badge::new(INTERRUPTED)) { println!("shell: Ctrl+C will not work"); } let mut sh = Shell { diff --git a/crates/apps/spawntest/Cargo.toml b/crates/apps/spawntest/Cargo.toml index 026ccec..5a6d149 100644 --- a/crates/apps/spawntest/Cargo.toml +++ b/crates/apps/spawntest/Cargo.toml @@ -11,4 +11,5 @@ bench = false [dependencies] types = { workspace = true } manifest = { workspace = true } +flaky = { path = "../flaky" } libkos = { workspace = true } diff --git a/crates/apps/spawntest/src/main.rs b/crates/apps/spawntest/src/main.rs index 9b74459..0d94d96 100644 --- a/crates/apps/spawntest/src/main.rs +++ b/crates/apps/spawntest/src/main.rs @@ -17,10 +17,11 @@ use types::budget::Limits; /// Where init put them (see init's manifest: `uses`, storage (the disk, /// the user's home), the spawn endpoint, then control). const PERMISSION: CapSlot = CapSlot::const_new(0); -const STORAGE: CapSlot = CapSlot::const_new(1); -const HOME: CapSlot = CapSlot::const_new(2); -const INIT: CapSlot = CapSlot::const_new(3); -const CONTROL: CapSlot = CapSlot::const_new(4); +const FLAKY: CapSlot = CapSlot::const_new(1); +const STORAGE: CapSlot = CapSlot::const_new(2); +const HOME: CapSlot = CapSlot::const_new(3); +const INIT: CapSlot = CapSlot::const_new(4); +const CONTROL: CapSlot = CapSlot::const_new(5); /// Where storage's buffers are mapped: the whole disk's, a narrowed one's. const STORAGE_BUF_VA: types::address::VirtualAddress = types::address::VirtualAddress::from_raw(0x0180_0000); const NARROW_BUF_VA: types::address::VirtualAddress = types::address::VirtualAddress::from_raw(0x0190_0000); @@ -172,6 +173,33 @@ fn encrypted_home(fs: &mut Storage, home_fs: &mut Storage) -> Result<(), &'stati Ok(()) } +/// How many requests the `flaky` answering now has served. +fn flaky() -> Result { + syscall::request(FLAKY, [flaky::op::WHO, 0, 0, 0]).map(|r| r[0]).map_err(|_| "flaky does not answer") +} + +/// A server that dies is started again on the same endpoint: the caller's +/// capability reaches the new one. Twice in a row, the second waits. +fn restarted_after_crashes() -> Result<(), &'static str> { + if flaky()? != 1 || flaky()? != 2 { + return Err("flaky counts wrong"); + } + // Dies mid-call: the call comes back with an error. + let _ = syscall::request(FLAKY, [flaky::op::CRASH, 0, 0, 0]); + if flaky()? != 1 { + return Err("the crashed server still answers"); + } + let crashed = syscall::get_time().raw(); + let _ = syscall::request(FLAKY, [flaky::op::CRASH, 0, 0, 0]); + let fresh = flaky()? == 1; + // QEMU virt's timebase is 10 MHz. + let waited_ms = (syscall::get_time().raw() - crashed) / 10_000; + if !fresh || waited_ms < 900 { + return Err("a second crash in a row was not backed off"); + } + Ok(()) +} + fn check() -> Result<(), &'static str> { let mut fs = Storage::attach(STORAGE, STORAGE_BUF_VA).map_err(|_| "cannot attach to storage")?; write_a_file(&mut fs)?; @@ -201,6 +229,9 @@ fn check() -> Result<(), &'static str> { revoke_stops_a_running_app(died)?; libkos::println!("spawntest: revoking stops a running app, allowing lets it run: ok"); + + restarted_after_crashes()?; + libkos::println!("spawntest: a crashed server is restarted, a crash loop backs off: ok"); Ok(()) } diff --git a/crates/init/src/factory.rs b/crates/init/src/factory.rs index 9fe1c0b..c137686 100644 --- a/crates/init/src/factory.rs +++ b/crates/init/src/factory.rs @@ -125,7 +125,7 @@ pub(crate) fn prepare(fs: &mut Storage, folder: types::user::UserFolder, buf: &m } fs.mkdir(path.as_bytes()).map_err(|_| "cannot make an area")?; copy_tree(fs, &Path::new(factory), &path, buf)?; - libkos::println!("init: set up {}", area); + crate::outln!("init: set up {}", area); } Ok(()) } diff --git a/crates/init/src/launch.rs b/crates/init/src/launch.rs index 0677a8d..d2f91a0 100644 --- a/crates/init/src/launch.rs +++ b/crates/init/src/launch.rs @@ -15,6 +15,7 @@ use crate::devices::Devices; use crate::load::{self, LoadedProcess}; use crate::manifest::{Device, Program, Scope}; use crate::spawn::Spawner; +use crate::supervise::{MAX_SUPERVISED, Supervised}; const USER_STACK_TOP: StackPointer = StackPointer::from_raw(0x10000); const MAX_SERVED: usize = 8; @@ -36,22 +37,28 @@ pub(crate) struct Launcher<'a> { /// Bound to init's thread: spawned children's deaths, a bit each. pub(crate) deaths: CapSlot, pub(crate) spawner: Spawner, - /// Who is signed in, and what runs for them (stopped at sign-out). + /// Who is signed in. pub(crate) user: Option, - pub(crate) running: [Option; MAX_SESSION], /// What a session runs after `SESSION`: the desktop, or the tests. pub(crate) desktop: &'static [Program], + /// Every program init restarts or stops at sign-out (see supervise.rs). + pub(crate) supervised: [Option; MAX_SUPERVISED], + /// The console, handed to the log server (again, if it restarts). + console: Option, + /// Signals the death notification's timer bit. + pub(crate) timer: CapSlot, } -/// Programs one session runs at once, at most. -pub(crate) const MAX_SESSION: usize = 4; - /// What became of starting a program. pub(crate) struct Launched { pub(crate) note: &'static str, pub(crate) process: CapSlot, /// Its endpoint, if it serves one. pub(crate) ep: Option, + /// Init's copies of its storage grants and its spawn client's badge, + /// to take back when it dies. + pub(crate) grants: [Option; 2], + pub(crate) spawn: Option, } impl<'a> Launcher<'a> { @@ -62,6 +69,8 @@ impl<'a> Launcher<'a> { ) -> Result { let spawn_ep = syscall::create_endpoint().ok_or("no spawn endpoint")?; let deaths = syscall::create_notification().map_err(|_| "no death notification")?; + let timer = syscall::mint(deaths, Badge::new(crate::supervise::TIMER_BIT), perms(&[Permission::Write])) + .map_err(|_| "no timer")?; // Storage, started by boot, is not among them: it is reached // through grants (a program's `storage` scope). let served = [None; MAX_SERVED]; @@ -75,8 +84,10 @@ impl<'a> Launcher<'a> { deaths, spawner: Spawner::new(), user: None, - running: [None; MAX_SESSION], desktop: &[], + supervised: [None; MAX_SUPERVISED], + console: libkos::start::console(), + timer, }) } @@ -84,11 +95,6 @@ impl<'a> Launcher<'a> { (self.storage, self.scratch) } - /// The endpoint of the running server `name`. - pub(crate) fn served(&self, name: &str) -> Option { - self.endpoint(name) - } - fn endpoint(&self, name: &str) -> Option { self.served.iter().flatten().find(|(n, _)| *n == name).map(|(_, ep)| *ep) } @@ -125,18 +131,20 @@ impl<'a> Launcher<'a> { syscall::drop_cap(copy); given.ok_or("cannot give an endpoint")?; } - // Storage issues its own badges, each naming a scope. - for scope in p.storage { + // Storage issues its own badges, each naming a scope; init keeps + // its copy, to take the grant back when the program dies. + let mut grants = [None; 2]; + for (scope, kept) in p.storage.iter().zip(grants.iter_mut()) { let mut path = [0u8; types::storage::MAX_PATH]; let path = self.scope_path(scope, &mut path).ok_or("a storage scope for nobody")?; let grant = self.storage.grant(path, scope.write).map_err(|_| "storage refused a grant")?; - let given = syscall::delegate(grant, child, perms(&[Permission::Execute, Permission::Delegate])); - syscall::drop_cap(grant); - given.ok_or("cannot give storage")?; - } - if let Some(allowance) = p.spawns { - self.give_spawn(child, allowance)?; + *kept = Some(grant); + syscall::delegate(grant, child, perms(&[Permission::Execute, Permission::Delegate])).ok_or("cannot give storage")?; } + let spawn = match p.spawns { + Some(allowance) => Some(self.give_spawn(child, allowance)?), + None => None, + }; if p.system_control { let control = crate::devices::system_control().ok_or("no SystemControl to give")?; syscall::delegate(control, child, perms(&[Permission::Execute])).ok_or("cannot give SystemControl")?; @@ -149,7 +157,17 @@ impl<'a> Launcher<'a> { syscall::drop_cap(copy); given.ok_or("cannot give signing out")?; } - let console = libkos::start::console() + if p.takes_ctrl_c { + let terminal = self.endpoint("terminal").ok_or("needs the terminal")?; + let badge = Badge::new(self.next_badge | types::terminal::CONTROL); + self.next_badge += 1; + let copy = syscall::mint(terminal, badge, perms(&[Permission::Execute, Permission::Delegate])).map_err(|_| "cannot mint")?; + // Delegate, to send its notification with the call. + let given = syscall::delegate(copy, child, perms(&[Permission::Execute, Permission::Delegate])); + syscall::drop_cap(copy); + given.ok_or("cannot give Ctrl+C")?; + } + let console = self.console .filter(|_| p.console) .and_then(|c| syscall::delegate(c, child, perms(&[Permission::Write]))); start(&loaded, console); @@ -168,7 +186,7 @@ impl<'a> Launcher<'a> { note = ", console handed over"; } } - Ok(Launched { note, process: child, ep }) + Ok(Launched { note, process: child, ep, grants, spawn }) } /// The path on disk `scope` names: inside the user's folder if it is a @@ -188,14 +206,14 @@ impl<'a> Launcher<'a> { /// A badged copy of init's endpoint: the badge names the client and so /// its allowance. - fn give_spawn(&mut self, child: CapSlot, allowance: types::budget::Limits) -> Result<(), &'static str> { + fn give_spawn(&mut self, child: CapSlot, allowance: types::budget::Limits) -> Result { let badge: Badge = self.spawner.add_client(allowance).ok_or("too many spawn clients")?; let minted = perms(&[Permission::Execute, Permission::Delegate]); let copy = syscall::mint(self.spawn_ep, badge, minted).map_err(|_| "cannot mint")?; // Delegate, to send the program file's grant with each request. let given = syscall::delegate(copy, child, perms(&[Permission::Execute, Permission::Delegate])); syscall::drop_cap(copy); - given.map(|_| ()).ok_or("cannot give the spawn endpoint") + given.map(|_| badge).ok_or("cannot give the spawn endpoint") } /// The driver waits for its device before serving, and runs without @@ -214,12 +232,10 @@ impl<'a> Launcher<'a> { } } -/// From here on init prints through the log server, like everyone else, -/// and holds no console of its own. +/// From here on init prints through the log server, like everyone else +/// (by way of its printer, see output.rs); it keeps the console only to +/// hand it to a restarted log server. fn hand_over_console(log: CapSlot) { - if let Some(console) = libkos::start::console() { - syscall::drop_cap(console); - } libkos::start::set_console(None); libkos::start::set_log(Some(log)); } diff --git a/crates/init/src/lib.rs b/crates/init/src/lib.rs index 06daef2..bb5a6e4 100644 --- a/crates/init/src/lib.rs +++ b/crates/init/src/lib.rs @@ -8,9 +8,11 @@ mod factory; mod launch; mod load; mod manifest; +mod output; mod selftest; mod session; mod spawn; +mod supervise; use libkos::fs::Storage; use libkos::syscall; @@ -25,37 +27,44 @@ pub(crate) const STORAGE_EP: CapSlot = CapSlot::const_new(2); pub fn run() -> ! { libkos::start::set_console(devices::console()); - libkos::println!("init: started (pid={:#x})", syscall::get_process_id().raw()); + let printer = output::start(); + crate::outln!("init: started (pid={:#x})", syscall::get_process_id().raw()); + if !printer { + crate::outln!("init: no printer thread; output waits on the log server"); + } // Storage waits for its disk before serving, so this comes first. let found = devices::discover(); let disk = libkos::device::give_device(STORAGE_EP, found.disk).is_ok() && found.disk.mmio.is_some(); - libkos::println!( + crate::outln!( "init: disk: {}", if disk { "handed to its driver" } else { "none" } ); let Ok(mut storage) = Storage::attach(STORAGE_EP, STORAGE_BUF_VA) else { - libkos::println!("init: cannot reach storage, nothing to load"); + crate::outln!("init: cannot reach storage, nothing to load"); idle(); }; // Storage has nothing to print with yet; init speaks for it. - libkos::println!("init: storage: up"); + crate::outln!("init: storage: up"); let Some(scratch) = load::scratch() else { - libkos::println!("init: no memory to load programs into"); + crate::outln!("init: no memory to load programs into"); idle(); }; let mut launcher = match launch::Launcher::new(&mut storage, scratch, found) { Ok(l) => l, Err(why) => { - libkos::println!("init: {}", why); + crate::outln!("init: {}", why); idle(); } }; launch_all(&mut launcher, manifest::SERVERS); give_storage_the_log(); + if !launcher.watch_storage() { + crate::outln!("init: cannot watch storage"); + } // The test image carries the SMP self-test; it and the tests run // instead of the desktop, and the last one halts the machine. @@ -64,7 +73,7 @@ pub fn run() -> ! { .open(b"bin/smptest") .is_ok_and(|h| storage.close(h).is_ok()) { - libkos::println!("init: running the SMP self-test"); + crate::outln!("init: running the SMP self-test"); selftest::run(storage, scratch); manifest::TESTS } else { @@ -72,22 +81,22 @@ pub fn run() -> ! { }; // Nothing that touches a user's files starts before someone is in. if let Err(why) = session::sign_in(&mut launcher) { - libkos::println!("init: STOP ({})", why); + crate::outln!("init: STOP ({})", why); idle(); } - libkos::println!("init: all loaded, serving spawns"); + crate::outln!("init: all loaded, serving spawns"); let _ = syscall::bind_notification(launcher.deaths); let ep = launcher.spawn_ep; server_framework::serve(&mut launcher, ep) } -fn launch_all(launcher: &mut launch::Launcher, programs: &[manifest::Program]) { +fn launch_all(launcher: &mut launch::Launcher, programs: &'static [manifest::Program]) { for p in programs { - libkos::print!("init: loading {}... ", p.name); - match launcher.launch(p) { - Ok(started) => libkos::println!("ok{}", started.note), - Err(why) => libkos::println!("SKIP ({})", why), + crate::out!("init: loading {}... ", p.name); + match launcher.run(p, false) { + Ok(note) => crate::outln!("ok{}", note), + Err(why) => crate::outln!("SKIP ({})", why), } } } diff --git a/crates/init/src/manifest.rs b/crates/init/src/manifest.rs index 745c728..3e9d8d4 100644 --- a/crates/init/src/manifest.rs +++ b/crates/init/src/manifest.rs @@ -49,8 +49,15 @@ pub struct Program { pub spawns: Option, /// May halt the machine. pub system_control: bool, - /// May sign the user out (`types::spawn::SIGN_OUT`), in the last slot. + /// May sign the user out (`types::spawn::SIGN_OUT`), in the next slot. pub signs_out: bool, + /// Takes Ctrl+C: a terminal copy with `types::terminal::CONTROL`, last. + pub takes_ctrl_c: bool, + /// Started again if it dies (see init's supervise.rs). + pub restart: bool, + /// Its clients keep state in it (a subscription, focus, a handle), so + /// they are restarted after it is. + pub restarts_clients: bool, } /// Enough for a small program: its image, stack and a little heap. @@ -75,22 +82,29 @@ const fn server(name: &'static str, file: &'static [u8], limits: Limits) -> Prog spawns: None, system_control: false, signs_out: false, + takes_ctrl_c: false, + restart: true, + restarts_clients: false, } } /// Started on every boot. The log server comes first so the rest can print. pub const SERVERS: &[Program] = &[ Program { console: true, ..server("log", b"bin/log", SMALL) }, - Program { devices: &[Device::Input], ..server("input", b"bin/input", Limits { regions: 4, ..SMALL }) }, + // The display subscribes to it. + Program { devices: &[Device::Input], restarts_clients: true, ..server("input", b"bin/input", Limits { regions: 4, ..SMALL }) }, // The input server's only client: it passes input on to the app in focus. Program { devices: &[Device::Gpu], uses: &["input"], + // Its clients subscribe and take focus. + restarts_clients: true, // The framebuffer alone is ~1000 pages at 1280x800. ..server("display", b"bin/display", Limits { pages: 1280, regions: 4, ..SMALL }) }, // Text on the display, lines from the keyboard (it takes input focus). - Program { uses: &["display"], ..server("terminal", b"bin/terminal", SMALL) }, + // Its clients wait on lines and Ctrl+C. + Program { uses: &["display"], restarts_clients: true, ..server("terminal", b"bin/terminal", SMALL) }, server("session", b"bin/session", SMALL), ]; @@ -102,6 +116,8 @@ pub const GATEKEEPER: Program = Program { devices: &[Device::Keystore, Device::Random], // A user's record is KiBs, and it handles a few at once. stack_pages: 12, + // It exits once it has handed over; a new one runs at each sign-in. + restart: false, // argon2's 16 MiB, its image and a little more. ..server("gatekeeper", b"bin/gatekeeper", Limits { pages: 4096 + 160, regions: 6, ..SMALL }) }; @@ -115,18 +131,22 @@ pub const SESSION: &[Program] = &[Program { storage: &[users(b"", true), disk(b"apps", false)], // Room for a few apps at a time. spawns: Some(Limits { pages: 512, endpoints: 8, notifications: 8, regions: 8 }), + // The shell holds handles to the apps it ran, which die with it. + restarts_clients: true, ..server("permission", b"bin/permission", Limits { pages: 96, ..SMALL }) }]; /// Run on the test image after the SMP self-test, instead of the desktop: -/// spawns through init, then reports and halts. -pub const TESTS: &[Program] = &[Program { +/// spawns through init, then reports and halts. `flaky` dies when asked, +/// for spawntest to see it restarted. +pub const TESTS: &[Program] = &[server("flaky", b"bin/flaky", SMALL), Program { serves: false, // The permission manager, to run an app the way the shell does. - uses: &["permission"], + uses: &["permission", "flaky"], storage: &[disk(b"", true), users(b"home", true)], spawns: Some(Limits { pages: 128, endpoints: 2, notifications: 2, regions: 2 }), system_control: true, + restart: false, ..server("spawntest", b"bin/spawntest", SMALL) }]; @@ -138,5 +158,6 @@ pub const DESKTOP: &[Program] = &[Program { uses: &["terminal", "permission"], storage: &[users(b"home", true)], signs_out: true, + takes_ctrl_c: true, ..server("shell", b"bin/shell", SMALL) }]; diff --git a/crates/init/src/output.rs b/crates/init/src/output.rs new file mode 100644 index 0000000..61db0eb --- /dev/null +++ b/crates/init/src/output.rs @@ -0,0 +1,102 @@ +//! What init prints never waits on the log server: init restarts the log +//! server, so a blocking call to it could wait on itself. Text goes into a +//! ring; a thread of init's own takes it to the log and waits there when +//! the log is down, instead of init. Until that thread runs (early boot), +//! init prints to the console directly. + +use core::fmt; +use core::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; +use libkos::syscall; +use types::bounded::CapSlot; + +const SIZE: usize = 4096; +/// Bytes waiting: written at `TAIL` by init's thread, read at `HEAD` by +/// the printer. +static RING: [AtomicU8; SIZE] = [const { AtomicU8::new(0) }; SIZE]; +static HEAD: AtomicUsize = AtomicUsize::new(0); +static TAIL: AtomicUsize = AtomicUsize::new(0); +/// Bytes dropped because the ring was full: the log was down a long time. +static LOST: AtomicUsize = AtomicUsize::new(0); +/// The printer's notification, once it runs. +static WAKE: AtomicUsize = AtomicUsize::new(usize::MAX); + +fn wake() -> Option { + u8::try_from(WAKE.load(Ordering::Acquire)).ok().and_then(CapSlot::new) +} + +pub(crate) fn write(bytes: &[u8]) { + let Some(wake) = wake() else { + libkos::io::write_bytes(bytes); + return; + }; + let tail = TAIL.load(Ordering::Relaxed); + let room = SIZE - (tail - HEAD.load(Ordering::Acquire)); + let n = bytes.len().min(room); + for (i, b) in bytes[..n].iter().enumerate() { + RING[(tail + i) % SIZE].store(*b, Ordering::Relaxed); + } + TAIL.store(tail + n, Ordering::Release); + LOST.fetch_add(bytes.len() - n, Ordering::Relaxed); + let _ = syscall::signal(wake); +} + +struct Ring; + +impl fmt::Write for Ring { + fn write_str(&mut self, s: &str) -> fmt::Result { + write(s.as_bytes()); + Ok(()) + } +} + +pub(crate) fn print(args: fmt::Arguments) { + let _ = fmt::Write::write_fmt(&mut Ring, args); +} + +/// Starts the printer; from here on init's output goes through the ring. +pub(crate) fn start() -> bool { + let Ok(wake) = syscall::create_notification() else { return false }; + WAKE.store(wake.value() as usize, Ordering::Release); + if libkos::thread::spawn(&WAKE, printer) { + return true; + } + WAKE.store(usize::MAX, Ordering::Release); + false +} + +fn printer(_: &'static AtomicUsize) -> ! { + let mut chunk = [0u8; 256]; + loop { + if let Some(wake) = wake() { + let _ = syscall::wait(wake); + } + loop { + let (head, tail) = (HEAD.load(Ordering::Relaxed), TAIL.load(Ordering::Acquire)); + if head == tail { + break; + } + let n = (tail - head).min(chunk.len()); + for (i, b) in chunk[..n].iter_mut().enumerate() { + *b = RING[(head + i) % SIZE].load(Ordering::Relaxed); + } + HEAD.store(head + n, Ordering::Release); + // Blocks while the log server is down; init goes on. + libkos::io::write_bytes(&chunk[..n]); + } + let lost = LOST.swap(0, Ordering::Relaxed); + if lost > 0 { + libkos::println!("init: {} bytes of output lost while the log was down", lost); + } + } +} + +#[macro_export] +macro_rules! out { + ($($arg:tt)*) => ($crate::output::print(format_args!($($arg)*))); +} + +#[macro_export] +macro_rules! outln { + () => ($crate::out!("\n")); + ($($arg:tt)*) => ($crate::out!("{}\n", format_args!($($arg)*))); +} diff --git a/crates/init/src/selftest.rs b/crates/init/src/selftest.rs index a2958cc..fca1d04 100644 --- a/crates/init/src/selftest.rs +++ b/crates/init/src/selftest.rs @@ -86,7 +86,7 @@ fn run_phases(storage: &mut Storage, scratch: &mut [u8]) -> Result<(), &'static if syscall::poll(events) != Ok(1) { return Err("nobody heard the spinner fault"); } - libkos::println!("smptest: revoke stops every hart: ok"); + crate::outln!("smptest: revoke stops every hart: ok"); // Kill must stop threads running on other harts before it returns. let spinner = start_spinner(storage, scratch, page, words)?; @@ -98,7 +98,7 @@ fn run_phases(storage: &mut Storage, scratch: &mut [u8]) -> Result<(), &'static if syscall::poll(events) != Ok(1) { return Err("nobody heard the spinner killed"); } - libkos::println!("smptest: kill stops every hart: ok"); + crate::outln!("smptest: kill stops every hart: ok"); Ok(()) } @@ -116,7 +116,7 @@ fn timer_is_on_time(events: CapSlot) -> Result<(), &'static str> { if woke - deadline > 20 * TICKS_PER_MS { return Err("the timer fired late"); } - libkos::println!("smptest: timer wakes on time: ok"); + crate::outln!("smptest: timer wakes on time: ok"); Ok(()) } @@ -126,11 +126,11 @@ pub fn run(storage: &mut Storage, scratch: &mut [u8]) { let Err(why) = run_phases(storage, scratch) else { return; }; - libkos::println!("SMPTEST FAIL: {}", why); + crate::outln!("SMPTEST FAIL: {}", why); match crate::devices::system_control() { Some(control) => syscall::halt(control), None => { - libkos::println!("smptest: no SystemControl capability to halt with"); + crate::outln!("smptest: no SystemControl capability to halt with"); crate::idle() } } diff --git a/crates/init/src/session.rs b/crates/init/src/session.rs index 66817a1..2d270ec 100644 --- a/crates/init/src/session.rs +++ b/crates/init/src/session.rs @@ -78,9 +78,9 @@ fn purge(fs: &mut Storage, folder: UserFolder) -> Result<(), &'static str> { /// Runs the gatekeeper until someone is in and starts their session. pub(crate) fn sign_in(l: &mut Launcher) -> Result<(), &'static str> { - libkos::print!("init: loading gatekeeper... "); + crate::out!("init: loading gatekeeper... "); let gatekeeper = l.launch(&GATEKEEPER)?; - libkos::println!("ok{}", gatekeeper.note); + crate::outln!("ok{}", gatekeeper.note); let handover = gatekeeper.ep.ok_or("the gatekeeper serves nothing").and_then(ask); // It exits on its own once it has handed over. syscall::drop_cap(gatekeeper.process); @@ -91,11 +91,11 @@ pub(crate) fn sign_in(l: &mut Launcher) -> Result<(), &'static str> { // While storage is locked: it removes trees only then. if h.flags & unlocked::FRESH != 0 { fs.purge(b"users").map_err(|_| "cannot clear the users of an old keystore")?; - libkos::println!("init: a new keystore, every earlier user's files removed"); + crate::outln!("init: a new keystore, every earlier user's files removed"); } for id in h.removed.iter().flatten() { purge(fs, UserFolder::of(*id))?; - libkos::println!("init: a removed user's files removed"); + crate::outln!("init: a removed user's files removed"); } if guest { purge(fs, UserFolder::GUEST)?; @@ -107,31 +107,20 @@ pub(crate) fn sign_in(l: &mut Launcher) -> Result<(), &'static str> { } crate::factory::prepare(fs, folder, scratch)?; let name = core::str::from_utf8(&h.name[..h.name_len]).unwrap_or("someone"); - libkos::println!("init: {} is in", name); + crate::outln!("init: {} is in", name); l.user = Some(User { folder, guest }); start_session(l); Ok(()) } -/// Starts what runs for a signed-in user, keeping each process to stop -/// at sign-out. +/// Starts what runs for a signed-in user; it is stopped at sign-out. fn start_session(l: &mut Launcher) { let desktop = l.desktop; for p in SESSION.iter().chain(desktop) { - libkos::print!("init: loading {}... ", p.name); - match l.launch(p) { - Ok(started) => { - libkos::println!("ok{}", started.note); - match l.running.iter_mut().find(|r| r.is_none()) { - Some(free) => *free = Some(started.process), - None => { - // Untracked, it could not be stopped at sign-out. - let _ = syscall::kill(started.process); - syscall::drop_cap(started.process); - } - } - } - Err(why) => libkos::println!("SKIP ({})", why), + crate::out!("init: loading {}... ", p.name); + match l.run(p, true) { + Ok(note) => crate::outln!("ok{}", note), + Err(why) => crate::outln!("SKIP ({})", why), } } } @@ -139,25 +128,19 @@ fn start_session(l: &mut Launcher) { /// Stops the session, locks storage, and lets the next one in. pub(crate) fn sign_out(l: &mut Launcher) { l.spawner.clear(); - for process in l.running.iter_mut().filter_map(Option::take) { - let _ = syscall::kill(process); - syscall::drop_cap(process); - } - if let Some(terminal) = l.served("terminal") { - let _ = syscall::request(terminal, [types::terminal::SIGNED_OUT, 0, 0, 0]); - } + l.stop_session(); let user = l.user.take(); let (fs, _) = l.storage(); let locked = fs.lock().is_ok(); // A guest leaves nothing behind. let cleared = !user.is_some_and(|u| u.guest) || purge(fs, UserFolder::GUEST).is_ok(); - libkos::println!("init: signed out"); + crate::outln!("init: signed out"); let next = match locked && cleared { true => sign_in(l), false => Err("storage would not lock"), }; if let Err(why) = next { - libkos::println!("init: STOP ({})", why); + crate::outln!("init: STOP ({})", why); crate::idle(); } } diff --git a/crates/init/src/spawn.rs b/crates/init/src/spawn.rs index b667ce7..ea08e95 100644 --- a/crates/init/src/spawn.rs +++ b/crates/init/src/spawn.rs @@ -20,6 +20,9 @@ const MAX_CHILDREN: usize = 8; struct Child { client: usize, + /// Stopped by init: its limits go back to nobody, since its client may + /// be gone, and its slot stays taken until its death is heard. + stopped: bool, limits: Limits, process: CapSlot, } @@ -71,11 +74,19 @@ impl Spawner { impl Spawner { /// Sign-out: every client goes, and every child is stopped. pub(crate) fn clear(&mut self) { - for child in self.children.iter_mut().filter_map(Option::take) { - let _ = syscall::kill(child.process); - syscall::drop_cap(child.process); + for client in 0..MAX_CLIENTS { + self.remove_client(Badge::new(client + 1)); } - self.allowances = [None; MAX_CLIENTS]; + } + + /// The client behind `badge` is gone: so are its children. + pub(crate) fn remove_client(&mut self, badge: Badge) { + let Some(client) = badge.raw().checked_sub(1).filter(|c| *c < MAX_CLIENTS) else { return }; + for child in self.children.iter_mut().flatten().filter(|c| c.client == client && !c.stopped) { + let _ = syscall::kill(child.process); + child.stopped = true; + } + self.allowances[client] = None; } } @@ -131,7 +142,7 @@ impl Launcher<'_> { let Some(loaded) = loaded else { return refused(if found { reply::ERROR } else { reply::NOT_FOUND }); }; - if !self.watch(loaded.process_cap, index) { + if !self.watch(loaded.process_cap, 1 << index) { let _ = syscall::kill(loaded.process_cap); syscall::drop_cap(loaded.process_cap); return refused(reply::ERROR); @@ -139,22 +150,11 @@ impl Launcher<'_> { let call_only = PermissionSet::empty().with(Permission::Execute); let log = libkos::start::log().and_then(|l| syscall::delegate(l, loaded.process_cap, call_only)); self.spawner.allowances[client] = Some(sub(left, r.limits)); - self.spawner.children[index] = Some(Child { client, limits: r.limits, process: loaded.process_cap }); + self.spawner.children[index] = Some(Child { client, stopped: false, limits: r.limits, process: loaded.process_cap }); let log = log.map_or(NONE, |s| s.value() as usize); ([reply::OK, loaded.entry_point.raw(), log, 0], Some(loaded.process_cap)) } - /// Has the child's death signal bit `index` on init's death notification. - fn watch(&self, process: CapSlot, index: usize) -> bool { - let signal = PermissionSet::empty().with(Permission::Write); - let Ok(copy) = syscall::mint(self.deaths, Badge::new(1 << index), signal) else { - return false; - }; - let watched = syscall::watch_death(process, copy).is_ok(); - syscall::drop_cap(copy); - watched - } - /// Children behind `bits` died: their limits go back to their clients. fn reap(&mut self, bits: usize) { for (i, slot) in self.spawner.children.iter_mut().enumerate() { @@ -162,7 +162,7 @@ impl Launcher<'_> { continue; } if let Some(child) = slot.take() { - if let Some(Some(left)) = self.spawner.allowances.get_mut(child.client) { + if let Some(Some(left)) = self.spawner.allowances.get_mut(child.client).filter(|_| !child.stopped) { *left = add(*left, child.limits); } syscall::drop_cap(child.process); @@ -202,6 +202,7 @@ impl Server for Launcher<'_> { } fn signal(&mut self, bits: usize) { - self.reap(bits); + self.reap(bits & ((1 << MAX_CHILDREN) - 1)); + self.supervise(bits); } } diff --git a/crates/init/src/supervise.rs b/crates/init/src/supervise.rs new file mode 100644 index 0000000..60d800d --- /dev/null +++ b/crates/init/src/supervise.rs @@ -0,0 +1,316 @@ +//! Restarting what dies (see the vision's Watchdog note). Init watches every +//! server and session program it starts; one that dies is started again, +//! on the same endpoint so its clients keep their capabilities, after a +//! back-off that grows with each crash in a row until init gives up on it. +//! A server whose clients keep state in it (a subscription, focus, a +//! handle) has them restarted after it, in turn; see `restarts_clients`. Storage +//! is the exception: it holds the key and every grant, so its death stops +//! the system. +//! +//! Death signals carry a bit per program. A program init stops itself +//! keeps its slot until its death is heard, so a late signal can never be +//! taken for the death of whatever started next. + +use libkos::syscall; +use types::badge::Badge; +use types::bounded::CapSlot; +use types::permission::{Permission, PermissionSet}; +use types::time::Ticks; + +use crate::launch::{Launched, Launcher}; +use crate::manifest::Program; + +/// Programs watched at once, at most. +pub(crate) const MAX_SUPERVISED: usize = 12; +/// Init's death notification: spawned children below, storage, then a bit +/// per supervised program, and the timer on top. +const FIRST_BIT: usize = 16; +const STORAGE_BIT: usize = 1 << 15; +pub(crate) const TIMER_BIT: usize = 1 << 62; +/// Where boot left storage's process. +const STORAGE_PROCESS: CapSlot = CapSlot::const_new(1); + +const TICKS_PER_MS: u64 = 10_000; +/// The wait before each restart in a row, then giving up. +const DELAYS_MS: [u64; 4] = [0, 1_000, 5_000, 30_000]; +/// A run this long was not part of a crash loop. +const STABLE_MS: u64 = 60_000; + +/// How many crashes in a row, and so how long before the next restart. +#[derive(Clone, Copy, Default)] +pub(crate) struct Backoff { + crashes: usize, +} + +impl Backoff { + /// After a crash `ran_ms` into a run: how long to wait before starting + /// again, or `None` to give up. + pub(crate) fn after_crash(&mut self, ran_ms: u64) -> Option { + if ran_ms >= STABLE_MS { + self.crashes = 0; + } + let delay = DELAYS_MS.get(self.crashes).copied()?; + self.crashes += 1; + Some(delay) + } +} + +#[derive(Clone, Copy)] +enum State { + Running { process: CapSlot, since: u64 }, + /// Dead, started again at this time. + Waiting { at: u64 }, + /// Stopped by init: started again once its death is heard (a + /// dependency was restarted), or freed (signed out). + Stopping { process: CapSlot, restart: bool }, +} + +#[derive(Clone, Copy)] +pub(crate) struct Supervised { + program: &'static Program, + state: State, + session: bool, + backoff: Backoff, + /// Init's copies of what it gave the program, taken back at its death. + grants: [Option; 2], + spawn: Option, + /// Until what this run held is let go of. + holding: bool, +} + +impl Supervised { + /// What this run held through init, the first time only: by the time + /// its death is heard, a slot it freed may hold someone else's + /// capability. + fn take_held(&mut self) -> Option<([Option; 2], Option)> { + let held = (self.grants, self.spawn); + self.grants = [None; 2]; + self.spawn = None; + core::mem::take(&mut self.holding).then_some(held) + } +} + +fn now_ms() -> u64 { + syscall::get_time().raw() / TICKS_PER_MS +} + +fn bit(index: usize) -> usize { + 1 << (FIRST_BIT + index) +} + +impl Launcher<'_> { + /// Starts `p` and, if it is to be restarted or stopped at sign-out, + /// watches it. + pub(crate) fn run(&mut self, p: &'static Program, session: bool) -> Result<&'static str, &'static str> { + let index = self.supervised.iter().position(Option::is_none).ok_or("too many programs to watch")?; + let started = self.launch(p)?; + let note = started.note; + if !self.watch(started.process, bit(index)) { + let _ = syscall::kill(started.process); + self.forget(started); + return Err("cannot watch it"); + } + self.supervised[index] = Some(Supervised { + program: p, + state: State::Running { process: started.process, since: now_ms() }, + session, + backoff: Backoff::default(), + grants: started.grants, + spawn: started.spawn, + holding: true, + }); + Ok(note) + } + + /// Has `process`'s death signal `bits` on the death notification. + pub(crate) fn watch(&self, process: CapSlot, bits: usize) -> bool { + let signal = PermissionSet::empty().with(Permission::Write); + let Ok(copy) = syscall::mint(self.deaths, Badge::new(bits), signal) else { + return false; + }; + let watched = syscall::watch_death(process, copy).is_ok(); + syscall::drop_cap(copy); + watched + } + + /// Storage's death ends everything; it is heard like any other. + pub(crate) fn watch_storage(&self) -> bool { + self.watch(STORAGE_PROCESS, STORAGE_BIT) + } + + /// Lets go of what a dead (or never started) program held through init. + fn forget(&mut self, gone: Launched) { + syscall::drop_cap(gone.process); + self.release(gone.grants, gone.spawn); + } + + fn release(&mut self, grants: [Option; 2], spawn: Option) { + for grant in grants.into_iter().flatten() { + let _ = libkos::fs::release(grant); + syscall::drop_cap(grant); + } + if let Some(badge) = spawn { + self.spawner.remove_client(badge); + } + } + + /// Takes back what supervised program `index` held through init. + fn let_go(&mut self, index: usize) { + // Never a call to a server: it may be the one that is down. + let Some((grants, spawn)) = self.supervised[index].as_mut().and_then(Supervised::take_held) else { return }; + self.release(grants, spawn); + } + + /// Stops a running program on purpose; `restart` it once it is dead. + fn stop(&mut self, index: usize, restart: bool) { + let Some(s) = self.supervised[index].as_mut() else { return }; + if let State::Running { process, .. } = s.state { + let _ = syscall::kill(process); + s.state = State::Stopping { process, restart }; + self.let_go(index); + } + } + + /// Sign-out: every session program is stopped, never to restart. + pub(crate) fn stop_session(&mut self) { + for index in 0..MAX_SUPERVISED { + match self.supervised[index] { + Some(s) if s.session && matches!(s.state, State::Waiting { .. }) => { + // Dead already, waiting to restart: nothing to hear. + self.supervised[index] = None; + } + Some(s) if s.session => self.stop(index, false), + _ => {} + } + } + } + + /// Death signals and the timer, from init's server loop. + pub(crate) fn supervise(&mut self, bits: usize) { + if bits & STORAGE_BIT != 0 { + crate::outln!("init: storage died; nothing can go on without it"); + crate::idle(); + } + for index in 0..MAX_SUPERVISED { + if bits & bit(index) != 0 { + self.died(index); + } + } + self.restart_due(); + } + + fn died(&mut self, index: usize) { + // Nothing left to let go of if init stopped it. + self.let_go(index); + let Some(s) = self.supervised[index] else { return }; + let (process, restart) = match s.state { + State::Running { process, since } => { + let ran = now_ms().saturating_sub(since); + (process, if s.program.restart { Some(ran) } else { None }) + } + // Restarted on purpose: no crash to count. + State::Stopping { process, restart: true } => (process, Some(u64::MAX)), + State::Stopping { process, restart: false } => (process, None), + // Heard twice: nothing more to do. + State::Waiting { .. } => return, + }; + syscall::drop_cap(process); + let crashed = matches!(s.state, State::Running { .. }); + let mut backoff = s.backoff; + let delay = restart.and_then(|ran| if crashed { backoff.after_crash(ran) } else { Some(0) }); + match (crashed, delay) { + (true, Some(_)) => crate::outln!("init: {} died, restarting it", s.program.name), + (true, None) if s.program.restart => crate::outln!("init: {} keeps dying; not restarting it again", s.program.name), + _ => {} + } + self.supervised[index] = delay.map(|d| Supervised { + state: State::Waiting { at: now_ms() + d }, + backoff, + grants: [None; 2], + spawn: None, + ..s + }); + } + + /// Starts again whatever is due, then whoever uses it; sets the timer + /// for the next one. + fn restart_due(&mut self) { + let now = now_ms(); + for index in 0..MAX_SUPERVISED { + let Some(s) = self.supervised[index] else { continue }; + let State::Waiting { at } = s.state else { continue }; + if at > now { + continue; + } + match self.launch(s.program) { + Ok(started) if self.watch(started.process, bit(index)) => { + crate::outln!("init: {} is back", s.program.name); + self.supervised[index] = Some(Supervised { + state: State::Running { process: started.process, since: now }, + grants: started.grants, + spawn: started.spawn, + holding: true, + ..s + }); + if s.program.restarts_clients { + self.restart_users_of(s.program.name); + } + } + Ok(started) => { + let _ = syscall::kill(started.process); + self.forget(started); + self.supervised[index] = None; + crate::outln!("init: cannot watch {} again; left stopped", s.program.name); + } + Err(why) => { + self.supervised[index] = None; + crate::outln!("init: cannot start {} again ({}); left stopped", s.program.name, why); + } + } + } + let next = self.supervised.iter().flatten().filter_map(|s| match s.state { + State::Waiting { at } => Some(at), + _ => None, + }).min(); + if let Some(at) = next { + self.wake_at(at); + } + } + + /// Whoever calls `name` kept state in the old one: restart them too. + fn restart_users_of(&mut self, name: &str) { + for index in 0..MAX_SUPERVISED { + if let Some(s) = self.supervised[index].filter(|s| s.program.uses.contains(&name)) { + // One that is not to restart is just stopped. + self.stop(index, s.program.restart); + } + } + } + + /// The timer bit fires at `at_ms`. + fn wake_at(&self, at_ms: u64) { + let _ = syscall::set_timer(self.timer, Ticks::from_raw(at_ms * TICKS_PER_MS)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn restarts_back_off_then_give_up() { + let mut b = Backoff::default(); + let delays: [Option; 5] = core::array::from_fn(|_| b.after_crash(10)); + assert_eq!(delays, [Some(0), Some(1_000), Some(5_000), Some(30_000), None]); + } + + #[test] + fn a_long_run_forgives_earlier_crashes() { + let mut b = Backoff::default(); + for _ in 0..3 { + b.after_crash(10); + } + assert_eq!(b.after_crash(STABLE_MS), Some(0)); + assert_eq!(b.after_crash(10), Some(1_000)); + } +} diff --git a/crates/servers/terminal-server/src/server.rs b/crates/servers/terminal-server/src/server.rs index fe1f1fa..caa6768 100644 --- a/crates/servers/terminal-server/src/server.rs +++ b/crates/servers/terminal-server/src/server.rs @@ -247,20 +247,15 @@ impl Server for TerminalServer { self.console.flush(); [0; 4] } - types::terminal::ON_INTERRUPT => match (cap, self.on_interrupt) { - (Some(n), None) => { - self.on_interrupt = Some(n); + types::terminal::ON_INTERRUPT => match cap { + Some(n) if badge & types::terminal::CONTROL != 0 => { + if let Some(old) = self.on_interrupt.replace(n) { + self.console.forget(old); + } [0; 4] } _ => [MALFORMED, 0, 0, 0], }, - types::terminal::SIGNED_OUT if badge == 0 => { - if let Some(n) = self.on_interrupt.take() { - self.console.forget(n); - } - [0; 4] - } - types::terminal::SIGNED_OUT => [MALFORMED, 0, 0, 0], types::terminal::CLEAR => { self.handle(Request::Clear); [0; 4] diff --git a/crates/servers/terminal-server/src/tests/server.rs b/crates/servers/terminal-server/src/tests/server.rs index 6f4274b..dc69b68 100644 --- a/crates/servers/terminal-server/src/tests/server.rs +++ b/crates/servers/terminal-server/src/tests/server.rs @@ -105,8 +105,9 @@ fn line_of(t: &mut TerminalServer) -> Vec { line } +/// Registered through the shell's control copy. fn on_interrupt(t: &mut TerminalServer, slot: u8) -> usize { - t.handle_call(1, [types::terminal::ON_INTERRUPT, 0, 0, 0], CapSlot::new(slot)).0[0] + t.handle_call(1 | types::terminal::CONTROL, [types::terminal::ON_INTERRUPT, 0, 0, 0], CapSlot::new(slot)).0[0] } #[test] @@ -119,16 +120,6 @@ fn ctrl_c_while_a_program_runs_interrupts_it() { assert_eq!(&t.grid().row(0)[..2], b"^C"); } -#[test] -fn only_the_first_interrupt_notification_is_taken() { - let mut t = term(&[]); - assert_eq!(on_interrupt(&mut t, 7), 0); - assert_ne!(on_interrupt(&mut t, 8), 0); - t.console_mut_for_test().typed = vec![CTRL_C]; - t.signal(KEYS_BIT); - assert_eq!(t.console().interrupted, [CapSlot::new(7).unwrap()]); -} - #[test] fn keys_typed_ahead_start_the_next_line() { let mut t = term(b"s\n"); @@ -192,15 +183,16 @@ fn a_secret_line_shows_stars_and_is_forgotten() { } #[test] -fn only_init_lets_the_next_session_register_for_ctrl_c() { +fn only_the_control_copy_takes_ctrl_c_and_the_latest_counts() { use server_framework::MALFORMED; let mut t = term(&[]); + let plain = |t: &mut TerminalServer, slot| t.handle_call(1, [types::terminal::ON_INTERRUPT, 0, 0, 0], CapSlot::new(slot)).0[0]; + assert_eq!(plain(&mut t, 6), MALFORMED, "an app, with the plain copy, cannot take it"); assert_eq!(on_interrupt(&mut t, 7), 0); - assert_eq!(on_interrupt(&mut t, 8), MALFORMED, "an app cannot take Ctrl+C over"); - let signed_out = |t: &mut TerminalServer, badge| t.handle_call(badge, [types::terminal::SIGNED_OUT, 0, 0, 0], None).0[0]; - assert_eq!(signed_out(&mut t, 1), MALFORMED, "nor say the user signed out"); - assert_eq!(on_interrupt(&mut t, 8), MALFORMED); - assert_eq!(signed_out(&mut t, 0), 0); + assert_eq!(plain(&mut t, 8), MALFORMED, "nor take it over"); + assert_eq!(on_interrupt(&mut t, 9), 0, "a restarted shell registers again"); assert_eq!(t.console().forgotten, [CapSlot::new(7).unwrap()]); - assert_eq!(on_interrupt(&mut t, 9), 0, "the next shell may"); + t.console_mut_for_test().typed = vec![CTRL_C]; + t.signal(KEYS_BIT); + assert_eq!(t.console().interrupted, [CapSlot::new(9).unwrap()]); } diff --git a/crates/types/src/terminal.rs b/crates/types/src/terminal.rs index 5578136..8ce0120 100644 --- a/crates/types/src/terminal.rs +++ b/crates/types/src/terminal.rs @@ -12,11 +12,13 @@ pub const READ_SECRET: usize = 14; /// that drew on the screen itself. pub const REDRAW: usize = 12; /// Carries a notification the terminal signals on Ctrl+C while no line is -/// being read. Accepted once a session (see `SIGNED_OUT`). +/// being read. Only through a copy whose badge has `CONTROL`; the latest +/// one counts, so a restarted shell simply registers again. pub const ON_INTERRUPT: usize = 13; -/// From the unbadged holder (init) only, at sign-out: forgets who -/// `ON_INTERRUPT` signals, so the next session's shell may register. -pub const SIGNED_OUT: usize = 15; +/// On the badge of the terminal copy init gives the program that takes +/// Ctrl+C (the shell). The copies it passes on to apps carry its badge +/// too, so it gives them its other, plain copy. +pub const CONTROL: usize = 1 << 62; /// What Ctrl+C types. pub const CTRL_C: u8 = 3; diff --git a/docs/ipc.md b/docs/ipc.md index f5bf114..aa062a5 100644 --- a/docs/ipc.md +++ b/docs/ipc.md @@ -118,11 +118,24 @@ stops there. Signing out (`types::spawn::SIGN_OUT`, sent one way through the copy of init's endpoint the shell holds) stops everything the session ran (the -shell, the permission manager, every app), tells the terminal -(`SIGNED_OUT`, init only), locks storage (`LOCK`, init only: every grant +shell, the permission manager, every app), locks storage (`LOCK`, init only: every grant taken back, every file closed, the key forgotten), removes a guest's files, and runs the gatekeeper again. +Init restarts what dies (`init/src/supervise.rs`): every server, and the +session's programs, except the gatekeeper and storage. A restarted server +gets the same endpoint, so its clients' capabilities reach the new one; +what it held in memory is gone. One whose clients keep state in it +(`restarts_clients`: input, display, terminal, the permission manager) has +them restarted after it, in turn. Crashes in a row wait longer each time +(at once, 1 s, 5 s, 30 s), then init gives up on it; a minute's run +forgives earlier crashes. What init gave a program through itself (its +storage grants, its spawn allowance and the children spawned from it) is +taken back as soon as it stops or dies. Storage's death stops the system: +it holds the key and every grant. Init's main thread never waits on a +server it restarts: its own output goes through a thread of its own +(`init/src/output.rs`), which waits on the log server instead. + A program's `uses` endpoints arrive as badged copies with `Execute` and `Delegate`: the badge (unique per copy) tells the server which client is calling, and `Delegate` lets the client send capabilities with its calls @@ -151,7 +164,9 @@ is whatever text endpoint a program was given. `READ` (`types::terminal`) blocks until a line is typed and returns it in chunks, the last ending in `\n`. Ctrl+C there drops the line. Keys typed while no line is read are kept for the next one, except Ctrl+C, which signals the notification the -session's first `ON_INTERRUPT` caller (its shell) gave. `REDRAW` repaints the screen and takes +latest `ON_INTERRUPT` caller gave. Only a terminal copy whose badge has +`CONTROL` may register: init mints one for the shell alone, and the copy +the shell passes on to apps is its plain one. `REDRAW` repaints the screen and takes input focus back. The desktop is the shell, given the terminal, the permission manager, diff --git a/scripts/make-disk-image.sh b/scripts/make-disk-image.sh index ddb8e9a..bc4a78c 100755 --- a/scripts/make-disk-image.sh +++ b/scripts/make-disk-image.sh @@ -62,6 +62,7 @@ if [ -n "${SMPTEST:-}" ]; then copy smptest smptest copy spawntest spawntest copy hello hello + copy flaky flaky fi rm -f "${IMG}"