feat(init): supervise servers+session progs: restart on death, same endpoint, backoff 0/1/5/30s then give up, restarts_clients cascade; release grants/spawn at stop (stale-slot safe); storage death fatal; init output via printer thread (never blocks on log); terminal Ctrl+C via CONTROL-badged copy, latest wins; flaky test server + spawntest restart/backoff check

This commit is contained in:
2026-10-28 01:44:24 +01:00
parent c4871320a0
commit 28f1ace00a
23 changed files with 736 additions and 144 deletions

View File

@@ -0,0 +1,14 @@
[package]
name = "flaky"
version = "0.1.0"
edition = "2024"
[[bin]]
name = "flaky"
test = false
bench = false
[dependencies]
types = { workspace = true }
libkos = { workspace = true }
server-framework = { workspace = true }

View File

@@ -0,0 +1,69 @@
//! A server that dies or hangs when asked: on the test image only, so the
//! self-test can watch init restart it (see init's supervise.rs).
#![no_std]
#![forbid(unsafe_code)]
use libkos::syscall;
use server_framework::{Decode, Encode, MALFORMED, Server};
use types::bounded::CapSlot;
/// Requests (word 0); a reply carries how many requests this instance has
/// served, this one included: a restarted server starts at 1.
pub mod op {
pub const WHO: usize = 1;
/// Panics: the process dies.
pub const CRASH: usize = 2;
/// Blocks for good inside the handler.
pub const HANG: usize = 3;
}
/// Everything goes through `handle_call`.
struct Never;
impl Decode for Never {
fn decode(_: [usize; 4]) -> Option<Self> {
None
}
}
impl Encode for Never {
fn encode(&self) -> [usize; 4] {
[MALFORMED, 0, 0, 0]
}
}
struct Flaky {
served: usize,
}
impl Server for Flaky {
type Request = Never;
type Response = Never;
fn handle(&mut self, request: Never) -> Never {
request
}
fn handle_call(&mut self, _badge: usize, words: [usize; 4], _cap: Option<CapSlot>) -> ([usize; 4], Option<syscall::Give>) {
match words[0] {
op::WHO => {}
op::CRASH => panic!("asked to crash"),
op::HANG => {
// A notification nobody signals.
if let Ok(never) = syscall::create_notification() {
loop {
let _ = syscall::wait(never);
}
}
}
_ => return ([MALFORMED, 0, 0, 0], None),
}
self.served += 1;
([self.served, 0, 0, 0], None)
}
}
pub fn run() -> ! {
server_framework::serve(&mut Flaky { served: 0 }, CapSlot::const_new(0))
}

View File

@@ -0,0 +1,11 @@
#![no_std]
#![forbid(unsafe_code)]
#![no_main]
extern crate libkos;
libkos::entry!(main);
fn main() {
flaky::run();
}

View File

@@ -25,6 +25,8 @@ const PERMISSION: CapSlot = CapSlot::const_new(1);
/// The user's files, their `home`: the shell's `/`.
const HOME: CapSlot = CapSlot::const_new(2);
const SESSION: CapSlot = CapSlot::const_new(3);
/// The terminal again, the copy that takes Ctrl+C; apps get the plain one.
const CONTROL: CapSlot = CapSlot::const_new(4);
/// Bits on the shell's notification: the program died (its death watch is
/// unbadged), or Ctrl+C was typed (the terminal's copy).
@@ -294,7 +296,7 @@ fn main() {
println!("shell: no permission manager");
return;
};
if !libkos::terminal::on_interrupt(TERMINAL, done, types::badge::Badge::new(INTERRUPTED)) {
if !libkos::terminal::on_interrupt(CONTROL, done, types::badge::Badge::new(INTERRUPTED)) {
println!("shell: Ctrl+C will not work");
}
let mut sh = Shell {

View File

@@ -11,4 +11,5 @@ bench = false
[dependencies]
types = { workspace = true }
manifest = { workspace = true }
flaky = { path = "../flaky" }
libkos = { workspace = true }

View File

@@ -17,10 +17,11 @@ use types::budget::Limits;
/// Where init put them (see init's manifest: `uses`, storage (the disk,
/// the user's home), the spawn endpoint, then control).
const PERMISSION: CapSlot = CapSlot::const_new(0);
const STORAGE: CapSlot = CapSlot::const_new(1);
const HOME: CapSlot = CapSlot::const_new(2);
const INIT: CapSlot = CapSlot::const_new(3);
const CONTROL: CapSlot = CapSlot::const_new(4);
const FLAKY: CapSlot = CapSlot::const_new(1);
const STORAGE: CapSlot = CapSlot::const_new(2);
const HOME: CapSlot = CapSlot::const_new(3);
const INIT: CapSlot = CapSlot::const_new(4);
const CONTROL: CapSlot = CapSlot::const_new(5);
/// Where storage's buffers are mapped: the whole disk's, a narrowed one's.
const STORAGE_BUF_VA: types::address::VirtualAddress = types::address::VirtualAddress::from_raw(0x0180_0000);
const NARROW_BUF_VA: types::address::VirtualAddress = types::address::VirtualAddress::from_raw(0x0190_0000);
@@ -172,6 +173,33 @@ fn encrypted_home(fs: &mut Storage, home_fs: &mut Storage) -> Result<(), &'stati
Ok(())
}
/// How many requests the `flaky` answering now has served.
fn flaky() -> Result<usize, &'static str> {
syscall::request(FLAKY, [flaky::op::WHO, 0, 0, 0]).map(|r| r[0]).map_err(|_| "flaky does not answer")
}
/// A server that dies is started again on the same endpoint: the caller's
/// capability reaches the new one. Twice in a row, the second waits.
fn restarted_after_crashes() -> Result<(), &'static str> {
if flaky()? != 1 || flaky()? != 2 {
return Err("flaky counts wrong");
}
// Dies mid-call: the call comes back with an error.
let _ = syscall::request(FLAKY, [flaky::op::CRASH, 0, 0, 0]);
if flaky()? != 1 {
return Err("the crashed server still answers");
}
let crashed = syscall::get_time().raw();
let _ = syscall::request(FLAKY, [flaky::op::CRASH, 0, 0, 0]);
let fresh = flaky()? == 1;
// QEMU virt's timebase is 10 MHz.
let waited_ms = (syscall::get_time().raw() - crashed) / 10_000;
if !fresh || waited_ms < 900 {
return Err("a second crash in a row was not backed off");
}
Ok(())
}
fn check() -> Result<(), &'static str> {
let mut fs = Storage::attach(STORAGE, STORAGE_BUF_VA).map_err(|_| "cannot attach to storage")?;
write_a_file(&mut fs)?;
@@ -201,6 +229,9 @@ fn check() -> Result<(), &'static str> {
revoke_stops_a_running_app(died)?;
libkos::println!("spawntest: revoking stops a running app, allowing lets it run: ok");
restarted_after_crashes()?;
libkos::println!("spawntest: a crashed server is restarted, a crash loop backs off: ok");
Ok(())
}

View File

@@ -125,7 +125,7 @@ pub(crate) fn prepare(fs: &mut Storage, folder: types::user::UserFolder, buf: &m
}
fs.mkdir(path.as_bytes()).map_err(|_| "cannot make an area")?;
copy_tree(fs, &Path::new(factory), &path, buf)?;
libkos::println!("init: set up {}", area);
crate::outln!("init: set up {}", area);
}
Ok(())
}

View File

@@ -15,6 +15,7 @@ use crate::devices::Devices;
use crate::load::{self, LoadedProcess};
use crate::manifest::{Device, Program, Scope};
use crate::spawn::Spawner;
use crate::supervise::{MAX_SUPERVISED, Supervised};
const USER_STACK_TOP: StackPointer = StackPointer::from_raw(0x10000);
const MAX_SERVED: usize = 8;
@@ -36,22 +37,28 @@ pub(crate) struct Launcher<'a> {
/// Bound to init's thread: spawned children's deaths, a bit each.
pub(crate) deaths: CapSlot,
pub(crate) spawner: Spawner,
/// Who is signed in, and what runs for them (stopped at sign-out).
/// Who is signed in.
pub(crate) user: Option<crate::session::User>,
pub(crate) running: [Option<CapSlot>; MAX_SESSION],
/// What a session runs after `SESSION`: the desktop, or the tests.
pub(crate) desktop: &'static [Program],
/// Every program init restarts or stops at sign-out (see supervise.rs).
pub(crate) supervised: [Option<Supervised>; MAX_SUPERVISED],
/// The console, handed to the log server (again, if it restarts).
console: Option<CapSlot>,
/// Signals the death notification's timer bit.
pub(crate) timer: CapSlot,
}
/// Programs one session runs at once, at most.
pub(crate) const MAX_SESSION: usize = 4;
/// What became of starting a program.
pub(crate) struct Launched {
pub(crate) note: &'static str,
pub(crate) process: CapSlot,
/// Its endpoint, if it serves one.
pub(crate) ep: Option<CapSlot>,
/// Init's copies of its storage grants and its spawn client's badge,
/// to take back when it dies.
pub(crate) grants: [Option<CapSlot>; 2],
pub(crate) spawn: Option<Badge>,
}
impl<'a> Launcher<'a> {
@@ -62,6 +69,8 @@ impl<'a> Launcher<'a> {
) -> Result<Self, &'static str> {
let spawn_ep = syscall::create_endpoint().ok_or("no spawn endpoint")?;
let deaths = syscall::create_notification().map_err(|_| "no death notification")?;
let timer = syscall::mint(deaths, Badge::new(crate::supervise::TIMER_BIT), perms(&[Permission::Write]))
.map_err(|_| "no timer")?;
// Storage, started by boot, is not among them: it is reached
// through grants (a program's `storage` scope).
let served = [None; MAX_SERVED];
@@ -75,8 +84,10 @@ impl<'a> Launcher<'a> {
deaths,
spawner: Spawner::new(),
user: None,
running: [None; MAX_SESSION],
desktop: &[],
supervised: [None; MAX_SUPERVISED],
console: libkos::start::console(),
timer,
})
}
@@ -84,11 +95,6 @@ impl<'a> Launcher<'a> {
(self.storage, self.scratch)
}
/// The endpoint of the running server `name`.
pub(crate) fn served(&self, name: &str) -> Option<CapSlot> {
self.endpoint(name)
}
fn endpoint(&self, name: &str) -> Option<CapSlot> {
self.served.iter().flatten().find(|(n, _)| *n == name).map(|(_, ep)| *ep)
}
@@ -125,18 +131,20 @@ impl<'a> Launcher<'a> {
syscall::drop_cap(copy);
given.ok_or("cannot give an endpoint")?;
}
// Storage issues its own badges, each naming a scope.
for scope in p.storage {
// Storage issues its own badges, each naming a scope; init keeps
// its copy, to take the grant back when the program dies.
let mut grants = [None; 2];
for (scope, kept) in p.storage.iter().zip(grants.iter_mut()) {
let mut path = [0u8; types::storage::MAX_PATH];
let path = self.scope_path(scope, &mut path).ok_or("a storage scope for nobody")?;
let grant = self.storage.grant(path, scope.write).map_err(|_| "storage refused a grant")?;
let given = syscall::delegate(grant, child, perms(&[Permission::Execute, Permission::Delegate]));
syscall::drop_cap(grant);
given.ok_or("cannot give storage")?;
}
if let Some(allowance) = p.spawns {
self.give_spawn(child, allowance)?;
*kept = Some(grant);
syscall::delegate(grant, child, perms(&[Permission::Execute, Permission::Delegate])).ok_or("cannot give storage")?;
}
let spawn = match p.spawns {
Some(allowance) => Some(self.give_spawn(child, allowance)?),
None => None,
};
if p.system_control {
let control = crate::devices::system_control().ok_or("no SystemControl to give")?;
syscall::delegate(control, child, perms(&[Permission::Execute])).ok_or("cannot give SystemControl")?;
@@ -149,7 +157,17 @@ impl<'a> Launcher<'a> {
syscall::drop_cap(copy);
given.ok_or("cannot give signing out")?;
}
let console = libkos::start::console()
if p.takes_ctrl_c {
let terminal = self.endpoint("terminal").ok_or("needs the terminal")?;
let badge = Badge::new(self.next_badge | types::terminal::CONTROL);
self.next_badge += 1;
let copy = syscall::mint(terminal, badge, perms(&[Permission::Execute, Permission::Delegate])).map_err(|_| "cannot mint")?;
// Delegate, to send its notification with the call.
let given = syscall::delegate(copy, child, perms(&[Permission::Execute, Permission::Delegate]));
syscall::drop_cap(copy);
given.ok_or("cannot give Ctrl+C")?;
}
let console = self.console
.filter(|_| p.console)
.and_then(|c| syscall::delegate(c, child, perms(&[Permission::Write])));
start(&loaded, console);
@@ -168,7 +186,7 @@ impl<'a> Launcher<'a> {
note = ", console handed over";
}
}
Ok(Launched { note, process: child, ep })
Ok(Launched { note, process: child, ep, grants, spawn })
}
/// The path on disk `scope` names: inside the user's folder if it is a
@@ -188,14 +206,14 @@ impl<'a> Launcher<'a> {
/// A badged copy of init's endpoint: the badge names the client and so
/// its allowance.
fn give_spawn(&mut self, child: CapSlot, allowance: types::budget::Limits) -> Result<(), &'static str> {
fn give_spawn(&mut self, child: CapSlot, allowance: types::budget::Limits) -> Result<Badge, &'static str> {
let badge: Badge = self.spawner.add_client(allowance).ok_or("too many spawn clients")?;
let minted = perms(&[Permission::Execute, Permission::Delegate]);
let copy = syscall::mint(self.spawn_ep, badge, minted).map_err(|_| "cannot mint")?;
// Delegate, to send the program file's grant with each request.
let given = syscall::delegate(copy, child, perms(&[Permission::Execute, Permission::Delegate]));
syscall::drop_cap(copy);
given.map(|_| ()).ok_or("cannot give the spawn endpoint")
given.map(|_| badge).ok_or("cannot give the spawn endpoint")
}
/// The driver waits for its device before serving, and runs without
@@ -214,12 +232,10 @@ impl<'a> Launcher<'a> {
}
}
/// From here on init prints through the log server, like everyone else,
/// and holds no console of its own.
/// From here on init prints through the log server, like everyone else
/// (by way of its printer, see output.rs); it keeps the console only to
/// hand it to a restarted log server.
fn hand_over_console(log: CapSlot) {
if let Some(console) = libkos::start::console() {
syscall::drop_cap(console);
}
libkos::start::set_console(None);
libkos::start::set_log(Some(log));
}

View File

@@ -8,9 +8,11 @@ mod factory;
mod launch;
mod load;
mod manifest;
mod output;
mod selftest;
mod session;
mod spawn;
mod supervise;
use libkos::fs::Storage;
use libkos::syscall;
@@ -25,37 +27,44 @@ pub(crate) const STORAGE_EP: CapSlot = CapSlot::const_new(2);
pub fn run() -> ! {
libkos::start::set_console(devices::console());
libkos::println!("init: started (pid={:#x})", syscall::get_process_id().raw());
let printer = output::start();
crate::outln!("init: started (pid={:#x})", syscall::get_process_id().raw());
if !printer {
crate::outln!("init: no printer thread; output waits on the log server");
}
// Storage waits for its disk before serving, so this comes first.
let found = devices::discover();
let disk =
libkos::device::give_device(STORAGE_EP, found.disk).is_ok() && found.disk.mmio.is_some();
libkos::println!(
crate::outln!(
"init: disk: {}",
if disk { "handed to its driver" } else { "none" }
);
let Ok(mut storage) = Storage::attach(STORAGE_EP, STORAGE_BUF_VA) else {
libkos::println!("init: cannot reach storage, nothing to load");
crate::outln!("init: cannot reach storage, nothing to load");
idle();
};
// Storage has nothing to print with yet; init speaks for it.
libkos::println!("init: storage: up");
crate::outln!("init: storage: up");
let Some(scratch) = load::scratch() else {
libkos::println!("init: no memory to load programs into");
crate::outln!("init: no memory to load programs into");
idle();
};
let mut launcher = match launch::Launcher::new(&mut storage, scratch, found) {
Ok(l) => l,
Err(why) => {
libkos::println!("init: {}", why);
crate::outln!("init: {}", why);
idle();
}
};
launch_all(&mut launcher, manifest::SERVERS);
give_storage_the_log();
if !launcher.watch_storage() {
crate::outln!("init: cannot watch storage");
}
// The test image carries the SMP self-test; it and the tests run
// instead of the desktop, and the last one halts the machine.
@@ -64,7 +73,7 @@ pub fn run() -> ! {
.open(b"bin/smptest")
.is_ok_and(|h| storage.close(h).is_ok())
{
libkos::println!("init: running the SMP self-test");
crate::outln!("init: running the SMP self-test");
selftest::run(storage, scratch);
manifest::TESTS
} else {
@@ -72,22 +81,22 @@ pub fn run() -> ! {
};
// Nothing that touches a user's files starts before someone is in.
if let Err(why) = session::sign_in(&mut launcher) {
libkos::println!("init: STOP ({})", why);
crate::outln!("init: STOP ({})", why);
idle();
}
libkos::println!("init: all loaded, serving spawns");
crate::outln!("init: all loaded, serving spawns");
let _ = syscall::bind_notification(launcher.deaths);
let ep = launcher.spawn_ep;
server_framework::serve(&mut launcher, ep)
}
fn launch_all(launcher: &mut launch::Launcher, programs: &[manifest::Program]) {
fn launch_all(launcher: &mut launch::Launcher, programs: &'static [manifest::Program]) {
for p in programs {
libkos::print!("init: loading {}... ", p.name);
match launcher.launch(p) {
Ok(started) => libkos::println!("ok{}", started.note),
Err(why) => libkos::println!("SKIP ({})", why),
crate::out!("init: loading {}... ", p.name);
match launcher.run(p, false) {
Ok(note) => crate::outln!("ok{}", note),
Err(why) => crate::outln!("SKIP ({})", why),
}
}
}

View File

@@ -49,8 +49,15 @@ pub struct Program {
pub spawns: Option<Limits>,
/// May halt the machine.
pub system_control: bool,
/// May sign the user out (`types::spawn::SIGN_OUT`), in the last slot.
/// May sign the user out (`types::spawn::SIGN_OUT`), in the next slot.
pub signs_out: bool,
/// Takes Ctrl+C: a terminal copy with `types::terminal::CONTROL`, last.
pub takes_ctrl_c: bool,
/// Started again if it dies (see init's supervise.rs).
pub restart: bool,
/// Its clients keep state in it (a subscription, focus, a handle), so
/// they are restarted after it is.
pub restarts_clients: bool,
}
/// Enough for a small program: its image, stack and a little heap.
@@ -75,22 +82,29 @@ const fn server(name: &'static str, file: &'static [u8], limits: Limits) -> Prog
spawns: None,
system_control: false,
signs_out: false,
takes_ctrl_c: false,
restart: true,
restarts_clients: false,
}
}
/// Started on every boot. The log server comes first so the rest can print.
pub const SERVERS: &[Program] = &[
Program { console: true, ..server("log", b"bin/log", SMALL) },
Program { devices: &[Device::Input], ..server("input", b"bin/input", Limits { regions: 4, ..SMALL }) },
// The display subscribes to it.
Program { devices: &[Device::Input], restarts_clients: true, ..server("input", b"bin/input", Limits { regions: 4, ..SMALL }) },
// The input server's only client: it passes input on to the app in focus.
Program {
devices: &[Device::Gpu],
uses: &["input"],
// Its clients subscribe and take focus.
restarts_clients: true,
// The framebuffer alone is ~1000 pages at 1280x800.
..server("display", b"bin/display", Limits { pages: 1280, regions: 4, ..SMALL })
},
// Text on the display, lines from the keyboard (it takes input focus).
Program { uses: &["display"], ..server("terminal", b"bin/terminal", SMALL) },
// Its clients wait on lines and Ctrl+C.
Program { uses: &["display"], restarts_clients: true, ..server("terminal", b"bin/terminal", SMALL) },
server("session", b"bin/session", SMALL),
];
@@ -102,6 +116,8 @@ pub const GATEKEEPER: Program = Program {
devices: &[Device::Keystore, Device::Random],
// A user's record is KiBs, and it handles a few at once.
stack_pages: 12,
// It exits once it has handed over; a new one runs at each sign-in.
restart: false,
// argon2's 16 MiB, its image and a little more.
..server("gatekeeper", b"bin/gatekeeper", Limits { pages: 4096 + 160, regions: 6, ..SMALL })
};
@@ -115,18 +131,22 @@ pub const SESSION: &[Program] = &[Program {
storage: &[users(b"", true), disk(b"apps", false)],
// Room for a few apps at a time.
spawns: Some(Limits { pages: 512, endpoints: 8, notifications: 8, regions: 8 }),
// The shell holds handles to the apps it ran, which die with it.
restarts_clients: true,
..server("permission", b"bin/permission", Limits { pages: 96, ..SMALL })
}];
/// Run on the test image after the SMP self-test, instead of the desktop:
/// spawns through init, then reports and halts.
pub const TESTS: &[Program] = &[Program {
/// spawns through init, then reports and halts. `flaky` dies when asked,
/// for spawntest to see it restarted.
pub const TESTS: &[Program] = &[server("flaky", b"bin/flaky", SMALL), Program {
serves: false,
// The permission manager, to run an app the way the shell does.
uses: &["permission"],
uses: &["permission", "flaky"],
storage: &[disk(b"", true), users(b"home", true)],
spawns: Some(Limits { pages: 128, endpoints: 2, notifications: 2, regions: 2 }),
system_control: true,
restart: false,
..server("spawntest", b"bin/spawntest", SMALL)
}];
@@ -138,5 +158,6 @@ pub const DESKTOP: &[Program] = &[Program {
uses: &["terminal", "permission"],
storage: &[users(b"home", true)],
signs_out: true,
takes_ctrl_c: true,
..server("shell", b"bin/shell", SMALL)
}];

102
crates/init/src/output.rs Normal file
View File

@@ -0,0 +1,102 @@
//! What init prints never waits on the log server: init restarts the log
//! server, so a blocking call to it could wait on itself. Text goes into a
//! ring; a thread of init's own takes it to the log and waits there when
//! the log is down, instead of init. Until that thread runs (early boot),
//! init prints to the console directly.
use core::fmt;
use core::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
use libkos::syscall;
use types::bounded::CapSlot;
const SIZE: usize = 4096;
/// Bytes waiting: written at `TAIL` by init's thread, read at `HEAD` by
/// the printer.
static RING: [AtomicU8; SIZE] = [const { AtomicU8::new(0) }; SIZE];
static HEAD: AtomicUsize = AtomicUsize::new(0);
static TAIL: AtomicUsize = AtomicUsize::new(0);
/// Bytes dropped because the ring was full: the log was down a long time.
static LOST: AtomicUsize = AtomicUsize::new(0);
/// The printer's notification, once it runs.
static WAKE: AtomicUsize = AtomicUsize::new(usize::MAX);
fn wake() -> Option<CapSlot> {
u8::try_from(WAKE.load(Ordering::Acquire)).ok().and_then(CapSlot::new)
}
pub(crate) fn write(bytes: &[u8]) {
let Some(wake) = wake() else {
libkos::io::write_bytes(bytes);
return;
};
let tail = TAIL.load(Ordering::Relaxed);
let room = SIZE - (tail - HEAD.load(Ordering::Acquire));
let n = bytes.len().min(room);
for (i, b) in bytes[..n].iter().enumerate() {
RING[(tail + i) % SIZE].store(*b, Ordering::Relaxed);
}
TAIL.store(tail + n, Ordering::Release);
LOST.fetch_add(bytes.len() - n, Ordering::Relaxed);
let _ = syscall::signal(wake);
}
struct Ring;
impl fmt::Write for Ring {
fn write_str(&mut self, s: &str) -> fmt::Result {
write(s.as_bytes());
Ok(())
}
}
pub(crate) fn print(args: fmt::Arguments) {
let _ = fmt::Write::write_fmt(&mut Ring, args);
}
/// Starts the printer; from here on init's output goes through the ring.
pub(crate) fn start() -> bool {
let Ok(wake) = syscall::create_notification() else { return false };
WAKE.store(wake.value() as usize, Ordering::Release);
if libkos::thread::spawn(&WAKE, printer) {
return true;
}
WAKE.store(usize::MAX, Ordering::Release);
false
}
fn printer(_: &'static AtomicUsize) -> ! {
let mut chunk = [0u8; 256];
loop {
if let Some(wake) = wake() {
let _ = syscall::wait(wake);
}
loop {
let (head, tail) = (HEAD.load(Ordering::Relaxed), TAIL.load(Ordering::Acquire));
if head == tail {
break;
}
let n = (tail - head).min(chunk.len());
for (i, b) in chunk[..n].iter_mut().enumerate() {
*b = RING[(head + i) % SIZE].load(Ordering::Relaxed);
}
HEAD.store(head + n, Ordering::Release);
// Blocks while the log server is down; init goes on.
libkos::io::write_bytes(&chunk[..n]);
}
let lost = LOST.swap(0, Ordering::Relaxed);
if lost > 0 {
libkos::println!("init: {} bytes of output lost while the log was down", lost);
}
}
}
#[macro_export]
macro_rules! out {
($($arg:tt)*) => ($crate::output::print(format_args!($($arg)*)));
}
#[macro_export]
macro_rules! outln {
() => ($crate::out!("\n"));
($($arg:tt)*) => ($crate::out!("{}\n", format_args!($($arg)*)));
}

View File

@@ -86,7 +86,7 @@ fn run_phases(storage: &mut Storage, scratch: &mut [u8]) -> Result<(), &'static
if syscall::poll(events) != Ok(1) {
return Err("nobody heard the spinner fault");
}
libkos::println!("smptest: revoke stops every hart: ok");
crate::outln!("smptest: revoke stops every hart: ok");
// Kill must stop threads running on other harts before it returns.
let spinner = start_spinner(storage, scratch, page, words)?;
@@ -98,7 +98,7 @@ fn run_phases(storage: &mut Storage, scratch: &mut [u8]) -> Result<(), &'static
if syscall::poll(events) != Ok(1) {
return Err("nobody heard the spinner killed");
}
libkos::println!("smptest: kill stops every hart: ok");
crate::outln!("smptest: kill stops every hart: ok");
Ok(())
}
@@ -116,7 +116,7 @@ fn timer_is_on_time(events: CapSlot) -> Result<(), &'static str> {
if woke - deadline > 20 * TICKS_PER_MS {
return Err("the timer fired late");
}
libkos::println!("smptest: timer wakes on time: ok");
crate::outln!("smptest: timer wakes on time: ok");
Ok(())
}
@@ -126,11 +126,11 @@ pub fn run(storage: &mut Storage, scratch: &mut [u8]) {
let Err(why) = run_phases(storage, scratch) else {
return;
};
libkos::println!("SMPTEST FAIL: {}", why);
crate::outln!("SMPTEST FAIL: {}", why);
match crate::devices::system_control() {
Some(control) => syscall::halt(control),
None => {
libkos::println!("smptest: no SystemControl capability to halt with");
crate::outln!("smptest: no SystemControl capability to halt with");
crate::idle()
}
}

View File

@@ -78,9 +78,9 @@ fn purge(fs: &mut Storage, folder: UserFolder) -> Result<(), &'static str> {
/// Runs the gatekeeper until someone is in and starts their session.
pub(crate) fn sign_in(l: &mut Launcher) -> Result<(), &'static str> {
libkos::print!("init: loading gatekeeper... ");
crate::out!("init: loading gatekeeper... ");
let gatekeeper = l.launch(&GATEKEEPER)?;
libkos::println!("ok{}", gatekeeper.note);
crate::outln!("ok{}", gatekeeper.note);
let handover = gatekeeper.ep.ok_or("the gatekeeper serves nothing").and_then(ask);
// It exits on its own once it has handed over.
syscall::drop_cap(gatekeeper.process);
@@ -91,11 +91,11 @@ pub(crate) fn sign_in(l: &mut Launcher) -> Result<(), &'static str> {
// While storage is locked: it removes trees only then.
if h.flags & unlocked::FRESH != 0 {
fs.purge(b"users").map_err(|_| "cannot clear the users of an old keystore")?;
libkos::println!("init: a new keystore, every earlier user's files removed");
crate::outln!("init: a new keystore, every earlier user's files removed");
}
for id in h.removed.iter().flatten() {
purge(fs, UserFolder::of(*id))?;
libkos::println!("init: a removed user's files removed");
crate::outln!("init: a removed user's files removed");
}
if guest {
purge(fs, UserFolder::GUEST)?;
@@ -107,31 +107,20 @@ pub(crate) fn sign_in(l: &mut Launcher) -> Result<(), &'static str> {
}
crate::factory::prepare(fs, folder, scratch)?;
let name = core::str::from_utf8(&h.name[..h.name_len]).unwrap_or("someone");
libkos::println!("init: {} is in", name);
crate::outln!("init: {} is in", name);
l.user = Some(User { folder, guest });
start_session(l);
Ok(())
}
/// Starts what runs for a signed-in user, keeping each process to stop
/// at sign-out.
/// Starts what runs for a signed-in user; it is stopped at sign-out.
fn start_session(l: &mut Launcher) {
let desktop = l.desktop;
for p in SESSION.iter().chain(desktop) {
libkos::print!("init: loading {}... ", p.name);
match l.launch(p) {
Ok(started) => {
libkos::println!("ok{}", started.note);
match l.running.iter_mut().find(|r| r.is_none()) {
Some(free) => *free = Some(started.process),
None => {
// Untracked, it could not be stopped at sign-out.
let _ = syscall::kill(started.process);
syscall::drop_cap(started.process);
}
}
}
Err(why) => libkos::println!("SKIP ({})", why),
crate::out!("init: loading {}... ", p.name);
match l.run(p, true) {
Ok(note) => crate::outln!("ok{}", note),
Err(why) => crate::outln!("SKIP ({})", why),
}
}
}
@@ -139,25 +128,19 @@ fn start_session(l: &mut Launcher) {
/// Stops the session, locks storage, and lets the next one in.
pub(crate) fn sign_out(l: &mut Launcher) {
l.spawner.clear();
for process in l.running.iter_mut().filter_map(Option::take) {
let _ = syscall::kill(process);
syscall::drop_cap(process);
}
if let Some(terminal) = l.served("terminal") {
let _ = syscall::request(terminal, [types::terminal::SIGNED_OUT, 0, 0, 0]);
}
l.stop_session();
let user = l.user.take();
let (fs, _) = l.storage();
let locked = fs.lock().is_ok();
// A guest leaves nothing behind.
let cleared = !user.is_some_and(|u| u.guest) || purge(fs, UserFolder::GUEST).is_ok();
libkos::println!("init: signed out");
crate::outln!("init: signed out");
let next = match locked && cleared {
true => sign_in(l),
false => Err("storage would not lock"),
};
if let Err(why) = next {
libkos::println!("init: STOP ({})", why);
crate::outln!("init: STOP ({})", why);
crate::idle();
}
}

View File

@@ -20,6 +20,9 @@ const MAX_CHILDREN: usize = 8;
struct Child {
client: usize,
/// Stopped by init: its limits go back to nobody, since its client may
/// be gone, and its slot stays taken until its death is heard.
stopped: bool,
limits: Limits,
process: CapSlot,
}
@@ -71,11 +74,19 @@ impl Spawner {
impl Spawner {
/// Sign-out: every client goes, and every child is stopped.
pub(crate) fn clear(&mut self) {
for child in self.children.iter_mut().filter_map(Option::take) {
let _ = syscall::kill(child.process);
syscall::drop_cap(child.process);
for client in 0..MAX_CLIENTS {
self.remove_client(Badge::new(client + 1));
}
self.allowances = [None; MAX_CLIENTS];
}
/// The client behind `badge` is gone: so are its children.
pub(crate) fn remove_client(&mut self, badge: Badge) {
let Some(client) = badge.raw().checked_sub(1).filter(|c| *c < MAX_CLIENTS) else { return };
for child in self.children.iter_mut().flatten().filter(|c| c.client == client && !c.stopped) {
let _ = syscall::kill(child.process);
child.stopped = true;
}
self.allowances[client] = None;
}
}
@@ -131,7 +142,7 @@ impl Launcher<'_> {
let Some(loaded) = loaded else {
return refused(if found { reply::ERROR } else { reply::NOT_FOUND });
};
if !self.watch(loaded.process_cap, index) {
if !self.watch(loaded.process_cap, 1 << index) {
let _ = syscall::kill(loaded.process_cap);
syscall::drop_cap(loaded.process_cap);
return refused(reply::ERROR);
@@ -139,22 +150,11 @@ impl Launcher<'_> {
let call_only = PermissionSet::empty().with(Permission::Execute);
let log = libkos::start::log().and_then(|l| syscall::delegate(l, loaded.process_cap, call_only));
self.spawner.allowances[client] = Some(sub(left, r.limits));
self.spawner.children[index] = Some(Child { client, limits: r.limits, process: loaded.process_cap });
self.spawner.children[index] = Some(Child { client, stopped: false, limits: r.limits, process: loaded.process_cap });
let log = log.map_or(NONE, |s| s.value() as usize);
([reply::OK, loaded.entry_point.raw(), log, 0], Some(loaded.process_cap))
}
/// Has the child's death signal bit `index` on init's death notification.
fn watch(&self, process: CapSlot, index: usize) -> bool {
let signal = PermissionSet::empty().with(Permission::Write);
let Ok(copy) = syscall::mint(self.deaths, Badge::new(1 << index), signal) else {
return false;
};
let watched = syscall::watch_death(process, copy).is_ok();
syscall::drop_cap(copy);
watched
}
/// Children behind `bits` died: their limits go back to their clients.
fn reap(&mut self, bits: usize) {
for (i, slot) in self.spawner.children.iter_mut().enumerate() {
@@ -162,7 +162,7 @@ impl Launcher<'_> {
continue;
}
if let Some(child) = slot.take() {
if let Some(Some(left)) = self.spawner.allowances.get_mut(child.client) {
if let Some(Some(left)) = self.spawner.allowances.get_mut(child.client).filter(|_| !child.stopped) {
*left = add(*left, child.limits);
}
syscall::drop_cap(child.process);
@@ -202,6 +202,7 @@ impl Server for Launcher<'_> {
}
fn signal(&mut self, bits: usize) {
self.reap(bits);
self.reap(bits & ((1 << MAX_CHILDREN) - 1));
self.supervise(bits);
}
}

View File

@@ -0,0 +1,316 @@
//! Restarting what dies (see the vision's Watchdog note). Init watches every
//! server and session program it starts; one that dies is started again,
//! on the same endpoint so its clients keep their capabilities, after a
//! back-off that grows with each crash in a row until init gives up on it.
//! A server whose clients keep state in it (a subscription, focus, a
//! handle) has them restarted after it, in turn; see `restarts_clients`. Storage
//! is the exception: it holds the key and every grant, so its death stops
//! the system.
//!
//! Death signals carry a bit per program. A program init stops itself
//! keeps its slot until its death is heard, so a late signal can never be
//! taken for the death of whatever started next.
use libkos::syscall;
use types::badge::Badge;
use types::bounded::CapSlot;
use types::permission::{Permission, PermissionSet};
use types::time::Ticks;
use crate::launch::{Launched, Launcher};
use crate::manifest::Program;
/// Programs watched at once, at most.
pub(crate) const MAX_SUPERVISED: usize = 12;
/// Init's death notification: spawned children below, storage, then a bit
/// per supervised program, and the timer on top.
const FIRST_BIT: usize = 16;
const STORAGE_BIT: usize = 1 << 15;
pub(crate) const TIMER_BIT: usize = 1 << 62;
/// Where boot left storage's process.
const STORAGE_PROCESS: CapSlot = CapSlot::const_new(1);
const TICKS_PER_MS: u64 = 10_000;
/// The wait before each restart in a row, then giving up.
const DELAYS_MS: [u64; 4] = [0, 1_000, 5_000, 30_000];
/// A run this long was not part of a crash loop.
const STABLE_MS: u64 = 60_000;
/// How many crashes in a row, and so how long before the next restart.
#[derive(Clone, Copy, Default)]
pub(crate) struct Backoff {
crashes: usize,
}
impl Backoff {
/// After a crash `ran_ms` into a run: how long to wait before starting
/// again, or `None` to give up.
pub(crate) fn after_crash(&mut self, ran_ms: u64) -> Option<u64> {
if ran_ms >= STABLE_MS {
self.crashes = 0;
}
let delay = DELAYS_MS.get(self.crashes).copied()?;
self.crashes += 1;
Some(delay)
}
}
#[derive(Clone, Copy)]
enum State {
Running { process: CapSlot, since: u64 },
/// Dead, started again at this time.
Waiting { at: u64 },
/// Stopped by init: started again once its death is heard (a
/// dependency was restarted), or freed (signed out).
Stopping { process: CapSlot, restart: bool },
}
#[derive(Clone, Copy)]
pub(crate) struct Supervised {
program: &'static Program,
state: State,
session: bool,
backoff: Backoff,
/// Init's copies of what it gave the program, taken back at its death.
grants: [Option<CapSlot>; 2],
spawn: Option<Badge>,
/// Until what this run held is let go of.
holding: bool,
}
impl Supervised {
/// What this run held through init, the first time only: by the time
/// its death is heard, a slot it freed may hold someone else's
/// capability.
fn take_held(&mut self) -> Option<([Option<CapSlot>; 2], Option<Badge>)> {
let held = (self.grants, self.spawn);
self.grants = [None; 2];
self.spawn = None;
core::mem::take(&mut self.holding).then_some(held)
}
}
fn now_ms() -> u64 {
syscall::get_time().raw() / TICKS_PER_MS
}
fn bit(index: usize) -> usize {
1 << (FIRST_BIT + index)
}
impl Launcher<'_> {
/// Starts `p` and, if it is to be restarted or stopped at sign-out,
/// watches it.
pub(crate) fn run(&mut self, p: &'static Program, session: bool) -> Result<&'static str, &'static str> {
let index = self.supervised.iter().position(Option::is_none).ok_or("too many programs to watch")?;
let started = self.launch(p)?;
let note = started.note;
if !self.watch(started.process, bit(index)) {
let _ = syscall::kill(started.process);
self.forget(started);
return Err("cannot watch it");
}
self.supervised[index] = Some(Supervised {
program: p,
state: State::Running { process: started.process, since: now_ms() },
session,
backoff: Backoff::default(),
grants: started.grants,
spawn: started.spawn,
holding: true,
});
Ok(note)
}
/// Has `process`'s death signal `bits` on the death notification.
pub(crate) fn watch(&self, process: CapSlot, bits: usize) -> bool {
let signal = PermissionSet::empty().with(Permission::Write);
let Ok(copy) = syscall::mint(self.deaths, Badge::new(bits), signal) else {
return false;
};
let watched = syscall::watch_death(process, copy).is_ok();
syscall::drop_cap(copy);
watched
}
/// Storage's death ends everything; it is heard like any other.
pub(crate) fn watch_storage(&self) -> bool {
self.watch(STORAGE_PROCESS, STORAGE_BIT)
}
/// Lets go of what a dead (or never started) program held through init.
fn forget(&mut self, gone: Launched) {
syscall::drop_cap(gone.process);
self.release(gone.grants, gone.spawn);
}
fn release(&mut self, grants: [Option<CapSlot>; 2], spawn: Option<Badge>) {
for grant in grants.into_iter().flatten() {
let _ = libkos::fs::release(grant);
syscall::drop_cap(grant);
}
if let Some(badge) = spawn {
self.spawner.remove_client(badge);
}
}
/// Takes back what supervised program `index` held through init.
fn let_go(&mut self, index: usize) {
// Never a call to a server: it may be the one that is down.
let Some((grants, spawn)) = self.supervised[index].as_mut().and_then(Supervised::take_held) else { return };
self.release(grants, spawn);
}
/// Stops a running program on purpose; `restart` it once it is dead.
fn stop(&mut self, index: usize, restart: bool) {
let Some(s) = self.supervised[index].as_mut() else { return };
if let State::Running { process, .. } = s.state {
let _ = syscall::kill(process);
s.state = State::Stopping { process, restart };
self.let_go(index);
}
}
/// Sign-out: every session program is stopped, never to restart.
pub(crate) fn stop_session(&mut self) {
for index in 0..MAX_SUPERVISED {
match self.supervised[index] {
Some(s) if s.session && matches!(s.state, State::Waiting { .. }) => {
// Dead already, waiting to restart: nothing to hear.
self.supervised[index] = None;
}
Some(s) if s.session => self.stop(index, false),
_ => {}
}
}
}
/// Death signals and the timer, from init's server loop.
pub(crate) fn supervise(&mut self, bits: usize) {
if bits & STORAGE_BIT != 0 {
crate::outln!("init: storage died; nothing can go on without it");
crate::idle();
}
for index in 0..MAX_SUPERVISED {
if bits & bit(index) != 0 {
self.died(index);
}
}
self.restart_due();
}
fn died(&mut self, index: usize) {
// Nothing left to let go of if init stopped it.
self.let_go(index);
let Some(s) = self.supervised[index] else { return };
let (process, restart) = match s.state {
State::Running { process, since } => {
let ran = now_ms().saturating_sub(since);
(process, if s.program.restart { Some(ran) } else { None })
}
// Restarted on purpose: no crash to count.
State::Stopping { process, restart: true } => (process, Some(u64::MAX)),
State::Stopping { process, restart: false } => (process, None),
// Heard twice: nothing more to do.
State::Waiting { .. } => return,
};
syscall::drop_cap(process);
let crashed = matches!(s.state, State::Running { .. });
let mut backoff = s.backoff;
let delay = restart.and_then(|ran| if crashed { backoff.after_crash(ran) } else { Some(0) });
match (crashed, delay) {
(true, Some(_)) => crate::outln!("init: {} died, restarting it", s.program.name),
(true, None) if s.program.restart => crate::outln!("init: {} keeps dying; not restarting it again", s.program.name),
_ => {}
}
self.supervised[index] = delay.map(|d| Supervised {
state: State::Waiting { at: now_ms() + d },
backoff,
grants: [None; 2],
spawn: None,
..s
});
}
/// Starts again whatever is due, then whoever uses it; sets the timer
/// for the next one.
fn restart_due(&mut self) {
let now = now_ms();
for index in 0..MAX_SUPERVISED {
let Some(s) = self.supervised[index] else { continue };
let State::Waiting { at } = s.state else { continue };
if at > now {
continue;
}
match self.launch(s.program) {
Ok(started) if self.watch(started.process, bit(index)) => {
crate::outln!("init: {} is back", s.program.name);
self.supervised[index] = Some(Supervised {
state: State::Running { process: started.process, since: now },
grants: started.grants,
spawn: started.spawn,
holding: true,
..s
});
if s.program.restarts_clients {
self.restart_users_of(s.program.name);
}
}
Ok(started) => {
let _ = syscall::kill(started.process);
self.forget(started);
self.supervised[index] = None;
crate::outln!("init: cannot watch {} again; left stopped", s.program.name);
}
Err(why) => {
self.supervised[index] = None;
crate::outln!("init: cannot start {} again ({}); left stopped", s.program.name, why);
}
}
}
let next = self.supervised.iter().flatten().filter_map(|s| match s.state {
State::Waiting { at } => Some(at),
_ => None,
}).min();
if let Some(at) = next {
self.wake_at(at);
}
}
/// Whoever calls `name` kept state in the old one: restart them too.
fn restart_users_of(&mut self, name: &str) {
for index in 0..MAX_SUPERVISED {
if let Some(s) = self.supervised[index].filter(|s| s.program.uses.contains(&name)) {
// One that is not to restart is just stopped.
self.stop(index, s.program.restart);
}
}
}
/// The timer bit fires at `at_ms`.
fn wake_at(&self, at_ms: u64) {
let _ = syscall::set_timer(self.timer, Ticks::from_raw(at_ms * TICKS_PER_MS));
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn restarts_back_off_then_give_up() {
let mut b = Backoff::default();
let delays: [Option<u64>; 5] = core::array::from_fn(|_| b.after_crash(10));
assert_eq!(delays, [Some(0), Some(1_000), Some(5_000), Some(30_000), None]);
}
#[test]
fn a_long_run_forgives_earlier_crashes() {
let mut b = Backoff::default();
for _ in 0..3 {
b.after_crash(10);
}
assert_eq!(b.after_crash(STABLE_MS), Some(0));
assert_eq!(b.after_crash(10), Some(1_000));
}
}

View File

@@ -247,20 +247,15 @@ impl<C: Console> Server for TerminalServer<C> {
self.console.flush();
[0; 4]
}
types::terminal::ON_INTERRUPT => match (cap, self.on_interrupt) {
(Some(n), None) => {
self.on_interrupt = Some(n);
types::terminal::ON_INTERRUPT => match cap {
Some(n) if badge & types::terminal::CONTROL != 0 => {
if let Some(old) = self.on_interrupt.replace(n) {
self.console.forget(old);
}
[0; 4]
}
_ => [MALFORMED, 0, 0, 0],
},
types::terminal::SIGNED_OUT if badge == 0 => {
if let Some(n) = self.on_interrupt.take() {
self.console.forget(n);
}
[0; 4]
}
types::terminal::SIGNED_OUT => [MALFORMED, 0, 0, 0],
types::terminal::CLEAR => {
self.handle(Request::Clear);
[0; 4]

View File

@@ -105,8 +105,9 @@ fn line_of(t: &mut TerminalServer<Fake>) -> Vec<u8> {
line
}
/// Registered through the shell's control copy.
fn on_interrupt(t: &mut TerminalServer<Fake>, slot: u8) -> usize {
t.handle_call(1, [types::terminal::ON_INTERRUPT, 0, 0, 0], CapSlot::new(slot)).0[0]
t.handle_call(1 | types::terminal::CONTROL, [types::terminal::ON_INTERRUPT, 0, 0, 0], CapSlot::new(slot)).0[0]
}
#[test]
@@ -119,16 +120,6 @@ fn ctrl_c_while_a_program_runs_interrupts_it() {
assert_eq!(&t.grid().row(0)[..2], b"^C");
}
#[test]
fn only_the_first_interrupt_notification_is_taken() {
let mut t = term(&[]);
assert_eq!(on_interrupt(&mut t, 7), 0);
assert_ne!(on_interrupt(&mut t, 8), 0);
t.console_mut_for_test().typed = vec![CTRL_C];
t.signal(KEYS_BIT);
assert_eq!(t.console().interrupted, [CapSlot::new(7).unwrap()]);
}
#[test]
fn keys_typed_ahead_start_the_next_line() {
let mut t = term(b"s\n");
@@ -192,15 +183,16 @@ fn a_secret_line_shows_stars_and_is_forgotten() {
}
#[test]
fn only_init_lets_the_next_session_register_for_ctrl_c() {
fn only_the_control_copy_takes_ctrl_c_and_the_latest_counts() {
use server_framework::MALFORMED;
let mut t = term(&[]);
let plain = |t: &mut TerminalServer<Fake>, slot| t.handle_call(1, [types::terminal::ON_INTERRUPT, 0, 0, 0], CapSlot::new(slot)).0[0];
assert_eq!(plain(&mut t, 6), MALFORMED, "an app, with the plain copy, cannot take it");
assert_eq!(on_interrupt(&mut t, 7), 0);
assert_eq!(on_interrupt(&mut t, 8), MALFORMED, "an app cannot take Ctrl+C over");
let signed_out = |t: &mut TerminalServer<Fake>, badge| t.handle_call(badge, [types::terminal::SIGNED_OUT, 0, 0, 0], None).0[0];
assert_eq!(signed_out(&mut t, 1), MALFORMED, "nor say the user signed out");
assert_eq!(on_interrupt(&mut t, 8), MALFORMED);
assert_eq!(signed_out(&mut t, 0), 0);
assert_eq!(plain(&mut t, 8), MALFORMED, "nor take it over");
assert_eq!(on_interrupt(&mut t, 9), 0, "a restarted shell registers again");
assert_eq!(t.console().forgotten, [CapSlot::new(7).unwrap()]);
assert_eq!(on_interrupt(&mut t, 9), 0, "the next shell may");
t.console_mut_for_test().typed = vec![CTRL_C];
t.signal(KEYS_BIT);
assert_eq!(t.console().interrupted, [CapSlot::new(9).unwrap()]);
}

View File

@@ -12,11 +12,13 @@ pub const READ_SECRET: usize = 14;
/// that drew on the screen itself.
pub const REDRAW: usize = 12;
/// Carries a notification the terminal signals on Ctrl+C while no line is
/// being read. Accepted once a session (see `SIGNED_OUT`).
/// being read. Only through a copy whose badge has `CONTROL`; the latest
/// one counts, so a restarted shell simply registers again.
pub const ON_INTERRUPT: usize = 13;
/// From the unbadged holder (init) only, at sign-out: forgets who
/// `ON_INTERRUPT` signals, so the next session's shell may register.
pub const SIGNED_OUT: usize = 15;
/// On the badge of the terminal copy init gives the program that takes
/// Ctrl+C (the shell). The copies it passes on to apps carry its badge
/// too, so it gives them its other, plain copy.
pub const CONTROL: usize = 1 << 62;
/// What Ctrl+C types.
pub const CTRL_C: u8 = 3;