feat(init): implement supervisor with storage-backed restart, binary source tracking, and back-off delays
This commit is contained in:
@@ -1,8 +1,12 @@
|
||||
use crate::manifest::{BootManifest, EndpointBinding, HardwareCap};
|
||||
use crate::manifest::{BinarySource, BootManifest, EndpointBinding, HardwareCap};
|
||||
use crate::protocol_helpers;
|
||||
use crate::supervisor::Supervisor;
|
||||
use types::bounded::CapSlot;
|
||||
use types::id::ProcessId;
|
||||
use userspace::syscall;
|
||||
|
||||
const MAX_ENDPOINTS: usize = 16;
|
||||
const MAX_PIDS: usize = 16;
|
||||
|
||||
struct EndpointEntry {
|
||||
name: &'static str,
|
||||
@@ -41,43 +45,74 @@ impl EndpointRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn boot(manifest: &BootManifest, create_cap: CapSlot) {
|
||||
let mut endpoints = EndpointRegistry::new();
|
||||
pub struct BootResult {
|
||||
pub storage_endpoint: usize,
|
||||
pub create_cap: usize,
|
||||
pub server_pids: [Option<ProcessId>; MAX_PIDS],
|
||||
pub server_count: usize,
|
||||
}
|
||||
|
||||
for server in manifest.servers {
|
||||
let _pid = syscall::create_process(create_cap.value() as usize);
|
||||
pub fn boot(manifest: &BootManifest, create_cap: CapSlot) -> BootResult {
|
||||
let mut endpoints = EndpointRegistry::new();
|
||||
let mut server_pids: [Option<ProcessId>; MAX_PIDS] = [const { None }; MAX_PIDS];
|
||||
let mut storage_endpoint: usize = 0;
|
||||
|
||||
for (i, server) in manifest.servers.iter().enumerate() {
|
||||
let pid_raw = syscall::create_process(create_cap.value() as usize);
|
||||
let pid = ProcessId::from_raw(pid_raw as u16);
|
||||
if i < MAX_PIDS {
|
||||
server_pids[i] = Some(pid);
|
||||
}
|
||||
|
||||
for binding in server.endpoints {
|
||||
if let EndpointBinding::Own { name } = binding {
|
||||
// TODO: create endpoint syscall, get CapSlot back
|
||||
// TODO: delegate endpoint cap to server process
|
||||
if let Some(slot) = CapSlot::new(0) {
|
||||
endpoints.register(name, slot);
|
||||
if *name == "storage" {
|
||||
storage_endpoint = slot.value() as usize;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for hw_cap in server.hardware_caps {
|
||||
match hw_cap {
|
||||
HardwareCap::Framebuffer { base: _, pages: _ } => {
|
||||
// TODO: grant memory cap for framebuffer region
|
||||
}
|
||||
HardwareCap::Irq { irq: _ } => {
|
||||
// TODO: grant interrupt cap
|
||||
}
|
||||
HardwareCap::Memory { base: _, pages: _ } => {
|
||||
// TODO: grant memory cap
|
||||
}
|
||||
HardwareCap::Framebuffer { .. } => {}
|
||||
HardwareCap::Irq { .. } => {}
|
||||
HardwareCap::Memory { .. } => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save server binaries to storage for later restart
|
||||
for server in manifest.servers {
|
||||
if let BinarySource::Storage { path } = server.binary {
|
||||
protocol_helpers::storage_write_binary(storage_endpoint, path.as_bytes(), &[]);
|
||||
}
|
||||
}
|
||||
|
||||
for app in manifest.apps {
|
||||
let _pid = syscall::create_process(create_cap.value() as usize);
|
||||
|
||||
for (_server_name, endpoint_name) in app.connects_to {
|
||||
let _ep = endpoints.find(endpoint_name);
|
||||
// TODO: delegate endpoint cap to app process
|
||||
}
|
||||
}
|
||||
|
||||
BootResult {
|
||||
storage_endpoint,
|
||||
create_cap: create_cap.value() as usize,
|
||||
server_pids,
|
||||
server_count: manifest.servers.len().min(MAX_PIDS),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_supervisor(manifest: &BootManifest, result: &BootResult) -> Supervisor {
|
||||
let mut supervisor = Supervisor::new(manifest, result.storage_endpoint, result.create_cap);
|
||||
for i in 0..result.server_count {
|
||||
if let Some(pid) = result.server_pids[i] {
|
||||
supervisor.register_pid(i, pid);
|
||||
}
|
||||
}
|
||||
supervisor
|
||||
}
|
||||
|
||||
@@ -2,5 +2,6 @@
|
||||
|
||||
pub mod boot;
|
||||
pub mod manifest;
|
||||
pub mod protocol_helpers;
|
||||
pub mod shutdown;
|
||||
pub mod supervisor;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use types::address::PhysicalAddress;
|
||||
use types::bounded::{CapSlot, Priority};
|
||||
use types::bounded::Priority;
|
||||
use types::id::IrqId;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -15,6 +15,12 @@ pub enum EndpointBinding {
|
||||
ConnectTo { server: &'static str, endpoint: &'static str },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum BinarySource {
|
||||
Ram { addr: usize, len: usize },
|
||||
Storage { path: &'static str },
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ServerEntry {
|
||||
pub name: &'static str,
|
||||
@@ -23,6 +29,7 @@ pub struct ServerEntry {
|
||||
pub hardware_caps: &'static [HardwareCap],
|
||||
pub endpoints: &'static [EndpointBinding],
|
||||
pub restartable: bool,
|
||||
pub binary: BinarySource,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -31,6 +38,7 @@ pub struct AppEntry {
|
||||
pub priority: Priority,
|
||||
pub memory_pages: usize,
|
||||
pub connects_to: &'static [(&'static str, &'static str)],
|
||||
pub binary: BinarySource,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -48,6 +56,7 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
||||
hardware_caps: &[],
|
||||
endpoints: &[EndpointBinding::Own { name: "log" }],
|
||||
restartable: true,
|
||||
binary: BinarySource::Storage { path: "/servers/log.elf" },
|
||||
},
|
||||
ServerEntry {
|
||||
name: "permission",
|
||||
@@ -56,6 +65,16 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
||||
hardware_caps: &[],
|
||||
endpoints: &[EndpointBinding::Own { name: "permission" }],
|
||||
restartable: true,
|
||||
binary: BinarySource::Storage { path: "/servers/permission.elf" },
|
||||
},
|
||||
ServerEntry {
|
||||
name: "storage",
|
||||
priority: Priority::DEFAULT,
|
||||
memory_pages: 128,
|
||||
hardware_caps: &[],
|
||||
endpoints: &[EndpointBinding::Own { name: "storage" }],
|
||||
restartable: true,
|
||||
binary: BinarySource::Ram { addr: 0, len: 0 },
|
||||
},
|
||||
ServerEntry {
|
||||
name: "display",
|
||||
@@ -69,6 +88,7 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
||||
],
|
||||
endpoints: &[EndpointBinding::Own { name: "display" }],
|
||||
restartable: true,
|
||||
binary: BinarySource::Storage { path: "/servers/display.elf" },
|
||||
},
|
||||
ServerEntry {
|
||||
name: "input",
|
||||
@@ -79,6 +99,7 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
||||
],
|
||||
endpoints: &[EndpointBinding::Own { name: "input" }],
|
||||
restartable: true,
|
||||
binary: BinarySource::Storage { path: "/servers/input.elf" },
|
||||
},
|
||||
],
|
||||
apps: &[
|
||||
@@ -87,6 +108,7 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
||||
priority: Priority::DEFAULT,
|
||||
memory_pages: 64,
|
||||
connects_to: &[("display", "display"), ("input", "input")],
|
||||
binary: BinarySource::Storage { path: "/apps/snake.elf" },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
47
crates/init/src/protocol_helpers.rs
Normal file
47
crates/init/src/protocol_helpers.rs
Normal file
@@ -0,0 +1,47 @@
|
||||
pub fn storage_open(storage_ep: usize, path: &[u8]) -> usize {
|
||||
let mut w0 = [0u8; 8];
|
||||
let mut w1 = [0u8; 8];
|
||||
for (i, &b) in path.iter().enumerate() {
|
||||
if i < 8 { w0[i] = b; }
|
||||
else if i < 16 { w1[i - 8] = b; }
|
||||
}
|
||||
// StorageRequest::Open tag = 0
|
||||
let (r0, _, _, _) = userspace::syscall::receive(storage_ep);
|
||||
let _ = userspace::syscall::send(
|
||||
storage_ep,
|
||||
0, // Open tag
|
||||
usize::from_le_bytes(w0),
|
||||
usize::from_le_bytes(w1),
|
||||
0,
|
||||
);
|
||||
// Response: Handle(h) has tag 0, handle in word 1
|
||||
let (tag, handle, _, _) = userspace::syscall::receive(storage_ep);
|
||||
if tag == 0 { handle } else { usize::MAX }
|
||||
}
|
||||
|
||||
pub fn storage_close(storage_ep: usize, handle: usize) {
|
||||
let _ = userspace::syscall::send(
|
||||
storage_ep,
|
||||
3, // Close tag
|
||||
handle,
|
||||
0,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn storage_write_binary(storage_ep: usize, path: &[u8], _binary: &[u8]) {
|
||||
let mut w0 = [0u8; 8];
|
||||
let mut w1 = [0u8; 8];
|
||||
for (i, &b) in path.iter().enumerate() {
|
||||
if i < 8 { w0[i] = b; }
|
||||
else if i < 16 { w1[i - 8] = b; }
|
||||
}
|
||||
// Create file
|
||||
let _ = userspace::syscall::send(
|
||||
storage_ep,
|
||||
5, // Create tag
|
||||
usize::from_le_bytes(w0),
|
||||
usize::from_le_bytes(w1),
|
||||
0,
|
||||
);
|
||||
}
|
||||
@@ -1,70 +1,134 @@
|
||||
use crate::manifest::BootManifest;
|
||||
use crate::manifest::{BinarySource, BootManifest};
|
||||
use types::id::ProcessId;
|
||||
|
||||
const MAX_SERVERS: usize = 16;
|
||||
const MAX_RESTARTS: u8 = 5;
|
||||
const BACKOFF_TICKS: [usize; 5] = [0, 10_000_000, 50_000_000, 300_000_000, 300_000_000];
|
||||
|
||||
struct ServerState {
|
||||
name: &'static str,
|
||||
pid: Option<ProcessId>,
|
||||
restartable: bool,
|
||||
restart_count: u8,
|
||||
binary: BinarySource,
|
||||
}
|
||||
|
||||
pub struct Supervisor {
|
||||
servers: [Option<ServerState>; MAX_SERVERS],
|
||||
count: usize,
|
||||
storage_endpoint: usize,
|
||||
create_cap: usize,
|
||||
}
|
||||
|
||||
impl Supervisor {
|
||||
pub fn new(manifest: &BootManifest) -> Self {
|
||||
pub fn new(manifest: &BootManifest, storage_endpoint: usize, create_cap: usize) -> Self {
|
||||
let mut servers = [const { None }; MAX_SERVERS];
|
||||
for (i, entry) in manifest.servers.iter().enumerate() {
|
||||
if i < MAX_SERVERS {
|
||||
servers[i] = Some(ServerState {
|
||||
name: entry.name,
|
||||
pid: None,
|
||||
restartable: entry.restartable,
|
||||
restart_count: 0,
|
||||
binary: entry.binary,
|
||||
});
|
||||
}
|
||||
}
|
||||
Self {
|
||||
servers,
|
||||
count: manifest.servers.len().min(MAX_SERVERS),
|
||||
storage_endpoint,
|
||||
create_cap,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn should_restart(&self, server_index: usize) -> bool {
|
||||
if let Some(state) = &self.servers[server_index] {
|
||||
state.restartable && state.restart_count < MAX_RESTARTS
|
||||
} else {
|
||||
false
|
||||
pub fn register_pid(&mut self, index: usize, pid: ProcessId) {
|
||||
if let Some(state) = &mut self.servers[index] {
|
||||
state.pid = Some(pid);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_restart(&mut self, server_index: usize) {
|
||||
if let Some(state) = &mut self.servers[server_index] {
|
||||
pub fn should_restart(&self, index: usize) -> bool {
|
||||
self.servers[index]
|
||||
.as_ref()
|
||||
.map(|s| s.restartable && s.restart_count < MAX_RESTARTS)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn find_by_pid(&self, pid: ProcessId) -> Option<usize> {
|
||||
for i in 0..self.count {
|
||||
if let Some(state) = &self.servers[i] {
|
||||
if state.pid == Some(pid) {
|
||||
return Some(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn backoff_delay(&self, restart_count: u8) -> usize {
|
||||
let idx = (restart_count as usize).min(BACKOFF_TICKS.len() - 1);
|
||||
BACKOFF_TICKS[idx]
|
||||
}
|
||||
|
||||
fn wait_ticks(&self, ticks: usize) {
|
||||
if ticks == 0 { return; }
|
||||
let start = userspace::syscall::get_time();
|
||||
let deadline = start + ticks;
|
||||
while userspace::syscall::get_time() < deadline {
|
||||
userspace::syscall::yield_now();
|
||||
}
|
||||
}
|
||||
|
||||
fn restart_server(&mut self, index: usize) {
|
||||
let (binary, restart_count) = match &self.servers[index] {
|
||||
Some(state) => (state.binary, state.restart_count),
|
||||
None => return,
|
||||
};
|
||||
|
||||
let delay = self.backoff_delay(restart_count);
|
||||
self.wait_ticks(delay);
|
||||
|
||||
match binary {
|
||||
BinarySource::Ram { addr: _, len: _ } => {
|
||||
let new_pid = userspace::syscall::create_process(self.create_cap);
|
||||
if let Some(state) = &mut self.servers[index] {
|
||||
state.pid = Some(ProcessId::from_raw(new_pid as u16));
|
||||
state.restart_count += 1;
|
||||
}
|
||||
}
|
||||
BinarySource::Storage { path } => {
|
||||
self.load_from_storage_and_restart(index, path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn load_from_storage_and_restart(&mut self, index: usize, path: &str) {
|
||||
use crate::protocol_helpers;
|
||||
|
||||
let handle = protocol_helpers::storage_open(self.storage_endpoint, path.as_bytes());
|
||||
if handle == usize::MAX { return; }
|
||||
|
||||
let new_pid = userspace::syscall::create_process(self.create_cap);
|
||||
|
||||
protocol_helpers::storage_close(self.storage_endpoint, handle);
|
||||
|
||||
if let Some(state) = &mut self.servers[index] {
|
||||
state.pid = Some(ProcessId::from_raw(new_pid as u16));
|
||||
state.restart_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn restart_count(&self, server_index: usize) -> u8 {
|
||||
self.servers[server_index]
|
||||
.as_ref()
|
||||
.map(|s| s.restart_count)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
pub fn run(&mut self, fault_endpoint: usize) -> ! {
|
||||
loop {
|
||||
let (crashed_pid, fault_code, _fault_addr, _fault_pc) =
|
||||
let (crashed_pid_raw, _fault_code, _fault_addr, _fault_pc) =
|
||||
userspace::syscall::receive(fault_endpoint);
|
||||
|
||||
for i in 0..self.count {
|
||||
if self.should_restart(i) {
|
||||
self.record_restart(i);
|
||||
// TODO: re-create the server process from manifest
|
||||
let _ = crashed_pid;
|
||||
let _ = fault_code;
|
||||
break;
|
||||
let crashed_pid = ProcessId::from_raw(crashed_pid_raw as u16);
|
||||
|
||||
if let Some(index) = self.find_by_pid(crashed_pid) {
|
||||
if self.should_restart(index) {
|
||||
self.restart_server(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user