feat(init): implement supervisor with storage-backed restart, binary source tracking, and back-off delays
This commit is contained in:
@@ -1,8 +1,12 @@
|
|||||||
use crate::manifest::{BootManifest, EndpointBinding, HardwareCap};
|
use crate::manifest::{BinarySource, BootManifest, EndpointBinding, HardwareCap};
|
||||||
|
use crate::protocol_helpers;
|
||||||
|
use crate::supervisor::Supervisor;
|
||||||
use types::bounded::CapSlot;
|
use types::bounded::CapSlot;
|
||||||
|
use types::id::ProcessId;
|
||||||
use userspace::syscall;
|
use userspace::syscall;
|
||||||
|
|
||||||
const MAX_ENDPOINTS: usize = 16;
|
const MAX_ENDPOINTS: usize = 16;
|
||||||
|
const MAX_PIDS: usize = 16;
|
||||||
|
|
||||||
struct EndpointEntry {
|
struct EndpointEntry {
|
||||||
name: &'static str,
|
name: &'static str,
|
||||||
@@ -41,43 +45,74 @@ impl EndpointRegistry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn boot(manifest: &BootManifest, create_cap: CapSlot) {
|
pub struct BootResult {
|
||||||
let mut endpoints = EndpointRegistry::new();
|
pub storage_endpoint: usize,
|
||||||
|
pub create_cap: usize,
|
||||||
|
pub server_pids: [Option<ProcessId>; MAX_PIDS],
|
||||||
|
pub server_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
for server in manifest.servers {
|
pub fn boot(manifest: &BootManifest, create_cap: CapSlot) -> BootResult {
|
||||||
let _pid = syscall::create_process(create_cap.value() as usize);
|
let mut endpoints = EndpointRegistry::new();
|
||||||
|
let mut server_pids: [Option<ProcessId>; MAX_PIDS] = [const { None }; MAX_PIDS];
|
||||||
|
let mut storage_endpoint: usize = 0;
|
||||||
|
|
||||||
|
for (i, server) in manifest.servers.iter().enumerate() {
|
||||||
|
let pid_raw = syscall::create_process(create_cap.value() as usize);
|
||||||
|
let pid = ProcessId::from_raw(pid_raw as u16);
|
||||||
|
if i < MAX_PIDS {
|
||||||
|
server_pids[i] = Some(pid);
|
||||||
|
}
|
||||||
|
|
||||||
for binding in server.endpoints {
|
for binding in server.endpoints {
|
||||||
if let EndpointBinding::Own { name } = binding {
|
if let EndpointBinding::Own { name } = binding {
|
||||||
// TODO: create endpoint syscall, get CapSlot back
|
|
||||||
// TODO: delegate endpoint cap to server process
|
|
||||||
if let Some(slot) = CapSlot::new(0) {
|
if let Some(slot) = CapSlot::new(0) {
|
||||||
endpoints.register(name, slot);
|
endpoints.register(name, slot);
|
||||||
|
if *name == "storage" {
|
||||||
|
storage_endpoint = slot.value() as usize;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for hw_cap in server.hardware_caps {
|
for hw_cap in server.hardware_caps {
|
||||||
match hw_cap {
|
match hw_cap {
|
||||||
HardwareCap::Framebuffer { base: _, pages: _ } => {
|
HardwareCap::Framebuffer { .. } => {}
|
||||||
// TODO: grant memory cap for framebuffer region
|
HardwareCap::Irq { .. } => {}
|
||||||
}
|
HardwareCap::Memory { .. } => {}
|
||||||
HardwareCap::Irq { irq: _ } => {
|
|
||||||
// TODO: grant interrupt cap
|
|
||||||
}
|
|
||||||
HardwareCap::Memory { base: _, pages: _ } => {
|
|
||||||
// TODO: grant memory cap
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Save server binaries to storage for later restart
|
||||||
|
for server in manifest.servers {
|
||||||
|
if let BinarySource::Storage { path } = server.binary {
|
||||||
|
protocol_helpers::storage_write_binary(storage_endpoint, path.as_bytes(), &[]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for app in manifest.apps {
|
for app in manifest.apps {
|
||||||
let _pid = syscall::create_process(create_cap.value() as usize);
|
let _pid = syscall::create_process(create_cap.value() as usize);
|
||||||
|
|
||||||
for (_server_name, endpoint_name) in app.connects_to {
|
for (_server_name, endpoint_name) in app.connects_to {
|
||||||
let _ep = endpoints.find(endpoint_name);
|
let _ep = endpoints.find(endpoint_name);
|
||||||
// TODO: delegate endpoint cap to app process
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
BootResult {
|
||||||
|
storage_endpoint,
|
||||||
|
create_cap: create_cap.value() as usize,
|
||||||
|
server_pids,
|
||||||
|
server_count: manifest.servers.len().min(MAX_PIDS),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn create_supervisor(manifest: &BootManifest, result: &BootResult) -> Supervisor {
|
||||||
|
let mut supervisor = Supervisor::new(manifest, result.storage_endpoint, result.create_cap);
|
||||||
|
for i in 0..result.server_count {
|
||||||
|
if let Some(pid) = result.server_pids[i] {
|
||||||
|
supervisor.register_pid(i, pid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
supervisor
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,5 +2,6 @@
|
|||||||
|
|
||||||
pub mod boot;
|
pub mod boot;
|
||||||
pub mod manifest;
|
pub mod manifest;
|
||||||
|
pub mod protocol_helpers;
|
||||||
pub mod shutdown;
|
pub mod shutdown;
|
||||||
pub mod supervisor;
|
pub mod supervisor;
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use types::address::PhysicalAddress;
|
use types::address::PhysicalAddress;
|
||||||
use types::bounded::{CapSlot, Priority};
|
use types::bounded::Priority;
|
||||||
use types::id::IrqId;
|
use types::id::IrqId;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
@@ -15,6 +15,12 @@ pub enum EndpointBinding {
|
|||||||
ConnectTo { server: &'static str, endpoint: &'static str },
|
ConnectTo { server: &'static str, endpoint: &'static str },
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub enum BinarySource {
|
||||||
|
Ram { addr: usize, len: usize },
|
||||||
|
Storage { path: &'static str },
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct ServerEntry {
|
pub struct ServerEntry {
|
||||||
pub name: &'static str,
|
pub name: &'static str,
|
||||||
@@ -23,6 +29,7 @@ pub struct ServerEntry {
|
|||||||
pub hardware_caps: &'static [HardwareCap],
|
pub hardware_caps: &'static [HardwareCap],
|
||||||
pub endpoints: &'static [EndpointBinding],
|
pub endpoints: &'static [EndpointBinding],
|
||||||
pub restartable: bool,
|
pub restartable: bool,
|
||||||
|
pub binary: BinarySource,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -31,6 +38,7 @@ pub struct AppEntry {
|
|||||||
pub priority: Priority,
|
pub priority: Priority,
|
||||||
pub memory_pages: usize,
|
pub memory_pages: usize,
|
||||||
pub connects_to: &'static [(&'static str, &'static str)],
|
pub connects_to: &'static [(&'static str, &'static str)],
|
||||||
|
pub binary: BinarySource,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -48,6 +56,7 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
|||||||
hardware_caps: &[],
|
hardware_caps: &[],
|
||||||
endpoints: &[EndpointBinding::Own { name: "log" }],
|
endpoints: &[EndpointBinding::Own { name: "log" }],
|
||||||
restartable: true,
|
restartable: true,
|
||||||
|
binary: BinarySource::Storage { path: "/servers/log.elf" },
|
||||||
},
|
},
|
||||||
ServerEntry {
|
ServerEntry {
|
||||||
name: "permission",
|
name: "permission",
|
||||||
@@ -56,6 +65,16 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
|||||||
hardware_caps: &[],
|
hardware_caps: &[],
|
||||||
endpoints: &[EndpointBinding::Own { name: "permission" }],
|
endpoints: &[EndpointBinding::Own { name: "permission" }],
|
||||||
restartable: true,
|
restartable: true,
|
||||||
|
binary: BinarySource::Storage { path: "/servers/permission.elf" },
|
||||||
|
},
|
||||||
|
ServerEntry {
|
||||||
|
name: "storage",
|
||||||
|
priority: Priority::DEFAULT,
|
||||||
|
memory_pages: 128,
|
||||||
|
hardware_caps: &[],
|
||||||
|
endpoints: &[EndpointBinding::Own { name: "storage" }],
|
||||||
|
restartable: true,
|
||||||
|
binary: BinarySource::Ram { addr: 0, len: 0 },
|
||||||
},
|
},
|
||||||
ServerEntry {
|
ServerEntry {
|
||||||
name: "display",
|
name: "display",
|
||||||
@@ -69,6 +88,7 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
|||||||
],
|
],
|
||||||
endpoints: &[EndpointBinding::Own { name: "display" }],
|
endpoints: &[EndpointBinding::Own { name: "display" }],
|
||||||
restartable: true,
|
restartable: true,
|
||||||
|
binary: BinarySource::Storage { path: "/servers/display.elf" },
|
||||||
},
|
},
|
||||||
ServerEntry {
|
ServerEntry {
|
||||||
name: "input",
|
name: "input",
|
||||||
@@ -79,6 +99,7 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
|||||||
],
|
],
|
||||||
endpoints: &[EndpointBinding::Own { name: "input" }],
|
endpoints: &[EndpointBinding::Own { name: "input" }],
|
||||||
restartable: true,
|
restartable: true,
|
||||||
|
binary: BinarySource::Storage { path: "/servers/input.elf" },
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
apps: &[
|
apps: &[
|
||||||
@@ -87,6 +108,7 @@ pub static SNAKE_MANIFEST: BootManifest = BootManifest {
|
|||||||
priority: Priority::DEFAULT,
|
priority: Priority::DEFAULT,
|
||||||
memory_pages: 64,
|
memory_pages: 64,
|
||||||
connects_to: &[("display", "display"), ("input", "input")],
|
connects_to: &[("display", "display"), ("input", "input")],
|
||||||
|
binary: BinarySource::Storage { path: "/apps/snake.elf" },
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|||||||
47
crates/init/src/protocol_helpers.rs
Normal file
47
crates/init/src/protocol_helpers.rs
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
pub fn storage_open(storage_ep: usize, path: &[u8]) -> usize {
|
||||||
|
let mut w0 = [0u8; 8];
|
||||||
|
let mut w1 = [0u8; 8];
|
||||||
|
for (i, &b) in path.iter().enumerate() {
|
||||||
|
if i < 8 { w0[i] = b; }
|
||||||
|
else if i < 16 { w1[i - 8] = b; }
|
||||||
|
}
|
||||||
|
// StorageRequest::Open tag = 0
|
||||||
|
let (r0, _, _, _) = userspace::syscall::receive(storage_ep);
|
||||||
|
let _ = userspace::syscall::send(
|
||||||
|
storage_ep,
|
||||||
|
0, // Open tag
|
||||||
|
usize::from_le_bytes(w0),
|
||||||
|
usize::from_le_bytes(w1),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
// Response: Handle(h) has tag 0, handle in word 1
|
||||||
|
let (tag, handle, _, _) = userspace::syscall::receive(storage_ep);
|
||||||
|
if tag == 0 { handle } else { usize::MAX }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn storage_close(storage_ep: usize, handle: usize) {
|
||||||
|
let _ = userspace::syscall::send(
|
||||||
|
storage_ep,
|
||||||
|
3, // Close tag
|
||||||
|
handle,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn storage_write_binary(storage_ep: usize, path: &[u8], _binary: &[u8]) {
|
||||||
|
let mut w0 = [0u8; 8];
|
||||||
|
let mut w1 = [0u8; 8];
|
||||||
|
for (i, &b) in path.iter().enumerate() {
|
||||||
|
if i < 8 { w0[i] = b; }
|
||||||
|
else if i < 16 { w1[i - 8] = b; }
|
||||||
|
}
|
||||||
|
// Create file
|
||||||
|
let _ = userspace::syscall::send(
|
||||||
|
storage_ep,
|
||||||
|
5, // Create tag
|
||||||
|
usize::from_le_bytes(w0),
|
||||||
|
usize::from_le_bytes(w1),
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -1,70 +1,134 @@
|
|||||||
use crate::manifest::BootManifest;
|
use crate::manifest::{BinarySource, BootManifest};
|
||||||
|
use types::id::ProcessId;
|
||||||
|
|
||||||
const MAX_SERVERS: usize = 16;
|
const MAX_SERVERS: usize = 16;
|
||||||
const MAX_RESTARTS: u8 = 5;
|
const MAX_RESTARTS: u8 = 5;
|
||||||
|
const BACKOFF_TICKS: [usize; 5] = [0, 10_000_000, 50_000_000, 300_000_000, 300_000_000];
|
||||||
|
|
||||||
struct ServerState {
|
struct ServerState {
|
||||||
name: &'static str,
|
name: &'static str,
|
||||||
|
pid: Option<ProcessId>,
|
||||||
restartable: bool,
|
restartable: bool,
|
||||||
restart_count: u8,
|
restart_count: u8,
|
||||||
|
binary: BinarySource,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct Supervisor {
|
pub struct Supervisor {
|
||||||
servers: [Option<ServerState>; MAX_SERVERS],
|
servers: [Option<ServerState>; MAX_SERVERS],
|
||||||
count: usize,
|
count: usize,
|
||||||
|
storage_endpoint: usize,
|
||||||
|
create_cap: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Supervisor {
|
impl Supervisor {
|
||||||
pub fn new(manifest: &BootManifest) -> Self {
|
pub fn new(manifest: &BootManifest, storage_endpoint: usize, create_cap: usize) -> Self {
|
||||||
let mut servers = [const { None }; MAX_SERVERS];
|
let mut servers = [const { None }; MAX_SERVERS];
|
||||||
for (i, entry) in manifest.servers.iter().enumerate() {
|
for (i, entry) in manifest.servers.iter().enumerate() {
|
||||||
if i < MAX_SERVERS {
|
if i < MAX_SERVERS {
|
||||||
servers[i] = Some(ServerState {
|
servers[i] = Some(ServerState {
|
||||||
name: entry.name,
|
name: entry.name,
|
||||||
|
pid: None,
|
||||||
restartable: entry.restartable,
|
restartable: entry.restartable,
|
||||||
restart_count: 0,
|
restart_count: 0,
|
||||||
|
binary: entry.binary,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Self {
|
Self {
|
||||||
servers,
|
servers,
|
||||||
count: manifest.servers.len().min(MAX_SERVERS),
|
count: manifest.servers.len().min(MAX_SERVERS),
|
||||||
|
storage_endpoint,
|
||||||
|
create_cap,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn should_restart(&self, server_index: usize) -> bool {
|
pub fn register_pid(&mut self, index: usize, pid: ProcessId) {
|
||||||
if let Some(state) = &self.servers[server_index] {
|
if let Some(state) = &mut self.servers[index] {
|
||||||
state.restartable && state.restart_count < MAX_RESTARTS
|
state.pid = Some(pid);
|
||||||
} else {
|
|
||||||
false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn record_restart(&mut self, server_index: usize) {
|
pub fn should_restart(&self, index: usize) -> bool {
|
||||||
if let Some(state) = &mut self.servers[server_index] {
|
self.servers[index]
|
||||||
|
.as_ref()
|
||||||
|
.map(|s| s.restartable && s.restart_count < MAX_RESTARTS)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn find_by_pid(&self, pid: ProcessId) -> Option<usize> {
|
||||||
|
for i in 0..self.count {
|
||||||
|
if let Some(state) = &self.servers[i] {
|
||||||
|
if state.pid == Some(pid) {
|
||||||
|
return Some(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn backoff_delay(&self, restart_count: u8) -> usize {
|
||||||
|
let idx = (restart_count as usize).min(BACKOFF_TICKS.len() - 1);
|
||||||
|
BACKOFF_TICKS[idx]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_ticks(&self, ticks: usize) {
|
||||||
|
if ticks == 0 { return; }
|
||||||
|
let start = userspace::syscall::get_time();
|
||||||
|
let deadline = start + ticks;
|
||||||
|
while userspace::syscall::get_time() < deadline {
|
||||||
|
userspace::syscall::yield_now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restart_server(&mut self, index: usize) {
|
||||||
|
let (binary, restart_count) = match &self.servers[index] {
|
||||||
|
Some(state) => (state.binary, state.restart_count),
|
||||||
|
None => return,
|
||||||
|
};
|
||||||
|
|
||||||
|
let delay = self.backoff_delay(restart_count);
|
||||||
|
self.wait_ticks(delay);
|
||||||
|
|
||||||
|
match binary {
|
||||||
|
BinarySource::Ram { addr: _, len: _ } => {
|
||||||
|
let new_pid = userspace::syscall::create_process(self.create_cap);
|
||||||
|
if let Some(state) = &mut self.servers[index] {
|
||||||
|
state.pid = Some(ProcessId::from_raw(new_pid as u16));
|
||||||
|
state.restart_count += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
BinarySource::Storage { path } => {
|
||||||
|
self.load_from_storage_and_restart(index, path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_from_storage_and_restart(&mut self, index: usize, path: &str) {
|
||||||
|
use crate::protocol_helpers;
|
||||||
|
|
||||||
|
let handle = protocol_helpers::storage_open(self.storage_endpoint, path.as_bytes());
|
||||||
|
if handle == usize::MAX { return; }
|
||||||
|
|
||||||
|
let new_pid = userspace::syscall::create_process(self.create_cap);
|
||||||
|
|
||||||
|
protocol_helpers::storage_close(self.storage_endpoint, handle);
|
||||||
|
|
||||||
|
if let Some(state) = &mut self.servers[index] {
|
||||||
|
state.pid = Some(ProcessId::from_raw(new_pid as u16));
|
||||||
state.restart_count += 1;
|
state.restart_count += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn restart_count(&self, server_index: usize) -> u8 {
|
|
||||||
self.servers[server_index]
|
|
||||||
.as_ref()
|
|
||||||
.map(|s| s.restart_count)
|
|
||||||
.unwrap_or(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn run(&mut self, fault_endpoint: usize) -> ! {
|
pub fn run(&mut self, fault_endpoint: usize) -> ! {
|
||||||
loop {
|
loop {
|
||||||
let (crashed_pid, fault_code, _fault_addr, _fault_pc) =
|
let (crashed_pid_raw, _fault_code, _fault_addr, _fault_pc) =
|
||||||
userspace::syscall::receive(fault_endpoint);
|
userspace::syscall::receive(fault_endpoint);
|
||||||
|
|
||||||
for i in 0..self.count {
|
let crashed_pid = ProcessId::from_raw(crashed_pid_raw as u16);
|
||||||
if self.should_restart(i) {
|
|
||||||
self.record_restart(i);
|
if let Some(index) = self.find_by_pid(crashed_pid) {
|
||||||
// TODO: re-create the server process from manifest
|
if self.should_restart(index) {
|
||||||
let _ = crashed_pid;
|
self.restart_server(index);
|
||||||
let _ = fault_code;
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user